@artemiskit/sdk 0.4.4 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/dist/index.js +153 -321
  3. package/package.json +3 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.5
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @artemiskit/core@0.5.2
9
+ - @artemiskit/redteam@0.3.6
10
+
3
11
  ## 0.4.4
4
12
 
5
13
  ### Patch Changes
package/dist/index.js CHANGED
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
26484
26484
  tags: arrayType(stringType()).optional().default([]),
26485
26485
  metadata: recordType(unknownType()).optional().default({}),
26486
26486
  timeout: numberType().optional(),
26487
- retries: numberType().optional().default(0),
26487
+ retries: numberType().int().min(0).max(99).optional().default(0),
26488
26488
  provider: ProviderSchema.optional(),
26489
26489
  model: stringType().optional(),
26490
26490
  variables: VariablesSchema.optional(),
@@ -27129,17 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
27129
27129
  }
27130
27130
  async function executeCase(testCase, context) {
27131
27131
  const { timeout, retries = 0 } = context;
27132
+ if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
27133
+ throw new RangeError("retries must be a whole number between 0 and 99");
27134
+ }
27132
27135
  const caseStartTime = Date.now();
27133
27136
  const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
27137
+ const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
27138
+ const repetitionIndex = context.repetition?.index ?? 1;
27139
+ const attemptEvidence = [];
27134
27140
  let lastError = null;
27135
27141
  for (let attempt = 0;attempt <= retries; attempt++) {
27142
+ const attemptStartTime = Date.now();
27136
27143
  try {
27137
27144
  const result = await executeCaseAttempt(testCase, context, timeout);
27138
- return { ...result, attempts: attempt + 1 };
27145
+ return withAttemptEvidence(result, attemptEvidence, {
27146
+ retryChainId,
27147
+ repetitionIndex,
27148
+ attemptNumber: attempt + 1,
27149
+ includedInOutcome: true,
27150
+ latencyMs: result.latencyMs
27151
+ });
27139
27152
  } catch (error) {
27140
27153
  lastError = error;
27141
- if (error instanceof ToolLoopError)
27142
- return { ...error.caseResult, attempts: attempt + 1 };
27154
+ if (error instanceof ToolLoopError) {
27155
+ return withAttemptEvidence(error.caseResult, attemptEvidence, {
27156
+ retryChainId,
27157
+ repetitionIndex,
27158
+ attemptNumber: attempt + 1,
27159
+ includedInOutcome: true,
27160
+ latencyMs: error.caseResult.latencyMs,
27161
+ errorCode: "tool_error"
27162
+ });
27163
+ }
27164
+ attemptEvidence.push({
27165
+ attempt_id: `${retryChainId}:${attempt + 1}`,
27166
+ retry_chain_id: retryChainId,
27167
+ repetition_index: repetitionIndex,
27168
+ attempt_number: attempt + 1,
27169
+ status: "error",
27170
+ included_in_outcome: false,
27171
+ latency_ms: Date.now() - attemptStartTime,
27172
+ error_code: error instanceof TimeoutError ? "timeout" : "target_error"
27173
+ });
27143
27174
  if (attempt < retries) {
27144
27175
  await sleep(2 ** attempt * 1000);
27145
27176
  }
@@ -27152,6 +27183,10 @@ async function executeCase(testCase, context) {
27152
27183
  ok: false,
27153
27184
  status: "error",
27154
27185
  attempts: retries + 1,
27186
+ attempt_evidence: attemptEvidence.map((entry, index) => ({
27187
+ ...entry,
27188
+ included_in_outcome: index === attemptEvidence.length - 1
27189
+ })),
27155
27190
  score: 0,
27156
27191
  matcherType: testCase.expected.type,
27157
27192
  reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -27165,6 +27200,28 @@ async function executeCase(testCase, context) {
27165
27200
  target: targetEvidence(context.client.provider, requestedModel)
27166
27201
  };
27167
27202
  }
27203
+ function withAttemptEvidence(result, priorAttempts, input) {
27204
+ return {
27205
+ ...result,
27206
+ attempts: input.attemptNumber,
27207
+ attempt_evidence: [
27208
+ ...priorAttempts,
27209
+ {
27210
+ attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
27211
+ retry_chain_id: input.retryChainId,
27212
+ repetition_index: input.repetitionIndex,
27213
+ attempt_number: input.attemptNumber,
27214
+ status: getTerminalStatus(result),
27215
+ included_in_outcome: input.includedInOutcome,
27216
+ latency_ms: input.latencyMs,
27217
+ ...input.errorCode ? { error_code: input.errorCode } : {}
27218
+ }
27219
+ ]
27220
+ };
27221
+ }
27222
+ function getTerminalStatus(result) {
27223
+ return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
27224
+ }
27168
27225
  async function executeCaseAttempt(testCase, context, timeout) {
27169
27226
  const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
27170
27227
  const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -27526,302 +27583,6 @@ function nanoid(size = 21) {
27526
27583
  }
27527
27584
  return id;
27528
27585
  }
27529
- var MODEL_PRICING = {
27530
- "gpt-5": {
27531
- promptPer1K: 0.00125,
27532
- completionPer1K: 0.01,
27533
- lastUpdated: "2026-01",
27534
- notes: "400K context window"
27535
- },
27536
- "gpt-5.1": {
27537
- promptPer1K: 0.00125,
27538
- completionPer1K: 0.01,
27539
- lastUpdated: "2026-01"
27540
- },
27541
- "gpt-5.2": {
27542
- promptPer1K: 0.00175,
27543
- completionPer1K: 0.014,
27544
- lastUpdated: "2026-01"
27545
- },
27546
- "gpt-5-mini": {
27547
- promptPer1K: 0.00025,
27548
- completionPer1K: 0.002,
27549
- lastUpdated: "2026-01"
27550
- },
27551
- "gpt-5-nano": {
27552
- promptPer1K: 0.00005,
27553
- completionPer1K: 0.0004,
27554
- lastUpdated: "2026-01"
27555
- },
27556
- "gpt-4.1": {
27557
- promptPer1K: 0.002,
27558
- completionPer1K: 0.008,
27559
- lastUpdated: "2026-01",
27560
- notes: "1M context window"
27561
- },
27562
- "gpt-4.1-mini": {
27563
- promptPer1K: 0.0004,
27564
- completionPer1K: 0.0016,
27565
- lastUpdated: "2026-01"
27566
- },
27567
- "gpt-4.1-nano": {
27568
- promptPer1K: 0.0001,
27569
- completionPer1K: 0.0004,
27570
- lastUpdated: "2026-01"
27571
- },
27572
- "gpt-4o": {
27573
- promptPer1K: 0.0025,
27574
- completionPer1K: 0.01,
27575
- lastUpdated: "2026-01",
27576
- notes: "128K context window"
27577
- },
27578
- "gpt-4o-mini": {
27579
- promptPer1K: 0.00015,
27580
- completionPer1K: 0.0006,
27581
- lastUpdated: "2026-01",
27582
- notes: "128K context window"
27583
- },
27584
- o1: {
27585
- promptPer1K: 0.015,
27586
- completionPer1K: 0.06,
27587
- lastUpdated: "2026-01",
27588
- notes: "Reasoning model - internal thinking tokens billed as output"
27589
- },
27590
- o3: {
27591
- promptPer1K: 0.002,
27592
- completionPer1K: 0.008,
27593
- lastUpdated: "2026-01"
27594
- },
27595
- "o3-mini": {
27596
- promptPer1K: 0.0011,
27597
- completionPer1K: 0.0044,
27598
- lastUpdated: "2026-01"
27599
- },
27600
- "o4-mini": {
27601
- promptPer1K: 0.0011,
27602
- completionPer1K: 0.0044,
27603
- lastUpdated: "2026-01"
27604
- },
27605
- "gpt-4-turbo": {
27606
- promptPer1K: 0.01,
27607
- completionPer1K: 0.03,
27608
- lastUpdated: "2026-01"
27609
- },
27610
- "gpt-4": {
27611
- promptPer1K: 0.03,
27612
- completionPer1K: 0.06,
27613
- lastUpdated: "2026-01"
27614
- },
27615
- "gpt-3.5-turbo": {
27616
- promptPer1K: 0.0005,
27617
- completionPer1K: 0.0015,
27618
- lastUpdated: "2026-01"
27619
- },
27620
- "claude-opus-4.5": {
27621
- promptPer1K: 0.005,
27622
- completionPer1K: 0.025,
27623
- lastUpdated: "2026-01",
27624
- notes: "Most capable Claude model"
27625
- },
27626
- "claude-sonnet-4.5": {
27627
- promptPer1K: 0.003,
27628
- completionPer1K: 0.015,
27629
- lastUpdated: "2026-01",
27630
- notes: "Balanced performance and cost"
27631
- },
27632
- "claude-haiku-4.5": {
27633
- promptPer1K: 0.001,
27634
- completionPer1K: 0.005,
27635
- lastUpdated: "2026-01",
27636
- notes: "Fastest Claude model"
27637
- },
27638
- "claude-opus-4": {
27639
- promptPer1K: 0.015,
27640
- completionPer1K: 0.075,
27641
- lastUpdated: "2026-01"
27642
- },
27643
- "claude-opus-4.1": {
27644
- promptPer1K: 0.015,
27645
- completionPer1K: 0.075,
27646
- lastUpdated: "2026-01"
27647
- },
27648
- "claude-sonnet-4": {
27649
- promptPer1K: 0.003,
27650
- completionPer1K: 0.015,
27651
- lastUpdated: "2026-01"
27652
- },
27653
- "claude-sonnet-3.7": {
27654
- promptPer1K: 0.003,
27655
- completionPer1K: 0.015,
27656
- lastUpdated: "2026-01"
27657
- },
27658
- "claude-3-7-sonnet": {
27659
- promptPer1K: 0.003,
27660
- completionPer1K: 0.015,
27661
- lastUpdated: "2026-01"
27662
- },
27663
- "claude-3-5-sonnet-20241022": {
27664
- promptPer1K: 0.003,
27665
- completionPer1K: 0.015,
27666
- lastUpdated: "2026-01"
27667
- },
27668
- "claude-3-5-haiku-20241022": {
27669
- promptPer1K: 0.0008,
27670
- completionPer1K: 0.004,
27671
- lastUpdated: "2026-01"
27672
- },
27673
- "claude-haiku-3.5": {
27674
- promptPer1K: 0.0008,
27675
- completionPer1K: 0.004,
27676
- lastUpdated: "2026-01"
27677
- },
27678
- "claude-3-opus": {
27679
- promptPer1K: 0.015,
27680
- completionPer1K: 0.075,
27681
- lastUpdated: "2026-01"
27682
- },
27683
- "claude-3-sonnet": {
27684
- promptPer1K: 0.003,
27685
- completionPer1K: 0.015,
27686
- lastUpdated: "2026-01"
27687
- },
27688
- "claude-3-haiku": {
27689
- promptPer1K: 0.00025,
27690
- completionPer1K: 0.00125,
27691
- lastUpdated: "2026-01"
27692
- },
27693
- "claude-3.5-sonnet": {
27694
- promptPer1K: 0.003,
27695
- completionPer1K: 0.015,
27696
- lastUpdated: "2026-01"
27697
- },
27698
- "claude-3.5-haiku": {
27699
- promptPer1K: 0.0008,
27700
- completionPer1K: 0.004,
27701
- lastUpdated: "2026-01"
27702
- }
27703
- };
27704
- var DEFAULT_PRICING = {
27705
- promptPer1K: 0.003,
27706
- completionPer1K: 0.015,
27707
- lastUpdated: "2026-01",
27708
- notes: "Default pricing - verify with provider"
27709
- };
27710
- function getModelPricing(model) {
27711
- if (MODEL_PRICING[model]) {
27712
- return MODEL_PRICING[model];
27713
- }
27714
- const lowerModel = model.toLowerCase();
27715
- for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
27716
- if (key.toLowerCase() === lowerModel) {
27717
- return pricing;
27718
- }
27719
- }
27720
- if (lowerModel.includes("gpt-5.2")) {
27721
- return MODEL_PRICING["gpt-5.2"];
27722
- }
27723
- if (lowerModel.includes("gpt-5.1")) {
27724
- return MODEL_PRICING["gpt-5.1"];
27725
- }
27726
- if (lowerModel.includes("gpt-5-mini")) {
27727
- return MODEL_PRICING["gpt-5-mini"];
27728
- }
27729
- if (lowerModel.includes("gpt-5-nano")) {
27730
- return MODEL_PRICING["gpt-5-nano"];
27731
- }
27732
- if (lowerModel.includes("gpt-5")) {
27733
- return MODEL_PRICING["gpt-5"];
27734
- }
27735
- if (lowerModel.includes("gpt-4.1-mini")) {
27736
- return MODEL_PRICING["gpt-4.1-mini"];
27737
- }
27738
- if (lowerModel.includes("gpt-4.1-nano")) {
27739
- return MODEL_PRICING["gpt-4.1-nano"];
27740
- }
27741
- if (lowerModel.includes("gpt-4.1")) {
27742
- return MODEL_PRICING["gpt-4.1"];
27743
- }
27744
- if (lowerModel.includes("gpt-4o-mini")) {
27745
- return MODEL_PRICING["gpt-4o-mini"];
27746
- }
27747
- if (lowerModel.includes("gpt-4o")) {
27748
- return MODEL_PRICING["gpt-4o"];
27749
- }
27750
- if (lowerModel.includes("o4-mini")) {
27751
- return MODEL_PRICING["o4-mini"];
27752
- }
27753
- if (lowerModel.includes("o3-mini")) {
27754
- return MODEL_PRICING["o3-mini"];
27755
- }
27756
- if (lowerModel.includes("o3")) {
27757
- return MODEL_PRICING.o3;
27758
- }
27759
- if (lowerModel.includes("o1")) {
27760
- return MODEL_PRICING.o1;
27761
- }
27762
- if (lowerModel.includes("gpt-4-turbo")) {
27763
- return MODEL_PRICING["gpt-4-turbo"];
27764
- }
27765
- if (lowerModel.includes("gpt-4")) {
27766
- return MODEL_PRICING["gpt-4"];
27767
- }
27768
- if (lowerModel.includes("gpt-3.5")) {
27769
- return MODEL_PRICING["gpt-3.5-turbo"];
27770
- }
27771
- if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
27772
- return MODEL_PRICING["claude-opus-4.5"];
27773
- }
27774
- if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
27775
- return MODEL_PRICING["claude-sonnet-4.5"];
27776
- }
27777
- if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
27778
- return MODEL_PRICING["claude-haiku-4.5"];
27779
- }
27780
- if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
27781
- return MODEL_PRICING["claude-opus-4.1"];
27782
- }
27783
- if (lowerModel.includes("opus-4")) {
27784
- return MODEL_PRICING["claude-opus-4"];
27785
- }
27786
- if (lowerModel.includes("sonnet-4")) {
27787
- return MODEL_PRICING["claude-sonnet-4"];
27788
- }
27789
- if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
27790
- return MODEL_PRICING["claude-sonnet-3.7"];
27791
- }
27792
- if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
27793
- return MODEL_PRICING["claude-3.5-sonnet"];
27794
- }
27795
- if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
27796
- return MODEL_PRICING["claude-3.5-haiku"];
27797
- }
27798
- if (lowerModel.includes("claude-3-opus")) {
27799
- return MODEL_PRICING["claude-3-opus"];
27800
- }
27801
- if (lowerModel.includes("claude-3-sonnet")) {
27802
- return MODEL_PRICING["claude-3-sonnet"];
27803
- }
27804
- if (lowerModel.includes("claude-3-haiku")) {
27805
- return MODEL_PRICING["claude-3-haiku"];
27806
- }
27807
- if (lowerModel.includes("claude")) {
27808
- return MODEL_PRICING["claude-sonnet-4.5"];
27809
- }
27810
- return DEFAULT_PRICING;
27811
- }
27812
- function estimateCost(promptTokens, completionTokens, model) {
27813
- const pricing = getModelPricing(model);
27814
- const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
27815
- const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
27816
- const totalUsd = promptCostUsd + completionCostUsd;
27817
- return {
27818
- totalUsd,
27819
- promptCostUsd,
27820
- completionCostUsd,
27821
- model,
27822
- pricing
27823
- };
27824
- }
27825
27586
  function getEnvironmentInfo() {
27826
27587
  return {
27827
27588
  node_version: process.version,
@@ -27880,6 +27641,12 @@ function assertRunManifestIntegrity(manifest) {
27880
27641
  if (manifest.execution_provenance !== undefined) {
27881
27642
  assertExecutionProvenance(manifest.execution_provenance);
27882
27643
  }
27644
+ if (manifest.attempt_evidence !== undefined) {
27645
+ assertRunAttemptEvidence(manifest.attempt_evidence);
27646
+ }
27647
+ if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
27648
+ assertCostProvenance(manifest.metrics.cost_provenance);
27649
+ }
27883
27650
  for (const [index, caseResult] of manifest.cases.entries()) {
27884
27651
  if (!isRecord2(caseResult)) {
27885
27652
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27893,8 +27660,61 @@ function assertRunManifestIntegrity(manifest) {
27893
27660
  if (caseResult.target !== undefined) {
27894
27661
  assertCaseTargetEvidence(caseResult.target);
27895
27662
  }
27663
+ if (caseResult.attempt_evidence !== undefined) {
27664
+ assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
27665
+ }
27896
27666
  }
27897
27667
  }
27668
+ function assertRunAttemptEvidence(evidence) {
27669
+ if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
27670
+ throw new Error("Invalid run manifest: malformed attempt evidence");
27671
+ }
27672
+ }
27673
+ function assertCaseAttemptEvidence(evidence, index) {
27674
+ if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
27675
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27676
+ }
27677
+ for (const attempt of evidence) {
27678
+ if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
27679
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27680
+ }
27681
+ }
27682
+ }
27683
+ function assertCostProvenance(cost) {
27684
+ if (!isRecord2(cost) || cost.schema_version !== "1") {
27685
+ throw new Error("Invalid run manifest: malformed cost provenance");
27686
+ }
27687
+ if (cost.status === "unavailable") {
27688
+ if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
27689
+ throw new Error("Invalid run manifest: malformed cost provenance");
27690
+ }
27691
+ return;
27692
+ }
27693
+ if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
27694
+ throw new Error("Invalid run manifest: malformed cost provenance");
27695
+ }
27696
+ }
27697
+ function isCaseEvaluationStatus(value) {
27698
+ return value === "passed" || value === "failed" || value === "invalid" || value === "error";
27699
+ }
27700
+ function isBoundedNonemptyString(value, maxLength) {
27701
+ return typeof value === "string" && value.length > 0 && value.length <= maxLength;
27702
+ }
27703
+ function isNonnegativeFiniteNumber(value) {
27704
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
27705
+ }
27706
+ function isPositiveFiniteNumber(value) {
27707
+ return isNonnegativeFiniteNumber(value) && value > 0;
27708
+ }
27709
+ function isNonnegativeSafeInteger(value) {
27710
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
27711
+ }
27712
+ function isPositiveSafeInteger(value) {
27713
+ return isNonnegativeSafeInteger(value) && value > 0;
27714
+ }
27715
+ function isIsoTimestamp(value) {
27716
+ return typeof value === "string" && Number.isFinite(Date.parse(value));
27717
+ }
27898
27718
  function assertCaseTargetEvidence(target) {
27899
27719
  if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
27900
27720
  throw new Error("Invalid run manifest: malformed target evidence");
@@ -27971,6 +27791,9 @@ function createRunManifest(options) {
27971
27791
  resolvedConfig,
27972
27792
  workloadIdentity,
27973
27793
  executionProvenance,
27794
+ attemptEvidence,
27795
+ costProvenance,
27796
+ runId,
27974
27797
  cases,
27975
27798
  startTime,
27976
27799
  endTime,
@@ -27978,13 +27801,12 @@ function createRunManifest(options) {
27978
27801
  runReason,
27979
27802
  redaction
27980
27803
  } = options;
27981
- const modelForCost = resolvedConfig?.model || config.model;
27982
- const metrics = calculateMetrics(cases, modelForCost);
27804
+ const metrics = calculateMetrics(cases, costProvenance);
27983
27805
  const git = getGitInfo();
27984
27806
  const environment = getEnvironmentInfo();
27985
27807
  return {
27986
- version: "1.3",
27987
- run_id: nanoid(12),
27808
+ version: "1.4",
27809
+ run_id: runId ?? nanoid(12),
27988
27810
  project,
27989
27811
  start_time: startTime.toISOString(),
27990
27812
  end_time: endTime.toISOString(),
@@ -27993,6 +27815,7 @@ function createRunManifest(options) {
27993
27815
  resolved_config: resolvedConfig,
27994
27816
  workload_identity: workloadIdentity,
27995
27817
  execution_provenance: executionProvenance,
27818
+ attempt_evidence: attemptEvidence,
27996
27819
  metrics,
27997
27820
  git,
27998
27821
  provenance: {
@@ -28005,7 +27828,7 @@ function createRunManifest(options) {
28005
27828
  redaction
28006
27829
  };
28007
27830
  }
28008
- function calculateMetrics(cases, model) {
27831
+ function calculateMetrics(cases, costProvenance) {
28009
27832
  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
28010
27833
  const validCases = cases.filter((c) => {
28011
27834
  const status = getCaseEvaluationStatus(c);
@@ -28017,21 +27840,11 @@ function calculateMetrics(cases, model) {
28017
27840
  const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
28018
27841
  const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
28019
27842
  const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
28020
- let cost;
28021
- if (model && !model.toLowerCase().includes("ling-") && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
28022
- const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
28023
- const pricing = getModelPricing(model);
28024
- cost = {
28025
- total_usd: costEstimate.totalUsd,
28026
- prompt_cost_usd: costEstimate.promptCostUsd,
28027
- completion_cost_usd: costEstimate.completionCostUsd,
28028
- model: costEstimate.model,
28029
- pricing: {
28030
- prompt_per_1k: pricing.promptPer1K,
28031
- completion_per_1k: pricing.completionPer1K
28032
- }
28033
- };
28034
- }
27843
+ const cost_provenance = costProvenance ?? {
27844
+ schema_version: "1",
27845
+ status: "unavailable",
27846
+ unavailable_reason: "provider_billing_not_recorded"
27847
+ };
28035
27848
  return {
28036
27849
  success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
28037
27850
  total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
@@ -28046,7 +27859,7 @@ function calculateMetrics(cases, model) {
28046
27859
  total_tokens: totalPromptTokens + totalCompletionTokens,
28047
27860
  total_prompt_tokens: totalPromptTokens,
28048
27861
  total_completion_tokens: totalCompletionTokens,
28049
- cost
27862
+ cost_provenance
28050
27863
  };
28051
27864
  }
28052
27865
  function detectCIEnvironment() {
@@ -28185,6 +27998,8 @@ async function runScenario(options) {
28185
27998
  concurrency = 1,
28186
27999
  timeout,
28187
28000
  retries,
28001
+ repetition = { index: 1, total: 1 },
28002
+ costProvenance,
28188
28003
  redaction,
28189
28004
  toolExecutor,
28190
28005
  onCaseComplete,
@@ -28200,6 +28015,7 @@ async function runScenario(options) {
28200
28015
  }
28201
28016
  onProgress?.(`Running ${cases.length} test cases...`);
28202
28017
  const startTime = new Date;
28018
+ const runId = nanoid(12);
28203
28019
  const results = [];
28204
28020
  if (concurrency === 1) {
28205
28021
  for (let i = 0;i < cases.length; i++) {
@@ -28210,6 +28026,8 @@ async function runScenario(options) {
28210
28026
  requestedModel: resolvedConfig?.model,
28211
28027
  timeout: testCase.timeout || timeout,
28212
28028
  retries: testCase.retries ?? retries,
28029
+ runId,
28030
+ repetition,
28213
28031
  redaction,
28214
28032
  toolExecutor
28215
28033
  });
@@ -28227,6 +28045,8 @@ async function runScenario(options) {
28227
28045
  requestedModel: resolvedConfig?.model,
28228
28046
  timeout: testCase.timeout || timeout,
28229
28047
  retries: testCase.retries ?? retries,
28048
+ runId,
28049
+ repetition,
28230
28050
  redaction,
28231
28051
  toolExecutor
28232
28052
  });
@@ -28277,9 +28097,21 @@ async function runScenario(options) {
28277
28097
  seed: scenario.seed,
28278
28098
  cases: results
28279
28099
  }),
28100
+ attemptEvidence: {
28101
+ schema_version: "1",
28102
+ repetition,
28103
+ retry_policy: {
28104
+ default_max_retries: retries ?? 0,
28105
+ backoff: "exponential",
28106
+ initial_delay_ms: 1000
28107
+ },
28108
+ ...timeout ? { timeout: { default_ms: timeout } } : {}
28109
+ },
28110
+ costProvenance,
28280
28111
  cases: results,
28281
28112
  startTime,
28282
28113
  endTime,
28114
+ runId,
28283
28115
  redaction: redactionInfo
28284
28116
  });
28285
28117
  const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.4",
3
+ "version": "0.4.5",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.5.1",
75
- "@artemiskit/redteam": "0.3.5"
74
+ "@artemiskit/core": "0.5.2",
75
+ "@artemiskit/redteam": "0.3.6"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",