@artemiskit/sdk 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/index.js +256 -331
  3. package/package.json +3 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.5
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @artemiskit/core@0.5.2
9
+ - @artemiskit/redteam@0.3.6
10
+
11
+ ## 0.4.4
12
+
13
+ ### Patch Changes
14
+
15
+ - Updated dependencies
16
+ - @artemiskit/core@0.5.1
17
+ - @artemiskit/redteam@0.3.5
18
+
3
19
  ## 0.4.3
4
20
 
5
21
  ### Patch Changes
package/dist/index.js CHANGED
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
26484
26484
  tags: arrayType(stringType()).optional().default([]),
26485
26485
  metadata: recordType(unknownType()).optional().default({}),
26486
26486
  timeout: numberType().optional(),
26487
- retries: numberType().optional().default(0),
26487
+ retries: numberType().int().min(0).max(99).optional().default(0),
26488
26488
  provider: ProviderSchema.optional(),
26489
26489
  model: stringType().optional(),
26490
26490
  variables: VariablesSchema.optional(),
@@ -27129,16 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
27129
27129
  }
27130
27130
  async function executeCase(testCase, context) {
27131
27131
  const { timeout, retries = 0 } = context;
27132
+ if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
27133
+ throw new RangeError("retries must be a whole number between 0 and 99");
27134
+ }
27132
27135
  const caseStartTime = Date.now();
27136
+ const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
27137
+ const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
27138
+ const repetitionIndex = context.repetition?.index ?? 1;
27139
+ const attemptEvidence = [];
27133
27140
  let lastError = null;
27134
27141
  for (let attempt = 0;attempt <= retries; attempt++) {
27142
+ const attemptStartTime = Date.now();
27135
27143
  try {
27136
27144
  const result = await executeCaseAttempt(testCase, context, timeout);
27137
- return { ...result, attempts: attempt + 1 };
27145
+ return withAttemptEvidence(result, attemptEvidence, {
27146
+ retryChainId,
27147
+ repetitionIndex,
27148
+ attemptNumber: attempt + 1,
27149
+ includedInOutcome: true,
27150
+ latencyMs: result.latencyMs
27151
+ });
27138
27152
  } catch (error) {
27139
27153
  lastError = error;
27140
- if (error instanceof ToolLoopError)
27141
- return { ...error.caseResult, attempts: attempt + 1 };
27154
+ if (error instanceof ToolLoopError) {
27155
+ return withAttemptEvidence(error.caseResult, attemptEvidence, {
27156
+ retryChainId,
27157
+ repetitionIndex,
27158
+ attemptNumber: attempt + 1,
27159
+ includedInOutcome: true,
27160
+ latencyMs: error.caseResult.latencyMs,
27161
+ errorCode: "tool_error"
27162
+ });
27163
+ }
27164
+ attemptEvidence.push({
27165
+ attempt_id: `${retryChainId}:${attempt + 1}`,
27166
+ retry_chain_id: retryChainId,
27167
+ repetition_index: repetitionIndex,
27168
+ attempt_number: attempt + 1,
27169
+ status: "error",
27170
+ included_in_outcome: false,
27171
+ latency_ms: Date.now() - attemptStartTime,
27172
+ error_code: error instanceof TimeoutError ? "timeout" : "target_error"
27173
+ });
27142
27174
  if (attempt < retries) {
27143
27175
  await sleep(2 ** attempt * 1000);
27144
27176
  }
@@ -27151,6 +27183,10 @@ async function executeCase(testCase, context) {
27151
27183
  ok: false,
27152
27184
  status: "error",
27153
27185
  attempts: retries + 1,
27186
+ attempt_evidence: attemptEvidence.map((entry, index) => ({
27187
+ ...entry,
27188
+ included_in_outcome: index === attemptEvidence.length - 1
27189
+ })),
27154
27190
  score: 0,
27155
27191
  matcherType: testCase.expected.type,
27156
27192
  reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -27160,11 +27196,34 @@ async function executeCase(testCase, context) {
27160
27196
  response: "",
27161
27197
  expected: testCase.expected,
27162
27198
  tags: testCase.tags,
27163
- error: lastError?.message
27199
+ error: lastError?.message,
27200
+ target: targetEvidence(context.client.provider, requestedModel)
27164
27201
  };
27165
27202
  }
27203
+ function withAttemptEvidence(result, priorAttempts, input) {
27204
+ return {
27205
+ ...result,
27206
+ attempts: input.attemptNumber,
27207
+ attempt_evidence: [
27208
+ ...priorAttempts,
27209
+ {
27210
+ attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
27211
+ retry_chain_id: input.retryChainId,
27212
+ repetition_index: input.repetitionIndex,
27213
+ attempt_number: input.attemptNumber,
27214
+ status: getTerminalStatus(result),
27215
+ included_in_outcome: input.includedInOutcome,
27216
+ latency_ms: input.latencyMs,
27217
+ ...input.errorCode ? { error_code: input.errorCode } : {}
27218
+ }
27219
+ ]
27220
+ };
27221
+ }
27222
+ function getTerminalStatus(result) {
27223
+ return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
27224
+ }
27166
27225
  async function executeCaseAttempt(testCase, context, timeout) {
27167
- const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
27226
+ const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
27168
27227
  const variables = mergeVariables(scenario.variables, testCase.variables);
27169
27228
  let prompt = substituteVariables(testCase.prompt, variables);
27170
27229
  if (scenario.setup?.systemPrompt && typeof prompt === "string") {
@@ -27182,7 +27241,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27182
27241
  const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
27183
27242
  const generate = () => client.generate({
27184
27243
  prompt: loopPrompt,
27185
- model: testCase.model || scenario.model,
27244
+ model: testCase.model || requestedModel || scenario.model,
27186
27245
  temperature: scenario.temperature,
27187
27246
  maxTokens: scenario.maxTokens,
27188
27247
  seed: scenario.seed,
@@ -27190,6 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27190
27249
  });
27191
27250
  const generatePromise = generate();
27192
27251
  let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
27252
+ const observedModels = [result.model];
27193
27253
  const generationMetrics = {
27194
27254
  latencyMs: result.latencyMs,
27195
27255
  tokens: { ...result.tokens }
@@ -27200,7 +27260,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27200
27260
  status: "error",
27201
27261
  steps: 0,
27202
27262
  terminationReason: "tool_error"
27203
- }, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
27263
+ }, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27204
27264
  }
27205
27265
  const executor = toolExecutor ?? new FixtureToolExecutor({
27206
27266
  tools,
@@ -27219,7 +27279,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27219
27279
  status: "error",
27220
27280
  steps: step,
27221
27281
  terminationReason: "duplicate_call"
27222
- }, generationMetrics, "TOOL_DUPLICATE_CALL");
27282
+ }, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27223
27283
  }
27224
27284
  seenCalls.add(fingerprint);
27225
27285
  const toolStart = Date.now();
@@ -27246,7 +27306,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27246
27306
  status: "error",
27247
27307
  steps: step + 1,
27248
27308
  terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
27249
- }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
27309
+ }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27250
27310
  }
27251
27311
  const content = JSON.stringify(execution.result ?? {});
27252
27312
  loopPrompt.push({
@@ -27258,7 +27318,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27258
27318
  }
27259
27319
  const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
27260
27320
  if (remainingLoopTime <= 0) {
27261
- throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
27321
+ throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27262
27322
  }
27263
27323
  const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
27264
27324
  try {
@@ -27269,8 +27329,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
27269
27329
  status: "error",
27270
27330
  steps: step + 1,
27271
27331
  terminationReason: timedOut ? "timeout" : "tool_error"
27272
- }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
27332
+ }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27273
27333
  }
27334
+ observedModels.push(result.model);
27274
27335
  generationMetrics.latencyMs += result.latencyMs;
27275
27336
  generationMetrics.tokens.prompt += result.tokens.prompt;
27276
27337
  generationMetrics.tokens.completion += result.tokens.completion;
@@ -27281,7 +27342,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27281
27342
  status: "error",
27282
27343
  steps: policy.maxSteps,
27283
27344
  terminationReason: "max_steps"
27284
- }, generationMetrics, "TOOL_LOOP_MAX_STEPS");
27345
+ }, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27285
27346
  }
27286
27347
  toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
27287
27348
  }
@@ -27377,11 +27438,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
27377
27438
  tags: testCase.tags,
27378
27439
  redaction: redactionInfo,
27379
27440
  evidence: finalEvidence,
27441
+ target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
27380
27442
  toolTrace: toolTrace.length ? toolTrace : undefined,
27381
27443
  toolLoop
27382
27444
  };
27383
27445
  }
27384
- function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code) {
27446
+ function targetEvidence(provider, requestedModel, observedModels) {
27447
+ const observed = [
27448
+ ...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
27449
+ ].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
27450
+ return {
27451
+ provider: sanitizeArtifactText(provider, 100) ?? "unknown",
27452
+ ...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
27453
+ ...observed.length ? { observed_models: observed } : {}
27454
+ };
27455
+ }
27456
+ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
27385
27457
  return new ToolLoopError(code, {
27386
27458
  id: testCase.id,
27387
27459
  name: testCase.name,
@@ -27397,6 +27469,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
27397
27469
  expected: testCase.expected,
27398
27470
  tags: testCase.tags,
27399
27471
  error: code,
27472
+ target,
27400
27473
  toolTrace,
27401
27474
  toolLoop
27402
27475
  });
@@ -27510,302 +27583,6 @@ function nanoid(size = 21) {
27510
27583
  }
27511
27584
  return id;
27512
27585
  }
27513
- var MODEL_PRICING = {
27514
- "gpt-5": {
27515
- promptPer1K: 0.00125,
27516
- completionPer1K: 0.01,
27517
- lastUpdated: "2026-01",
27518
- notes: "400K context window"
27519
- },
27520
- "gpt-5.1": {
27521
- promptPer1K: 0.00125,
27522
- completionPer1K: 0.01,
27523
- lastUpdated: "2026-01"
27524
- },
27525
- "gpt-5.2": {
27526
- promptPer1K: 0.00175,
27527
- completionPer1K: 0.014,
27528
- lastUpdated: "2026-01"
27529
- },
27530
- "gpt-5-mini": {
27531
- promptPer1K: 0.00025,
27532
- completionPer1K: 0.002,
27533
- lastUpdated: "2026-01"
27534
- },
27535
- "gpt-5-nano": {
27536
- promptPer1K: 0.00005,
27537
- completionPer1K: 0.0004,
27538
- lastUpdated: "2026-01"
27539
- },
27540
- "gpt-4.1": {
27541
- promptPer1K: 0.002,
27542
- completionPer1K: 0.008,
27543
- lastUpdated: "2026-01",
27544
- notes: "1M context window"
27545
- },
27546
- "gpt-4.1-mini": {
27547
- promptPer1K: 0.0004,
27548
- completionPer1K: 0.0016,
27549
- lastUpdated: "2026-01"
27550
- },
27551
- "gpt-4.1-nano": {
27552
- promptPer1K: 0.0001,
27553
- completionPer1K: 0.0004,
27554
- lastUpdated: "2026-01"
27555
- },
27556
- "gpt-4o": {
27557
- promptPer1K: 0.0025,
27558
- completionPer1K: 0.01,
27559
- lastUpdated: "2026-01",
27560
- notes: "128K context window"
27561
- },
27562
- "gpt-4o-mini": {
27563
- promptPer1K: 0.00015,
27564
- completionPer1K: 0.0006,
27565
- lastUpdated: "2026-01",
27566
- notes: "128K context window"
27567
- },
27568
- o1: {
27569
- promptPer1K: 0.015,
27570
- completionPer1K: 0.06,
27571
- lastUpdated: "2026-01",
27572
- notes: "Reasoning model - internal thinking tokens billed as output"
27573
- },
27574
- o3: {
27575
- promptPer1K: 0.002,
27576
- completionPer1K: 0.008,
27577
- lastUpdated: "2026-01"
27578
- },
27579
- "o3-mini": {
27580
- promptPer1K: 0.0011,
27581
- completionPer1K: 0.0044,
27582
- lastUpdated: "2026-01"
27583
- },
27584
- "o4-mini": {
27585
- promptPer1K: 0.0011,
27586
- completionPer1K: 0.0044,
27587
- lastUpdated: "2026-01"
27588
- },
27589
- "gpt-4-turbo": {
27590
- promptPer1K: 0.01,
27591
- completionPer1K: 0.03,
27592
- lastUpdated: "2026-01"
27593
- },
27594
- "gpt-4": {
27595
- promptPer1K: 0.03,
27596
- completionPer1K: 0.06,
27597
- lastUpdated: "2026-01"
27598
- },
27599
- "gpt-3.5-turbo": {
27600
- promptPer1K: 0.0005,
27601
- completionPer1K: 0.0015,
27602
- lastUpdated: "2026-01"
27603
- },
27604
- "claude-opus-4.5": {
27605
- promptPer1K: 0.005,
27606
- completionPer1K: 0.025,
27607
- lastUpdated: "2026-01",
27608
- notes: "Most capable Claude model"
27609
- },
27610
- "claude-sonnet-4.5": {
27611
- promptPer1K: 0.003,
27612
- completionPer1K: 0.015,
27613
- lastUpdated: "2026-01",
27614
- notes: "Balanced performance and cost"
27615
- },
27616
- "claude-haiku-4.5": {
27617
- promptPer1K: 0.001,
27618
- completionPer1K: 0.005,
27619
- lastUpdated: "2026-01",
27620
- notes: "Fastest Claude model"
27621
- },
27622
- "claude-opus-4": {
27623
- promptPer1K: 0.015,
27624
- completionPer1K: 0.075,
27625
- lastUpdated: "2026-01"
27626
- },
27627
- "claude-opus-4.1": {
27628
- promptPer1K: 0.015,
27629
- completionPer1K: 0.075,
27630
- lastUpdated: "2026-01"
27631
- },
27632
- "claude-sonnet-4": {
27633
- promptPer1K: 0.003,
27634
- completionPer1K: 0.015,
27635
- lastUpdated: "2026-01"
27636
- },
27637
- "claude-sonnet-3.7": {
27638
- promptPer1K: 0.003,
27639
- completionPer1K: 0.015,
27640
- lastUpdated: "2026-01"
27641
- },
27642
- "claude-3-7-sonnet": {
27643
- promptPer1K: 0.003,
27644
- completionPer1K: 0.015,
27645
- lastUpdated: "2026-01"
27646
- },
27647
- "claude-3-5-sonnet-20241022": {
27648
- promptPer1K: 0.003,
27649
- completionPer1K: 0.015,
27650
- lastUpdated: "2026-01"
27651
- },
27652
- "claude-3-5-haiku-20241022": {
27653
- promptPer1K: 0.0008,
27654
- completionPer1K: 0.004,
27655
- lastUpdated: "2026-01"
27656
- },
27657
- "claude-haiku-3.5": {
27658
- promptPer1K: 0.0008,
27659
- completionPer1K: 0.004,
27660
- lastUpdated: "2026-01"
27661
- },
27662
- "claude-3-opus": {
27663
- promptPer1K: 0.015,
27664
- completionPer1K: 0.075,
27665
- lastUpdated: "2026-01"
27666
- },
27667
- "claude-3-sonnet": {
27668
- promptPer1K: 0.003,
27669
- completionPer1K: 0.015,
27670
- lastUpdated: "2026-01"
27671
- },
27672
- "claude-3-haiku": {
27673
- promptPer1K: 0.00025,
27674
- completionPer1K: 0.00125,
27675
- lastUpdated: "2026-01"
27676
- },
27677
- "claude-3.5-sonnet": {
27678
- promptPer1K: 0.003,
27679
- completionPer1K: 0.015,
27680
- lastUpdated: "2026-01"
27681
- },
27682
- "claude-3.5-haiku": {
27683
- promptPer1K: 0.0008,
27684
- completionPer1K: 0.004,
27685
- lastUpdated: "2026-01"
27686
- }
27687
- };
27688
- var DEFAULT_PRICING = {
27689
- promptPer1K: 0.003,
27690
- completionPer1K: 0.015,
27691
- lastUpdated: "2026-01",
27692
- notes: "Default pricing - verify with provider"
27693
- };
27694
- function getModelPricing(model) {
27695
- if (MODEL_PRICING[model]) {
27696
- return MODEL_PRICING[model];
27697
- }
27698
- const lowerModel = model.toLowerCase();
27699
- for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
27700
- if (key.toLowerCase() === lowerModel) {
27701
- return pricing;
27702
- }
27703
- }
27704
- if (lowerModel.includes("gpt-5.2")) {
27705
- return MODEL_PRICING["gpt-5.2"];
27706
- }
27707
- if (lowerModel.includes("gpt-5.1")) {
27708
- return MODEL_PRICING["gpt-5.1"];
27709
- }
27710
- if (lowerModel.includes("gpt-5-mini")) {
27711
- return MODEL_PRICING["gpt-5-mini"];
27712
- }
27713
- if (lowerModel.includes("gpt-5-nano")) {
27714
- return MODEL_PRICING["gpt-5-nano"];
27715
- }
27716
- if (lowerModel.includes("gpt-5")) {
27717
- return MODEL_PRICING["gpt-5"];
27718
- }
27719
- if (lowerModel.includes("gpt-4.1-mini")) {
27720
- return MODEL_PRICING["gpt-4.1-mini"];
27721
- }
27722
- if (lowerModel.includes("gpt-4.1-nano")) {
27723
- return MODEL_PRICING["gpt-4.1-nano"];
27724
- }
27725
- if (lowerModel.includes("gpt-4.1")) {
27726
- return MODEL_PRICING["gpt-4.1"];
27727
- }
27728
- if (lowerModel.includes("gpt-4o-mini")) {
27729
- return MODEL_PRICING["gpt-4o-mini"];
27730
- }
27731
- if (lowerModel.includes("gpt-4o")) {
27732
- return MODEL_PRICING["gpt-4o"];
27733
- }
27734
- if (lowerModel.includes("o4-mini")) {
27735
- return MODEL_PRICING["o4-mini"];
27736
- }
27737
- if (lowerModel.includes("o3-mini")) {
27738
- return MODEL_PRICING["o3-mini"];
27739
- }
27740
- if (lowerModel.includes("o3")) {
27741
- return MODEL_PRICING.o3;
27742
- }
27743
- if (lowerModel.includes("o1")) {
27744
- return MODEL_PRICING.o1;
27745
- }
27746
- if (lowerModel.includes("gpt-4-turbo")) {
27747
- return MODEL_PRICING["gpt-4-turbo"];
27748
- }
27749
- if (lowerModel.includes("gpt-4")) {
27750
- return MODEL_PRICING["gpt-4"];
27751
- }
27752
- if (lowerModel.includes("gpt-3.5")) {
27753
- return MODEL_PRICING["gpt-3.5-turbo"];
27754
- }
27755
- if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
27756
- return MODEL_PRICING["claude-opus-4.5"];
27757
- }
27758
- if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
27759
- return MODEL_PRICING["claude-sonnet-4.5"];
27760
- }
27761
- if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
27762
- return MODEL_PRICING["claude-haiku-4.5"];
27763
- }
27764
- if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
27765
- return MODEL_PRICING["claude-opus-4.1"];
27766
- }
27767
- if (lowerModel.includes("opus-4")) {
27768
- return MODEL_PRICING["claude-opus-4"];
27769
- }
27770
- if (lowerModel.includes("sonnet-4")) {
27771
- return MODEL_PRICING["claude-sonnet-4"];
27772
- }
27773
- if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
27774
- return MODEL_PRICING["claude-sonnet-3.7"];
27775
- }
27776
- if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
27777
- return MODEL_PRICING["claude-3.5-sonnet"];
27778
- }
27779
- if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
27780
- return MODEL_PRICING["claude-3.5-haiku"];
27781
- }
27782
- if (lowerModel.includes("claude-3-opus")) {
27783
- return MODEL_PRICING["claude-3-opus"];
27784
- }
27785
- if (lowerModel.includes("claude-3-sonnet")) {
27786
- return MODEL_PRICING["claude-3-sonnet"];
27787
- }
27788
- if (lowerModel.includes("claude-3-haiku")) {
27789
- return MODEL_PRICING["claude-3-haiku"];
27790
- }
27791
- if (lowerModel.includes("claude")) {
27792
- return MODEL_PRICING["claude-sonnet-4.5"];
27793
- }
27794
- return DEFAULT_PRICING;
27795
- }
27796
- function estimateCost(promptTokens, completionTokens, model) {
27797
- const pricing = getModelPricing(model);
27798
- const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
27799
- const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
27800
- const totalUsd = promptCostUsd + completionCostUsd;
27801
- return {
27802
- totalUsd,
27803
- promptCostUsd,
27804
- completionCostUsd,
27805
- model,
27806
- pricing
27807
- };
27808
- }
27809
27586
  function getEnvironmentInfo() {
27810
27587
  return {
27811
27588
  node_version: process.version,
@@ -27861,6 +27638,15 @@ function assertRunManifestIntegrity(manifest) {
27861
27638
  if (manifest.workload_identity !== undefined) {
27862
27639
  assertWorkloadIdentity(manifest.workload_identity);
27863
27640
  }
27641
+ if (manifest.execution_provenance !== undefined) {
27642
+ assertExecutionProvenance(manifest.execution_provenance);
27643
+ }
27644
+ if (manifest.attempt_evidence !== undefined) {
27645
+ assertRunAttemptEvidence(manifest.attempt_evidence);
27646
+ }
27647
+ if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
27648
+ assertCostProvenance(manifest.metrics.cost_provenance);
27649
+ }
27864
27650
  for (const [index, caseResult] of manifest.cases.entries()) {
27865
27651
  if (!isRecord2(caseResult)) {
27866
27652
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27871,8 +27657,91 @@ function assertRunManifestIntegrity(manifest) {
27871
27657
  if (caseResult.evidence !== undefined) {
27872
27658
  assertCaseEvaluationEvidence(caseResult.evidence, index);
27873
27659
  }
27660
+ if (caseResult.target !== undefined) {
27661
+ assertCaseTargetEvidence(caseResult.target);
27662
+ }
27663
+ if (caseResult.attempt_evidence !== undefined) {
27664
+ assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
27665
+ }
27666
+ }
27667
+ }
27668
+ function assertRunAttemptEvidence(evidence) {
27669
+ if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
27670
+ throw new Error("Invalid run manifest: malformed attempt evidence");
27671
+ }
27672
+ }
27673
+ function assertCaseAttemptEvidence(evidence, index) {
27674
+ if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
27675
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27676
+ }
27677
+ for (const attempt of evidence) {
27678
+ if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
27679
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27680
+ }
27681
+ }
27682
+ }
27683
+ function assertCostProvenance(cost) {
27684
+ if (!isRecord2(cost) || cost.schema_version !== "1") {
27685
+ throw new Error("Invalid run manifest: malformed cost provenance");
27686
+ }
27687
+ if (cost.status === "unavailable") {
27688
+ if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
27689
+ throw new Error("Invalid run manifest: malformed cost provenance");
27690
+ }
27691
+ return;
27692
+ }
27693
+ if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
27694
+ throw new Error("Invalid run manifest: malformed cost provenance");
27695
+ }
27696
+ }
27697
+ function isCaseEvaluationStatus(value) {
27698
+ return value === "passed" || value === "failed" || value === "invalid" || value === "error";
27699
+ }
27700
+ function isBoundedNonemptyString(value, maxLength) {
27701
+ return typeof value === "string" && value.length > 0 && value.length <= maxLength;
27702
+ }
27703
+ function isNonnegativeFiniteNumber(value) {
27704
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
27705
+ }
27706
+ function isPositiveFiniteNumber(value) {
27707
+ return isNonnegativeFiniteNumber(value) && value > 0;
27708
+ }
27709
+ function isNonnegativeSafeInteger(value) {
27710
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
27711
+ }
27712
+ function isPositiveSafeInteger(value) {
27713
+ return isNonnegativeSafeInteger(value) && value > 0;
27714
+ }
27715
+ function isIsoTimestamp(value) {
27716
+ return typeof value === "string" && Number.isFinite(Date.parse(value));
27717
+ }
27718
+ function assertCaseTargetEvidence(target) {
27719
+ if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
27720
+ throw new Error("Invalid run manifest: malformed target evidence");
27721
+ }
27722
+ }
27723
+ function assertExecutionProvenance(provenance) {
27724
+ if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
27725
+ throw new Error("Invalid run manifest: malformed execution provenance");
27726
+ }
27727
+ const target = provenance.target;
27728
+ if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
27729
+ throw new Error("Invalid run manifest: malformed execution provenance");
27730
+ }
27731
+ if (provenance.evaluator !== undefined) {
27732
+ if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
27733
+ throw new Error("Invalid run manifest: malformed execution provenance");
27734
+ }
27874
27735
  }
27875
27736
  }
27737
+ function isBoundedStringList(value) {
27738
+ return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
27739
+ }
27740
+ function isGenerationConfig(value) {
27741
+ if (!isRecord2(value))
27742
+ return false;
27743
+ return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
27744
+ }
27876
27745
  function assertWorkloadIdentity(identity) {
27877
27746
  if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
27878
27747
  throw new Error("Invalid run manifest: malformed workload identity");
@@ -27921,6 +27790,10 @@ function createRunManifest(options) {
27921
27790
  config,
27922
27791
  resolvedConfig,
27923
27792
  workloadIdentity,
27793
+ executionProvenance,
27794
+ attemptEvidence,
27795
+ costProvenance,
27796
+ runId,
27924
27797
  cases,
27925
27798
  startTime,
27926
27799
  endTime,
@@ -27928,13 +27801,12 @@ function createRunManifest(options) {
27928
27801
  runReason,
27929
27802
  redaction
27930
27803
  } = options;
27931
- const modelForCost = resolvedConfig?.model || config.model;
27932
- const metrics = calculateMetrics(cases, modelForCost);
27804
+ const metrics = calculateMetrics(cases, costProvenance);
27933
27805
  const git = getGitInfo();
27934
27806
  const environment = getEnvironmentInfo();
27935
27807
  return {
27936
- version: "1.2",
27937
- run_id: nanoid(12),
27808
+ version: "1.4",
27809
+ run_id: runId ?? nanoid(12),
27938
27810
  project,
27939
27811
  start_time: startTime.toISOString(),
27940
27812
  end_time: endTime.toISOString(),
@@ -27942,6 +27814,8 @@ function createRunManifest(options) {
27942
27814
  config,
27943
27815
  resolved_config: resolvedConfig,
27944
27816
  workload_identity: workloadIdentity,
27817
+ execution_provenance: executionProvenance,
27818
+ attempt_evidence: attemptEvidence,
27945
27819
  metrics,
27946
27820
  git,
27947
27821
  provenance: {
@@ -27954,7 +27828,7 @@ function createRunManifest(options) {
27954
27828
  redaction
27955
27829
  };
27956
27830
  }
27957
- function calculateMetrics(cases, model) {
27831
+ function calculateMetrics(cases, costProvenance) {
27958
27832
  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
27959
27833
  const validCases = cases.filter((c) => {
27960
27834
  const status = getCaseEvaluationStatus(c);
@@ -27966,21 +27840,11 @@ function calculateMetrics(cases, model) {
27966
27840
  const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
27967
27841
  const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
27968
27842
  const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
27969
- let cost;
27970
- if (model && !model.toLowerCase().includes("ling-") && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
27971
- const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
27972
- const pricing = getModelPricing(model);
27973
- cost = {
27974
- total_usd: costEstimate.totalUsd,
27975
- prompt_cost_usd: costEstimate.promptCostUsd,
27976
- completion_cost_usd: costEstimate.completionCostUsd,
27977
- model: costEstimate.model,
27978
- pricing: {
27979
- prompt_per_1k: pricing.promptPer1K,
27980
- completion_per_1k: pricing.completionPer1K
27981
- }
27982
- };
27983
- }
27843
+ const cost_provenance = costProvenance ?? {
27844
+ schema_version: "1",
27845
+ status: "unavailable",
27846
+ unavailable_reason: "provider_billing_not_recorded"
27847
+ };
27984
27848
  return {
27985
27849
  success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
27986
27850
  total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
@@ -27995,7 +27859,7 @@ function calculateMetrics(cases, model) {
27995
27859
  total_tokens: totalPromptTokens + totalCompletionTokens,
27996
27860
  total_prompt_tokens: totalPromptTokens,
27997
27861
  total_completion_tokens: totalCompletionTokens,
27998
- cost
27862
+ cost_provenance
27999
27863
  };
28000
27864
  }
28001
27865
  function detectCIEnvironment() {
@@ -28092,6 +27956,38 @@ function canonicalize(value, patterns, key) {
28092
27956
  }
28093
27957
  return value;
28094
27958
  }
27959
+ function createExecutionProvenance(options) {
27960
+ const requestedModels = uniqueStrings([
27961
+ options.requestedModel,
27962
+ ...options.cases.map((caseResult) => caseResult.target?.requested_model)
27963
+ ]);
27964
+ const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
27965
+ const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
27966
+ const generation = omitUndefined({
27967
+ temperature: options.temperature,
27968
+ max_tokens: options.maxTokens,
27969
+ seed: options.seed
27970
+ });
27971
+ return {
27972
+ schema_version: "1",
27973
+ target: {
27974
+ provider: boundedString(options.provider, 100) ?? "unknown",
27975
+ ...requestedModels.length ? { requested_models: requestedModels } : {},
27976
+ ...observedModels.length ? { observed_models: observedModels } : {},
27977
+ ...Object.keys(generation).length ? { generation } : {}
27978
+ },
27979
+ ...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
27980
+ };
27981
+ }
27982
+ function uniqueStrings(values) {
27983
+ return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
27984
+ }
27985
+ function boundedString(value, maxLength) {
27986
+ return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
27987
+ }
27988
+ function omitUndefined(value) {
27989
+ return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
27990
+ }
28095
27991
  async function runScenario(options) {
28096
27992
  const {
28097
27993
  scenario,
@@ -28102,6 +27998,8 @@ async function runScenario(options) {
28102
27998
  concurrency = 1,
28103
27999
  timeout,
28104
28000
  retries,
28001
+ repetition = { index: 1, total: 1 },
28002
+ costProvenance,
28105
28003
  redaction,
28106
28004
  toolExecutor,
28107
28005
  onCaseComplete,
@@ -28117,6 +28015,7 @@ async function runScenario(options) {
28117
28015
  }
28118
28016
  onProgress?.(`Running ${cases.length} test cases...`);
28119
28017
  const startTime = new Date;
28018
+ const runId = nanoid(12);
28120
28019
  const results = [];
28121
28020
  if (concurrency === 1) {
28122
28021
  for (let i = 0;i < cases.length; i++) {
@@ -28124,8 +28023,11 @@ async function runScenario(options) {
28124
28023
  const result = await executeCase(testCase, {
28125
28024
  client,
28126
28025
  scenario,
28026
+ requestedModel: resolvedConfig?.model,
28127
28027
  timeout: testCase.timeout || timeout,
28128
28028
  retries: testCase.retries ?? retries,
28029
+ runId,
28030
+ repetition,
28129
28031
  redaction,
28130
28032
  toolExecutor
28131
28033
  });
@@ -28140,8 +28042,11 @@ async function runScenario(options) {
28140
28042
  const result = await executeCase(testCase, {
28141
28043
  client,
28142
28044
  scenario,
28045
+ requestedModel: resolvedConfig?.model,
28143
28046
  timeout: testCase.timeout || timeout,
28144
28047
  retries: testCase.retries ?? retries,
28048
+ runId,
28049
+ repetition,
28145
28050
  redaction,
28146
28051
  toolExecutor
28147
28052
  });
@@ -28184,9 +28089,29 @@ async function runScenario(options) {
28184
28089
  },
28185
28090
  resolvedConfig,
28186
28091
  workloadIdentity: createWorkloadIdentity(scenario),
28092
+ executionProvenance: createExecutionProvenance({
28093
+ provider: client.provider,
28094
+ requestedModel: resolvedConfig?.model || scenario.model,
28095
+ temperature: resolvedConfig?.temperature ?? scenario.temperature,
28096
+ maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
28097
+ seed: scenario.seed,
28098
+ cases: results
28099
+ }),
28100
+ attemptEvidence: {
28101
+ schema_version: "1",
28102
+ repetition,
28103
+ retry_policy: {
28104
+ default_max_retries: retries ?? 0,
28105
+ backoff: "exponential",
28106
+ initial_delay_ms: 1000
28107
+ },
28108
+ ...timeout ? { timeout: { default_ms: timeout } } : {}
28109
+ },
28110
+ costProvenance,
28187
28111
  cases: results,
28188
28112
  startTime,
28189
28113
  endTime,
28114
+ runId,
28190
28115
  redaction: redactionInfo
28191
28116
  });
28192
28117
  const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.3",
3
+ "version": "0.4.5",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.5.0",
75
- "@artemiskit/redteam": "0.3.4"
74
+ "@artemiskit/core": "0.5.2",
75
+ "@artemiskit/redteam": "0.3.6"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",