@artemiskit/sdk 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/index.js +153 -321
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
package/dist/index.js
CHANGED
|
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
|
|
|
26484
26484
|
tags: arrayType(stringType()).optional().default([]),
|
|
26485
26485
|
metadata: recordType(unknownType()).optional().default({}),
|
|
26486
26486
|
timeout: numberType().optional(),
|
|
26487
|
-
retries: numberType().optional().default(0),
|
|
26487
|
+
retries: numberType().int().min(0).max(99).optional().default(0),
|
|
26488
26488
|
provider: ProviderSchema.optional(),
|
|
26489
26489
|
model: stringType().optional(),
|
|
26490
26490
|
variables: VariablesSchema.optional(),
|
|
@@ -27129,17 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
27129
27129
|
}
|
|
27130
27130
|
async function executeCase(testCase, context) {
|
|
27131
27131
|
const { timeout, retries = 0 } = context;
|
|
27132
|
+
if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
|
|
27133
|
+
throw new RangeError("retries must be a whole number between 0 and 99");
|
|
27134
|
+
}
|
|
27132
27135
|
const caseStartTime = Date.now();
|
|
27133
27136
|
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
27137
|
+
const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
|
|
27138
|
+
const repetitionIndex = context.repetition?.index ?? 1;
|
|
27139
|
+
const attemptEvidence = [];
|
|
27134
27140
|
let lastError = null;
|
|
27135
27141
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
27142
|
+
const attemptStartTime = Date.now();
|
|
27136
27143
|
try {
|
|
27137
27144
|
const result = await executeCaseAttempt(testCase, context, timeout);
|
|
27138
|
-
return
|
|
27145
|
+
return withAttemptEvidence(result, attemptEvidence, {
|
|
27146
|
+
retryChainId,
|
|
27147
|
+
repetitionIndex,
|
|
27148
|
+
attemptNumber: attempt + 1,
|
|
27149
|
+
includedInOutcome: true,
|
|
27150
|
+
latencyMs: result.latencyMs
|
|
27151
|
+
});
|
|
27139
27152
|
} catch (error) {
|
|
27140
27153
|
lastError = error;
|
|
27141
|
-
if (error instanceof ToolLoopError)
|
|
27142
|
-
return
|
|
27154
|
+
if (error instanceof ToolLoopError) {
|
|
27155
|
+
return withAttemptEvidence(error.caseResult, attemptEvidence, {
|
|
27156
|
+
retryChainId,
|
|
27157
|
+
repetitionIndex,
|
|
27158
|
+
attemptNumber: attempt + 1,
|
|
27159
|
+
includedInOutcome: true,
|
|
27160
|
+
latencyMs: error.caseResult.latencyMs,
|
|
27161
|
+
errorCode: "tool_error"
|
|
27162
|
+
});
|
|
27163
|
+
}
|
|
27164
|
+
attemptEvidence.push({
|
|
27165
|
+
attempt_id: `${retryChainId}:${attempt + 1}`,
|
|
27166
|
+
retry_chain_id: retryChainId,
|
|
27167
|
+
repetition_index: repetitionIndex,
|
|
27168
|
+
attempt_number: attempt + 1,
|
|
27169
|
+
status: "error",
|
|
27170
|
+
included_in_outcome: false,
|
|
27171
|
+
latency_ms: Date.now() - attemptStartTime,
|
|
27172
|
+
error_code: error instanceof TimeoutError ? "timeout" : "target_error"
|
|
27173
|
+
});
|
|
27143
27174
|
if (attempt < retries) {
|
|
27144
27175
|
await sleep(2 ** attempt * 1000);
|
|
27145
27176
|
}
|
|
@@ -27152,6 +27183,10 @@ async function executeCase(testCase, context) {
|
|
|
27152
27183
|
ok: false,
|
|
27153
27184
|
status: "error",
|
|
27154
27185
|
attempts: retries + 1,
|
|
27186
|
+
attempt_evidence: attemptEvidence.map((entry, index) => ({
|
|
27187
|
+
...entry,
|
|
27188
|
+
included_in_outcome: index === attemptEvidence.length - 1
|
|
27189
|
+
})),
|
|
27155
27190
|
score: 0,
|
|
27156
27191
|
matcherType: testCase.expected.type,
|
|
27157
27192
|
reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
|
|
@@ -27165,6 +27200,28 @@ async function executeCase(testCase, context) {
|
|
|
27165
27200
|
target: targetEvidence(context.client.provider, requestedModel)
|
|
27166
27201
|
};
|
|
27167
27202
|
}
|
|
27203
|
+
function withAttemptEvidence(result, priorAttempts, input) {
|
|
27204
|
+
return {
|
|
27205
|
+
...result,
|
|
27206
|
+
attempts: input.attemptNumber,
|
|
27207
|
+
attempt_evidence: [
|
|
27208
|
+
...priorAttempts,
|
|
27209
|
+
{
|
|
27210
|
+
attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
|
|
27211
|
+
retry_chain_id: input.retryChainId,
|
|
27212
|
+
repetition_index: input.repetitionIndex,
|
|
27213
|
+
attempt_number: input.attemptNumber,
|
|
27214
|
+
status: getTerminalStatus(result),
|
|
27215
|
+
included_in_outcome: input.includedInOutcome,
|
|
27216
|
+
latency_ms: input.latencyMs,
|
|
27217
|
+
...input.errorCode ? { error_code: input.errorCode } : {}
|
|
27218
|
+
}
|
|
27219
|
+
]
|
|
27220
|
+
};
|
|
27221
|
+
}
|
|
27222
|
+
function getTerminalStatus(result) {
|
|
27223
|
+
return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
|
|
27224
|
+
}
|
|
27168
27225
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
27169
27226
|
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
27170
27227
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
@@ -27526,302 +27583,6 @@ function nanoid(size = 21) {
|
|
|
27526
27583
|
}
|
|
27527
27584
|
return id;
|
|
27528
27585
|
}
|
|
27529
|
-
var MODEL_PRICING = {
|
|
27530
|
-
"gpt-5": {
|
|
27531
|
-
promptPer1K: 0.00125,
|
|
27532
|
-
completionPer1K: 0.01,
|
|
27533
|
-
lastUpdated: "2026-01",
|
|
27534
|
-
notes: "400K context window"
|
|
27535
|
-
},
|
|
27536
|
-
"gpt-5.1": {
|
|
27537
|
-
promptPer1K: 0.00125,
|
|
27538
|
-
completionPer1K: 0.01,
|
|
27539
|
-
lastUpdated: "2026-01"
|
|
27540
|
-
},
|
|
27541
|
-
"gpt-5.2": {
|
|
27542
|
-
promptPer1K: 0.00175,
|
|
27543
|
-
completionPer1K: 0.014,
|
|
27544
|
-
lastUpdated: "2026-01"
|
|
27545
|
-
},
|
|
27546
|
-
"gpt-5-mini": {
|
|
27547
|
-
promptPer1K: 0.00025,
|
|
27548
|
-
completionPer1K: 0.002,
|
|
27549
|
-
lastUpdated: "2026-01"
|
|
27550
|
-
},
|
|
27551
|
-
"gpt-5-nano": {
|
|
27552
|
-
promptPer1K: 0.00005,
|
|
27553
|
-
completionPer1K: 0.0004,
|
|
27554
|
-
lastUpdated: "2026-01"
|
|
27555
|
-
},
|
|
27556
|
-
"gpt-4.1": {
|
|
27557
|
-
promptPer1K: 0.002,
|
|
27558
|
-
completionPer1K: 0.008,
|
|
27559
|
-
lastUpdated: "2026-01",
|
|
27560
|
-
notes: "1M context window"
|
|
27561
|
-
},
|
|
27562
|
-
"gpt-4.1-mini": {
|
|
27563
|
-
promptPer1K: 0.0004,
|
|
27564
|
-
completionPer1K: 0.0016,
|
|
27565
|
-
lastUpdated: "2026-01"
|
|
27566
|
-
},
|
|
27567
|
-
"gpt-4.1-nano": {
|
|
27568
|
-
promptPer1K: 0.0001,
|
|
27569
|
-
completionPer1K: 0.0004,
|
|
27570
|
-
lastUpdated: "2026-01"
|
|
27571
|
-
},
|
|
27572
|
-
"gpt-4o": {
|
|
27573
|
-
promptPer1K: 0.0025,
|
|
27574
|
-
completionPer1K: 0.01,
|
|
27575
|
-
lastUpdated: "2026-01",
|
|
27576
|
-
notes: "128K context window"
|
|
27577
|
-
},
|
|
27578
|
-
"gpt-4o-mini": {
|
|
27579
|
-
promptPer1K: 0.00015,
|
|
27580
|
-
completionPer1K: 0.0006,
|
|
27581
|
-
lastUpdated: "2026-01",
|
|
27582
|
-
notes: "128K context window"
|
|
27583
|
-
},
|
|
27584
|
-
o1: {
|
|
27585
|
-
promptPer1K: 0.015,
|
|
27586
|
-
completionPer1K: 0.06,
|
|
27587
|
-
lastUpdated: "2026-01",
|
|
27588
|
-
notes: "Reasoning model - internal thinking tokens billed as output"
|
|
27589
|
-
},
|
|
27590
|
-
o3: {
|
|
27591
|
-
promptPer1K: 0.002,
|
|
27592
|
-
completionPer1K: 0.008,
|
|
27593
|
-
lastUpdated: "2026-01"
|
|
27594
|
-
},
|
|
27595
|
-
"o3-mini": {
|
|
27596
|
-
promptPer1K: 0.0011,
|
|
27597
|
-
completionPer1K: 0.0044,
|
|
27598
|
-
lastUpdated: "2026-01"
|
|
27599
|
-
},
|
|
27600
|
-
"o4-mini": {
|
|
27601
|
-
promptPer1K: 0.0011,
|
|
27602
|
-
completionPer1K: 0.0044,
|
|
27603
|
-
lastUpdated: "2026-01"
|
|
27604
|
-
},
|
|
27605
|
-
"gpt-4-turbo": {
|
|
27606
|
-
promptPer1K: 0.01,
|
|
27607
|
-
completionPer1K: 0.03,
|
|
27608
|
-
lastUpdated: "2026-01"
|
|
27609
|
-
},
|
|
27610
|
-
"gpt-4": {
|
|
27611
|
-
promptPer1K: 0.03,
|
|
27612
|
-
completionPer1K: 0.06,
|
|
27613
|
-
lastUpdated: "2026-01"
|
|
27614
|
-
},
|
|
27615
|
-
"gpt-3.5-turbo": {
|
|
27616
|
-
promptPer1K: 0.0005,
|
|
27617
|
-
completionPer1K: 0.0015,
|
|
27618
|
-
lastUpdated: "2026-01"
|
|
27619
|
-
},
|
|
27620
|
-
"claude-opus-4.5": {
|
|
27621
|
-
promptPer1K: 0.005,
|
|
27622
|
-
completionPer1K: 0.025,
|
|
27623
|
-
lastUpdated: "2026-01",
|
|
27624
|
-
notes: "Most capable Claude model"
|
|
27625
|
-
},
|
|
27626
|
-
"claude-sonnet-4.5": {
|
|
27627
|
-
promptPer1K: 0.003,
|
|
27628
|
-
completionPer1K: 0.015,
|
|
27629
|
-
lastUpdated: "2026-01",
|
|
27630
|
-
notes: "Balanced performance and cost"
|
|
27631
|
-
},
|
|
27632
|
-
"claude-haiku-4.5": {
|
|
27633
|
-
promptPer1K: 0.001,
|
|
27634
|
-
completionPer1K: 0.005,
|
|
27635
|
-
lastUpdated: "2026-01",
|
|
27636
|
-
notes: "Fastest Claude model"
|
|
27637
|
-
},
|
|
27638
|
-
"claude-opus-4": {
|
|
27639
|
-
promptPer1K: 0.015,
|
|
27640
|
-
completionPer1K: 0.075,
|
|
27641
|
-
lastUpdated: "2026-01"
|
|
27642
|
-
},
|
|
27643
|
-
"claude-opus-4.1": {
|
|
27644
|
-
promptPer1K: 0.015,
|
|
27645
|
-
completionPer1K: 0.075,
|
|
27646
|
-
lastUpdated: "2026-01"
|
|
27647
|
-
},
|
|
27648
|
-
"claude-sonnet-4": {
|
|
27649
|
-
promptPer1K: 0.003,
|
|
27650
|
-
completionPer1K: 0.015,
|
|
27651
|
-
lastUpdated: "2026-01"
|
|
27652
|
-
},
|
|
27653
|
-
"claude-sonnet-3.7": {
|
|
27654
|
-
promptPer1K: 0.003,
|
|
27655
|
-
completionPer1K: 0.015,
|
|
27656
|
-
lastUpdated: "2026-01"
|
|
27657
|
-
},
|
|
27658
|
-
"claude-3-7-sonnet": {
|
|
27659
|
-
promptPer1K: 0.003,
|
|
27660
|
-
completionPer1K: 0.015,
|
|
27661
|
-
lastUpdated: "2026-01"
|
|
27662
|
-
},
|
|
27663
|
-
"claude-3-5-sonnet-20241022": {
|
|
27664
|
-
promptPer1K: 0.003,
|
|
27665
|
-
completionPer1K: 0.015,
|
|
27666
|
-
lastUpdated: "2026-01"
|
|
27667
|
-
},
|
|
27668
|
-
"claude-3-5-haiku-20241022": {
|
|
27669
|
-
promptPer1K: 0.0008,
|
|
27670
|
-
completionPer1K: 0.004,
|
|
27671
|
-
lastUpdated: "2026-01"
|
|
27672
|
-
},
|
|
27673
|
-
"claude-haiku-3.5": {
|
|
27674
|
-
promptPer1K: 0.0008,
|
|
27675
|
-
completionPer1K: 0.004,
|
|
27676
|
-
lastUpdated: "2026-01"
|
|
27677
|
-
},
|
|
27678
|
-
"claude-3-opus": {
|
|
27679
|
-
promptPer1K: 0.015,
|
|
27680
|
-
completionPer1K: 0.075,
|
|
27681
|
-
lastUpdated: "2026-01"
|
|
27682
|
-
},
|
|
27683
|
-
"claude-3-sonnet": {
|
|
27684
|
-
promptPer1K: 0.003,
|
|
27685
|
-
completionPer1K: 0.015,
|
|
27686
|
-
lastUpdated: "2026-01"
|
|
27687
|
-
},
|
|
27688
|
-
"claude-3-haiku": {
|
|
27689
|
-
promptPer1K: 0.00025,
|
|
27690
|
-
completionPer1K: 0.00125,
|
|
27691
|
-
lastUpdated: "2026-01"
|
|
27692
|
-
},
|
|
27693
|
-
"claude-3.5-sonnet": {
|
|
27694
|
-
promptPer1K: 0.003,
|
|
27695
|
-
completionPer1K: 0.015,
|
|
27696
|
-
lastUpdated: "2026-01"
|
|
27697
|
-
},
|
|
27698
|
-
"claude-3.5-haiku": {
|
|
27699
|
-
promptPer1K: 0.0008,
|
|
27700
|
-
completionPer1K: 0.004,
|
|
27701
|
-
lastUpdated: "2026-01"
|
|
27702
|
-
}
|
|
27703
|
-
};
|
|
27704
|
-
var DEFAULT_PRICING = {
|
|
27705
|
-
promptPer1K: 0.003,
|
|
27706
|
-
completionPer1K: 0.015,
|
|
27707
|
-
lastUpdated: "2026-01",
|
|
27708
|
-
notes: "Default pricing - verify with provider"
|
|
27709
|
-
};
|
|
27710
|
-
function getModelPricing(model) {
|
|
27711
|
-
if (MODEL_PRICING[model]) {
|
|
27712
|
-
return MODEL_PRICING[model];
|
|
27713
|
-
}
|
|
27714
|
-
const lowerModel = model.toLowerCase();
|
|
27715
|
-
for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
|
|
27716
|
-
if (key.toLowerCase() === lowerModel) {
|
|
27717
|
-
return pricing;
|
|
27718
|
-
}
|
|
27719
|
-
}
|
|
27720
|
-
if (lowerModel.includes("gpt-5.2")) {
|
|
27721
|
-
return MODEL_PRICING["gpt-5.2"];
|
|
27722
|
-
}
|
|
27723
|
-
if (lowerModel.includes("gpt-5.1")) {
|
|
27724
|
-
return MODEL_PRICING["gpt-5.1"];
|
|
27725
|
-
}
|
|
27726
|
-
if (lowerModel.includes("gpt-5-mini")) {
|
|
27727
|
-
return MODEL_PRICING["gpt-5-mini"];
|
|
27728
|
-
}
|
|
27729
|
-
if (lowerModel.includes("gpt-5-nano")) {
|
|
27730
|
-
return MODEL_PRICING["gpt-5-nano"];
|
|
27731
|
-
}
|
|
27732
|
-
if (lowerModel.includes("gpt-5")) {
|
|
27733
|
-
return MODEL_PRICING["gpt-5"];
|
|
27734
|
-
}
|
|
27735
|
-
if (lowerModel.includes("gpt-4.1-mini")) {
|
|
27736
|
-
return MODEL_PRICING["gpt-4.1-mini"];
|
|
27737
|
-
}
|
|
27738
|
-
if (lowerModel.includes("gpt-4.1-nano")) {
|
|
27739
|
-
return MODEL_PRICING["gpt-4.1-nano"];
|
|
27740
|
-
}
|
|
27741
|
-
if (lowerModel.includes("gpt-4.1")) {
|
|
27742
|
-
return MODEL_PRICING["gpt-4.1"];
|
|
27743
|
-
}
|
|
27744
|
-
if (lowerModel.includes("gpt-4o-mini")) {
|
|
27745
|
-
return MODEL_PRICING["gpt-4o-mini"];
|
|
27746
|
-
}
|
|
27747
|
-
if (lowerModel.includes("gpt-4o")) {
|
|
27748
|
-
return MODEL_PRICING["gpt-4o"];
|
|
27749
|
-
}
|
|
27750
|
-
if (lowerModel.includes("o4-mini")) {
|
|
27751
|
-
return MODEL_PRICING["o4-mini"];
|
|
27752
|
-
}
|
|
27753
|
-
if (lowerModel.includes("o3-mini")) {
|
|
27754
|
-
return MODEL_PRICING["o3-mini"];
|
|
27755
|
-
}
|
|
27756
|
-
if (lowerModel.includes("o3")) {
|
|
27757
|
-
return MODEL_PRICING.o3;
|
|
27758
|
-
}
|
|
27759
|
-
if (lowerModel.includes("o1")) {
|
|
27760
|
-
return MODEL_PRICING.o1;
|
|
27761
|
-
}
|
|
27762
|
-
if (lowerModel.includes("gpt-4-turbo")) {
|
|
27763
|
-
return MODEL_PRICING["gpt-4-turbo"];
|
|
27764
|
-
}
|
|
27765
|
-
if (lowerModel.includes("gpt-4")) {
|
|
27766
|
-
return MODEL_PRICING["gpt-4"];
|
|
27767
|
-
}
|
|
27768
|
-
if (lowerModel.includes("gpt-3.5")) {
|
|
27769
|
-
return MODEL_PRICING["gpt-3.5-turbo"];
|
|
27770
|
-
}
|
|
27771
|
-
if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
|
|
27772
|
-
return MODEL_PRICING["claude-opus-4.5"];
|
|
27773
|
-
}
|
|
27774
|
-
if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
|
|
27775
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27776
|
-
}
|
|
27777
|
-
if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
|
|
27778
|
-
return MODEL_PRICING["claude-haiku-4.5"];
|
|
27779
|
-
}
|
|
27780
|
-
if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
|
|
27781
|
-
return MODEL_PRICING["claude-opus-4.1"];
|
|
27782
|
-
}
|
|
27783
|
-
if (lowerModel.includes("opus-4")) {
|
|
27784
|
-
return MODEL_PRICING["claude-opus-4"];
|
|
27785
|
-
}
|
|
27786
|
-
if (lowerModel.includes("sonnet-4")) {
|
|
27787
|
-
return MODEL_PRICING["claude-sonnet-4"];
|
|
27788
|
-
}
|
|
27789
|
-
if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
|
|
27790
|
-
return MODEL_PRICING["claude-sonnet-3.7"];
|
|
27791
|
-
}
|
|
27792
|
-
if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
|
|
27793
|
-
return MODEL_PRICING["claude-3.5-sonnet"];
|
|
27794
|
-
}
|
|
27795
|
-
if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
|
|
27796
|
-
return MODEL_PRICING["claude-3.5-haiku"];
|
|
27797
|
-
}
|
|
27798
|
-
if (lowerModel.includes("claude-3-opus")) {
|
|
27799
|
-
return MODEL_PRICING["claude-3-opus"];
|
|
27800
|
-
}
|
|
27801
|
-
if (lowerModel.includes("claude-3-sonnet")) {
|
|
27802
|
-
return MODEL_PRICING["claude-3-sonnet"];
|
|
27803
|
-
}
|
|
27804
|
-
if (lowerModel.includes("claude-3-haiku")) {
|
|
27805
|
-
return MODEL_PRICING["claude-3-haiku"];
|
|
27806
|
-
}
|
|
27807
|
-
if (lowerModel.includes("claude")) {
|
|
27808
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27809
|
-
}
|
|
27810
|
-
return DEFAULT_PRICING;
|
|
27811
|
-
}
|
|
27812
|
-
function estimateCost(promptTokens, completionTokens, model) {
|
|
27813
|
-
const pricing = getModelPricing(model);
|
|
27814
|
-
const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
|
|
27815
|
-
const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
|
|
27816
|
-
const totalUsd = promptCostUsd + completionCostUsd;
|
|
27817
|
-
return {
|
|
27818
|
-
totalUsd,
|
|
27819
|
-
promptCostUsd,
|
|
27820
|
-
completionCostUsd,
|
|
27821
|
-
model,
|
|
27822
|
-
pricing
|
|
27823
|
-
};
|
|
27824
|
-
}
|
|
27825
27586
|
function getEnvironmentInfo() {
|
|
27826
27587
|
return {
|
|
27827
27588
|
node_version: process.version,
|
|
@@ -27880,6 +27641,12 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27880
27641
|
if (manifest.execution_provenance !== undefined) {
|
|
27881
27642
|
assertExecutionProvenance(manifest.execution_provenance);
|
|
27882
27643
|
}
|
|
27644
|
+
if (manifest.attempt_evidence !== undefined) {
|
|
27645
|
+
assertRunAttemptEvidence(manifest.attempt_evidence);
|
|
27646
|
+
}
|
|
27647
|
+
if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
|
|
27648
|
+
assertCostProvenance(manifest.metrics.cost_provenance);
|
|
27649
|
+
}
|
|
27883
27650
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27884
27651
|
if (!isRecord2(caseResult)) {
|
|
27885
27652
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27893,8 +27660,61 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27893
27660
|
if (caseResult.target !== undefined) {
|
|
27894
27661
|
assertCaseTargetEvidence(caseResult.target);
|
|
27895
27662
|
}
|
|
27663
|
+
if (caseResult.attempt_evidence !== undefined) {
|
|
27664
|
+
assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
|
|
27665
|
+
}
|
|
27896
27666
|
}
|
|
27897
27667
|
}
|
|
27668
|
+
function assertRunAttemptEvidence(evidence) {
|
|
27669
|
+
if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
|
|
27670
|
+
throw new Error("Invalid run manifest: malformed attempt evidence");
|
|
27671
|
+
}
|
|
27672
|
+
}
|
|
27673
|
+
function assertCaseAttemptEvidence(evidence, index) {
|
|
27674
|
+
if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
|
|
27675
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27676
|
+
}
|
|
27677
|
+
for (const attempt of evidence) {
|
|
27678
|
+
if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
|
|
27679
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27680
|
+
}
|
|
27681
|
+
}
|
|
27682
|
+
}
|
|
27683
|
+
function assertCostProvenance(cost) {
|
|
27684
|
+
if (!isRecord2(cost) || cost.schema_version !== "1") {
|
|
27685
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27686
|
+
}
|
|
27687
|
+
if (cost.status === "unavailable") {
|
|
27688
|
+
if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
|
|
27689
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27690
|
+
}
|
|
27691
|
+
return;
|
|
27692
|
+
}
|
|
27693
|
+
if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
|
|
27694
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27695
|
+
}
|
|
27696
|
+
}
|
|
27697
|
+
function isCaseEvaluationStatus(value) {
|
|
27698
|
+
return value === "passed" || value === "failed" || value === "invalid" || value === "error";
|
|
27699
|
+
}
|
|
27700
|
+
function isBoundedNonemptyString(value, maxLength) {
|
|
27701
|
+
return typeof value === "string" && value.length > 0 && value.length <= maxLength;
|
|
27702
|
+
}
|
|
27703
|
+
function isNonnegativeFiniteNumber(value) {
|
|
27704
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
27705
|
+
}
|
|
27706
|
+
function isPositiveFiniteNumber(value) {
|
|
27707
|
+
return isNonnegativeFiniteNumber(value) && value > 0;
|
|
27708
|
+
}
|
|
27709
|
+
function isNonnegativeSafeInteger(value) {
|
|
27710
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
27711
|
+
}
|
|
27712
|
+
function isPositiveSafeInteger(value) {
|
|
27713
|
+
return isNonnegativeSafeInteger(value) && value > 0;
|
|
27714
|
+
}
|
|
27715
|
+
function isIsoTimestamp(value) {
|
|
27716
|
+
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
27717
|
+
}
|
|
27898
27718
|
function assertCaseTargetEvidence(target) {
|
|
27899
27719
|
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
27900
27720
|
throw new Error("Invalid run manifest: malformed target evidence");
|
|
@@ -27971,6 +27791,9 @@ function createRunManifest(options) {
|
|
|
27971
27791
|
resolvedConfig,
|
|
27972
27792
|
workloadIdentity,
|
|
27973
27793
|
executionProvenance,
|
|
27794
|
+
attemptEvidence,
|
|
27795
|
+
costProvenance,
|
|
27796
|
+
runId,
|
|
27974
27797
|
cases,
|
|
27975
27798
|
startTime,
|
|
27976
27799
|
endTime,
|
|
@@ -27978,13 +27801,12 @@ function createRunManifest(options) {
|
|
|
27978
27801
|
runReason,
|
|
27979
27802
|
redaction
|
|
27980
27803
|
} = options;
|
|
27981
|
-
const
|
|
27982
|
-
const metrics = calculateMetrics(cases, modelForCost);
|
|
27804
|
+
const metrics = calculateMetrics(cases, costProvenance);
|
|
27983
27805
|
const git = getGitInfo();
|
|
27984
27806
|
const environment = getEnvironmentInfo();
|
|
27985
27807
|
return {
|
|
27986
|
-
version: "1.
|
|
27987
|
-
run_id: nanoid(12),
|
|
27808
|
+
version: "1.4",
|
|
27809
|
+
run_id: runId ?? nanoid(12),
|
|
27988
27810
|
project,
|
|
27989
27811
|
start_time: startTime.toISOString(),
|
|
27990
27812
|
end_time: endTime.toISOString(),
|
|
@@ -27993,6 +27815,7 @@ function createRunManifest(options) {
|
|
|
27993
27815
|
resolved_config: resolvedConfig,
|
|
27994
27816
|
workload_identity: workloadIdentity,
|
|
27995
27817
|
execution_provenance: executionProvenance,
|
|
27818
|
+
attempt_evidence: attemptEvidence,
|
|
27996
27819
|
metrics,
|
|
27997
27820
|
git,
|
|
27998
27821
|
provenance: {
|
|
@@ -28005,7 +27828,7 @@ function createRunManifest(options) {
|
|
|
28005
27828
|
redaction
|
|
28006
27829
|
};
|
|
28007
27830
|
}
|
|
28008
|
-
function calculateMetrics(cases,
|
|
27831
|
+
function calculateMetrics(cases, costProvenance) {
|
|
28009
27832
|
const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
|
|
28010
27833
|
const validCases = cases.filter((c) => {
|
|
28011
27834
|
const status = getCaseEvaluationStatus(c);
|
|
@@ -28017,21 +27840,11 @@ function calculateMetrics(cases, model) {
|
|
|
28017
27840
|
const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
|
|
28018
27841
|
const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
|
|
28019
27842
|
const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
|
|
28020
|
-
|
|
28021
|
-
|
|
28022
|
-
|
|
28023
|
-
|
|
28024
|
-
|
|
28025
|
-
total_usd: costEstimate.totalUsd,
|
|
28026
|
-
prompt_cost_usd: costEstimate.promptCostUsd,
|
|
28027
|
-
completion_cost_usd: costEstimate.completionCostUsd,
|
|
28028
|
-
model: costEstimate.model,
|
|
28029
|
-
pricing: {
|
|
28030
|
-
prompt_per_1k: pricing.promptPer1K,
|
|
28031
|
-
completion_per_1k: pricing.completionPer1K
|
|
28032
|
-
}
|
|
28033
|
-
};
|
|
28034
|
-
}
|
|
27843
|
+
const cost_provenance = costProvenance ?? {
|
|
27844
|
+
schema_version: "1",
|
|
27845
|
+
status: "unavailable",
|
|
27846
|
+
unavailable_reason: "provider_billing_not_recorded"
|
|
27847
|
+
};
|
|
28035
27848
|
return {
|
|
28036
27849
|
success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
|
|
28037
27850
|
total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
|
|
@@ -28046,7 +27859,7 @@ function calculateMetrics(cases, model) {
|
|
|
28046
27859
|
total_tokens: totalPromptTokens + totalCompletionTokens,
|
|
28047
27860
|
total_prompt_tokens: totalPromptTokens,
|
|
28048
27861
|
total_completion_tokens: totalCompletionTokens,
|
|
28049
|
-
|
|
27862
|
+
cost_provenance
|
|
28050
27863
|
};
|
|
28051
27864
|
}
|
|
28052
27865
|
function detectCIEnvironment() {
|
|
@@ -28185,6 +27998,8 @@ async function runScenario(options) {
|
|
|
28185
27998
|
concurrency = 1,
|
|
28186
27999
|
timeout,
|
|
28187
28000
|
retries,
|
|
28001
|
+
repetition = { index: 1, total: 1 },
|
|
28002
|
+
costProvenance,
|
|
28188
28003
|
redaction,
|
|
28189
28004
|
toolExecutor,
|
|
28190
28005
|
onCaseComplete,
|
|
@@ -28200,6 +28015,7 @@ async function runScenario(options) {
|
|
|
28200
28015
|
}
|
|
28201
28016
|
onProgress?.(`Running ${cases.length} test cases...`);
|
|
28202
28017
|
const startTime = new Date;
|
|
28018
|
+
const runId = nanoid(12);
|
|
28203
28019
|
const results = [];
|
|
28204
28020
|
if (concurrency === 1) {
|
|
28205
28021
|
for (let i = 0;i < cases.length; i++) {
|
|
@@ -28210,6 +28026,8 @@ async function runScenario(options) {
|
|
|
28210
28026
|
requestedModel: resolvedConfig?.model,
|
|
28211
28027
|
timeout: testCase.timeout || timeout,
|
|
28212
28028
|
retries: testCase.retries ?? retries,
|
|
28029
|
+
runId,
|
|
28030
|
+
repetition,
|
|
28213
28031
|
redaction,
|
|
28214
28032
|
toolExecutor
|
|
28215
28033
|
});
|
|
@@ -28227,6 +28045,8 @@ async function runScenario(options) {
|
|
|
28227
28045
|
requestedModel: resolvedConfig?.model,
|
|
28228
28046
|
timeout: testCase.timeout || timeout,
|
|
28229
28047
|
retries: testCase.retries ?? retries,
|
|
28048
|
+
runId,
|
|
28049
|
+
repetition,
|
|
28230
28050
|
redaction,
|
|
28231
28051
|
toolExecutor
|
|
28232
28052
|
});
|
|
@@ -28277,9 +28097,21 @@ async function runScenario(options) {
|
|
|
28277
28097
|
seed: scenario.seed,
|
|
28278
28098
|
cases: results
|
|
28279
28099
|
}),
|
|
28100
|
+
attemptEvidence: {
|
|
28101
|
+
schema_version: "1",
|
|
28102
|
+
repetition,
|
|
28103
|
+
retry_policy: {
|
|
28104
|
+
default_max_retries: retries ?? 0,
|
|
28105
|
+
backoff: "exponential",
|
|
28106
|
+
initial_delay_ms: 1000
|
|
28107
|
+
},
|
|
28108
|
+
...timeout ? { timeout: { default_ms: timeout } } : {}
|
|
28109
|
+
},
|
|
28110
|
+
costProvenance,
|
|
28280
28111
|
cases: results,
|
|
28281
28112
|
startTime,
|
|
28282
28113
|
endTime,
|
|
28114
|
+
runId,
|
|
28283
28115
|
redaction: redactionInfo
|
|
28284
28116
|
});
|
|
28285
28117
|
const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.5",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.5.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.2",
|
|
75
|
+
"@artemiskit/redteam": "0.3.6"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|