@artemiskit/sdk 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.js +246 -323
- package/dist/types.d.ts +8 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/artemiskit.ts +32 -0
- package/src/types.ts +8 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.4.6
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies
|
|
8
|
+
- @artemiskit/core@0.5.3
|
|
9
|
+
- @artemiskit/redteam@0.3.7
|
|
10
|
+
|
|
11
|
+
## 0.4.5
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Updated dependencies
|
|
16
|
+
- @artemiskit/core@0.5.2
|
|
17
|
+
- @artemiskit/redteam@0.3.6
|
|
18
|
+
|
|
3
19
|
## 0.4.4
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAuBzC;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.js
CHANGED
|
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
|
|
|
26484
26484
|
tags: arrayType(stringType()).optional().default([]),
|
|
26485
26485
|
metadata: recordType(unknownType()).optional().default({}),
|
|
26486
26486
|
timeout: numberType().optional(),
|
|
26487
|
-
retries: numberType().optional().default(0),
|
|
26487
|
+
retries: numberType().int().min(0).max(99).optional().default(0),
|
|
26488
26488
|
provider: ProviderSchema.optional(),
|
|
26489
26489
|
model: stringType().optional(),
|
|
26490
26490
|
variables: VariablesSchema.optional(),
|
|
@@ -27129,17 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
27129
27129
|
}
|
|
27130
27130
|
async function executeCase(testCase, context) {
|
|
27131
27131
|
const { timeout, retries = 0 } = context;
|
|
27132
|
+
if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
|
|
27133
|
+
throw new RangeError("retries must be a whole number between 0 and 99");
|
|
27134
|
+
}
|
|
27132
27135
|
const caseStartTime = Date.now();
|
|
27133
27136
|
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
27137
|
+
const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
|
|
27138
|
+
const repetitionIndex = context.repetition?.index ?? 1;
|
|
27139
|
+
const attemptEvidence = [];
|
|
27134
27140
|
let lastError = null;
|
|
27135
27141
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
27142
|
+
const attemptStartTime = Date.now();
|
|
27136
27143
|
try {
|
|
27137
27144
|
const result = await executeCaseAttempt(testCase, context, timeout);
|
|
27138
|
-
return
|
|
27145
|
+
return withAttemptEvidence(result, attemptEvidence, {
|
|
27146
|
+
retryChainId,
|
|
27147
|
+
repetitionIndex,
|
|
27148
|
+
attemptNumber: attempt + 1,
|
|
27149
|
+
includedInOutcome: true,
|
|
27150
|
+
latencyMs: result.latencyMs
|
|
27151
|
+
});
|
|
27139
27152
|
} catch (error) {
|
|
27140
27153
|
lastError = error;
|
|
27141
|
-
if (error instanceof ToolLoopError)
|
|
27142
|
-
return
|
|
27154
|
+
if (error instanceof ToolLoopError) {
|
|
27155
|
+
return withAttemptEvidence(error.caseResult, attemptEvidence, {
|
|
27156
|
+
retryChainId,
|
|
27157
|
+
repetitionIndex,
|
|
27158
|
+
attemptNumber: attempt + 1,
|
|
27159
|
+
includedInOutcome: true,
|
|
27160
|
+
latencyMs: error.caseResult.latencyMs,
|
|
27161
|
+
errorCode: "tool_error"
|
|
27162
|
+
});
|
|
27163
|
+
}
|
|
27164
|
+
attemptEvidence.push({
|
|
27165
|
+
attempt_id: `${retryChainId}:${attempt + 1}`,
|
|
27166
|
+
retry_chain_id: retryChainId,
|
|
27167
|
+
repetition_index: repetitionIndex,
|
|
27168
|
+
attempt_number: attempt + 1,
|
|
27169
|
+
status: "error",
|
|
27170
|
+
included_in_outcome: false,
|
|
27171
|
+
latency_ms: Date.now() - attemptStartTime,
|
|
27172
|
+
error_code: error instanceof TimeoutError ? "timeout" : "target_error"
|
|
27173
|
+
});
|
|
27143
27174
|
if (attempt < retries) {
|
|
27144
27175
|
await sleep(2 ** attempt * 1000);
|
|
27145
27176
|
}
|
|
@@ -27152,6 +27183,10 @@ async function executeCase(testCase, context) {
|
|
|
27152
27183
|
ok: false,
|
|
27153
27184
|
status: "error",
|
|
27154
27185
|
attempts: retries + 1,
|
|
27186
|
+
attempt_evidence: attemptEvidence.map((entry, index) => ({
|
|
27187
|
+
...entry,
|
|
27188
|
+
included_in_outcome: index === attemptEvidence.length - 1
|
|
27189
|
+
})),
|
|
27155
27190
|
score: 0,
|
|
27156
27191
|
matcherType: testCase.expected.type,
|
|
27157
27192
|
reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
|
|
@@ -27165,6 +27200,28 @@ async function executeCase(testCase, context) {
|
|
|
27165
27200
|
target: targetEvidence(context.client.provider, requestedModel)
|
|
27166
27201
|
};
|
|
27167
27202
|
}
|
|
27203
|
+
function withAttemptEvidence(result, priorAttempts, input) {
|
|
27204
|
+
return {
|
|
27205
|
+
...result,
|
|
27206
|
+
attempts: input.attemptNumber,
|
|
27207
|
+
attempt_evidence: [
|
|
27208
|
+
...priorAttempts,
|
|
27209
|
+
{
|
|
27210
|
+
attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
|
|
27211
|
+
retry_chain_id: input.retryChainId,
|
|
27212
|
+
repetition_index: input.repetitionIndex,
|
|
27213
|
+
attempt_number: input.attemptNumber,
|
|
27214
|
+
status: getTerminalStatus(result),
|
|
27215
|
+
included_in_outcome: input.includedInOutcome,
|
|
27216
|
+
latency_ms: input.latencyMs,
|
|
27217
|
+
...input.errorCode ? { error_code: input.errorCode } : {}
|
|
27218
|
+
}
|
|
27219
|
+
]
|
|
27220
|
+
};
|
|
27221
|
+
}
|
|
27222
|
+
function getTerminalStatus(result) {
|
|
27223
|
+
return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
|
|
27224
|
+
}
|
|
27168
27225
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
27169
27226
|
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
27170
27227
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
@@ -27526,302 +27583,6 @@ function nanoid(size = 21) {
|
|
|
27526
27583
|
}
|
|
27527
27584
|
return id;
|
|
27528
27585
|
}
|
|
27529
|
-
var MODEL_PRICING = {
|
|
27530
|
-
"gpt-5": {
|
|
27531
|
-
promptPer1K: 0.00125,
|
|
27532
|
-
completionPer1K: 0.01,
|
|
27533
|
-
lastUpdated: "2026-01",
|
|
27534
|
-
notes: "400K context window"
|
|
27535
|
-
},
|
|
27536
|
-
"gpt-5.1": {
|
|
27537
|
-
promptPer1K: 0.00125,
|
|
27538
|
-
completionPer1K: 0.01,
|
|
27539
|
-
lastUpdated: "2026-01"
|
|
27540
|
-
},
|
|
27541
|
-
"gpt-5.2": {
|
|
27542
|
-
promptPer1K: 0.00175,
|
|
27543
|
-
completionPer1K: 0.014,
|
|
27544
|
-
lastUpdated: "2026-01"
|
|
27545
|
-
},
|
|
27546
|
-
"gpt-5-mini": {
|
|
27547
|
-
promptPer1K: 0.00025,
|
|
27548
|
-
completionPer1K: 0.002,
|
|
27549
|
-
lastUpdated: "2026-01"
|
|
27550
|
-
},
|
|
27551
|
-
"gpt-5-nano": {
|
|
27552
|
-
promptPer1K: 0.00005,
|
|
27553
|
-
completionPer1K: 0.0004,
|
|
27554
|
-
lastUpdated: "2026-01"
|
|
27555
|
-
},
|
|
27556
|
-
"gpt-4.1": {
|
|
27557
|
-
promptPer1K: 0.002,
|
|
27558
|
-
completionPer1K: 0.008,
|
|
27559
|
-
lastUpdated: "2026-01",
|
|
27560
|
-
notes: "1M context window"
|
|
27561
|
-
},
|
|
27562
|
-
"gpt-4.1-mini": {
|
|
27563
|
-
promptPer1K: 0.0004,
|
|
27564
|
-
completionPer1K: 0.0016,
|
|
27565
|
-
lastUpdated: "2026-01"
|
|
27566
|
-
},
|
|
27567
|
-
"gpt-4.1-nano": {
|
|
27568
|
-
promptPer1K: 0.0001,
|
|
27569
|
-
completionPer1K: 0.0004,
|
|
27570
|
-
lastUpdated: "2026-01"
|
|
27571
|
-
},
|
|
27572
|
-
"gpt-4o": {
|
|
27573
|
-
promptPer1K: 0.0025,
|
|
27574
|
-
completionPer1K: 0.01,
|
|
27575
|
-
lastUpdated: "2026-01",
|
|
27576
|
-
notes: "128K context window"
|
|
27577
|
-
},
|
|
27578
|
-
"gpt-4o-mini": {
|
|
27579
|
-
promptPer1K: 0.00015,
|
|
27580
|
-
completionPer1K: 0.0006,
|
|
27581
|
-
lastUpdated: "2026-01",
|
|
27582
|
-
notes: "128K context window"
|
|
27583
|
-
},
|
|
27584
|
-
o1: {
|
|
27585
|
-
promptPer1K: 0.015,
|
|
27586
|
-
completionPer1K: 0.06,
|
|
27587
|
-
lastUpdated: "2026-01",
|
|
27588
|
-
notes: "Reasoning model - internal thinking tokens billed as output"
|
|
27589
|
-
},
|
|
27590
|
-
o3: {
|
|
27591
|
-
promptPer1K: 0.002,
|
|
27592
|
-
completionPer1K: 0.008,
|
|
27593
|
-
lastUpdated: "2026-01"
|
|
27594
|
-
},
|
|
27595
|
-
"o3-mini": {
|
|
27596
|
-
promptPer1K: 0.0011,
|
|
27597
|
-
completionPer1K: 0.0044,
|
|
27598
|
-
lastUpdated: "2026-01"
|
|
27599
|
-
},
|
|
27600
|
-
"o4-mini": {
|
|
27601
|
-
promptPer1K: 0.0011,
|
|
27602
|
-
completionPer1K: 0.0044,
|
|
27603
|
-
lastUpdated: "2026-01"
|
|
27604
|
-
},
|
|
27605
|
-
"gpt-4-turbo": {
|
|
27606
|
-
promptPer1K: 0.01,
|
|
27607
|
-
completionPer1K: 0.03,
|
|
27608
|
-
lastUpdated: "2026-01"
|
|
27609
|
-
},
|
|
27610
|
-
"gpt-4": {
|
|
27611
|
-
promptPer1K: 0.03,
|
|
27612
|
-
completionPer1K: 0.06,
|
|
27613
|
-
lastUpdated: "2026-01"
|
|
27614
|
-
},
|
|
27615
|
-
"gpt-3.5-turbo": {
|
|
27616
|
-
promptPer1K: 0.0005,
|
|
27617
|
-
completionPer1K: 0.0015,
|
|
27618
|
-
lastUpdated: "2026-01"
|
|
27619
|
-
},
|
|
27620
|
-
"claude-opus-4.5": {
|
|
27621
|
-
promptPer1K: 0.005,
|
|
27622
|
-
completionPer1K: 0.025,
|
|
27623
|
-
lastUpdated: "2026-01",
|
|
27624
|
-
notes: "Most capable Claude model"
|
|
27625
|
-
},
|
|
27626
|
-
"claude-sonnet-4.5": {
|
|
27627
|
-
promptPer1K: 0.003,
|
|
27628
|
-
completionPer1K: 0.015,
|
|
27629
|
-
lastUpdated: "2026-01",
|
|
27630
|
-
notes: "Balanced performance and cost"
|
|
27631
|
-
},
|
|
27632
|
-
"claude-haiku-4.5": {
|
|
27633
|
-
promptPer1K: 0.001,
|
|
27634
|
-
completionPer1K: 0.005,
|
|
27635
|
-
lastUpdated: "2026-01",
|
|
27636
|
-
notes: "Fastest Claude model"
|
|
27637
|
-
},
|
|
27638
|
-
"claude-opus-4": {
|
|
27639
|
-
promptPer1K: 0.015,
|
|
27640
|
-
completionPer1K: 0.075,
|
|
27641
|
-
lastUpdated: "2026-01"
|
|
27642
|
-
},
|
|
27643
|
-
"claude-opus-4.1": {
|
|
27644
|
-
promptPer1K: 0.015,
|
|
27645
|
-
completionPer1K: 0.075,
|
|
27646
|
-
lastUpdated: "2026-01"
|
|
27647
|
-
},
|
|
27648
|
-
"claude-sonnet-4": {
|
|
27649
|
-
promptPer1K: 0.003,
|
|
27650
|
-
completionPer1K: 0.015,
|
|
27651
|
-
lastUpdated: "2026-01"
|
|
27652
|
-
},
|
|
27653
|
-
"claude-sonnet-3.7": {
|
|
27654
|
-
promptPer1K: 0.003,
|
|
27655
|
-
completionPer1K: 0.015,
|
|
27656
|
-
lastUpdated: "2026-01"
|
|
27657
|
-
},
|
|
27658
|
-
"claude-3-7-sonnet": {
|
|
27659
|
-
promptPer1K: 0.003,
|
|
27660
|
-
completionPer1K: 0.015,
|
|
27661
|
-
lastUpdated: "2026-01"
|
|
27662
|
-
},
|
|
27663
|
-
"claude-3-5-sonnet-20241022": {
|
|
27664
|
-
promptPer1K: 0.003,
|
|
27665
|
-
completionPer1K: 0.015,
|
|
27666
|
-
lastUpdated: "2026-01"
|
|
27667
|
-
},
|
|
27668
|
-
"claude-3-5-haiku-20241022": {
|
|
27669
|
-
promptPer1K: 0.0008,
|
|
27670
|
-
completionPer1K: 0.004,
|
|
27671
|
-
lastUpdated: "2026-01"
|
|
27672
|
-
},
|
|
27673
|
-
"claude-haiku-3.5": {
|
|
27674
|
-
promptPer1K: 0.0008,
|
|
27675
|
-
completionPer1K: 0.004,
|
|
27676
|
-
lastUpdated: "2026-01"
|
|
27677
|
-
},
|
|
27678
|
-
"claude-3-opus": {
|
|
27679
|
-
promptPer1K: 0.015,
|
|
27680
|
-
completionPer1K: 0.075,
|
|
27681
|
-
lastUpdated: "2026-01"
|
|
27682
|
-
},
|
|
27683
|
-
"claude-3-sonnet": {
|
|
27684
|
-
promptPer1K: 0.003,
|
|
27685
|
-
completionPer1K: 0.015,
|
|
27686
|
-
lastUpdated: "2026-01"
|
|
27687
|
-
},
|
|
27688
|
-
"claude-3-haiku": {
|
|
27689
|
-
promptPer1K: 0.00025,
|
|
27690
|
-
completionPer1K: 0.00125,
|
|
27691
|
-
lastUpdated: "2026-01"
|
|
27692
|
-
},
|
|
27693
|
-
"claude-3.5-sonnet": {
|
|
27694
|
-
promptPer1K: 0.003,
|
|
27695
|
-
completionPer1K: 0.015,
|
|
27696
|
-
lastUpdated: "2026-01"
|
|
27697
|
-
},
|
|
27698
|
-
"claude-3.5-haiku": {
|
|
27699
|
-
promptPer1K: 0.0008,
|
|
27700
|
-
completionPer1K: 0.004,
|
|
27701
|
-
lastUpdated: "2026-01"
|
|
27702
|
-
}
|
|
27703
|
-
};
|
|
27704
|
-
var DEFAULT_PRICING = {
|
|
27705
|
-
promptPer1K: 0.003,
|
|
27706
|
-
completionPer1K: 0.015,
|
|
27707
|
-
lastUpdated: "2026-01",
|
|
27708
|
-
notes: "Default pricing - verify with provider"
|
|
27709
|
-
};
|
|
27710
|
-
function getModelPricing(model) {
|
|
27711
|
-
if (MODEL_PRICING[model]) {
|
|
27712
|
-
return MODEL_PRICING[model];
|
|
27713
|
-
}
|
|
27714
|
-
const lowerModel = model.toLowerCase();
|
|
27715
|
-
for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
|
|
27716
|
-
if (key.toLowerCase() === lowerModel) {
|
|
27717
|
-
return pricing;
|
|
27718
|
-
}
|
|
27719
|
-
}
|
|
27720
|
-
if (lowerModel.includes("gpt-5.2")) {
|
|
27721
|
-
return MODEL_PRICING["gpt-5.2"];
|
|
27722
|
-
}
|
|
27723
|
-
if (lowerModel.includes("gpt-5.1")) {
|
|
27724
|
-
return MODEL_PRICING["gpt-5.1"];
|
|
27725
|
-
}
|
|
27726
|
-
if (lowerModel.includes("gpt-5-mini")) {
|
|
27727
|
-
return MODEL_PRICING["gpt-5-mini"];
|
|
27728
|
-
}
|
|
27729
|
-
if (lowerModel.includes("gpt-5-nano")) {
|
|
27730
|
-
return MODEL_PRICING["gpt-5-nano"];
|
|
27731
|
-
}
|
|
27732
|
-
if (lowerModel.includes("gpt-5")) {
|
|
27733
|
-
return MODEL_PRICING["gpt-5"];
|
|
27734
|
-
}
|
|
27735
|
-
if (lowerModel.includes("gpt-4.1-mini")) {
|
|
27736
|
-
return MODEL_PRICING["gpt-4.1-mini"];
|
|
27737
|
-
}
|
|
27738
|
-
if (lowerModel.includes("gpt-4.1-nano")) {
|
|
27739
|
-
return MODEL_PRICING["gpt-4.1-nano"];
|
|
27740
|
-
}
|
|
27741
|
-
if (lowerModel.includes("gpt-4.1")) {
|
|
27742
|
-
return MODEL_PRICING["gpt-4.1"];
|
|
27743
|
-
}
|
|
27744
|
-
if (lowerModel.includes("gpt-4o-mini")) {
|
|
27745
|
-
return MODEL_PRICING["gpt-4o-mini"];
|
|
27746
|
-
}
|
|
27747
|
-
if (lowerModel.includes("gpt-4o")) {
|
|
27748
|
-
return MODEL_PRICING["gpt-4o"];
|
|
27749
|
-
}
|
|
27750
|
-
if (lowerModel.includes("o4-mini")) {
|
|
27751
|
-
return MODEL_PRICING["o4-mini"];
|
|
27752
|
-
}
|
|
27753
|
-
if (lowerModel.includes("o3-mini")) {
|
|
27754
|
-
return MODEL_PRICING["o3-mini"];
|
|
27755
|
-
}
|
|
27756
|
-
if (lowerModel.includes("o3")) {
|
|
27757
|
-
return MODEL_PRICING.o3;
|
|
27758
|
-
}
|
|
27759
|
-
if (lowerModel.includes("o1")) {
|
|
27760
|
-
return MODEL_PRICING.o1;
|
|
27761
|
-
}
|
|
27762
|
-
if (lowerModel.includes("gpt-4-turbo")) {
|
|
27763
|
-
return MODEL_PRICING["gpt-4-turbo"];
|
|
27764
|
-
}
|
|
27765
|
-
if (lowerModel.includes("gpt-4")) {
|
|
27766
|
-
return MODEL_PRICING["gpt-4"];
|
|
27767
|
-
}
|
|
27768
|
-
if (lowerModel.includes("gpt-3.5")) {
|
|
27769
|
-
return MODEL_PRICING["gpt-3.5-turbo"];
|
|
27770
|
-
}
|
|
27771
|
-
if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
|
|
27772
|
-
return MODEL_PRICING["claude-opus-4.5"];
|
|
27773
|
-
}
|
|
27774
|
-
if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
|
|
27775
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27776
|
-
}
|
|
27777
|
-
if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
|
|
27778
|
-
return MODEL_PRICING["claude-haiku-4.5"];
|
|
27779
|
-
}
|
|
27780
|
-
if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
|
|
27781
|
-
return MODEL_PRICING["claude-opus-4.1"];
|
|
27782
|
-
}
|
|
27783
|
-
if (lowerModel.includes("opus-4")) {
|
|
27784
|
-
return MODEL_PRICING["claude-opus-4"];
|
|
27785
|
-
}
|
|
27786
|
-
if (lowerModel.includes("sonnet-4")) {
|
|
27787
|
-
return MODEL_PRICING["claude-sonnet-4"];
|
|
27788
|
-
}
|
|
27789
|
-
if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
|
|
27790
|
-
return MODEL_PRICING["claude-sonnet-3.7"];
|
|
27791
|
-
}
|
|
27792
|
-
if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
|
|
27793
|
-
return MODEL_PRICING["claude-3.5-sonnet"];
|
|
27794
|
-
}
|
|
27795
|
-
if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
|
|
27796
|
-
return MODEL_PRICING["claude-3.5-haiku"];
|
|
27797
|
-
}
|
|
27798
|
-
if (lowerModel.includes("claude-3-opus")) {
|
|
27799
|
-
return MODEL_PRICING["claude-3-opus"];
|
|
27800
|
-
}
|
|
27801
|
-
if (lowerModel.includes("claude-3-sonnet")) {
|
|
27802
|
-
return MODEL_PRICING["claude-3-sonnet"];
|
|
27803
|
-
}
|
|
27804
|
-
if (lowerModel.includes("claude-3-haiku")) {
|
|
27805
|
-
return MODEL_PRICING["claude-3-haiku"];
|
|
27806
|
-
}
|
|
27807
|
-
if (lowerModel.includes("claude")) {
|
|
27808
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27809
|
-
}
|
|
27810
|
-
return DEFAULT_PRICING;
|
|
27811
|
-
}
|
|
27812
|
-
function estimateCost(promptTokens, completionTokens, model) {
|
|
27813
|
-
const pricing = getModelPricing(model);
|
|
27814
|
-
const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
|
|
27815
|
-
const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
|
|
27816
|
-
const totalUsd = promptCostUsd + completionCostUsd;
|
|
27817
|
-
return {
|
|
27818
|
-
totalUsd,
|
|
27819
|
-
promptCostUsd,
|
|
27820
|
-
completionCostUsd,
|
|
27821
|
-
model,
|
|
27822
|
-
pricing
|
|
27823
|
-
};
|
|
27824
|
-
}
|
|
27825
27586
|
function getEnvironmentInfo() {
|
|
27826
27587
|
return {
|
|
27827
27588
|
node_version: process.version,
|
|
@@ -27880,6 +27641,12 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27880
27641
|
if (manifest.execution_provenance !== undefined) {
|
|
27881
27642
|
assertExecutionProvenance(manifest.execution_provenance);
|
|
27882
27643
|
}
|
|
27644
|
+
if (manifest.attempt_evidence !== undefined) {
|
|
27645
|
+
assertRunAttemptEvidence(manifest.attempt_evidence);
|
|
27646
|
+
}
|
|
27647
|
+
if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
|
|
27648
|
+
assertCostProvenance(manifest.metrics.cost_provenance);
|
|
27649
|
+
}
|
|
27883
27650
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27884
27651
|
if (!isRecord2(caseResult)) {
|
|
27885
27652
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27893,7 +27660,60 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27893
27660
|
if (caseResult.target !== undefined) {
|
|
27894
27661
|
assertCaseTargetEvidence(caseResult.target);
|
|
27895
27662
|
}
|
|
27663
|
+
if (caseResult.attempt_evidence !== undefined) {
|
|
27664
|
+
assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
|
|
27665
|
+
}
|
|
27666
|
+
}
|
|
27667
|
+
}
|
|
27668
|
+
function assertRunAttemptEvidence(evidence) {
|
|
27669
|
+
if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
|
|
27670
|
+
throw new Error("Invalid run manifest: malformed attempt evidence");
|
|
27671
|
+
}
|
|
27672
|
+
}
|
|
27673
|
+
function assertCaseAttemptEvidence(evidence, index) {
|
|
27674
|
+
if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
|
|
27675
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27676
|
+
}
|
|
27677
|
+
for (const attempt of evidence) {
|
|
27678
|
+
if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
|
|
27679
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27680
|
+
}
|
|
27681
|
+
}
|
|
27682
|
+
}
|
|
27683
|
+
function assertCostProvenance(cost) {
|
|
27684
|
+
if (!isRecord2(cost) || cost.schema_version !== "1") {
|
|
27685
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27686
|
+
}
|
|
27687
|
+
if (cost.status === "unavailable") {
|
|
27688
|
+
if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
|
|
27689
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27690
|
+
}
|
|
27691
|
+
return;
|
|
27896
27692
|
}
|
|
27693
|
+
if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
|
|
27694
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27695
|
+
}
|
|
27696
|
+
}
|
|
27697
|
+
function isCaseEvaluationStatus(value) {
|
|
27698
|
+
return value === "passed" || value === "failed" || value === "invalid" || value === "error";
|
|
27699
|
+
}
|
|
27700
|
+
function isBoundedNonemptyString(value, maxLength) {
|
|
27701
|
+
return typeof value === "string" && value.length > 0 && value.length <= maxLength;
|
|
27702
|
+
}
|
|
27703
|
+
function isNonnegativeFiniteNumber(value) {
|
|
27704
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
27705
|
+
}
|
|
27706
|
+
function isPositiveFiniteNumber(value) {
|
|
27707
|
+
return isNonnegativeFiniteNumber(value) && value > 0;
|
|
27708
|
+
}
|
|
27709
|
+
function isNonnegativeSafeInteger(value) {
|
|
27710
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
27711
|
+
}
|
|
27712
|
+
function isPositiveSafeInteger(value) {
|
|
27713
|
+
return isNonnegativeSafeInteger(value) && value > 0;
|
|
27714
|
+
}
|
|
27715
|
+
function isIsoTimestamp(value) {
|
|
27716
|
+
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
27897
27717
|
}
|
|
27898
27718
|
function assertCaseTargetEvidence(target) {
|
|
27899
27719
|
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
@@ -27971,6 +27791,9 @@ function createRunManifest(options) {
|
|
|
27971
27791
|
resolvedConfig,
|
|
27972
27792
|
workloadIdentity,
|
|
27973
27793
|
executionProvenance,
|
|
27794
|
+
attemptEvidence,
|
|
27795
|
+
costProvenance,
|
|
27796
|
+
runId,
|
|
27974
27797
|
cases,
|
|
27975
27798
|
startTime,
|
|
27976
27799
|
endTime,
|
|
@@ -27978,13 +27801,12 @@ function createRunManifest(options) {
|
|
|
27978
27801
|
runReason,
|
|
27979
27802
|
redaction
|
|
27980
27803
|
} = options;
|
|
27981
|
-
const
|
|
27982
|
-
const metrics = calculateMetrics(cases, modelForCost);
|
|
27804
|
+
const metrics = calculateMetrics(cases, costProvenance);
|
|
27983
27805
|
const git = getGitInfo();
|
|
27984
27806
|
const environment = getEnvironmentInfo();
|
|
27985
27807
|
return {
|
|
27986
|
-
version: "1.
|
|
27987
|
-
run_id: nanoid(12),
|
|
27808
|
+
version: "1.4",
|
|
27809
|
+
run_id: runId ?? nanoid(12),
|
|
27988
27810
|
project,
|
|
27989
27811
|
start_time: startTime.toISOString(),
|
|
27990
27812
|
end_time: endTime.toISOString(),
|
|
@@ -27993,6 +27815,7 @@ function createRunManifest(options) {
|
|
|
27993
27815
|
resolved_config: resolvedConfig,
|
|
27994
27816
|
workload_identity: workloadIdentity,
|
|
27995
27817
|
execution_provenance: executionProvenance,
|
|
27818
|
+
attempt_evidence: attemptEvidence,
|
|
27996
27819
|
metrics,
|
|
27997
27820
|
git,
|
|
27998
27821
|
provenance: {
|
|
@@ -28005,7 +27828,7 @@ function createRunManifest(options) {
|
|
|
28005
27828
|
redaction
|
|
28006
27829
|
};
|
|
28007
27830
|
}
|
|
28008
|
-
function calculateMetrics(cases,
|
|
27831
|
+
function calculateMetrics(cases, costProvenance) {
|
|
28009
27832
|
const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
|
|
28010
27833
|
const validCases = cases.filter((c) => {
|
|
28011
27834
|
const status = getCaseEvaluationStatus(c);
|
|
@@ -28017,21 +27840,11 @@ function calculateMetrics(cases, model) {
|
|
|
28017
27840
|
const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
|
|
28018
27841
|
const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
|
|
28019
27842
|
const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
|
|
28020
|
-
|
|
28021
|
-
|
|
28022
|
-
|
|
28023
|
-
|
|
28024
|
-
|
|
28025
|
-
total_usd: costEstimate.totalUsd,
|
|
28026
|
-
prompt_cost_usd: costEstimate.promptCostUsd,
|
|
28027
|
-
completion_cost_usd: costEstimate.completionCostUsd,
|
|
28028
|
-
model: costEstimate.model,
|
|
28029
|
-
pricing: {
|
|
28030
|
-
prompt_per_1k: pricing.promptPer1K,
|
|
28031
|
-
completion_per_1k: pricing.completionPer1K
|
|
28032
|
-
}
|
|
28033
|
-
};
|
|
28034
|
-
}
|
|
27843
|
+
const cost_provenance = costProvenance ?? {
|
|
27844
|
+
schema_version: "1",
|
|
27845
|
+
status: "unavailable",
|
|
27846
|
+
unavailable_reason: "provider_billing_not_recorded"
|
|
27847
|
+
};
|
|
28035
27848
|
return {
|
|
28036
27849
|
success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
|
|
28037
27850
|
total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
|
|
@@ -28046,7 +27859,7 @@ function calculateMetrics(cases, model) {
|
|
|
28046
27859
|
total_tokens: totalPromptTokens + totalCompletionTokens,
|
|
28047
27860
|
total_prompt_tokens: totalPromptTokens,
|
|
28048
27861
|
total_completion_tokens: totalCompletionTokens,
|
|
28049
|
-
|
|
27862
|
+
cost_provenance
|
|
28050
27863
|
};
|
|
28051
27864
|
}
|
|
28052
27865
|
function detectCIEnvironment() {
|
|
@@ -28185,6 +27998,8 @@ async function runScenario(options) {
|
|
|
28185
27998
|
concurrency = 1,
|
|
28186
27999
|
timeout,
|
|
28187
28000
|
retries,
|
|
28001
|
+
repetition = { index: 1, total: 1 },
|
|
28002
|
+
costProvenance,
|
|
28188
28003
|
redaction,
|
|
28189
28004
|
toolExecutor,
|
|
28190
28005
|
onCaseComplete,
|
|
@@ -28200,6 +28015,7 @@ async function runScenario(options) {
|
|
|
28200
28015
|
}
|
|
28201
28016
|
onProgress?.(`Running ${cases.length} test cases...`);
|
|
28202
28017
|
const startTime = new Date;
|
|
28018
|
+
const runId = nanoid(12);
|
|
28203
28019
|
const results = [];
|
|
28204
28020
|
if (concurrency === 1) {
|
|
28205
28021
|
for (let i = 0;i < cases.length; i++) {
|
|
@@ -28210,6 +28026,8 @@ async function runScenario(options) {
|
|
|
28210
28026
|
requestedModel: resolvedConfig?.model,
|
|
28211
28027
|
timeout: testCase.timeout || timeout,
|
|
28212
28028
|
retries: testCase.retries ?? retries,
|
|
28029
|
+
runId,
|
|
28030
|
+
repetition,
|
|
28213
28031
|
redaction,
|
|
28214
28032
|
toolExecutor
|
|
28215
28033
|
});
|
|
@@ -28227,6 +28045,8 @@ async function runScenario(options) {
|
|
|
28227
28045
|
requestedModel: resolvedConfig?.model,
|
|
28228
28046
|
timeout: testCase.timeout || timeout,
|
|
28229
28047
|
retries: testCase.retries ?? retries,
|
|
28048
|
+
runId,
|
|
28049
|
+
repetition,
|
|
28230
28050
|
redaction,
|
|
28231
28051
|
toolExecutor
|
|
28232
28052
|
});
|
|
@@ -28277,9 +28097,21 @@ async function runScenario(options) {
|
|
|
28277
28097
|
seed: scenario.seed,
|
|
28278
28098
|
cases: results
|
|
28279
28099
|
}),
|
|
28100
|
+
attemptEvidence: {
|
|
28101
|
+
schema_version: "1",
|
|
28102
|
+
repetition,
|
|
28103
|
+
retry_policy: {
|
|
28104
|
+
default_max_retries: retries ?? 0,
|
|
28105
|
+
backoff: "exponential",
|
|
28106
|
+
initial_delay_ms: 1000
|
|
28107
|
+
},
|
|
28108
|
+
...timeout ? { timeout: { default_ms: timeout } } : {}
|
|
28109
|
+
},
|
|
28110
|
+
costProvenance,
|
|
28280
28111
|
cases: results,
|
|
28281
28112
|
startTime,
|
|
28282
28113
|
endTime,
|
|
28114
|
+
runId,
|
|
28283
28115
|
redaction: redactionInfo
|
|
28284
28116
|
});
|
|
28285
28117
|
const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
|
|
@@ -28296,6 +28128,62 @@ function chunkArray(array, size) {
|
|
|
28296
28128
|
}
|
|
28297
28129
|
return chunks;
|
|
28298
28130
|
}
|
|
28131
|
+
function assessComparisonEligibility(baseline, current) {
|
|
28132
|
+
const reasons = [];
|
|
28133
|
+
if (baseline.config.scenario !== current.config.scenario) {
|
|
28134
|
+
reasons.push({ code: "scenario_mismatch" });
|
|
28135
|
+
}
|
|
28136
|
+
const baselineIdentity = baseline.workload_identity;
|
|
28137
|
+
const currentIdentity = current.workload_identity;
|
|
28138
|
+
if (!baselineIdentity || !currentIdentity) {
|
|
28139
|
+
if (!baselineIdentity || !currentIdentity)
|
|
28140
|
+
reasons.push({ code: "workload_identity_missing" });
|
|
28141
|
+
if (!baselineIdentity || !currentIdentity)
|
|
28142
|
+
reasons.push({ code: "rubric_identity_missing" });
|
|
28143
|
+
} else {
|
|
28144
|
+
if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
|
|
28145
|
+
reasons.push({ code: "workload_mismatch" });
|
|
28146
|
+
}
|
|
28147
|
+
if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
|
|
28148
|
+
reasons.push({ code: "rubric_mismatch" });
|
|
28149
|
+
}
|
|
28150
|
+
}
|
|
28151
|
+
if (reasons.some((reason) => isIncomparableReason(reason.code))) {
|
|
28152
|
+
return { schema_version: "1", status: "incomparable", reasons };
|
|
28153
|
+
}
|
|
28154
|
+
const baselineExecution = baseline.execution_provenance;
|
|
28155
|
+
const currentExecution = current.execution_provenance;
|
|
28156
|
+
if (!baselineExecution || !currentExecution) {
|
|
28157
|
+
reasons.push({ code: "execution_provenance_missing" });
|
|
28158
|
+
} else {
|
|
28159
|
+
if (baselineExecution.target.provider !== currentExecution.target.provider) {
|
|
28160
|
+
reasons.push({ code: "target_provider_changed" });
|
|
28161
|
+
}
|
|
28162
|
+
if (!sameStringSet(baselineExecution.target.requested_models, currentExecution.target.requested_models)) {
|
|
28163
|
+
reasons.push({ code: "target_model_changed" });
|
|
28164
|
+
}
|
|
28165
|
+
if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
|
|
28166
|
+
reasons.push({ code: "generation_settings_changed" });
|
|
28167
|
+
}
|
|
28168
|
+
}
|
|
28169
|
+
return {
|
|
28170
|
+
schema_version: "1",
|
|
28171
|
+
status: reasons.length === 0 ? "compatible" : "qualified",
|
|
28172
|
+
reasons
|
|
28173
|
+
};
|
|
28174
|
+
}
|
|
28175
|
+
function isComparisonAvailable(eligibility) {
|
|
28176
|
+
return eligibility.status !== "incomparable";
|
|
28177
|
+
}
|
|
28178
|
+
function isIncomparableReason(code) {
|
|
28179
|
+
return code === "scenario_mismatch" || code === "workload_mismatch" || code === "rubric_mismatch";
|
|
28180
|
+
}
|
|
28181
|
+
function sameStringSet(left, right) {
|
|
28182
|
+
return JSON.stringify([...left ?? []].sort()) === JSON.stringify([...right ?? []].sort());
|
|
28183
|
+
}
|
|
28184
|
+
function sameGeneration(left, right) {
|
|
28185
|
+
return left?.temperature === right?.temperature && left?.max_tokens === right?.max_tokens && left?.seed === right?.seed;
|
|
28186
|
+
}
|
|
28299
28187
|
function getManifestType(manifest) {
|
|
28300
28188
|
if ("type" in manifest) {
|
|
28301
28189
|
if (manifest.type === "redteam")
|
|
@@ -28435,9 +28323,14 @@ class LocalStorageAdapter {
|
|
|
28435
28323
|
this.loadRun(baselineId),
|
|
28436
28324
|
this.loadRun(currentId)
|
|
28437
28325
|
]);
|
|
28326
|
+
const eligibility = assessComparisonEligibility(baseline, current);
|
|
28327
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
28328
|
+
return { baseline, current, eligibility };
|
|
28329
|
+
}
|
|
28438
28330
|
return {
|
|
28439
28331
|
baseline,
|
|
28440
28332
|
current,
|
|
28333
|
+
eligibility,
|
|
28441
28334
|
delta: {
|
|
28442
28335
|
successRate: current.metrics.success_rate - baseline.metrics.success_rate,
|
|
28443
28336
|
latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
|
|
@@ -28536,7 +28429,7 @@ class LocalStorageAdapter {
|
|
|
28536
28429
|
return null;
|
|
28537
28430
|
}
|
|
28538
28431
|
const comparison = await this.compare(baseline.runId, runId);
|
|
28539
|
-
const hasRegression = comparison.delta.successRate < -regressionThreshold;
|
|
28432
|
+
const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
|
|
28540
28433
|
return {
|
|
28541
28434
|
baseline,
|
|
28542
28435
|
comparison,
|
|
@@ -37127,9 +37020,14 @@ class SupabaseStorageAdapter {
|
|
|
37127
37020
|
}
|
|
37128
37021
|
async compare(baselineId, currentId) {
|
|
37129
37022
|
const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
|
|
37023
|
+
const eligibility = assessComparisonEligibility(baseline, current);
|
|
37024
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
37025
|
+
return { baseline, current, eligibility };
|
|
37026
|
+
}
|
|
37130
37027
|
return {
|
|
37131
37028
|
baseline,
|
|
37132
37029
|
current,
|
|
37030
|
+
eligibility,
|
|
37133
37031
|
delta: {
|
|
37134
37032
|
successRate: current.metrics.success_rate - baseline.metrics.success_rate,
|
|
37135
37033
|
latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
|
|
@@ -37260,7 +37158,7 @@ class SupabaseStorageAdapter {
|
|
|
37260
37158
|
return null;
|
|
37261
37159
|
}
|
|
37262
37160
|
const comparison = await this.compare(baseline.runId, runId);
|
|
37263
|
-
const hasRegression = comparison.delta.successRate < -regressionThreshold;
|
|
37161
|
+
const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
|
|
37264
37162
|
return {
|
|
37265
37163
|
baseline,
|
|
37266
37164
|
comparison,
|
|
@@ -51643,6 +51541,29 @@ class ArtemisKit {
|
|
|
51643
51541
|
});
|
|
51644
51542
|
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
51645
51543
|
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
51544
|
+
const eligibility = assessComparisonEligibility(baselineManifest, currentManifest);
|
|
51545
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
51546
|
+
this.emit("progress", {
|
|
51547
|
+
message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(", ")}`,
|
|
51548
|
+
phase: "teardown",
|
|
51549
|
+
progress: 100
|
|
51550
|
+
});
|
|
51551
|
+
return {
|
|
51552
|
+
baseline: baselineMetrics,
|
|
51553
|
+
current: currentMetrics,
|
|
51554
|
+
comparison: {
|
|
51555
|
+
comparisonAvailable: false,
|
|
51556
|
+
newFailures: [],
|
|
51557
|
+
newPasses: [],
|
|
51558
|
+
unchanged: [],
|
|
51559
|
+
addedCases: [],
|
|
51560
|
+
removedCases: []
|
|
51561
|
+
},
|
|
51562
|
+
eligibility,
|
|
51563
|
+
hasRegression: false,
|
|
51564
|
+
threshold
|
|
51565
|
+
};
|
|
51566
|
+
}
|
|
51646
51567
|
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
51647
51568
|
this.emit("progress", {
|
|
51648
51569
|
message: "Comparing cases...",
|
|
@@ -51660,9 +51581,11 @@ class ArtemisKit {
|
|
|
51660
51581
|
baseline: baselineMetrics,
|
|
51661
51582
|
current: currentMetrics,
|
|
51662
51583
|
comparison: {
|
|
51584
|
+
comparisonAvailable: true,
|
|
51663
51585
|
successRateDelta,
|
|
51664
51586
|
...caseComparison
|
|
51665
51587
|
},
|
|
51588
|
+
eligibility,
|
|
51666
51589
|
hasRegression,
|
|
51667
51590
|
threshold
|
|
51668
51591
|
};
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Type definitions for the programmatic SDK
|
|
4
4
|
*/
|
|
5
|
-
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
5
|
+
import type { AdapterConfig, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
6
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
7
|
import type { Scenario } from '@artemiskit/core';
|
|
8
8
|
/**
|
|
@@ -327,8 +327,10 @@ export interface RunSummary {
|
|
|
327
327
|
* Comparison details between two runs
|
|
328
328
|
*/
|
|
329
329
|
export interface ComparisonDetails {
|
|
330
|
-
/**
|
|
331
|
-
|
|
330
|
+
/** True when the runs are eligible for a metric/case delta. */
|
|
331
|
+
comparisonAvailable: boolean;
|
|
332
|
+
/** Change in success rate (current - baseline), absent for incomparable runs. */
|
|
333
|
+
successRateDelta?: number;
|
|
332
334
|
/** Cases that passed in baseline but failed in current */
|
|
333
335
|
newFailures: Array<{
|
|
334
336
|
caseId: string;
|
|
@@ -371,7 +373,9 @@ export interface CompareResult {
|
|
|
371
373
|
current: RunSummary;
|
|
372
374
|
/** Detailed comparison */
|
|
373
375
|
comparison: ComparisonDetails;
|
|
374
|
-
/**
|
|
376
|
+
/** Versioned compatibility decision that governs the comparison. */
|
|
377
|
+
eligibility: ComparisonEligibility;
|
|
378
|
+
/** Whether a regression was detected; always false for incomparable runs. */
|
|
375
379
|
hasRegression: boolean;
|
|
376
380
|
/** Threshold used for regression detection */
|
|
377
381
|
threshold: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.6",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.5.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.3",
|
|
75
|
+
"@artemiskit/redteam": "0.3.7"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
package/src/artemiskit.ts
CHANGED
|
@@ -15,10 +15,12 @@ import {
|
|
|
15
15
|
type StressManifest,
|
|
16
16
|
type StressMetrics,
|
|
17
17
|
type StressRequestResult,
|
|
18
|
+
assessComparisonEligibility,
|
|
18
19
|
runScenario as coreRunScenario,
|
|
19
20
|
createAdapter,
|
|
20
21
|
createStorageAdapter,
|
|
21
22
|
getGitInfo,
|
|
23
|
+
isComparisonAvailable,
|
|
22
24
|
parseScenarioFile,
|
|
23
25
|
resolveScenarioPaths,
|
|
24
26
|
} from '@artemiskit/core';
|
|
@@ -1025,6 +1027,34 @@ export class ArtemisKit {
|
|
|
1025
1027
|
// Extract metrics from manifests
|
|
1026
1028
|
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
1027
1029
|
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
1030
|
+
const eligibility = assessComparisonEligibility(
|
|
1031
|
+
baselineManifest as RunManifest,
|
|
1032
|
+
currentManifest as RunManifest
|
|
1033
|
+
);
|
|
1034
|
+
|
|
1035
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
1036
|
+
this.emit('progress', {
|
|
1037
|
+
message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(', ')}`,
|
|
1038
|
+
phase: 'teardown',
|
|
1039
|
+
progress: 100,
|
|
1040
|
+
});
|
|
1041
|
+
|
|
1042
|
+
return {
|
|
1043
|
+
baseline: baselineMetrics,
|
|
1044
|
+
current: currentMetrics,
|
|
1045
|
+
comparison: {
|
|
1046
|
+
comparisonAvailable: false,
|
|
1047
|
+
newFailures: [],
|
|
1048
|
+
newPasses: [],
|
|
1049
|
+
unchanged: [],
|
|
1050
|
+
addedCases: [],
|
|
1051
|
+
removedCases: [],
|
|
1052
|
+
},
|
|
1053
|
+
eligibility,
|
|
1054
|
+
hasRegression: false,
|
|
1055
|
+
threshold,
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1028
1058
|
|
|
1029
1059
|
// Calculate success rate delta
|
|
1030
1060
|
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
@@ -1054,9 +1084,11 @@ export class ArtemisKit {
|
|
|
1054
1084
|
baseline: baselineMetrics,
|
|
1055
1085
|
current: currentMetrics,
|
|
1056
1086
|
comparison: {
|
|
1087
|
+
comparisonAvailable: true,
|
|
1057
1088
|
successRateDelta,
|
|
1058
1089
|
...caseComparison,
|
|
1059
1090
|
},
|
|
1091
|
+
eligibility,
|
|
1060
1092
|
hasRegression,
|
|
1061
1093
|
threshold,
|
|
1062
1094
|
};
|
package/src/types.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import type {
|
|
7
7
|
AdapterConfig,
|
|
8
8
|
CaseResult,
|
|
9
|
+
ComparisonEligibility,
|
|
9
10
|
ModelClient,
|
|
10
11
|
RedTeamManifest,
|
|
11
12
|
RunManifest,
|
|
@@ -393,8 +394,10 @@ export interface RunSummary {
|
|
|
393
394
|
* Comparison details between two runs
|
|
394
395
|
*/
|
|
395
396
|
export interface ComparisonDetails {
|
|
396
|
-
/**
|
|
397
|
-
|
|
397
|
+
/** True when the runs are eligible for a metric/case delta. */
|
|
398
|
+
comparisonAvailable: boolean;
|
|
399
|
+
/** Change in success rate (current - baseline), absent for incomparable runs. */
|
|
400
|
+
successRateDelta?: number;
|
|
398
401
|
/** Cases that passed in baseline but failed in current */
|
|
399
402
|
newFailures: Array<{
|
|
400
403
|
caseId: string;
|
|
@@ -438,7 +441,9 @@ export interface CompareResult {
|
|
|
438
441
|
current: RunSummary;
|
|
439
442
|
/** Detailed comparison */
|
|
440
443
|
comparison: ComparisonDetails;
|
|
441
|
-
/**
|
|
444
|
+
/** Versioned compatibility decision that governs the comparison. */
|
|
445
|
+
eligibility: ComparisonEligibility;
|
|
446
|
+
/** Whether a regression was detected; always false for incomparable runs. */
|
|
442
447
|
hasRegression: boolean;
|
|
443
448
|
/** Threshold used for regression detection */
|
|
444
449
|
threshold: number;
|