@artemiskit/sdk 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.js +256 -331
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.4.5
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies
|
|
8
|
+
- @artemiskit/core@0.5.2
|
|
9
|
+
- @artemiskit/redteam@0.3.6
|
|
10
|
+
|
|
11
|
+
## 0.4.4
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Updated dependencies
|
|
16
|
+
- @artemiskit/core@0.5.1
|
|
17
|
+
- @artemiskit/redteam@0.3.5
|
|
18
|
+
|
|
3
19
|
## 0.4.3
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/dist/index.js
CHANGED
|
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
|
|
|
26484
26484
|
tags: arrayType(stringType()).optional().default([]),
|
|
26485
26485
|
metadata: recordType(unknownType()).optional().default({}),
|
|
26486
26486
|
timeout: numberType().optional(),
|
|
26487
|
-
retries: numberType().optional().default(0),
|
|
26487
|
+
retries: numberType().int().min(0).max(99).optional().default(0),
|
|
26488
26488
|
provider: ProviderSchema.optional(),
|
|
26489
26489
|
model: stringType().optional(),
|
|
26490
26490
|
variables: VariablesSchema.optional(),
|
|
@@ -27129,16 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
27129
27129
|
}
|
|
27130
27130
|
async function executeCase(testCase, context) {
|
|
27131
27131
|
const { timeout, retries = 0 } = context;
|
|
27132
|
+
if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
|
|
27133
|
+
throw new RangeError("retries must be a whole number between 0 and 99");
|
|
27134
|
+
}
|
|
27132
27135
|
const caseStartTime = Date.now();
|
|
27136
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
27137
|
+
const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
|
|
27138
|
+
const repetitionIndex = context.repetition?.index ?? 1;
|
|
27139
|
+
const attemptEvidence = [];
|
|
27133
27140
|
let lastError = null;
|
|
27134
27141
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
27142
|
+
const attemptStartTime = Date.now();
|
|
27135
27143
|
try {
|
|
27136
27144
|
const result = await executeCaseAttempt(testCase, context, timeout);
|
|
27137
|
-
return
|
|
27145
|
+
return withAttemptEvidence(result, attemptEvidence, {
|
|
27146
|
+
retryChainId,
|
|
27147
|
+
repetitionIndex,
|
|
27148
|
+
attemptNumber: attempt + 1,
|
|
27149
|
+
includedInOutcome: true,
|
|
27150
|
+
latencyMs: result.latencyMs
|
|
27151
|
+
});
|
|
27138
27152
|
} catch (error) {
|
|
27139
27153
|
lastError = error;
|
|
27140
|
-
if (error instanceof ToolLoopError)
|
|
27141
|
-
return
|
|
27154
|
+
if (error instanceof ToolLoopError) {
|
|
27155
|
+
return withAttemptEvidence(error.caseResult, attemptEvidence, {
|
|
27156
|
+
retryChainId,
|
|
27157
|
+
repetitionIndex,
|
|
27158
|
+
attemptNumber: attempt + 1,
|
|
27159
|
+
includedInOutcome: true,
|
|
27160
|
+
latencyMs: error.caseResult.latencyMs,
|
|
27161
|
+
errorCode: "tool_error"
|
|
27162
|
+
});
|
|
27163
|
+
}
|
|
27164
|
+
attemptEvidence.push({
|
|
27165
|
+
attempt_id: `${retryChainId}:${attempt + 1}`,
|
|
27166
|
+
retry_chain_id: retryChainId,
|
|
27167
|
+
repetition_index: repetitionIndex,
|
|
27168
|
+
attempt_number: attempt + 1,
|
|
27169
|
+
status: "error",
|
|
27170
|
+
included_in_outcome: false,
|
|
27171
|
+
latency_ms: Date.now() - attemptStartTime,
|
|
27172
|
+
error_code: error instanceof TimeoutError ? "timeout" : "target_error"
|
|
27173
|
+
});
|
|
27142
27174
|
if (attempt < retries) {
|
|
27143
27175
|
await sleep(2 ** attempt * 1000);
|
|
27144
27176
|
}
|
|
@@ -27151,6 +27183,10 @@ async function executeCase(testCase, context) {
|
|
|
27151
27183
|
ok: false,
|
|
27152
27184
|
status: "error",
|
|
27153
27185
|
attempts: retries + 1,
|
|
27186
|
+
attempt_evidence: attemptEvidence.map((entry, index) => ({
|
|
27187
|
+
...entry,
|
|
27188
|
+
included_in_outcome: index === attemptEvidence.length - 1
|
|
27189
|
+
})),
|
|
27154
27190
|
score: 0,
|
|
27155
27191
|
matcherType: testCase.expected.type,
|
|
27156
27192
|
reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
|
|
@@ -27160,11 +27196,34 @@ async function executeCase(testCase, context) {
|
|
|
27160
27196
|
response: "",
|
|
27161
27197
|
expected: testCase.expected,
|
|
27162
27198
|
tags: testCase.tags,
|
|
27163
|
-
error: lastError?.message
|
|
27199
|
+
error: lastError?.message,
|
|
27200
|
+
target: targetEvidence(context.client.provider, requestedModel)
|
|
27164
27201
|
};
|
|
27165
27202
|
}
|
|
27203
|
+
function withAttemptEvidence(result, priorAttempts, input) {
|
|
27204
|
+
return {
|
|
27205
|
+
...result,
|
|
27206
|
+
attempts: input.attemptNumber,
|
|
27207
|
+
attempt_evidence: [
|
|
27208
|
+
...priorAttempts,
|
|
27209
|
+
{
|
|
27210
|
+
attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
|
|
27211
|
+
retry_chain_id: input.retryChainId,
|
|
27212
|
+
repetition_index: input.repetitionIndex,
|
|
27213
|
+
attempt_number: input.attemptNumber,
|
|
27214
|
+
status: getTerminalStatus(result),
|
|
27215
|
+
included_in_outcome: input.includedInOutcome,
|
|
27216
|
+
latency_ms: input.latencyMs,
|
|
27217
|
+
...input.errorCode ? { error_code: input.errorCode } : {}
|
|
27218
|
+
}
|
|
27219
|
+
]
|
|
27220
|
+
};
|
|
27221
|
+
}
|
|
27222
|
+
function getTerminalStatus(result) {
|
|
27223
|
+
return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
|
|
27224
|
+
}
|
|
27166
27225
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
27167
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
27226
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
27168
27227
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
27169
27228
|
let prompt = substituteVariables(testCase.prompt, variables);
|
|
27170
27229
|
if (scenario.setup?.systemPrompt && typeof prompt === "string") {
|
|
@@ -27182,7 +27241,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27182
27241
|
const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
|
27183
27242
|
const generate = () => client.generate({
|
|
27184
27243
|
prompt: loopPrompt,
|
|
27185
|
-
model: testCase.model || scenario.model,
|
|
27244
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
27186
27245
|
temperature: scenario.temperature,
|
|
27187
27246
|
maxTokens: scenario.maxTokens,
|
|
27188
27247
|
seed: scenario.seed,
|
|
@@ -27190,6 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27190
27249
|
});
|
|
27191
27250
|
const generatePromise = generate();
|
|
27192
27251
|
let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
|
|
27252
|
+
const observedModels = [result.model];
|
|
27193
27253
|
const generationMetrics = {
|
|
27194
27254
|
latencyMs: result.latencyMs,
|
|
27195
27255
|
tokens: { ...result.tokens }
|
|
@@ -27200,7 +27260,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27200
27260
|
status: "error",
|
|
27201
27261
|
steps: 0,
|
|
27202
27262
|
terminationReason: "tool_error"
|
|
27203
|
-
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
|
|
27263
|
+
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27204
27264
|
}
|
|
27205
27265
|
const executor = toolExecutor ?? new FixtureToolExecutor({
|
|
27206
27266
|
tools,
|
|
@@ -27219,7 +27279,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27219
27279
|
status: "error",
|
|
27220
27280
|
steps: step,
|
|
27221
27281
|
terminationReason: "duplicate_call"
|
|
27222
|
-
}, generationMetrics, "TOOL_DUPLICATE_CALL");
|
|
27282
|
+
}, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27223
27283
|
}
|
|
27224
27284
|
seenCalls.add(fingerprint);
|
|
27225
27285
|
const toolStart = Date.now();
|
|
@@ -27246,7 +27306,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27246
27306
|
status: "error",
|
|
27247
27307
|
steps: step + 1,
|
|
27248
27308
|
terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
|
|
27249
|
-
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
|
|
27309
|
+
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27250
27310
|
}
|
|
27251
27311
|
const content = JSON.stringify(execution.result ?? {});
|
|
27252
27312
|
loopPrompt.push({
|
|
@@ -27258,7 +27318,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27258
27318
|
}
|
|
27259
27319
|
const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
|
|
27260
27320
|
if (remainingLoopTime <= 0) {
|
|
27261
|
-
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
|
|
27321
|
+
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27262
27322
|
}
|
|
27263
27323
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
27264
27324
|
try {
|
|
@@ -27269,8 +27329,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27269
27329
|
status: "error",
|
|
27270
27330
|
steps: step + 1,
|
|
27271
27331
|
terminationReason: timedOut ? "timeout" : "tool_error"
|
|
27272
|
-
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
|
|
27332
|
+
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27273
27333
|
}
|
|
27334
|
+
observedModels.push(result.model);
|
|
27274
27335
|
generationMetrics.latencyMs += result.latencyMs;
|
|
27275
27336
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
27276
27337
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -27281,7 +27342,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27281
27342
|
status: "error",
|
|
27282
27343
|
steps: policy.maxSteps,
|
|
27283
27344
|
terminationReason: "max_steps"
|
|
27284
|
-
}, generationMetrics, "TOOL_LOOP_MAX_STEPS");
|
|
27345
|
+
}, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27285
27346
|
}
|
|
27286
27347
|
toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
|
|
27287
27348
|
}
|
|
@@ -27377,11 +27438,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27377
27438
|
tags: testCase.tags,
|
|
27378
27439
|
redaction: redactionInfo,
|
|
27379
27440
|
evidence: finalEvidence,
|
|
27441
|
+
target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
|
|
27380
27442
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
27381
27443
|
toolLoop
|
|
27382
27444
|
};
|
|
27383
27445
|
}
|
|
27384
|
-
function
|
|
27446
|
+
function targetEvidence(provider, requestedModel, observedModels) {
|
|
27447
|
+
const observed = [
|
|
27448
|
+
...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
|
|
27449
|
+
].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
|
|
27450
|
+
return {
|
|
27451
|
+
provider: sanitizeArtifactText(provider, 100) ?? "unknown",
|
|
27452
|
+
...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
|
|
27453
|
+
...observed.length ? { observed_models: observed } : {}
|
|
27454
|
+
};
|
|
27455
|
+
}
|
|
27456
|
+
function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
|
|
27385
27457
|
return new ToolLoopError(code, {
|
|
27386
27458
|
id: testCase.id,
|
|
27387
27459
|
name: testCase.name,
|
|
@@ -27397,6 +27469,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
27397
27469
|
expected: testCase.expected,
|
|
27398
27470
|
tags: testCase.tags,
|
|
27399
27471
|
error: code,
|
|
27472
|
+
target,
|
|
27400
27473
|
toolTrace,
|
|
27401
27474
|
toolLoop
|
|
27402
27475
|
});
|
|
@@ -27510,302 +27583,6 @@ function nanoid(size = 21) {
|
|
|
27510
27583
|
}
|
|
27511
27584
|
return id;
|
|
27512
27585
|
}
|
|
27513
|
-
var MODEL_PRICING = {
|
|
27514
|
-
"gpt-5": {
|
|
27515
|
-
promptPer1K: 0.00125,
|
|
27516
|
-
completionPer1K: 0.01,
|
|
27517
|
-
lastUpdated: "2026-01",
|
|
27518
|
-
notes: "400K context window"
|
|
27519
|
-
},
|
|
27520
|
-
"gpt-5.1": {
|
|
27521
|
-
promptPer1K: 0.00125,
|
|
27522
|
-
completionPer1K: 0.01,
|
|
27523
|
-
lastUpdated: "2026-01"
|
|
27524
|
-
},
|
|
27525
|
-
"gpt-5.2": {
|
|
27526
|
-
promptPer1K: 0.00175,
|
|
27527
|
-
completionPer1K: 0.014,
|
|
27528
|
-
lastUpdated: "2026-01"
|
|
27529
|
-
},
|
|
27530
|
-
"gpt-5-mini": {
|
|
27531
|
-
promptPer1K: 0.00025,
|
|
27532
|
-
completionPer1K: 0.002,
|
|
27533
|
-
lastUpdated: "2026-01"
|
|
27534
|
-
},
|
|
27535
|
-
"gpt-5-nano": {
|
|
27536
|
-
promptPer1K: 0.00005,
|
|
27537
|
-
completionPer1K: 0.0004,
|
|
27538
|
-
lastUpdated: "2026-01"
|
|
27539
|
-
},
|
|
27540
|
-
"gpt-4.1": {
|
|
27541
|
-
promptPer1K: 0.002,
|
|
27542
|
-
completionPer1K: 0.008,
|
|
27543
|
-
lastUpdated: "2026-01",
|
|
27544
|
-
notes: "1M context window"
|
|
27545
|
-
},
|
|
27546
|
-
"gpt-4.1-mini": {
|
|
27547
|
-
promptPer1K: 0.0004,
|
|
27548
|
-
completionPer1K: 0.0016,
|
|
27549
|
-
lastUpdated: "2026-01"
|
|
27550
|
-
},
|
|
27551
|
-
"gpt-4.1-nano": {
|
|
27552
|
-
promptPer1K: 0.0001,
|
|
27553
|
-
completionPer1K: 0.0004,
|
|
27554
|
-
lastUpdated: "2026-01"
|
|
27555
|
-
},
|
|
27556
|
-
"gpt-4o": {
|
|
27557
|
-
promptPer1K: 0.0025,
|
|
27558
|
-
completionPer1K: 0.01,
|
|
27559
|
-
lastUpdated: "2026-01",
|
|
27560
|
-
notes: "128K context window"
|
|
27561
|
-
},
|
|
27562
|
-
"gpt-4o-mini": {
|
|
27563
|
-
promptPer1K: 0.00015,
|
|
27564
|
-
completionPer1K: 0.0006,
|
|
27565
|
-
lastUpdated: "2026-01",
|
|
27566
|
-
notes: "128K context window"
|
|
27567
|
-
},
|
|
27568
|
-
o1: {
|
|
27569
|
-
promptPer1K: 0.015,
|
|
27570
|
-
completionPer1K: 0.06,
|
|
27571
|
-
lastUpdated: "2026-01",
|
|
27572
|
-
notes: "Reasoning model - internal thinking tokens billed as output"
|
|
27573
|
-
},
|
|
27574
|
-
o3: {
|
|
27575
|
-
promptPer1K: 0.002,
|
|
27576
|
-
completionPer1K: 0.008,
|
|
27577
|
-
lastUpdated: "2026-01"
|
|
27578
|
-
},
|
|
27579
|
-
"o3-mini": {
|
|
27580
|
-
promptPer1K: 0.0011,
|
|
27581
|
-
completionPer1K: 0.0044,
|
|
27582
|
-
lastUpdated: "2026-01"
|
|
27583
|
-
},
|
|
27584
|
-
"o4-mini": {
|
|
27585
|
-
promptPer1K: 0.0011,
|
|
27586
|
-
completionPer1K: 0.0044,
|
|
27587
|
-
lastUpdated: "2026-01"
|
|
27588
|
-
},
|
|
27589
|
-
"gpt-4-turbo": {
|
|
27590
|
-
promptPer1K: 0.01,
|
|
27591
|
-
completionPer1K: 0.03,
|
|
27592
|
-
lastUpdated: "2026-01"
|
|
27593
|
-
},
|
|
27594
|
-
"gpt-4": {
|
|
27595
|
-
promptPer1K: 0.03,
|
|
27596
|
-
completionPer1K: 0.06,
|
|
27597
|
-
lastUpdated: "2026-01"
|
|
27598
|
-
},
|
|
27599
|
-
"gpt-3.5-turbo": {
|
|
27600
|
-
promptPer1K: 0.0005,
|
|
27601
|
-
completionPer1K: 0.0015,
|
|
27602
|
-
lastUpdated: "2026-01"
|
|
27603
|
-
},
|
|
27604
|
-
"claude-opus-4.5": {
|
|
27605
|
-
promptPer1K: 0.005,
|
|
27606
|
-
completionPer1K: 0.025,
|
|
27607
|
-
lastUpdated: "2026-01",
|
|
27608
|
-
notes: "Most capable Claude model"
|
|
27609
|
-
},
|
|
27610
|
-
"claude-sonnet-4.5": {
|
|
27611
|
-
promptPer1K: 0.003,
|
|
27612
|
-
completionPer1K: 0.015,
|
|
27613
|
-
lastUpdated: "2026-01",
|
|
27614
|
-
notes: "Balanced performance and cost"
|
|
27615
|
-
},
|
|
27616
|
-
"claude-haiku-4.5": {
|
|
27617
|
-
promptPer1K: 0.001,
|
|
27618
|
-
completionPer1K: 0.005,
|
|
27619
|
-
lastUpdated: "2026-01",
|
|
27620
|
-
notes: "Fastest Claude model"
|
|
27621
|
-
},
|
|
27622
|
-
"claude-opus-4": {
|
|
27623
|
-
promptPer1K: 0.015,
|
|
27624
|
-
completionPer1K: 0.075,
|
|
27625
|
-
lastUpdated: "2026-01"
|
|
27626
|
-
},
|
|
27627
|
-
"claude-opus-4.1": {
|
|
27628
|
-
promptPer1K: 0.015,
|
|
27629
|
-
completionPer1K: 0.075,
|
|
27630
|
-
lastUpdated: "2026-01"
|
|
27631
|
-
},
|
|
27632
|
-
"claude-sonnet-4": {
|
|
27633
|
-
promptPer1K: 0.003,
|
|
27634
|
-
completionPer1K: 0.015,
|
|
27635
|
-
lastUpdated: "2026-01"
|
|
27636
|
-
},
|
|
27637
|
-
"claude-sonnet-3.7": {
|
|
27638
|
-
promptPer1K: 0.003,
|
|
27639
|
-
completionPer1K: 0.015,
|
|
27640
|
-
lastUpdated: "2026-01"
|
|
27641
|
-
},
|
|
27642
|
-
"claude-3-7-sonnet": {
|
|
27643
|
-
promptPer1K: 0.003,
|
|
27644
|
-
completionPer1K: 0.015,
|
|
27645
|
-
lastUpdated: "2026-01"
|
|
27646
|
-
},
|
|
27647
|
-
"claude-3-5-sonnet-20241022": {
|
|
27648
|
-
promptPer1K: 0.003,
|
|
27649
|
-
completionPer1K: 0.015,
|
|
27650
|
-
lastUpdated: "2026-01"
|
|
27651
|
-
},
|
|
27652
|
-
"claude-3-5-haiku-20241022": {
|
|
27653
|
-
promptPer1K: 0.0008,
|
|
27654
|
-
completionPer1K: 0.004,
|
|
27655
|
-
lastUpdated: "2026-01"
|
|
27656
|
-
},
|
|
27657
|
-
"claude-haiku-3.5": {
|
|
27658
|
-
promptPer1K: 0.0008,
|
|
27659
|
-
completionPer1K: 0.004,
|
|
27660
|
-
lastUpdated: "2026-01"
|
|
27661
|
-
},
|
|
27662
|
-
"claude-3-opus": {
|
|
27663
|
-
promptPer1K: 0.015,
|
|
27664
|
-
completionPer1K: 0.075,
|
|
27665
|
-
lastUpdated: "2026-01"
|
|
27666
|
-
},
|
|
27667
|
-
"claude-3-sonnet": {
|
|
27668
|
-
promptPer1K: 0.003,
|
|
27669
|
-
completionPer1K: 0.015,
|
|
27670
|
-
lastUpdated: "2026-01"
|
|
27671
|
-
},
|
|
27672
|
-
"claude-3-haiku": {
|
|
27673
|
-
promptPer1K: 0.00025,
|
|
27674
|
-
completionPer1K: 0.00125,
|
|
27675
|
-
lastUpdated: "2026-01"
|
|
27676
|
-
},
|
|
27677
|
-
"claude-3.5-sonnet": {
|
|
27678
|
-
promptPer1K: 0.003,
|
|
27679
|
-
completionPer1K: 0.015,
|
|
27680
|
-
lastUpdated: "2026-01"
|
|
27681
|
-
},
|
|
27682
|
-
"claude-3.5-haiku": {
|
|
27683
|
-
promptPer1K: 0.0008,
|
|
27684
|
-
completionPer1K: 0.004,
|
|
27685
|
-
lastUpdated: "2026-01"
|
|
27686
|
-
}
|
|
27687
|
-
};
|
|
27688
|
-
var DEFAULT_PRICING = {
|
|
27689
|
-
promptPer1K: 0.003,
|
|
27690
|
-
completionPer1K: 0.015,
|
|
27691
|
-
lastUpdated: "2026-01",
|
|
27692
|
-
notes: "Default pricing - verify with provider"
|
|
27693
|
-
};
|
|
27694
|
-
function getModelPricing(model) {
|
|
27695
|
-
if (MODEL_PRICING[model]) {
|
|
27696
|
-
return MODEL_PRICING[model];
|
|
27697
|
-
}
|
|
27698
|
-
const lowerModel = model.toLowerCase();
|
|
27699
|
-
for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
|
|
27700
|
-
if (key.toLowerCase() === lowerModel) {
|
|
27701
|
-
return pricing;
|
|
27702
|
-
}
|
|
27703
|
-
}
|
|
27704
|
-
if (lowerModel.includes("gpt-5.2")) {
|
|
27705
|
-
return MODEL_PRICING["gpt-5.2"];
|
|
27706
|
-
}
|
|
27707
|
-
if (lowerModel.includes("gpt-5.1")) {
|
|
27708
|
-
return MODEL_PRICING["gpt-5.1"];
|
|
27709
|
-
}
|
|
27710
|
-
if (lowerModel.includes("gpt-5-mini")) {
|
|
27711
|
-
return MODEL_PRICING["gpt-5-mini"];
|
|
27712
|
-
}
|
|
27713
|
-
if (lowerModel.includes("gpt-5-nano")) {
|
|
27714
|
-
return MODEL_PRICING["gpt-5-nano"];
|
|
27715
|
-
}
|
|
27716
|
-
if (lowerModel.includes("gpt-5")) {
|
|
27717
|
-
return MODEL_PRICING["gpt-5"];
|
|
27718
|
-
}
|
|
27719
|
-
if (lowerModel.includes("gpt-4.1-mini")) {
|
|
27720
|
-
return MODEL_PRICING["gpt-4.1-mini"];
|
|
27721
|
-
}
|
|
27722
|
-
if (lowerModel.includes("gpt-4.1-nano")) {
|
|
27723
|
-
return MODEL_PRICING["gpt-4.1-nano"];
|
|
27724
|
-
}
|
|
27725
|
-
if (lowerModel.includes("gpt-4.1")) {
|
|
27726
|
-
return MODEL_PRICING["gpt-4.1"];
|
|
27727
|
-
}
|
|
27728
|
-
if (lowerModel.includes("gpt-4o-mini")) {
|
|
27729
|
-
return MODEL_PRICING["gpt-4o-mini"];
|
|
27730
|
-
}
|
|
27731
|
-
if (lowerModel.includes("gpt-4o")) {
|
|
27732
|
-
return MODEL_PRICING["gpt-4o"];
|
|
27733
|
-
}
|
|
27734
|
-
if (lowerModel.includes("o4-mini")) {
|
|
27735
|
-
return MODEL_PRICING["o4-mini"];
|
|
27736
|
-
}
|
|
27737
|
-
if (lowerModel.includes("o3-mini")) {
|
|
27738
|
-
return MODEL_PRICING["o3-mini"];
|
|
27739
|
-
}
|
|
27740
|
-
if (lowerModel.includes("o3")) {
|
|
27741
|
-
return MODEL_PRICING.o3;
|
|
27742
|
-
}
|
|
27743
|
-
if (lowerModel.includes("o1")) {
|
|
27744
|
-
return MODEL_PRICING.o1;
|
|
27745
|
-
}
|
|
27746
|
-
if (lowerModel.includes("gpt-4-turbo")) {
|
|
27747
|
-
return MODEL_PRICING["gpt-4-turbo"];
|
|
27748
|
-
}
|
|
27749
|
-
if (lowerModel.includes("gpt-4")) {
|
|
27750
|
-
return MODEL_PRICING["gpt-4"];
|
|
27751
|
-
}
|
|
27752
|
-
if (lowerModel.includes("gpt-3.5")) {
|
|
27753
|
-
return MODEL_PRICING["gpt-3.5-turbo"];
|
|
27754
|
-
}
|
|
27755
|
-
if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
|
|
27756
|
-
return MODEL_PRICING["claude-opus-4.5"];
|
|
27757
|
-
}
|
|
27758
|
-
if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
|
|
27759
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27760
|
-
}
|
|
27761
|
-
if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
|
|
27762
|
-
return MODEL_PRICING["claude-haiku-4.5"];
|
|
27763
|
-
}
|
|
27764
|
-
if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
|
|
27765
|
-
return MODEL_PRICING["claude-opus-4.1"];
|
|
27766
|
-
}
|
|
27767
|
-
if (lowerModel.includes("opus-4")) {
|
|
27768
|
-
return MODEL_PRICING["claude-opus-4"];
|
|
27769
|
-
}
|
|
27770
|
-
if (lowerModel.includes("sonnet-4")) {
|
|
27771
|
-
return MODEL_PRICING["claude-sonnet-4"];
|
|
27772
|
-
}
|
|
27773
|
-
if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
|
|
27774
|
-
return MODEL_PRICING["claude-sonnet-3.7"];
|
|
27775
|
-
}
|
|
27776
|
-
if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
|
|
27777
|
-
return MODEL_PRICING["claude-3.5-sonnet"];
|
|
27778
|
-
}
|
|
27779
|
-
if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
|
|
27780
|
-
return MODEL_PRICING["claude-3.5-haiku"];
|
|
27781
|
-
}
|
|
27782
|
-
if (lowerModel.includes("claude-3-opus")) {
|
|
27783
|
-
return MODEL_PRICING["claude-3-opus"];
|
|
27784
|
-
}
|
|
27785
|
-
if (lowerModel.includes("claude-3-sonnet")) {
|
|
27786
|
-
return MODEL_PRICING["claude-3-sonnet"];
|
|
27787
|
-
}
|
|
27788
|
-
if (lowerModel.includes("claude-3-haiku")) {
|
|
27789
|
-
return MODEL_PRICING["claude-3-haiku"];
|
|
27790
|
-
}
|
|
27791
|
-
if (lowerModel.includes("claude")) {
|
|
27792
|
-
return MODEL_PRICING["claude-sonnet-4.5"];
|
|
27793
|
-
}
|
|
27794
|
-
return DEFAULT_PRICING;
|
|
27795
|
-
}
|
|
27796
|
-
function estimateCost(promptTokens, completionTokens, model) {
|
|
27797
|
-
const pricing = getModelPricing(model);
|
|
27798
|
-
const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
|
|
27799
|
-
const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
|
|
27800
|
-
const totalUsd = promptCostUsd + completionCostUsd;
|
|
27801
|
-
return {
|
|
27802
|
-
totalUsd,
|
|
27803
|
-
promptCostUsd,
|
|
27804
|
-
completionCostUsd,
|
|
27805
|
-
model,
|
|
27806
|
-
pricing
|
|
27807
|
-
};
|
|
27808
|
-
}
|
|
27809
27586
|
function getEnvironmentInfo() {
|
|
27810
27587
|
return {
|
|
27811
27588
|
node_version: process.version,
|
|
@@ -27861,6 +27638,15 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27861
27638
|
if (manifest.workload_identity !== undefined) {
|
|
27862
27639
|
assertWorkloadIdentity(manifest.workload_identity);
|
|
27863
27640
|
}
|
|
27641
|
+
if (manifest.execution_provenance !== undefined) {
|
|
27642
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
27643
|
+
}
|
|
27644
|
+
if (manifest.attempt_evidence !== undefined) {
|
|
27645
|
+
assertRunAttemptEvidence(manifest.attempt_evidence);
|
|
27646
|
+
}
|
|
27647
|
+
if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
|
|
27648
|
+
assertCostProvenance(manifest.metrics.cost_provenance);
|
|
27649
|
+
}
|
|
27864
27650
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27865
27651
|
if (!isRecord2(caseResult)) {
|
|
27866
27652
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27871,8 +27657,91 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27871
27657
|
if (caseResult.evidence !== undefined) {
|
|
27872
27658
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
27873
27659
|
}
|
|
27660
|
+
if (caseResult.target !== undefined) {
|
|
27661
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
27662
|
+
}
|
|
27663
|
+
if (caseResult.attempt_evidence !== undefined) {
|
|
27664
|
+
assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
|
|
27665
|
+
}
|
|
27666
|
+
}
|
|
27667
|
+
}
|
|
27668
|
+
function assertRunAttemptEvidence(evidence) {
|
|
27669
|
+
if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
|
|
27670
|
+
throw new Error("Invalid run manifest: malformed attempt evidence");
|
|
27671
|
+
}
|
|
27672
|
+
}
|
|
27673
|
+
function assertCaseAttemptEvidence(evidence, index) {
|
|
27674
|
+
if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
|
|
27675
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27676
|
+
}
|
|
27677
|
+
for (const attempt of evidence) {
|
|
27678
|
+
if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
|
|
27679
|
+
throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
|
|
27680
|
+
}
|
|
27681
|
+
}
|
|
27682
|
+
}
|
|
27683
|
+
function assertCostProvenance(cost) {
|
|
27684
|
+
if (!isRecord2(cost) || cost.schema_version !== "1") {
|
|
27685
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27686
|
+
}
|
|
27687
|
+
if (cost.status === "unavailable") {
|
|
27688
|
+
if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
|
|
27689
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27690
|
+
}
|
|
27691
|
+
return;
|
|
27692
|
+
}
|
|
27693
|
+
if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
|
|
27694
|
+
throw new Error("Invalid run manifest: malformed cost provenance");
|
|
27695
|
+
}
|
|
27696
|
+
}
|
|
27697
|
+
function isCaseEvaluationStatus(value) {
|
|
27698
|
+
return value === "passed" || value === "failed" || value === "invalid" || value === "error";
|
|
27699
|
+
}
|
|
27700
|
+
function isBoundedNonemptyString(value, maxLength) {
|
|
27701
|
+
return typeof value === "string" && value.length > 0 && value.length <= maxLength;
|
|
27702
|
+
}
|
|
27703
|
+
function isNonnegativeFiniteNumber(value) {
|
|
27704
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
27705
|
+
}
|
|
27706
|
+
function isPositiveFiniteNumber(value) {
|
|
27707
|
+
return isNonnegativeFiniteNumber(value) && value > 0;
|
|
27708
|
+
}
|
|
27709
|
+
function isNonnegativeSafeInteger(value) {
|
|
27710
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
27711
|
+
}
|
|
27712
|
+
function isPositiveSafeInteger(value) {
|
|
27713
|
+
return isNonnegativeSafeInteger(value) && value > 0;
|
|
27714
|
+
}
|
|
27715
|
+
function isIsoTimestamp(value) {
|
|
27716
|
+
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
27717
|
+
}
|
|
27718
|
+
function assertCaseTargetEvidence(target) {
|
|
27719
|
+
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
27720
|
+
throw new Error("Invalid run manifest: malformed target evidence");
|
|
27721
|
+
}
|
|
27722
|
+
}
|
|
27723
|
+
function assertExecutionProvenance(provenance) {
|
|
27724
|
+
if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
|
|
27725
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27726
|
+
}
|
|
27727
|
+
const target = provenance.target;
|
|
27728
|
+
if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
|
|
27729
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27730
|
+
}
|
|
27731
|
+
if (provenance.evaluator !== undefined) {
|
|
27732
|
+
if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
27733
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27734
|
+
}
|
|
27874
27735
|
}
|
|
27875
27736
|
}
|
|
27737
|
+
function isBoundedStringList(value) {
|
|
27738
|
+
return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
|
|
27739
|
+
}
|
|
27740
|
+
function isGenerationConfig(value) {
|
|
27741
|
+
if (!isRecord2(value))
|
|
27742
|
+
return false;
|
|
27743
|
+
return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
|
|
27744
|
+
}
|
|
27876
27745
|
function assertWorkloadIdentity(identity) {
|
|
27877
27746
|
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
27878
27747
|
throw new Error("Invalid run manifest: malformed workload identity");
|
|
@@ -27921,6 +27790,10 @@ function createRunManifest(options) {
|
|
|
27921
27790
|
config,
|
|
27922
27791
|
resolvedConfig,
|
|
27923
27792
|
workloadIdentity,
|
|
27793
|
+
executionProvenance,
|
|
27794
|
+
attemptEvidence,
|
|
27795
|
+
costProvenance,
|
|
27796
|
+
runId,
|
|
27924
27797
|
cases,
|
|
27925
27798
|
startTime,
|
|
27926
27799
|
endTime,
|
|
@@ -27928,13 +27801,12 @@ function createRunManifest(options) {
|
|
|
27928
27801
|
runReason,
|
|
27929
27802
|
redaction
|
|
27930
27803
|
} = options;
|
|
27931
|
-
const
|
|
27932
|
-
const metrics = calculateMetrics(cases, modelForCost);
|
|
27804
|
+
const metrics = calculateMetrics(cases, costProvenance);
|
|
27933
27805
|
const git = getGitInfo();
|
|
27934
27806
|
const environment = getEnvironmentInfo();
|
|
27935
27807
|
return {
|
|
27936
|
-
version: "1.
|
|
27937
|
-
run_id: nanoid(12),
|
|
27808
|
+
version: "1.4",
|
|
27809
|
+
run_id: runId ?? nanoid(12),
|
|
27938
27810
|
project,
|
|
27939
27811
|
start_time: startTime.toISOString(),
|
|
27940
27812
|
end_time: endTime.toISOString(),
|
|
@@ -27942,6 +27814,8 @@ function createRunManifest(options) {
|
|
|
27942
27814
|
config,
|
|
27943
27815
|
resolved_config: resolvedConfig,
|
|
27944
27816
|
workload_identity: workloadIdentity,
|
|
27817
|
+
execution_provenance: executionProvenance,
|
|
27818
|
+
attempt_evidence: attemptEvidence,
|
|
27945
27819
|
metrics,
|
|
27946
27820
|
git,
|
|
27947
27821
|
provenance: {
|
|
@@ -27954,7 +27828,7 @@ function createRunManifest(options) {
|
|
|
27954
27828
|
redaction
|
|
27955
27829
|
};
|
|
27956
27830
|
}
|
|
27957
|
-
function calculateMetrics(cases,
|
|
27831
|
+
function calculateMetrics(cases, costProvenance) {
|
|
27958
27832
|
const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
|
|
27959
27833
|
const validCases = cases.filter((c) => {
|
|
27960
27834
|
const status = getCaseEvaluationStatus(c);
|
|
@@ -27966,21 +27840,11 @@ function calculateMetrics(cases, model) {
|
|
|
27966
27840
|
const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
|
|
27967
27841
|
const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
|
|
27968
27842
|
const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
|
|
27969
|
-
|
|
27970
|
-
|
|
27971
|
-
|
|
27972
|
-
|
|
27973
|
-
|
|
27974
|
-
total_usd: costEstimate.totalUsd,
|
|
27975
|
-
prompt_cost_usd: costEstimate.promptCostUsd,
|
|
27976
|
-
completion_cost_usd: costEstimate.completionCostUsd,
|
|
27977
|
-
model: costEstimate.model,
|
|
27978
|
-
pricing: {
|
|
27979
|
-
prompt_per_1k: pricing.promptPer1K,
|
|
27980
|
-
completion_per_1k: pricing.completionPer1K
|
|
27981
|
-
}
|
|
27982
|
-
};
|
|
27983
|
-
}
|
|
27843
|
+
const cost_provenance = costProvenance ?? {
|
|
27844
|
+
schema_version: "1",
|
|
27845
|
+
status: "unavailable",
|
|
27846
|
+
unavailable_reason: "provider_billing_not_recorded"
|
|
27847
|
+
};
|
|
27984
27848
|
return {
|
|
27985
27849
|
success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
|
|
27986
27850
|
total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
|
|
@@ -27995,7 +27859,7 @@ function calculateMetrics(cases, model) {
|
|
|
27995
27859
|
total_tokens: totalPromptTokens + totalCompletionTokens,
|
|
27996
27860
|
total_prompt_tokens: totalPromptTokens,
|
|
27997
27861
|
total_completion_tokens: totalCompletionTokens,
|
|
27998
|
-
|
|
27862
|
+
cost_provenance
|
|
27999
27863
|
};
|
|
28000
27864
|
}
|
|
28001
27865
|
function detectCIEnvironment() {
|
|
@@ -28092,6 +27956,38 @@ function canonicalize(value, patterns, key) {
|
|
|
28092
27956
|
}
|
|
28093
27957
|
return value;
|
|
28094
27958
|
}
|
|
27959
|
+
function createExecutionProvenance(options) {
|
|
27960
|
+
const requestedModels = uniqueStrings([
|
|
27961
|
+
options.requestedModel,
|
|
27962
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model)
|
|
27963
|
+
]);
|
|
27964
|
+
const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
|
|
27965
|
+
const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
|
|
27966
|
+
const generation = omitUndefined({
|
|
27967
|
+
temperature: options.temperature,
|
|
27968
|
+
max_tokens: options.maxTokens,
|
|
27969
|
+
seed: options.seed
|
|
27970
|
+
});
|
|
27971
|
+
return {
|
|
27972
|
+
schema_version: "1",
|
|
27973
|
+
target: {
|
|
27974
|
+
provider: boundedString(options.provider, 100) ?? "unknown",
|
|
27975
|
+
...requestedModels.length ? { requested_models: requestedModels } : {},
|
|
27976
|
+
...observedModels.length ? { observed_models: observedModels } : {},
|
|
27977
|
+
...Object.keys(generation).length ? { generation } : {}
|
|
27978
|
+
},
|
|
27979
|
+
...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
|
|
27980
|
+
};
|
|
27981
|
+
}
|
|
27982
|
+
function uniqueStrings(values) {
|
|
27983
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
|
|
27984
|
+
}
|
|
27985
|
+
function boundedString(value, maxLength) {
|
|
27986
|
+
return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
27987
|
+
}
|
|
27988
|
+
function omitUndefined(value) {
|
|
27989
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
|
27990
|
+
}
|
|
28095
27991
|
async function runScenario(options) {
|
|
28096
27992
|
const {
|
|
28097
27993
|
scenario,
|
|
@@ -28102,6 +27998,8 @@ async function runScenario(options) {
|
|
|
28102
27998
|
concurrency = 1,
|
|
28103
27999
|
timeout,
|
|
28104
28000
|
retries,
|
|
28001
|
+
repetition = { index: 1, total: 1 },
|
|
28002
|
+
costProvenance,
|
|
28105
28003
|
redaction,
|
|
28106
28004
|
toolExecutor,
|
|
28107
28005
|
onCaseComplete,
|
|
@@ -28117,6 +28015,7 @@ async function runScenario(options) {
|
|
|
28117
28015
|
}
|
|
28118
28016
|
onProgress?.(`Running ${cases.length} test cases...`);
|
|
28119
28017
|
const startTime = new Date;
|
|
28018
|
+
const runId = nanoid(12);
|
|
28120
28019
|
const results = [];
|
|
28121
28020
|
if (concurrency === 1) {
|
|
28122
28021
|
for (let i = 0;i < cases.length; i++) {
|
|
@@ -28124,8 +28023,11 @@ async function runScenario(options) {
|
|
|
28124
28023
|
const result = await executeCase(testCase, {
|
|
28125
28024
|
client,
|
|
28126
28025
|
scenario,
|
|
28026
|
+
requestedModel: resolvedConfig?.model,
|
|
28127
28027
|
timeout: testCase.timeout || timeout,
|
|
28128
28028
|
retries: testCase.retries ?? retries,
|
|
28029
|
+
runId,
|
|
28030
|
+
repetition,
|
|
28129
28031
|
redaction,
|
|
28130
28032
|
toolExecutor
|
|
28131
28033
|
});
|
|
@@ -28140,8 +28042,11 @@ async function runScenario(options) {
|
|
|
28140
28042
|
const result = await executeCase(testCase, {
|
|
28141
28043
|
client,
|
|
28142
28044
|
scenario,
|
|
28045
|
+
requestedModel: resolvedConfig?.model,
|
|
28143
28046
|
timeout: testCase.timeout || timeout,
|
|
28144
28047
|
retries: testCase.retries ?? retries,
|
|
28048
|
+
runId,
|
|
28049
|
+
repetition,
|
|
28145
28050
|
redaction,
|
|
28146
28051
|
toolExecutor
|
|
28147
28052
|
});
|
|
@@ -28184,9 +28089,29 @@ async function runScenario(options) {
|
|
|
28184
28089
|
},
|
|
28185
28090
|
resolvedConfig,
|
|
28186
28091
|
workloadIdentity: createWorkloadIdentity(scenario),
|
|
28092
|
+
executionProvenance: createExecutionProvenance({
|
|
28093
|
+
provider: client.provider,
|
|
28094
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
28095
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
28096
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
28097
|
+
seed: scenario.seed,
|
|
28098
|
+
cases: results
|
|
28099
|
+
}),
|
|
28100
|
+
attemptEvidence: {
|
|
28101
|
+
schema_version: "1",
|
|
28102
|
+
repetition,
|
|
28103
|
+
retry_policy: {
|
|
28104
|
+
default_max_retries: retries ?? 0,
|
|
28105
|
+
backoff: "exponential",
|
|
28106
|
+
initial_delay_ms: 1000
|
|
28107
|
+
},
|
|
28108
|
+
...timeout ? { timeout: { default_ms: timeout } } : {}
|
|
28109
|
+
},
|
|
28110
|
+
costProvenance,
|
|
28187
28111
|
cases: results,
|
|
28188
28112
|
startTime,
|
|
28189
28113
|
endTime,
|
|
28114
|
+
runId,
|
|
28190
28115
|
redaction: redactionInfo
|
|
28191
28116
|
});
|
|
28192
28117
|
const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.5",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.5.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.2",
|
|
75
|
+
"@artemiskit/redteam": "0.3.6"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|