@artemiskit/sdk 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/index.js +104 -11
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
package/dist/index.js
CHANGED
|
@@ -27130,6 +27130,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
27130
27130
|
async function executeCase(testCase, context) {
|
|
27131
27131
|
const { timeout, retries = 0 } = context;
|
|
27132
27132
|
const caseStartTime = Date.now();
|
|
27133
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
27133
27134
|
let lastError = null;
|
|
27134
27135
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
27135
27136
|
try {
|
|
@@ -27160,11 +27161,12 @@ async function executeCase(testCase, context) {
|
|
|
27160
27161
|
response: "",
|
|
27161
27162
|
expected: testCase.expected,
|
|
27162
27163
|
tags: testCase.tags,
|
|
27163
|
-
error: lastError?.message
|
|
27164
|
+
error: lastError?.message,
|
|
27165
|
+
target: targetEvidence(context.client.provider, requestedModel)
|
|
27164
27166
|
};
|
|
27165
27167
|
}
|
|
27166
27168
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
27167
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
27169
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
27168
27170
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
27169
27171
|
let prompt = substituteVariables(testCase.prompt, variables);
|
|
27170
27172
|
if (scenario.setup?.systemPrompt && typeof prompt === "string") {
|
|
@@ -27182,7 +27184,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27182
27184
|
const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
|
27183
27185
|
const generate = () => client.generate({
|
|
27184
27186
|
prompt: loopPrompt,
|
|
27185
|
-
model: testCase.model || scenario.model,
|
|
27187
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
27186
27188
|
temperature: scenario.temperature,
|
|
27187
27189
|
maxTokens: scenario.maxTokens,
|
|
27188
27190
|
seed: scenario.seed,
|
|
@@ -27190,6 +27192,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27190
27192
|
});
|
|
27191
27193
|
const generatePromise = generate();
|
|
27192
27194
|
let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
|
|
27195
|
+
const observedModels = [result.model];
|
|
27193
27196
|
const generationMetrics = {
|
|
27194
27197
|
latencyMs: result.latencyMs,
|
|
27195
27198
|
tokens: { ...result.tokens }
|
|
@@ -27200,7 +27203,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27200
27203
|
status: "error",
|
|
27201
27204
|
steps: 0,
|
|
27202
27205
|
terminationReason: "tool_error"
|
|
27203
|
-
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
|
|
27206
|
+
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27204
27207
|
}
|
|
27205
27208
|
const executor = toolExecutor ?? new FixtureToolExecutor({
|
|
27206
27209
|
tools,
|
|
@@ -27219,7 +27222,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27219
27222
|
status: "error",
|
|
27220
27223
|
steps: step,
|
|
27221
27224
|
terminationReason: "duplicate_call"
|
|
27222
|
-
}, generationMetrics, "TOOL_DUPLICATE_CALL");
|
|
27225
|
+
}, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27223
27226
|
}
|
|
27224
27227
|
seenCalls.add(fingerprint);
|
|
27225
27228
|
const toolStart = Date.now();
|
|
@@ -27246,7 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27246
27249
|
status: "error",
|
|
27247
27250
|
steps: step + 1,
|
|
27248
27251
|
terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
|
|
27249
|
-
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
|
|
27252
|
+
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27250
27253
|
}
|
|
27251
27254
|
const content = JSON.stringify(execution.result ?? {});
|
|
27252
27255
|
loopPrompt.push({
|
|
@@ -27258,7 +27261,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27258
27261
|
}
|
|
27259
27262
|
const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
|
|
27260
27263
|
if (remainingLoopTime <= 0) {
|
|
27261
|
-
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
|
|
27264
|
+
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27262
27265
|
}
|
|
27263
27266
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
27264
27267
|
try {
|
|
@@ -27269,8 +27272,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27269
27272
|
status: "error",
|
|
27270
27273
|
steps: step + 1,
|
|
27271
27274
|
terminationReason: timedOut ? "timeout" : "tool_error"
|
|
27272
|
-
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
|
|
27275
|
+
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27273
27276
|
}
|
|
27277
|
+
observedModels.push(result.model);
|
|
27274
27278
|
generationMetrics.latencyMs += result.latencyMs;
|
|
27275
27279
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
27276
27280
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -27281,7 +27285,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27281
27285
|
status: "error",
|
|
27282
27286
|
steps: policy.maxSteps,
|
|
27283
27287
|
terminationReason: "max_steps"
|
|
27284
|
-
}, generationMetrics, "TOOL_LOOP_MAX_STEPS");
|
|
27288
|
+
}, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27285
27289
|
}
|
|
27286
27290
|
toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
|
|
27287
27291
|
}
|
|
@@ -27377,11 +27381,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27377
27381
|
tags: testCase.tags,
|
|
27378
27382
|
redaction: redactionInfo,
|
|
27379
27383
|
evidence: finalEvidence,
|
|
27384
|
+
target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
|
|
27380
27385
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
27381
27386
|
toolLoop
|
|
27382
27387
|
};
|
|
27383
27388
|
}
|
|
27384
|
-
function
|
|
27389
|
+
function targetEvidence(provider, requestedModel, observedModels) {
|
|
27390
|
+
const observed = [
|
|
27391
|
+
...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
|
|
27392
|
+
].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
|
|
27393
|
+
return {
|
|
27394
|
+
provider: sanitizeArtifactText(provider, 100) ?? "unknown",
|
|
27395
|
+
...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
|
|
27396
|
+
...observed.length ? { observed_models: observed } : {}
|
|
27397
|
+
};
|
|
27398
|
+
}
|
|
27399
|
+
function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
|
|
27385
27400
|
return new ToolLoopError(code, {
|
|
27386
27401
|
id: testCase.id,
|
|
27387
27402
|
name: testCase.name,
|
|
@@ -27397,6 +27412,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
27397
27412
|
expected: testCase.expected,
|
|
27398
27413
|
tags: testCase.tags,
|
|
27399
27414
|
error: code,
|
|
27415
|
+
target,
|
|
27400
27416
|
toolTrace,
|
|
27401
27417
|
toolLoop
|
|
27402
27418
|
});
|
|
@@ -27861,6 +27877,9 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27861
27877
|
if (manifest.workload_identity !== undefined) {
|
|
27862
27878
|
assertWorkloadIdentity(manifest.workload_identity);
|
|
27863
27879
|
}
|
|
27880
|
+
if (manifest.execution_provenance !== undefined) {
|
|
27881
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
27882
|
+
}
|
|
27864
27883
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27865
27884
|
if (!isRecord2(caseResult)) {
|
|
27866
27885
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27871,8 +27890,38 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27871
27890
|
if (caseResult.evidence !== undefined) {
|
|
27872
27891
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
27873
27892
|
}
|
|
27893
|
+
if (caseResult.target !== undefined) {
|
|
27894
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
27895
|
+
}
|
|
27896
|
+
}
|
|
27897
|
+
}
|
|
27898
|
+
function assertCaseTargetEvidence(target) {
|
|
27899
|
+
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
27900
|
+
throw new Error("Invalid run manifest: malformed target evidence");
|
|
27901
|
+
}
|
|
27902
|
+
}
|
|
27903
|
+
function assertExecutionProvenance(provenance) {
|
|
27904
|
+
if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
|
|
27905
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27906
|
+
}
|
|
27907
|
+
const target = provenance.target;
|
|
27908
|
+
if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
|
|
27909
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27910
|
+
}
|
|
27911
|
+
if (provenance.evaluator !== undefined) {
|
|
27912
|
+
if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
27913
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27914
|
+
}
|
|
27874
27915
|
}
|
|
27875
27916
|
}
|
|
27917
|
+
function isBoundedStringList(value) {
|
|
27918
|
+
return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
|
|
27919
|
+
}
|
|
27920
|
+
function isGenerationConfig(value) {
|
|
27921
|
+
if (!isRecord2(value))
|
|
27922
|
+
return false;
|
|
27923
|
+
return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
|
|
27924
|
+
}
|
|
27876
27925
|
function assertWorkloadIdentity(identity) {
|
|
27877
27926
|
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
27878
27927
|
throw new Error("Invalid run manifest: malformed workload identity");
|
|
@@ -27921,6 +27970,7 @@ function createRunManifest(options) {
|
|
|
27921
27970
|
config,
|
|
27922
27971
|
resolvedConfig,
|
|
27923
27972
|
workloadIdentity,
|
|
27973
|
+
executionProvenance,
|
|
27924
27974
|
cases,
|
|
27925
27975
|
startTime,
|
|
27926
27976
|
endTime,
|
|
@@ -27933,7 +27983,7 @@ function createRunManifest(options) {
|
|
|
27933
27983
|
const git = getGitInfo();
|
|
27934
27984
|
const environment = getEnvironmentInfo();
|
|
27935
27985
|
return {
|
|
27936
|
-
version: "1.
|
|
27986
|
+
version: "1.3",
|
|
27937
27987
|
run_id: nanoid(12),
|
|
27938
27988
|
project,
|
|
27939
27989
|
start_time: startTime.toISOString(),
|
|
@@ -27942,6 +27992,7 @@ function createRunManifest(options) {
|
|
|
27942
27992
|
config,
|
|
27943
27993
|
resolved_config: resolvedConfig,
|
|
27944
27994
|
workload_identity: workloadIdentity,
|
|
27995
|
+
execution_provenance: executionProvenance,
|
|
27945
27996
|
metrics,
|
|
27946
27997
|
git,
|
|
27947
27998
|
provenance: {
|
|
@@ -28092,6 +28143,38 @@ function canonicalize(value, patterns, key) {
|
|
|
28092
28143
|
}
|
|
28093
28144
|
return value;
|
|
28094
28145
|
}
|
|
28146
|
+
function createExecutionProvenance(options) {
|
|
28147
|
+
const requestedModels = uniqueStrings([
|
|
28148
|
+
options.requestedModel,
|
|
28149
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model)
|
|
28150
|
+
]);
|
|
28151
|
+
const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
|
|
28152
|
+
const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
|
|
28153
|
+
const generation = omitUndefined({
|
|
28154
|
+
temperature: options.temperature,
|
|
28155
|
+
max_tokens: options.maxTokens,
|
|
28156
|
+
seed: options.seed
|
|
28157
|
+
});
|
|
28158
|
+
return {
|
|
28159
|
+
schema_version: "1",
|
|
28160
|
+
target: {
|
|
28161
|
+
provider: boundedString(options.provider, 100) ?? "unknown",
|
|
28162
|
+
...requestedModels.length ? { requested_models: requestedModels } : {},
|
|
28163
|
+
...observedModels.length ? { observed_models: observedModels } : {},
|
|
28164
|
+
...Object.keys(generation).length ? { generation } : {}
|
|
28165
|
+
},
|
|
28166
|
+
...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
|
|
28167
|
+
};
|
|
28168
|
+
}
|
|
28169
|
+
function uniqueStrings(values) {
|
|
28170
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
|
|
28171
|
+
}
|
|
28172
|
+
function boundedString(value, maxLength) {
|
|
28173
|
+
return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
28174
|
+
}
|
|
28175
|
+
function omitUndefined(value) {
|
|
28176
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
|
28177
|
+
}
|
|
28095
28178
|
async function runScenario(options) {
|
|
28096
28179
|
const {
|
|
28097
28180
|
scenario,
|
|
@@ -28124,6 +28207,7 @@ async function runScenario(options) {
|
|
|
28124
28207
|
const result = await executeCase(testCase, {
|
|
28125
28208
|
client,
|
|
28126
28209
|
scenario,
|
|
28210
|
+
requestedModel: resolvedConfig?.model,
|
|
28127
28211
|
timeout: testCase.timeout || timeout,
|
|
28128
28212
|
retries: testCase.retries ?? retries,
|
|
28129
28213
|
redaction,
|
|
@@ -28140,6 +28224,7 @@ async function runScenario(options) {
|
|
|
28140
28224
|
const result = await executeCase(testCase, {
|
|
28141
28225
|
client,
|
|
28142
28226
|
scenario,
|
|
28227
|
+
requestedModel: resolvedConfig?.model,
|
|
28143
28228
|
timeout: testCase.timeout || timeout,
|
|
28144
28229
|
retries: testCase.retries ?? retries,
|
|
28145
28230
|
redaction,
|
|
@@ -28184,6 +28269,14 @@ async function runScenario(options) {
|
|
|
28184
28269
|
},
|
|
28185
28270
|
resolvedConfig,
|
|
28186
28271
|
workloadIdentity: createWorkloadIdentity(scenario),
|
|
28272
|
+
executionProvenance: createExecutionProvenance({
|
|
28273
|
+
provider: client.provider,
|
|
28274
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
28275
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
28276
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
28277
|
+
seed: scenario.seed,
|
|
28278
|
+
cases: results
|
|
28279
|
+
}),
|
|
28187
28280
|
cases: results,
|
|
28188
28281
|
startTime,
|
|
28189
28282
|
endTime,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.4",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.5.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.1",
|
|
75
|
+
"@artemiskit/redteam": "0.3.5"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|