@artemiskit/sdk 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.js +182 -11
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.4.4
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies
|
|
8
|
+
- @artemiskit/core@0.5.1
|
|
9
|
+
- @artemiskit/redteam@0.3.5
|
|
10
|
+
|
|
11
|
+
## 0.4.3
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Updated dependencies [b826698]
|
|
16
|
+
- @artemiskit/core@0.5.0
|
|
17
|
+
- @artemiskit/redteam@0.3.4
|
|
18
|
+
|
|
3
19
|
## 0.4.2
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/dist/index.js
CHANGED
|
@@ -6924,6 +6924,7 @@ import { join, resolve } from "path";
|
|
|
6924
6924
|
import { createHash } from "crypto";
|
|
6925
6925
|
import { webcrypto as crypto2 } from "crypto";
|
|
6926
6926
|
import { execSync } from "child_process";
|
|
6927
|
+
import { createHash as createHash2 } from "crypto";
|
|
6927
6928
|
import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
|
|
6928
6929
|
import { join as join2, resolve as resolve2 } from "path";
|
|
6929
6930
|
import { formatWithOptions } from "util";
|
|
@@ -27129,6 +27130,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
27129
27130
|
async function executeCase(testCase, context) {
|
|
27130
27131
|
const { timeout, retries = 0 } = context;
|
|
27131
27132
|
const caseStartTime = Date.now();
|
|
27133
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
27132
27134
|
let lastError = null;
|
|
27133
27135
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
27134
27136
|
try {
|
|
@@ -27159,11 +27161,12 @@ async function executeCase(testCase, context) {
|
|
|
27159
27161
|
response: "",
|
|
27160
27162
|
expected: testCase.expected,
|
|
27161
27163
|
tags: testCase.tags,
|
|
27162
|
-
error: lastError?.message
|
|
27164
|
+
error: lastError?.message,
|
|
27165
|
+
target: targetEvidence(context.client.provider, requestedModel)
|
|
27163
27166
|
};
|
|
27164
27167
|
}
|
|
27165
27168
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
27166
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
27169
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
27167
27170
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
27168
27171
|
let prompt = substituteVariables(testCase.prompt, variables);
|
|
27169
27172
|
if (scenario.setup?.systemPrompt && typeof prompt === "string") {
|
|
@@ -27181,7 +27184,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27181
27184
|
const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
|
27182
27185
|
const generate = () => client.generate({
|
|
27183
27186
|
prompt: loopPrompt,
|
|
27184
|
-
model: testCase.model || scenario.model,
|
|
27187
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
27185
27188
|
temperature: scenario.temperature,
|
|
27186
27189
|
maxTokens: scenario.maxTokens,
|
|
27187
27190
|
seed: scenario.seed,
|
|
@@ -27189,6 +27192,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27189
27192
|
});
|
|
27190
27193
|
const generatePromise = generate();
|
|
27191
27194
|
let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
|
|
27195
|
+
const observedModels = [result.model];
|
|
27192
27196
|
const generationMetrics = {
|
|
27193
27197
|
latencyMs: result.latencyMs,
|
|
27194
27198
|
tokens: { ...result.tokens }
|
|
@@ -27199,7 +27203,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27199
27203
|
status: "error",
|
|
27200
27204
|
steps: 0,
|
|
27201
27205
|
terminationReason: "tool_error"
|
|
27202
|
-
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
|
|
27206
|
+
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27203
27207
|
}
|
|
27204
27208
|
const executor = toolExecutor ?? new FixtureToolExecutor({
|
|
27205
27209
|
tools,
|
|
@@ -27218,7 +27222,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27218
27222
|
status: "error",
|
|
27219
27223
|
steps: step,
|
|
27220
27224
|
terminationReason: "duplicate_call"
|
|
27221
|
-
}, generationMetrics, "TOOL_DUPLICATE_CALL");
|
|
27225
|
+
}, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27222
27226
|
}
|
|
27223
27227
|
seenCalls.add(fingerprint);
|
|
27224
27228
|
const toolStart = Date.now();
|
|
@@ -27245,7 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27245
27249
|
status: "error",
|
|
27246
27250
|
steps: step + 1,
|
|
27247
27251
|
terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
|
|
27248
|
-
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
|
|
27252
|
+
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27249
27253
|
}
|
|
27250
27254
|
const content = JSON.stringify(execution.result ?? {});
|
|
27251
27255
|
loopPrompt.push({
|
|
@@ -27257,7 +27261,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27257
27261
|
}
|
|
27258
27262
|
const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
|
|
27259
27263
|
if (remainingLoopTime <= 0) {
|
|
27260
|
-
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
|
|
27264
|
+
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27261
27265
|
}
|
|
27262
27266
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
27263
27267
|
try {
|
|
@@ -27268,8 +27272,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27268
27272
|
status: "error",
|
|
27269
27273
|
steps: step + 1,
|
|
27270
27274
|
terminationReason: timedOut ? "timeout" : "tool_error"
|
|
27271
|
-
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
|
|
27275
|
+
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27272
27276
|
}
|
|
27277
|
+
observedModels.push(result.model);
|
|
27273
27278
|
generationMetrics.latencyMs += result.latencyMs;
|
|
27274
27279
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
27275
27280
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -27280,7 +27285,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27280
27285
|
status: "error",
|
|
27281
27286
|
steps: policy.maxSteps,
|
|
27282
27287
|
terminationReason: "max_steps"
|
|
27283
|
-
}, generationMetrics, "TOOL_LOOP_MAX_STEPS");
|
|
27288
|
+
}, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
27284
27289
|
}
|
|
27285
27290
|
toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
|
|
27286
27291
|
}
|
|
@@ -27376,11 +27381,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27376
27381
|
tags: testCase.tags,
|
|
27377
27382
|
redaction: redactionInfo,
|
|
27378
27383
|
evidence: finalEvidence,
|
|
27384
|
+
target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
|
|
27379
27385
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
27380
27386
|
toolLoop
|
|
27381
27387
|
};
|
|
27382
27388
|
}
|
|
27383
|
-
function
|
|
27389
|
+
function targetEvidence(provider, requestedModel, observedModels) {
|
|
27390
|
+
const observed = [
|
|
27391
|
+
...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
|
|
27392
|
+
].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
|
|
27393
|
+
return {
|
|
27394
|
+
provider: sanitizeArtifactText(provider, 100) ?? "unknown",
|
|
27395
|
+
...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
|
|
27396
|
+
...observed.length ? { observed_models: observed } : {}
|
|
27397
|
+
};
|
|
27398
|
+
}
|
|
27399
|
+
function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
|
|
27384
27400
|
return new ToolLoopError(code, {
|
|
27385
27401
|
id: testCase.id,
|
|
27386
27402
|
name: testCase.name,
|
|
@@ -27396,6 +27412,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
27396
27412
|
expected: testCase.expected,
|
|
27397
27413
|
tags: testCase.tags,
|
|
27398
27414
|
error: code,
|
|
27415
|
+
target,
|
|
27399
27416
|
toolTrace,
|
|
27400
27417
|
toolLoop
|
|
27401
27418
|
});
|
|
@@ -27857,6 +27874,12 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27857
27874
|
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
27858
27875
|
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
27859
27876
|
}
|
|
27877
|
+
if (manifest.workload_identity !== undefined) {
|
|
27878
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
27879
|
+
}
|
|
27880
|
+
if (manifest.execution_provenance !== undefined) {
|
|
27881
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
27882
|
+
}
|
|
27860
27883
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27861
27884
|
if (!isRecord2(caseResult)) {
|
|
27862
27885
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27867,8 +27890,46 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27867
27890
|
if (caseResult.evidence !== undefined) {
|
|
27868
27891
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
27869
27892
|
}
|
|
27893
|
+
if (caseResult.target !== undefined) {
|
|
27894
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
27895
|
+
}
|
|
27870
27896
|
}
|
|
27871
27897
|
}
|
|
27898
|
+
function assertCaseTargetEvidence(target) {
|
|
27899
|
+
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
27900
|
+
throw new Error("Invalid run manifest: malformed target evidence");
|
|
27901
|
+
}
|
|
27902
|
+
}
|
|
27903
|
+
function assertExecutionProvenance(provenance) {
|
|
27904
|
+
if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
|
|
27905
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27906
|
+
}
|
|
27907
|
+
const target = provenance.target;
|
|
27908
|
+
if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
|
|
27909
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27910
|
+
}
|
|
27911
|
+
if (provenance.evaluator !== undefined) {
|
|
27912
|
+
if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
27913
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
27914
|
+
}
|
|
27915
|
+
}
|
|
27916
|
+
}
|
|
27917
|
+
function isBoundedStringList(value) {
|
|
27918
|
+
return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
|
|
27919
|
+
}
|
|
27920
|
+
function isGenerationConfig(value) {
|
|
27921
|
+
if (!isRecord2(value))
|
|
27922
|
+
return false;
|
|
27923
|
+
return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
|
|
27924
|
+
}
|
|
27925
|
+
function assertWorkloadIdentity(identity) {
|
|
27926
|
+
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
27927
|
+
throw new Error("Invalid run manifest: malformed workload identity");
|
|
27928
|
+
}
|
|
27929
|
+
}
|
|
27930
|
+
function isContentIdentity(value) {
|
|
27931
|
+
return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
|
|
27932
|
+
}
|
|
27872
27933
|
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
27873
27934
|
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
27874
27935
|
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
@@ -27908,6 +27969,8 @@ function createRunManifest(options) {
|
|
|
27908
27969
|
project,
|
|
27909
27970
|
config,
|
|
27910
27971
|
resolvedConfig,
|
|
27972
|
+
workloadIdentity,
|
|
27973
|
+
executionProvenance,
|
|
27911
27974
|
cases,
|
|
27912
27975
|
startTime,
|
|
27913
27976
|
endTime,
|
|
@@ -27920,7 +27983,7 @@ function createRunManifest(options) {
|
|
|
27920
27983
|
const git = getGitInfo();
|
|
27921
27984
|
const environment = getEnvironmentInfo();
|
|
27922
27985
|
return {
|
|
27923
|
-
version: "1.
|
|
27986
|
+
version: "1.3",
|
|
27924
27987
|
run_id: nanoid(12),
|
|
27925
27988
|
project,
|
|
27926
27989
|
start_time: startTime.toISOString(),
|
|
@@ -27928,6 +27991,8 @@ function createRunManifest(options) {
|
|
|
27928
27991
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
27929
27992
|
config,
|
|
27930
27993
|
resolved_config: resolvedConfig,
|
|
27994
|
+
workload_identity: workloadIdentity,
|
|
27995
|
+
execution_provenance: executionProvenance,
|
|
27931
27996
|
metrics,
|
|
27932
27997
|
git,
|
|
27933
27998
|
provenance: {
|
|
@@ -28015,6 +28080,101 @@ function detectCIEnvironment() {
|
|
|
28015
28080
|
}
|
|
28016
28081
|
return;
|
|
28017
28082
|
}
|
|
28083
|
+
var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
28084
|
+
var REDACTED_SECRET = "[REDACTED_SECRET]";
|
|
28085
|
+
function createWorkloadIdentity(scenario) {
|
|
28086
|
+
const patterns = resolvePatterns([
|
|
28087
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
28088
|
+
...scenario.redaction?.patterns ?? []
|
|
28089
|
+
]).map(({ regex }) => regex);
|
|
28090
|
+
const workload = {
|
|
28091
|
+
name: scenario.name,
|
|
28092
|
+
version: scenario.version,
|
|
28093
|
+
description: scenario.description,
|
|
28094
|
+
tags: scenario.tags,
|
|
28095
|
+
variables: scenario.variables,
|
|
28096
|
+
setup: scenario.setup,
|
|
28097
|
+
cases: scenario.cases.map((testCase) => ({
|
|
28098
|
+
id: testCase.id,
|
|
28099
|
+
name: testCase.name,
|
|
28100
|
+
description: testCase.description,
|
|
28101
|
+
prompt: testCase.prompt,
|
|
28102
|
+
tags: testCase.tags,
|
|
28103
|
+
metadata: testCase.metadata,
|
|
28104
|
+
timeout: testCase.timeout,
|
|
28105
|
+
retries: testCase.retries,
|
|
28106
|
+
variables: testCase.variables
|
|
28107
|
+
}))
|
|
28108
|
+
};
|
|
28109
|
+
const rubric = {
|
|
28110
|
+
cases: scenario.cases.map((testCase) => ({
|
|
28111
|
+
id: testCase.id,
|
|
28112
|
+
expected: testCase.expected
|
|
28113
|
+
}))
|
|
28114
|
+
};
|
|
28115
|
+
return {
|
|
28116
|
+
schema_version: "1",
|
|
28117
|
+
workload: createContentIdentity(workload, patterns),
|
|
28118
|
+
rubric: createContentIdentity(rubric, patterns)
|
|
28119
|
+
};
|
|
28120
|
+
}
|
|
28121
|
+
function createContentIdentity(value, patterns) {
|
|
28122
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
28123
|
+
return {
|
|
28124
|
+
schema_version: "1",
|
|
28125
|
+
algorithm: "sha256",
|
|
28126
|
+
digest: createHash2("sha256").update(canonical).digest("hex")
|
|
28127
|
+
};
|
|
28128
|
+
}
|
|
28129
|
+
function canonicalize(value, patterns, key) {
|
|
28130
|
+
if (key && SENSITIVE_KEY.test(key))
|
|
28131
|
+
return REDACTED_SECRET;
|
|
28132
|
+
if (typeof value === "string") {
|
|
28133
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
28134
|
+
}
|
|
28135
|
+
if (Array.isArray(value)) {
|
|
28136
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
28137
|
+
}
|
|
28138
|
+
if (value && typeof value === "object") {
|
|
28139
|
+
return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
|
|
28140
|
+
nestedKey,
|
|
28141
|
+
canonicalize(nestedValue, patterns, nestedKey)
|
|
28142
|
+
]));
|
|
28143
|
+
}
|
|
28144
|
+
return value;
|
|
28145
|
+
}
|
|
28146
|
+
function createExecutionProvenance(options) {
|
|
28147
|
+
const requestedModels = uniqueStrings([
|
|
28148
|
+
options.requestedModel,
|
|
28149
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model)
|
|
28150
|
+
]);
|
|
28151
|
+
const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
|
|
28152
|
+
const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
|
|
28153
|
+
const generation = omitUndefined({
|
|
28154
|
+
temperature: options.temperature,
|
|
28155
|
+
max_tokens: options.maxTokens,
|
|
28156
|
+
seed: options.seed
|
|
28157
|
+
});
|
|
28158
|
+
return {
|
|
28159
|
+
schema_version: "1",
|
|
28160
|
+
target: {
|
|
28161
|
+
provider: boundedString(options.provider, 100) ?? "unknown",
|
|
28162
|
+
...requestedModels.length ? { requested_models: requestedModels } : {},
|
|
28163
|
+
...observedModels.length ? { observed_models: observedModels } : {},
|
|
28164
|
+
...Object.keys(generation).length ? { generation } : {}
|
|
28165
|
+
},
|
|
28166
|
+
...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
|
|
28167
|
+
};
|
|
28168
|
+
}
|
|
28169
|
+
function uniqueStrings(values) {
|
|
28170
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
|
|
28171
|
+
}
|
|
28172
|
+
function boundedString(value, maxLength) {
|
|
28173
|
+
return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
28174
|
+
}
|
|
28175
|
+
function omitUndefined(value) {
|
|
28176
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
|
28177
|
+
}
|
|
28018
28178
|
async function runScenario(options) {
|
|
28019
28179
|
const {
|
|
28020
28180
|
scenario,
|
|
@@ -28047,6 +28207,7 @@ async function runScenario(options) {
|
|
|
28047
28207
|
const result = await executeCase(testCase, {
|
|
28048
28208
|
client,
|
|
28049
28209
|
scenario,
|
|
28210
|
+
requestedModel: resolvedConfig?.model,
|
|
28050
28211
|
timeout: testCase.timeout || timeout,
|
|
28051
28212
|
retries: testCase.retries ?? retries,
|
|
28052
28213
|
redaction,
|
|
@@ -28063,6 +28224,7 @@ async function runScenario(options) {
|
|
|
28063
28224
|
const result = await executeCase(testCase, {
|
|
28064
28225
|
client,
|
|
28065
28226
|
scenario,
|
|
28227
|
+
requestedModel: resolvedConfig?.model,
|
|
28066
28228
|
timeout: testCase.timeout || timeout,
|
|
28067
28229
|
retries: testCase.retries ?? retries,
|
|
28068
28230
|
redaction,
|
|
@@ -28106,6 +28268,15 @@ async function runScenario(options) {
|
|
|
28106
28268
|
seed: scenario.seed
|
|
28107
28269
|
},
|
|
28108
28270
|
resolvedConfig,
|
|
28271
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
28272
|
+
executionProvenance: createExecutionProvenance({
|
|
28273
|
+
provider: client.provider,
|
|
28274
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
28275
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
28276
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
28277
|
+
seed: scenario.seed,
|
|
28278
|
+
cases: results
|
|
28279
|
+
}),
|
|
28109
28280
|
cases: results,
|
|
28110
28281
|
startTime,
|
|
28111
28282
|
endTime,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.4",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.1",
|
|
75
|
+
"@artemiskit/redteam": "0.3.5"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|