@artemiskit/sdk 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.4
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @artemiskit/core@0.5.1
9
+ - @artemiskit/redteam@0.3.5
10
+
3
11
  ## 0.4.3
4
12
 
5
13
  ### Patch Changes
package/dist/index.js CHANGED
@@ -27130,6 +27130,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
27130
27130
  async function executeCase(testCase, context) {
27131
27131
  const { timeout, retries = 0 } = context;
27132
27132
  const caseStartTime = Date.now();
27133
+ const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
27133
27134
  let lastError = null;
27134
27135
  for (let attempt = 0;attempt <= retries; attempt++) {
27135
27136
  try {
@@ -27160,11 +27161,12 @@ async function executeCase(testCase, context) {
27160
27161
  response: "",
27161
27162
  expected: testCase.expected,
27162
27163
  tags: testCase.tags,
27163
- error: lastError?.message
27164
+ error: lastError?.message,
27165
+ target: targetEvidence(context.client.provider, requestedModel)
27164
27166
  };
27165
27167
  }
27166
27168
  async function executeCaseAttempt(testCase, context, timeout) {
27167
- const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
27169
+ const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
27168
27170
  const variables = mergeVariables(scenario.variables, testCase.variables);
27169
27171
  let prompt = substituteVariables(testCase.prompt, variables);
27170
27172
  if (scenario.setup?.systemPrompt && typeof prompt === "string") {
@@ -27182,7 +27184,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27182
27184
  const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
27183
27185
  const generate = () => client.generate({
27184
27186
  prompt: loopPrompt,
27185
- model: testCase.model || scenario.model,
27187
+ model: testCase.model || requestedModel || scenario.model,
27186
27188
  temperature: scenario.temperature,
27187
27189
  maxTokens: scenario.maxTokens,
27188
27190
  seed: scenario.seed,
@@ -27190,6 +27192,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27190
27192
  });
27191
27193
  const generatePromise = generate();
27192
27194
  let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
27195
+ const observedModels = [result.model];
27193
27196
  const generationMetrics = {
27194
27197
  latencyMs: result.latencyMs,
27195
27198
  tokens: { ...result.tokens }
@@ -27200,7 +27203,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27200
27203
  status: "error",
27201
27204
  steps: 0,
27202
27205
  terminationReason: "tool_error"
27203
- }, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
27206
+ }, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27204
27207
  }
27205
27208
  const executor = toolExecutor ?? new FixtureToolExecutor({
27206
27209
  tools,
@@ -27219,7 +27222,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27219
27222
  status: "error",
27220
27223
  steps: step,
27221
27224
  terminationReason: "duplicate_call"
27222
- }, generationMetrics, "TOOL_DUPLICATE_CALL");
27225
+ }, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27223
27226
  }
27224
27227
  seenCalls.add(fingerprint);
27225
27228
  const toolStart = Date.now();
@@ -27246,7 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27246
27249
  status: "error",
27247
27250
  steps: step + 1,
27248
27251
  terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
27249
- }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
27252
+ }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27250
27253
  }
27251
27254
  const content = JSON.stringify(execution.result ?? {});
27252
27255
  loopPrompt.push({
@@ -27258,7 +27261,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27258
27261
  }
27259
27262
  const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
27260
27263
  if (remainingLoopTime <= 0) {
27261
- throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
27264
+ throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27262
27265
  }
27263
27266
  const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
27264
27267
  try {
@@ -27269,8 +27272,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
27269
27272
  status: "error",
27270
27273
  steps: step + 1,
27271
27274
  terminationReason: timedOut ? "timeout" : "tool_error"
27272
- }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
27275
+ }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27273
27276
  }
27277
+ observedModels.push(result.model);
27274
27278
  generationMetrics.latencyMs += result.latencyMs;
27275
27279
  generationMetrics.tokens.prompt += result.tokens.prompt;
27276
27280
  generationMetrics.tokens.completion += result.tokens.completion;
@@ -27281,7 +27285,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27281
27285
  status: "error",
27282
27286
  steps: policy.maxSteps,
27283
27287
  terminationReason: "max_steps"
27284
- }, generationMetrics, "TOOL_LOOP_MAX_STEPS");
27288
+ }, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27285
27289
  }
27286
27290
  toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
27287
27291
  }
@@ -27377,11 +27381,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
27377
27381
  tags: testCase.tags,
27378
27382
  redaction: redactionInfo,
27379
27383
  evidence: finalEvidence,
27384
+ target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
27380
27385
  toolTrace: toolTrace.length ? toolTrace : undefined,
27381
27386
  toolLoop
27382
27387
  };
27383
27388
  }
27384
- function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code) {
27389
+ function targetEvidence(provider, requestedModel, observedModels) {
27390
+ const observed = [
27391
+ ...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
27392
+ ].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
27393
+ return {
27394
+ provider: sanitizeArtifactText(provider, 100) ?? "unknown",
27395
+ ...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
27396
+ ...observed.length ? { observed_models: observed } : {}
27397
+ };
27398
+ }
27399
+ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
27385
27400
  return new ToolLoopError(code, {
27386
27401
  id: testCase.id,
27387
27402
  name: testCase.name,
@@ -27397,6 +27412,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
27397
27412
  expected: testCase.expected,
27398
27413
  tags: testCase.tags,
27399
27414
  error: code,
27415
+ target,
27400
27416
  toolTrace,
27401
27417
  toolLoop
27402
27418
  });
@@ -27861,6 +27877,9 @@ function assertRunManifestIntegrity(manifest) {
27861
27877
  if (manifest.workload_identity !== undefined) {
27862
27878
  assertWorkloadIdentity(manifest.workload_identity);
27863
27879
  }
27880
+ if (manifest.execution_provenance !== undefined) {
27881
+ assertExecutionProvenance(manifest.execution_provenance);
27882
+ }
27864
27883
  for (const [index, caseResult] of manifest.cases.entries()) {
27865
27884
  if (!isRecord2(caseResult)) {
27866
27885
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27871,8 +27890,38 @@ function assertRunManifestIntegrity(manifest) {
27871
27890
  if (caseResult.evidence !== undefined) {
27872
27891
  assertCaseEvaluationEvidence(caseResult.evidence, index);
27873
27892
  }
27893
+ if (caseResult.target !== undefined) {
27894
+ assertCaseTargetEvidence(caseResult.target);
27895
+ }
27896
+ }
27897
+ }
27898
+ function assertCaseTargetEvidence(target) {
27899
+ if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
27900
+ throw new Error("Invalid run manifest: malformed target evidence");
27901
+ }
27902
+ }
27903
+ function assertExecutionProvenance(provenance) {
27904
+ if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
27905
+ throw new Error("Invalid run manifest: malformed execution provenance");
27906
+ }
27907
+ const target = provenance.target;
27908
+ if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
27909
+ throw new Error("Invalid run manifest: malformed execution provenance");
27910
+ }
27911
+ if (provenance.evaluator !== undefined) {
27912
+ if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
27913
+ throw new Error("Invalid run manifest: malformed execution provenance");
27914
+ }
27874
27915
  }
27875
27916
  }
27917
+ function isBoundedStringList(value) {
27918
+ return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
27919
+ }
27920
+ function isGenerationConfig(value) {
27921
+ if (!isRecord2(value))
27922
+ return false;
27923
+ return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
27924
+ }
27876
27925
  function assertWorkloadIdentity(identity) {
27877
27926
  if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
27878
27927
  throw new Error("Invalid run manifest: malformed workload identity");
@@ -27921,6 +27970,7 @@ function createRunManifest(options) {
27921
27970
  config,
27922
27971
  resolvedConfig,
27923
27972
  workloadIdentity,
27973
+ executionProvenance,
27924
27974
  cases,
27925
27975
  startTime,
27926
27976
  endTime,
@@ -27933,7 +27983,7 @@ function createRunManifest(options) {
27933
27983
  const git = getGitInfo();
27934
27984
  const environment = getEnvironmentInfo();
27935
27985
  return {
27936
- version: "1.2",
27986
+ version: "1.3",
27937
27987
  run_id: nanoid(12),
27938
27988
  project,
27939
27989
  start_time: startTime.toISOString(),
@@ -27942,6 +27992,7 @@ function createRunManifest(options) {
27942
27992
  config,
27943
27993
  resolved_config: resolvedConfig,
27944
27994
  workload_identity: workloadIdentity,
27995
+ execution_provenance: executionProvenance,
27945
27996
  metrics,
27946
27997
  git,
27947
27998
  provenance: {
@@ -28092,6 +28143,38 @@ function canonicalize(value, patterns, key) {
28092
28143
  }
28093
28144
  return value;
28094
28145
  }
28146
+ function createExecutionProvenance(options) {
28147
+ const requestedModels = uniqueStrings([
28148
+ options.requestedModel,
28149
+ ...options.cases.map((caseResult) => caseResult.target?.requested_model)
28150
+ ]);
28151
+ const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
28152
+ const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
28153
+ const generation = omitUndefined({
28154
+ temperature: options.temperature,
28155
+ max_tokens: options.maxTokens,
28156
+ seed: options.seed
28157
+ });
28158
+ return {
28159
+ schema_version: "1",
28160
+ target: {
28161
+ provider: boundedString(options.provider, 100) ?? "unknown",
28162
+ ...requestedModels.length ? { requested_models: requestedModels } : {},
28163
+ ...observedModels.length ? { observed_models: observedModels } : {},
28164
+ ...Object.keys(generation).length ? { generation } : {}
28165
+ },
28166
+ ...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
28167
+ };
28168
+ }
28169
+ function uniqueStrings(values) {
28170
+ return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
28171
+ }
28172
+ function boundedString(value, maxLength) {
28173
+ return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
28174
+ }
28175
+ function omitUndefined(value) {
28176
+ return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
28177
+ }
28095
28178
  async function runScenario(options) {
28096
28179
  const {
28097
28180
  scenario,
@@ -28124,6 +28207,7 @@ async function runScenario(options) {
28124
28207
  const result = await executeCase(testCase, {
28125
28208
  client,
28126
28209
  scenario,
28210
+ requestedModel: resolvedConfig?.model,
28127
28211
  timeout: testCase.timeout || timeout,
28128
28212
  retries: testCase.retries ?? retries,
28129
28213
  redaction,
@@ -28140,6 +28224,7 @@ async function runScenario(options) {
28140
28224
  const result = await executeCase(testCase, {
28141
28225
  client,
28142
28226
  scenario,
28227
+ requestedModel: resolvedConfig?.model,
28143
28228
  timeout: testCase.timeout || timeout,
28144
28229
  retries: testCase.retries ?? retries,
28145
28230
  redaction,
@@ -28184,6 +28269,14 @@ async function runScenario(options) {
28184
28269
  },
28185
28270
  resolvedConfig,
28186
28271
  workloadIdentity: createWorkloadIdentity(scenario),
28272
+ executionProvenance: createExecutionProvenance({
28273
+ provider: client.provider,
28274
+ requestedModel: resolvedConfig?.model || scenario.model,
28275
+ temperature: resolvedConfig?.temperature ?? scenario.temperature,
28276
+ maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
28277
+ seed: scenario.seed,
28278
+ cases: results
28279
+ }),
28187
28280
  cases: results,
28188
28281
  startTime,
28189
28282
  endTime,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.3",
3
+ "version": "0.4.4",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.5.0",
75
- "@artemiskit/redteam": "0.3.4"
74
+ "@artemiskit/core": "0.5.1",
75
+ "@artemiskit/redteam": "0.3.5"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",