@artemiskit/sdk 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/index.js +182 -11
  3. package/package.json +3 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.4
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @artemiskit/core@0.5.1
9
+ - @artemiskit/redteam@0.3.5
10
+
11
+ ## 0.4.3
12
+
13
+ ### Patch Changes
14
+
15
+ - Updated dependencies [b826698]
16
+ - @artemiskit/core@0.5.0
17
+ - @artemiskit/redteam@0.3.4
18
+
3
19
  ## 0.4.2
4
20
 
5
21
  ### Patch Changes
package/dist/index.js CHANGED
@@ -6924,6 +6924,7 @@ import { join, resolve } from "path";
6924
6924
  import { createHash } from "crypto";
6925
6925
  import { webcrypto as crypto2 } from "crypto";
6926
6926
  import { execSync } from "child_process";
6927
+ import { createHash as createHash2 } from "crypto";
6927
6928
  import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
6928
6929
  import { join as join2, resolve as resolve2 } from "path";
6929
6930
  import { formatWithOptions } from "util";
@@ -27129,6 +27130,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
27129
27130
  async function executeCase(testCase, context) {
27130
27131
  const { timeout, retries = 0 } = context;
27131
27132
  const caseStartTime = Date.now();
27133
+ const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
27132
27134
  let lastError = null;
27133
27135
  for (let attempt = 0;attempt <= retries; attempt++) {
27134
27136
  try {
@@ -27159,11 +27161,12 @@ async function executeCase(testCase, context) {
27159
27161
  response: "",
27160
27162
  expected: testCase.expected,
27161
27163
  tags: testCase.tags,
27162
- error: lastError?.message
27164
+ error: lastError?.message,
27165
+ target: targetEvidence(context.client.provider, requestedModel)
27163
27166
  };
27164
27167
  }
27165
27168
  async function executeCaseAttempt(testCase, context, timeout) {
27166
- const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
27169
+ const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
27167
27170
  const variables = mergeVariables(scenario.variables, testCase.variables);
27168
27171
  let prompt = substituteVariables(testCase.prompt, variables);
27169
27172
  if (scenario.setup?.systemPrompt && typeof prompt === "string") {
@@ -27181,7 +27184,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27181
27184
  const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
27182
27185
  const generate = () => client.generate({
27183
27186
  prompt: loopPrompt,
27184
- model: testCase.model || scenario.model,
27187
+ model: testCase.model || requestedModel || scenario.model,
27185
27188
  temperature: scenario.temperature,
27186
27189
  maxTokens: scenario.maxTokens,
27187
27190
  seed: scenario.seed,
@@ -27189,6 +27192,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27189
27192
  });
27190
27193
  const generatePromise = generate();
27191
27194
  let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
27195
+ const observedModels = [result.model];
27192
27196
  const generationMetrics = {
27193
27197
  latencyMs: result.latencyMs,
27194
27198
  tokens: { ...result.tokens }
@@ -27199,7 +27203,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27199
27203
  status: "error",
27200
27204
  steps: 0,
27201
27205
  terminationReason: "tool_error"
27202
- }, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
27206
+ }, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27203
27207
  }
27204
27208
  const executor = toolExecutor ?? new FixtureToolExecutor({
27205
27209
  tools,
@@ -27218,7 +27222,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27218
27222
  status: "error",
27219
27223
  steps: step,
27220
27224
  terminationReason: "duplicate_call"
27221
- }, generationMetrics, "TOOL_DUPLICATE_CALL");
27225
+ }, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27222
27226
  }
27223
27227
  seenCalls.add(fingerprint);
27224
27228
  const toolStart = Date.now();
@@ -27245,7 +27249,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27245
27249
  status: "error",
27246
27250
  steps: step + 1,
27247
27251
  terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
27248
- }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
27252
+ }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27249
27253
  }
27250
27254
  const content = JSON.stringify(execution.result ?? {});
27251
27255
  loopPrompt.push({
@@ -27257,7 +27261,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27257
27261
  }
27258
27262
  const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
27259
27263
  if (remainingLoopTime <= 0) {
27260
- throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
27264
+ throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27261
27265
  }
27262
27266
  const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
27263
27267
  try {
@@ -27268,8 +27272,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
27268
27272
  status: "error",
27269
27273
  steps: step + 1,
27270
27274
  terminationReason: timedOut ? "timeout" : "tool_error"
27271
- }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
27275
+ }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27272
27276
  }
27277
+ observedModels.push(result.model);
27273
27278
  generationMetrics.latencyMs += result.latencyMs;
27274
27279
  generationMetrics.tokens.prompt += result.tokens.prompt;
27275
27280
  generationMetrics.tokens.completion += result.tokens.completion;
@@ -27280,7 +27285,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27280
27285
  status: "error",
27281
27286
  steps: policy.maxSteps,
27282
27287
  terminationReason: "max_steps"
27283
- }, generationMetrics, "TOOL_LOOP_MAX_STEPS");
27288
+ }, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
27284
27289
  }
27285
27290
  toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
27286
27291
  }
@@ -27376,11 +27381,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
27376
27381
  tags: testCase.tags,
27377
27382
  redaction: redactionInfo,
27378
27383
  evidence: finalEvidence,
27384
+ target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
27379
27385
  toolTrace: toolTrace.length ? toolTrace : undefined,
27380
27386
  toolLoop
27381
27387
  };
27382
27388
  }
27383
- function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code) {
27389
+ function targetEvidence(provider, requestedModel, observedModels) {
27390
+ const observed = [
27391
+ ...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
27392
+ ].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
27393
+ return {
27394
+ provider: sanitizeArtifactText(provider, 100) ?? "unknown",
27395
+ ...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
27396
+ ...observed.length ? { observed_models: observed } : {}
27397
+ };
27398
+ }
27399
+ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
27384
27400
  return new ToolLoopError(code, {
27385
27401
  id: testCase.id,
27386
27402
  name: testCase.name,
@@ -27396,6 +27412,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
27396
27412
  expected: testCase.expected,
27397
27413
  tags: testCase.tags,
27398
27414
  error: code,
27415
+ target,
27399
27416
  toolTrace,
27400
27417
  toolLoop
27401
27418
  });
@@ -27857,6 +27874,12 @@ function assertRunManifestIntegrity(manifest) {
27857
27874
  if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
27858
27875
  throw new Error("Invalid run manifest: expected an object with a cases array");
27859
27876
  }
27877
+ if (manifest.workload_identity !== undefined) {
27878
+ assertWorkloadIdentity(manifest.workload_identity);
27879
+ }
27880
+ if (manifest.execution_provenance !== undefined) {
27881
+ assertExecutionProvenance(manifest.execution_provenance);
27882
+ }
27860
27883
  for (const [index, caseResult] of manifest.cases.entries()) {
27861
27884
  if (!isRecord2(caseResult)) {
27862
27885
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27867,8 +27890,46 @@ function assertRunManifestIntegrity(manifest) {
27867
27890
  if (caseResult.evidence !== undefined) {
27868
27891
  assertCaseEvaluationEvidence(caseResult.evidence, index);
27869
27892
  }
27893
+ if (caseResult.target !== undefined) {
27894
+ assertCaseTargetEvidence(caseResult.target);
27895
+ }
27870
27896
  }
27871
27897
  }
27898
+ function assertCaseTargetEvidence(target) {
27899
+ if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
27900
+ throw new Error("Invalid run manifest: malformed target evidence");
27901
+ }
27902
+ }
27903
+ function assertExecutionProvenance(provenance) {
27904
+ if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
27905
+ throw new Error("Invalid run manifest: malformed execution provenance");
27906
+ }
27907
+ const target = provenance.target;
27908
+ if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
27909
+ throw new Error("Invalid run manifest: malformed execution provenance");
27910
+ }
27911
+ if (provenance.evaluator !== undefined) {
27912
+ if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
27913
+ throw new Error("Invalid run manifest: malformed execution provenance");
27914
+ }
27915
+ }
27916
+ }
27917
+ function isBoundedStringList(value) {
27918
+ return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
27919
+ }
27920
+ function isGenerationConfig(value) {
27921
+ if (!isRecord2(value))
27922
+ return false;
27923
+ return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
27924
+ }
27925
+ function assertWorkloadIdentity(identity) {
27926
+ if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
27927
+ throw new Error("Invalid run manifest: malformed workload identity");
27928
+ }
27929
+ }
27930
+ function isContentIdentity(value) {
27931
+ return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
27932
+ }
27872
27933
  function assertCaseEvaluationEvidence(evidence, caseIndex) {
27873
27934
  if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
27874
27935
  throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
@@ -27908,6 +27969,8 @@ function createRunManifest(options) {
27908
27969
  project,
27909
27970
  config,
27910
27971
  resolvedConfig,
27972
+ workloadIdentity,
27973
+ executionProvenance,
27911
27974
  cases,
27912
27975
  startTime,
27913
27976
  endTime,
@@ -27920,7 +27983,7 @@ function createRunManifest(options) {
27920
27983
  const git = getGitInfo();
27921
27984
  const environment = getEnvironmentInfo();
27922
27985
  return {
27923
- version: "1.1",
27986
+ version: "1.3",
27924
27987
  run_id: nanoid(12),
27925
27988
  project,
27926
27989
  start_time: startTime.toISOString(),
@@ -27928,6 +27991,8 @@ function createRunManifest(options) {
27928
27991
  duration_ms: endTime.getTime() - startTime.getTime(),
27929
27992
  config,
27930
27993
  resolved_config: resolvedConfig,
27994
+ workload_identity: workloadIdentity,
27995
+ execution_provenance: executionProvenance,
27931
27996
  metrics,
27932
27997
  git,
27933
27998
  provenance: {
@@ -28015,6 +28080,101 @@ function detectCIEnvironment() {
28015
28080
  }
28016
28081
  return;
28017
28082
  }
28083
+ var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
28084
+ var REDACTED_SECRET = "[REDACTED_SECRET]";
28085
+ function createWorkloadIdentity(scenario) {
28086
+ const patterns = resolvePatterns([
28087
+ ...DEFAULT_REDACTION_PATTERNS,
28088
+ ...scenario.redaction?.patterns ?? []
28089
+ ]).map(({ regex }) => regex);
28090
+ const workload = {
28091
+ name: scenario.name,
28092
+ version: scenario.version,
28093
+ description: scenario.description,
28094
+ tags: scenario.tags,
28095
+ variables: scenario.variables,
28096
+ setup: scenario.setup,
28097
+ cases: scenario.cases.map((testCase) => ({
28098
+ id: testCase.id,
28099
+ name: testCase.name,
28100
+ description: testCase.description,
28101
+ prompt: testCase.prompt,
28102
+ tags: testCase.tags,
28103
+ metadata: testCase.metadata,
28104
+ timeout: testCase.timeout,
28105
+ retries: testCase.retries,
28106
+ variables: testCase.variables
28107
+ }))
28108
+ };
28109
+ const rubric = {
28110
+ cases: scenario.cases.map((testCase) => ({
28111
+ id: testCase.id,
28112
+ expected: testCase.expected
28113
+ }))
28114
+ };
28115
+ return {
28116
+ schema_version: "1",
28117
+ workload: createContentIdentity(workload, patterns),
28118
+ rubric: createContentIdentity(rubric, patterns)
28119
+ };
28120
+ }
28121
+ function createContentIdentity(value, patterns) {
28122
+ const canonical = JSON.stringify(canonicalize(value, patterns));
28123
+ return {
28124
+ schema_version: "1",
28125
+ algorithm: "sha256",
28126
+ digest: createHash2("sha256").update(canonical).digest("hex")
28127
+ };
28128
+ }
28129
+ function canonicalize(value, patterns, key) {
28130
+ if (key && SENSITIVE_KEY.test(key))
28131
+ return REDACTED_SECRET;
28132
+ if (typeof value === "string") {
28133
+ return redactText(value, patterns, REDACTED_SECRET).text;
28134
+ }
28135
+ if (Array.isArray(value)) {
28136
+ return value.map((item) => canonicalize(item, patterns));
28137
+ }
28138
+ if (value && typeof value === "object") {
28139
+ return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
28140
+ nestedKey,
28141
+ canonicalize(nestedValue, patterns, nestedKey)
28142
+ ]));
28143
+ }
28144
+ return value;
28145
+ }
28146
+ function createExecutionProvenance(options) {
28147
+ const requestedModels = uniqueStrings([
28148
+ options.requestedModel,
28149
+ ...options.cases.map((caseResult) => caseResult.target?.requested_model)
28150
+ ]);
28151
+ const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
28152
+ const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
28153
+ const generation = omitUndefined({
28154
+ temperature: options.temperature,
28155
+ max_tokens: options.maxTokens,
28156
+ seed: options.seed
28157
+ });
28158
+ return {
28159
+ schema_version: "1",
28160
+ target: {
28161
+ provider: boundedString(options.provider, 100) ?? "unknown",
28162
+ ...requestedModels.length ? { requested_models: requestedModels } : {},
28163
+ ...observedModels.length ? { observed_models: observedModels } : {},
28164
+ ...Object.keys(generation).length ? { generation } : {}
28165
+ },
28166
+ ...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
28167
+ };
28168
+ }
28169
+ function uniqueStrings(values) {
28170
+ return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
28171
+ }
28172
+ function boundedString(value, maxLength) {
28173
+ return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
28174
+ }
28175
+ function omitUndefined(value) {
28176
+ return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
28177
+ }
28018
28178
  async function runScenario(options) {
28019
28179
  const {
28020
28180
  scenario,
@@ -28047,6 +28207,7 @@ async function runScenario(options) {
28047
28207
  const result = await executeCase(testCase, {
28048
28208
  client,
28049
28209
  scenario,
28210
+ requestedModel: resolvedConfig?.model,
28050
28211
  timeout: testCase.timeout || timeout,
28051
28212
  retries: testCase.retries ?? retries,
28052
28213
  redaction,
@@ -28063,6 +28224,7 @@ async function runScenario(options) {
28063
28224
  const result = await executeCase(testCase, {
28064
28225
  client,
28065
28226
  scenario,
28227
+ requestedModel: resolvedConfig?.model,
28066
28228
  timeout: testCase.timeout || timeout,
28067
28229
  retries: testCase.retries ?? retries,
28068
28230
  redaction,
@@ -28106,6 +28268,15 @@ async function runScenario(options) {
28106
28268
  seed: scenario.seed
28107
28269
  },
28108
28270
  resolvedConfig,
28271
+ workloadIdentity: createWorkloadIdentity(scenario),
28272
+ executionProvenance: createExecutionProvenance({
28273
+ provider: client.provider,
28274
+ requestedModel: resolvedConfig?.model || scenario.model,
28275
+ temperature: resolvedConfig?.temperature ?? scenario.temperature,
28276
+ maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
28277
+ seed: scenario.seed,
28278
+ cases: results
28279
+ }),
28109
28280
  cases: results,
28110
28281
  startTime,
28111
28282
  endTime,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.2",
3
+ "version": "0.4.4",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.4.2",
75
- "@artemiskit/redteam": "0.3.3"
74
+ "@artemiskit/core": "0.5.1",
75
+ "@artemiskit/redteam": "0.3.5"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",