@artemiskit/sdk 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/index.js +121 -16
  3. package/package.json +3 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.2
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies [8716ca6]
8
+ - @artemiskit/core@0.4.2
9
+ - @artemiskit/redteam@0.3.3
10
+
11
+ ## 0.4.1
12
+
13
+ ### Patch Changes
14
+
15
+ - Updated dependencies
16
+ - @artemiskit/core@0.4.1
17
+ - @artemiskit/redteam@0.3.2
18
+
3
19
  ## 0.4.0
4
20
 
5
21
  ### Minor Changes
package/dist/index.js CHANGED
@@ -27306,11 +27306,14 @@ async function executeCaseAttempt(testCase, context, timeout) {
27306
27306
  const effectiveRedaction = mergeRedactionConfig(scenario.redaction, testCase.redaction, cliRedaction);
27307
27307
  let finalPrompt = testCase.prompt;
27308
27308
  let finalResponse = result.text;
27309
+ let finalReason = sanitizeArtifactText(evalResult.reason, 1000);
27310
+ let finalEvidence = sanitizeEvidence(testCase.expected.type, evalResult);
27309
27311
  let redactionInfo;
27310
27312
  if (effectiveRedaction.enabled) {
27311
27313
  const redactor = new Redactor(effectiveRedaction);
27312
27314
  let promptRedacted = false;
27313
27315
  let responseRedacted = false;
27316
+ let reasonRedacted = false;
27314
27317
  let totalRedactions = 0;
27315
27318
  if (effectiveRedaction.redactPrompts) {
27316
27319
  if (typeof finalPrompt === "string") {
@@ -27338,10 +27341,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
27338
27341
  responseRedacted = responseResult.wasRedacted;
27339
27342
  totalRedactions += responseResult.redactionCount;
27340
27343
  }
27344
+ if (finalReason) {
27345
+ const reasonResult = redactor.redact(finalReason);
27346
+ finalReason = reasonResult.text;
27347
+ reasonRedacted = reasonResult.wasRedacted;
27348
+ totalRedactions += reasonResult.redactionCount;
27349
+ }
27350
+ finalEvidence = redactEvidence(finalEvidence, redactor, (count) => {
27351
+ if (count > 0)
27352
+ reasonRedacted = true;
27353
+ totalRedactions += count;
27354
+ });
27341
27355
  redactionInfo = {
27342
- redacted: promptRedacted || responseRedacted,
27356
+ redacted: promptRedacted || responseRedacted || reasonRedacted,
27343
27357
  promptRedacted,
27344
27358
  responseRedacted,
27359
+ reasonRedacted,
27345
27360
  redactionCount: totalRedactions
27346
27361
  };
27347
27362
  }
@@ -27352,7 +27367,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27352
27367
  status: evaluationStatus(evalResult),
27353
27368
  score: validScore(evalResult.score),
27354
27369
  matcherType: testCase.expected.type,
27355
- reason: evalResult.reason,
27370
+ reason: finalReason,
27356
27371
  latencyMs: generationMetrics.latencyMs,
27357
27372
  tokens: generationMetrics.tokens,
27358
27373
  prompt: finalPrompt,
@@ -27360,7 +27375,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
27360
27375
  expected: testCase.expected,
27361
27376
  tags: testCase.tags,
27362
27377
  redaction: redactionInfo,
27363
- evidence: sanitizeEvidence(testCase.expected.type, evalResult),
27378
+ evidence: finalEvidence,
27364
27379
  toolTrace: toolTrace.length ? toolTrace : undefined,
27365
27380
  toolLoop
27366
27381
  };
@@ -27386,30 +27401,69 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
27386
27401
  });
27387
27402
  }
27388
27403
  function evaluationStatus(result) {
27389
- return result.status ?? (result.passed ? "passed" : "failed");
27404
+ if (result.status === "passed" || result.status === "failed" || result.status === "invalid") {
27405
+ return result.status;
27406
+ }
27407
+ return result.passed ? "passed" : "failed";
27390
27408
  }
27391
27409
  function validScore(score) {
27392
27410
  return Number.isFinite(score) && score >= 0 && score <= 1 ? score : 0;
27393
27411
  }
27394
27412
  function sanitizeEvidence(evaluator, result) {
27395
- const evidence = { evaluator };
27396
- const score = validScore(result.score);
27397
- if (Number.isFinite(result.score) && result.score >= 0 && result.score <= 1) {
27398
- evidence.score = score;
27413
+ const evidence = {
27414
+ evaluator: sanitizeArtifactText(evaluator, 100) ?? "unknown"
27415
+ };
27416
+ if (isUnitIntervalNumber(result.score)) {
27417
+ evidence.score = result.score;
27399
27418
  }
27400
- if (result.evidence?.threshold !== undefined && validScore(result.evidence.threshold) === result.evidence.threshold) {
27419
+ if (!isRecord(result.evidence))
27420
+ return evidence;
27421
+ if (isUnitIntervalNumber(result.evidence.threshold)) {
27401
27422
  evidence.threshold = result.evidence.threshold;
27402
27423
  }
27403
- if (result.evidence?.model)
27404
- evidence.model = result.evidence.model.slice(0, 200);
27405
- if (result.evidence?.validation) {
27424
+ if (typeof result.evidence.model === "string" && result.evidence.model) {
27425
+ evidence.model = sanitizeArtifactText(result.evidence.model, 200);
27426
+ }
27427
+ if (isRecord(result.evidence.validation)) {
27428
+ const status = result.evidence.validation.status;
27429
+ const code = result.evidence.validation.code;
27430
+ if (status !== "valid" && status !== "invalid")
27431
+ return evidence;
27406
27432
  evidence.validation = {
27407
- status: result.evidence.validation.status,
27408
- ...result.evidence.validation.code ? { code: result.evidence.validation.code.slice(0, 100) } : {}
27433
+ status,
27434
+ ...typeof code === "string" && code ? { code: sanitizeArtifactText(code, 100) } : {}
27409
27435
  };
27410
27436
  }
27411
27437
  return evidence;
27412
27438
  }
27439
+ function redactEvidence(evidence, redactor, onRedactions) {
27440
+ const redactValue = (value) => {
27441
+ if (!value)
27442
+ return value;
27443
+ const result = redactor.redact(value);
27444
+ onRedactions(result.redactionCount);
27445
+ return result.text;
27446
+ };
27447
+ return {
27448
+ ...evidence,
27449
+ ...evidence.model ? { model: redactValue(evidence.model) } : {},
27450
+ ...evidence.validation ? {
27451
+ validation: {
27452
+ ...evidence.validation,
27453
+ ...evidence.validation.code ? { code: redactValue(evidence.validation.code) } : {}
27454
+ }
27455
+ } : {}
27456
+ };
27457
+ }
27458
+ function sanitizeArtifactText(value, maxLength) {
27459
+ return typeof value === "string" ? value.slice(0, maxLength) : undefined;
27460
+ }
27461
+ function isUnitIntervalNumber(value) {
27462
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
27463
+ }
27464
+ function isRecord(value) {
27465
+ return typeof value === "object" && value !== null && !Array.isArray(value);
27466
+ }
27413
27467
 
27414
27468
  class ToolLoopError extends Error {
27415
27469
  caseResult;
@@ -27792,12 +27846,54 @@ function execGit(command, allowFailure = false) {
27792
27846
  }
27793
27847
  }
27794
27848
  function getCaseEvaluationStatus(caseResult) {
27795
- if (caseResult.status)
27849
+ if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
27796
27850
  return caseResult.status;
27851
+ }
27797
27852
  if (caseResult.ok)
27798
27853
  return "passed";
27799
27854
  return caseResult.error ? "error" : "failed";
27800
27855
  }
27856
+ function assertRunManifestIntegrity(manifest) {
27857
+ if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
27858
+ throw new Error("Invalid run manifest: expected an object with a cases array");
27859
+ }
27860
+ for (const [index, caseResult] of manifest.cases.entries()) {
27861
+ if (!isRecord2(caseResult)) {
27862
+ throw new Error(`Invalid run manifest: case ${index} is not an object`);
27863
+ }
27864
+ if (caseResult.status !== undefined && caseResult.status !== "passed" && caseResult.status !== "failed" && caseResult.status !== "invalid" && caseResult.status !== "error") {
27865
+ throw new Error(`Invalid run manifest: case ${index} has an unknown status`);
27866
+ }
27867
+ if (caseResult.evidence !== undefined) {
27868
+ assertCaseEvaluationEvidence(caseResult.evidence, index);
27869
+ }
27870
+ }
27871
+ }
27872
+ function assertCaseEvaluationEvidence(evidence, caseIndex) {
27873
+ if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
27874
+ throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
27875
+ }
27876
+ if (evidence.score !== undefined && !isUnitIntervalNumber2(evidence.score)) {
27877
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence score`);
27878
+ }
27879
+ if (evidence.threshold !== undefined && !isUnitIntervalNumber2(evidence.threshold)) {
27880
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence threshold`);
27881
+ }
27882
+ if (evidence.model !== undefined && (typeof evidence.model !== "string" || evidence.model.length > 200)) {
27883
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence model`);
27884
+ }
27885
+ if (evidence.validation !== undefined) {
27886
+ if (!isRecord2(evidence.validation) || evidence.validation.status !== "valid" && evidence.validation.status !== "invalid" || evidence.validation.code !== undefined && (typeof evidence.validation.code !== "string" || evidence.validation.code.length > 100)) {
27887
+ throw new Error(`Invalid run manifest: case ${caseIndex} has invalid evidence validation`);
27888
+ }
27889
+ }
27890
+ }
27891
+ function isUnitIntervalNumber2(value) {
27892
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
27893
+ }
27894
+ function isRecord2(value) {
27895
+ return typeof value === "object" && value !== null && !Array.isArray(value);
27896
+ }
27801
27897
  function isRunManifest(manifest) {
27802
27898
  return !("type" in manifest) || manifest.type === undefined;
27803
27899
  }
@@ -27986,6 +28082,7 @@ async function runScenario(options) {
27986
28082
  const redactor = new Redactor(effectiveRedaction);
27987
28083
  const promptsRedacted = results.filter((r) => r.redaction?.promptRedacted).length;
27988
28084
  const responsesRedacted = results.filter((r) => r.redaction?.responseRedacted).length;
28085
+ const reasonsRedacted = results.filter((r) => r.redaction?.reasonRedacted).length;
27989
28086
  const totalRedactions = results.reduce((sum, r) => sum + (r.redaction?.redactionCount ?? 0), 0);
27990
28087
  redactionInfo = {
27991
28088
  enabled: true,
@@ -27994,6 +28091,7 @@ async function runScenario(options) {
27994
28091
  summary: {
27995
28092
  promptsRedacted,
27996
28093
  responsesRedacted,
28094
+ reasonsRedacted,
27997
28095
  totalRedactions
27998
28096
  }
27999
28097
  };
@@ -28068,6 +28166,9 @@ class LocalStorageAdapter {
28068
28166
  this.baselinesPath = join2(this.basePath, ".artemis", "baselines.json");
28069
28167
  }
28070
28168
  async save(manifest) {
28169
+ if (isRunManifest(manifest)) {
28170
+ assertRunManifestIntegrity(manifest);
28171
+ }
28071
28172
  const dir = join2(this.basePath, manifest.project);
28072
28173
  await mkdir(dir, { recursive: true });
28073
28174
  const filePath = join2(dir, `${manifest.run_id}.json`);
@@ -28090,6 +28191,7 @@ class LocalStorageAdapter {
28090
28191
  if (getManifestType(manifest) !== "run") {
28091
28192
  throw new Error(`Run ${runId} is not a standard run manifest`);
28092
28193
  }
28194
+ assertRunManifestIntegrity(manifest);
28093
28195
  return manifest;
28094
28196
  }
28095
28197
  async loadRedTeam(runId) {
@@ -36762,6 +36864,7 @@ class SupabaseStorageAdapter {
36762
36864
  this.project = project || "default";
36763
36865
  }
36764
36866
  async save(manifest) {
36867
+ assertRunManifestIntegrity(manifest);
36765
36868
  const filePath = `${manifest.project}/${manifest.run_id}.json`;
36766
36869
  const { error: uploadError } = await this.client.storage.from(this.bucket).upload(filePath, JSON.stringify(manifest, null, 2), {
36767
36870
  contentType: "application/json",
@@ -36815,7 +36918,9 @@ class SupabaseStorageAdapter {
36815
36918
  throw new Error(`Failed to download manifest: ${downloadError?.message}`);
36816
36919
  }
36817
36920
  const text = await data.text();
36818
- return JSON.parse(text);
36921
+ const manifest = JSON.parse(text);
36922
+ assertRunManifestIntegrity(manifest);
36923
+ return manifest;
36819
36924
  }
36820
36925
  async list(options) {
36821
36926
  let query = this.client.from("runs").select("run_id, scenario, success_rate, started_at").order("started_at", { ascending: false });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.0",
3
+ "version": "0.4.2",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.4.0",
75
- "@artemiskit/redteam": "0.3.1"
74
+ "@artemiskit/core": "0.4.2",
75
+ "@artemiskit/redteam": "0.3.3"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",