@artemiskit/sdk 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.3
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies [b826698]
8
+ - @artemiskit/core@0.5.0
9
+ - @artemiskit/redteam@0.3.4
10
+
3
11
  ## 0.4.2
4
12
 
5
13
  ### Patch Changes
package/dist/index.js CHANGED
@@ -6924,6 +6924,7 @@ import { join, resolve } from "path";
6924
6924
  import { createHash } from "crypto";
6925
6925
  import { webcrypto as crypto2 } from "crypto";
6926
6926
  import { execSync } from "child_process";
6927
+ import { createHash as createHash2 } from "crypto";
6927
6928
  import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
6928
6929
  import { join as join2, resolve as resolve2 } from "path";
6929
6930
  import { formatWithOptions } from "util";
@@ -27857,6 +27858,9 @@ function assertRunManifestIntegrity(manifest) {
27857
27858
  if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
27858
27859
  throw new Error("Invalid run manifest: expected an object with a cases array");
27859
27860
  }
27861
+ if (manifest.workload_identity !== undefined) {
27862
+ assertWorkloadIdentity(manifest.workload_identity);
27863
+ }
27860
27864
  for (const [index, caseResult] of manifest.cases.entries()) {
27861
27865
  if (!isRecord2(caseResult)) {
27862
27866
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27869,6 +27873,14 @@ function assertRunManifestIntegrity(manifest) {
27869
27873
  }
27870
27874
  }
27871
27875
  }
27876
+ function assertWorkloadIdentity(identity) {
27877
+ if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
27878
+ throw new Error("Invalid run manifest: malformed workload identity");
27879
+ }
27880
+ }
27881
+ function isContentIdentity(value) {
27882
+ return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
27883
+ }
27872
27884
  function assertCaseEvaluationEvidence(evidence, caseIndex) {
27873
27885
  if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
27874
27886
  throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
@@ -27908,6 +27920,7 @@ function createRunManifest(options) {
27908
27920
  project,
27909
27921
  config,
27910
27922
  resolvedConfig,
27923
+ workloadIdentity,
27911
27924
  cases,
27912
27925
  startTime,
27913
27926
  endTime,
@@ -27920,7 +27933,7 @@ function createRunManifest(options) {
27920
27933
  const git = getGitInfo();
27921
27934
  const environment = getEnvironmentInfo();
27922
27935
  return {
27923
- version: "1.1",
27936
+ version: "1.2",
27924
27937
  run_id: nanoid(12),
27925
27938
  project,
27926
27939
  start_time: startTime.toISOString(),
@@ -27928,6 +27941,7 @@ function createRunManifest(options) {
27928
27941
  duration_ms: endTime.getTime() - startTime.getTime(),
27929
27942
  config,
27930
27943
  resolved_config: resolvedConfig,
27944
+ workload_identity: workloadIdentity,
27931
27945
  metrics,
27932
27946
  git,
27933
27947
  provenance: {
@@ -28015,6 +28029,69 @@ function detectCIEnvironment() {
28015
28029
  }
28016
28030
  return;
28017
28031
  }
28032
+ var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
28033
+ var REDACTED_SECRET = "[REDACTED_SECRET]";
28034
+ function createWorkloadIdentity(scenario) {
28035
+ const patterns = resolvePatterns([
28036
+ ...DEFAULT_REDACTION_PATTERNS,
28037
+ ...scenario.redaction?.patterns ?? []
28038
+ ]).map(({ regex }) => regex);
28039
+ const workload = {
28040
+ name: scenario.name,
28041
+ version: scenario.version,
28042
+ description: scenario.description,
28043
+ tags: scenario.tags,
28044
+ variables: scenario.variables,
28045
+ setup: scenario.setup,
28046
+ cases: scenario.cases.map((testCase) => ({
28047
+ id: testCase.id,
28048
+ name: testCase.name,
28049
+ description: testCase.description,
28050
+ prompt: testCase.prompt,
28051
+ tags: testCase.tags,
28052
+ metadata: testCase.metadata,
28053
+ timeout: testCase.timeout,
28054
+ retries: testCase.retries,
28055
+ variables: testCase.variables
28056
+ }))
28057
+ };
28058
+ const rubric = {
28059
+ cases: scenario.cases.map((testCase) => ({
28060
+ id: testCase.id,
28061
+ expected: testCase.expected
28062
+ }))
28063
+ };
28064
+ return {
28065
+ schema_version: "1",
28066
+ workload: createContentIdentity(workload, patterns),
28067
+ rubric: createContentIdentity(rubric, patterns)
28068
+ };
28069
+ }
28070
+ function createContentIdentity(value, patterns) {
28071
+ const canonical = JSON.stringify(canonicalize(value, patterns));
28072
+ return {
28073
+ schema_version: "1",
28074
+ algorithm: "sha256",
28075
+ digest: createHash2("sha256").update(canonical).digest("hex")
28076
+ };
28077
+ }
28078
+ function canonicalize(value, patterns, key) {
28079
+ if (key && SENSITIVE_KEY.test(key))
28080
+ return REDACTED_SECRET;
28081
+ if (typeof value === "string") {
28082
+ return redactText(value, patterns, REDACTED_SECRET).text;
28083
+ }
28084
+ if (Array.isArray(value)) {
28085
+ return value.map((item) => canonicalize(item, patterns));
28086
+ }
28087
+ if (value && typeof value === "object") {
28088
+ return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
28089
+ nestedKey,
28090
+ canonicalize(nestedValue, patterns, nestedKey)
28091
+ ]));
28092
+ }
28093
+ return value;
28094
+ }
28018
28095
  async function runScenario(options) {
28019
28096
  const {
28020
28097
  scenario,
@@ -28106,6 +28183,7 @@ async function runScenario(options) {
28106
28183
  seed: scenario.seed
28107
28184
  },
28108
28185
  resolvedConfig,
28186
+ workloadIdentity: createWorkloadIdentity(scenario),
28109
28187
  cases: results,
28110
28188
  startTime,
28111
28189
  endTime,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.2",
3
+ "version": "0.4.3",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.4.2",
75
- "@artemiskit/redteam": "0.3.3"
74
+ "@artemiskit/core": "0.5.0",
75
+ "@artemiskit/redteam": "0.3.4"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",