@artemiskit/sdk 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.js +79 -1
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.4.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [b826698]
|
|
8
|
+
- @artemiskit/core@0.5.0
|
|
9
|
+
- @artemiskit/redteam@0.3.4
|
|
10
|
+
|
|
11
|
+
## 0.4.2
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Updated dependencies [8716ca6]
|
|
16
|
+
- @artemiskit/core@0.4.2
|
|
17
|
+
- @artemiskit/redteam@0.3.3
|
|
18
|
+
|
|
3
19
|
## 0.4.1
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/dist/index.js
CHANGED
|
@@ -6924,6 +6924,7 @@ import { join, resolve } from "path";
|
|
|
6924
6924
|
import { createHash } from "crypto";
|
|
6925
6925
|
import { webcrypto as crypto2 } from "crypto";
|
|
6926
6926
|
import { execSync } from "child_process";
|
|
6927
|
+
import { createHash as createHash2 } from "crypto";
|
|
6927
6928
|
import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
|
|
6928
6929
|
import { join as join2, resolve as resolve2 } from "path";
|
|
6929
6930
|
import { formatWithOptions } from "util";
|
|
@@ -27857,6 +27858,9 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27857
27858
|
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
27858
27859
|
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
27859
27860
|
}
|
|
27861
|
+
if (manifest.workload_identity !== undefined) {
|
|
27862
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
27863
|
+
}
|
|
27860
27864
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27861
27865
|
if (!isRecord2(caseResult)) {
|
|
27862
27866
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -27869,6 +27873,14 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
27869
27873
|
}
|
|
27870
27874
|
}
|
|
27871
27875
|
}
|
|
27876
|
+
function assertWorkloadIdentity(identity) {
|
|
27877
|
+
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
27878
|
+
throw new Error("Invalid run manifest: malformed workload identity");
|
|
27879
|
+
}
|
|
27880
|
+
}
|
|
27881
|
+
function isContentIdentity(value) {
|
|
27882
|
+
return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
|
|
27883
|
+
}
|
|
27872
27884
|
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
27873
27885
|
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
27874
27886
|
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
@@ -27908,6 +27920,7 @@ function createRunManifest(options) {
|
|
|
27908
27920
|
project,
|
|
27909
27921
|
config,
|
|
27910
27922
|
resolvedConfig,
|
|
27923
|
+
workloadIdentity,
|
|
27911
27924
|
cases,
|
|
27912
27925
|
startTime,
|
|
27913
27926
|
endTime,
|
|
@@ -27920,7 +27933,7 @@ function createRunManifest(options) {
|
|
|
27920
27933
|
const git = getGitInfo();
|
|
27921
27934
|
const environment = getEnvironmentInfo();
|
|
27922
27935
|
return {
|
|
27923
|
-
version: "1.
|
|
27936
|
+
version: "1.2",
|
|
27924
27937
|
run_id: nanoid(12),
|
|
27925
27938
|
project,
|
|
27926
27939
|
start_time: startTime.toISOString(),
|
|
@@ -27928,6 +27941,7 @@ function createRunManifest(options) {
|
|
|
27928
27941
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
27929
27942
|
config,
|
|
27930
27943
|
resolved_config: resolvedConfig,
|
|
27944
|
+
workload_identity: workloadIdentity,
|
|
27931
27945
|
metrics,
|
|
27932
27946
|
git,
|
|
27933
27947
|
provenance: {
|
|
@@ -28015,6 +28029,69 @@ function detectCIEnvironment() {
|
|
|
28015
28029
|
}
|
|
28016
28030
|
return;
|
|
28017
28031
|
}
|
|
28032
|
+
var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
28033
|
+
var REDACTED_SECRET = "[REDACTED_SECRET]";
|
|
28034
|
+
function createWorkloadIdentity(scenario) {
|
|
28035
|
+
const patterns = resolvePatterns([
|
|
28036
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
28037
|
+
...scenario.redaction?.patterns ?? []
|
|
28038
|
+
]).map(({ regex }) => regex);
|
|
28039
|
+
const workload = {
|
|
28040
|
+
name: scenario.name,
|
|
28041
|
+
version: scenario.version,
|
|
28042
|
+
description: scenario.description,
|
|
28043
|
+
tags: scenario.tags,
|
|
28044
|
+
variables: scenario.variables,
|
|
28045
|
+
setup: scenario.setup,
|
|
28046
|
+
cases: scenario.cases.map((testCase) => ({
|
|
28047
|
+
id: testCase.id,
|
|
28048
|
+
name: testCase.name,
|
|
28049
|
+
description: testCase.description,
|
|
28050
|
+
prompt: testCase.prompt,
|
|
28051
|
+
tags: testCase.tags,
|
|
28052
|
+
metadata: testCase.metadata,
|
|
28053
|
+
timeout: testCase.timeout,
|
|
28054
|
+
retries: testCase.retries,
|
|
28055
|
+
variables: testCase.variables
|
|
28056
|
+
}))
|
|
28057
|
+
};
|
|
28058
|
+
const rubric = {
|
|
28059
|
+
cases: scenario.cases.map((testCase) => ({
|
|
28060
|
+
id: testCase.id,
|
|
28061
|
+
expected: testCase.expected
|
|
28062
|
+
}))
|
|
28063
|
+
};
|
|
28064
|
+
return {
|
|
28065
|
+
schema_version: "1",
|
|
28066
|
+
workload: createContentIdentity(workload, patterns),
|
|
28067
|
+
rubric: createContentIdentity(rubric, patterns)
|
|
28068
|
+
};
|
|
28069
|
+
}
|
|
28070
|
+
function createContentIdentity(value, patterns) {
|
|
28071
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
28072
|
+
return {
|
|
28073
|
+
schema_version: "1",
|
|
28074
|
+
algorithm: "sha256",
|
|
28075
|
+
digest: createHash2("sha256").update(canonical).digest("hex")
|
|
28076
|
+
};
|
|
28077
|
+
}
|
|
28078
|
+
function canonicalize(value, patterns, key) {
|
|
28079
|
+
if (key && SENSITIVE_KEY.test(key))
|
|
28080
|
+
return REDACTED_SECRET;
|
|
28081
|
+
if (typeof value === "string") {
|
|
28082
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
28083
|
+
}
|
|
28084
|
+
if (Array.isArray(value)) {
|
|
28085
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
28086
|
+
}
|
|
28087
|
+
if (value && typeof value === "object") {
|
|
28088
|
+
return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
|
|
28089
|
+
nestedKey,
|
|
28090
|
+
canonicalize(nestedValue, patterns, nestedKey)
|
|
28091
|
+
]));
|
|
28092
|
+
}
|
|
28093
|
+
return value;
|
|
28094
|
+
}
|
|
28018
28095
|
async function runScenario(options) {
|
|
28019
28096
|
const {
|
|
28020
28097
|
scenario,
|
|
@@ -28106,6 +28183,7 @@ async function runScenario(options) {
|
|
|
28106
28183
|
seed: scenario.seed
|
|
28107
28184
|
},
|
|
28108
28185
|
resolvedConfig,
|
|
28186
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
28109
28187
|
cases: results,
|
|
28110
28188
|
startTime,
|
|
28111
28189
|
endTime,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.3",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.0",
|
|
75
|
+
"@artemiskit/redteam": "0.3.4"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|