@artemiskit/sdk 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.js +121 -16
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.4.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [8716ca6]
|
|
8
|
+
- @artemiskit/core@0.4.2
|
|
9
|
+
- @artemiskit/redteam@0.3.3
|
|
10
|
+
|
|
11
|
+
## 0.4.1
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Updated dependencies
|
|
16
|
+
- @artemiskit/core@0.4.1
|
|
17
|
+
- @artemiskit/redteam@0.3.2
|
|
18
|
+
|
|
3
19
|
## 0.4.0
|
|
4
20
|
|
|
5
21
|
### Minor Changes
|
package/dist/index.js
CHANGED
|
@@ -27306,11 +27306,14 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27306
27306
|
const effectiveRedaction = mergeRedactionConfig(scenario.redaction, testCase.redaction, cliRedaction);
|
|
27307
27307
|
let finalPrompt = testCase.prompt;
|
|
27308
27308
|
let finalResponse = result.text;
|
|
27309
|
+
let finalReason = sanitizeArtifactText(evalResult.reason, 1000);
|
|
27310
|
+
let finalEvidence = sanitizeEvidence(testCase.expected.type, evalResult);
|
|
27309
27311
|
let redactionInfo;
|
|
27310
27312
|
if (effectiveRedaction.enabled) {
|
|
27311
27313
|
const redactor = new Redactor(effectiveRedaction);
|
|
27312
27314
|
let promptRedacted = false;
|
|
27313
27315
|
let responseRedacted = false;
|
|
27316
|
+
let reasonRedacted = false;
|
|
27314
27317
|
let totalRedactions = 0;
|
|
27315
27318
|
if (effectiveRedaction.redactPrompts) {
|
|
27316
27319
|
if (typeof finalPrompt === "string") {
|
|
@@ -27338,10 +27341,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27338
27341
|
responseRedacted = responseResult.wasRedacted;
|
|
27339
27342
|
totalRedactions += responseResult.redactionCount;
|
|
27340
27343
|
}
|
|
27344
|
+
if (finalReason) {
|
|
27345
|
+
const reasonResult = redactor.redact(finalReason);
|
|
27346
|
+
finalReason = reasonResult.text;
|
|
27347
|
+
reasonRedacted = reasonResult.wasRedacted;
|
|
27348
|
+
totalRedactions += reasonResult.redactionCount;
|
|
27349
|
+
}
|
|
27350
|
+
finalEvidence = redactEvidence(finalEvidence, redactor, (count) => {
|
|
27351
|
+
if (count > 0)
|
|
27352
|
+
reasonRedacted = true;
|
|
27353
|
+
totalRedactions += count;
|
|
27354
|
+
});
|
|
27341
27355
|
redactionInfo = {
|
|
27342
|
-
redacted: promptRedacted || responseRedacted,
|
|
27356
|
+
redacted: promptRedacted || responseRedacted || reasonRedacted,
|
|
27343
27357
|
promptRedacted,
|
|
27344
27358
|
responseRedacted,
|
|
27359
|
+
reasonRedacted,
|
|
27345
27360
|
redactionCount: totalRedactions
|
|
27346
27361
|
};
|
|
27347
27362
|
}
|
|
@@ -27352,7 +27367,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27352
27367
|
status: evaluationStatus(evalResult),
|
|
27353
27368
|
score: validScore(evalResult.score),
|
|
27354
27369
|
matcherType: testCase.expected.type,
|
|
27355
|
-
reason:
|
|
27370
|
+
reason: finalReason,
|
|
27356
27371
|
latencyMs: generationMetrics.latencyMs,
|
|
27357
27372
|
tokens: generationMetrics.tokens,
|
|
27358
27373
|
prompt: finalPrompt,
|
|
@@ -27360,7 +27375,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
27360
27375
|
expected: testCase.expected,
|
|
27361
27376
|
tags: testCase.tags,
|
|
27362
27377
|
redaction: redactionInfo,
|
|
27363
|
-
evidence:
|
|
27378
|
+
evidence: finalEvidence,
|
|
27364
27379
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
27365
27380
|
toolLoop
|
|
27366
27381
|
};
|
|
@@ -27386,30 +27401,69 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
27386
27401
|
});
|
|
27387
27402
|
}
|
|
27388
27403
|
function evaluationStatus(result) {
|
|
27389
|
-
|
|
27404
|
+
if (result.status === "passed" || result.status === "failed" || result.status === "invalid") {
|
|
27405
|
+
return result.status;
|
|
27406
|
+
}
|
|
27407
|
+
return result.passed ? "passed" : "failed";
|
|
27390
27408
|
}
|
|
27391
27409
|
function validScore(score) {
|
|
27392
27410
|
return Number.isFinite(score) && score >= 0 && score <= 1 ? score : 0;
|
|
27393
27411
|
}
|
|
27394
27412
|
function sanitizeEvidence(evaluator, result) {
|
|
27395
|
-
const evidence = {
|
|
27396
|
-
|
|
27397
|
-
|
|
27398
|
-
|
|
27413
|
+
const evidence = {
|
|
27414
|
+
evaluator: sanitizeArtifactText(evaluator, 100) ?? "unknown"
|
|
27415
|
+
};
|
|
27416
|
+
if (isUnitIntervalNumber(result.score)) {
|
|
27417
|
+
evidence.score = result.score;
|
|
27399
27418
|
}
|
|
27400
|
-
if (
|
|
27419
|
+
if (!isRecord(result.evidence))
|
|
27420
|
+
return evidence;
|
|
27421
|
+
if (isUnitIntervalNumber(result.evidence.threshold)) {
|
|
27401
27422
|
evidence.threshold = result.evidence.threshold;
|
|
27402
27423
|
}
|
|
27403
|
-
if (result.evidence
|
|
27404
|
-
evidence.model = result.evidence.model
|
|
27405
|
-
|
|
27424
|
+
if (typeof result.evidence.model === "string" && result.evidence.model) {
|
|
27425
|
+
evidence.model = sanitizeArtifactText(result.evidence.model, 200);
|
|
27426
|
+
}
|
|
27427
|
+
if (isRecord(result.evidence.validation)) {
|
|
27428
|
+
const status = result.evidence.validation.status;
|
|
27429
|
+
const code = result.evidence.validation.code;
|
|
27430
|
+
if (status !== "valid" && status !== "invalid")
|
|
27431
|
+
return evidence;
|
|
27406
27432
|
evidence.validation = {
|
|
27407
|
-
status
|
|
27408
|
-
...
|
|
27433
|
+
status,
|
|
27434
|
+
...typeof code === "string" && code ? { code: sanitizeArtifactText(code, 100) } : {}
|
|
27409
27435
|
};
|
|
27410
27436
|
}
|
|
27411
27437
|
return evidence;
|
|
27412
27438
|
}
|
|
27439
|
+
function redactEvidence(evidence, redactor, onRedactions) {
|
|
27440
|
+
const redactValue = (value) => {
|
|
27441
|
+
if (!value)
|
|
27442
|
+
return value;
|
|
27443
|
+
const result = redactor.redact(value);
|
|
27444
|
+
onRedactions(result.redactionCount);
|
|
27445
|
+
return result.text;
|
|
27446
|
+
};
|
|
27447
|
+
return {
|
|
27448
|
+
...evidence,
|
|
27449
|
+
...evidence.model ? { model: redactValue(evidence.model) } : {},
|
|
27450
|
+
...evidence.validation ? {
|
|
27451
|
+
validation: {
|
|
27452
|
+
...evidence.validation,
|
|
27453
|
+
...evidence.validation.code ? { code: redactValue(evidence.validation.code) } : {}
|
|
27454
|
+
}
|
|
27455
|
+
} : {}
|
|
27456
|
+
};
|
|
27457
|
+
}
|
|
27458
|
+
function sanitizeArtifactText(value, maxLength) {
|
|
27459
|
+
return typeof value === "string" ? value.slice(0, maxLength) : undefined;
|
|
27460
|
+
}
|
|
27461
|
+
function isUnitIntervalNumber(value) {
|
|
27462
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
27463
|
+
}
|
|
27464
|
+
function isRecord(value) {
|
|
27465
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
27466
|
+
}
|
|
27413
27467
|
|
|
27414
27468
|
class ToolLoopError extends Error {
|
|
27415
27469
|
caseResult;
|
|
@@ -27792,12 +27846,54 @@ function execGit(command, allowFailure = false) {
|
|
|
27792
27846
|
}
|
|
27793
27847
|
}
|
|
27794
27848
|
function getCaseEvaluationStatus(caseResult) {
|
|
27795
|
-
if (caseResult.status)
|
|
27849
|
+
if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
|
|
27796
27850
|
return caseResult.status;
|
|
27851
|
+
}
|
|
27797
27852
|
if (caseResult.ok)
|
|
27798
27853
|
return "passed";
|
|
27799
27854
|
return caseResult.error ? "error" : "failed";
|
|
27800
27855
|
}
|
|
27856
|
+
function assertRunManifestIntegrity(manifest) {
|
|
27857
|
+
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
27858
|
+
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
27859
|
+
}
|
|
27860
|
+
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
27861
|
+
if (!isRecord2(caseResult)) {
|
|
27862
|
+
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
27863
|
+
}
|
|
27864
|
+
if (caseResult.status !== undefined && caseResult.status !== "passed" && caseResult.status !== "failed" && caseResult.status !== "invalid" && caseResult.status !== "error") {
|
|
27865
|
+
throw new Error(`Invalid run manifest: case ${index} has an unknown status`);
|
|
27866
|
+
}
|
|
27867
|
+
if (caseResult.evidence !== undefined) {
|
|
27868
|
+
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
27869
|
+
}
|
|
27870
|
+
}
|
|
27871
|
+
}
|
|
27872
|
+
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
27873
|
+
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
27874
|
+
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
27875
|
+
}
|
|
27876
|
+
if (evidence.score !== undefined && !isUnitIntervalNumber2(evidence.score)) {
|
|
27877
|
+
throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence score`);
|
|
27878
|
+
}
|
|
27879
|
+
if (evidence.threshold !== undefined && !isUnitIntervalNumber2(evidence.threshold)) {
|
|
27880
|
+
throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence threshold`);
|
|
27881
|
+
}
|
|
27882
|
+
if (evidence.model !== undefined && (typeof evidence.model !== "string" || evidence.model.length > 200)) {
|
|
27883
|
+
throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence model`);
|
|
27884
|
+
}
|
|
27885
|
+
if (evidence.validation !== undefined) {
|
|
27886
|
+
if (!isRecord2(evidence.validation) || evidence.validation.status !== "valid" && evidence.validation.status !== "invalid" || evidence.validation.code !== undefined && (typeof evidence.validation.code !== "string" || evidence.validation.code.length > 100)) {
|
|
27887
|
+
throw new Error(`Invalid run manifest: case ${caseIndex} has invalid evidence validation`);
|
|
27888
|
+
}
|
|
27889
|
+
}
|
|
27890
|
+
}
|
|
27891
|
+
function isUnitIntervalNumber2(value) {
|
|
27892
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
27893
|
+
}
|
|
27894
|
+
function isRecord2(value) {
|
|
27895
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
27896
|
+
}
|
|
27801
27897
|
function isRunManifest(manifest) {
|
|
27802
27898
|
return !("type" in manifest) || manifest.type === undefined;
|
|
27803
27899
|
}
|
|
@@ -27986,6 +28082,7 @@ async function runScenario(options) {
|
|
|
27986
28082
|
const redactor = new Redactor(effectiveRedaction);
|
|
27987
28083
|
const promptsRedacted = results.filter((r) => r.redaction?.promptRedacted).length;
|
|
27988
28084
|
const responsesRedacted = results.filter((r) => r.redaction?.responseRedacted).length;
|
|
28085
|
+
const reasonsRedacted = results.filter((r) => r.redaction?.reasonRedacted).length;
|
|
27989
28086
|
const totalRedactions = results.reduce((sum, r) => sum + (r.redaction?.redactionCount ?? 0), 0);
|
|
27990
28087
|
redactionInfo = {
|
|
27991
28088
|
enabled: true,
|
|
@@ -27994,6 +28091,7 @@ async function runScenario(options) {
|
|
|
27994
28091
|
summary: {
|
|
27995
28092
|
promptsRedacted,
|
|
27996
28093
|
responsesRedacted,
|
|
28094
|
+
reasonsRedacted,
|
|
27997
28095
|
totalRedactions
|
|
27998
28096
|
}
|
|
27999
28097
|
};
|
|
@@ -28068,6 +28166,9 @@ class LocalStorageAdapter {
|
|
|
28068
28166
|
this.baselinesPath = join2(this.basePath, ".artemis", "baselines.json");
|
|
28069
28167
|
}
|
|
28070
28168
|
async save(manifest) {
|
|
28169
|
+
if (isRunManifest(manifest)) {
|
|
28170
|
+
assertRunManifestIntegrity(manifest);
|
|
28171
|
+
}
|
|
28071
28172
|
const dir = join2(this.basePath, manifest.project);
|
|
28072
28173
|
await mkdir(dir, { recursive: true });
|
|
28073
28174
|
const filePath = join2(dir, `${manifest.run_id}.json`);
|
|
@@ -28090,6 +28191,7 @@ class LocalStorageAdapter {
|
|
|
28090
28191
|
if (getManifestType(manifest) !== "run") {
|
|
28091
28192
|
throw new Error(`Run ${runId} is not a standard run manifest`);
|
|
28092
28193
|
}
|
|
28194
|
+
assertRunManifestIntegrity(manifest);
|
|
28093
28195
|
return manifest;
|
|
28094
28196
|
}
|
|
28095
28197
|
async loadRedTeam(runId) {
|
|
@@ -36762,6 +36864,7 @@ class SupabaseStorageAdapter {
|
|
|
36762
36864
|
this.project = project || "default";
|
|
36763
36865
|
}
|
|
36764
36866
|
async save(manifest) {
|
|
36867
|
+
assertRunManifestIntegrity(manifest);
|
|
36765
36868
|
const filePath = `${manifest.project}/${manifest.run_id}.json`;
|
|
36766
36869
|
const { error: uploadError } = await this.client.storage.from(this.bucket).upload(filePath, JSON.stringify(manifest, null, 2), {
|
|
36767
36870
|
contentType: "application/json",
|
|
@@ -36815,7 +36918,9 @@ class SupabaseStorageAdapter {
|
|
|
36815
36918
|
throw new Error(`Failed to download manifest: ${downloadError?.message}`);
|
|
36816
36919
|
}
|
|
36817
36920
|
const text = await data.text();
|
|
36818
|
-
|
|
36921
|
+
const manifest = JSON.parse(text);
|
|
36922
|
+
assertRunManifestIntegrity(manifest);
|
|
36923
|
+
return manifest;
|
|
36819
36924
|
}
|
|
36820
36925
|
async list(options) {
|
|
36821
36926
|
let query = this.client.from("runs").select("run_id, scenario, success_rate, started_at").order("started_at", { ascending: false });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.4.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.4.2",
|
|
75
|
+
"@artemiskit/redteam": "0.3.3"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|