@artemiskit/core 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/artifacts/manifest.d.ts +2 -1
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +23 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/index.js +92 -2
- package/dist/provenance/index.d.ts +1 -0
- package/dist/provenance/index.d.ts.map +1 -1
- package/dist/provenance/workload-identity.d.ts +12 -0
- package/dist/provenance/workload-identity.d.ts.map +1 -0
- package/dist/runner/runner.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/artifacts/manifest.test.ts +31 -1
- package/src/artifacts/manifest.ts +5 -1
- package/src/artifacts/types.ts +63 -0
- package/src/provenance/index.ts +1 -0
- package/src/provenance/workload-identity.test.ts +69 -0
- package/src/provenance/workload-identity.ts +92 -0
- package/src/runner/release-validation.test.ts +5 -0
- package/src/runner/runner.ts +2 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/core
|
|
2
2
|
|
|
3
|
+
## 0.5.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- b826698: Add versioned, redacted SHA-256 workload and rubric identities to normal scenario-run manifests.
|
|
8
|
+
The new manifest v1.2 field distinguishes changed declared inputs from changed evaluation criteria
|
|
9
|
+
without inferring identity for historical artifacts.
|
|
10
|
+
|
|
11
|
+
## 0.4.2
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- 8716ca6: Harden assessment evidence handling with a disposable database migration proof,
|
|
16
|
+
explicit invalid-measurement and execution-error presentation, and consistent
|
|
17
|
+
integrity counts across supported report formats.
|
|
18
|
+
|
|
3
19
|
## 0.4.1
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run manifest generation utilities
|
|
3
3
|
*/
|
|
4
|
-
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest } from './types';
|
|
4
|
+
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
|
|
5
5
|
/**
|
|
6
6
|
* Create a new run manifest
|
|
7
7
|
*/
|
|
@@ -9,6 +9,7 @@ export declare function createRunManifest(options: {
|
|
|
9
9
|
project: string;
|
|
10
10
|
config: RunConfig;
|
|
11
11
|
resolvedConfig?: ResolvedConfig;
|
|
12
|
+
workloadIdentity?: WorkloadIdentity;
|
|
12
13
|
cases: CaseResult[];
|
|
13
14
|
startTime: Date;
|
|
14
15
|
endTime: Date;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CAyCd"}
|
|
@@ -35,6 +35,23 @@ export interface ManifestRedactionInfo {
|
|
|
35
35
|
totalRedactions: number;
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
|
+
/** A versioned SHA-256 digest of canonical, redacted assessment material. */
|
|
39
|
+
export interface ContentIdentity {
|
|
40
|
+
schema_version: '1';
|
|
41
|
+
algorithm: 'sha256';
|
|
42
|
+
digest: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Separates the scenario workload from the criteria used to judge it.
|
|
46
|
+
*
|
|
47
|
+
* The digests prove matching declared, sanitized inputs. They are not a
|
|
48
|
+
* signature or an attestation of provider behaviour.
|
|
49
|
+
*/
|
|
50
|
+
export interface WorkloadIdentity {
|
|
51
|
+
schema_version: '1';
|
|
52
|
+
workload: ContentIdentity;
|
|
53
|
+
rubric: ContentIdentity;
|
|
54
|
+
}
|
|
38
55
|
/**
|
|
39
56
|
* Terminal status of a case measurement.
|
|
40
57
|
*
|
|
@@ -42,6 +59,8 @@ export interface ManifestRedactionInfo {
|
|
|
42
59
|
* reliably. `error` means execution did not produce a usable target response.
|
|
43
60
|
*/
|
|
44
61
|
export type CaseEvaluationStatus = 'passed' | 'failed' | 'invalid' | 'error';
|
|
62
|
+
/** Human-readable status labels shared by CLI and report consumers. */
|
|
63
|
+
export declare const CASE_EVALUATION_STATUS_LABELS: Record<CaseEvaluationStatus, string>;
|
|
45
64
|
/** Reviewed, bounded evaluator evidence retained in a run artifact. */
|
|
46
65
|
export interface CaseEvaluationEvidence {
|
|
47
66
|
evaluator: string;
|
|
@@ -235,6 +254,8 @@ export interface RunManifest {
|
|
|
235
254
|
config: RunConfig;
|
|
236
255
|
/** Resolved configuration with full provider details and source tracking */
|
|
237
256
|
resolved_config?: ResolvedConfig;
|
|
257
|
+
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
258
|
+
workload_identity?: WorkloadIdentity;
|
|
238
259
|
metrics: RunMetrics;
|
|
239
260
|
git: GitInfo;
|
|
240
261
|
provenance: ProvenanceInfo;
|
|
@@ -253,6 +274,8 @@ export interface RunManifest {
|
|
|
253
274
|
* criteria unless they set the legacy `error` field.
|
|
254
275
|
*/
|
|
255
276
|
export declare function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationStatus;
|
|
277
|
+
/** Return the stable human-readable label for a case measurement status. */
|
|
278
|
+
export declare function getCaseEvaluationStatusLabel(caseResult: CaseResult): string;
|
|
256
279
|
/**
|
|
257
280
|
* Reject untrusted integrity-bearing fields before a run manifest is persisted
|
|
258
281
|
* or treated as a standard run. Historical manifests remain supported because
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CA4B7F;AAqED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
package/dist/index.js
CHANGED
|
@@ -21271,6 +21271,12 @@ function execGit(command, allowFailure = false) {
|
|
|
21271
21271
|
}
|
|
21272
21272
|
|
|
21273
21273
|
// src/artifacts/types.ts
|
|
21274
|
+
var CASE_EVALUATION_STATUS_LABELS = {
|
|
21275
|
+
passed: "Passed",
|
|
21276
|
+
failed: "Failed criteria",
|
|
21277
|
+
invalid: "Invalid measurement",
|
|
21278
|
+
error: "Execution error"
|
|
21279
|
+
};
|
|
21274
21280
|
function getCaseEvaluationStatus(caseResult) {
|
|
21275
21281
|
if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
|
|
21276
21282
|
return caseResult.status;
|
|
@@ -21279,10 +21285,16 @@ function getCaseEvaluationStatus(caseResult) {
|
|
|
21279
21285
|
return "passed";
|
|
21280
21286
|
return caseResult.error ? "error" : "failed";
|
|
21281
21287
|
}
|
|
21288
|
+
function getCaseEvaluationStatusLabel(caseResult) {
|
|
21289
|
+
return CASE_EVALUATION_STATUS_LABELS[getCaseEvaluationStatus(caseResult)];
|
|
21290
|
+
}
|
|
21282
21291
|
function assertRunManifestIntegrity(manifest) {
|
|
21283
21292
|
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
21284
21293
|
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
21285
21294
|
}
|
|
21295
|
+
if (manifest.workload_identity !== undefined) {
|
|
21296
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
21297
|
+
}
|
|
21286
21298
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
21287
21299
|
if (!isRecord2(caseResult)) {
|
|
21288
21300
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -21295,6 +21307,14 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21295
21307
|
}
|
|
21296
21308
|
}
|
|
21297
21309
|
}
|
|
21310
|
+
function assertWorkloadIdentity(identity) {
|
|
21311
|
+
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
21312
|
+
throw new Error("Invalid run manifest: malformed workload identity");
|
|
21313
|
+
}
|
|
21314
|
+
}
|
|
21315
|
+
function isContentIdentity(value) {
|
|
21316
|
+
return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
|
|
21317
|
+
}
|
|
21298
21318
|
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
21299
21319
|
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
21300
21320
|
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
@@ -21336,6 +21356,7 @@ function createRunManifest(options) {
|
|
|
21336
21356
|
project,
|
|
21337
21357
|
config,
|
|
21338
21358
|
resolvedConfig,
|
|
21359
|
+
workloadIdentity,
|
|
21339
21360
|
cases,
|
|
21340
21361
|
startTime,
|
|
21341
21362
|
endTime,
|
|
@@ -21348,7 +21369,7 @@ function createRunManifest(options) {
|
|
|
21348
21369
|
const git = getGitInfo();
|
|
21349
21370
|
const environment = getEnvironmentInfo();
|
|
21350
21371
|
return {
|
|
21351
|
-
version: "1.
|
|
21372
|
+
version: "1.2",
|
|
21352
21373
|
run_id: nanoid(12),
|
|
21353
21374
|
project,
|
|
21354
21375
|
start_time: startTime.toISOString(),
|
|
@@ -21356,6 +21377,7 @@ function createRunManifest(options) {
|
|
|
21356
21377
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
21357
21378
|
config,
|
|
21358
21379
|
resolved_config: resolvedConfig,
|
|
21380
|
+
workload_identity: workloadIdentity,
|
|
21359
21381
|
metrics,
|
|
21360
21382
|
git,
|
|
21361
21383
|
provenance: {
|
|
@@ -21443,7 +21465,71 @@ function detectCIEnvironment() {
|
|
|
21443
21465
|
}
|
|
21444
21466
|
return;
|
|
21445
21467
|
}
|
|
21446
|
-
|
|
21468
|
+
// src/provenance/workload-identity.ts
|
|
21469
|
+
import { createHash as createHash2 } from "crypto";
|
|
21470
|
+
var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
21471
|
+
var REDACTED_SECRET = "[REDACTED_SECRET]";
|
|
21472
|
+
function createWorkloadIdentity(scenario) {
|
|
21473
|
+
const patterns = resolvePatterns([
|
|
21474
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
21475
|
+
...scenario.redaction?.patterns ?? []
|
|
21476
|
+
]).map(({ regex }) => regex);
|
|
21477
|
+
const workload = {
|
|
21478
|
+
name: scenario.name,
|
|
21479
|
+
version: scenario.version,
|
|
21480
|
+
description: scenario.description,
|
|
21481
|
+
tags: scenario.tags,
|
|
21482
|
+
variables: scenario.variables,
|
|
21483
|
+
setup: scenario.setup,
|
|
21484
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21485
|
+
id: testCase.id,
|
|
21486
|
+
name: testCase.name,
|
|
21487
|
+
description: testCase.description,
|
|
21488
|
+
prompt: testCase.prompt,
|
|
21489
|
+
tags: testCase.tags,
|
|
21490
|
+
metadata: testCase.metadata,
|
|
21491
|
+
timeout: testCase.timeout,
|
|
21492
|
+
retries: testCase.retries,
|
|
21493
|
+
variables: testCase.variables
|
|
21494
|
+
}))
|
|
21495
|
+
};
|
|
21496
|
+
const rubric = {
|
|
21497
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21498
|
+
id: testCase.id,
|
|
21499
|
+
expected: testCase.expected
|
|
21500
|
+
}))
|
|
21501
|
+
};
|
|
21502
|
+
return {
|
|
21503
|
+
schema_version: "1",
|
|
21504
|
+
workload: createContentIdentity(workload, patterns),
|
|
21505
|
+
rubric: createContentIdentity(rubric, patterns)
|
|
21506
|
+
};
|
|
21507
|
+
}
|
|
21508
|
+
function createContentIdentity(value, patterns) {
|
|
21509
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
21510
|
+
return {
|
|
21511
|
+
schema_version: "1",
|
|
21512
|
+
algorithm: "sha256",
|
|
21513
|
+
digest: createHash2("sha256").update(canonical).digest("hex")
|
|
21514
|
+
};
|
|
21515
|
+
}
|
|
21516
|
+
function canonicalize(value, patterns, key) {
|
|
21517
|
+
if (key && SENSITIVE_KEY.test(key))
|
|
21518
|
+
return REDACTED_SECRET;
|
|
21519
|
+
if (typeof value === "string") {
|
|
21520
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
21521
|
+
}
|
|
21522
|
+
if (Array.isArray(value)) {
|
|
21523
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
21524
|
+
}
|
|
21525
|
+
if (value && typeof value === "object") {
|
|
21526
|
+
return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
|
|
21527
|
+
nestedKey,
|
|
21528
|
+
canonicalize(nestedValue, patterns, nestedKey)
|
|
21529
|
+
]));
|
|
21530
|
+
}
|
|
21531
|
+
return value;
|
|
21532
|
+
}
|
|
21447
21533
|
// src/runner/runner.ts
|
|
21448
21534
|
async function runScenario(options) {
|
|
21449
21535
|
const {
|
|
@@ -21536,6 +21622,7 @@ async function runScenario(options) {
|
|
|
21536
21622
|
seed: scenario.seed
|
|
21537
21623
|
},
|
|
21538
21624
|
resolvedConfig,
|
|
21625
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
21539
21626
|
cases: results,
|
|
21540
21627
|
startTime,
|
|
21541
21628
|
endTime,
|
|
@@ -32582,6 +32669,7 @@ export {
|
|
|
32582
32669
|
ArtemisError,
|
|
32583
32670
|
BUILTIN_PATTERNS,
|
|
32584
32671
|
BUILTIN_REGEX_PATTERNS,
|
|
32672
|
+
CASE_EVALUATION_STATUS_LABELS,
|
|
32585
32673
|
ChatMessageSchema,
|
|
32586
32674
|
CombinedEvaluator,
|
|
32587
32675
|
ContainsEvaluator,
|
|
@@ -32622,11 +32710,13 @@ export {
|
|
|
32622
32710
|
createRunManifest,
|
|
32623
32711
|
createStorageAdapter,
|
|
32624
32712
|
createStorageFromEnv,
|
|
32713
|
+
createWorkloadIdentity,
|
|
32625
32714
|
discoverScenarios,
|
|
32626
32715
|
estimateCost,
|
|
32627
32716
|
executeCase,
|
|
32628
32717
|
formatCost,
|
|
32629
32718
|
getCaseEvaluationStatus,
|
|
32719
|
+
getCaseEvaluationStatusLabel,
|
|
32630
32720
|
getEnvironmentInfo,
|
|
32631
32721
|
getEvaluator,
|
|
32632
32722
|
getGitInfo,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical, redacted workload identities for reproducible assessment evidence.
|
|
3
|
+
*/
|
|
4
|
+
import type { WorkloadIdentity } from '../artifacts/types';
|
|
5
|
+
import type { Scenario } from '../scenario/schema';
|
|
6
|
+
/**
|
|
7
|
+
* Create independent identities for the declared workload and its evaluation
|
|
8
|
+
* criteria. Input is sorted, redacted, and hashed locally; raw material is
|
|
9
|
+
* never retained in the identity artifact.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity;
|
|
12
|
+
//# sourceMappingURL=workload-identity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workload-identity.d.ts","sourceRoot":"","sources":["../../src/provenance/workload-identity.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAKnD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,gBAAgB,CAsC3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA4HzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
|
package/package.json
CHANGED
|
@@ -55,7 +55,7 @@ describe('createRunManifest', () => {
|
|
|
55
55
|
endTime,
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
expect(manifest.version).toBe('1.
|
|
58
|
+
expect(manifest.version).toBe('1.2');
|
|
59
59
|
expect(manifest.project).toBe('test-project');
|
|
60
60
|
expect(manifest.run_id).toBeTruthy();
|
|
61
61
|
expect(manifest.run_id.length).toBe(12);
|
|
@@ -163,6 +163,32 @@ describe('createRunManifest', () => {
|
|
|
163
163
|
expect(manifest.resolved_config?.source.model).toBe('config');
|
|
164
164
|
});
|
|
165
165
|
|
|
166
|
+
test('retains a validated workload identity when provided', () => {
|
|
167
|
+
const manifest = createRunManifest({
|
|
168
|
+
project: 'test-project',
|
|
169
|
+
config: { scenario: 'test-scenario', provider: 'openai' },
|
|
170
|
+
workloadIdentity: {
|
|
171
|
+
schema_version: '1',
|
|
172
|
+
workload: {
|
|
173
|
+
schema_version: '1',
|
|
174
|
+
algorithm: 'sha256',
|
|
175
|
+
digest: 'a'.repeat(64),
|
|
176
|
+
},
|
|
177
|
+
rubric: {
|
|
178
|
+
schema_version: '1',
|
|
179
|
+
algorithm: 'sha256',
|
|
180
|
+
digest: 'b'.repeat(64),
|
|
181
|
+
},
|
|
182
|
+
},
|
|
183
|
+
cases: mockCases,
|
|
184
|
+
startTime: new Date(),
|
|
185
|
+
endTime: new Date(),
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
expect(manifest.workload_identity?.workload.digest).toBe('a'.repeat(64));
|
|
189
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
190
|
+
});
|
|
191
|
+
|
|
166
192
|
test('includes provenance information', () => {
|
|
167
193
|
const manifest = createRunManifest({
|
|
168
194
|
project: 'test-project',
|
|
@@ -273,5 +299,9 @@ describe('createRunManifest', () => {
|
|
|
273
299
|
],
|
|
274
300
|
};
|
|
275
301
|
expect(() => assertRunManifestIntegrity(malformed)).toThrow('invalid evidence validation');
|
|
302
|
+
|
|
303
|
+
expect(() =>
|
|
304
|
+
assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
|
|
305
|
+
).toThrow('malformed workload identity');
|
|
276
306
|
});
|
|
277
307
|
});
|
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
RunConfig,
|
|
15
15
|
RunManifest,
|
|
16
16
|
RunMetrics,
|
|
17
|
+
WorkloadIdentity,
|
|
17
18
|
} from './types';
|
|
18
19
|
import { getCaseEvaluationStatus } from './types';
|
|
19
20
|
|
|
@@ -24,6 +25,7 @@ export function createRunManifest(options: {
|
|
|
24
25
|
project: string;
|
|
25
26
|
config: RunConfig;
|
|
26
27
|
resolvedConfig?: ResolvedConfig;
|
|
28
|
+
workloadIdentity?: WorkloadIdentity;
|
|
27
29
|
cases: CaseResult[];
|
|
28
30
|
startTime: Date;
|
|
29
31
|
endTime: Date;
|
|
@@ -35,6 +37,7 @@ export function createRunManifest(options: {
|
|
|
35
37
|
project,
|
|
36
38
|
config,
|
|
37
39
|
resolvedConfig,
|
|
40
|
+
workloadIdentity,
|
|
38
41
|
cases,
|
|
39
42
|
startTime,
|
|
40
43
|
endTime,
|
|
@@ -50,7 +53,7 @@ export function createRunManifest(options: {
|
|
|
50
53
|
const environment = getEnvironmentInfo();
|
|
51
54
|
|
|
52
55
|
return {
|
|
53
|
-
version: '1.
|
|
56
|
+
version: '1.2',
|
|
54
57
|
run_id: nanoid(12),
|
|
55
58
|
project,
|
|
56
59
|
start_time: startTime.toISOString(),
|
|
@@ -58,6 +61,7 @@ export function createRunManifest(options: {
|
|
|
58
61
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
59
62
|
config,
|
|
60
63
|
resolved_config: resolvedConfig,
|
|
64
|
+
workload_identity: workloadIdentity,
|
|
61
65
|
metrics,
|
|
62
66
|
git,
|
|
63
67
|
provenance: {
|
package/src/artifacts/types.ts
CHANGED
|
@@ -42,6 +42,29 @@ export interface ManifestRedactionInfo {
|
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
// ============================================================================
|
|
46
|
+
// Reproducible Evidence Types
|
|
47
|
+
// ============================================================================
|
|
48
|
+
|
|
49
|
+
/** A versioned SHA-256 digest of canonical, redacted assessment material. */
|
|
50
|
+
export interface ContentIdentity {
|
|
51
|
+
schema_version: '1';
|
|
52
|
+
algorithm: 'sha256';
|
|
53
|
+
digest: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Separates the scenario workload from the criteria used to judge it.
|
|
58
|
+
*
|
|
59
|
+
* The digests prove matching declared, sanitized inputs. They are not a
|
|
60
|
+
* signature or an attestation of provider behaviour.
|
|
61
|
+
*/
|
|
62
|
+
export interface WorkloadIdentity {
|
|
63
|
+
schema_version: '1';
|
|
64
|
+
workload: ContentIdentity;
|
|
65
|
+
rubric: ContentIdentity;
|
|
66
|
+
}
|
|
67
|
+
|
|
45
68
|
// ============================================================================
|
|
46
69
|
// Case Result Types
|
|
47
70
|
// ============================================================================
|
|
@@ -54,6 +77,14 @@ export interface ManifestRedactionInfo {
|
|
|
54
77
|
*/
|
|
55
78
|
export type CaseEvaluationStatus = 'passed' | 'failed' | 'invalid' | 'error';
|
|
56
79
|
|
|
80
|
+
/** Human-readable status labels shared by CLI and report consumers. */
|
|
81
|
+
export const CASE_EVALUATION_STATUS_LABELS: Record<CaseEvaluationStatus, string> = {
|
|
82
|
+
passed: 'Passed',
|
|
83
|
+
failed: 'Failed criteria',
|
|
84
|
+
invalid: 'Invalid measurement',
|
|
85
|
+
error: 'Execution error',
|
|
86
|
+
};
|
|
87
|
+
|
|
57
88
|
/** Reviewed, bounded evaluator evidence retained in a run artifact. */
|
|
58
89
|
export interface CaseEvaluationEvidence {
|
|
59
90
|
evaluator: string;
|
|
@@ -269,6 +300,8 @@ export interface RunManifest {
|
|
|
269
300
|
config: RunConfig;
|
|
270
301
|
/** Resolved configuration with full provider details and source tracking */
|
|
271
302
|
resolved_config?: ResolvedConfig;
|
|
303
|
+
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
304
|
+
workload_identity?: WorkloadIdentity;
|
|
272
305
|
metrics: RunMetrics;
|
|
273
306
|
git: GitInfo;
|
|
274
307
|
provenance: ProvenanceInfo;
|
|
@@ -300,6 +333,11 @@ export function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationS
|
|
|
300
333
|
return caseResult.error ? 'error' : 'failed';
|
|
301
334
|
}
|
|
302
335
|
|
|
336
|
+
/** Return the stable human-readable label for a case measurement status. */
|
|
337
|
+
export function getCaseEvaluationStatusLabel(caseResult: CaseResult): string {
|
|
338
|
+
return CASE_EVALUATION_STATUS_LABELS[getCaseEvaluationStatus(caseResult)];
|
|
339
|
+
}
|
|
340
|
+
|
|
303
341
|
/**
|
|
304
342
|
* Reject untrusted integrity-bearing fields before a run manifest is persisted
|
|
305
343
|
* or treated as a standard run. Historical manifests remain supported because
|
|
@@ -310,6 +348,10 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
310
348
|
throw new Error('Invalid run manifest: expected an object with a cases array');
|
|
311
349
|
}
|
|
312
350
|
|
|
351
|
+
if (manifest.workload_identity !== undefined) {
|
|
352
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
353
|
+
}
|
|
354
|
+
|
|
313
355
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
314
356
|
if (!isRecord(caseResult)) {
|
|
315
357
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -331,6 +373,27 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
331
373
|
}
|
|
332
374
|
}
|
|
333
375
|
|
|
376
|
+
function assertWorkloadIdentity(identity: unknown): void {
|
|
377
|
+
if (
|
|
378
|
+
!isRecord(identity) ||
|
|
379
|
+
identity.schema_version !== '1' ||
|
|
380
|
+
!isContentIdentity(identity.workload) ||
|
|
381
|
+
!isContentIdentity(identity.rubric)
|
|
382
|
+
) {
|
|
383
|
+
throw new Error('Invalid run manifest: malformed workload identity');
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function isContentIdentity(value: unknown): boolean {
|
|
388
|
+
return (
|
|
389
|
+
isRecord(value) &&
|
|
390
|
+
value.schema_version === '1' &&
|
|
391
|
+
value.algorithm === 'sha256' &&
|
|
392
|
+
typeof value.digest === 'string' &&
|
|
393
|
+
/^[a-f0-9]{64}$/.test(value.digest)
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
|
|
334
397
|
function assertCaseEvaluationEvidence(evidence: unknown, caseIndex: number): void {
|
|
335
398
|
if (
|
|
336
399
|
!isRecord(evidence) ||
|
package/src/provenance/index.ts
CHANGED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { validateScenario } from '../scenario';
|
|
3
|
+
import { createWorkloadIdentity } from './workload-identity';
|
|
4
|
+
|
|
5
|
+
const baseScenario = () =>
|
|
6
|
+
validateScenario({
|
|
7
|
+
name: 'Customer-service policy test',
|
|
8
|
+
version: '2026-09',
|
|
9
|
+
providerConfig: { apiKey: 'sk-live-very-secret-value' },
|
|
10
|
+
setup: {
|
|
11
|
+
systemPrompt: 'Use token=internal-secret only for this fixture.',
|
|
12
|
+
},
|
|
13
|
+
cases: [
|
|
14
|
+
{
|
|
15
|
+
id: 'refund-policy',
|
|
16
|
+
prompt: 'Can I receive a refund?',
|
|
17
|
+
expected: {
|
|
18
|
+
type: 'llm_grader',
|
|
19
|
+
rubric: 'Approve only eligible refunds.',
|
|
20
|
+
strict: true,
|
|
21
|
+
threshold: 0.8,
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
],
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe('createWorkloadIdentity', () => {
|
|
28
|
+
test('is deterministic for equivalent declared scenario material', () => {
|
|
29
|
+
const first = createWorkloadIdentity(baseScenario());
|
|
30
|
+
const second = createWorkloadIdentity(baseScenario());
|
|
31
|
+
|
|
32
|
+
expect(first).toEqual(second);
|
|
33
|
+
expect(first.workload.digest).toHaveLength(64);
|
|
34
|
+
expect(first.rubric.digest).toHaveLength(64);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('separates workload changes from rubric changes', () => {
|
|
38
|
+
const base = baseScenario();
|
|
39
|
+
const changedPrompt = structuredClone(base);
|
|
40
|
+
changedPrompt.cases[0].prompt = 'Can I receive a replacement?';
|
|
41
|
+
const changedRubric = structuredClone(base);
|
|
42
|
+
changedRubric.cases[0].expected = {
|
|
43
|
+
type: 'llm_grader',
|
|
44
|
+
rubric: 'Approve only eligible replacements.',
|
|
45
|
+
strict: true,
|
|
46
|
+
threshold: 0.8,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const baseIdentity = createWorkloadIdentity(base);
|
|
50
|
+
const promptIdentity = createWorkloadIdentity(changedPrompt);
|
|
51
|
+
const rubricIdentity = createWorkloadIdentity(changedRubric);
|
|
52
|
+
|
|
53
|
+
expect(promptIdentity.workload.digest).not.toBe(baseIdentity.workload.digest);
|
|
54
|
+
expect(promptIdentity.rubric.digest).toBe(baseIdentity.rubric.digest);
|
|
55
|
+
expect(rubricIdentity.workload.digest).toBe(baseIdentity.workload.digest);
|
|
56
|
+
expect(rubricIdentity.rubric.digest).not.toBe(baseIdentity.rubric.digest);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('excludes sensitive configuration and recognized secret text before hashing', () => {
|
|
60
|
+
const first = baseScenario();
|
|
61
|
+
first.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-one' };
|
|
62
|
+
const second = structuredClone(first);
|
|
63
|
+
second.providerConfig = { apiKey: 'sk-live-another-secret-value' };
|
|
64
|
+
second.setup = { systemPrompt: 'Use token=different-secret only for this fixture.' };
|
|
65
|
+
second.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-two' };
|
|
66
|
+
|
|
67
|
+
expect(createWorkloadIdentity(second)).toEqual(createWorkloadIdentity(first));
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical, redacted workload identities for reproducible assessment evidence.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
import type { WorkloadIdentity } from '../artifacts/types';
|
|
7
|
+
import { redactText, resolvePatterns } from '../redaction/redactor';
|
|
8
|
+
import { DEFAULT_REDACTION_PATTERNS } from '../redaction/types';
|
|
9
|
+
import type { Scenario } from '../scenario/schema';
|
|
10
|
+
|
|
11
|
+
const SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
12
|
+
const REDACTED_SECRET = '[REDACTED_SECRET]';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Create independent identities for the declared workload and its evaluation
|
|
16
|
+
* criteria. Input is sorted, redacted, and hashed locally; raw material is
|
|
17
|
+
* never retained in the identity artifact.
|
|
18
|
+
*/
|
|
19
|
+
export function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity {
|
|
20
|
+
const patterns = resolvePatterns([
|
|
21
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
22
|
+
...(scenario.redaction?.patterns ?? []),
|
|
23
|
+
]).map(({ regex }) => regex);
|
|
24
|
+
|
|
25
|
+
const workload = {
|
|
26
|
+
name: scenario.name,
|
|
27
|
+
version: scenario.version,
|
|
28
|
+
description: scenario.description,
|
|
29
|
+
tags: scenario.tags,
|
|
30
|
+
variables: scenario.variables,
|
|
31
|
+
setup: scenario.setup,
|
|
32
|
+
cases: scenario.cases.map((testCase) => ({
|
|
33
|
+
id: testCase.id,
|
|
34
|
+
name: testCase.name,
|
|
35
|
+
description: testCase.description,
|
|
36
|
+
prompt: testCase.prompt,
|
|
37
|
+
tags: testCase.tags,
|
|
38
|
+
metadata: testCase.metadata,
|
|
39
|
+
timeout: testCase.timeout,
|
|
40
|
+
retries: testCase.retries,
|
|
41
|
+
variables: testCase.variables,
|
|
42
|
+
})),
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const rubric = {
|
|
46
|
+
cases: scenario.cases.map((testCase) => ({
|
|
47
|
+
id: testCase.id,
|
|
48
|
+
expected: testCase.expected,
|
|
49
|
+
})),
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
schema_version: '1',
|
|
54
|
+
workload: createContentIdentity(workload, patterns),
|
|
55
|
+
rubric: createContentIdentity(rubric, patterns),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function createContentIdentity(value: unknown, patterns: RegExp[]) {
|
|
60
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
61
|
+
return {
|
|
62
|
+
schema_version: '1' as const,
|
|
63
|
+
algorithm: 'sha256' as const,
|
|
64
|
+
digest: createHash('sha256').update(canonical).digest('hex'),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function canonicalize(value: unknown, patterns: RegExp[], key?: string): unknown {
|
|
69
|
+
if (key && SENSITIVE_KEY.test(key)) return REDACTED_SECRET;
|
|
70
|
+
|
|
71
|
+
if (typeof value === 'string') {
|
|
72
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (Array.isArray(value)) {
|
|
76
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (value && typeof value === 'object') {
|
|
80
|
+
return Object.fromEntries(
|
|
81
|
+
Object.entries(value)
|
|
82
|
+
.filter(([, nestedValue]) => nestedValue !== undefined)
|
|
83
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
84
|
+
.map(([nestedKey, nestedValue]) => [
|
|
85
|
+
nestedKey,
|
|
86
|
+
canonicalize(nestedValue, patterns, nestedKey),
|
|
87
|
+
])
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return value;
|
|
92
|
+
}
|
|
@@ -91,6 +91,11 @@ describe('release validation: fixture-backed workflow cases', () => {
|
|
|
91
91
|
failed_cases: 2,
|
|
92
92
|
success_rate: 0.6,
|
|
93
93
|
});
|
|
94
|
+
expect(result.manifest.workload_identity).toMatchObject({
|
|
95
|
+
schema_version: '1',
|
|
96
|
+
workload: { algorithm: 'sha256' },
|
|
97
|
+
rubric: { algorithm: 'sha256' },
|
|
98
|
+
});
|
|
94
99
|
});
|
|
95
100
|
|
|
96
101
|
test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
|
package/src/runner/runner.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { createRunManifest } from '../artifacts/manifest';
|
|
6
6
|
import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
|
|
7
|
+
import { createWorkloadIdentity } from '../provenance';
|
|
7
8
|
import { Redactor } from '../redaction';
|
|
8
9
|
import { executeCase } from './executor';
|
|
9
10
|
import type { RunOptions, RunResult } from './types';
|
|
@@ -120,6 +121,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
|
|
|
120
121
|
seed: scenario.seed,
|
|
121
122
|
},
|
|
122
123
|
resolvedConfig,
|
|
124
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
123
125
|
cases: results,
|
|
124
126
|
startTime,
|
|
125
127
|
endTime,
|