@artemiskit/core 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/artifacts/manifest.d.ts +2 -1
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +19 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/index.js +81 -2
- package/dist/provenance/index.d.ts +1 -0
- package/dist/provenance/index.d.ts.map +1 -1
- package/dist/provenance/workload-identity.d.ts +12 -0
- package/dist/provenance/workload-identity.d.ts.map +1 -0
- package/dist/runner/runner.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/artifacts/manifest.test.ts +31 -1
- package/src/artifacts/manifest.ts +5 -1
- package/src/artifacts/types.ts +50 -0
- package/src/provenance/index.ts +1 -0
- package/src/provenance/workload-identity.test.ts +69 -0
- package/src/provenance/workload-identity.ts +92 -0
- package/src/runner/release-validation.test.ts +5 -0
- package/src/runner/runner.ts +2 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @artemiskit/core
|
|
2
2
|
|
|
3
|
+
## 0.5.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- b826698: Add versioned, redacted SHA-256 workload and rubric identities to normal scenario-run manifests.
|
|
8
|
+
The new manifest v1.2 field distinguishes changed declared inputs from changed evaluation criteria
|
|
9
|
+
without inferring identity for historical artifacts.
|
|
10
|
+
|
|
3
11
|
## 0.4.2
|
|
4
12
|
|
|
5
13
|
### Patch Changes
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run manifest generation utilities
|
|
3
3
|
*/
|
|
4
|
-
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest } from './types';
|
|
4
|
+
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
|
|
5
5
|
/**
|
|
6
6
|
* Create a new run manifest
|
|
7
7
|
*/
|
|
@@ -9,6 +9,7 @@ export declare function createRunManifest(options: {
|
|
|
9
9
|
project: string;
|
|
10
10
|
config: RunConfig;
|
|
11
11
|
resolvedConfig?: ResolvedConfig;
|
|
12
|
+
workloadIdentity?: WorkloadIdentity;
|
|
12
13
|
cases: CaseResult[];
|
|
13
14
|
startTime: Date;
|
|
14
15
|
endTime: Date;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CAyCd"}
|
|
@@ -35,6 +35,23 @@ export interface ManifestRedactionInfo {
|
|
|
35
35
|
totalRedactions: number;
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
|
+
/** A versioned SHA-256 digest of canonical, redacted assessment material. */
|
|
39
|
+
export interface ContentIdentity {
|
|
40
|
+
schema_version: '1';
|
|
41
|
+
algorithm: 'sha256';
|
|
42
|
+
digest: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Separates the scenario workload from the criteria used to judge it.
|
|
46
|
+
*
|
|
47
|
+
* The digests prove matching declared, sanitized inputs. They are not a
|
|
48
|
+
* signature or an attestation of provider behaviour.
|
|
49
|
+
*/
|
|
50
|
+
export interface WorkloadIdentity {
|
|
51
|
+
schema_version: '1';
|
|
52
|
+
workload: ContentIdentity;
|
|
53
|
+
rubric: ContentIdentity;
|
|
54
|
+
}
|
|
38
55
|
/**
|
|
39
56
|
* Terminal status of a case measurement.
|
|
40
57
|
*
|
|
@@ -237,6 +254,8 @@ export interface RunManifest {
|
|
|
237
254
|
config: RunConfig;
|
|
238
255
|
/** Resolved configuration with full provider details and source tracking */
|
|
239
256
|
resolved_config?: ResolvedConfig;
|
|
257
|
+
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
258
|
+
workload_identity?: WorkloadIdentity;
|
|
240
259
|
metrics: RunMetrics;
|
|
241
260
|
git: GitInfo;
|
|
242
261
|
provenance: ProvenanceInfo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CA4B7F;AAqED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
package/dist/index.js
CHANGED
|
@@ -21292,6 +21292,9 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21292
21292
|
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
21293
21293
|
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
21294
21294
|
}
|
|
21295
|
+
if (manifest.workload_identity !== undefined) {
|
|
21296
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
21297
|
+
}
|
|
21295
21298
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
21296
21299
|
if (!isRecord2(caseResult)) {
|
|
21297
21300
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -21304,6 +21307,14 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21304
21307
|
}
|
|
21305
21308
|
}
|
|
21306
21309
|
}
|
|
21310
|
+
function assertWorkloadIdentity(identity) {
|
|
21311
|
+
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
21312
|
+
throw new Error("Invalid run manifest: malformed workload identity");
|
|
21313
|
+
}
|
|
21314
|
+
}
|
|
21315
|
+
function isContentIdentity(value) {
|
|
21316
|
+
return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
|
|
21317
|
+
}
|
|
21307
21318
|
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
21308
21319
|
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
21309
21320
|
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
@@ -21345,6 +21356,7 @@ function createRunManifest(options) {
|
|
|
21345
21356
|
project,
|
|
21346
21357
|
config,
|
|
21347
21358
|
resolvedConfig,
|
|
21359
|
+
workloadIdentity,
|
|
21348
21360
|
cases,
|
|
21349
21361
|
startTime,
|
|
21350
21362
|
endTime,
|
|
@@ -21357,7 +21369,7 @@ function createRunManifest(options) {
|
|
|
21357
21369
|
const git = getGitInfo();
|
|
21358
21370
|
const environment = getEnvironmentInfo();
|
|
21359
21371
|
return {
|
|
21360
|
-
version: "1.
|
|
21372
|
+
version: "1.2",
|
|
21361
21373
|
run_id: nanoid(12),
|
|
21362
21374
|
project,
|
|
21363
21375
|
start_time: startTime.toISOString(),
|
|
@@ -21365,6 +21377,7 @@ function createRunManifest(options) {
|
|
|
21365
21377
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
21366
21378
|
config,
|
|
21367
21379
|
resolved_config: resolvedConfig,
|
|
21380
|
+
workload_identity: workloadIdentity,
|
|
21368
21381
|
metrics,
|
|
21369
21382
|
git,
|
|
21370
21383
|
provenance: {
|
|
@@ -21452,7 +21465,71 @@ function detectCIEnvironment() {
|
|
|
21452
21465
|
}
|
|
21453
21466
|
return;
|
|
21454
21467
|
}
|
|
21455
|
-
|
|
21468
|
+
// src/provenance/workload-identity.ts
|
|
21469
|
+
import { createHash as createHash2 } from "crypto";
|
|
21470
|
+
var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
21471
|
+
var REDACTED_SECRET = "[REDACTED_SECRET]";
|
|
21472
|
+
function createWorkloadIdentity(scenario) {
|
|
21473
|
+
const patterns = resolvePatterns([
|
|
21474
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
21475
|
+
...scenario.redaction?.patterns ?? []
|
|
21476
|
+
]).map(({ regex }) => regex);
|
|
21477
|
+
const workload = {
|
|
21478
|
+
name: scenario.name,
|
|
21479
|
+
version: scenario.version,
|
|
21480
|
+
description: scenario.description,
|
|
21481
|
+
tags: scenario.tags,
|
|
21482
|
+
variables: scenario.variables,
|
|
21483
|
+
setup: scenario.setup,
|
|
21484
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21485
|
+
id: testCase.id,
|
|
21486
|
+
name: testCase.name,
|
|
21487
|
+
description: testCase.description,
|
|
21488
|
+
prompt: testCase.prompt,
|
|
21489
|
+
tags: testCase.tags,
|
|
21490
|
+
metadata: testCase.metadata,
|
|
21491
|
+
timeout: testCase.timeout,
|
|
21492
|
+
retries: testCase.retries,
|
|
21493
|
+
variables: testCase.variables
|
|
21494
|
+
}))
|
|
21495
|
+
};
|
|
21496
|
+
const rubric = {
|
|
21497
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21498
|
+
id: testCase.id,
|
|
21499
|
+
expected: testCase.expected
|
|
21500
|
+
}))
|
|
21501
|
+
};
|
|
21502
|
+
return {
|
|
21503
|
+
schema_version: "1",
|
|
21504
|
+
workload: createContentIdentity(workload, patterns),
|
|
21505
|
+
rubric: createContentIdentity(rubric, patterns)
|
|
21506
|
+
};
|
|
21507
|
+
}
|
|
21508
|
+
function createContentIdentity(value, patterns) {
|
|
21509
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
21510
|
+
return {
|
|
21511
|
+
schema_version: "1",
|
|
21512
|
+
algorithm: "sha256",
|
|
21513
|
+
digest: createHash2("sha256").update(canonical).digest("hex")
|
|
21514
|
+
};
|
|
21515
|
+
}
|
|
21516
|
+
function canonicalize(value, patterns, key) {
|
|
21517
|
+
if (key && SENSITIVE_KEY.test(key))
|
|
21518
|
+
return REDACTED_SECRET;
|
|
21519
|
+
if (typeof value === "string") {
|
|
21520
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
21521
|
+
}
|
|
21522
|
+
if (Array.isArray(value)) {
|
|
21523
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
21524
|
+
}
|
|
21525
|
+
if (value && typeof value === "object") {
|
|
21526
|
+
return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
|
|
21527
|
+
nestedKey,
|
|
21528
|
+
canonicalize(nestedValue, patterns, nestedKey)
|
|
21529
|
+
]));
|
|
21530
|
+
}
|
|
21531
|
+
return value;
|
|
21532
|
+
}
|
|
21456
21533
|
// src/runner/runner.ts
|
|
21457
21534
|
async function runScenario(options) {
|
|
21458
21535
|
const {
|
|
@@ -21545,6 +21622,7 @@ async function runScenario(options) {
|
|
|
21545
21622
|
seed: scenario.seed
|
|
21546
21623
|
},
|
|
21547
21624
|
resolvedConfig,
|
|
21625
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
21548
21626
|
cases: results,
|
|
21549
21627
|
startTime,
|
|
21550
21628
|
endTime,
|
|
@@ -32632,6 +32710,7 @@ export {
|
|
|
32632
32710
|
createRunManifest,
|
|
32633
32711
|
createStorageAdapter,
|
|
32634
32712
|
createStorageFromEnv,
|
|
32713
|
+
createWorkloadIdentity,
|
|
32635
32714
|
discoverScenarios,
|
|
32636
32715
|
estimateCost,
|
|
32637
32716
|
executeCase,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical, redacted workload identities for reproducible assessment evidence.
|
|
3
|
+
*/
|
|
4
|
+
import type { WorkloadIdentity } from '../artifacts/types';
|
|
5
|
+
import type { Scenario } from '../scenario/schema';
|
|
6
|
+
/**
|
|
7
|
+
* Create independent identities for the declared workload and its evaluation
|
|
8
|
+
* criteria. Input is sorted, redacted, and hashed locally; raw material is
|
|
9
|
+
* never retained in the identity artifact.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity;
|
|
12
|
+
//# sourceMappingURL=workload-identity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workload-identity.d.ts","sourceRoot":"","sources":["../../src/provenance/workload-identity.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAKnD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,gBAAgB,CAsC3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA4HzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
|
package/package.json
CHANGED
|
@@ -55,7 +55,7 @@ describe('createRunManifest', () => {
|
|
|
55
55
|
endTime,
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
expect(manifest.version).toBe('1.
|
|
58
|
+
expect(manifest.version).toBe('1.2');
|
|
59
59
|
expect(manifest.project).toBe('test-project');
|
|
60
60
|
expect(manifest.run_id).toBeTruthy();
|
|
61
61
|
expect(manifest.run_id.length).toBe(12);
|
|
@@ -163,6 +163,32 @@ describe('createRunManifest', () => {
|
|
|
163
163
|
expect(manifest.resolved_config?.source.model).toBe('config');
|
|
164
164
|
});
|
|
165
165
|
|
|
166
|
+
test('retains a validated workload identity when provided', () => {
|
|
167
|
+
const manifest = createRunManifest({
|
|
168
|
+
project: 'test-project',
|
|
169
|
+
config: { scenario: 'test-scenario', provider: 'openai' },
|
|
170
|
+
workloadIdentity: {
|
|
171
|
+
schema_version: '1',
|
|
172
|
+
workload: {
|
|
173
|
+
schema_version: '1',
|
|
174
|
+
algorithm: 'sha256',
|
|
175
|
+
digest: 'a'.repeat(64),
|
|
176
|
+
},
|
|
177
|
+
rubric: {
|
|
178
|
+
schema_version: '1',
|
|
179
|
+
algorithm: 'sha256',
|
|
180
|
+
digest: 'b'.repeat(64),
|
|
181
|
+
},
|
|
182
|
+
},
|
|
183
|
+
cases: mockCases,
|
|
184
|
+
startTime: new Date(),
|
|
185
|
+
endTime: new Date(),
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
expect(manifest.workload_identity?.workload.digest).toBe('a'.repeat(64));
|
|
189
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
190
|
+
});
|
|
191
|
+
|
|
166
192
|
test('includes provenance information', () => {
|
|
167
193
|
const manifest = createRunManifest({
|
|
168
194
|
project: 'test-project',
|
|
@@ -273,5 +299,9 @@ describe('createRunManifest', () => {
|
|
|
273
299
|
],
|
|
274
300
|
};
|
|
275
301
|
expect(() => assertRunManifestIntegrity(malformed)).toThrow('invalid evidence validation');
|
|
302
|
+
|
|
303
|
+
expect(() =>
|
|
304
|
+
assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
|
|
305
|
+
).toThrow('malformed workload identity');
|
|
276
306
|
});
|
|
277
307
|
});
|
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
RunConfig,
|
|
15
15
|
RunManifest,
|
|
16
16
|
RunMetrics,
|
|
17
|
+
WorkloadIdentity,
|
|
17
18
|
} from './types';
|
|
18
19
|
import { getCaseEvaluationStatus } from './types';
|
|
19
20
|
|
|
@@ -24,6 +25,7 @@ export function createRunManifest(options: {
|
|
|
24
25
|
project: string;
|
|
25
26
|
config: RunConfig;
|
|
26
27
|
resolvedConfig?: ResolvedConfig;
|
|
28
|
+
workloadIdentity?: WorkloadIdentity;
|
|
27
29
|
cases: CaseResult[];
|
|
28
30
|
startTime: Date;
|
|
29
31
|
endTime: Date;
|
|
@@ -35,6 +37,7 @@ export function createRunManifest(options: {
|
|
|
35
37
|
project,
|
|
36
38
|
config,
|
|
37
39
|
resolvedConfig,
|
|
40
|
+
workloadIdentity,
|
|
38
41
|
cases,
|
|
39
42
|
startTime,
|
|
40
43
|
endTime,
|
|
@@ -50,7 +53,7 @@ export function createRunManifest(options: {
|
|
|
50
53
|
const environment = getEnvironmentInfo();
|
|
51
54
|
|
|
52
55
|
return {
|
|
53
|
-
version: '1.
|
|
56
|
+
version: '1.2',
|
|
54
57
|
run_id: nanoid(12),
|
|
55
58
|
project,
|
|
56
59
|
start_time: startTime.toISOString(),
|
|
@@ -58,6 +61,7 @@ export function createRunManifest(options: {
|
|
|
58
61
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
59
62
|
config,
|
|
60
63
|
resolved_config: resolvedConfig,
|
|
64
|
+
workload_identity: workloadIdentity,
|
|
61
65
|
metrics,
|
|
62
66
|
git,
|
|
63
67
|
provenance: {
|
package/src/artifacts/types.ts
CHANGED
|
@@ -42,6 +42,29 @@ export interface ManifestRedactionInfo {
|
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
// ============================================================================
|
|
46
|
+
// Reproducible Evidence Types
|
|
47
|
+
// ============================================================================
|
|
48
|
+
|
|
49
|
+
/** A versioned SHA-256 digest of canonical, redacted assessment material. */
|
|
50
|
+
export interface ContentIdentity {
|
|
51
|
+
schema_version: '1';
|
|
52
|
+
algorithm: 'sha256';
|
|
53
|
+
digest: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Separates the scenario workload from the criteria used to judge it.
|
|
58
|
+
*
|
|
59
|
+
* The digests prove matching declared, sanitized inputs. They are not a
|
|
60
|
+
* signature or an attestation of provider behaviour.
|
|
61
|
+
*/
|
|
62
|
+
export interface WorkloadIdentity {
|
|
63
|
+
schema_version: '1';
|
|
64
|
+
workload: ContentIdentity;
|
|
65
|
+
rubric: ContentIdentity;
|
|
66
|
+
}
|
|
67
|
+
|
|
45
68
|
// ============================================================================
|
|
46
69
|
// Case Result Types
|
|
47
70
|
// ============================================================================
|
|
@@ -277,6 +300,8 @@ export interface RunManifest {
|
|
|
277
300
|
config: RunConfig;
|
|
278
301
|
/** Resolved configuration with full provider details and source tracking */
|
|
279
302
|
resolved_config?: ResolvedConfig;
|
|
303
|
+
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
304
|
+
workload_identity?: WorkloadIdentity;
|
|
280
305
|
metrics: RunMetrics;
|
|
281
306
|
git: GitInfo;
|
|
282
307
|
provenance: ProvenanceInfo;
|
|
@@ -323,6 +348,10 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
323
348
|
throw new Error('Invalid run manifest: expected an object with a cases array');
|
|
324
349
|
}
|
|
325
350
|
|
|
351
|
+
if (manifest.workload_identity !== undefined) {
|
|
352
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
353
|
+
}
|
|
354
|
+
|
|
326
355
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
327
356
|
if (!isRecord(caseResult)) {
|
|
328
357
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -344,6 +373,27 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
344
373
|
}
|
|
345
374
|
}
|
|
346
375
|
|
|
376
|
+
function assertWorkloadIdentity(identity: unknown): void {
|
|
377
|
+
if (
|
|
378
|
+
!isRecord(identity) ||
|
|
379
|
+
identity.schema_version !== '1' ||
|
|
380
|
+
!isContentIdentity(identity.workload) ||
|
|
381
|
+
!isContentIdentity(identity.rubric)
|
|
382
|
+
) {
|
|
383
|
+
throw new Error('Invalid run manifest: malformed workload identity');
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function isContentIdentity(value: unknown): boolean {
|
|
388
|
+
return (
|
|
389
|
+
isRecord(value) &&
|
|
390
|
+
value.schema_version === '1' &&
|
|
391
|
+
value.algorithm === 'sha256' &&
|
|
392
|
+
typeof value.digest === 'string' &&
|
|
393
|
+
/^[a-f0-9]{64}$/.test(value.digest)
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
|
|
347
397
|
function assertCaseEvaluationEvidence(evidence: unknown, caseIndex: number): void {
|
|
348
398
|
if (
|
|
349
399
|
!isRecord(evidence) ||
|
package/src/provenance/index.ts
CHANGED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { validateScenario } from '../scenario';
|
|
3
|
+
import { createWorkloadIdentity } from './workload-identity';
|
|
4
|
+
|
|
5
|
+
const baseScenario = () =>
|
|
6
|
+
validateScenario({
|
|
7
|
+
name: 'Customer-service policy test',
|
|
8
|
+
version: '2026-09',
|
|
9
|
+
providerConfig: { apiKey: 'sk-live-very-secret-value' },
|
|
10
|
+
setup: {
|
|
11
|
+
systemPrompt: 'Use token=internal-secret only for this fixture.',
|
|
12
|
+
},
|
|
13
|
+
cases: [
|
|
14
|
+
{
|
|
15
|
+
id: 'refund-policy',
|
|
16
|
+
prompt: 'Can I receive a refund?',
|
|
17
|
+
expected: {
|
|
18
|
+
type: 'llm_grader',
|
|
19
|
+
rubric: 'Approve only eligible refunds.',
|
|
20
|
+
strict: true,
|
|
21
|
+
threshold: 0.8,
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
],
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe('createWorkloadIdentity', () => {
|
|
28
|
+
test('is deterministic for equivalent declared scenario material', () => {
|
|
29
|
+
const first = createWorkloadIdentity(baseScenario());
|
|
30
|
+
const second = createWorkloadIdentity(baseScenario());
|
|
31
|
+
|
|
32
|
+
expect(first).toEqual(second);
|
|
33
|
+
expect(first.workload.digest).toHaveLength(64);
|
|
34
|
+
expect(first.rubric.digest).toHaveLength(64);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('separates workload changes from rubric changes', () => {
|
|
38
|
+
const base = baseScenario();
|
|
39
|
+
const changedPrompt = structuredClone(base);
|
|
40
|
+
changedPrompt.cases[0].prompt = 'Can I receive a replacement?';
|
|
41
|
+
const changedRubric = structuredClone(base);
|
|
42
|
+
changedRubric.cases[0].expected = {
|
|
43
|
+
type: 'llm_grader',
|
|
44
|
+
rubric: 'Approve only eligible replacements.',
|
|
45
|
+
strict: true,
|
|
46
|
+
threshold: 0.8,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const baseIdentity = createWorkloadIdentity(base);
|
|
50
|
+
const promptIdentity = createWorkloadIdentity(changedPrompt);
|
|
51
|
+
const rubricIdentity = createWorkloadIdentity(changedRubric);
|
|
52
|
+
|
|
53
|
+
expect(promptIdentity.workload.digest).not.toBe(baseIdentity.workload.digest);
|
|
54
|
+
expect(promptIdentity.rubric.digest).toBe(baseIdentity.rubric.digest);
|
|
55
|
+
expect(rubricIdentity.workload.digest).toBe(baseIdentity.workload.digest);
|
|
56
|
+
expect(rubricIdentity.rubric.digest).not.toBe(baseIdentity.rubric.digest);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('excludes sensitive configuration and recognized secret text before hashing', () => {
|
|
60
|
+
const first = baseScenario();
|
|
61
|
+
first.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-one' };
|
|
62
|
+
const second = structuredClone(first);
|
|
63
|
+
second.providerConfig = { apiKey: 'sk-live-another-secret-value' };
|
|
64
|
+
second.setup = { systemPrompt: 'Use token=different-secret only for this fixture.' };
|
|
65
|
+
second.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-two' };
|
|
66
|
+
|
|
67
|
+
expect(createWorkloadIdentity(second)).toEqual(createWorkloadIdentity(first));
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical, redacted workload identities for reproducible assessment evidence.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
import type { WorkloadIdentity } from '../artifacts/types';
|
|
7
|
+
import { redactText, resolvePatterns } from '../redaction/redactor';
|
|
8
|
+
import { DEFAULT_REDACTION_PATTERNS } from '../redaction/types';
|
|
9
|
+
import type { Scenario } from '../scenario/schema';
|
|
10
|
+
|
|
11
|
+
const SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
12
|
+
const REDACTED_SECRET = '[REDACTED_SECRET]';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Create independent identities for the declared workload and its evaluation
|
|
16
|
+
* criteria. Input is sorted, redacted, and hashed locally; raw material is
|
|
17
|
+
* never retained in the identity artifact.
|
|
18
|
+
*/
|
|
19
|
+
export function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity {
|
|
20
|
+
const patterns = resolvePatterns([
|
|
21
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
22
|
+
...(scenario.redaction?.patterns ?? []),
|
|
23
|
+
]).map(({ regex }) => regex);
|
|
24
|
+
|
|
25
|
+
const workload = {
|
|
26
|
+
name: scenario.name,
|
|
27
|
+
version: scenario.version,
|
|
28
|
+
description: scenario.description,
|
|
29
|
+
tags: scenario.tags,
|
|
30
|
+
variables: scenario.variables,
|
|
31
|
+
setup: scenario.setup,
|
|
32
|
+
cases: scenario.cases.map((testCase) => ({
|
|
33
|
+
id: testCase.id,
|
|
34
|
+
name: testCase.name,
|
|
35
|
+
description: testCase.description,
|
|
36
|
+
prompt: testCase.prompt,
|
|
37
|
+
tags: testCase.tags,
|
|
38
|
+
metadata: testCase.metadata,
|
|
39
|
+
timeout: testCase.timeout,
|
|
40
|
+
retries: testCase.retries,
|
|
41
|
+
variables: testCase.variables,
|
|
42
|
+
})),
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const rubric = {
|
|
46
|
+
cases: scenario.cases.map((testCase) => ({
|
|
47
|
+
id: testCase.id,
|
|
48
|
+
expected: testCase.expected,
|
|
49
|
+
})),
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
schema_version: '1',
|
|
54
|
+
workload: createContentIdentity(workload, patterns),
|
|
55
|
+
rubric: createContentIdentity(rubric, patterns),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function createContentIdentity(value: unknown, patterns: RegExp[]) {
|
|
60
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
61
|
+
return {
|
|
62
|
+
schema_version: '1' as const,
|
|
63
|
+
algorithm: 'sha256' as const,
|
|
64
|
+
digest: createHash('sha256').update(canonical).digest('hex'),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function canonicalize(value: unknown, patterns: RegExp[], key?: string): unknown {
|
|
69
|
+
if (key && SENSITIVE_KEY.test(key)) return REDACTED_SECRET;
|
|
70
|
+
|
|
71
|
+
if (typeof value === 'string') {
|
|
72
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (Array.isArray(value)) {
|
|
76
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (value && typeof value === 'object') {
|
|
80
|
+
return Object.fromEntries(
|
|
81
|
+
Object.entries(value)
|
|
82
|
+
.filter(([, nestedValue]) => nestedValue !== undefined)
|
|
83
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
84
|
+
.map(([nestedKey, nestedValue]) => [
|
|
85
|
+
nestedKey,
|
|
86
|
+
canonicalize(nestedValue, patterns, nestedKey),
|
|
87
|
+
])
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return value;
|
|
92
|
+
}
|
|
@@ -91,6 +91,11 @@ describe('release validation: fixture-backed workflow cases', () => {
|
|
|
91
91
|
failed_cases: 2,
|
|
92
92
|
success_rate: 0.6,
|
|
93
93
|
});
|
|
94
|
+
expect(result.manifest.workload_identity).toMatchObject({
|
|
95
|
+
schema_version: '1',
|
|
96
|
+
workload: { algorithm: 'sha256' },
|
|
97
|
+
rubric: { algorithm: 'sha256' },
|
|
98
|
+
});
|
|
94
99
|
});
|
|
95
100
|
|
|
96
101
|
test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
|
package/src/runner/runner.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { createRunManifest } from '../artifacts/manifest';
|
|
6
6
|
import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
|
|
7
|
+
import { createWorkloadIdentity } from '../provenance';
|
|
7
8
|
import { Redactor } from '../redaction';
|
|
8
9
|
import { executeCase } from './executor';
|
|
9
10
|
import type { RunOptions, RunResult } from './types';
|
|
@@ -120,6 +121,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
|
|
|
120
121
|
seed: scenario.seed,
|
|
121
122
|
},
|
|
122
123
|
resolvedConfig,
|
|
124
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
123
125
|
cases: results,
|
|
124
126
|
startTime,
|
|
125
127
|
endTime,
|