@artemiskit/core 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/dist/artifacts/types.d.ts +10 -0
  3. package/dist/artifacts/types.d.ts.map +1 -1
  4. package/dist/index.js +122 -16
  5. package/dist/runner/runner.d.ts.map +1 -1
  6. package/dist/storage/local.d.ts +1 -1
  7. package/dist/storage/local.d.ts.map +1 -1
  8. package/dist/storage/supabase.d.ts.map +1 -1
  9. package/package.json +1 -1
  10. package/src/artifacts/manifest.test.ts +24 -0
  11. package/src/artifacts/types.ts +85 -1
  12. package/src/runner/executor.test.ts +34 -0
  13. package/src/runner/executor.ts +83 -25
  14. package/src/runner/runner.ts +2 -0
  15. package/src/storage/local.test.ts +24 -0
  16. package/src/storage/local.ts +13 -2
  17. package/src/storage/supabase.test.ts +46 -0
  18. package/src/storage/supabase.ts +10 -2
package/CHANGELOG.md CHANGED
@@ -1,5 +1,12 @@
1
1
  # @artemiskit/core
2
2
 
3
+ ## 0.4.1
4
+
5
+ ### Patch Changes
6
+
7
+ - Harden evaluation-integrity artifacts: redact bounded evaluator evidence, validate status and
8
+ evidence at storage and JSON-report boundaries, and make non-interactive publishes exit cleanly.
9
+
3
10
  ## 0.4.0
4
11
 
5
12
  ### Minor Changes
package/dist/artifacts/types.d.ts CHANGED
@@ -11,6 +11,8 @@ export interface CaseRedactionInfo {
11
11
  promptRedacted: boolean;
12
12
  /** Whether response was redacted */
13
13
  responseRedacted: boolean;
14
+ /** Whether evaluator reason text was redacted */
15
+ reasonRedacted?: boolean;
14
16
  /** Number of redactions in this case */
15
17
  redactionCount: number;
16
18
  }
@@ -28,6 +30,8 @@ export interface ManifestRedactionInfo {
28
30
  summary: {
29
31
  promptsRedacted: number;
30
32
  responsesRedacted: number;
33
+ /** Cases whose evaluator reason text was redacted. */
34
+ reasonsRedacted?: number;
31
35
  totalRedactions: number;
32
36
  };
33
37
  }
@@ -249,6 +253,12 @@ export interface RunManifest {
249
253
  * criteria unless they set the legacy `error` field.
250
254
  */
251
255
  export declare function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationStatus;
256
+ /**
257
+ * Reject untrusted integrity-bearing fields before a run manifest is persisted
258
+ * or treated as a standard run. Historical manifests remain supported because
259
+ * status and evidence are optional in the v1.0 contract.
260
+ */
261
+ export declare function assertRunManifestIntegrity(manifest: unknown): asserts manifest is RunManifest;
252
262
  /**
253
263
  * Red team test result status
254
264
  */
package/dist/artifacts/types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAIpF;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAwB7F;AAgDD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
package/dist/index.js CHANGED
@@ -20702,11 +20702,14 @@ async function executeCaseAttempt(testCase, context, timeout) {
20702
20702
  const effectiveRedaction = mergeRedactionConfig(scenario.redaction, testCase.redaction, cliRedaction);
20703
20703
  let finalPrompt = testCase.prompt;
20704
20704
  let finalResponse = result.text;
20705
+ let finalReason = sanitizeArtifactText(evalResult.reason, 1000);
20706
+ let finalEvidence = sanitizeEvidence(testCase.expected.type, evalResult);
20705
20707
  let redactionInfo;
20706
20708
  if (effectiveRedaction.enabled) {
20707
20709
  const redactor = new Redactor(effectiveRedaction);
20708
20710
  let promptRedacted = false;
20709
20711
  let responseRedacted = false;
20712
+ let reasonRedacted = false;
20710
20713
  let totalRedactions = 0;
20711
20714
  if (effectiveRedaction.redactPrompts) {
20712
20715
  if (typeof finalPrompt === "string") {
@@ -20734,10 +20737,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
20734
20737
  responseRedacted = responseResult.wasRedacted;
20735
20738
  totalRedactions += responseResult.redactionCount;
20736
20739
  }
20740
+ if (finalReason) {
20741
+ const reasonResult = redactor.redact(finalReason);
20742
+ finalReason = reasonResult.text;
20743
+ reasonRedacted = reasonResult.wasRedacted;
20744
+ totalRedactions += reasonResult.redactionCount;
20745
+ }
20746
+ finalEvidence = redactEvidence(finalEvidence, redactor, (count) => {
20747
+ if (count > 0)
20748
+ reasonRedacted = true;
20749
+ totalRedactions += count;
20750
+ });
20737
20751
  redactionInfo = {
20738
- redacted: promptRedacted || responseRedacted,
20752
+ redacted: promptRedacted || responseRedacted || reasonRedacted,
20739
20753
  promptRedacted,
20740
20754
  responseRedacted,
20755
+ reasonRedacted,
20741
20756
  redactionCount: totalRedactions
20742
20757
  };
20743
20758
  }
@@ -20748,7 +20763,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20748
20763
  status: evaluationStatus(evalResult),
20749
20764
  score: validScore(evalResult.score),
20750
20765
  matcherType: testCase.expected.type,
20751
- reason: evalResult.reason,
20766
+ reason: finalReason,
20752
20767
  latencyMs: generationMetrics.latencyMs,
20753
20768
  tokens: generationMetrics.tokens,
20754
20769
  prompt: finalPrompt,
@@ -20756,7 +20771,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20756
20771
  expected: testCase.expected,
20757
20772
  tags: testCase.tags,
20758
20773
  redaction: redactionInfo,
20759
- evidence: sanitizeEvidence(testCase.expected.type, evalResult),
20774
+ evidence: finalEvidence,
20760
20775
  toolTrace: toolTrace.length ? toolTrace : undefined,
20761
20776
  toolLoop
20762
20777
  };
@@ -20782,30 +20797,69 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
20782
20797
  });
20783
20798
  }
20784
20799
  function evaluationStatus(result) {
20785
- return result.status ?? (result.passed ? "passed" : "failed");
20800
+ if (result.status === "passed" || result.status === "failed" || result.status === "invalid") {
20801
+ return result.status;
20802
+ }
20803
+ return result.passed ? "passed" : "failed";
20786
20804
  }
20787
20805
  function validScore(score) {
20788
20806
  return Number.isFinite(score) && score >= 0 && score <= 1 ? score : 0;
20789
20807
  }
20790
20808
  function sanitizeEvidence(evaluator, result) {
20791
- const evidence = { evaluator };
20792
- const score = validScore(result.score);
20793
- if (Number.isFinite(result.score) && result.score >= 0 && result.score <= 1) {
20794
- evidence.score = score;
20809
+ const evidence = {
20810
+ evaluator: sanitizeArtifactText(evaluator, 100) ?? "unknown"
20811
+ };
20812
+ if (isUnitIntervalNumber(result.score)) {
20813
+ evidence.score = result.score;
20795
20814
  }
20796
- if (result.evidence?.threshold !== undefined && validScore(result.evidence.threshold) === result.evidence.threshold) {
20815
+ if (!isRecord(result.evidence))
20816
+ return evidence;
20817
+ if (isUnitIntervalNumber(result.evidence.threshold)) {
20797
20818
  evidence.threshold = result.evidence.threshold;
20798
20819
  }
20799
- if (result.evidence?.model)
20800
- evidence.model = result.evidence.model.slice(0, 200);
20801
- if (result.evidence?.validation) {
20820
+ if (typeof result.evidence.model === "string" && result.evidence.model) {
20821
+ evidence.model = sanitizeArtifactText(result.evidence.model, 200);
20822
+ }
20823
+ if (isRecord(result.evidence.validation)) {
20824
+ const status = result.evidence.validation.status;
20825
+ const code = result.evidence.validation.code;
20826
+ if (status !== "valid" && status !== "invalid")
20827
+ return evidence;
20802
20828
  evidence.validation = {
20803
- status: result.evidence.validation.status,
20804
- ...result.evidence.validation.code ? { code: result.evidence.validation.code.slice(0, 100) } : {}
20829
+ status,
20830
+ ...typeof code === "string" && code ? { code: sanitizeArtifactText(code, 100) } : {}
20805
20831
  };
20806
20832
  }
20807
20833
  return evidence;
20808
20834
  }
20835
+ function redactEvidence(evidence, redactor, onRedactions) {
20836
+ const redactValue = (value) => {
20837
+ if (!value)
20838
+ return value;
20839
+ const result = redactor.redact(value);
20840
+ onRedactions(result.redactionCount);
20841
+ return result.text;
20842
+ };
20843
+ return {
20844
+ ...evidence,
20845
+ ...evidence.model ? { model: redactValue(evidence.model) } : {},
20846
+ ...evidence.validation ? {
20847
+ validation: {
20848
+ ...evidence.validation,
20849
+ ...evidence.validation.code ? { code: redactValue(evidence.validation.code) } : {}
20850
+ }
20851
+ } : {}
20852
+ };
20853
+ }
20854
+ function sanitizeArtifactText(value, maxLength) {
20855
+ return typeof value === "string" ? value.slice(0, maxLength) : undefined;
20856
+ }
20857
+ function isUnitIntervalNumber(value) {
20858
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
20859
+ }
20860
+ function isRecord(value) {
20861
+ return typeof value === "object" && value !== null && !Array.isArray(value);
20862
+ }
20809
20863
 
20810
20864
  class ToolLoopError extends Error {
20811
20865
  caseResult;
@@ -21218,12 +21272,54 @@ function execGit(command, allowFailure = false) {
21218
21272
 
21219
21273
  // src/artifacts/types.ts
21220
21274
  function getCaseEvaluationStatus(caseResult) {
21221
- if (caseResult.status)
21275
+ if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
21222
21276
  return caseResult.status;
21277
+ }
21223
21278
  if (caseResult.ok)
21224
21279
  return "passed";
21225
21280
  return caseResult.error ? "error" : "failed";
21226
21281
  }
21282
+ function assertRunManifestIntegrity(manifest) {
21283
+ if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
21284
+ throw new Error("Invalid run manifest: expected an object with a cases array");
21285
+ }
21286
+ for (const [index, caseResult] of manifest.cases.entries()) {
21287
+ if (!isRecord2(caseResult)) {
21288
+ throw new Error(`Invalid run manifest: case ${index} is not an object`);
21289
+ }
21290
+ if (caseResult.status !== undefined && caseResult.status !== "passed" && caseResult.status !== "failed" && caseResult.status !== "invalid" && caseResult.status !== "error") {
21291
+ throw new Error(`Invalid run manifest: case ${index} has an unknown status`);
21292
+ }
21293
+ if (caseResult.evidence !== undefined) {
21294
+ assertCaseEvaluationEvidence(caseResult.evidence, index);
21295
+ }
21296
+ }
21297
+ }
21298
+ function assertCaseEvaluationEvidence(evidence, caseIndex) {
21299
+ if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
21300
+ throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
21301
+ }
21302
+ if (evidence.score !== undefined && !isUnitIntervalNumber2(evidence.score)) {
21303
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence score`);
21304
+ }
21305
+ if (evidence.threshold !== undefined && !isUnitIntervalNumber2(evidence.threshold)) {
21306
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence threshold`);
21307
+ }
21308
+ if (evidence.model !== undefined && (typeof evidence.model !== "string" || evidence.model.length > 200)) {
21309
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence model`);
21310
+ }
21311
+ if (evidence.validation !== undefined) {
21312
+ if (!isRecord2(evidence.validation) || evidence.validation.status !== "valid" && evidence.validation.status !== "invalid" || evidence.validation.code !== undefined && (typeof evidence.validation.code !== "string" || evidence.validation.code.length > 100)) {
21313
+ throw new Error(`Invalid run manifest: case ${caseIndex} has invalid evidence validation`);
21314
+ }
21315
+ }
21316
+ }
21317
+ function isUnitIntervalNumber2(value) {
21318
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
21319
+ }
21320
+ function isRecord2(value) {
21321
+ return typeof value === "object" && value !== null && !Array.isArray(value);
21322
+ }
21227
21323
  function isRunManifest(manifest) {
21228
21324
  return !("type" in manifest) || manifest.type === undefined;
21229
21325
  }
@@ -21416,6 +21512,7 @@ async function runScenario(options) {
21416
21512
  const redactor = new Redactor(effectiveRedaction);
21417
21513
  const promptsRedacted = results.filter((r) => r.redaction?.promptRedacted).length;
21418
21514
  const responsesRedacted = results.filter((r) => r.redaction?.responseRedacted).length;
21515
+ const reasonsRedacted = results.filter((r) => r.redaction?.reasonRedacted).length;
21419
21516
  const totalRedactions = results.reduce((sum, r) => sum + (r.redaction?.redactionCount ?? 0), 0);
21420
21517
  redactionInfo = {
21421
21518
  enabled: true,
@@ -21424,6 +21521,7 @@ async function runScenario(options) {
21424
21521
  summary: {
21425
21522
  promptsRedacted,
21426
21523
  responsesRedacted,
21524
+ reasonsRedacted,
21427
21525
  totalRedactions
21428
21526
  }
21429
21527
  };
@@ -21509,6 +21607,9 @@ class LocalStorageAdapter {
21509
21607
  this.baselinesPath = join2(this.basePath, ".artemis", "baselines.json");
21510
21608
  }
21511
21609
  async save(manifest) {
21610
+ if (isRunManifest(manifest)) {
21611
+ assertRunManifestIntegrity(manifest);
21612
+ }
21512
21613
  const dir = join2(this.basePath, manifest.project);
21513
21614
  await mkdir(dir, { recursive: true });
21514
21615
  const filePath = join2(dir, `${manifest.run_id}.json`);
@@ -21531,6 +21632,7 @@ class LocalStorageAdapter {
21531
21632
  if (getManifestType(manifest) !== "run") {
21532
21633
  throw new Error(`Run ${runId} is not a standard run manifest`);
21533
21634
  }
21635
+ assertRunManifestIntegrity(manifest);
21534
21636
  return manifest;
21535
21637
  }
21536
21638
  async loadRedTeam(runId) {
@@ -30258,6 +30360,7 @@ class SupabaseStorageAdapter {
30258
30360
  this.project = project || "default";
30259
30361
  }
30260
30362
  async save(manifest) {
30363
+ assertRunManifestIntegrity(manifest);
30261
30364
  const filePath = `${manifest.project}/${manifest.run_id}.json`;
30262
30365
  const { error: uploadError } = await this.client.storage.from(this.bucket).upload(filePath, JSON.stringify(manifest, null, 2), {
30263
30366
  contentType: "application/json",
@@ -30311,7 +30414,9 @@ class SupabaseStorageAdapter {
30311
30414
  throw new Error(`Failed to download manifest: ${downloadError?.message}`);
30312
30415
  }
30313
30416
  const text = await data.text();
30314
- return JSON.parse(text);
30417
+ const manifest = JSON.parse(text);
30418
+ assertRunManifestIntegrity(manifest);
30419
+ return manifest;
30315
30420
  }
30316
30421
  async list(options) {
30317
30422
  let query = this.client.from("runs").select("run_id, scenario, success_rate, started_at").order("started_at", { ascending: false });
@@ -32509,6 +32614,7 @@ export {
32509
32614
  VariablesSchema,
32510
32615
  actionBudgetExceeded,
32511
32616
  adapterRegistry,
32617
+ assertRunManifestIntegrity,
32512
32618
  createAdapter,
32513
32619
  createDefaultRedactor,
32514
32620
  createNoOpRedactor,
package/dist/runner/runner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAyHzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA2HzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
package/dist/storage/local.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Local filesystem storage adapter
3
3
  */
4
- import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
4
+ import { type AnyManifest, type RedTeamManifest, type RunManifest, type StressManifest } from '../artifacts/types';
5
5
  import type { BaselineMetadata, BaselineStorageAdapter, ComparisonResult, ListOptions, RunListItem } from './types';
6
6
  export declare class LocalStorageAdapter implements BaselineStorageAdapter {
7
7
  private basePath;
package/dist/storage/local.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/storage/local.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpG,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,WAAW,EACX,WAAW,EACZ,MAAM,SAAS,CAAC;AAyDjB,qBAAa,mBAAoB,YAAW,sBAAsB;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;gBAElB,QAAQ,SAAmB;IAKjC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAU5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAczC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAQ5C,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQpD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQlD,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA0DnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAiBjE,eAAe;YASf,SAAS;IAWvB;;OAEG;YACW,iBAAiB;IAS/B;;OAEG;YACW,iBAAiB;IAM/B;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4B3F;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKrE;;OAEG;IACG,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMzE;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAOlD;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUxD;;OAEG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5D;;OAEG;IACG,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;CAwBV"}
1
+ {"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/storage/local.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,cAAc,EAGpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,WAAW,EACX,WAAW,EACZ,MAAM,SAAS,CAAC;AAyDjB,qBAAa,mBAAoB,YAAW,sBAAsB;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;gBAElB,QAAQ,SAAmB;IAKjC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAa5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAczC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAS5C,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQpD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQlD,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA0DnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAiBjE,eAAe;YASf,SAAS;IAWvB;;OAEG;YACW,iBAAiB;IAS/B;;OAEG;YACW,iBAAiB;IAM/B;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4B3F;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKrE;;OAEG;IACG,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMzE;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAOlD;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUxD;;OAEG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5D;;OAEG;IACG,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;CAwBV"}
package/dist/storage/supabase.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"supabase.d.ts","sourceRoot":"","sources":["../../src/storage/supabase.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAmB,KAAK,WAAW,EAA2B,MAAM,oBAAoB,CAAC;AAChG,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,mBAAmB,EACnB,WAAW,EACX,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD,qBAAa,sBAAuB,YAAW,uBAAuB;IACpE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,MAAM;IAUrD,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAuD5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAuBzC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAiCnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAepC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkBzE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkDrF,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA4B/D,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA2BnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA2B5C,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAclD,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUtD,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;IAoCH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiCzD,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoC/D,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAiC1D,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwD9E,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAgC/D,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsCxE,kBAAkB,CACtB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAyC5B,qBAAqB,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC;CA0E5B"}
1
+ {"version":3,"file":"supabase.d.ts","sourceRoot":"","sources":["../../src/storage/supabase.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAEL,KAAK,WAAW,EAGjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,mBAAmB,EACnB,WAAW,EACX,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD,qBAAa,sBAAuB,YAAW,uBAAuB;IACpE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,MAAM;IAUrD,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAwD5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAyBzC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAiCnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAepC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkBzE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkDrF,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA4B/D,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA2BnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA2B5C,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAclD,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUtD,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;IAoCH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiCzD,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoC/D,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAiC1D,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwD9E,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAgC/D,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsCxE,kBAAkB,CACtB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAyC5B,qBAAqB,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC;CA0E5B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/core",
3
- "version": "0.4.0",
3
+ "version": "0.4.1",
4
4
  "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
package/src/artifacts/manifest.test.ts CHANGED
@@ -4,6 +4,7 @@
4
4
 
5
5
  import { describe, expect, test } from 'bun:test';
6
6
  import { createRunManifest } from './manifest';
7
+ import { assertRunManifestIntegrity } from './types';
7
8
  import type { CaseResult } from './types';
8
9
 
9
10
  describe('createRunManifest', () => {
@@ -250,4 +251,27 @@ describe('createRunManifest', () => {
250
251
  success_rate: 1,
251
252
  });
252
253
  });
254
+
255
+ test('accepts historical cases without integrity fields but rejects malformed new evidence', () => {
256
+ const historical = createRunManifest({
257
+ project: 'test-project',
258
+ config: { scenario: 'legacy', provider: 'openai' },
259
+ cases: [{ ...mockCases[0], status: undefined, evidence: undefined }],
260
+ startTime: new Date(),
261
+ endTime: new Date(),
262
+ });
263
+ expect(() => assertRunManifestIntegrity(historical)).not.toThrow();
264
+
265
+ const malformed = {
266
+ ...historical,
267
+ cases: [
268
+ {
269
+ ...mockCases[0],
270
+ status: 'passed',
271
+ evidence: { evaluator: 'custom', validation: { status: 'unknown' } },
272
+ },
273
+ ],
274
+ };
275
+ expect(() => assertRunManifestIntegrity(malformed)).toThrow('invalid evidence validation');
276
+ });
253
277
  });
package/src/artifacts/types.ts CHANGED
@@ -16,6 +16,8 @@ export interface CaseRedactionInfo {
16
16
  promptRedacted: boolean;
17
17
  /** Whether response was redacted */
18
18
  responseRedacted: boolean;
19
+ /** Whether evaluator reason text was redacted */
20
+ reasonRedacted?: boolean;
19
21
  /** Number of redactions in this case */
20
22
  redactionCount: number;
21
23
  }
@@ -34,6 +36,8 @@ export interface ManifestRedactionInfo {
34
36
  summary: {
35
37
  promptsRedacted: number;
36
38
  responsesRedacted: number;
39
+ /** Cases whose evaluator reason text was redacted. */
40
+ reasonsRedacted?: number;
37
41
  totalRedactions: number;
38
42
  };
39
43
  }
@@ -284,11 +288,91 @@ export interface RunManifest {
284
288
  * criteria unless they set the legacy `error` field.
285
289
  */
286
290
  export function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationStatus {
287
- if (caseResult.status) return caseResult.status;
291
+ if (
292
+ caseResult.status === 'passed' ||
293
+ caseResult.status === 'failed' ||
294
+ caseResult.status === 'invalid' ||
295
+ caseResult.status === 'error'
296
+ ) {
297
+ return caseResult.status;
298
+ }
288
299
  if (caseResult.ok) return 'passed';
289
300
  return caseResult.error ? 'error' : 'failed';
290
301
  }
291
302
 
303
+ /**
304
+ * Reject untrusted integrity-bearing fields before a run manifest is persisted
305
+ * or treated as a standard run. Historical manifests remain supported because
306
+ * status and evidence are optional in the v1.0 contract.
307
+ */
308
+ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest is RunManifest {
309
+ if (!isRecord(manifest) || !Array.isArray(manifest.cases)) {
310
+ throw new Error('Invalid run manifest: expected an object with a cases array');
311
+ }
312
+
313
+ for (const [index, caseResult] of manifest.cases.entries()) {
314
+ if (!isRecord(caseResult)) {
315
+ throw new Error(`Invalid run manifest: case ${index} is not an object`);
316
+ }
317
+
318
+ if (
319
+ caseResult.status !== undefined &&
320
+ caseResult.status !== 'passed' &&
321
+ caseResult.status !== 'failed' &&
322
+ caseResult.status !== 'invalid' &&
323
+ caseResult.status !== 'error'
324
+ ) {
325
+ throw new Error(`Invalid run manifest: case ${index} has an unknown status`);
326
+ }
327
+
328
+ if (caseResult.evidence !== undefined) {
329
+ assertCaseEvaluationEvidence(caseResult.evidence, index);
330
+ }
331
+ }
332
+ }
333
+
334
+ function assertCaseEvaluationEvidence(evidence: unknown, caseIndex: number): void {
335
+ if (
336
+ !isRecord(evidence) ||
337
+ typeof evidence.evaluator !== 'string' ||
338
+ evidence.evaluator.length > 100
339
+ ) {
340
+ throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
341
+ }
342
+
343
+ if (evidence.score !== undefined && !isUnitIntervalNumber(evidence.score)) {
344
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence score`);
345
+ }
346
+ if (evidence.threshold !== undefined && !isUnitIntervalNumber(evidence.threshold)) {
347
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence threshold`);
348
+ }
349
+ if (
350
+ evidence.model !== undefined &&
351
+ (typeof evidence.model !== 'string' || evidence.model.length > 200)
352
+ ) {
353
+ throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence model`);
354
+ }
355
+
356
+ if (evidence.validation !== undefined) {
357
+ if (
358
+ !isRecord(evidence.validation) ||
359
+ (evidence.validation.status !== 'valid' && evidence.validation.status !== 'invalid') ||
360
+ (evidence.validation.code !== undefined &&
361
+ (typeof evidence.validation.code !== 'string' || evidence.validation.code.length > 100))
362
+ ) {
363
+ throw new Error(`Invalid run manifest: case ${caseIndex} has invalid evidence validation`);
364
+ }
365
+ }
366
+ }
367
+
368
+ function isUnitIntervalNumber(value: unknown): value is number {
369
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
370
+ }
371
+
372
+ function isRecord(value: unknown): value is Record<string, unknown> {
373
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
374
+ }
375
+
292
376
  // ============================================================================
293
377
  // Red Team Manifest Types
294
378
  // ============================================================================
package/src/runner/executor.test.ts CHANGED
@@ -337,4 +337,38 @@ describe('executeCase measurement integrity', () => {
337
337
  expect(JSON.stringify(result)).not.toContain('secret judge transcript');
338
338
  expect(JSON.stringify(result)).not.toContain('secret rubric');
339
339
  });
340
+
341
+ it('redacts evaluator reason text and discards malformed runtime evidence', async () => {
342
+ registerEvaluator('custom', {
343
+ type: 'custom',
344
+ evaluate: async () =>
345
+ ({
346
+ passed: false,
347
+ score: 0.4,
348
+ status: 'invalid-status',
349
+ reason: 'judge_error: token=super-secret-token',
350
+ evidence: {
351
+ threshold: '0.7',
352
+ model: 'reviewer@example.com',
353
+ validation: { status: 'unknown', code: 'token=super-secret-token' },
354
+ },
355
+ }) as never,
356
+ });
357
+ const redactedScenario = ScenarioSchema.parse({
358
+ ...scenario,
359
+ redaction: { enabled: true, patterns: ['email', 'secrets'] },
360
+ });
361
+
362
+ const result = await executeCase(redactedScenario.cases[0], {
363
+ client,
364
+ scenario: redactedScenario,
365
+ });
366
+
367
+ expect(result.status).toBe('failed');
368
+ expect(result.reason).toContain('[REDACTED]');
369
+ expect(result.evidence).toEqual({ evaluator: 'custom', score: 0.4, model: '[REDACTED]' });
370
+ expect(result.redaction).toMatchObject({ redacted: true, reasonRedacted: true });
371
+ expect(JSON.stringify(result)).not.toContain('super-secret-token');
372
+ expect(JSON.stringify(result)).not.toContain('reviewer@example.com');
373
+ });
340
374
  });
package/src/runner/executor.ts CHANGED
@@ -353,6 +353,8 @@ async function executeCaseAttempt(
353
353
  // Apply redaction if enabled
354
354
  let finalPrompt: string | object = testCase.prompt;
355
355
  let finalResponse = result.text;
356
+ let finalReason = sanitizeArtifactText(evalResult.reason, 1000);
357
+ let finalEvidence = sanitizeEvidence(testCase.expected.type, evalResult);
356
358
  let redactionInfo: CaseRedactionInfo | undefined;
357
359
 
358
360
  if (effectiveRedaction.enabled) {
@@ -360,6 +362,7 @@ async function executeCaseAttempt(
360
362
 
361
363
  let promptRedacted = false;
362
364
  let responseRedacted = false;
365
+ let reasonRedacted = false;
363
366
  let totalRedactions = 0;
364
367
 
365
368
  // Redact prompt if configured
@@ -393,10 +396,25 @@ async function executeCaseAttempt(
393
396
  totalRedactions += responseResult.redactionCount;
394
397
  }
395
398
 
399
+ // Evaluator reasons can contain judge output or provider error text. They
400
+ // are retained evidence, so they are always redacted when run redaction is
401
+ // enabled, independent of the legacy metadata toggle.
402
+ if (finalReason) {
403
+ const reasonResult = redactor.redact(finalReason);
404
+ finalReason = reasonResult.text;
405
+ reasonRedacted = reasonResult.wasRedacted;
406
+ totalRedactions += reasonResult.redactionCount;
407
+ }
408
+ finalEvidence = redactEvidence(finalEvidence, redactor, (count) => {
409
+ if (count > 0) reasonRedacted = true;
410
+ totalRedactions += count;
411
+ });
412
+
396
413
  redactionInfo = {
397
- redacted: promptRedacted || responseRedacted,
414
+ redacted: promptRedacted || responseRedacted || reasonRedacted,
398
415
  promptRedacted,
399
416
  responseRedacted,
417
+ reasonRedacted,
400
418
  redactionCount: totalRedactions,
401
419
  };
402
420
  }
@@ -408,7 +426,7 @@ async function executeCaseAttempt(
408
426
  status: evaluationStatus(evalResult),
409
427
  score: validScore(evalResult.score),
410
428
  matcherType: testCase.expected.type,
411
- reason: evalResult.reason,
429
+ reason: finalReason,
412
430
  latencyMs: generationMetrics.latencyMs,
413
431
  tokens: generationMetrics.tokens,
414
432
  prompt: finalPrompt,
@@ -416,7 +434,7 @@ async function executeCaseAttempt(
416
434
  expected: testCase.expected,
417
435
  tags: testCase.tags,
418
436
  redaction: redactionInfo,
419
- evidence: sanitizeEvidence(testCase.expected.type, evalResult),
437
+ evidence: finalEvidence,
420
438
  toolTrace: toolTrace.length ? toolTrace : undefined,
421
439
  toolLoop,
422
440
  };
@@ -451,9 +469,12 @@ function createToolLoopError(
451
469
 
452
470
  function evaluationStatus(result: {
453
471
  passed: boolean;
454
- status?: 'passed' | 'failed' | 'invalid';
472
+ status?: unknown;
455
473
  }): CaseEvaluationStatus {
456
- return result.status ?? (result.passed ? 'passed' : 'failed');
474
+ if (result.status === 'passed' || result.status === 'failed' || result.status === 'invalid') {
475
+ return result.status;
476
+ }
477
+ return result.passed ? 'passed' : 'failed';
457
478
  }
458
479
 
459
480
  function validScore(score: number): number {
@@ -463,37 +484,74 @@ function validScore(score: number): number {
463
484
  function sanitizeEvidence(
464
485
  evaluator: string,
465
486
  result: {
466
- score: number;
467
- evidence?: {
468
- threshold?: number;
469
- model?: string;
470
- validation?: { status: 'valid' | 'invalid'; code?: string };
471
- };
487
+ score: unknown;
488
+ evidence?: unknown;
472
489
  }
473
490
  ): CaseEvaluationEvidence {
474
- const evidence: CaseEvaluationEvidence = { evaluator };
475
- const score = validScore(result.score);
476
- if (Number.isFinite(result.score) && result.score >= 0 && result.score <= 1) {
477
- evidence.score = score;
491
+ const evidence: CaseEvaluationEvidence = {
492
+ evaluator: sanitizeArtifactText(evaluator, 100) ?? 'unknown',
493
+ };
494
+ if (isUnitIntervalNumber(result.score)) {
495
+ evidence.score = result.score;
478
496
  }
479
- if (
480
- result.evidence?.threshold !== undefined &&
481
- validScore(result.evidence.threshold) === result.evidence.threshold
482
- ) {
497
+ if (!isRecord(result.evidence)) return evidence;
498
+
499
+ if (isUnitIntervalNumber(result.evidence.threshold)) {
483
500
  evidence.threshold = result.evidence.threshold;
484
501
  }
485
- if (result.evidence?.model) evidence.model = result.evidence.model.slice(0, 200);
486
- if (result.evidence?.validation) {
502
+ if (typeof result.evidence.model === 'string' && result.evidence.model) {
503
+ evidence.model = sanitizeArtifactText(result.evidence.model, 200);
504
+ }
505
+ if (isRecord(result.evidence.validation)) {
506
+ const status = result.evidence.validation.status;
507
+ const code = result.evidence.validation.code;
508
+ if (status !== 'valid' && status !== 'invalid') return evidence;
487
509
  evidence.validation = {
488
- status: result.evidence.validation.status,
489
- ...(result.evidence.validation.code
490
- ? { code: result.evidence.validation.code.slice(0, 100) }
491
- : {}),
510
+ status,
511
+ ...(typeof code === 'string' && code ? { code: sanitizeArtifactText(code, 100) } : {}),
492
512
  };
493
513
  }
494
514
  return evidence;
495
515
  }
496
516
 
517
+ function redactEvidence(
518
+ evidence: CaseEvaluationEvidence,
519
+ redactor: Redactor,
520
+ onRedactions: (count: number) => void
521
+ ): CaseEvaluationEvidence {
522
+ const redactValue = (value: string | undefined): string | undefined => {
523
+ if (!value) return value;
524
+ const result = redactor.redact(value);
525
+ onRedactions(result.redactionCount);
526
+ return result.text;
527
+ };
528
+
529
+ return {
530
+ ...evidence,
531
+ ...(evidence.model ? { model: redactValue(evidence.model) } : {}),
532
+ ...(evidence.validation
533
+ ? {
534
+ validation: {
535
+ ...evidence.validation,
536
+ ...(evidence.validation.code ? { code: redactValue(evidence.validation.code) } : {}),
537
+ },
538
+ }
539
+ : {}),
540
+ };
541
+ }
542
+
543
+ function sanitizeArtifactText(value: unknown, maxLength: number): string | undefined {
544
+ return typeof value === 'string' ? value.slice(0, maxLength) : undefined;
545
+ }
546
+
547
+ function isUnitIntervalNumber(value: unknown): value is number {
548
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
549
+ }
550
+
551
+ function isRecord(value: unknown): value is Record<string, unknown> {
552
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
553
+ }
554
+
497
555
  class ToolLoopError extends Error {
498
556
  constructor(
499
557
  message: string,
package/src/runner/runner.ts CHANGED
@@ -93,6 +93,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
93
93
  const redactor = new Redactor(effectiveRedaction);
94
94
  const promptsRedacted = results.filter((r) => r.redaction?.promptRedacted).length;
95
95
  const responsesRedacted = results.filter((r) => r.redaction?.responseRedacted).length;
96
+ const reasonsRedacted = results.filter((r) => r.redaction?.reasonRedacted).length;
96
97
  const totalRedactions = results.reduce((sum, r) => sum + (r.redaction?.redactionCount ?? 0), 0);
97
98
 
98
99
  redactionInfo = {
@@ -102,6 +103,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
102
103
  summary: {
103
104
  promptsRedacted,
104
105
  responsesRedacted,
106
+ reasonsRedacted,
105
107
  totalRedactions,
106
108
  },
107
109
  };
package/src/storage/local.test.ts CHANGED
@@ -77,6 +77,30 @@ describe('LocalStorageAdapter', () => {
77
77
  expect(loaded.metrics.success_rate).toBe(0.8);
78
78
  });
79
79
 
80
+ test('rejects malformed evaluator evidence before writing a standard run', async () => {
81
+ const malformed = {
82
+ ...mockManifest,
83
+ run_id: 'invalid-evidence',
84
+ cases: [
85
+ {
86
+ id: 'case-1',
87
+ ok: false,
88
+ score: 0,
89
+ matcherType: 'custom',
90
+ latencyMs: 1,
91
+ tokens: { prompt: 0, completion: 0, total: 0 },
92
+ prompt: 'prompt',
93
+ response: 'response',
94
+ expected: {},
95
+ tags: [],
96
+ evidence: { evaluator: 'custom', threshold: 2 },
97
+ },
98
+ ],
99
+ } as RunManifest;
100
+
101
+ await expect(storage.save(malformed)).rejects.toThrow('invalid evidence threshold');
102
+ });
103
+
80
104
  test('throws error for non-existent run', async () => {
81
105
  await expect(storage.load('non-existent-run')).rejects.toThrow('Run not found');
82
106
  });
package/src/storage/local.ts CHANGED
@@ -4,7 +4,14 @@
4
4
 
5
5
  import { mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises';
6
6
  import { join, resolve } from 'node:path';
7
- import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
7
+ import {
8
+ type AnyManifest,
9
+ type RedTeamManifest,
10
+ type RunManifest,
11
+ type StressManifest,
12
+ assertRunManifestIntegrity,
13
+ isRunManifest,
14
+ } from '../artifacts/types';
8
15
  import type {
9
16
  BaselineMetadata,
10
17
  BaselineStorageAdapter,
@@ -78,6 +85,9 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
78
85
  }
79
86
 
80
87
  async save(manifest: AnyManifest): Promise<string> {
88
+ if (isRunManifest(manifest)) {
89
+ assertRunManifestIntegrity(manifest);
90
+ }
81
91
  const dir = join(this.basePath, manifest.project);
82
92
  await mkdir(dir, { recursive: true });
83
93
 
@@ -106,7 +116,8 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
106
116
  if (getManifestType(manifest) !== 'run') {
107
117
  throw new Error(`Run ${runId} is not a standard run manifest`);
108
118
  }
109
- return manifest as RunManifest;
119
+ assertRunManifestIntegrity(manifest);
120
+ return manifest;
110
121
  }
111
122
 
112
123
  async loadRedTeam(runId: string): Promise<RedTeamManifest> {
package/src/storage/supabase.test.ts CHANGED
@@ -241,6 +241,22 @@ describe('SupabaseStorageAdapter', () => {
241
241
 
242
242
  await expect(adapter.save(manifest)).rejects.toThrow('Failed to save run metadata');
243
243
  });
244
+
245
+ it('rejects malformed evaluator evidence before uploading a manifest', async () => {
246
+ const manifest = createMockManifest({
247
+ cases: [
248
+ {
249
+ ...createMockCaseResult('case-invalid-evidence', false),
250
+ evidence: { evaluator: 'custom', validation: { status: 'unknown' } },
251
+ } as never,
252
+ ],
253
+ });
254
+ const upload = mock(() => Promise.resolve({ error: null }));
255
+ mockStorage.mockReturnValue({ upload });
256
+
257
+ await expect(adapter.save(manifest)).rejects.toThrow('invalid evidence validation');
258
+ expect(upload).not.toHaveBeenCalled();
259
+ });
244
260
  });
245
261
 
246
262
  describe('load', () => {
@@ -288,6 +304,36 @@ describe('SupabaseStorageAdapter', () => {
288
304
 
289
305
  await expect(adapter.load('non-existent')).rejects.toThrow('Run not found');
290
306
  });
307
+
308
+ it('rejects a downloaded manifest with malformed evaluator evidence', async () => {
309
+ const manifest = createMockManifest({
310
+ cases: [
311
+ {
312
+ ...createMockCaseResult('case-invalid-evidence', false),
313
+ evidence: { evaluator: 'custom', score: Number.NaN },
314
+ } as never,
315
+ ],
316
+ });
317
+ mockFrom.mockReturnValue({
318
+ select: mock(() => ({
319
+ eq: mock(() => ({
320
+ single: mock(() =>
321
+ Promise.resolve({
322
+ data: { manifest_path: 'test-project/test-run-123.json' },
323
+ error: null,
324
+ })
325
+ ),
326
+ })),
327
+ })),
328
+ });
329
+ mockStorage.mockReturnValue({
330
+ download: mock(() =>
331
+ Promise.resolve({ data: new Blob([JSON.stringify(manifest)]), error: null })
332
+ ),
333
+ });
334
+
335
+ await expect(adapter.load('test-run-123')).rejects.toThrow('invalid evidence score');
336
+ });
291
337
  });
292
338
 
293
339
  describe('list', () => {
package/src/storage/supabase.ts CHANGED
@@ -3,7 +3,12 @@
3
3
  */
4
4
 
5
5
  import { type SupabaseClient, createClient } from '@supabase/supabase-js';
6
- import { type CaseResult, type RunManifest, getCaseEvaluationStatus } from '../artifacts/types';
6
+ import {
7
+ type CaseResult,
8
+ type RunManifest,
9
+ assertRunManifestIntegrity,
10
+ getCaseEvaluationStatus,
11
+ } from '../artifacts/types';
7
12
  import type {
8
13
  AnalyticsStorageAdapter,
9
14
  BaselineMetadata,
@@ -63,6 +68,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
63
68
  // ============================================================================
64
69
 
65
70
  async save(manifest: RunManifest): Promise<string> {
71
+ assertRunManifestIntegrity(manifest);
66
72
  const filePath = `${manifest.project}/${manifest.run_id}.json`;
67
73
 
68
74
  const { error: uploadError } = await this.client.storage
@@ -137,7 +143,9 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
137
143
  }
138
144
 
139
145
  const text = await data.text();
140
- return JSON.parse(text);
146
+ const manifest: unknown = JSON.parse(text);
147
+ assertRunManifestIntegrity(manifest);
148
+ return manifest;
141
149
  }
142
150
 
143
151
  async list(options?: ListOptions): Promise<RunListItem[]> {