@vercel/agent-eval 1.2.0 → 1.3.0

package/dist/lib/fingerprint.d.ts

@@ -5,11 +5,49 @@
  * If the fingerprint matches and the result is valid, the eval can be skipped.
  */
 import type { RunnableExperimentConfig } from './types.js';
+/**
+ * Compute a fingerprint of ONLY the eval's files (no config).
+ *
+ * This is the "did the eval itself change" signal: it changes when an eval is
+ * edited or re-synced with different content, but NOT when a benign config field
+ * (e.g. timeout) is bumped. Result reuse/refingerprinting uses it to carry forward
+ * config-only changes without ever masking a real eval change. Hex SHA-256.
+ */
+export declare function computeContentFingerprint(evalPath: string): string;
 /**
  * Compute a fingerprint for an (eval, config) pair.
  *
  * Hashes: all eval directory files + config fields that affect results.
- * Returns a hex SHA-256 digest.
+ * Returns a hex SHA-256 digest. The file-hashing is identical to
+ * {@link computeContentFingerprint}; config is appended after, so this value is
+ * byte-for-byte unchanged from before the content/config split (existing cached
+ * fingerprints stay valid).
  */
 export declare function computeFingerprint(evalPath: string, config: RunnableExperimentConfig): string;
+/** What `refingerprint` should do to one cached result's summary. */
+export interface RefingerprintDecision {
+    /** New combined fingerprint to write, or undefined to leave as-is. */
+    fingerprint?: string;
+    /** New content fingerprint to write, or undefined to leave as-is. */
+    contentFingerprint?: string;
+    /** True when the eval content changed and the result was left stale (not re-stamped). */
+    stale: boolean;
+}
+/**
+ * Decide how to refingerprint one cached result — the content-aware replacement for
+ * the old "re-stamp everything" behavior. The whole point: carry forward benign
+ * config changes WITHOUT masking a real eval-content change.
+ *
+ *   - content unchanged → carry the new combined fingerprint (a config-only change)
+ *   - content changed    → leave it stale (don't touch); honest "this eval changed"
+ *   - legacy (no stored content fp) → adopt the current content fp only if the result
+ *     is already fully current; otherwise leave stale (we can't prove content matches)
+ */
+export declare function decideRefingerprint(stored: {
+    fingerprint?: string;
+    contentFingerprint?: string;
+}, current: {
+    fingerprint: string;
+    contentFingerprint: string;
+}): RefingerprintDecision;
 //# sourceMappingURL=fingerprint.d.ts.map

package/dist/lib/fingerprint.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAyC3D;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAiC7F"}
+{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAyD3D;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIlE;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAiC7F;AAED,qEAAqE;AACrE,MAAM,WAAW,qBAAqB;IACpC,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,yFAAyF;IACzF,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAAE,EAC7D,OAAO,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAA;CAAE,GAC3D,qBAAqB,CAcvB"}

package/dist/lib/fingerprint.js

@@ -30,20 +30,45 @@ function collectFiles(dir, basePath = '') {
     return files;
 }
 /**
- * Compute a fingerprint for an (eval, config) pair.
- *
- * Hashes: all eval directory files + config fields that affect results.
- * Returns a hex SHA-256 digest.
+ * Hash every eval-directory file into the given hash, in deterministic order.
+ * Shared by {@link computeContentFingerprint} and {@link computeFingerprint} so
+ * the two stay in lockstep — the combined fingerprint's file bytes are byte-for-byte
+ * what the content fingerprint hashes, just with config appended.
  */
-export function computeFingerprint(evalPath, config) {
-    const hash = createHash('sha256');
-    // Hash all files in the eval directory (sorted for determinism)
+function hashEvalFiles(hash, evalPath) {
     const files = collectFiles(evalPath);
     for (const file of files) {
         hash.update(`file:${file.relativePath}\n`);
         hash.update(file.content);
         hash.update('\0');
     }
+}
+/**
+ * Compute a fingerprint of ONLY the eval's files (no config).
+ *
+ * This is the "did the eval itself change" signal: it changes when an eval is
+ * edited or re-synced with different content, but NOT when a benign config field
+ * (e.g. timeout) is bumped. Result reuse/refingerprinting uses it to carry forward
+ * config-only changes without ever masking a real eval change. Hex SHA-256.
+ */
+export function computeContentFingerprint(evalPath) {
+    const hash = createHash('sha256');
+    hashEvalFiles(hash, evalPath);
+    return hash.digest('hex');
+}
+/**
+ * Compute a fingerprint for an (eval, config) pair.
+ *
+ * Hashes: all eval directory files + config fields that affect results.
+ * Returns a hex SHA-256 digest. The file-hashing is identical to
+ * {@link computeContentFingerprint}; config is appended after, so this value is
+ * byte-for-byte unchanged from before the content/config split (existing cached
+ * fingerprints stay valid).
+ */
+export function computeFingerprint(evalPath, config) {
+    const hash = createHash('sha256');
+    // Hash all files in the eval directory (sorted for determinism)
+    hashEvalFiles(hash, evalPath);
     // Hash config fields that affect results
     const configForHash = {
         agent: config.agent,
@@ -63,7 +88,37 @@ export function computeFingerprint(evalPath, config) {
     if (config.webResearch) {
         configForHash.webResearch = true;
     }
+    // Included only when the judge is pinned, so existing (unpinned) fingerprints are
+    // unchanged. Changing the judge agent/model invalidates the cache → re-runs.
+    if (config.judge) {
+        configForHash.judge = { agent: config.judge.agent, model: config.judge.model };
+    }
     hash.update(`config:${JSON.stringify(configForHash)}`);
     return hash.digest('hex');
 }
+/**
+ * Decide how to refingerprint one cached result — the content-aware replacement for
+ * the old "re-stamp everything" behavior. The whole point: carry forward benign
+ * config changes WITHOUT masking a real eval-content change.
+ *
+ *   - content unchanged → carry the new combined fingerprint (a config-only change)
+ *   - content changed    → leave it stale (don't touch); honest "this eval changed"
+ *   - legacy (no stored content fp) → adopt the current content fp only if the result
+ *     is already fully current; otherwise leave stale (we can't prove content matches)
+ */
+export function decideRefingerprint(stored, current) {
+    if (stored.contentFingerprint === undefined) {
+        if (stored.fingerprint === current.fingerprint) {
+            return { contentFingerprint: current.contentFingerprint, stale: false };
+        }
+        return { stale: true };
+    }
+    if (stored.contentFingerprint === current.contentFingerprint) {
+        if (stored.fingerprint !== current.fingerprint) {
+            return { fingerprint: current.fingerprint, stale: false };
+        }
+        return { stale: false };
+    }
+    return { stale: true };
+}
 //# sourceMappingURL=fingerprint.js.map

package/dist/lib/fingerprint.js.map

@@ -1 +1 @@
-{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAkB5B;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,WAAmB,EAAE;IACtD,MAAM,KAAK,GAAqD,EAAE,CAAC;IACnE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,KAAK,cAAc,IAAI,KAAK,KAAK,MAAM;YAAE,SAAS;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAgC;IACnF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,gEAAgE;IAChE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,yCAAyC;IACzC,MAAM,aAAa,GAA0B;QAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACnC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC;IACF,IAAI,MAAM,CAAC,WAAW,KAAK,gBAAgB,EAAE,CAAC;QAC5C,aAAa,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACjD,CAAC;IACD,wEAAwE;IACxE,mEAAmE;IACnE,qEAAqE;IACrE,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;IACnC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAEvD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}
+{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/lib/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAmB5B;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,WAAmB,EAAE;IACtD,MAAM,KAAK,GAAqD,EAAE,CAAC;IACnE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,KAAK,cAAc,IAAI,KAAK,KAAK,MAAM;YAAE,SAAS;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,IAAmC,EAAE,QAAgB;IAC1E,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAgB;IACxD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAgC;IACnF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,gEAAgE;IAChE,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAE9B,yCAAyC;IACzC,MAAM,aAAa,GAA0B;QAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACnC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC;IACF,IAAI,MAAM,CAAC,WAAW,KAAK,gBAAgB,EAAE,CAAC;QAC5C,aAAa,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACjD,CAAC;IACD,wEAAwE;IACxE,mEAAmE;IACnE,qEAAqE;IACrE,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;IACnC,CAAC;IACD,kFAAkF;IAClF,6EAA6E;IAC7E,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,aAAa,CAAC,KAAK,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAEvD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAA6D,EAC7D,OAA4D;IAE5D,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,WAAW,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC;YAC/C,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC1E,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7D,IAAI,MAAM,CAAC,WAAW,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC;YAC/C,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC5D,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC"}

package/dist/lib/results.d.ts

@@ -25,6 +25,10 @@ export interface SaveResultsOptions {
     experimentName: string;
     /** Per-eval fingerprints (eval name -> fingerprint hash) */
     fingerprints?: Record<string, string>;
+    /** Per-eval content fingerprints (eval name -> eval-files-only hash). Lets
+     * refingerprinting carry forward config-only changes without masking a real
+     * eval change, and lets staleness be detected against the eval content alone. */
+    contentFingerprints?: Record<string, string>;
     /** Per-eval validity flags (eval name -> valid). Defaults to true. */
     validity?: Record<string, boolean>;
     /** Whether this is a smoke test run. Smoke results are excluded from reuse. */
@@ -81,7 +85,7 @@ export interface ReusableResult {
  * Scan existing results for an experiment to find reusable eval results.
  *
  * A result is reusable if:
- * 1. Its fingerprint matches the current fingerprint
+ * 1. Its fingerprint matches the current fingerprint (fresh)
  * 2. It is "valid" (not marked as invalid by the classifier)
  * 3. It has passedRuns > 0 (successful result worth reusing)
  *

package/dist/lib/results.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,OAAO,KAAK,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACxB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAKxD;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,WAAW,EAAE,cAAc,GACzB,WAAW,CAyCb;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EAAE,GACpB,WAAW,CAab;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GACf,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6FAA6F;IAC7F,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GACzB,MAAM,CAgMR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8DrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CASR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAClC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CA4E7B"}
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,OAAO,KAAK,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACxB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAKxD;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,WAAW,EAAE,cAAc,GACzB,WAAW,CAyCb;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EAAE,GACpB,WAAW,CAab;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GACf,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC;;qFAEiF;IACjF,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6FAA6F;IAC7F,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GACzB,MAAM,CAoMR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8DrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CASR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAClC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CA8E7B"}

package/dist/lib/results.js

@@ -96,6 +96,7 @@ export function saveResults(results, options) {
         mkdirSync(evalDir, { recursive: true });
         // Save summary (simplified format per design)
         const fingerprint = options.fingerprints?.[evalSummary.name];
+        const contentFingerprint = options.contentFingerprints?.[evalSummary.name];
         const valid = options.validity?.[evalSummary.name];
         const summaryForFile = {
             totalRuns: evalSummary.totalRuns,
@@ -106,6 +107,9 @@ export function saveResults(results, options) {
         if (fingerprint) {
             summaryForFile.fingerprint = fingerprint;
         }
+        if (contentFingerprint) {
+            summaryForFile.contentFingerprint = contentFingerprint;
+        }
         if (valid === false) {
             summaryForFile.valid = false;
         }
@@ -292,7 +296,7 @@ export function createProgressDisplay(evalName, runNumber, totalRuns, context) {
  * Scan existing results for an experiment to find reusable eval results.
  *
  * A result is reusable if:
- * 1. Its fingerprint matches the current fingerprint
+ * 1. Its fingerprint matches the current fingerprint (fresh)
  * 2. It is "valid" (not marked as invalid by the classifier)
  * 3. It has passedRuns > 0 (successful result worth reusing)
  *
@@ -336,7 +340,9 @@ export function scanReusableResults(resultsDir, experimentName, fingerprints) {
             const summaryPath = join(tsDir, evalDir, 'summary.json');
             try {
                 const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
-                // Check fingerprint match
+                // Reuse only when fresh (fingerprint matches). A changed eval is left to
+                // re-run; "which staleness is acceptable" is the consumer's policy (via
+                // `agent-eval status --json` in CI), not the framework's.
                 if (summary.fingerprint !== expectedFingerprint)
                     continue;
                 // Check validity (valid defaults to true if not explicitly set to false)

package/dist/lib/results.js.map

@@ -1 +1 @@
-{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACN,SAAS,EACT,aAAa,EACb,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,GACV,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACvC,WAA2B;IAE3B,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,qEAAqE;IACrE,IACC,WAAW,CAAC,cAAc;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EACjD,CAAC;QACF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1C,WAAW,CAAC,cAAc,CAC1B,EAAE,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAkB;QAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACjD,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;KAC5D,CAAC;IACF,IAAI,WAAW,CAAC,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;IAClD,CAAC;IAED,OAAO;QACN,MAAM;QACN,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EACZ,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAClE,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;KACtC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACN,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACL,CAAC;AACH,CAAC;AAoBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CAC1B,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CACzB,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,cAAc,EACtB,SAAS,CACT,CAAC;IAEF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC/C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACtC,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YACjB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACrB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,aAAa,CACZ,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACvC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;YAClE,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YACxD,MAAM,KAAK,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAChH,MAAM,eAAe,GAEjB;gBACH,GAAG,UAAU;gBACb,KAAK;aACL,CAAC;YACF,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;gBACtC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC1C,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC;gBAChD,eAAe,CAAC,aAAa,GAAG,aAAa,CAAC;YAC/C,CAAC;YAED,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CACjC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACL,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EACpC,OAAO,CAAC,UAAU,CAClB,CAAC;gBACF,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC3C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC3B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChC,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAC5B,OAAO,CAAC,aAAa,CAAC,IAAI,CAC1B,CAAC;oBACF,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBACzC,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC3C,OAAO,CAAC,aAAa,CAAC,OAAO,CAC7B,EAAE,CAAC;wBACH,IAAI,OAAO,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAClB,IAAI,CACJ,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBACrC,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IACC,WAAW,CAAC,IAAI;oBAChB,CAAC,WAAW,CAAC,OAAO;wBACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC5C,CAAC;oBACF,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3C,MAAM,iBAAiB,GACtB,OAAO,CAAC,cAAc;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,MAAM,eAAe,GACpB,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IACC,SAAS;gBACT,SAAS,KAAK,MAAM;gBACpB,CAAC,iBAAiB,IAAI,eAAe,CAAC,EACrC,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3C,oDAAoD;gBACpD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,WAAW,EAAE,CAAC;wBACjB,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;wBACnD,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,YAAY,EAAE,CAAC;4BAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;4BAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAClC,CAAC;oBACF,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBAC1B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;4BAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC1B,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACtB,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC/C,OAAO,CAAC,cAAc,CACtB,EAAE,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,QAAQ,GACb,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,SAAS,GACd,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;YAC/C,CAAC,CAAC,KAAK,CAAC,KAAK;YACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEd,KAAK,CAAC,IAAI,CACT,SAAS,CACR,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IACzD,WAAW,CAAC,SACb,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC/C,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACT,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1D,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GACjB,eAAe,KAAK,GAAG;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,eAAe,IAAI,EAAE;YACvB,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACd,KAAK,CAAC,IAAI,CACT,YAAY,CACX,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CACtE,CAAC,CACD,IAAI,CACL,CACD,CAAC;IAEF,MAAM,QAAQ,GACb,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC;IACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,OAAqE;IAErE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IAEZ,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IAClE,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,IAAI,CACjB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAClE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,KAAK,CAAC,GAAG,CAChB,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EACrC,EAAE,CACF,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACpC,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,OAAqE;IAErE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IACZ,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,OAAO,KAAK,CAAC,IAAI,CAChB,WAAW,MAAM,KAAK,SAAS,IAAI,SAAS,IAAI,MAAM,KAAK,CAC3D,CAAC;AACH,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAClC,UAAkB,EAClB,cAAsB,EACtB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACJ,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAE1D,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;oBAAE,SAAS;gBAErC,kEAAkE;gBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAAE,SAAS;gBAEtD,oEAAoE;gBACpE,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC;oBAAE,SAAS;gBAErC,4EAA4E;gBAC5E,iFAAiF;gBACjF,IACC,OAAO,CAAC,UAAU,KAAK,CAAC;oBACxB,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;oBAExD,SAAS;gBAEV,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACrB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACT,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,yBAAyB;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC"}
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACN,SAAS,EACT,aAAa,EACb,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,GACV,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACvC,WAA2B;IAE3B,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,qEAAqE;IACrE,IACC,WAAW,CAAC,cAAc;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EACjD,CAAC;QACF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1C,WAAW,CAAC,cAAc,CAC1B,EAAE,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAkB;QAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACjD,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;KAC5D,CAAC;IACF,IAAI,WAAW,CAAC,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;IAClD,CAAC;IAED,OAAO;QACN,MAAM;QACN,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EACZ,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAClE,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;KACtC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACN,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACL,CAAC;AACH,CAAC;AAwBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CAC1B,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CACzB,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,cAAc,EACtB,SAAS,CACT,CAAC;IAEF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC/C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACtC,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YACjB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACxB,cAAc,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QACxD,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACrB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,aAAa,CACZ,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACvC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;YAClE,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YACxD,MAAM,KAAK,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAChH,MAAM,eAAe,GAEjB;gBACH,GAAG,UAAU;gBACb,KAAK;aACL,CAAC;YACF,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;gBACtC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC1C,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC;gBAChD,eAAe,CAAC,aAAa,GAAG,aAAa,CAAC;YAC/C,CAAC;YAED,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CACjC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACL,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EACpC,OAAO,CAAC,UAAU,CAClB,CAAC;gBACF,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC3C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC3B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChC,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAC5B,OAAO,CAAC,aAAa,CAAC,IAAI,CAC1B,CAAC;oBACF,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBACzC,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC3C,OAAO,CAAC,aAAa,CAAC,OAAO,CAC7B,EAAE,CAAC;wBACH,IAAI,OAAO,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAClB,IAAI,CACJ,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBACrC,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IACC,WAAW,CAAC,IAAI;oBAChB,CAAC,WAAW,CAAC,OAAO;wBACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC5C,CAAC;oBACF,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3C,MAAM,iBAAiB,GACtB,OAAO,CAAC,cAAc;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,MAAM,eAAe,GACpB,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IACC,SAAS;gBACT,SAAS,KAAK,MAAM;gBACpB,CAAC,iBAAiB,IAAI,eAAe,CAAC,EACrC,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3C,oDAAoD;gBACpD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,WAAW,EAAE,CAAC;wBACjB,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;wBACnD,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,YAAY,EAAE,CAAC;4BAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;4BAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAClC,CAAC;oBACF,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBAC1B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;4BAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC1B,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACtB,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC/C,OAAO,CAAC,cAAc,CACtB,EAAE,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,QAAQ,GACb,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,SAAS,GACd,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;YAC/C,CAAC,CAAC,KAAK,CAAC,KAAK;YACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEd,KAAK,CAAC,IAAI,CACT,SAAS,CACR,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IACzD,WAAW,CAAC,SACb,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC/C,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACT,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1D,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GACjB,eAAe,KAAK,GAAG;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,eAAe,IAAI,EAAE;YACvB,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACd,KAAK,CAAC,IAAI,CACT,YAAY,CACX,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CACtE,CAAC,CACD,IAAI,CACL,CACD,CAAC;IAEF,MAAM,QAAQ,GACb,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC;IACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,OAAqE;IAErE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IAEZ,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IAClE,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,IAAI,CACjB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAClE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,KAAK,CAAC,GAAG,CAChB,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EACrC,EAAE,CACF,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACpC,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,OAAqE;IAErE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IACZ,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,OAAO,KAAK,CAAC,IAAI,CAChB,WAAW,MAAM,KAAK,SAAS,IAAI,SAAS,IAAI,MAAM,KAAK,CAC3D,CAAC;AACH,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAClC,UAAkB,EAClB,cAAsB,EACtB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACJ,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,yEAAyE;gBACzE,wEAAwE;gBACxE,0DAA0D;gBAC1D,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAE1D,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;oBAAE,SAAS;gBAErC,kEAAkE;gBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAAE,SAAS;gBAEtD,oEAAoE;gBACpE,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC;oBAAE,SAAS;gBAErC,4EAA4E;gBAC5E,iFAAiF;gBACjF,IACC,OAAO,CAAC,UAAU,KAAK,CAAC;oBACxB,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;oBAExD,SAAS;gBAEV,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACrB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACT,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,yBAAyB;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC"}

package/dist/lib/runner.d.ts

@@ -40,6 +40,8 @@ export interface RunExperimentOptions {
     experimentName: string;
     /** Per-eval fingerprints (eval name -> hash) for result reuse */
     fingerprints?: Record<string, string>;
+    /** Per-eval content fingerprints (eval name -> eval-files-only hash) */
+    contentFingerprints?: Record<string, string>;
     /** Callback for progress updates */
     onProgress?: (event: ProgressEvent) => void;
     /** Whether to run in verbose mode */
@@ -70,5 +72,6 @@ export declare function runSingleEval<T extends ResolvedExperimentConfig['model'
     verbose?: boolean;
     agentOptions?: ResolvedExperimentConfig['agentOptions'];
     webResearch?: ResolvedExperimentConfig['webResearch'];
+    judge?: ResolvedExperimentConfig['judge'];
 }): Promise<T extends Array<unknown> ? EvalRunData[] : EvalRunData>;
 //# sourceMappingURL=runner.d.ts.map

package/dist/lib/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAwP5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,wBAAwB,CAAC,YAAY,CAAC,CAAC;IACpD,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;CACvD,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CAiCjE"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CA0P5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,wBAAwB,CAAC,YAAY,CAAC,CAAC;IACpD,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;IACtD,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;CAC3C,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CAkCjE"}

package/dist/lib/runner.js

@@ -58,7 +58,7 @@ export class StartRateLimiter {
  * With earlyExit enabled, remaining attempts for a fixture are aborted once one passes.
  */
 export async function runExperiment(options) {
-    const { config, fixtures, apiKey, resultsDir, experimentName, fingerprints, onProgress, smoke, rateLimiter } = options;
+    const { config, fixtures, apiKey, resultsDir, experimentName, fingerprints, contentFingerprints, onProgress, smoke, rateLimiter } = options;
     const startedAt = new Date();
     // Get the agent from registry
     const agent = getAgent(config.agent);
@@ -120,6 +120,7 @@ export async function runExperiment(options) {
                 sandbox: config.sandbox,
                 agentOptions: config.agentOptions,
                 webResearch: config.webResearch,
+                judge: config.judge,
             }),
             new Promise((_, reject) => {
                 timeoutId = setTimeout(() => {
@@ -260,6 +261,7 @@ export async function runExperiment(options) {
         resultsDir,
         experimentName,
         fingerprints,
+        contentFingerprints,
         smoke,
         fixturePaths,
     });
@@ -288,6 +290,7 @@ export async function runSingleEval(fixture, options) {
             sandbox: options.sandbox,
             agentOptions: options.agentOptions,
             webResearch: options.webResearch,
+            judge: options.judge,
         });
         results.push(agentResultToEvalRunData(agentResult));
     }

package/dist/lib/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAMR;IACA;IANX,KAAK,GAAmB,EAAE,CAAC;IAC3B,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAA0C,IAAI,CAAC;IAE5D,YACmB,eAAuB,EACvB,QAAgB;QADhB,oBAAe,GAAf,eAAe,CAAQ;QACvB,aAAQ,GAAR,QAAQ,CAAQ;IAChC,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;YACjB,qDAAqD;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;gBACpE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,EAAE,CAAC;YACxB,CAAC;YACD,uCAAuC;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC;gBAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;CACF;AA8CD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACvH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,CAAC,KAAoB,EAAE,EAAE;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAExH,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;QAC1D,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACnF,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAC9E,KAAK,EAAE,cAAc;gBACrB,WAAW;gBACX,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEpD,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;oBAChD,OAAO;oBACP,QAAQ;oBACR,MAAM;oBACN,OAAO;iBACR,CAAC,CAAC;gBACH,IAAI,cAAc,EAAE,CAAC;oBACnB,OAAO,GAAG,cAAc,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG;oBACR,GAAG,OAAO;oBACV,MAAM,EAAE;wBACN,GAAG,OAAO,CAAC,MAAM;wBACjB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;qBACzF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,sGAAsG;IACtG,oEAAoE;IACpE,MAAM,WAAW,GAAG,CAAC,CAAC;IACtB,MAAM,kBAAkB,GAAG,KAAK,CAAC;IACjC,MAAM,mBAAmB,GAAG,CAAC,CAAC;IAE9B,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACjF,KAAK,IAAI,KAAK,GAAG,CAAC,GAAI,KAAK,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;YAEzC,MAAM,kBAAkB,GACtB,CAAC,MAAM,CAAC,OAAO;gBACf,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ;gBACzC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,mBAAmB;gBACpD,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEtD,IAAI,CAAC,kBAAkB,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;gBAChD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,GAAG,GAAG,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,4EAA4E;IAC5E,MAAM,MAAM,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACpE,sDAAsD;QACtD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,WAAW,CAAC,WAAW,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAExJ,4EAA4E;YAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,KAAK,EAAE,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAExD,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5C,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;QACd,YAAY;QACZ,KAAK;QACL,YAAY;KACb,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9C,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAEjE,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OAaC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAa,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAExF,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACjD,MAAM;YACN,KAAK;YACL,WAAW,EAAE,eAAe;YAC5B,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;YAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,OAAO,CAAC,WAAW;SAChC,CAAC,CAAC;QAEA,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,gHAAgH;IAChH,IAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC,CAAC,CAA2D,CAAC;IAC3E,CAAC;IAED,OAAO,OAAiE,CAAC;AAC3E,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAMR;IACA;IANX,KAAK,GAAmB,EAAE,CAAC;IAC3B,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAA0C,IAAI,CAAC;IAE5D,YACmB,eAAuB,EACvB,QAAgB;QADhB,oBAAe,GAAf,eAAe,CAAQ;QACvB,aAAQ,GAAR,QAAQ,CAAQ;IAChC,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;YACjB,qDAAqD;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;gBACpE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,EAAE,CAAC;YACxB,CAAC;YACD,uCAAuC;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC;gBAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;CACF;AAgDD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,mBAAmB,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAC5I,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,CAAC,KAAoB,EAAE,EAAE;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAExH,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;QAC1D,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACnF,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAC9E,KAAK,EAAE,cAAc;gBACrB,WAAW;gBACX,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEpD,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;oBAChD,OAAO;oBACP,QAAQ;oBACR,MAAM;oBACN,OAAO;iBACR,CAAC,CAAC;gBACH,IAAI,cAAc,EAAE,CAAC;oBACnB,OAAO,GAAG,cAAc,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG;oBACR,GAAG,OAAO;oBACV,MAAM,EAAE;wBACN,GAAG,OAAO,CAAC,MAAM;wBACjB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;qBACzF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,sGAAsG;IACtG,oEAAoE;IACpE,MAAM,WAAW,GAAG,CAAC,CAAC;IACtB,MAAM,kBAAkB,GAAG,KAAK,CAAC;IACjC,MAAM,mBAAmB,GAAG,CAAC,CAAC;IAE9B,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACjF,KAAK,IAAI,KAAK,GAAG,CAAC,GAAI,KAAK,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;YAEzC,MAAM,kBAAkB,GACtB,CAAC,MAAM,CAAC,OAAO;gBACf,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ;gBACzC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,mBAAmB;gBACpD,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEtD,IAAI,CAAC,kBAAkB,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;gBAChD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,GAAG,GAAG,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,4EAA4E;IAC5E,MAAM,MAAM,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACpE,sDAAsD;QACtD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,WAAW,CAAC,WAAW,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAExJ,4EAA4E;YAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,KAAK,EAAE,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAExD,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5C,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;QACd,YAAY;QACZ,mBAAmB;QACnB,KAAK;QACL,YAAY;KACb,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9C,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAEjE,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OAcC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAa,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAExF,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACjD,MAAM;YACN,KAAK;YACL,WAAW,EAAE,eAAe;YAC5B,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;YAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;SACpB,CAAC,CAAC;QAEA,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,gHAAgH;IAChH,IAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC,CAAC,CAA2D,CAAC;IAC3E,CAAC;IAED,OAAO,OAAiE,CAAC;AAC3E,CAAC"}

package/dist/lib/types.d.ts

@@ -70,6 +70,21 @@ export type RunCompleteHook = (context: RunCompleteContext) => Promise<EvalRunDa
  * Sandbox backend type.
  */
 export type SandboxBackend = 'vercel' | 'docker';
+/**
+ * Configures the agentic LLM judge used by `expect(environment|transcript)`
+ * matchers in EVAL.ts. By default the judge runs as the SAME agent+model that
+ * generated the code (self-grading). Pin it here to grade every model with one
+ * fixed judge — the apples-to-apples choice for cross-model dashboards.
+ */
+export interface JudgeConfig {
+    /** Judge harness. Defaults to the codegen agent (same harness, model pinned).
+     * Set a Claude variant (e.g. 'vercel-ai-gateway/claude-code') to judge codex,
+     * gemini, etc. runs with Claude. */
+    agent?: AgentType;
+    /** Model the judge grades with (e.g. 'claude-opus-4-8'). Required — pinning the
+     * judge model is the whole point. */
+    model: ModelTier;
+}
 /**
  * Experiment configuration.
  * Defines what to test and how.
@@ -115,6 +130,9 @@ export interface ExperimentConfig {
     brands?: BrandConfig[];
     /** Optional hook for custom post-run analysis before results are saved. */
     onRunComplete?: RunCompleteHook;
+    /** Pin the agentic LLM judge to a fixed agent+model. @default undefined
+     * (judge self-grades with the codegen agent+model). */
+    judge?: JudgeConfig;
 }
 /**
  * Resolved experiment config with all defaults applied.
@@ -137,6 +155,7 @@ export interface ResolvedExperimentConfig {
     webResearch?: boolean;
     brands?: BrandConfig[];
     onRunComplete?: RunCompleteHook;
+    judge?: JudgeConfig;
 }
 /**
  * Resolved experiment config with all defaults applied.
@@ -159,6 +178,7 @@ export interface RunnableExperimentConfig {
     webResearch?: boolean;
     brands?: BrandConfig[];
     onRunComplete?: RunCompleteHook;
+    judge?: JudgeConfig;
 }
 /**
  * Required files for a valid eval fixture.

package/dist/lib/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,GAC5B,QAAQ,GACR,QAAQ,CAAC;AAEb;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,gBAAgB,CAAC;AAE7D,kFAAkF;AAClF,eAAO,MAAM,oBAAoB,EAAG,gBAAyB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;IAC9B,gEAAgE;IAChE,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,wBAAwB,CAAC;IACjC,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GAAG,CAC5B,OAAO,EAAE,kBAAkB,KACxB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;gFAC4E;IAC5E,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,kEAAkE;IAClE,UAAU,CAAC,EAAE,cAAc,CAAC;IAE5B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAExC;;;;yBAIqB;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IAEvC;6EACyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;0EAEsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,4EAA4E;IAC5E,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAEvB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,wGAAwG;IACxG,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,GAC5B,QAAQ,GACR,QAAQ,CAAC;AAEb;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,gBAAgB,CAAC;AAE7D,kFAAkF;AAClF,eAAO,MAAM,oBAAoB,EAAG,gBAAyB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;IAC9B,gEAAgE;IAChE,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,wBAAwB,CAAC;IACjC,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GAAG,CAC5B,OAAO,EAAE,kBAAkB,KACxB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B;;wCAEoC;IACpC,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB;yCACqC;IACrC,KAAK,EAAE,SAAS,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;gFAC4E;IAC5E,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,kEAAkE;IAClE,UAAU,CAAC,EAAE,cAAc,CAAC;IAE5B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAExC;;;;yBAIqB;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IAEvC;6EACyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;0EAEsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,4EAA4E;IAC5E,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAEvB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,eAAe,CAAC;IAEhC;2DACuD;IACvD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,wGAAwG;IACxG,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}

package/dist/lib/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,kFAAkF;AAClF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gBAAyB,CAAC;AAyK9D;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,kFAAkF;AAClF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gBAAyB,CAAC;AA+L9D;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/agent-eval",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Framework for testing AI coding agents in isolated sandboxes",
   "repository": {
     "type": "git",