@nexus-cortex/core 4.130.0 → 4.131.1

package/dist/training/AutoResearchGuards.js

@@ -0,0 +1,81 @@
+import { normalQuantile } from './AutoResearchConfirm.js';
+function round4(n) { return Math.round(n * 10000) / 10000; }
+/** Student-t quantile via the Cornish–Fisher expansion (df >= 1; exact enough for bounds). */
+export function tQuantile(p, df) {
+    const z = normalQuantile(p);
+    if (!Number.isFinite(df) || df > 1e6)
+        return z;
+    const z3 = z ** 3, z5 = z ** 5, z7 = z ** 7;
+    return z + (z3 + z) / (4 * df) + (5 * z5 + 16 * z3 + 3 * z) / (96 * df ** 2)
+        + (3 * z7 + 19 * z5 + 17 * z3 - 15 * z) / (384 * df ** 3);
+}
+function variance(xs) {
+    const m = xs.reduce((a, b) => a + b, 0) / xs.length;
+    return xs.reduce((a, b) => a + (b - m) ** 2, 0) / (xs.length - 1);
+}
+const avg = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+/** Per-family effects: mean over the family's tasks of (candMean - baseMean), its SE from
+ *  the sample variances, Welch–Satterthwaite df, and the one-sided upper bound at
+ *  alpha / nFamilies. */
+export function familyEffects(tasks, opts = {}) {
+    const minRuns = opts.minRunsPerArm ?? 2;
+    const usable = tasks.filter((t) => t.baseScores.length >= minRuns && t.candScores.length >= minRuns);
+    const groups = new Map();
+    for (const t of usable) {
+        const f = t.taskFamily ?? 'unlabeled';
+        groups.set(f, [...(groups.get(f) ?? []), t]);
+    }
+    const nFam = Math.max(1, groups.size);
+    const alpha = opts.alpha ?? 0.05;
+    const out = [];
+    for (const [family, arms] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const k = arms.length;
+        const effect = avg(arms.map((t) => avg(t.candScores) - avg(t.baseScores)));
+        // Var(effect) = Σ_t (s_b²/n_b + s_c²/n_c) / k²; components feed Welch–Satterthwaite.
+        const comps = arms.flatMap((t) => [
+            { v: variance(t.baseScores) / t.baseScores.length / (k * k), df: t.baseScores.length - 1 },
+            { v: variance(t.candScores) / t.candScores.length / (k * k), df: t.candScores.length - 1 },
+        ]);
+        const vSum = comps.reduce((a, c) => a + c.v, 0);
+        const dfDen = comps.reduce((a, c) => a + (c.v * c.v) / c.df, 0);
+        const df = vSum === 0 || dfDen === 0 ? Infinity : (vSum * vSum) / dfDen;
+        const se = Math.sqrt(vSum);
+        const upper = effect + tQuantile(1 - alpha / nFam, Math.max(1, df)) * se;
+        out.push({ family, nTasks: k, effect: round4(effect), se: round4(se), df: Number.isFinite(df) ? round4(df) : 1e9, upper: round4(upper) });
+    }
+    return out;
+}
+function verdict(scope, family, effect, se, q, margin) {
+    const upper = round4(effect + q * se);
+    return { family, scope, effect: round4(effect), se: round4(se), upper, margin, blocked: effect < -margin && upper < 0 };
+}
+/** The STEP guard: one verdict per family of this candidate vs its base. */
+export function stepGuards(families, opts = {}) {
+    const p = 1 - (opts.alpha ?? 0.05) / Math.max(1, families.length);
+    return families.map((f) => verdict('step', f.family, f.effect, f.se, tQuantile(p, Math.max(1, f.df)), opts.margin ?? 2));
+}
+/** The TOTAL guard: the ledger plus this step, per family this step measured. */
+export function totalGuards(ledger, families, opts = {}) {
+    const p = 1 - (opts.alpha ?? 0.05) / Math.max(1, families.length);
+    return families.map((f) => {
+        const prior = ledger.get(f.family);
+        // conservative df for the sum: the smallest df of any step in it
+        const df = Math.max(1, Math.min(f.df, prior?.df ?? Infinity));
+        return verdict('total', f.family, (prior?.effect ?? 0) + f.effect, Math.sqrt((prior?.variance ?? 0) + f.se ** 2), tQuantile(p, df), opts.totalMargin ?? 5);
+    });
+}
+/** Add a merged step's family effects to the drift ledger (mutates and returns it). */
+export function recordDrift(ledger, families) {
+    for (const f of families) {
+        const prior = ledger.get(f.family) ?? { effect: 0, variance: 0, steps: 0, df: Infinity };
+        ledger.set(f.family, { effect: round4(prior.effect + f.effect), variance: prior.variance + f.se ** 2, steps: prior.steps + 1, df: Math.min(prior.df, f.df) });
+    }
+    return ledger;
+}
+/** One-line summary of the blocking guards (empty when none block). */
+export function describeGuardBlocks(vs) {
+    return vs.filter((v) => v.blocked)
+        .map((v) => `${v.family} ${v.scope === 'step' ? 'step' : 'total'} ${v.effect} (upper ${v.upper} < 0, margin -${v.margin})`)
+        .join('; ');
+}
+//# sourceMappingURL=AutoResearchGuards.js.map

package/dist/training/AutoResearchGuards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutoResearchGuards.js","sourceRoot":"","sources":["../../src/training/AutoResearchGuards.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAyC1D,SAAS,MAAM,CAAC,CAAS,IAAY,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAE5E,8FAA8F;AAC9F,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,EAAU;IAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,GAAG;QAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;UACxE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,EAAqB;IACrC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpD,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC;AACD,MAAM,GAAG,GAAG,CAAC,EAAqB,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AAEjF;;yBAEyB;AACzB,MAAM,UAAU,aAAa,CAAC,KAA0B,EAAE,OAAqB,EAAE;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,WAAW,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3E,qFAAqF;QACrF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAChC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1F,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;SAC3F,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;QACxE,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACzE,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5I,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CAAC,KAAuB,EAAE,MAAc,EAAE,MAAc,EAAE,EAAU,EAAE,CAAS,EAAE,MAAc;IAC7G,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;AAC1H,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,UAAU,CAAC,QAAiC,EAAE,OAAqB,EAAE;IACnF,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3H,CAAC;AAKD,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,MAAmB,EAAE,QAAiC,EAAE,OAAqB,EAAE;IACzG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnC,iEAAiE;QACjE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,QAAQ,CAAC,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC7J,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,WAAW,CAAC,MAAmB,EAAE,QAAiC;IAChF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;QACzF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChK,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,mBAAmB,CAAC,EAA2B;IAC7D,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,KAAK,iBAAiB,CAAC,CAAC,MAAM,GAAG,CAAC;SAC1H,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/training/BenchRunner.d.ts

@@ -193,6 +193,8 @@ export interface BenchTaskSummary {
     runs: number;
     passRate: number;
     meanScore: number;
+    /** sample SD of the per-run qualitative scores (run-count planning). */
+    scoreSd?: number;
 }
 export interface BenchSummary {
     experimentTag: string;
@@ -203,13 +205,27 @@ export interface BenchSummary {
     /** Number of deficiencies seeded into `opts.backlog` this run (0 when no sink). */
     seededDeficiencies: number;
 }
+export declare function runBench(tasks: TaskSpec[], runner: HarnessRunner, matrix: ModelRouterMatrix, opts: RunBenchOptions): Promise<BenchSummary>;
+export interface PairedBenchArm {
+    runner: HarnessRunner;
+    /** this arm's per-arm bench options (harnessRef, strategy, …); the shared fields come
+     *  from the `opts` passed to runBenchPaired. */
+    harnessRef?: string;
+}
 /**
- * Run each task `runs` times through `runner`, grade each output, and write a
- * REAL scored BenchmarkRecord to the matrix (harnessRef auto-stamped unless
- * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
- * comparability key the gate joins on).
+ * Bench a BASE and a CANDIDATE arm INTERLEAVED: for each task and each run index the
+ * two arms run back to back, in a seeded random order. Provider drift (latency, load,
+ * a silent model update, rate limiting) then lands on both arms instead of all on the
+ * one that ran second. Records are identical to two runBench calls (same matrix rows,
+ * same summaries); only the run order differs.
  */
-export declare function runBench(tasks: TaskSpec[], runner: HarnessRunner, matrix: ModelRouterMatrix, opts: RunBenchOptions): Promise<BenchSummary>;
+export declare function runBenchPaired(tasks: TaskSpec[], base: PairedBenchArm, candidate: PairedBenchArm, matrix: ModelRouterMatrix, opts: RunBenchOptions & {
+    orderSeed?: number;
+}): Promise<{
+    base: BenchSummary;
+    candidate: BenchSummary;
+    order: ('base' | 'candidate')[];
+}>;
 /** Validate a parsed task-set JSON value, throwing on malformed entries. */
 export declare function parseTaskSet(raw: unknown, source?: string): TaskSpec[];
 //# sourceMappingURL=BenchRunner.d.ts.map

package/dist/training/BenchRunner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"BenchRunner.d.ts","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,sBAAsB,CAAC;AAMpE,MAAM,MAAM,QAAQ;AAClB,yDAAyD;AACvD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE;AAC1D,6DAA6D;GAC3D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD;;8EAE8E;GAC5E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE;AAChE,oEAAoE;GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACvC;;;;;;;6CAO6C;GAC3C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,UAAU,GAAG,UAAU,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;AAC3H;;;;;;;+EAO+E;GAC7E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE/G,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CA2E9H;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB;+EAC2E;IAC3E,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC3E;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC;CAChC;AAMD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8CAA8C;IAC9C,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACjG;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,mFAAmF;IACnF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,YAAY,CAAC,CA+FvB;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAe,GAAG,QAAQ,EAAE,CAmC5E"}
+{"version":3,"file":"BenchRunner.d.ts","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAI3D,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,sBAAsB,CAAC;AAMpE,MAAM,MAAM,QAAQ;AAClB,yDAAyD;AACvD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE;AAC1D,6DAA6D;GAC3D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD;;8EAE8E;GAC5E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE;AAChE,oEAAoE;GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACvC;;;;;;;6CAO6C;GAC3C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,UAAU,GAAG,UAAU,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;AAC3H;;;;;;;+EAO+E;GAC7E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE/G,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CA2E9H;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB;+EAC2E;IAC3E,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC3E;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC;CAChC;AAMD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8CAA8C;IAC9C,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACjG;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,mFAAmF;IACnF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AA8HD,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,YAAY,CAAC,CASvB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,aAAa,CAAC;IACtB;oDACgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,QAAQ,EAAE,EACjB,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,cAAc,EACzB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,CAAA;CAAE,CAAC,CAsB3F;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAe,GAAG,QAAQ,EAAE,CAmC5E"}

package/dist/training/BenchRunner.js

@@ -21,6 +21,8 @@
  */
 import { ModelRouterMatrix } from './ModelRouterMatrix.js';
 import { classifyTask } from './TaskClassifier.js';
+import { mulberry32 } from './AutoResearchStats.js';
+import { sampleSd } from './AutoResearchConfirm.js';
 function normalizeWs(s) {
     return s.replace(/\s+/g, ' ').trim().toLowerCase();
 }
@@ -117,50 +119,67 @@ export async function gradeRun(output, verifier, judge, extras) {
  * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
  * comparability key the gate joins on).
  */
-export async function runBench(tasks, runner, matrix, opts) {
-    const runs = opts.runs ?? 2;
-    const split = opts.split ?? 'train';
-    const taskSummaries = [];
-    let totalRuns = 0;
-    let seededDeficiencies = 0;
-    for (const task of tasks) {
-        const taskType = task.taskType ?? classifyTask(task.prompt).taskType;
-        const taskFingerprint = ModelRouterMatrix.fingerprintTask(task.prompt, JSON.stringify(task.verifier));
-        let passes = 0;
-        let scoreSum = 0;
-        let sampleFailure;
-        let observedModelId = opts.modelId;
-        for (let i = 0; i < runs; i++) {
-            const res = await runner.run(task.prompt, { model: opts.modelId });
-            const grade = await gradeRun(res.text, task.verifier, opts.judge, { toolUses: res.toolUses });
-            if (!grade.pass && sampleFailure === undefined)
-                sampleFailure = res.text;
-            observedModelId = res.modelId ?? observedModelId;
-            passes += grade.pass ? 1 : 0;
-            scoreSum += grade.qualitativeScore;
-            totalRuns++;
-            matrix.record({
-                modelId: res.modelId ?? opts.modelId ?? 'unknown',
-                taskType,
-                toolCallCount: res.toolCallCount,
-                inputTokens: res.inputTokens,
-                outputTokens: res.outputTokens,
-                latencyMs: res.latencyMs,
-                pass: grade.pass,
-                qualitativeScore: grade.qualitativeScore,
-                taskFingerprint,
-                experimentTag: opts.experimentTag,
-                split,
-                benchmarkSource: opts.benchmarkSource,
-                ...(opts.harnessRef ? { harnessRef: opts.harnessRef } : {}),
-                ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
-                ...(opts.strategy ? { strategy: opts.strategy } : {}),
-                ...(opts.toolProfile && opts.toolProfile !== 'full' ? { toolProfile: opts.toolProfile } : {}),
-            });
-            opts.onRun?.({ taskId: task.id, run: i + 1, pass: grade.pass, qualitativeScore: grade.qualitativeScore });
-        }
-        const passRate = Math.round((passes / runs) * 1000) / 1000;
-        const meanScore = Math.round((scoreSum / runs) * 100) / 100;
+/** Per-task accumulator shared by runBench and runBenchPaired: one arm × one task. */
+class TaskBench {
+    task;
+    runner;
+    matrix;
+    opts;
+    taskType;
+    taskFingerprint;
+    passes = 0;
+    scores = [];
+    sampleFailure;
+    observedModelId;
+    constructor(task, runner, matrix, opts) {
+        this.task = task;
+        this.runner = runner;
+        this.matrix = matrix;
+        this.opts = opts;
+        this.taskType = task.taskType ?? classifyTask(task.prompt).taskType;
+        this.taskFingerprint = ModelRouterMatrix.fingerprintTask(task.prompt, JSON.stringify(task.verifier));
+        this.observedModelId = opts.modelId;
+    }
+    /** One run: execute, grade, record to the matrix, notify. */
+    async runOnce() {
+        const { task, opts } = this;
+        const res = await this.runner.run(task.prompt, { model: opts.modelId });
+        const grade = await gradeRun(res.text, task.verifier, opts.judge, { toolUses: res.toolUses });
+        if (!grade.pass && this.sampleFailure === undefined)
+            this.sampleFailure = res.text;
+        this.observedModelId = res.modelId ?? this.observedModelId;
+        this.passes += grade.pass ? 1 : 0;
+        this.scores.push(grade.qualitativeScore);
+        this.matrix.record({
+            modelId: res.modelId ?? opts.modelId ?? 'unknown',
+            taskType: this.taskType,
+            toolCallCount: res.toolCallCount,
+            inputTokens: res.inputTokens,
+            outputTokens: res.outputTokens,
+            latencyMs: res.latencyMs,
+            pass: grade.pass,
+            qualitativeScore: grade.qualitativeScore,
+            taskFingerprint: this.taskFingerprint,
+            experimentTag: opts.experimentTag,
+            split: opts.split ?? 'train',
+            benchmarkSource: opts.benchmarkSource,
+            ...(opts.harnessRef ? { harnessRef: opts.harnessRef } : {}),
+            ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
+            ...(opts.strategy ? { strategy: opts.strategy } : {}),
+            ...(opts.toolProfile && opts.toolProfile !== 'full' ? { toolProfile: opts.toolProfile } : {}),
+        });
+        opts.onRun?.({ taskId: task.id, run: this.scores.length, pass: grade.pass, qualitativeScore: grade.qualitativeScore });
+    }
+    /** Summarize (and seed the backlog on a train failure). Returns the seeded count (0|1). */
+    finish() {
+        const { task, opts } = this;
+        const runs = this.scores.length;
+        const split = opts.split ?? 'train';
+        const scoreSum = this.scores.reduce((a, b) => a + b, 0);
+        const passRate = runs ? Math.round((this.passes / runs) * 1000) / 1000 : 0;
+        const meanScore = runs ? Math.round((scoreSum / runs) * 100) / 100 : 0;
+        const scoreSd = Math.round(sampleSd(this.scores) * 100) / 100;
+        let seeded = 0;
         // Deterministic backlog seeding: a sub-100% pass-rate is direct evidence of a
         // harness weakness, so seed it here rather than relying on the model to call the
         // ResearchBacklog tool. Idempotent by title (`Bench failure: <taskId>`), so a
@@ -172,12 +191,12 @@ export async function runBench(tasks, runner, matrix, opts) {
             const sampleWeight = Math.min(1, runs / 3); // 1 run = weak evidence; ~3+ = solid
             const confidence = Math.round(Math.min(0.9, Math.max(0.25, 0.3 + 0.6 * failRate * sampleWeight)) * 100) / 100;
             const severity = meanScore <= 10 ? 'high' : meanScore < 60 ? 'medium' : 'low';
-            const clip = (s, n) => (s.length > n ? s.slice(0, n) + '…' : s);
+            const clip = (x, n) => (x.length > n ? x.slice(0, n) + '…' : x);
             try {
                 opts.backlog.add({
                     title: `Bench failure: ${task.id}`,
                     description: `Harness failed benchmark task "${task.id}" (${task.verifier.type} verifier). ` +
-                        `passRate=${passes}/${runs}, meanScore=${meanScore}. Prompt: ${clip(task.prompt, 200)}`,
+                        `passRate=${this.passes}/${runs}, meanScore=${meanScore}. Prompt: ${clip(task.prompt, 200)}`,
                     bugClass: opts.deficiencyBugClass ?? 'Other',
                     severity,
                     impact: 3,
@@ -185,27 +204,80 @@ export async function runBench(tasks, runner, matrix, opts) {
                     confidence,
                     discoveredRound: opts.discoveredRound,
                     discoveredRef: opts.discoveredRef,
-                    affectedModels: observedModelId ? [observedModelId] : undefined,
-                    affectedTaskFingerprints: [taskFingerprint],
-                    notes: `taskType=${taskType}; split=${split}; sample output: ${clip(sampleFailure ?? '(empty output)', 300)}`,
+                    affectedModels: this.observedModelId ? [this.observedModelId] : undefined,
+                    affectedTaskFingerprints: [this.taskFingerprint],
+                    notes: `taskType=${this.taskType}; split=${split}; sample output: ${clip(this.sampleFailure ?? '(empty output)', 300)}`,
                 });
-                seededDeficiencies++;
+                seeded = 1;
             }
             catch {
                 // A backlog write error must never fail the benchmark itself.
             }
         }
-        taskSummaries.push({ taskId: task.id, taskFingerprint, taskType, runs, passRate, meanScore });
+        return {
+            summary: { taskId: task.id, taskFingerprint: this.taskFingerprint, taskType: this.taskType, runs, passRate, meanScore, scoreSd },
+            seeded,
+        };
+    }
+}
+function summarize(opts, benches) {
+    const tasks = [];
+    let seededDeficiencies = 0;
+    for (const b of benches) {
+        const { summary, seeded } = b.finish();
+        tasks.push(summary);
+        seededDeficiencies += seeded;
     }
     return {
         experimentTag: opts.experimentTag,
-        split,
+        split: opts.split ?? 'train',
         harnessRef: opts.harnessRef,
-        totalRuns,
-        tasks: taskSummaries,
+        totalRuns: tasks.reduce((s, t) => s + t.runs, 0),
+        tasks,
         seededDeficiencies,
     };
 }
+export async function runBench(tasks, runner, matrix, opts) {
+    const runs = opts.runs ?? 2;
+    const benches = [];
+    for (const task of tasks) {
+        const b = new TaskBench(task, runner, matrix, opts);
+        for (let i = 0; i < runs; i++)
+            await b.runOnce();
+        benches.push(b);
+    }
+    return summarize(opts, benches);
+}
+/**
+ * Bench a BASE and a CANDIDATE arm INTERLEAVED: for each task and each run index the
+ * two arms run back to back, in a seeded random order. Provider drift (latency, load,
+ * a silent model update, rate limiting) then lands on both arms instead of all on the
+ * one that ran second. Records are identical to two runBench calls (same matrix rows,
+ * same summaries); only the run order differs.
+ */
+export async function runBenchPaired(tasks, base, candidate, matrix, opts) {
+    const runs = opts.runs ?? 2;
+    const rng = mulberry32(opts.orderSeed ?? 0x5eed);
+    const baseOpts = { ...opts, harnessRef: base.harnessRef };
+    const candOpts = { ...opts, harnessRef: candidate.harnessRef };
+    const baseBenches = [];
+    const candBenches = [];
+    const order = [];
+    for (const task of tasks) {
+        const b = new TaskBench(task, base.runner, matrix, baseOpts);
+        const c = new TaskBench(task, candidate.runner, matrix, candOpts);
+        for (let i = 0; i < runs; i++) {
+            const baseFirst = rng() < 0.5;
+            for (const arm of baseFirst ? ['base', 'candidate'] : ['candidate', 'base']) {
+                order.push(arm);
+                await (arm === 'base' ? b : c).runOnce();
+            }
+        }
+        baseBenches.push(b);
+        candBenches.push(c);
+    }
+    return { base: summarize(baseOpts, baseBenches), candidate: summarize(candOpts, candBenches), order };
+}
 /** Validate a parsed task-set JSON value, throwing on malformed entries. */
 export function parseTaskSet(raw, source = '<task-set>') {
     const arr = Array.isArray(raw) ? raw : [raw];

package/dist/training/BenchRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"BenchRunner.js","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAuDnD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAOD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,QAAkB,EAAE,KAAe,EAAE,MAAoB;IACtG,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,UAAU,EAAE,CAAC;QAC5G,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YAChG,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,GAAG,CAAC;YAChB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC;oBAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC9D,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM;oBAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACtF,CAAC;YACD,gFAAgF;YAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACpE,IAAI,KAAa,CAAC;YAClB,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpF,KAAK,GAAG,KAAK,KAAK,MAAM;oBACtB,CAAC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACjF,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC,CAAC,qEAAqE;YACzF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9F,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;YAC3F,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,sBAAsB,KAAK,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrH,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC;YAC3H,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7G,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5H,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,QAAQ,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAsGD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,MAAqB,EACrB,MAAyB,EACzB,IAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;IACpC,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QACrE,MAAM,eAAe,GAAG,iBAAiB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,aAAiC,CAAC;QACtC,IAAI,eAAe,GAAuB,IAAI,CAAC,OAAO,CAAC;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,aAAa,KAAK,SAAS;gBAAE,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC;YACzE,eAAe,GAAG,GAAG,CAAC,OAAO,IAAI,eAAe,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,QAAQ,IAAI,KAAK,CAAC,gBAAgB,CAAC;YACnC,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,MAAM,CAAC;gBACZ,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,SAAS;gBACjD,QAAQ;gBACR,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;gBAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,eAAe;gBACf,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK;gBACL,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5E,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9F,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC5G,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAE5D,8EAA8E;QAC9E,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9G,MAAM,QAAQ,GAAa,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YACxF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBACf,KAAK,EAAE,kBAAkB,IAAI,CAAC,EAAE,EAAE;oBAClC,WAAW,EACT,kCAAkC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc;wBAC/E,YAAY,MAAM,IAAI,IAAI,eAAe,SAAS,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;oBACzF,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,OAAO;oBAC5C,QAAQ;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;oBACT,UAAU;oBACV,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBAC/D,wBAAwB,EAAE,CAAC,eAAe,CAAC;oBAC3C,KAAK,EAAE,YAAY,QAAQ,WAAW,KAAK,oBAAoB,IAAI,CAAC,aAAa,IAAI,gBAAgB,EAAE,GAAG,CAAC,EAAE;iBAC9G,CAAC,CAAC;gBACH,kBAAkB,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,8DAA8D;YAChE,CAAC;QACH,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK;QACL,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS;QACT,KAAK,EAAE,aAAa;QACpB,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,MAAM,GAAG,YAAY;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnF,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9G,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,CAAC,CAAC,QAAmC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4CAA4C,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClH,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,wDAAwD,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,8CAA8C,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,0DAA0D,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4EAA4E,CAAC,CAAC;QACxH,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAA8B,EAAE,CAAC;IACtH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"BenchRunner.js","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAuDpD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAOD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,QAAkB,EAAE,KAAe,EAAE,MAAoB;IACtG,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,UAAU,EAAE,CAAC;QAC5G,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YAChG,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,GAAG,CAAC;YAChB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC;oBAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC9D,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM;oBAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACtF,CAAC;YACD,gFAAgF;YAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACpE,IAAI,KAAa,CAAC;YAClB,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpF,KAAK,GAAG,KAAK,KAAK,MAAM;oBACtB,CAAC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACjF,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC,CAAC,qEAAqE;YACzF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9F,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;YAC3F,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,sBAAsB,KAAK,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrH,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC;YAC3H,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7G,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5H,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,QAAQ,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAwGD;;;;;GAKG;AACH,sFAAsF;AACtF,MAAM,SAAS;IAQQ;IAAiC;IAAwC;IAA4C;IAPjI,QAAQ,CAAS;IACjB,eAAe,CAAS;IACzB,MAAM,GAAG,CAAC,CAAC;IACX,MAAM,GAAa,EAAE,CAAC;IACtB,aAAa,CAAqB;IAClC,eAAe,CAAqB;IAE5C,YAAqB,IAAc,EAAmB,MAAqB,EAAmB,MAAyB,EAAmB,IAAqB;QAA1I,SAAI,GAAJ,IAAI,CAAU;QAAmB,WAAM,GAAN,MAAM,CAAe;QAAmB,WAAM,GAAN,MAAM,CAAmB;QAAmB,SAAI,GAAJ,IAAI,CAAiB;QAC7J,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QACpE,IAAI,CAAC,eAAe,GAAG,iBAAiB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC;IACtC,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,OAAO;QACX,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC;QACnF,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC;QAC3D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YACjB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,SAAS;YACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO;YAC5B,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9F,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACzH,CAAC;IAED,2FAA2F;IAC3F,MAAM;QACJ,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAC9D,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,8EAA8E;QAC9E,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9G,MAAM,QAAQ,GAAa,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YACxF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBACf,KAAK,EAAE,kBAAkB,IAAI,CAAC,EAAE,EAAE;oBAClC,WAAW,EACT,kCAAkC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc;wBAC/E,YAAY,IAAI,CAAC,MAAM,IAAI,IAAI,eAAe,SAAS,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;oBAC9F,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,OAAO;oBAC5C,QAAQ;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;oBACT,UAAU;oBACV,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,cAAc,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACzE,wBAAwB,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;oBAChD,KAAK,EAAE,YAAY,IAAI,CAAC,QAAQ,WAAW,KAAK,oBAAoB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,gBAAgB,EAAE,GAAG,CAAC,EAAE;iBACxH,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,8DAA8D;YAChE,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE;YAChI,MAAM;SACP,CAAC;IACJ,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAqB,EAAE,OAAoB;IAC5D,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,kBAAkB,IAAI,MAAM,CAAC;IAC/B,CAAC;IACD,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO;QAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK;QACL,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,MAAqB,EACrB,MAAyB,EACzB,IAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE;YAAE,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AASD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAiB,EACjB,IAAoB,EACpB,SAAyB,EACzB,MAAyB,EACzB,IAA8C;IAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAoB,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IAC3E,MAAM,QAAQ,GAAoB,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,CAAC;IAChF,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,MAAM,KAAK,GAA6B,EAAE,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,WAAW,CAAW,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,MAAM,CAAW,EAAE,CAAC;gBAClG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,KAAK,EAAE,CAAC;AACxG,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,MAAM,GAAG,YAAY;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnF,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9G,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,CAAC,CAAC,QAAmC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4CAA4C,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClH,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,wDAAwD,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,8CAA8C,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,0DAA0D,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4EAA4E,CAAC,CAAC;QACxH,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAA8B,EAAE,CAAC;IACtH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/training/EvalGuard.d.ts

@@ -0,0 +1,43 @@
+/** Convert a glob to an anchored RegExp over repo-relative POSIX paths.
+ *  `**` matches across directories (and `dir/**` also matches `dir` itself), `*` and `?`
+ *  stay inside one path segment. A glob with no slash matches the basename at any depth. */
+export declare function globToRegExp(glob: string): RegExp;
+export declare function matchesAny(path: string, globs: readonly string[]): boolean;
+export interface GuardViolation {
+    path: string;
+    rule: 'protected' | 'outside-editable';
+}
+export interface GuardPolicy {
+    /** globs no candidate may touch (the task-sets' in-repo paths are added by the loop). */
+    protect: readonly string[];
+    /** when non-empty, every changed path must match one of these. */
+    editable?: readonly string[];
+}
+/** Check a candidate's changed paths (repo-relative, POSIX) against the policy. */
+export declare function checkCandidateChanges(changed: readonly string[], policy: GuardPolicy): {
+    ok: boolean;
+    violations: GuardViolation[];
+};
+/** One-line human summary of the violations (capped). */
+export declare function describeViolations(v: readonly GuardViolation[], max?: number): string;
+/** The repo-relative POSIX path of `absPath` when it lies inside `repoDir`, else undefined. */
+export declare function repoRelative(absPath: string, repoDir: string): string | undefined;
+/** Glob that protects a task-set path (a file, or every file under a directory). */
+export declare function protectGlobFor(relPath: string, isDir: boolean): string;
+/** sha256 over a file, or over a directory's files (sorted relative path + content). */
+export declare function hashPath(p: string): string;
+export interface SealedSet {
+    /** where the loop read it from */
+    source: string;
+    /** the private copy every experiment grades against */
+    sealed: string;
+    sha256: string;
+    isDir: boolean;
+    /** repo-relative path when the source lies inside the repo (then it is hidden + protected) */
+    inRepo?: string;
+}
+/** Copy a task-set (file or dir) into `sealDir/<label>-<basename>` and fingerprint it. */
+export declare function sealTaskSet(source: string, sealDir: string, label: string, repoDir: string): SealedSet;
+/** Task ids declared in a task-set (file or dir of *.json; array or single object). */
+export declare function taskIdsIn(p: string): Set<string>;
+//# sourceMappingURL=EvalGuard.d.ts.map

package/dist/training/EvalGuard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalGuard.d.ts","sourceRoot":"","sources":["../../src/training/EvalGuard.ts"],"names":[],"mappings":"AA0BA;;4FAE4F;AAC5F,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuBjD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAG1E;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,kBAAkB,CAAC;CACxC;AAED,MAAM,WAAW,WAAW;IAC1B,yFAAyF;IACzF,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9B;AAED,mFAAmF;AACnF,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,cAAc,EAAE,CAAA;CAAE,CAQpI;AAED,yDAAyD;AACzD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,SAAI,GAAG,MAAM,CAGhF;AAED,+FAA+F;AAC/F,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAIjF;AAED,oFAAoF;AACpF,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAEtE;AAED,wFAAwF;AACxF,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAa1C;AAED,MAAM,WAAW,SAAS;IACxB,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,0FAA0F;AAC1F,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAKtG;AAED,uFAAuF;AACvF,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAYhD"}

package/dist/training/EvalGuard.js

@@ -0,0 +1,131 @@
+/**
+ * EvalGuard — keep the Fixer away from the evaluation (autoresearch hardening A1).
+ *
+ * The statistical gate can only be trusted if the thing being improved cannot reach
+ * the thing that grades it. Library home of the three mechanisms, so every entry point
+ * can apply them (the CLI `cortex autoresearch loop` applies all three):
+ *
+ *  1. SEAL — the train/holdout task-sets are copied ONCE, at loop start, into a private
+ *     temp dir outside the repo, and every experiment grades against those sealed copies.
+ *     Editing the in-repo task-set (or the main checkout) changes nothing the gate reads.
+ *  2. HIDE — a task-set that lives inside the repo is deleted from each candidate worktree
+ *     before the Fixer runs and restored to the base version before the candidate is
+ *     committed, so the Fixer neither reads it in the tree nor ships an edit to it.
+ *     (Best-effort secrecy: the files are still in git history. For a holdout the Fixer
+ *     must never see, keep it OUTSIDE the repo; the loop warns when it is inside.)
+ *  3. REFUSE — before a candidate is benched, its changed paths are checked against the
+ *     protected globs (verifier scripts, graders, benches: `--protect`) and the optional
+ *     editable allowlist (`--editable`). A candidate that touches a protected path or
+ *     strays outside the allowlist is refused unbenched (`skipped: 'eval-guard'`).
+ *
+ * Pure helpers + small fs helpers; no git here (the caller owns git).
+ */
+import { createHash } from 'node:crypto';
+import { cpSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { basename, isAbsolute, join, relative, sep } from 'node:path';
+/** Convert a glob to an anchored RegExp over repo-relative POSIX paths.
+ *  `**` matches across directories (and `dir/**` also matches `dir` itself), `*` and `?`
+ *  stay inside one path segment. A glob with no slash matches the basename at any depth. */
+export function globToRegExp(glob) {
+    let g = glob.trim().replace(/\\/g, '/').replace(/^\.\//, '');
+    if (!g.includes('/'))
+        g = `**/${g}`;
+    let re = '';
+    for (let i = 0; i < g.length; i++) {
+        const c = g[i];
+        if (c === '*') {
+            if (g[i + 1] === '*') {
+                const slashAfter = g[i + 2] === '/';
+                re += slashAfter ? '(?:.*/)?' : '.*';
+                i += slashAfter ? 2 : 1;
+            }
+            else {
+                re += '[^/]*';
+            }
+        }
+        else if (c === '?') {
+            re += '[^/]';
+        }
+        else {
+            re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+        }
+    }
+    // `dir/**` also covers the directory path itself.
+    if (re.endsWith('/.*'))
+        re = `${re.slice(0, -3)}(?:/.*)?`;
+    return new RegExp(`^${re}$`);
+}
+export function matchesAny(path, globs) {
+    const p = path.replace(/\\/g, '/').replace(/^\.\//, '');
+    return globs.some((g) => globToRegExp(g).test(p));
+}
+/** Check a candidate's changed paths (repo-relative, POSIX) against the policy. */
+export function checkCandidateChanges(changed, policy) {
+    const violations = [];
+    const editable = policy.editable ?? [];
+    for (const path of changed) {
+        if (matchesAny(path, policy.protect))
+            violations.push({ path, rule: 'protected' });
+        else if (editable.length && !matchesAny(path, editable))
+            violations.push({ path, rule: 'outside-editable' });
+    }
+    return { ok: violations.length === 0, violations };
+}
+/** One-line human summary of the violations (capped). */
+export function describeViolations(v, max = 5) {
+    const head = v.slice(0, max).map((x) => `${x.path} (${x.rule})`).join(', ');
+    return v.length > max ? `${head}, +${v.length - max} more` : head;
+}
+/** The repo-relative POSIX path of `absPath` when it lies inside `repoDir`, else undefined. */
+export function repoRelative(absPath, repoDir) {
+    const rel = relative(repoDir, absPath);
+    if (!rel || rel.startsWith('..') || isAbsolute(rel))
+        return undefined;
+    return rel.split(sep).join('/');
+}
+/** Glob that protects a task-set path (a file, or every file under a directory). */
+export function protectGlobFor(relPath, isDir) {
+    return isDir ? `${relPath.replace(/\/+$/, '')}/**` : relPath;
+}
+/** sha256 over a file, or over a directory's files (sorted relative path + content). */
+export function hashPath(p) {
+    const h = createHash('sha256');
+    const walk = (abs, rel) => {
+        if (statSync(abs).isDirectory()) {
+            for (const name of readdirSync(abs).sort())
+                walk(join(abs, name), rel ? `${rel}/${name}` : name);
+        }
+        else {
+            h.update(`${rel}\0`);
+            h.update(readFileSync(abs));
+            h.update('\0');
+        }
+    };
+    walk(p, '');
+    return h.digest('hex');
+}
+/** Copy a task-set (file or dir) into `sealDir/<label>-<basename>` and fingerprint it. */
+export function sealTaskSet(source, sealDir, label, repoDir) {
+    const isDir = statSync(source).isDirectory();
+    const sealed = join(sealDir, `${label}-${basename(source)}`);
+    cpSync(source, sealed, { recursive: true });
+    return { source, sealed, sha256: hashPath(sealed), isDir, inRepo: repoRelative(source, repoDir) };
+}
+/** Task ids declared in a task-set (file or dir of *.json; array or single object). */
+export function taskIdsIn(p) {
+    const ids = new Set();
+    const files = statSync(p).isDirectory()
+        ? readdirSync(p).filter((f) => f.endsWith('.json')).map((f) => join(p, f))
+        : [p];
+    for (const f of files) {
+        try {
+            const raw = JSON.parse(readFileSync(f, 'utf8'));
+            for (const t of Array.isArray(raw) ? raw : [raw])
+                if (t && typeof t.id === 'string')
+                    ids.add(t.id);
+        }
+        catch { /* unreadable file: the experiment reports it */ }
+    }
+    return ids;
+}
+//# sourceMappingURL=EvalGuard.js.map