@nexus-cortex/core 4.130.0 → 4.131.2

package/dist/training/AutoResearchGate.js

@@ -17,6 +17,7 @@
  * `verifyOnHoldout` is the separate, mandatory second gate — a kept candidate is
  * only merge-eligible after it ALSO clears held-out. `fixed` ≠ `verified`.
  */
+import { familyEffects, stepGuards, describeGuardBlocks } from './AutoResearchGuards.js';
 import { regressionScan, toTaskResult } from './VersionComparison.js';
 import { decideExperiment } from './AutoResearchStats.js';
 /**
@@ -55,9 +56,14 @@ export function evaluateExperiment(matrix, ledger, input) {
     //    that flagged in the reason — the loop should not silently merge collateral
     //    damage even when the aggregate effect is positive.
     const regressedTasks = regression.regressions.map(r => r.taskFingerprint);
-    const reason = verdict.decision === 'keep' && regressedTasks.length > 0
+    const families = familyEffects(arms, { seed: input.gate?.seed, minRunsPerArm: input.gate?.minRunsPerArm });
+    const guards = stepGuards(families);
+    const blocks = describeGuardBlocks(guards);
+    let reason = verdict.decision === 'keep' && regressedTasks.length > 0
         ? `${verdict.reason} | WARNING: ${regressedTasks.length} task(s) regressed: ${regressedTasks.join(', ')}`
         : verdict.reason;
+    if (blocks)
+        reason += ` | GUARD: ${blocks}`;
     const record = ledger.decide(experimentTag, {
         decision: verdict.decision,
         pValue: verdict.pValue,
@@ -67,7 +73,7 @@ export function evaluateExperiment(matrix, ledger, input) {
         nRuns: verdict.nRuns,
         reason,
     });
-    return { record, verdict, regression, regressedTasks };
+    return { record, verdict, regression, regressedTasks, families, stepGuards: guards };
 }
 /**
  * Held-out verification — the SECOND, mandatory gate before a kept candidate may

package/dist/training/AutoResearchGate.js.map

@@ -1 +1 @@
-{"version":3,"file":"AutoResearchGate.js","sourceRoot":"","sources":["../../src/training/AutoResearchGate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAE,cAAc,EAAE,YAAY,EAA8C,MAAM,wBAAwB,CAAC;AAClH,OAAO,EAAE,gBAAgB,EAA2D,MAAM,wBAAwB,CAAC;AA6BnH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAyB,EACzB,MAAwB,EACxB,KAAoB;IAEpB,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAE/D,iFAAiF;IACjF,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE;QAC/D,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAC,CAAC;IACH,MAAM,WAAW,GAAqB;QACpC,GAAG,UAAU,CAAC,YAAY;QAC1B,GAAG,UAAU,CAAC,WAAW;QACzB,GAAG,UAAU,CAAC,OAAO;KACtB,CAAC;IAEF,+DAA+D;IAC/D,MAAM,CAAC,IAAI,CAAC;QACV,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM;QAC5C,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;KAC9D,CAAC,CAAC;IAEH,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,IAAI,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAClI,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE;QACrC,GAAG,KAAK,CAAC,IAAI;QACb,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,CAAC;KAClD,CAAC,CAAC;IAEH,4EAA4E;IAC5E,gFAAgF;IAChF,wDAAwD;IACxD,MAAM,cAAc,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QACrE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,eAAe,cAAc,CAAC,MAAM,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAEnB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE;QAC1C,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM;KACP,CAAE,CAAC;IAEJ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC;AACzD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAyB,EACzB,KAAkI;IAElI,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE;QACxE,KAAK,EAAE,SAAS;QAChB,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1F,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sCAAsC;IACjF,MAAM,IAAI,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAClI,OAAO,gBAAgB,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,CAAC,EAAE,CAAC,CAAC;AACtG,CAAC"}
+{"version":3,"file":"AutoResearchGate.js","sourceRoot":"","sources":["../../src/training/AutoResearchGate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,mBAAmB,EAAwC,MAAM,yBAAyB,CAAC;AAC/H,OAAO,EAAE,cAAc,EAAE,YAAY,EAA8C,MAAM,wBAAwB,CAAC;AAClH,OAAO,EAAE,gBAAgB,EAA2D,MAAM,wBAAwB,CAAC;AAkCnH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAyB,EACzB,MAAwB,EACxB,KAAoB;IAEpB,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAE/D,iFAAiF;IACjF,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE;QAC/D,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAC,CAAC;IACH,MAAM,WAAW,GAAqB;QACpC,GAAG,UAAU,CAAC,YAAY;QAC1B,GAAG,UAAU,CAAC,WAAW;QACzB,GAAG,UAAU,CAAC,OAAO;KACtB,CAAC;IAEF,+DAA+D;IAC/D,MAAM,CAAC,IAAI,CAAC;QACV,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM;QAC5C,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;KAC9D,CAAC,CAAC;IAEH,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,IAAI,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAClI,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE;QACrC,GAAG,KAAK,CAAC,IAAI;QACb,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,CAAC;KAClD,CAAC,CAAC;IAEH,4EAA4E;IAC5E,gFAAgF;IAChF,wDAAwD;IACxD,MAAM,cAAc,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;IAC3G,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QACnE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,eAAe,cAAc,CAAC,MAAM,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACnB,IAAI,MAAM;QAAE,MAAM,IAAI,aAAa,MAAM,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE;QAC1C,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM;KACP,CAAE,CAAC;IAEJ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACvF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAyB,EACzB,KAAkI;IAElI,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE;QACxE,KAAK,EAAE,SAAS;QAChB,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1F,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sCAAsC;IACjF,MAAM,IAAI,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAClI,OAAO,gBAAgB,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,CAAC,EAAE,CAAC,CAAC;AACtG,CAAC"}

package/dist/training/AutoResearchGuards.d.ts

@@ -0,0 +1,86 @@
+/**
+ * AutoResearchGuards — a gain on the aggregate must not hide a loss in one task family
+ * (autoresearch hardening B4).
+ *
+ * The gate keeps a candidate on the AGGREGATE effect (mean over tasks). A change can
+ * lift that mean while one family of tasks (TaskClassifier T1..T5 / GENERAL) gets worse,
+ * and a loop that merges such changes round after round can drift a family down in
+ * steps too small to notice one at a time. Two guards, both per family:
+ *
+ *   STEP  — this candidate vs its base: the family's effect.
+ *   TOTAL — the running sum of the family's effect over every merged round plus this
+ *           one, i.e. the drift from the round-0 base, built from each step's own
+ *           (interleaved) measurement, so no extra runs and no cross-time comparison.
+ *
+ * Rule (both guards): a family BLOCKS when its loss is MATERIAL (estimate < -margin) AND
+ * EVIDENT (its one-sided upper confidence bound is < 0, at alpha split across the
+ * families). Model variance is large: a point-estimate rule alone (est < -2 with an SE
+ * of ~5) would block ~34% of no-change families, and a non-inferiority rule (CI low >
+ * -margin) would block most of them. Requiring evidence keeps false blocks <= alpha per
+ * round, at the price of missing small real losses when runs are few — the run planner
+ * (planRuns) says how many runs the gate needs.
+ *
+ * SEs are analytic (sample variances, Welch–Satterthwaite df, t quantiles), not
+ * bootstrapped: with 3-4 runs per arm a percentile bootstrap understates the spread and
+ * normal quantiles are too tight — that combination blocked ~9% of no-change rounds in
+ * the test instead of <= 5%.
+ */
+import type { RNG, TaskArms } from './AutoResearchStats.js';
+export interface FamilyEffect {
+    family: string;
+    nTasks: number;
+    /** mean over the family's tasks of (candMean - baseMean), score points */
+    effect: number;
+    /** standard error of `effect` (sample variances of the per-run scores) */
+    se: number;
+    /** Welch–Satterthwaite degrees of freedom of `se` */
+    df: number;
+    /** one-sided upper confidence bound at the family-split alpha */
+    upper: number;
+}
+export interface GuardVerdict {
+    family: string;
+    /** 'step' = this candidate vs its base; 'total' = running drift from the round-0 base */
+    scope: 'step' | 'total';
+    effect: number;
+    se: number;
+    upper: number;
+    margin: number;
+    blocked: boolean;
+}
+export interface GuardOptions {
+    /** per-step loss margin, score points (default 2) */
+    margin?: number;
+    /** running-total loss margin, score points (default 5) */
+    totalMargin?: number;
+    /** guard error rate per round, split across families (default 0.05) */
+    alpha?: number;
+    /** unused (kept for call-site compatibility with the gate options) */
+    iterations?: number;
+    seed?: number;
+    rng?: RNG;
+    /** minimum runs per arm for a task to count (default 2) */
+    minRunsPerArm?: number;
+}
+/** Student-t quantile via the Cornish–Fisher expansion (df >= 1; exact enough for bounds). */
+export declare function tQuantile(p: number, df: number): number;
+/** Per-family effects: mean over the family's tasks of (candMean - baseMean), its SE from
+ *  the sample variances, Welch–Satterthwaite df, and the one-sided upper bound at
+ *  alpha / nFamilies. */
+export declare function familyEffects(tasks: readonly TaskArms[], opts?: GuardOptions): FamilyEffect[];
+/** The STEP guard: one verdict per family of this candidate vs its base. */
+export declare function stepGuards(families: readonly FamilyEffect[], opts?: GuardOptions): GuardVerdict[];
+/** Running per-family drift from the round-0 base: Σ effect and Σ variance of merged steps. */
+export type DriftLedger = Map<string, {
+    effect: number;
+    variance: number;
+    steps: number;
+    df: number;
+}>;
+/** The TOTAL guard: the ledger plus this step, per family this step measured. */
+export declare function totalGuards(ledger: DriftLedger, families: readonly FamilyEffect[], opts?: GuardOptions): GuardVerdict[];
+/** Add a merged step's family effects to the drift ledger (mutates and returns it). */
+export declare function recordDrift(ledger: DriftLedger, families: readonly FamilyEffect[]): DriftLedger;
+/** One-line summary of the blocking guards (empty when none block). */
+export declare function describeGuardBlocks(vs: readonly GuardVerdict[]): string;
+//# sourceMappingURL=AutoResearchGuards.d.ts.map

package/dist/training/AutoResearchGuards.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutoResearchGuards.d.ts","sourceRoot":"","sources":["../../src/training/AutoResearchGuards.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAG5D,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,EAAE,EAAE,MAAM,CAAC;IACX,qDAAqD;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAID,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAMvD;AAQD;;yBAEyB;AACzB,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,GAAE,YAAiB,GAAG,YAAY,EAAE,CA2BjG;AAOD,4EAA4E;AAC5E,wBAAgB,UAAU,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,EAAE,IAAI,GAAE,YAAiB,GAAG,YAAY,EAAE,CAGrG;AAED,+FAA+F;AAC/F,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEvG,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,YAAY,EAAE,EAAE,IAAI,GAAE,YAAiB,GAAG,YAAY,EAAE,CAQ3H;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,YAAY,EAAE,GAAG,WAAW,CAM/F;AAED,uEAAuE;AACvE,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,CAIvE"}

package/dist/training/AutoResearchGuards.js

@@ -0,0 +1,81 @@
+import { normalQuantile } from './AutoResearchConfirm.js';
+function round4(n) { return Math.round(n * 10000) / 10000; }
+/** Student-t quantile via the Cornish–Fisher expansion (df >= 1; exact enough for bounds). */
+export function tQuantile(p, df) {
+    const z = normalQuantile(p);
+    if (!Number.isFinite(df) || df > 1e6)
+        return z;
+    const z3 = z ** 3, z5 = z ** 5, z7 = z ** 7;
+    return z + (z3 + z) / (4 * df) + (5 * z5 + 16 * z3 + 3 * z) / (96 * df ** 2)
+        + (3 * z7 + 19 * z5 + 17 * z3 - 15 * z) / (384 * df ** 3);
+}
+function variance(xs) {
+    const m = xs.reduce((a, b) => a + b, 0) / xs.length;
+    return xs.reduce((a, b) => a + (b - m) ** 2, 0) / (xs.length - 1);
+}
+const avg = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+/** Per-family effects: mean over the family's tasks of (candMean - baseMean), its SE from
+ *  the sample variances, Welch–Satterthwaite df, and the one-sided upper bound at
+ *  alpha / nFamilies. */
+export function familyEffects(tasks, opts = {}) {
+    const minRuns = opts.minRunsPerArm ?? 2;
+    const usable = tasks.filter((t) => t.baseScores.length >= minRuns && t.candScores.length >= minRuns);
+    const groups = new Map();
+    for (const t of usable) {
+        const f = t.taskFamily ?? 'unlabeled';
+        groups.set(f, [...(groups.get(f) ?? []), t]);
+    }
+    const nFam = Math.max(1, groups.size);
+    const alpha = opts.alpha ?? 0.05;
+    const out = [];
+    for (const [family, arms] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const k = arms.length;
+        const effect = avg(arms.map((t) => avg(t.candScores) - avg(t.baseScores)));
+        // Var(effect) = Σ_t (s_b²/n_b + s_c²/n_c) / k²; components feed Welch–Satterthwaite.
+        const comps = arms.flatMap((t) => [
+            { v: variance(t.baseScores) / t.baseScores.length / (k * k), df: t.baseScores.length - 1 },
+            { v: variance(t.candScores) / t.candScores.length / (k * k), df: t.candScores.length - 1 },
+        ]);
+        const vSum = comps.reduce((a, c) => a + c.v, 0);
+        const dfDen = comps.reduce((a, c) => a + (c.v * c.v) / c.df, 0);
+        const df = vSum === 0 || dfDen === 0 ? Infinity : (vSum * vSum) / dfDen;
+        const se = Math.sqrt(vSum);
+        const upper = effect + tQuantile(1 - alpha / nFam, Math.max(1, df)) * se;
+        out.push({ family, nTasks: k, effect: round4(effect), se: round4(se), df: Number.isFinite(df) ? round4(df) : 1e9, upper: round4(upper) });
+    }
+    return out;
+}
+function verdict(scope, family, effect, se, q, margin) {
+    const upper = round4(effect + q * se);
+    return { family, scope, effect: round4(effect), se: round4(se), upper, margin, blocked: effect < -margin && upper < 0 };
+}
+/** The STEP guard: one verdict per family of this candidate vs its base. */
+export function stepGuards(families, opts = {}) {
+    const p = 1 - (opts.alpha ?? 0.05) / Math.max(1, families.length);
+    return families.map((f) => verdict('step', f.family, f.effect, f.se, tQuantile(p, Math.max(1, f.df)), opts.margin ?? 2));
+}
+/** The TOTAL guard: the ledger plus this step, per family this step measured. */
+export function totalGuards(ledger, families, opts = {}) {
+    const p = 1 - (opts.alpha ?? 0.05) / Math.max(1, families.length);
+    return families.map((f) => {
+        const prior = ledger.get(f.family);
+        // conservative df for the sum: the smallest df of any step in it
+        const df = Math.max(1, Math.min(f.df, prior?.df ?? Infinity));
+        return verdict('total', f.family, (prior?.effect ?? 0) + f.effect, Math.sqrt((prior?.variance ?? 0) + f.se ** 2), tQuantile(p, df), opts.totalMargin ?? 5);
+    });
+}
+/** Add a merged step's family effects to the drift ledger (mutates and returns it). */
+export function recordDrift(ledger, families) {
+    for (const f of families) {
+        const prior = ledger.get(f.family) ?? { effect: 0, variance: 0, steps: 0, df: Infinity };
+        ledger.set(f.family, { effect: round4(prior.effect + f.effect), variance: prior.variance + f.se ** 2, steps: prior.steps + 1, df: Math.min(prior.df, f.df) });
+    }
+    return ledger;
+}
+/** One-line summary of the blocking guards (empty when none block). */
+export function describeGuardBlocks(vs) {
+    return vs.filter((v) => v.blocked)
+        .map((v) => `${v.family} ${v.scope === 'step' ? 'step' : 'total'} ${v.effect} (upper ${v.upper} < 0, margin -${v.margin})`)
+        .join('; ');
+}
+//# sourceMappingURL=AutoResearchGuards.js.map

package/dist/training/AutoResearchGuards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutoResearchGuards.js","sourceRoot":"","sources":["../../src/training/AutoResearchGuards.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAyC1D,SAAS,MAAM,CAAC,CAAS,IAAY,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAE5E,8FAA8F;AAC9F,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,EAAU;IAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,GAAG;QAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;UACxE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,EAAqB;IACrC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpD,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC;AACD,MAAM,GAAG,GAAG,CAAC,EAAqB,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AAEjF;;yBAEyB;AACzB,MAAM,UAAU,aAAa,CAAC,KAA0B,EAAE,OAAqB,EAAE;IAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,WAAW,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3E,qFAAqF;QACrF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAChC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1F,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;SAC3F,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;QACxE,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACzE,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5I,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CAAC,KAAuB,EAAE,MAAc,EAAE,MAAc,EAAE,EAAU,EAAE,CAAS,EAAE,MAAc;IAC7G,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;AAC1H,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,UAAU,CAAC,QAAiC,EAAE,OAAqB,EAAE;IACnF,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3H,CAAC;AAKD,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,MAAmB,EAAE,QAAiC,EAAE,OAAqB,EAAE;IACzG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnC,iEAAiE;QACjE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,QAAQ,CAAC,CAAC,CAAC;QAC9D,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC7J,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,WAAW,CAAC,MAAmB,EAAE,QAAiC;IAChF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;QACzF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChK,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,mBAAmB,CAAC,EAA2B;IAC7D,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,KAAK,iBAAiB,CAAC,CAAC,MAAM,GAAG,CAAC;SAC1H,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/training/BenchRunner.d.ts

@@ -193,6 +193,8 @@ export interface BenchTaskSummary {
     runs: number;
     passRate: number;
     meanScore: number;
+    /** sample SD of the per-run qualitative scores (run-count planning). */
+    scoreSd?: number;
 }
 export interface BenchSummary {
     experimentTag: string;
@@ -203,13 +205,27 @@ export interface BenchSummary {
     /** Number of deficiencies seeded into `opts.backlog` this run (0 when no sink). */
     seededDeficiencies: number;
 }
+export declare function runBench(tasks: TaskSpec[], runner: HarnessRunner, matrix: ModelRouterMatrix, opts: RunBenchOptions): Promise<BenchSummary>;
+export interface PairedBenchArm {
+    runner: HarnessRunner;
+    /** this arm's per-arm bench options (harnessRef, strategy, …); the shared fields come
+     *  from the `opts` passed to runBenchPaired. */
+    harnessRef?: string;
+}
 /**
- * Run each task `runs` times through `runner`, grade each output, and write a
- * REAL scored BenchmarkRecord to the matrix (harnessRef auto-stamped unless
- * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
- * comparability key the gate joins on).
+ * Bench a BASE and a CANDIDATE arm INTERLEAVED: for each task and each run index the
+ * two arms run back to back, in a seeded random order. Provider drift (latency, load,
+ * a silent model update, rate limiting) then lands on both arms instead of all on the
+ * one that ran second. Records are identical to two runBench calls (same matrix rows,
+ * same summaries); only the run order differs.
  */
-export declare function runBench(tasks: TaskSpec[], runner: HarnessRunner, matrix: ModelRouterMatrix, opts: RunBenchOptions): Promise<BenchSummary>;
+export declare function runBenchPaired(tasks: TaskSpec[], base: PairedBenchArm, candidate: PairedBenchArm, matrix: ModelRouterMatrix, opts: RunBenchOptions & {
+    orderSeed?: number;
+}): Promise<{
+    base: BenchSummary;
+    candidate: BenchSummary;
+    order: ('base' | 'candidate')[];
+}>;
 /** Validate a parsed task-set JSON value, throwing on malformed entries. */
 export declare function parseTaskSet(raw: unknown, source?: string): TaskSpec[];
 //# sourceMappingURL=BenchRunner.d.ts.map

package/dist/training/BenchRunner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"BenchRunner.d.ts","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,sBAAsB,CAAC;AAMpE,MAAM,MAAM,QAAQ;AAClB,yDAAyD;AACvD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE;AAC1D,6DAA6D;GAC3D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD;;8EAE8E;GAC5E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE;AAChE,oEAAoE;GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACvC;;;;;;;6CAO6C;GAC3C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,UAAU,GAAG,UAAU,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;AAC3H;;;;;;;+EAO+E;GAC7E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE/G,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CA2E9H;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB;+EAC2E;IAC3E,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC3E;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC;CAChC;AAMD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8CAA8C;IAC9C,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACjG;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,mFAAmF;IACnF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,YAAY,CAAC,CA+FvB;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAe,GAAG,QAAQ,EAAE,CAmC5E"}
+{"version":3,"file":"BenchRunner.d.ts","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAI3D,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,sBAAsB,CAAC;AAMpE,MAAM,MAAM,QAAQ;AAClB,yDAAyD;AACvD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE;AAC1D,6DAA6D;GAC3D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD;;8EAE8E;GAC5E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE;AAChE,oEAAoE;GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACvC;;;;;;;6CAO6C;GAC3C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,UAAU,GAAG,UAAU,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;AAC3H;;;;;;;+EAO+E;GAC7E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE/G,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CA2E9H;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB;+EAC2E;IAC3E,QAAQ,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC3E;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC;CAChC;AAMD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8CAA8C;IAC9C,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACjG;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,mFAAmF;IACnF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AA8HD,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,YAAY,CAAC,CASvB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,aAAa,CAAC;IACtB;oDACgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,QAAQ,EAAE,EACjB,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,cAAc,EACzB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,CAAA;CAAE,CAAC,CAsB3F;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAe,GAAG,QAAQ,EAAE,CAmC5E"}

package/dist/training/BenchRunner.js

@@ -21,6 +21,8 @@
  */
 import { ModelRouterMatrix } from './ModelRouterMatrix.js';
 import { classifyTask } from './TaskClassifier.js';
+import { mulberry32 } from './AutoResearchStats.js';
+import { sampleSd } from './AutoResearchConfirm.js';
 function normalizeWs(s) {
     return s.replace(/\s+/g, ' ').trim().toLowerCase();
 }
@@ -117,50 +119,67 @@ export async function gradeRun(output, verifier, judge, extras) {
  * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
  * comparability key the gate joins on).
  */
-export async function runBench(tasks, runner, matrix, opts) {
-    const runs = opts.runs ?? 2;
-    const split = opts.split ?? 'train';
-    const taskSummaries = [];
-    let totalRuns = 0;
-    let seededDeficiencies = 0;
-    for (const task of tasks) {
-        const taskType = task.taskType ?? classifyTask(task.prompt).taskType;
-        const taskFingerprint = ModelRouterMatrix.fingerprintTask(task.prompt, JSON.stringify(task.verifier));
-        let passes = 0;
-        let scoreSum = 0;
-        let sampleFailure;
-        let observedModelId = opts.modelId;
-        for (let i = 0; i < runs; i++) {
-            const res = await runner.run(task.prompt, { model: opts.modelId });
-            const grade = await gradeRun(res.text, task.verifier, opts.judge, { toolUses: res.toolUses });
-            if (!grade.pass && sampleFailure === undefined)
-                sampleFailure = res.text;
-            observedModelId = res.modelId ?? observedModelId;
-            passes += grade.pass ? 1 : 0;
-            scoreSum += grade.qualitativeScore;
-            totalRuns++;
-            matrix.record({
-                modelId: res.modelId ?? opts.modelId ?? 'unknown',
-                taskType,
-                toolCallCount: res.toolCallCount,
-                inputTokens: res.inputTokens,
-                outputTokens: res.outputTokens,
-                latencyMs: res.latencyMs,
-                pass: grade.pass,
-                qualitativeScore: grade.qualitativeScore,
-                taskFingerprint,
-                experimentTag: opts.experimentTag,
-                split,
-                benchmarkSource: opts.benchmarkSource,
-                ...(opts.harnessRef ? { harnessRef: opts.harnessRef } : {}),
-                ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
-                ...(opts.strategy ? { strategy: opts.strategy } : {}),
-                ...(opts.toolProfile && opts.toolProfile !== 'full' ? { toolProfile: opts.toolProfile } : {}),
-            });
-            opts.onRun?.({ taskId: task.id, run: i + 1, pass: grade.pass, qualitativeScore: grade.qualitativeScore });
-        }
-        const passRate = Math.round((passes / runs) * 1000) / 1000;
-        const meanScore = Math.round((scoreSum / runs) * 100) / 100;
+/** Per-task accumulator shared by runBench and runBenchPaired: one arm × one task. */
+class TaskBench {
+    task;
+    runner;
+    matrix;
+    opts;
+    taskType;
+    taskFingerprint;
+    passes = 0;
+    scores = [];
+    sampleFailure;
+    observedModelId;
+    constructor(task, runner, matrix, opts) {
+        this.task = task;
+        this.runner = runner;
+        this.matrix = matrix;
+        this.opts = opts;
+        this.taskType = task.taskType ?? classifyTask(task.prompt).taskType;
+        this.taskFingerprint = ModelRouterMatrix.fingerprintTask(task.prompt, JSON.stringify(task.verifier));
+        this.observedModelId = opts.modelId;
+    }
+    /** One run: execute, grade, record to the matrix, notify. */
+    async runOnce() {
+        const { task, opts } = this;
+        const res = await this.runner.run(task.prompt, { model: opts.modelId });
+        const grade = await gradeRun(res.text, task.verifier, opts.judge, { toolUses: res.toolUses });
+        if (!grade.pass && this.sampleFailure === undefined)
+            this.sampleFailure = res.text;
+        this.observedModelId = res.modelId ?? this.observedModelId;
+        this.passes += grade.pass ? 1 : 0;
+        this.scores.push(grade.qualitativeScore);
+        this.matrix.record({
+            modelId: res.modelId ?? opts.modelId ?? 'unknown',
+            taskType: this.taskType,
+            toolCallCount: res.toolCallCount,
+            inputTokens: res.inputTokens,
+            outputTokens: res.outputTokens,
+            latencyMs: res.latencyMs,
+            pass: grade.pass,
+            qualitativeScore: grade.qualitativeScore,
+            taskFingerprint: this.taskFingerprint,
+            experimentTag: opts.experimentTag,
+            split: opts.split ?? 'train',
+            benchmarkSource: opts.benchmarkSource,
+            ...(opts.harnessRef ? { harnessRef: opts.harnessRef } : {}),
+            ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
+            ...(opts.strategy ? { strategy: opts.strategy } : {}),
+            ...(opts.toolProfile && opts.toolProfile !== 'full' ? { toolProfile: opts.toolProfile } : {}),
+        });
+        opts.onRun?.({ taskId: task.id, run: this.scores.length, pass: grade.pass, qualitativeScore: grade.qualitativeScore });
+    }
+    /** Summarize (and seed the backlog on a train failure). Returns the seeded count (0|1). */
+    finish() {
+        const { task, opts } = this;
+        const runs = this.scores.length;
+        const split = opts.split ?? 'train';
+        const scoreSum = this.scores.reduce((a, b) => a + b, 0);
+        const passRate = runs ? Math.round((this.passes / runs) * 1000) / 1000 : 0;
+        const meanScore = runs ? Math.round((scoreSum / runs) * 100) / 100 : 0;
+        const scoreSd = Math.round(sampleSd(this.scores) * 100) / 100;
+        let seeded = 0;
         // Deterministic backlog seeding: a sub-100% pass-rate is direct evidence of a
         // harness weakness, so seed it here rather than relying on the model to call the
         // ResearchBacklog tool. Idempotent by title (`Bench failure: <taskId>`), so a
@@ -172,12 +191,12 @@ export async function runBench(tasks, runner, matrix, opts) {
             const sampleWeight = Math.min(1, runs / 3); // 1 run = weak evidence; ~3+ = solid
             const confidence = Math.round(Math.min(0.9, Math.max(0.25, 0.3 + 0.6 * failRate * sampleWeight)) * 100) / 100;
             const severity = meanScore <= 10 ? 'high' : meanScore < 60 ? 'medium' : 'low';
-            const clip = (s, n) => (s.length > n ? s.slice(0, n) + '…' : s);
+            const clip = (x, n) => (x.length > n ? x.slice(0, n) + '…' : x);
             try {
                 opts.backlog.add({
                     title: `Bench failure: ${task.id}`,
                     description: `Harness failed benchmark task "${task.id}" (${task.verifier.type} verifier). ` +
-                        `passRate=${passes}/${runs}, meanScore=${meanScore}. Prompt: ${clip(task.prompt, 200)}`,
+                        `passRate=${this.passes}/${runs}, meanScore=${meanScore}. Prompt: ${clip(task.prompt, 200)}`,
                     bugClass: opts.deficiencyBugClass ?? 'Other',
                     severity,
                     impact: 3,
@@ -185,27 +204,80 @@ export async function runBench(tasks, runner, matrix, opts) {
                     confidence,
                     discoveredRound: opts.discoveredRound,
                     discoveredRef: opts.discoveredRef,
-                    affectedModels: observedModelId ? [observedModelId] : undefined,
-                    affectedTaskFingerprints: [taskFingerprint],
-                    notes: `taskType=${taskType}; split=${split}; sample output: ${clip(sampleFailure ?? '(empty output)', 300)}`,
+                    affectedModels: this.observedModelId ? [this.observedModelId] : undefined,
+                    affectedTaskFingerprints: [this.taskFingerprint],
+                    notes: `taskType=${this.taskType}; split=${split}; sample output: ${clip(this.sampleFailure ?? '(empty output)', 300)}`,
                 });
-                seededDeficiencies++;
+                seeded = 1;
             }
             catch {
                 // A backlog write error must never fail the benchmark itself.
             }
         }
-        taskSummaries.push({ taskId: task.id, taskFingerprint, taskType, runs, passRate, meanScore });
+        return {
+            summary: { taskId: task.id, taskFingerprint: this.taskFingerprint, taskType: this.taskType, runs, passRate, meanScore, scoreSd },
+            seeded,
+        };
+    }
+}
+function summarize(opts, benches) {
+    const tasks = [];
+    let seededDeficiencies = 0;
+    for (const b of benches) {
+        const { summary, seeded } = b.finish();
+        tasks.push(summary);
+        seededDeficiencies += seeded;
     }
     return {
         experimentTag: opts.experimentTag,
-        split,
+        split: opts.split ?? 'train',
         harnessRef: opts.harnessRef,
-        totalRuns,
-        tasks: taskSummaries,
+        totalRuns: tasks.reduce((s, t) => s + t.runs, 0),
+        tasks,
         seededDeficiencies,
     };
 }
+export async function runBench(tasks, runner, matrix, opts) {
+    const runs = opts.runs ?? 2;
+    const benches = [];
+    for (const task of tasks) {
+        const b = new TaskBench(task, runner, matrix, opts);
+        for (let i = 0; i < runs; i++)
+            await b.runOnce();
+        benches.push(b);
+    }
+    return summarize(opts, benches);
+}
+/**
+ * Bench a BASE and a CANDIDATE arm INTERLEAVED: for each task and each run index the
+ * two arms run back to back, in a seeded random order. Provider drift (latency, load,
+ * a silent model update, rate limiting) then lands on both arms instead of all on the
+ * one that ran second. Records are identical to two runBench calls (same matrix rows,
+ * same summaries); only the run order differs.
+ */
+export async function runBenchPaired(tasks, base, candidate, matrix, opts) {
+    const runs = opts.runs ?? 2;
+    const rng = mulberry32(opts.orderSeed ?? 0x5eed);
+    const baseOpts = { ...opts, harnessRef: base.harnessRef };
+    const candOpts = { ...opts, harnessRef: candidate.harnessRef };
+    const baseBenches = [];
+    const candBenches = [];
+    const order = [];
+    for (const task of tasks) {
+        const b = new TaskBench(task, base.runner, matrix, baseOpts);
+        const c = new TaskBench(task, candidate.runner, matrix, candOpts);
+        for (let i = 0; i < runs; i++) {
+            const baseFirst = rng() < 0.5;
+            for (const arm of baseFirst ? ['base', 'candidate'] : ['candidate', 'base']) {
+                order.push(arm);
+                await (arm === 'base' ? b : c).runOnce();
+            }
+        }
+        baseBenches.push(b);
+        candBenches.push(c);
+    }
+    return { base: summarize(baseOpts, baseBenches), candidate: summarize(candOpts, candBenches), order };
+}
 /** Validate a parsed task-set JSON value, throwing on malformed entries. */
 export function parseTaskSet(raw, source = '<task-set>') {
     const arr = Array.isArray(raw) ? raw : [raw];

package/dist/training/BenchRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"BenchRunner.js","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAuDnD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAOD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,QAAkB,EAAE,KAAe,EAAE,MAAoB;IACtG,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,UAAU,EAAE,CAAC;QAC5G,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YAChG,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,GAAG,CAAC;YAChB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC;oBAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC9D,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM;oBAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACtF,CAAC;YACD,gFAAgF;YAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACpE,IAAI,KAAa,CAAC;YAClB,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpF,KAAK,GAAG,KAAK,KAAK,MAAM;oBACtB,CAAC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACjF,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC,CAAC,qEAAqE;YACzF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9F,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;YAC3F,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,sBAAsB,KAAK,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrH,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC;YAC3H,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7G,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5H,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,QAAQ,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAsGD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,MAAqB,EACrB,MAAyB,EACzB,IAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;IACpC,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QACrE,MAAM,eAAe,GAAG,iBAAiB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,aAAiC,CAAC;QACtC,IAAI,eAAe,GAAuB,IAAI,CAAC,OAAO,CAAC;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,aAAa,KAAK,SAAS;gBAAE,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC;YACzE,eAAe,GAAG,GAAG,CAAC,OAAO,IAAI,eAAe,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,QAAQ,IAAI,KAAK,CAAC,gBAAgB,CAAC;YACnC,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,MAAM,CAAC;gBACZ,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,SAAS;gBACjD,QAAQ;gBACR,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;gBAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,eAAe;gBACf,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK;gBACL,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5E,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9F,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC5G,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAE5D,8EAA8E;QAC9E,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9G,MAAM,QAAQ,GAAa,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YACxF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBACf,KAAK,EAAE,kBAAkB,IAAI,CAAC,EAAE,EAAE;oBAClC,WAAW,EACT,kCAAkC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc;wBAC/E,YAAY,MAAM,IAAI,IAAI,eAAe,SAAS,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;oBACzF,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,OAAO;oBAC5C,QAAQ;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;oBACT,UAAU;oBACV,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBAC/D,wBAAwB,EAAE,CAAC,eAAe,CAAC;oBAC3C,KAAK,EAAE,YAAY,QAAQ,WAAW,KAAK,oBAAoB,IAAI,CAAC,aAAa,IAAI,gBAAgB,EAAE,GAAG,CAAC,EAAE;iBAC9G,CAAC,CAAC;gBACH,kBAAkB,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,8DAA8D;YAChE,CAAC;QACH,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK;QACL,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS;QACT,KAAK,EAAE,aAAa;QACpB,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,MAAM,GAAG,YAAY;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnF,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9G,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,CAAC,CAAC,QAAmC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4CAA4C,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClH,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,wDAAwD,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,8CAA8C,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,0DAA0D,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4EAA4E,CAAC,CAAC;QACxH,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAA8B,EAAE,CAAC;IACtH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"BenchRunner.js","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAuDpD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAOD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,QAAkB,EAAE,KAAe,EAAE,MAAoB;IACtG,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,UAAU,EAAE,CAAC;QAC5G,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YAChG,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,GAAG,CAAC;YAChB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC;oBAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC9D,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM;oBAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACtF,CAAC;YACD,gFAAgF;YAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACpE,IAAI,KAAa,CAAC;YAClB,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpF,KAAK,GAAG,KAAK,KAAK,MAAM;oBACtB,CAAC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACjF,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC,CAAC,qEAAqE;YACzF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9F,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;YAC3F,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,sBAAsB,KAAK,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrH,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC;YAC3H,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7G,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5H,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,QAAQ,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAwGD;;;;;GAKG;AACH,sFAAsF;AACtF,MAAM,SAAS;IAQQ;IAAiC;IAAwC;IAA4C;IAPjI,QAAQ,CAAS;IACjB,eAAe,CAAS;IACzB,MAAM,GAAG,CAAC,CAAC;IACX,MAAM,GAAa,EAAE,CAAC;IACtB,aAAa,CAAqB;IAClC,eAAe,CAAqB;IAE5C,YAAqB,IAAc,EAAmB,MAAqB,EAAmB,MAAyB,EAAmB,IAAqB;QAA1I,SAAI,GAAJ,IAAI,CAAU;QAAmB,WAAM,GAAN,MAAM,CAAe;QAAmB,WAAM,GAAN,MAAM,CAAmB;QAAmB,SAAI,GAAJ,IAAI,CAAiB;QAC7J,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QACpE,IAAI,CAAC,eAAe,GAAG,iBAAiB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC;IACtC,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,OAAO;QACX,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC;QACnF,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC;QAC3D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YACjB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,SAAS;YACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO;YAC5B,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9F,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACzH,CAAC;IAED,2FAA2F;IAC3F,MAAM;QACJ,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAC9D,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,8EAA8E;QAC9E,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9G,MAAM,QAAQ,GAAa,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YACxF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBACf,KAAK,EAAE,kBAAkB,IAAI,CAAC,EAAE,EAAE;oBAClC,WAAW,EACT,kCAAkC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc;wBAC/E,YAAY,IAAI,CAAC,MAAM,IAAI,IAAI,eAAe,SAAS,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;oBAC9F,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,OAAO;oBAC5C,QAAQ;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;oBACT,UAAU;oBACV,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,cAAc,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACzE,wBAAwB,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;oBAChD,KAAK,EAAE,YAAY,IAAI,CAAC,QAAQ,WAAW,KAAK,oBAAoB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,gBAAgB,EAAE,GAAG,CAAC,EAAE;iBACxH,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,8DAA8D;YAChE,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE;YAChI,MAAM;SACP,CAAC;IACJ,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAqB,EAAE,OAAoB;IAC5D,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,kBAAkB,IAAI,MAAM,CAAC;IAC/B,CAAC;IACD,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO;QAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK;QACL,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,MAAqB,EACrB,MAAyB,EACzB,IAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE;YAAE,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AASD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAiB,EACjB,IAAoB,EACpB,SAAyB,EACzB,MAAyB,EACzB,IAA8C;IAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAoB,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IAC3E,MAAM,QAAQ,GAAoB,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,CAAC;IAChF,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,MAAM,KAAK,GAA6B,EAAE,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,WAAW,CAAW,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,MAAM,CAAW,EAAE,CAAC;gBAClG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,KAAK,EAAE,CAAC;AACxG,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,MAAM,GAAG,YAAY;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnF,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9G,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,CAAC,CAAC,QAAmC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4CAA4C,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClH,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,wDAAwD,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,8CAA8C,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,0DAA0D,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,4EAA4E,CAAC,CAAC;QACxH,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAA8B,EAAE,CAAC;IACtH,CAAC,CAAC,CAAC;AACL,CAAC"}