@nexus-cortex/core 4.26.0

package/dist/training/AutoResearchStats.js

@@ -0,0 +1,210 @@
+/**
+ * AutoResearchStats — the Monte-Carlo keep/discard gate.
+ *
+ * This is the statistical immune system of the recursive auto-research loop. A
+ * raw `delta > 0` WILL fool a self-improvement loop: with enough parallel
+ * experiments, some "win" on noise alone, the loop merges the noise, and it
+ * compounds. This module replaces raw deltas with three rigorous tests:
+ *
+ *   1. BOOTSTRAP CI on the aggregate score delta — keep only if the CI excludes 0.
+ *   2. PERMUTATION TEST — a one-sided p-value for "candidate is better", computed
+ *      under the exchangeability null (base/candidate labels are swappable).
+ *   3. N-AWARE FWER — when N experiments run in parallel (a swarm), the keep bar
+ *      tightens to control the family-wise error rate. Without this a swarm
+ *      confidently merges noise (multiple-comparisons trap).
+ *
+ * REPRODUCIBILITY: every test is driven by a SEEDED PRNG (mulberry32). Same data
+ * + same seed → same p-value and CI. This is deliberate: the auto-research record
+ * is meant to be public + verifiable (Layer 3), so a decision must be
+ * reproducible by anyone re-running the gate on the same records.
+ *
+ * STRUCTURE: resampling is done at the RUN level WITHIN each task, then averaged
+ * across tasks. This respects the per-task structure (a task with more runs
+ * doesn't dominate; within-task noise is captured) — the correct unit for "did
+ * the harness version improve the eval".
+ *
+ * Pure functions, no I/O. The caller (VersionComparison → ExperimentLedger)
+ * feeds per-task score arrays and writes the returned verdict via
+ * `ExperimentLedger.decide()`.
+ */
+/** mulberry32 — tiny, fast, well-distributed seeded PRNG. Deterministic. */
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return function () {
+        a |= 0;
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+const DEFAULT_SEED = 0x9e3779b9;
+function mean(xs) {
+    if (xs.length === 0)
+        return 0;
+    return xs.reduce((s, x) => s + x, 0) / xs.length;
+}
+function round4(n) { return Math.round(n * 10000) / 10000; }
+/** Resample an array with replacement (bootstrap). */
+function resample(xs, rng) {
+    const n = xs.length;
+    const out = new Array(n);
+    for (let i = 0; i < n; i++)
+        out[i] = xs[(rng() * n) | 0];
+    return out;
+}
+/** Fisher-Yates shuffle (in place) using the seeded RNG. */
+function shuffle(xs, rng) {
+    for (let i = xs.length - 1; i > 0; i--) {
+        const j = (rng() * (i + 1)) | 0;
+        const tmp = xs[i];
+        xs[i] = xs[j];
+        xs[j] = tmp;
+    }
+}
+function percentile(sorted, q) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.min(sorted.length - 1, Math.max(0, Math.round(q * (sorted.length - 1))));
+    return sorted[idx];
+}
+/** Observed aggregate effect: mean across tasks of (candMean - baseMean). */
+export function aggregateEffect(tasks) {
+    if (tasks.length === 0)
+        return 0;
+    return mean(tasks.map(t => mean(t.candScores) - mean(t.baseScores)));
+}
+/**
+ * Bootstrap percentile CI on the aggregate effect. Resamples runs within each
+ * arm of each task, recomputes the aggregate, repeats. Keep requires ciLow > 0.
+ */
+export function bootstrapCI(tasks, opts) {
+    const iterations = opts.iterations ?? 2000;
+    const ci = opts.ci ?? 0.95;
+    const stats = [];
+    for (let b = 0; b < iterations; b++) {
+        const taskDeltas = tasks.map(t => mean(resample(t.candScores, opts.rng)) - mean(resample(t.baseScores, opts.rng)));
+        stats.push(mean(taskDeltas));
+    }
+    stats.sort((x, y) => x - y);
+    const lowQ = (1 - ci) / 2;
+    return {
+        ciLow: round4(percentile(stats, lowQ)),
+        ciHigh: round4(percentile(stats, 1 - lowQ)),
+        effect: round4(aggregateEffect(tasks)),
+    };
+}
+/**
+ * One-sided permutation p-value for "candidate > base". Within each task, pools
+ * the two arms and relabels at random into the original arm sizes (the
+ * exchangeability null), recomputes the aggregate effect, and measures how often
+ * the permuted effect meets or beats the observed one. Add-one estimator
+ * (never returns exactly 0 — a permutation p-value can't certify impossibility).
+ */
+export function permutationPValue(tasks, opts) {
+    const iterations = opts.iterations ?? 2000;
+    const observed = aggregateEffect(tasks);
+    let atLeastAsExtreme = 0;
+    for (let p = 0; p < iterations; p++) {
+        const permDeltas = tasks.map(t => {
+            const pooled = [...t.baseScores, ...t.candScores];
+            shuffle(pooled, opts.rng);
+            const nb = t.baseScores.length;
+            const permBase = pooled.slice(0, nb);
+            const permCand = pooled.slice(nb);
+            return mean(permCand) - mean(permBase);
+        });
+        if (mean(permDeltas) >= observed)
+            atLeastAsExtreme++;
+    }
+    return round4((atLeastAsExtreme + 1) / (iterations + 1));
+}
+/**
+ * Šidák family-wise threshold: the per-experiment alpha that holds the
+ * family-wise error at `alpha` across N independent experiments.
+ * `1 - (1 - alpha)^(1/N)`. Tighter than alpha, slightly looser than Bonferroni.
+ */
+export function sidakThreshold(alpha, nExperiments) {
+    const n = Math.max(1, nExperiments);
+    return round4(1 - Math.pow(1 - alpha, 1 / n));
+}
+/**
+ * Monte-Carlo FWER threshold. Simulates the joint null of N experiments and
+ * returns the per-experiment p-value cutoff that controls the family-wise error
+ * at `alpha` (the alpha-quantile of the family minimum p-value). With independent
+ * experiments this reproduces Šidák — and that's the point: it VALIDATES the
+ * analytic bar and is the extension point for correlated experiments (shared
+ * tasks), where the true threshold is less conservative than Bonferroni/Šidák.
+ * A `correlation` hook can later inject a copula; for now experiments are modeled
+ * independent (conservative, correct).
+ */
+export function mcFwerThreshold(alpha, nExperiments, opts) {
+    const n = Math.max(1, nExperiments);
+    if (n === 1)
+        return round4(alpha);
+    const iterations = opts.iterations ?? 5000;
+    const minPs = [];
+    for (let i = 0; i < iterations; i++) {
+        let minP = 1;
+        for (let j = 0; j < n; j++) {
+            const u = opts.rng(); // null p-value ~ Uniform(0,1)
+            if (u < minP)
+                minP = u;
+        }
+        minPs.push(minP);
+    }
+    minPs.sort((a, b) => a - b);
+    // cutoff = alpha-quantile of the family-minimum-p distribution
+    return round4(percentile(minPs, alpha));
+}
+/**
+ * The full gate. Produces the keep/discard/pending verdict for one experiment,
+ * tested against the FWER-adjusted threshold for a family of
+ * `nFamilyExperiments`. Decision rule:
+ *   - PENDING  if no task has >= minRunsPerArm in both arms (not enough data).
+ *   - KEEP     if ciLow > 0 AND pValue <= alphaAdjusted (real, significant gain).
+ *   - DISCARD  otherwise.
+ *
+ * The returned object maps directly onto ExperimentLedger.decide()'s
+ * DecisionUpdate (decision/pValue/ciLow/ciHigh/fwerAdjusted/nRuns/reason).
+ */
+export function decideExperiment(tasks, opts = {}) {
+    const alpha = opts.alpha ?? 0.05;
+    const nFamily = opts.nFamilyExperiments ?? 1;
+    const minRuns = opts.minRunsPerArm ?? 2;
+    const rng = opts.rng ?? mulberry32(opts.seed ?? DEFAULT_SEED);
+    const usable = tasks.filter(t => t.baseScores.length >= minRuns && t.candScores.length >= minRuns);
+    const nRuns = tasks.reduce((s, t) => s + t.baseScores.length + t.candScores.length, 0);
+    if (usable.length === 0) {
+        return {
+            decision: 'pending',
+            effect: round4(aggregateEffect(tasks)),
+            fwerAdjusted: false,
+            nRuns,
+            nTasks: tasks.length,
+            reason: `insufficient data: no task has >= ${minRuns} runs in both arms`,
+        };
+    }
+    const { ciLow, ciHigh, effect } = bootstrapCI(usable, { iterations: opts.bootstrapIterations, ci: opts.ci, rng });
+    const pValue = permutationPValue(usable, { iterations: opts.permutationIterations, rng });
+    const alphaAdjusted = mcFwerThreshold(alpha, nFamily, { rng });
+    const ciExcludesZero = ciLow > 0;
+    const significant = pValue <= alphaAdjusted;
+    const keep = ciExcludesZero && significant;
+    const reason = keep
+        ? `keep: effect +${effect} (95% CI [${ciLow}, ${ciHigh}] excludes 0), p=${pValue} <= alpha_adj=${alphaAdjusted} (N=${nFamily})`
+        : `discard: ${!ciExcludesZero ? `CI [${ciLow}, ${ciHigh}] includes 0` : `p=${pValue} > alpha_adj=${alphaAdjusted} (N=${nFamily})`}`;
+    return {
+        decision: keep ? 'keep' : 'discard',
+        effect,
+        pValue,
+        ciLow,
+        ciHigh,
+        fwerAdjusted: true,
+        alphaAdjusted,
+        nRuns,
+        nTasks: usable.length,
+        reason,
+    };
+}
+//# sourceMappingURL=AutoResearchStats.js.map

package/dist/training/AutoResearchStats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutoResearchStats.js","sourceRoot":"","sources":["../../src/training/AutoResearchStats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAKH,4EAA4E;AAC5E,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;IACnB,OAAO;QACL,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAyCD,MAAM,YAAY,GAAG,UAAU,CAAC;AAEhC,SAAS,IAAI,CAAC,EAAY;IACxB,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACnD,CAAC;AAED,SAAS,MAAM,CAAC,CAAS,IAAY,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAE5E,sDAAsD;AACtD,SAAS,QAAQ,CAAC,EAAY,EAAE,GAAQ;IACtC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;IAC1D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,SAAS,OAAO,CAAI,EAAO,EAAE,GAAQ;IACnC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,OAAO,MAAM,CAAC,GAAG,CAAE,CAAC;AACtB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAAiB,EACjB,IAAoD;IAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC;IAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACnH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;KACvC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAiB,EACjB,IAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YAClD,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,QAAQ;YAAE,gBAAgB,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,MAAM,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,YAAoB;IAChE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,YAAoB,EACpB,IAAuC;IAEvC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,8BAA8B;YACpD,IAAI,CAAC,GAAG,IAAI;gBAAE,IAAI,GAAG,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,+DAA+D;IAC/D,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAiB,EAAE,OAAoB,EAAE;IACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEvF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACtC,YAAY,EAAE,KAAK;YACnB,KAAK;YACL,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,qCAAqC,OAAO,oBAAoB;SACzE,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IAClH,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1F,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IAE/D,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,MAAM,IAAI,aAAa,CAAC;IAC5C,MAAM,IAAI,GAAG,cAAc,IAAI,WAAW,CAAC;IAE3C,MAAM,MAAM,GAAG,IAAI;QACjB,CAAC,CAAC,iBAAiB,MAAM,aAAa,KAAK,KAAK,MAAM,oBAAoB,MAAM,iBAAiB,aAAa,OAAO,OAAO,GAAG;QAC/H,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC,CAAC,CAAC,KAAK,MAAM,gBAAgB,aAAa,OAAO,OAAO,GAAG,EAAE,CAAC;IAEtI,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACnC,MAAM;QACN,MAAM;QACN,KAAK;QACL,MAAM;QACN,YAAY,EAAE,IAAI;QAClB,aAAa;QACb,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/training/BenchRunner.d.ts

@@ -0,0 +1,177 @@
+/**
+ * BenchRunner — the missing GRADER that produces REAL scored matrix records.
+ *
+ * The orchestrator's auto-record (MODEL_ROUTER_RECORD) writes a liveness STUB
+ * (`qualitativeScore = hasText ? 75 : 0`) — fine for "is the model alive", useless
+ * for "did the candidate harness get better". The auto-research decision layer
+ * consumes `qualitativeScore` as the per-run signal, so without a real grader
+ * every base/candidate comparison is a flat tie → the gate always says discard.
+ *
+ * This module runs a TASK SET through the harness and grades each run with a
+ * deterministic (or LLM-judge) verifier, writing the REAL pass/qualitativeScore
+ * to `router-matrix.jsonl` under a given experimentTag/split. The CLI
+ * (`cortex autoresearch bench`) wires a concrete HarnessRunner; here everything
+ * is pure + injectable so it unit-tests with no network.
+ *
+ * FLOW: orchestrator (nexus side) runs this in the base worktree (→ records under
+ * harnessRef=base) and the candidate worktree (→ harnessRef=candidate), then calls
+ * `cortex autoresearch evaluate --base … --candidate …` which compares them.
+ * One bench run = one harness build; the base/candidate axis is the git ref the
+ * record is auto-stamped with (override via opts.harnessRef for single-box tests).
+ */
+import { ModelRouterMatrix } from './ModelRouterMatrix.js';
+import type { NewDeficiency } from './ResearchBacklog.js';
+export type Verifier = 
+/** output, normalized, must equal `expected` exactly. */
+{
+    type: 'exact';
+    expected: string;
+    normalize?: boolean;
+}
+/** `pattern` (RegExp) must match somewhere in the output. */
+ | {
+    type: 'regex';
+    pattern: string;
+    flags?: string;
+}
+/** every string in `all` must appear — PARTIAL CREDIT: score = found/total×100.
+ *  This is the workhorse: graded continuous scores (not just pass/fail) give the
+ *  bootstrap/permutation gate real signal to separate base from candidate. */
+ | {
+    type: 'contains';
+    all: string[];
+    caseInsensitive?: boolean;
+}
+/** delegated to an injected judge fn (e.g. the deepseek helper). */
+ | {
+    type: 'llm-judge';
+    rubric: string;
+}
+/** Extract a NUMBER from the output and score it — the continuous-metric path for
+ *  non-cortex targets (ROI, latency, accuracy, tour length, …). `extract` is a regex
+ *  whose capture group 1 (or whole match) is the number; default = the LAST number in
+ *  the output (evals usually print the metric last). `direction` orients "better".
+ *  `best`/`worst` linearly map value→0-100 (omit → the raw oriented value is the score;
+ *  the gate is relative either way). `target` sets the pass threshold (>= for maximize,
+ *  <= for minimize); omit → any extracted number passes, so a crashed/non-numeric run
+ *  fails (score 0) and seeds the backlog. */
+ | {
+    type: 'numeric';
+    direction: 'maximize' | 'minimize';
+    extract?: string;
+    best?: number;
+    worst?: number;
+    target?: number;
+};
+export interface TaskSpec {
+    id: string;
+    prompt: string;
+    verifier: Verifier;
+    /** T1–T5; if omitted, classifyTask(prompt) decides. */
+    taskType?: string;
+}
+/** Optional LLM judge — injected so core stays network-free. */
+export type JudgeFn = (output: string, rubric: string) => Promise<{
+    pass: boolean;
+    qualitativeScore: number;
+}>;
+export interface GradeResult {
+    pass: boolean;
+    /** 0–100. Continuous where possible (partial credit) so the gate can separate arms. */
+    qualitativeScore: number;
+    detail?: string;
+}
+/**
+ * Grade one model output against a verifier. Pure for the deterministic types;
+ * `contains` gives partial credit. `llm-judge` requires an injected judge.
+ */
+export declare function gradeRun(output: string, verifier: Verifier, judge?: JudgeFn): Promise<GradeResult>;
+export interface HarnessRunResult {
+    text: string;
+    modelId?: string;
+    inputTokens: number;
+    outputTokens: number;
+    toolCallCount: number;
+    latencyMs: number;
+}
+export interface HarnessRunner {
+    run(prompt: string, opts?: {
+        model?: string;
+    }): Promise<HarnessRunResult>;
+}
+/**
+ * Deterministic backlog seeding sink. `ResearchBacklog` satisfies this directly
+ * (its `add(d: NewDeficiency)`); injected (not imported) so `runBench` stays pure
+ * and fs/network-free for unit tests.
+ */
+export interface DeficiencySink {
+    add(d: NewDeficiency): unknown;
+}
+export interface RunBenchOptions {
+    experimentTag: string;
+    runs?: number;
+    split?: 'train' | 'holdout';
+    modelId?: string;
+    benchmarkSource?: string;
+    /** override the auto-stamped git SHA (single-box base/candidate simulation). */
+    harnessRef?: string;
+    judge?: JudgeFn;
+    /** progress callback (one per graded run). */
+    onRun?: (info: {
+        taskId: string;
+        run: number;
+        pass: boolean;
+        qualitativeScore: number;
+    }) => void;
+    /**
+     * Deterministic backlog seeding. When provided, any task whose pass-rate < 1 across
+     * its runs idempotently seeds a deficiency here — so a failing verifier records a
+     * harness weakness even if the model never calls the ResearchBacklog tool. Confidence
+     * scales with failure consistency × run count, so a flaky one-off lands low-priority
+     * (and triage's confidence weighting sinks it) rather than as a false positive. Omit
+     * the sink ⇒ no seeding (unchanged behavior). Only the `train` split seeds — a holdout
+     * failure is verification signal for a specific fix, not a newly-discovered deficiency.
+     */
+    backlog?: DeficiencySink;
+    /** Provenance stamped onto seeded deficiencies (e.g. the bench round / harness SHA). */
+    discoveredRound?: string;
+    discoveredRef?: string;
+    /** Harness area for seeded deficiencies (ResearchBacklog `bugClass`; default 'Other'). */
+    deficiencyBugClass?: string;
+    /**
+     * Effectiveness-layer arm identity. When the auto-research PM dispatches a varied
+     * arm (a distinct persona/strategy at a chosen temperature), pass them here so each
+     * scored record carries the arm — getStrategyScores/recommendStrategy then rank the
+     * (model × temperature × strategy) combinations, not just models. Both optional and
+     * back-compatible; omit them and records score exactly as before.
+     */
+    temperature?: number;
+    strategy?: string;
+}
+export interface BenchTaskSummary {
+    taskId: string;
+    taskFingerprint: string;
+    taskType: string;
+    runs: number;
+    passRate: number;
+    meanScore: number;
+}
+export interface BenchSummary {
+    experimentTag: string;
+    split: 'train' | 'holdout';
+    harnessRef?: string;
+    totalRuns: number;
+    tasks: BenchTaskSummary[];
+    /** Number of deficiencies seeded into `opts.backlog` this run (0 when no sink). */
+    seededDeficiencies: number;
+}
+/**
+ * Run each task `runs` times through `runner`, grade each output, and write a
+ * REAL scored BenchmarkRecord to the matrix (harnessRef auto-stamped unless
+ * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
+ * comparability key the gate joins on).
+ */
+export declare function runBench(tasks: TaskSpec[], runner: HarnessRunner, matrix: ModelRouterMatrix, opts: RunBenchOptions): Promise<BenchSummary>;
+/** Validate a parsed task-set JSON value, throwing on malformed entries. */
+export declare function parseTaskSet(raw: unknown, source?: string): TaskSpec[];
+//# sourceMappingURL=BenchRunner.d.ts.map

package/dist/training/BenchRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BenchRunner.d.ts","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,sBAAsB,CAAC;AAMpE,MAAM,MAAM,QAAQ;AAClB,yDAAyD;AACvD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE;AAC1D,6DAA6D;GAC3D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD;;8EAE8E;GAC5E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE;AAChE,oEAAoE;GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACvC;;;;;;;6CAO6C;GAC3C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,UAAU,GAAG,UAAU,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9H,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE/G,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,CA2DxG;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC3E;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC;CAChC;AAMD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,8CAA8C;IAC9C,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACjG;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,mFAAmF;IACnF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,YAAY,CAAC,CA8FvB;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAe,GAAG,QAAQ,EAAE,CAe5E"}

package/dist/training/BenchRunner.js

@@ -0,0 +1,211 @@
+/**
+ * BenchRunner — the missing GRADER that produces REAL scored matrix records.
+ *
+ * The orchestrator's auto-record (MODEL_ROUTER_RECORD) writes a liveness STUB
+ * (`qualitativeScore = hasText ? 75 : 0`) — fine for "is the model alive", useless
+ * for "did the candidate harness get better". The auto-research decision layer
+ * consumes `qualitativeScore` as the per-run signal, so without a real grader
+ * every base/candidate comparison is a flat tie → the gate always says discard.
+ *
+ * This module runs a TASK SET through the harness and grades each run with a
+ * deterministic (or LLM-judge) verifier, writing the REAL pass/qualitativeScore
+ * to `router-matrix.jsonl` under a given experimentTag/split. The CLI
+ * (`cortex autoresearch bench`) wires a concrete HarnessRunner; here everything
+ * is pure + injectable so it unit-tests with no network.
+ *
+ * FLOW: orchestrator (nexus side) runs this in the base worktree (→ records under
+ * harnessRef=base) and the candidate worktree (→ harnessRef=candidate), then calls
+ * `cortex autoresearch evaluate --base … --candidate …` which compares them.
+ * One bench run = one harness build; the base/candidate axis is the git ref the
+ * record is auto-stamped with (override via opts.harnessRef for single-box tests).
+ */
+import { ModelRouterMatrix } from './ModelRouterMatrix.js';
+import { classifyTask } from './TaskClassifier.js';
+function normalizeWs(s) {
+    return s.replace(/\s+/g, ' ').trim().toLowerCase();
+}
+/**
+ * Grade one model output against a verifier. Pure for the deterministic types;
+ * `contains` gives partial credit. `llm-judge` requires an injected judge.
+ */
+export async function gradeRun(output, verifier, judge) {
+    switch (verifier.type) {
+        case 'exact': {
+            const a = verifier.normalize ? normalizeWs(output) : output.trim();
+            const b = verifier.normalize ? normalizeWs(verifier.expected) : verifier.expected.trim();
+            const pass = a === b;
+            return { pass, qualitativeScore: pass ? 100 : 0 };
+        }
+        case 'regex': {
+            const re = new RegExp(verifier.pattern, verifier.flags);
+            const pass = re.test(output);
+            return { pass, qualitativeScore: pass ? 100 : 0 };
+        }
+        case 'contains': {
+            const hay = verifier.caseInsensitive ? output.toLowerCase() : output;
+            const needles = verifier.all.map(n => (verifier.caseInsensitive ? n.toLowerCase() : n));
+            const found = needles.filter(n => hay.includes(n)).length;
+            const total = needles.length || 1;
+            const qualitativeScore = Math.round((found / total) * 100);
+            return { pass: found === needles.length, qualitativeScore, detail: `${found}/${needles.length} matched` };
+        }
+        case 'llm-judge': {
+            if (!judge)
+                throw new Error("gradeRun: verifier type 'llm-judge' requires an injected JudgeFn");
+            return judge(output, verifier.rubric);
+        }
+        case 'numeric': {
+            let value = NaN;
+            if (verifier.extract) {
+                const m = output.match(new RegExp(verifier.extract));
+                if (m)
+                    value = parseFloat(m[1] ?? m[0]);
+            }
+            else {
+                const all = output.match(/-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/g);
+                if (all && all.length)
+                    value = parseFloat(all[all.length - 1]);
+            }
+            if (!Number.isFinite(value)) {
+                return { pass: false, qualitativeScore: 0, detail: 'numeric: no number extracted' };
+            }
+            // Orient so higher = better, then optionally linear-map [worst,best] → [0,100].
+            const oriented = verifier.direction === 'minimize' ? -value : value;
+            let score;
+            if (verifier.best != null && verifier.worst != null) {
+                const bestO = verifier.direction === 'minimize' ? -verifier.best : verifier.best;
+                const worstO = verifier.direction === 'minimize' ? -verifier.worst : verifier.worst;
+                score = bestO === worstO
+                    ? (oriented >= bestO ? 100 : 0)
+                    : Math.max(0, Math.min(100, ((oriented - worstO) / (bestO - worstO)) * 100));
+            }
+            else {
+                score = oriented; // raw oriented value; the gate compares base vs candidate relatively
+            }
+            const pass = verifier.target == null
+                ? true
+                : (verifier.direction === 'minimize' ? value <= verifier.target : value >= verifier.target);
+            return { pass, qualitativeScore: Math.round(score * 100) / 100, detail: `value=${value}` };
+        }
+        default: {
+            const _exhaustive = verifier;
+            throw new Error(`gradeRun: unknown verifier ${JSON.stringify(_exhaustive)}`);
+        }
+    }
+}
+/**
+ * Run each task `runs` times through `runner`, grade each output, and write a
+ * REAL scored BenchmarkRecord to the matrix (harnessRef auto-stamped unless
+ * overridden; taskFingerprint = hash of prompt+verifier so it's the stable
+ * comparability key the gate joins on).
+ */
+export async function runBench(tasks, runner, matrix, opts) {
+    const runs = opts.runs ?? 2;
+    const split = opts.split ?? 'train';
+    const taskSummaries = [];
+    let totalRuns = 0;
+    let seededDeficiencies = 0;
+    for (const task of tasks) {
+        const taskType = task.taskType ?? classifyTask(task.prompt).taskType;
+        const taskFingerprint = ModelRouterMatrix.fingerprintTask(task.prompt, JSON.stringify(task.verifier));
+        let passes = 0;
+        let scoreSum = 0;
+        let sampleFailure;
+        let observedModelId = opts.modelId;
+        for (let i = 0; i < runs; i++) {
+            const res = await runner.run(task.prompt, { model: opts.modelId });
+            const grade = await gradeRun(res.text, task.verifier, opts.judge);
+            if (!grade.pass && sampleFailure === undefined)
+                sampleFailure = res.text;
+            observedModelId = res.modelId ?? observedModelId;
+            passes += grade.pass ? 1 : 0;
+            scoreSum += grade.qualitativeScore;
+            totalRuns++;
+            matrix.record({
+                modelId: res.modelId ?? opts.modelId ?? 'unknown',
+                taskType,
+                toolCallCount: res.toolCallCount,
+                inputTokens: res.inputTokens,
+                outputTokens: res.outputTokens,
+                latencyMs: res.latencyMs,
+                pass: grade.pass,
+                qualitativeScore: grade.qualitativeScore,
+                taskFingerprint,
+                experimentTag: opts.experimentTag,
+                split,
+                benchmarkSource: opts.benchmarkSource,
+                ...(opts.harnessRef ? { harnessRef: opts.harnessRef } : {}),
+                ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
+                ...(opts.strategy ? { strategy: opts.strategy } : {}),
+            });
+            opts.onRun?.({ taskId: task.id, run: i + 1, pass: grade.pass, qualitativeScore: grade.qualitativeScore });
+        }
+        const passRate = Math.round((passes / runs) * 1000) / 1000;
+        const meanScore = Math.round((scoreSum / runs) * 100) / 100;
+        // Deterministic backlog seeding: a sub-100% pass-rate is direct evidence of a
+        // harness weakness, so seed it here rather than relying on the model to call the
+        // ResearchBacklog tool. Idempotent by title (`Bench failure: <taskId>`), so a
+        // re-run updates the same record. Confidence = how consistently it failed × how
+        // many runs back it, so a flaky one-off (or a possibly-too-strict verifier on a
+        // near-miss) sinks in triage instead of registering as a confident false positive.
+        if (opts.backlog && split !== 'holdout' && passRate < 1) {
+            const failRate = 1 - passRate;
+            const sampleWeight = Math.min(1, runs / 3); // 1 run = weak evidence; ~3+ = solid
+            const confidence = Math.round(Math.min(0.9, Math.max(0.25, 0.3 + 0.6 * failRate * sampleWeight)) * 100) / 100;
+            const severity = meanScore <= 10 ? 'high' : meanScore < 60 ? 'medium' : 'low';
+            const clip = (s, n) => (s.length > n ? s.slice(0, n) + '…' : s);
+            try {
+                opts.backlog.add({
+                    title: `Bench failure: ${task.id}`,
+                    description: `Harness failed benchmark task "${task.id}" (${task.verifier.type} verifier). ` +
+                        `passRate=${passes}/${runs}, meanScore=${meanScore}. Prompt: ${clip(task.prompt, 200)}`,
+                    bugClass: opts.deficiencyBugClass ?? 'Other',
+                    severity,
+                    impact: 3,
+                    effort: 3,
+                    confidence,
+                    discoveredRound: opts.discoveredRound,
+                    discoveredRef: opts.discoveredRef,
+                    affectedModels: observedModelId ? [observedModelId] : undefined,
+                    affectedTaskFingerprints: [taskFingerprint],
+                    notes: `taskType=${taskType}; split=${split}; sample output: ${clip(sampleFailure ?? '(empty output)', 300)}`,
+                });
+                seededDeficiencies++;
+            }
+            catch {
+                // A backlog write error must never fail the benchmark itself.
+            }
+        }
+        taskSummaries.push({ taskId: task.id, taskFingerprint, taskType, runs, passRate, meanScore });
+    }
+    return {
+        experimentTag: opts.experimentTag,
+        split,
+        harnessRef: opts.harnessRef,
+        totalRuns,
+        tasks: taskSummaries,
+        seededDeficiencies,
+    };
+}
+/** Validate a parsed task-set JSON value, throwing on malformed entries. */
+export function parseTaskSet(raw, source = '<task-set>') {
+    const arr = Array.isArray(raw) ? raw : [raw];
+    return arr.map((t, i) => {
+        if (!t || typeof t !== 'object')
+            throw new Error(`${source}[${i}]: not an object`);
+        const o = t;
+        if (typeof o.id !== 'string' || !o.id)
+            throw new Error(`${source}[${i}]: missing 'id'`);
+        if (typeof o.prompt !== 'string' || !o.prompt)
+            throw new Error(`${source}[${i}] (${o.id}): missing 'prompt'`);
+        if (!o.verifier || typeof o.verifier !== 'object')
+            throw new Error(`${source}[${i}] (${o.id}): missing 'verifier'`);
+        const v = o.verifier;
+        const validTypes = ['exact', 'regex', 'contains', 'llm-judge', 'numeric'];
+        if (typeof v.type !== 'string' || !validTypes.includes(v.type)) {
+            throw new Error(`${source}[${i}] (${o.id}): verifier.type must be one of ${validTypes.join('|')}`);
+        }
+        return { id: o.id, prompt: o.prompt, verifier: o.verifier, taskType: o.taskType };
+    });
+}
+//# sourceMappingURL=BenchRunner.js.map

package/dist/training/BenchRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BenchRunner.js","sourceRoot":"","sources":["../../src/training/BenchRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AA8CnD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc,EAAE,QAAkB,EAAE,KAAe;IAChF,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,UAAU,EAAE,CAAC;QAC5G,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;YAChG,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,GAAG,CAAC;YAChB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC;oBAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC9D,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM;oBAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACtF,CAAC;YACD,gFAAgF;YAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACpE,IAAI,KAAa,CAAC;YAClB,IAAI,QAAQ,CAAC,IAAI,IAAI,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpF,KAAK,GAAG,KAAK,KAAK,MAAM;oBACtB,CAAC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACjF,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC,CAAC,qEAAqE;YACzF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9F,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,CAAC;QAC7F,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,QAAQ,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAwFD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,MAAqB,EACrB,MAAyB,EACzB,IAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;IACpC,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QACrE,MAAM,eAAe,GAAG,iBAAiB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,aAAiC,CAAC;QACtC,IAAI,eAAe,GAAuB,IAAI,CAAC,OAAO,CAAC;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAClE,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,aAAa,KAAK,SAAS;gBAAE,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC;YACzE,eAAe,GAAG,GAAG,CAAC,OAAO,IAAI,eAAe,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,QAAQ,IAAI,KAAK,CAAC,gBAAgB,CAAC;YACnC,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,MAAM,CAAC;gBACZ,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,SAAS;gBACjD,QAAQ;gBACR,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;gBAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,eAAe;gBACf,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK;gBACL,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5E,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC5G,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAE5D,8EAA8E;QAC9E,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9G,MAAM,QAAQ,GAAa,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YACxF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBACf,KAAK,EAAE,kBAAkB,IAAI,CAAC,EAAE,EAAE;oBAClC,WAAW,EACT,kCAAkC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc;wBAC/E,YAAY,MAAM,IAAI,IAAI,eAAe,SAAS,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;oBACzF,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,OAAO;oBAC5C,QAAQ;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;oBACT,UAAU;oBACV,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBAC/D,wBAAwB,EAAE,CAAC,eAAe,CAAC;oBAC3C,KAAK,EAAE,YAAY,QAAQ,WAAW,KAAK,oBAAoB,IAAI,CAAC,aAAa,IAAI,gBAAgB,EAAE,GAAG,CAAC,EAAE;iBAC9G,CAAC,CAAC;gBACH,kBAAkB,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,8DAA8D;YAChE,CAAC;QACH,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK;QACL,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS;QACT,KAAK,EAAE,aAAa;QACpB,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,MAAM,GAAG,YAAY;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnF,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9G,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,CAAC,CAAC,QAAmC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAA8B,EAAE,CAAC;IACtH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/training/DecisionPriorInjector.d.ts

@@ -0,0 +1,18 @@
+/**
+ * DecisionPriorInjector — formats a <system-reminder> block from past
+ * decision stats so the orchestrator can prepend it to the next tool
+ * result, giving the model evidence-based priors about how a given
+ * (toolName, input) pattern has fared in past sessions.
+ *
+ * Only emits a reminder when at least one prior FAILED — pure-success
+ * histories produce no output (no actionable signal, just noise).
+ *
+ * When `recent` is supplied, the reminder body includes up to 3 specific
+ * recent outcomes (newest first) — matching the
+ * "up to 3 prior outcomes" pattern. Specific
+ * recent entries give the model far more actionable signal than the
+ * aggregate count alone (e.g. "the last 2 attempts both timed out").
+ */
+import type { Decision, DecisionStats } from './DecisionStore.js';
+export declare function formatPriorReminder(toolName: string, stats: DecisionStats, recent?: Decision[]): string | null;
+//# sourceMappingURL=DecisionPriorInjector.d.ts.map

package/dist/training/DecisionPriorInjector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DecisionPriorInjector.d.ts","sourceRoot":"","sources":["../../src/training/DecisionPriorInjector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAIlE,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,aAAa,EACpB,MAAM,CAAC,EAAE,QAAQ,EAAE,GAClB,MAAM,GAAG,IAAI,CAoBf"}