@tachikomagundam/abathur 0.2.4 → 0.2.6

package/dist/core/stats.js

@@ -26,15 +26,39 @@
 //  - Pareto over (trainScore, tokens, wallS); budget-truncated generations
 //    (complete:false) never enter the set.
 //
+// ACCEPTANCE SEMANTICS (2026-09-24 redesign; design doc lives in the target
+// genome repo's evidence folder, not shipped here): when EvaluateInput.bank
+// carries the historical per-unit variance bank, precision/regression checks
+// price NOISE
+// with bank σ instead of per-arm t(df=n−1) CIs (the campaign-9/10 resolution
+// floor: t(0.05,1)=12.706 turned a 2-point sample into a 1.5883 half-width).
+// A val unit with bank σ clears when P(Δ ≤ −halfWidth) < q_pair AND the
+// acceptance band Z(q)·σ/√n_candidate ≤ halfWidth; the aggregate additionally
+// requires P(gain shift > 0) = Φ(gain/SE) ≥ q_pair, q_pair = 1−(1−ACCEPT_Q)/nPairs.
+// EVERY fail-closed rule ABOVE the bank path is untouched and can never be
+// bypassed by it: budget⇒inconclusive, one-sided n<2⇒indeterminate,
+// both-unmeasured⇒symmetric exclusion. A unit the bank cannot price falls back
+// to the exact legacy t-CI line; a null/absent bank reproduces the pre-2026-09-24
+// gate verbatim. minEffect 0.1 / halfWidth 0.15 point semantics stand.
+//
 // The inverse Student-t lives in stats-math.ts and the Pareto ranking in
 // stats-pareto.ts; both are re-exported here so the public surface of this
 // module — the one todos 9/10/12 consume — is unchanged. Accuracy: |t − table|
 // < 1e-3 for the t-distribution rows checked in stats.test.ts (df=1..30, α=0.05).
 import { EXIT_BLOCKED, EXIT_CANNOT_ANSWER, EXIT_OK } from "../exit.js";
 /** Per-unit replicate scores (train and val units alike). */
-import { studentTQuantile } from "./stats-math.js";
+import { studentTQuantile, normalCdf, normalQuantile } from "./stats-math.js";
 import { seededRandom, paretoFrontier } from "./stats-pareto.js";
-export { studentTQuantile, seededRandom, paretoFrontier };
+export { studentTQuantile, normalCdf, normalQuantile, seededRandom, paretoFrontier };
+/** Nomination acceptance probability (per finalist pair after Bonferroni). */
+export const ACCEPT_Q = 0.9;
+/** q_pair = 1 − (1 − ACCEPT_Q)/nPairs — Bonferroni on the acceptance ERROR rate. */
+export function bonferroniQ(nPairs) {
+    if (!Number.isInteger(nPairs) || nPairs < 1) {
+        throw new RangeError(`bonferroniQ: nPairs=${String(nPairs)} must be an integer >= 1`);
+    }
+    return Math.min(0.9999, 1 - (1 - ACCEPT_Q) / nPairs);
+}
 /** Family-wise alpha across finalist pairs (bonferroniAlpha divides it). */
 export const FAMILY_ALPHA = 0.05;
 /** Verdict → process exit code. inconclusive = 2 (cannot answer), never 0. */
@@ -132,53 +156,121 @@ export function evaluate(input) {
     const candByUnit = new Map(candidate.units.map((u) => [u.unitId, u]));
     const incByUnit = new Map(incumbent.units.map((u) => [u.unitId, u]));
     const failures = [];
-    // 2) HARD RULE nReps ≥ 2: n<2 on ANY candidate unit (or a val unit missing
-    //    from the candidate ⇒ n=0) leaves the variance undefined.
-    for (const unit of candidate.units) {
-        if (unit.scores.length < 2) {
-            failures.push(`n=${unit.scores.length} on unit '${unit.unitId}': variance undefined — indeterminate, never nominated`);
+    // 2) Cross-arm replicate rule. A unit measured cleanly (n>=2) on ONE arm but
+    //    not on the other leaves that comparison's variance undefined ⇒
+    //    indeterminate, never a silent cull and never a nomination (campaign-6:
+    //    the candidate's train unit scored while the incumbent's had timed out;
+    //    the old one-sided drop shifted the incumbent aggregate to the val
+    //    fallback and minted a bogus negative gain). A unit unmeasured on BOTH
+    //    arms is excluded symmetrically — every aggregate and comparison then
+    //    runs on a pool both arms share replicate-for-replicate.
+    const excluded = new Set();
+    for (const unitId of new Set([...incByUnit.keys(), ...candByUnit.keys()])) {
+        const cn = candByUnit.get(unitId)?.scores.length ?? 0;
+        const inn = incByUnit.get(unitId)?.scores.length ?? 0;
+        if (cn >= 2 && inn < 2) {
+            failures.push(`n=${inn} on incumbent unit '${unitId}': baseline variance undefined — indeterminate, never nominated`);
         }
-    }
-    for (const [unitId, incUnit] of incByUnit) {
-        if (incUnit.split === "val" && !candByUnit.has(unitId)) {
-            failures.push(`no replicates for val unit '${unitId}' on candidate '${candidate.runId}' — variance undefined, never nominated`);
+        else if (inn >= 2 && cn < 2) {
+            if (incByUnit.get(unitId)?.split === "val") {
+                failures.push(`no replicates for val unit '${unitId}' on candidate '${candidate.runId}' — variance undefined, never nominated`);
+            }
+            else {
+                failures.push(`n=${cn} on unit '${unitId}': variance undefined — indeterminate, never nominated`);
+            }
+        }
+        else if (cn < 2 && inn < 2) {
+            excluded.add(unitId);
         }
     }
     if (failures.length > 0) {
-        return { ...base, verdict: "indeterminate", exitCode: needsExit("indeterminate"), gain: aggregateScore(candidate.units) - aggregateScore(incumbent.units), unitComparisons: [], failures };
+        return { ...base, verdict: "indeterminate", exitCode: needsExit("indeterminate"), gain: null, unitComparisons: [], failures };
     }
-    // 3) Per-val-unit comparisons + the two precision gates.
+    const candKept = candidate.units.filter((u) => !excluded.has(u.unitId));
+    const incKept = incumbent.units.filter((u) => !excluded.has(u.unitId));
+    const candByUnitKept = new Map(candKept.map((u) => [u.unitId, u]));
+    // 3) Per-val-unit comparisons + the two precision gates. With a bank that
+    //    knows the unit, acceptance semantics (see module header) replace the
+    //    per-arm t-CI machinery FOR THAT UNIT ONLY; bank-unknown units keep the
+    //    legacy lines verbatim. The fail-closed rules above run before and
+    //    independently of any bank.
+    const bank = input.bank ?? null;
+    const qPair = bonferroniQ(nPairs);
+    const acceptanceUnits = [];
     const unitComparisons = [];
     for (const [unitId, incUnit] of incByUnit) {
-        if (incUnit.split !== "val")
+        if (incUnit.split !== "val" || excluded.has(unitId))
             continue;
-        const candUnit = candByUnit.get(unitId);
+        const candUnit = candByUnitKept.get(unitId);
         if (candUnit === undefined)
-            continue; // already ruled indeterminate above
-        const cs = summarizeUnit(candUnit.scores, alpha);
+            continue; // excluded/unreachable
         const incumbentMean = summarizeUnit(incUnit.scores).mean;
+        const cs = summarizeUnit(candUnit.scores, alpha);
         const delta = cs.mean - incumbentMean;
-        unitComparisons.push({
-            unitId,
-            candidateMean: cs.mean,
-            incumbentMean,
-            ciHalfWidth: cs.ciHalfWidth,
-            delta,
-            ciPasses: cs.ciHalfWidth <= stats.halfWidth,
-            passes: delta >= -cs.ciHalfWidth, // ties allowed
-        });
-        if (cs.ciHalfWidth > stats.halfWidth) {
-            failures.push(`CI half-width ${fmt(cs.ciHalfWidth)} > halfWidth ${fmt(stats.halfWidth)} on unit '${unitId}'`);
+        const banked = bank?.get(unitId) ?? null;
+        if (banked === null) {
+            unitComparisons.push({
+                unitId,
+                candidateMean: cs.mean,
+                incumbentMean,
+                ciHalfWidth: cs.ciHalfWidth,
+                delta,
+                ciPasses: cs.ciHalfWidth <= stats.halfWidth,
+                passes: delta >= -cs.ciHalfWidth, // ties allowed
+            });
+            if (cs.ciHalfWidth > stats.halfWidth) {
+                failures.push(`CI half-width ${fmt(cs.ciHalfWidth)} > halfWidth ${fmt(stats.halfWidth)} on unit '${unitId}'`);
+            }
+            if (delta < -cs.ciHalfWidth) {
+                failures.push(`regression on unit '${unitId}': delta ${fmt(delta)} < ${fmt(-cs.ciHalfWidth)} (mean ${fmt(cs.mean)} < ${fmt(incumbentMean)} - ${fmt(cs.ciHalfWidth)})`);
+            }
+            continue;
         }
-        if (delta < -cs.ciHalfWidth) {
-            failures.push(`regression on unit '${unitId}': delta ${fmt(delta)} < ${fmt(-cs.ciHalfWidth)} (mean ${fmt(cs.mean)} < ${fmt(incumbentMean)} - ${fmt(cs.ciHalfWidth)})`);
+        const se = banked.sigma * Math.sqrt(1 / candUnit.scores.length + 1 / incUnit.scores.length);
+        const band = (normalQuantile(qPair) * banked.sigma) / Math.sqrt(candUnit.scores.length);
+        const pRegression = se === 0 ? (delta <= -stats.halfWidth ? 1 : 0) : normalCdf((-stats.halfWidth - delta) / se);
+        const ciPasses = band <= stats.halfWidth;
+        const passes = pRegression < qPair; // ties at −halfWidth sit at Φ(0)=0.5 < q ⇒ still pass
+        unitComparisons.push({ unitId, candidateMean: cs.mean, incumbentMean, ciHalfWidth: band, delta, ciPasses, passes });
+        acceptanceUnits.push({ unitId, sigma: banked.sigma, df: banked.df, delta, se, band, pRegression });
+        if (!ciPasses) {
+            failures.push(`acceptance band ${fmt(band)} > halfWidth ${fmt(stats.halfWidth)} on unit '${unitId}' (bank sigma ${fmt(banked.sigma)}, df ${String(banked.df)})`);
+        }
+        else if (!passes) {
+            failures.push(`regression on unit '${unitId}': delta ${fmt(delta)} — P(Δ ≤ −${fmt(stats.halfWidth)}) = ${fmt(pRegression)} ≥ q ${fmt(qPair)} (bank sigma ${fmt(banked.sigma)})`);
         }
     }
-    // 4) Effect-size floor on the aggregate gain.
-    const gain = aggregateScore(candidate.units) - aggregateScore(incumbent.units);
+    // 4) Effect-size floor on the aggregate gain (shared, symmetric pool) —
+    //    unchanged — plus the acceptance gate: when the bank can price EVERY
+    //    unit of the gain pool, nomination additionally requires P(Δ>0) ≥ q.
+    //    A bank-priced point mass exactly at 0 (se=0, gain=0) fails (pShift=0):
+    //    demonstrated-identical arms are never nominated on a tie.
+    const gain = aggregateScore(candKept) - aggregateScore(incKept);
     if (gain < stats.minEffect) {
         failures.push(`gain ${fmt(gain)} < minEffect ${fmt(stats.minEffect)}`);
     }
+    let gainSe = null;
+    let pShift = null;
+    if (bank !== null) {
+        const train = candKept.filter((u) => u.split === "train");
+        const pool = train.length > 0 ? train : candKept;
+        const incCounts = new Map(incKept.map((u) => [u.unitId, u.scores.length]));
+        const priced = pool.map((u) => bank.get(u.unitId) ?? null);
+        if (priced.every((s) => s !== null)) {
+            const k = pool.length;
+            let variance = 0;
+            pool.forEach((u, index) => {
+                const s = priced[index];
+                variance += ((s.sigma * s.sigma) * (1 / u.scores.length + 1 / (incCounts.get(u.unitId) ?? 0))) / (k * k);
+            });
+            gainSe = Math.sqrt(variance);
+            pShift = gainSe === 0 ? (gain > 0 ? 1 : 0) : normalCdf(gain / gainSe);
+            if (pShift < qPair) {
+                failures.push(`acceptance: P(gain shift > 0) = ${fmt(pShift)} < q ${fmt(qPair)} (se ${fmt(gainSe)}, gain ${fmt(gain)})`);
+            }
+        }
+    }
     const verdict = failures.length === 0 ? "nominated" : "culled";
-    return { ...base, verdict, exitCode: needsExit(verdict), gain, unitComparisons, failures };
+    const acceptance = bank === null ? undefined : { q: qPair, gainSe, pShift, units: acceptanceUnits };
+    return { ...base, verdict, exitCode: needsExit(verdict), gain, unitComparisons, failures, ...(acceptance === undefined ? {} : { acceptance }) };
 }