@tangle-network/agent-eval 0.135.1 → 0.135.2

package/dist/{statistics-RwRNu2__.js → statistics-CnGCLLqc.js}

@@ -1379,6 +1379,39 @@ function wilson(successes, n, confidence = .95) {
 	};
 }
 /**
+* Are these per-item outcomes binary (every value exactly 0 or 1)?
+*
+* The discriminator a promotion gate needs before choosing a paired statistic.
+* On binary outcomes the paired delta vector lives in {-1, 0, +1} and is
+* normally dominated by zeros (both arms solve, or both arms miss, most items),
+* so its MEDIAN is pinned at exactly 0 no matter how large the real shift in
+* success rate is — and a bootstrap CI on that median collapses to [0, 0].
+* A gate keying on `ci.low > threshold` is then structurally unable to see
+* either a gain or a regression. Detect this shape and switch to the
+* paired-binary estimators ({@link mcnemar}, {@link pairedRiskDifference})
+* instead of silently answering "no" forever.
+*
+* Empty input is NOT binary: there is no evidence of the outcome's shape, and
+* defaulting an empty vector into the binary branch would pick a statistic on
+* no data at all.
+*
+* NOT the right discriminator for a gate. It recognises the literal {0, 1}
+* encoding and nothing else, so a pass/fail dimension emitted on 0-100 — which
+* judges in this codebase do routinely — reads as non-binary, and a single
+* partial-credit score in an otherwise pass/fail vector flips it to false while
+* leaving the median just as blind. Gates want {@link pairedBinaryScale} (any
+* two-point encoding). This predicate remains for callers that specifically
+* mean "literally 0/1".
+*/
+function isBinaryOutcomeVector(values) {
+	if (values.length === 0) return false;
+	for (let i = 0; i < values.length; i++) {
+		const v = values[i];
+		if (v !== 0 && v !== 1) return false;
+	}
+	return true;
+}
+/**
 * McNemar's test for paired binary outcomes — the correct significance test for
 * "does treatment change the success rate vs control on the SAME items". Only
 * discordant pairs (one arm right, the other wrong) carry information; concordant
@@ -1419,6 +1452,13 @@ function mcnemar(control, treatment) {
 * the discordant counts, not the independent-samples formula (which overstates
 * the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
 * arrays, control first. Throws on unequal lengths.
+*
+* REPORTING ONLY — do NOT decide a promotion on this interval. The CI is a Wald
+* normal approximation, which badly UNDERCOVERS when only a handful of pairs are
+* discordant: at n = 3 with b = 2, c = 0 it returns [0.133, 1.000], excluding 0,
+* while McNemar's exact test on the same data gives p = 0.50. A gate keying on
+* `lower > 0` would promote noise. Use {@link pairedRiskDifferenceExact}, whose
+* interval is dual to the exact test by construction, for any decision.
 */
 function pairedRiskDifference(control, treatment, confidence = .95) {
 	if (control.length !== treatment.length) throw new Error(`pairedRiskDifference: unequal sample sizes (${control.length} vs ${treatment.length})`);
@@ -1454,6 +1494,279 @@ function pairedRiskDifference(control, treatment, confidence = .95) {
 	};
 }
 /**
+* Paired risk difference with the EXACT CONDITIONAL interval — the estimator a
+* promotion gate may decide on.
+*
+* Conditional on the number of discordant pairs m = b + c, the treatment-win
+* count b is Binomial(m, π) with π = P(treatment wins | discordant), and the
+* risk difference is an exact reparameterisation: RD = (2π − 1)·m/n. So a
+* Clopper-Pearson exact interval for π maps straight onto RD. This buys the
+* property the Wald interval in {@link pairedRiskDifference} does not have:
+*
+*   **`lower > 0` ⟺ McNemar's exact test rejects at α = 1 − confidence.**
+*
+* Clopper-Pearson excludes π = 0.5 exactly when the two-sided exact binomial
+* test of π = 0.5 rejects, and that test IS {@link mcnemar}'s p-value — so the
+* interval and the test can never disagree, and a gate keyed on `lower` cannot
+* promote what the exact test refuses. The exact p is returned in the same
+* object so the two are impossible to compute apart.
+*
+* The interval is conservative (exact intervals over-cover; conditioning on m
+* discards the concordant pairs' information about m itself). That is the
+* correct direction for a promotion gate: it refuses more often, never less.
+*
+* With m = 0 there are no discordant pairs and π is not identified: the result
+* is the degenerate [0, 0] with p = 1. That is NOT evidence of equivalence —
+* callers must treat a zero-width interval as "cannot decide", not as "no
+* difference". Inputs are paired 0/1 (or boolean) arrays, control first.
+* Throws on unequal lengths.
+*/
+function pairedRiskDifferenceExact(control, treatment, confidence = .95) {
+	if (control.length !== treatment.length) throw new Error(`pairedRiskDifferenceExact: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	if (confidence <= 0 || confidence >= 1) throw new Error(`pairedRiskDifferenceExact: confidence must be in (0,1), got ${confidence}`);
+	const n = control.length;
+	if (n === 0) return {
+		n: 0,
+		b: 0,
+		c: 0,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: 0,
+		upper: 0,
+		confidence,
+		pValue: 1
+	};
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const m = b + c;
+	const riskDifference = (b - c) / n;
+	const pValue = binomialSignTwoSided(b, c);
+	if (m === 0) return {
+		n,
+		b,
+		c,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: 0,
+		upper: 0,
+		confidence,
+		pValue
+	};
+	const alpha = 1 - confidence;
+	const piLow = b === 0 ? 0 : betaQuantile(alpha / 2, b, m - b + 1);
+	const piHigh = b === m ? 1 : betaQuantile(1 - alpha / 2, b + 1, m - b);
+	const scale = m / n;
+	return {
+		n,
+		b,
+		c,
+		nDiscordant: m,
+		riskDifference,
+		lower: Math.max(-1, (2 * piLow - 1) * scale),
+		upper: Math.min(1, (2 * piHigh - 1) * scale),
+		confidence,
+		pValue
+	};
+}
+/** Inverse regularized incomplete beta by bisection on
+*  {@link regularizedIncompleteBeta}, which is monotone increasing in x. 80
+*  halvings of [0,1] resolve to ~8e-25, far past the continued fraction's own
+*  3e-15 tolerance, so the quantile is as exact as the CDF it inverts. */
+function betaQuantile(p, a, b) {
+	if (p <= 0) return 0;
+	if (p >= 1) return 1;
+	let lo = 0;
+	let hi = 1;
+	for (let i = 0; i < 80; i++) {
+		const mid = (lo + hi) / 2;
+		if (regularizedIncompleteBeta(mid, a, b) < p) lo = mid;
+		else hi = mid;
+	}
+	return (lo + hi) / 2;
+}
+/**
+* Constrained MLE of q = P(treatment loses) under the hypothesis RD = `delta`.
+*
+* Profiling the two concordant cells out of the multinomial leaves
+* `L(q) = b·log(q+delta) + c·log(q) + e·log(1 − 2q − delta)` with `e = n − b − c`,
+* whose stationary point is the positive root of
+* `2n·q² − [(b + c) − delta·(b + 3c + 2e)]·q − c·delta·(1 − delta) = 0`.
+* At `delta = 0` this returns `(b + c) / 2n`, the familiar null.
+*/
+function constrainedLossRate(b, c, n, delta) {
+	const e = n - b - c;
+	const quadratic = 2 * n;
+	const linear = -(b + c - delta * (b + 3 * c + 2 * e));
+	const constant = -c * delta * (1 - delta);
+	const discriminant = linear * linear - 4 * quadratic * constant;
+	const root = discriminant > 0 ? Math.sqrt(discriminant) : 0;
+	const q = (-linear + root) / (2 * quadratic);
+	return Math.min(Math.max(q, Math.max(0, -delta)), Math.max(0, (1 - delta) / 2));
+}
+/** Tango's score statistic for H0: RD = `delta`. `Var(b − c) = n·(2q + delta −
+*  delta²)` under that hypothesis, evaluated at the constrained MLE of q. */
+function tangoScore(b, c, n, delta) {
+	const numerator = b - c - n * delta;
+	const variance = n * (2 * constrainedLossRate(b, c, n, delta) + delta * (1 - delta));
+	if (!(variance > 0)) {
+		if (numerator === 0) return 0;
+		return numerator > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
+	}
+	return numerator / Math.sqrt(variance);
+}
+/**
+* Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a
+* promotion gate may decide on **at a nonzero margin**.
+*
+* {@link pairedRiskDifferenceExact} conditions on the observed discordant count
+* `m = b + c`, builds a Clopper-Pearson interval for the win share among those
+* `m` pairs, and multiplies by the observed `m/n`. That is exact for testing
+* RD = 0 — it is dual to McNemar — but it is NOT a confidence interval for the
+* population risk difference at a nonzero margin, because the sampling
+* variability of `m/n` itself is discarded. The gap is not academic: with the
+* production caller's `pairedDeltaThreshold: -0.05`, a process whose true risk
+* difference sits exactly on that margin clears a nominal-95 % `lower > margin`
+* check 24.75 % of the time at n = 40 and 43.95 % at n = 76 (2000 replicates
+* each) when the conditional interval decides.
+*
+* Tango's interval inverts the score test of RD = delta, which estimates the
+* nuisance loss rate under each hypothesised delta instead of fixing it at the
+* observed value, so `m` contributes its own uncertainty. It is the method
+* `ratesci::scorepairci` uses for paired risk-difference noninferiority, and it
+* is not conditional, so it stays valid as the margin moves away from zero.
+*
+* The bounds are found by bisecting `tangoScore(delta) = ±z` — the score is
+* monotone decreasing in delta, so each crossing is unique. Inputs are paired
+* 0/1 (or boolean) arrays, control first. Throws on unequal lengths.
+*/
+function pairedRiskDifferenceScore(control, treatment, confidence = .95) {
+	if (control.length !== treatment.length) throw new Error(`pairedRiskDifferenceScore: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	if (confidence <= 0 || confidence >= 1) throw new Error(`pairedRiskDifferenceScore: confidence must be in (0,1), got ${confidence}`);
+	const n = control.length;
+	if (n === 0) return {
+		n: 0,
+		b: 0,
+		c: 0,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: -1,
+		upper: 1,
+		confidence
+	};
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const riskDifference = (b - c) / n;
+	const z = zQuantile(1 - (1 - confidence) / 2);
+	let lo = -1;
+	let hi = riskDifference;
+	for (let i = 0; i < 200; i++) {
+		const mid = (lo + hi) / 2;
+		if (tangoScore(b, c, n, mid) > z) lo = mid;
+		else hi = mid;
+	}
+	const lower = (lo + hi) / 2;
+	let ulo = riskDifference;
+	let uhi = 1;
+	for (let i = 0; i < 200; i++) {
+		const mid = (ulo + uhi) / 2;
+		if (tangoScore(b, c, n, mid) > -z) ulo = mid;
+		else uhi = mid;
+	}
+	const upper = (ulo + uhi) / 2;
+	return {
+		n,
+		b,
+		c,
+		nDiscordant: b + c,
+		riskDifference,
+		lower: Math.max(-1, lower),
+		upper: Math.min(1, upper),
+		confidence
+	};
+}
+/**
+* The common positive level `s` such that EVERY value across both paired arms is
+* exactly 0 or `s` — i.e. the outcome is pass/fail, whatever encoding it arrived
+* in. Returns null when the outcomes are not two-point, when the two arms use
+* different levels, or when no positive value was observed at all (all-zero
+* arms: the level is not identified, and there is nothing to decide anyway).
+*
+* This is the scale-aware successor to {@link isBinaryOutcomeVector}, which only
+* recognises literal {0, 1}. Judges in this codebase emit dimensions on 0-100 as
+* well as [0,1] (see `detectScale` in `campaign/gates/statistical-heldout.ts`),
+* so a pass/fail dimension routinely arrives as {0, 100} and a {0,1}-only test
+* silently sends it down the median path that cannot see it. Any positive level
+* is accepted, not just 1 and 100: for a two-point {0, s} outcome the mean paired
+* delta is exactly s·(b − c)/n, so the binary estimators apply after dividing by
+* s and rescaling the result back into the caller's native units.
+*
+* Non-finite values ⇒ null: an unusable outcome must not be classified as a
+* clean pass/fail shape.
+*/
+function pairedBinaryScale(before, after) {
+	let level = null;
+	for (const arm of [before, after]) for (let i = 0; i < arm.length; i++) {
+		const v = arm[i];
+		if (!Number.isFinite(v)) return null;
+		if (v === 0) continue;
+		if (v < 0) return null;
+		if (level === null) level = v;
+		else if (v !== level) return null;
+	}
+	return level;
+}
+/** Fraction of paired observations whose delta is an exact tie (|after − before|
+*  < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */
+function pairedDeltaTieFraction(before, after) {
+	if (before.length !== after.length) throw new Error(`pairedDeltaTieFraction: unequal sample sizes (${before.length} vs ${after.length})`);
+	const n = before.length;
+	if (n === 0) return 0;
+	let ties = 0;
+	for (let i = 0; i < n; i++) if (Math.abs(after[i] - before[i]) < 1e-9) ties++;
+	return ties / n;
+}
+/**
+* The paired-delta statistic a DECISION is computed on, package-wide.
+*
+* The mean paired delta is the estimator that answers the question a promotion
+* gate asks — "by how much did the candidate move the score" — in the caller's
+* own units, and it equals the aggregate lift everyone quotes. The MEDIAN
+* answers a different question and loses the answer to this one in every regime
+* eval data actually lands in:
+*   - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in
+*     {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI
+*     are pinned at exactly 0 however large the shift. (Decide these on
+*     {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)
+*   - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by
+*     construction, and `ci.low > threshold` then answers "no" forever at a
+*     non-negative threshold and "yes" forever at a negative one.
+*   - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on
+*     integer 0-100, and block scores like {⅔, 1} from averaging pass/fail
+*     leaves, put the median on a coarse lattice whose bootstrap percentiles
+*     land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real
+*     +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —
+*     lower bound exactly 0, so a gate at threshold 0 refuses a real lift.
+* That last case is why there is no tie-fraction threshold here: any cutoff on
+* ties leaves the lattice case open on the other side of it.
+*
+* `heldoutSignificance` has defaulted to the mean since #316 for the same
+* reason. The median remains available per call site for callers who
+* specifically want outlier robustness and accept the blindness.
+*/
+const DECISION_PAIRED_DELTA_STATISTIC = "mean";
+/**
 * Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
 * Language Models Trained on Code"). Given `n` independent samples for one
 * problem of which `c` pass, the probability that at least one of a random k of
@@ -1894,6 +2207,6 @@ function mulberry32(seed) {
 	};
 }
 //#endregion
-export { passAtK as A, studentTCdf as B, pairedBootstrap as C, pairedSignTest as D, pairedRiskDifference as E, spearmanR as F, continuousAgreement as G, normalCdf as H, weightedComposite as I, verbosityBias as J, positionalBias as K, weightedMean as L, ranks as M, requiredPairedSampleSize as N, pairedTTest as O, requiredSampleSize as P, wilcoxonSignedRank as R, normalizeScores as S, pairedMde as T, calibrateJudge as U, studentTQuantile as V, calibrateJudgeContinuous as W, mannWhitneyU as _, WILCOXON_EXACT_MAX_N as a, mcnemarRequiredN as b, cliffsDelta as c, corpusInterRaterAgreement as d, corpusInterRaterAgreementFromJudgeScores as f, interpretCliffs as g, interRaterReliability as h, MANN_WHITNEY_EXACT_MAX_WORK as i, pearsonR as j, partialCredit as k, cohensD as l, holm as m, DEFAULT_PERMUTATIONS as n, benjaminiHochberg as o, eProcess as p, selfPreference as q, MANN_WHITNEY_EXACT_MAX_STATES as r, bonferroni as s, BOOTSTRAP_GATE_MIN_N as t, confidenceInterval as u, mcnemar as v, pairedCohensDz as w, mulberry32 as x, mcnemarPower as y, wilson as z };
+export { selfPreference as $, pairedRiskDifference as A, requiredSampleSize as B, mulberry32 as C, pairedCohensDz as D, pairedBootstrap as E, partialCredit as F, wilson as G, weightedComposite as H, passAtK as I, normalCdf as J, studentTCdf as K, pearsonR as L, pairedRiskDifferenceScore as M, pairedSignTest as N, pairedDeltaTieFraction as O, pairedTTest as P, positionalBias as Q, ranks as R, mcnemarRequiredN as S, pairedBinaryScale as T, weightedMean as U, spearmanR as V, wilcoxonSignedRank as W, calibrateJudgeContinuous as X, calibrateJudge as Y, continuousAgreement as Z, interpretCliffs as _, MANN_WHITNEY_EXACT_MAX_WORK as a, mcnemar as b, bonferroni as c, confidenceInterval as d, verbosityBias as et, corpusInterRaterAgreement as f, interRaterReliability as g, holm as h, MANN_WHITNEY_EXACT_MAX_STATES as i, pairedRiskDifferenceExact as j, pairedMde as k, cliffsDelta as l, eProcess as m, DECISION_PAIRED_DELTA_STATISTIC as n, WILCOXON_EXACT_MAX_N as o, corpusInterRaterAgreementFromJudgeScores as p, studentTQuantile as q, DEFAULT_PERMUTATIONS as r, benjaminiHochberg as s, BOOTSTRAP_GATE_MIN_N as t, cohensD as u, isBinaryOutcomeVector as v, normalizeScores as w, mcnemarPower as x, mannWhitneyU as y, requiredPairedSampleSize as z };
 
-//# sourceMappingURL=statistics-RwRNu2__.js.map
+//# sourceMappingURL=statistics-CnGCLLqc.js.map