@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/paired-arms-D-XRF_fy.js

@@ -0,0 +1,1045 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { i as makeRng, l as lnGamma, o as symmetricTwoSampleSeed, r as binomialSignTwoSided, s as zQuantile, t as assertFiniteSample, u as regularizedIncompleteBeta } from "./internal-BDHPCnjk.js";
+import { r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
+//#region src/statistics/paired-binary.ts
+/**
+* Wilson score interval for a binomial proportion. Correct at small n and near
+* 0/1, where the normal (Wald) approximation produces bounds outside [0, 1] and
+* understates coverage. Use this for any pass-rate / hit-rate / realness-rate
+* CI — the continuous `confidenceInterval` assumes the wrong distribution for a
+* proportion. `n = 0 ⇒ {0, 0, 0}`.
+*/
+function wilson(successes, n, confidence = .95) {
+	if (n <= 0) return {
+		estimate: 0,
+		lower: 0,
+		upper: 0
+	};
+	if (successes < 0 || successes > n) throw new Error(`wilson: successes (${successes}) must be in [0, ${n}]`);
+	const z = zQuantile(1 - (1 - confidence) / 2);
+	const p = successes / n;
+	const z2 = z * z;
+	const denom = 1 + z2 / n;
+	const center = (p + z2 / (2 * n)) / denom;
+	const half = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n) / denom;
+	return {
+		estimate: p,
+		lower: Math.max(0, center - half),
+		upper: Math.min(1, center + half)
+	};
+}
+/**
+* Are these per-item outcomes binary (every value exactly 0 or 1)?
+*
+* The discriminator a promotion gate needs before choosing a paired statistic.
+* On binary outcomes the paired delta vector lives in {-1, 0, +1} and is
+* normally dominated by zeros (both arms solve, or both arms miss, most items),
+* so its MEDIAN is pinned at exactly 0 no matter how large the real shift in
+* success rate is — and a bootstrap CI on that median collapses to [0, 0].
+* A gate keying on `ci.low > threshold` is then structurally unable to see
+* either a gain or a regression. Detect this shape and switch to the
+* paired-binary estimators ({@link mcnemar}, {@link pairedRiskDifference})
+* instead of silently answering "no" forever.
+*
+* Empty input is NOT binary: there is no evidence of the outcome's shape, and
+* defaulting an empty vector into the binary branch would pick a statistic on
+* no data at all.
+*
+* NOT the right discriminator for a gate. It recognises the literal {0, 1}
+* encoding and nothing else, so a pass/fail dimension emitted on 0-100 — which
+* judges in this codebase do routinely — reads as non-binary, and a single
+* partial-credit score in an otherwise pass/fail vector flips it to false while
+* leaving the median just as blind. Gates want {@link pairedBinaryScale} (any
+* two-point encoding). This predicate remains for callers that specifically
+* mean "literally 0/1".
+*/
+function isBinaryOutcomeVector(values) {
+	if (values.length === 0) return false;
+	for (let i = 0; i < values.length; i++) {
+		const v = values[i];
+		if (v !== 0 && v !== 1) return false;
+	}
+	return true;
+}
+/**
+* McNemar's test for paired binary outcomes — the correct significance test for
+* "does treatment change the success rate vs control on the SAME items". Only
+* discordant pairs (one arm right, the other wrong) carry information; concordant
+* pairs are uninformative, so a paired t-test / two-proportion z-test on the raw
+* rates is wrong here. The p-value is exact: under H0 the b "treatment-wins" are
+* Binomial(b + c, 0.5), so the two-sided p is the doubled binomial tail — correct
+* at the small discordant counts typical of eval runs (no continuity-corrected
+* chi-square approximation needed, though it is returned as `statistic` for
+* reference). Inputs are paired 0/1 (or boolean) arrays, control first to match
+* the module's (before, after) convention. Throws on unequal lengths.
+*/
+function mcnemar(control, treatment) {
+	if (control.length !== treatment.length) throw new Error(`mcnemar: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	const n = control.length;
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const nDiscordant = b + c;
+	const statistic = nDiscordant === 0 ? 0 : (Math.abs(b - c) - 1) ** 2 / nDiscordant;
+	return {
+		n,
+		nDiscordant,
+		b,
+		c,
+		statistic,
+		pValue: binomialSignTwoSided(b, c)
+	};
+}
+/**
+* Paired risk difference (the effect-size companion to {@link mcnemar}): the
+* change in success rate p(treatment) − p(control) on matched items, which for
+* paired binary data equals (b − c) / n. The CI uses the paired variance from
+* the discordant counts, not the independent-samples formula (which overstates
+* the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
+* arrays, control first. Throws on unequal lengths.
+*
+* REPORTING ONLY — do NOT decide a promotion on this interval. The CI is a Wald
+* normal approximation, which badly UNDERCOVERS when only a handful of pairs are
+* discordant: at n = 3 with b = 2, c = 0 it returns [0.133, 1.000], excluding 0,
+* while McNemar's exact test on the same data gives p = 0.50. A gate keying on
+* `lower > 0` would promote noise. Use {@link pairedRiskDifferenceExact}, whose
+* interval is dual to the exact test by construction, for any decision.
+*/
+function pairedRiskDifference(control, treatment, confidence = .95) {
+	if (control.length !== treatment.length) throw new Error(`pairedRiskDifference: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	const n = control.length;
+	if (n === 0) return {
+		n: 0,
+		b: 0,
+		c: 0,
+		riskDifference: 0,
+		lower: 0,
+		upper: 0,
+		confidence
+	};
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const rd = (b - c) / n;
+	const variance = (b + c - (b - c) ** 2 / n) / (n * n);
+	const half = zQuantile(1 - (1 - confidence) / 2) * Math.sqrt(Math.max(0, variance));
+	return {
+		n,
+		b,
+		c,
+		riskDifference: rd,
+		lower: Math.max(-1, rd - half),
+		upper: Math.min(1, rd + half),
+		confidence
+	};
+}
+/**
+* Paired risk difference with the EXACT CONDITIONAL interval — the estimator a
+* promotion gate may decide on.
+*
+* Conditional on the number of discordant pairs m = b + c, the treatment-win
+* count b is Binomial(m, π) with π = P(treatment wins | discordant), and the
+* risk difference is an exact reparameterisation: RD = (2π − 1)·m/n. So a
+* Clopper-Pearson exact interval for π maps straight onto RD. This buys the
+* property the Wald interval in {@link pairedRiskDifference} does not have:
+*
+*   **`lower > 0` ⟺ McNemar's exact test rejects at α = 1 − confidence.**
+*
+* Clopper-Pearson excludes π = 0.5 exactly when the two-sided exact binomial
+* test of π = 0.5 rejects, and that test IS {@link mcnemar}'s p-value — so the
+* interval and the test can never disagree, and a gate keyed on `lower` cannot
+* promote what the exact test refuses. The exact p is returned in the same
+* object so the two are impossible to compute apart.
+*
+* The interval is conservative (exact intervals over-cover; conditioning on m
+* discards the concordant pairs' information about m itself). That is the
+* correct direction for a promotion gate: it refuses more often, never less.
+*
+* With m = 0 there are no discordant pairs and π is not identified: the result
+* is the degenerate [0, 0] with p = 1. That is NOT evidence of equivalence —
+* callers must treat a zero-width interval as "cannot decide", not as "no
+* difference". Inputs are paired 0/1 (or boolean) arrays, control first.
+* Throws on unequal lengths.
+*/
+function pairedRiskDifferenceExact(control, treatment, confidence = .95) {
+	if (control.length !== treatment.length) throw new Error(`pairedRiskDifferenceExact: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	if (confidence <= 0 || confidence >= 1) throw new Error(`pairedRiskDifferenceExact: confidence must be in (0,1), got ${confidence}`);
+	const n = control.length;
+	if (n === 0) return {
+		n: 0,
+		b: 0,
+		c: 0,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: 0,
+		upper: 0,
+		confidence,
+		pValue: 1
+	};
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const m = b + c;
+	const riskDifference = (b - c) / n;
+	const pValue = binomialSignTwoSided(b, c);
+	if (m === 0) return {
+		n,
+		b,
+		c,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: 0,
+		upper: 0,
+		confidence,
+		pValue
+	};
+	const alpha = 1 - confidence;
+	const piLow = b === 0 ? 0 : betaQuantile(alpha / 2, b, m - b + 1);
+	const piHigh = b === m ? 1 : betaQuantile(1 - alpha / 2, b + 1, m - b);
+	const scale = m / n;
+	return {
+		n,
+		b,
+		c,
+		nDiscordant: m,
+		riskDifference,
+		lower: Math.max(-1, (2 * piLow - 1) * scale),
+		upper: Math.min(1, (2 * piHigh - 1) * scale),
+		confidence,
+		pValue
+	};
+}
+/** Inverse regularized incomplete beta by bisection on
+*  {@link regularizedIncompleteBeta}, which is monotone increasing in x. 80
+*  halvings of [0,1] resolve to ~8e-25, far past the continued fraction's own
+*  3e-15 tolerance, so the quantile is as exact as the CDF it inverts. */
+function betaQuantile(p, a, b) {
+	if (p <= 0) return 0;
+	if (p >= 1) return 1;
+	let lo = 0;
+	let hi = 1;
+	for (let i = 0; i < 80; i++) {
+		const mid = (lo + hi) / 2;
+		if (regularizedIncompleteBeta(mid, a, b) < p) lo = mid;
+		else hi = mid;
+	}
+	return (lo + hi) / 2;
+}
+/**
+* Constrained MLE of q = P(treatment loses) under the hypothesis RD = `delta`.
+*
+* Profiling the two concordant cells out of the multinomial leaves
+* `L(q) = b·log(q+delta) + c·log(q) + e·log(1 − 2q − delta)` with `e = n − b − c`,
+* whose stationary point is the positive root of
+* `2n·q² − [(b + c) − delta·(b + 3c + 2e)]·q − c·delta·(1 − delta) = 0`.
+* At `delta = 0` this returns `(b + c) / 2n`, the familiar null.
+*/
+function constrainedLossRate(b, c, n, delta) {
+	const e = n - b - c;
+	const quadratic = 2 * n;
+	const linear = -(b + c - delta * (b + 3 * c + 2 * e));
+	const constant = -c * delta * (1 - delta);
+	const discriminant = linear * linear - 4 * quadratic * constant;
+	const root = discriminant > 0 ? Math.sqrt(discriminant) : 0;
+	const q = (-linear + root) / (2 * quadratic);
+	return Math.min(Math.max(q, Math.max(0, -delta)), Math.max(0, (1 - delta) / 2));
+}
+/** Tango's score statistic for H0: RD = `delta`. `Var(b − c) = n·(2q + delta −
+*  delta²)` under that hypothesis, evaluated at the constrained MLE of q. */
+function tangoScore(b, c, n, delta) {
+	const numerator = b - c - n * delta;
+	const variance = n * (2 * constrainedLossRate(b, c, n, delta) + delta * (1 - delta));
+	if (!(variance > 0)) {
+		if (numerator === 0) return 0;
+		return numerator > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
+	}
+	return numerator / Math.sqrt(variance);
+}
+/**
+* Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a
+* promotion gate may decide on **at a nonzero margin**.
+*
+* {@link pairedRiskDifferenceExact} conditions on the observed discordant count
+* `m = b + c`, builds a Clopper-Pearson interval for the win share among those
+* `m` pairs, and multiplies by the observed `m/n`. That is exact for testing
+* RD = 0 — it is dual to McNemar — but it is NOT a confidence interval for the
+* population risk difference at a nonzero margin, because the sampling
+* variability of `m/n` itself is discarded. The gap is not academic: with the
+* production caller's `pairedDeltaThreshold: -0.05`, a process whose true risk
+* difference sits exactly on that margin clears a nominal-95 % `lower > margin`
+* check 24.75 % of the time at n = 40 and 43.95 % at n = 76 (2000 replicates
+* each) when the conditional interval decides.
+*
+* Tango's interval inverts the score test of RD = delta, which estimates the
+* nuisance loss rate under each hypothesised delta instead of fixing it at the
+* observed value, so `m` contributes its own uncertainty. It is the method
+* `ratesci::scorepairci` uses for paired risk-difference noninferiority, and it
+* is not conditional, so it stays valid as the margin moves away from zero.
+*
+* The bounds are found by bisecting `tangoScore(delta) = ±z` — the score is
+* monotone decreasing in delta, so each crossing is unique. Inputs are paired
+* 0/1 (or boolean) arrays, control first. Throws on unequal lengths.
+*/
+function pairedRiskDifferenceScore(control, treatment, confidence = .95) {
+	if (control.length !== treatment.length) throw new Error(`pairedRiskDifferenceScore: unequal sample sizes (${control.length} vs ${treatment.length})`);
+	if (confidence <= 0 || confidence >= 1) throw new Error(`pairedRiskDifferenceScore: confidence must be in (0,1), got ${confidence}`);
+	const n = control.length;
+	if (n === 0) return {
+		n: 0,
+		b: 0,
+		c: 0,
+		nDiscordant: 0,
+		riskDifference: 0,
+		lower: -1,
+		upper: 1,
+		confidence
+	};
+	let b = 0;
+	let c = 0;
+	for (let i = 0; i < n; i++) {
+		const ctrl = control[i] ? 1 : 0;
+		const treat = treatment[i] ? 1 : 0;
+		if (treat === 1 && ctrl === 0) b++;
+		else if (treat === 0 && ctrl === 1) c++;
+	}
+	const riskDifference = (b - c) / n;
+	const z = zQuantile(1 - (1 - confidence) / 2);
+	let lo = -1;
+	let hi = riskDifference;
+	for (let i = 0; i < 200; i++) {
+		const mid = (lo + hi) / 2;
+		if (tangoScore(b, c, n, mid) > z) lo = mid;
+		else hi = mid;
+	}
+	const lower = (lo + hi) / 2;
+	let ulo = riskDifference;
+	let uhi = 1;
+	for (let i = 0; i < 200; i++) {
+		const mid = (ulo + uhi) / 2;
+		if (tangoScore(b, c, n, mid) > -z) ulo = mid;
+		else uhi = mid;
+	}
+	const upper = (ulo + uhi) / 2;
+	return {
+		n,
+		b,
+		c,
+		nDiscordant: b + c,
+		riskDifference,
+		lower: Math.max(-1, lower),
+		upper: Math.min(1, upper),
+		confidence
+	};
+}
+/**
+* The common positive level `s` such that EVERY value across both paired arms is
+* exactly 0 or `s` — i.e. the outcome is pass/fail, whatever encoding it arrived
+* in. Returns null when the outcomes are not two-point, when the two arms use
+* different levels, or when no positive value was observed at all (all-zero
+* arms: the level is not identified, and there is nothing to decide anyway).
+*
+* This is the scale-aware successor to {@link isBinaryOutcomeVector}, which only
+* recognises literal {0, 1}. Judges in this codebase emit dimensions on 0-100 as
+* well as [0,1] (see `detectScale` in `campaign/gates/statistical-heldout.ts`),
+* so a pass/fail dimension routinely arrives as {0, 100} and a {0,1}-only test
+* silently sends it down the median path that cannot see it. Any positive level
+* is accepted, not just 1 and 100: for a two-point {0, s} outcome the mean paired
+* delta is exactly s·(b − c)/n, so the binary estimators apply after dividing by
+* s and rescaling the result back into the caller's native units.
+*
+* Non-finite values ⇒ null: an unusable outcome must not be classified as a
+* clean pass/fail shape.
+*/
+function pairedBinaryScale(before, after) {
+	let level = null;
+	for (const arm of [before, after]) for (let i = 0; i < arm.length; i++) {
+		const v = arm[i];
+		if (!Number.isFinite(v)) return null;
+		if (v === 0) continue;
+		if (v < 0) return null;
+		if (level === null) level = v;
+		else if (v !== level) return null;
+	}
+	return level;
+}
+/**
+* Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
+* Language Models Trained on Code"). Given `n` independent samples for one
+* problem of which `c` pass, the probability that at least one of a random k of
+* them passes is 1 − C(n−c, k) / C(n, k). Estimating pass@k as "did any of the
+* first k pass" is biased high at small n; this is the variance-reduced estimator
+* averaged implicitly over all k-subsets. Average the per-problem values across
+* the suite for the corpus pass@k. Computed in the numerically stable product
+* form. Requires 1 ≤ k ≤ n and 0 ≤ c ≤ n.
+*/
+function passAtK(n, c, k) {
+	if (!Number.isInteger(n) || !Number.isInteger(c) || !Number.isInteger(k)) throw new Error(`passAtK: n, c, k must be integers (got n=${n}, c=${c}, k=${k})`);
+	if (k < 1 || k > n || c < 0 || c > n) throw new Error(`passAtK: require 1 ≤ k ≤ n and 0 ≤ c ≤ n (got n=${n}, c=${c}, k=${k})`);
+	if (n - c < k) return 1;
+	let prob = 1;
+	for (let i = n - c + 1; i <= n; i++) prob *= 1 - k / i;
+	return 1 - prob;
+}
+//#endregion
+//#region src/math/normal.ts
+/**
+* Standard normal cumulative distribution using Abramowitz and Stegun 7.1.26.
+*
+* The approximation is evaluated as erf(x / sqrt(2)). Computing the negative
+* tail from the complementary term avoids cancellation when x is far below 0.
+* The maximum absolute CDF error is approximately 7.5e-8.
+*/
+function normalCdf(x) {
+	if (x === 0) return .5;
+	const a1 = .254829592;
+	const a2 = -.284496736;
+	const a3 = 1.421413741;
+	const a4 = -1.453152027;
+	const a5 = 1.061405429;
+	const p = .3275911;
+	const scaled = Math.abs(x) / Math.SQRT2;
+	const t = 1 / (1 + p * scaled);
+	const complement = ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-scaled * scaled);
+	return x < 0 ? complement / 2 : 1 - complement / 2;
+}
+//#endregion
+//#region src/statistics/rank-tests.ts
+/** Maximum dynamic-programming cells used by an exact two-sample rank test. */
+const MANN_WHITNEY_EXACT_MAX_STATES = 8192;
+/** Maximum inner-loop transitions used by an exact two-sample rank test. */
+const MANN_WHITNEY_EXACT_MAX_WORK = 25e4;
+/** Non-zero differences up to which the signed-rank null is enumerated exactly. */
+const WILCOXON_EXACT_MAX_N = 20;
+/** Resamples used when a rank test falls back to Monte Carlo permutation. */
+const DEFAULT_PERMUTATIONS = 1e5;
+/**
+* Mann-Whitney U — two independent samples, no distributional assumption.
+*
+* Exact conditional (permutation) p by default when the dynamic program fits
+* {@link MANN_WHITNEY_EXACT_MAX_STATES} cells and
+* {@link MANN_WHITNEY_EXACT_MAX_WORK} transitions, seeded Monte Carlo
+* permutation above those limits. This keeps imbalanced designs such as 1+24
+* exact without admitting expensive balanced designs merely because they have
+* the same total size. Throws on non-finite input and on `method:
+* 'asymptotic'` where an exact answer is available. Empty input yields `p = 1,
+* pFloor = 1` — no design, no attainable evidence.
+*/
+function mannWhitneyU(a, b, opts = {}) {
+	assertFiniteSample("mannWhitneyU", "a", a);
+	assertFiniteSample("mannWhitneyU", "b", b);
+	const n1 = a.length;
+	const n2 = b.length;
+	if (n1 === 0 || n2 === 0) return {
+		u: 0,
+		uA: 0,
+		p: 1,
+		method: "exact",
+		pFloor: 1
+	};
+	const total = n1 + n2;
+	const combined = [...a.map((v) => ({
+		v,
+		fromA: true
+	})), ...b.map((v) => ({
+		v,
+		fromA: false
+	}))].sort((x, y) => x.v - y.v);
+	const { midranks, tieTerm } = midranksWithTieTerm(combined.map((entry) => entry.v));
+	let rankSumA = 0;
+	for (let k = 0; k < total; k++) if (combined[k].fromA) rankSumA += midranks[k];
+	const uA = rankSumA - n1 * (n1 + 1) / 2;
+	const u = Math.min(uA, n1 * n2 - uA);
+	const doubled = midranks.map((rank) => Math.round(rank * 2));
+	const doubledDeviation = Math.abs(2 * uA - n1 * n2);
+	const selectedN = Math.min(n1, n2);
+	const otherN = total - selectedN;
+	const exactCost = exactTwoSampleCost(doubled, selectedN);
+	const designFloor = exactTwoSampleFloor(doubled, selectedN);
+	const method = selectRankTestMethod("mannWhitneyU", opts.method ?? "auto", `n1=${n1}, n2=${n2}`, exactCost.states <= 8192 && exactCost.work <= 25e4, designFloor, `${MANN_WHITNEY_EXACT_MAX_STATES.toLocaleString("en-US")} states and ${MANN_WHITNEY_EXACT_MAX_WORK.toLocaleString("en-US")} transitions`);
+	if (method === "exact") {
+		const { p, pFloor } = exactTwoSampleP(doubled, selectedN, otherN, doubledDeviation);
+		return {
+			u,
+			uA,
+			p,
+			method,
+			pFloor
+		};
+	}
+	if (method === "asymptotic") return {
+		u,
+		uA,
+		p: asymptoticTwoSidedP(doubledDeviation / 2, twoSampleSigma(n1, n2, total, tieTerm)),
+		method,
+		pFloor: designFloor
+	};
+	const permutations = resolvePermutations("mannWhitneyU", opts.permutations);
+	const rng = opts.seed === void 0 ? makeRng(symmetricTwoSampleSeed(a, b)) : makeRng(opts.seed);
+	let atLeastAsExtreme = 0;
+	const pool = [...doubled];
+	for (let iteration = 0; iteration < permutations; iteration++) {
+		let doubledRankSum = 0;
+		for (let k = 0; k < selectedN; k++) {
+			const pick = k + Math.floor(rng() * (total - k));
+			const swapped = pool[pick];
+			pool[pick] = pool[k];
+			pool[k] = swapped;
+			doubledRankSum += swapped;
+		}
+		if (Math.abs(doubledRankSum - selectedN * (selectedN + 1) - selectedN * otherN) >= doubledDeviation) atLeastAsExtreme++;
+	}
+	const pFloor = Math.max(1 / (permutations + 1), designFloor);
+	return {
+		u,
+		uA,
+		p: Math.max((1 + atLeastAsExtreme) / (permutations + 1), pFloor),
+		method,
+		pFloor
+	};
+}
+/**
+* Wilcoxon signed-rank — paired, no distributional assumption on the deltas.
+*
+* Exact conditional (sign-flip) p by default at `n ≤
+* {@link WILCOXON_EXACT_MAX_N}` non-zero differences, seeded Monte Carlo
+* permutation above it. Throws on non-finite input and on `method:
+* 'asymptotic'` where an exact answer is available.
+*
+* `n` is the count of NON-ZERO differences: exact ties are dropped before
+* ranking, so a run of tied pairs shrinks the design and raises `pFloor`.
+* All-tied input yields `p = 1, pFloor = 1` — no attainable evidence, which
+* `pFloor` states rather than leaving `p = 1` to be read as a measured null.
+*/
+function wilcoxonSignedRank(before, after, opts = {}) {
+	if (before.length !== after.length) throw new ValidationError(`wilcoxonSignedRank: unequal sample sizes (${before.length} vs ${after.length})`);
+	assertFiniteSample("wilcoxonSignedRank", "before", before);
+	assertFiniteSample("wilcoxonSignedRank", "after", after);
+	const diffs = before.map((b, i) => after[i] - b).filter((d) => d !== 0);
+	const n = diffs.length;
+	if (n === 0) return {
+		w: 0,
+		p: 1,
+		method: "exact",
+		pFloor: 1,
+		nNonZero: 0
+	};
+	const order = diffs.map((d, i) => ({
+		abs: Math.abs(d),
+		i
+	})).sort((x, y) => x.abs - y.abs);
+	const { midranks, tieTerm } = midranksWithTieTerm(order.map((entry) => entry.abs));
+	const ranks = new Array(n);
+	for (let k = 0; k < n; k++) ranks[order[k].i] = midranks[k];
+	let wPlus = 0;
+	for (let k = 0; k < n; k++) if (diffs[k] > 0) wPlus += ranks[k];
+	const doubled = midranks.map((rank) => Math.round(rank * 2));
+	const doubledDeviation = Math.abs(2 * wPlus - n * (n + 1) / 2);
+	const designFloor = Math.min(1, 2 ** (1 - n));
+	const method = selectRankTestMethod("wilcoxonSignedRank", opts.method ?? "auto", `n=${n} non-zero differences`, n <= 20, designFloor, `20 non-zero differences`);
+	if (method === "exact") {
+		const { p, pFloor } = exactSignedRankP(doubled, doubledDeviation);
+		return {
+			w: wPlus,
+			p,
+			method,
+			pFloor,
+			nNonZero: n
+		};
+	}
+	if (method === "asymptotic") {
+		const variance = n * (n + 1) * (2 * n + 1) / 24 - tieTerm / 48;
+		return {
+			w: wPlus,
+			p: asymptoticTwoSidedP(doubledDeviation / 2, Math.sqrt(variance)),
+			method,
+			pFloor: designFloor,
+			nNonZero: n
+		};
+	}
+	const permutations = resolvePermutations("wilcoxonSignedRank", opts.permutations);
+	const rng = makeRng(opts.seed, before, after);
+	const doubledCentre = n * (n + 1) / 2;
+	let atLeastAsExtreme = 0;
+	for (let iteration = 0; iteration < permutations; iteration++) {
+		let doubledWPlus = 0;
+		for (let k = 0; k < n; k++) if (rng() < .5) doubledWPlus += doubled[k];
+		if (Math.abs(doubledWPlus - doubledCentre) >= doubledDeviation) atLeastAsExtreme++;
+	}
+	return {
+		w: wPlus,
+		p: (1 + atLeastAsExtreme) / (permutations + 1),
+		method,
+		pFloor: Math.max(1 / (permutations + 1), designFloor),
+		nNonZero: n
+	};
+}
+/**
+* Average ranks over an ASCENDING-sorted array, plus `Σ(t³ − t)` over tie
+* groups of size `t` — the correction term both asymptotic rank-test variances
+* need.
+*/
+function midranksWithTieTerm(sorted) {
+	const midranks = new Array(sorted.length);
+	let tieTerm = 0;
+	let i = 0;
+	while (i < sorted.length) {
+		let j = i;
+		while (j < sorted.length && sorted[j] === sorted[i]) j++;
+		const average = (i + 1 + j) / 2;
+		for (let k = i; k < j; k++) midranks[k] = average;
+		const groupSize = j - i;
+		if (groupSize > 1) tieTerm += groupSize ** 3 - groupSize;
+		i = j;
+	}
+	return {
+		midranks,
+		tieTerm
+	};
+}
+function selectRankTestMethod(fn, request, design, exactFeasible, designFloor, threshold) {
+	if (request === "auto") return exactFeasible ? "exact" : "permutation";
+	if (request === "exact") {
+		if (exactFeasible) return "exact";
+		throw new ValidationError(`${fn}: method 'exact' is out of range at ${design} — enumeration is bounded by ${threshold}. Use 'auto' for the seeded Monte Carlo permutation, which converges to the same answer.`);
+	}
+	if (exactFeasible) throw new ValidationError(`${fn}: method 'asymptotic' is refused at ${design} — the exact p-grid at this design starts at ${formatProbability(designFloor)}, so an asymptotic p below it describes no attainable outcome. Use method 'exact' (the default) or add repetitions past ${threshold}.`);
+	return "asymptotic";
+}
+function resolvePermutations(fn, permutations) {
+	if (permutations === void 0) return DEFAULT_PERMUTATIONS;
+	if (!Number.isInteger(permutations) || permutations < 1) throw new ValidationError(`${fn}: permutations must be a positive integer, got ${permutations}`);
+	return permutations;
+}
+/** Two-sided normal-approximation tail with the continuity correction. */
+function asymptoticTwoSidedP(deviation, sigma) {
+	if (!(sigma > 0)) return 1;
+	return Math.min(1, 2 * (1 - normalCdf(Math.max(0, deviation - .5) / sigma)));
+}
+/** SD of U under the permutation null, corrected for the realised ties. The
+*  tie term reduces (N+1) and reaches it exactly when every value is tied, so
+*  the variance floors at 0 rather than going negative. */
+function twoSampleSigma(n1, n2, total, tieTerm) {
+	if (total < 2) return 0;
+	const variance = n1 * n2 / 12 * (total + 1 - tieTerm / (total * (total - 1)));
+	return Math.sqrt(Math.max(0, variance));
+}
+function logChoose(n, k) {
+	return lnGamma(n + 1) - lnGamma(k + 1) - lnGamma(n - k + 1);
+}
+function formatProbability(value) {
+	return value >= 1e-4 || value === 0 ? value.toFixed(4) : value.toExponential(3);
+}
+/**
+* Exact DP allocation and loop count for this observed rank vector.
+*
+* The smaller arm is sufficient because selecting its complement produces the
+* same two-sided U deviation while using fewer rows in the state table.
+*/
+function exactTwoSampleCost(doubledRanks, selectedN) {
+	const maxSum = doubledRanks.reduce((sum, rank) => sum + rank, 0);
+	let work = 0;
+	for (let placed = 0; placed < doubledRanks.length; placed++) work += Math.min(selectedN, placed + 1) * (maxSum - doubledRanks[placed] + 1);
+	return {
+		states: (selectedN + 1) * (maxSum + 1),
+		work
+	};
+}
+/**
+* Smallest attainable two-sided p under the observed ties.
+*
+* Only subsets with the minimum or maximum rank sum can attain the largest
+* deviation. Their multiplicity is the number of ways to choose within the
+* tie group at each boundary, so this calculation is exact without allocating
+* the full null distribution.
+*/
+function exactTwoSampleFloor(doubledRanks, selectedN) {
+	const total = doubledRanks.length;
+	const otherN = total - selectedN;
+	const minimumSum = doubledRanks.slice(0, selectedN).reduce((sum, rank) => sum + rank, 0);
+	const maximumSum = doubledRanks.slice(total - selectedN).reduce((sum, rank) => sum + rank, 0);
+	if (minimumSum === maximumSum) return 1;
+	const centre = selectedN * (selectedN + 1) + selectedN * otherN;
+	const minimumDeviation = Math.abs(minimumSum - centre);
+	const maximumDeviation = Math.abs(maximumSum - centre);
+	const totalLogWays = logChoose(total, selectedN);
+	const minimumMass = Math.exp(logExtremeSubsetWays(doubledRanks, selectedN, "minimum") - totalLogWays);
+	const maximumMass = Math.exp(logExtremeSubsetWays(doubledRanks, selectedN, "maximum") - totalLogWays);
+	if (minimumDeviation > maximumDeviation) return minimumMass;
+	if (maximumDeviation > minimumDeviation) return maximumMass;
+	return Math.min(1, minimumMass + maximumMass);
+}
+function logExtremeSubsetWays(sortedRanks, selectedN, side) {
+	const boundaryIndex = side === "minimum" ? selectedN - 1 : sortedRanks.length - selectedN;
+	const boundary = sortedRanks[boundaryIndex];
+	let first = boundaryIndex;
+	let afterLast = boundaryIndex + 1;
+	while (first > 0 && sortedRanks[first - 1] === boundary) first--;
+	while (afterLast < sortedRanks.length && sortedRanks[afterLast] === boundary) afterLast++;
+	return logChoose(afterLast - first, selectedN - (side === "minimum" ? first : sortedRanks.length - afterLast));
+}
+/**
+* Exact conditional two-sided p for the two-sample rank test.
+*
+* Convolves the observed doubled midranks into the null distribution of group
+* a's rank sum over every `C(n₁+n₂, n₁)` split — identical to enumerating the
+* splits, but `O(N·n₁·ΣR)` instead of `O(C(N,n₁)·n₁n₂)`. Conditioning on the
+* realised multiset makes the tie handling exact rather than a correction.
+*
+* The null is symmetric about `n₁n₂/2` (negating every value maps `U → n₁n₂ −
+* U` and permutes the split set onto itself), so the two-sided p is the mass
+* at least as far from the centre as the observation.
+*/
+function exactTwoSampleP(doubledRanks, n1, n2, doubledDeviation) {
+	const maxSum = doubledRanks.reduce((sum, rank) => sum + rank, 0);
+	const width = maxSum + 1;
+	const ways = Array.from({ length: n1 + 1 }, () => new Float64Array(width));
+	ways[0][0] = 1;
+	let placed = 0;
+	for (const rank of doubledRanks) {
+		for (let k = Math.min(n1, placed + 1); k >= 1; k--) {
+			const from = ways[k - 1];
+			const into = ways[k];
+			for (let sum = maxSum - rank; sum >= 0; sum--) {
+				const count = from[sum];
+				if (count !== 0) into[sum + rank] += count;
+			}
+		}
+		placed++;
+	}
+	const shift = n1 * (n1 + 1) + n1 * n2;
+	const chosen = ways[n1];
+	let totalWays = 0;
+	let extremeWays = 0;
+	let tailWays = 0;
+	let maxDeviation = -1;
+	for (let sum = 0; sum < width; sum++) {
+		const count = chosen[sum];
+		if (count === 0) continue;
+		totalWays += count;
+		const deviation = Math.abs(sum - shift);
+		if (deviation >= doubledDeviation) tailWays += count;
+		if (deviation > maxDeviation) {
+			maxDeviation = deviation;
+			extremeWays = count;
+		} else if (deviation === maxDeviation) extremeWays += count;
+	}
+	return {
+		p: tailWays / totalWays,
+		pFloor: extremeWays / totalWays
+	};
+}
+/**
+* Exact conditional two-sided p for the paired signed-rank test.
+*
+* Convolves the observed doubled absolute midranks over all `2ⁿ` sign
+* assignments in `O(n·ΣR)`. Probabilities rather than counts keep `2ⁿ` off the
+* arithmetic. The null is symmetric about `n(n+1)/4`.
+*/
+function exactSignedRankP(doubledRanks, doubledDeviation) {
+	const maxSum = doubledRanks.reduce((sum, rank) => sum + rank, 0);
+	const width = maxSum + 1;
+	let mass = new Float64Array(width);
+	mass[0] = 1;
+	for (const rank of doubledRanks) {
+		const next = new Float64Array(width);
+		for (let sum = 0; sum < width; sum++) {
+			const probability = mass[sum];
+			if (probability === 0) continue;
+			next[sum] += probability * .5;
+			next[sum + rank] += probability * .5;
+		}
+		mass = next;
+	}
+	const centre = maxSum / 2;
+	let tail = 0;
+	let extreme = 0;
+	let maxDeviation = -1;
+	for (let sum = 0; sum < width; sum++) {
+		const probability = mass[sum];
+		if (probability === 0) continue;
+		const deviation = Math.abs(sum - centre);
+		if (deviation >= doubledDeviation) tail += probability;
+		if (deviation > maxDeviation) {
+			maxDeviation = deviation;
+			extreme = probability;
+		} else if (deviation === maxDeviation) extreme += probability;
+	}
+	return {
+		p: Math.min(1, tail),
+		pFloor: Math.min(1, extreme)
+	};
+}
+//#endregion
+//#region src/paired-arms.ts
+/**
+* Matched-pair arm comparison — "did the treatment arm beat the baseline arm
+* on the SAME work items?"
+*
+* An arm A/B over run records is only trustworthy when it is PAIRED: the same
+* task/scenario/seed evaluated under both arms, compared item-by-item, so
+* inter-item difficulty variance cancels instead of masquerading as an arm
+* effect. This module owns the two error-prone steps every consumer otherwise
+* hand-rolls:
+*
+*   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`
+*      within multi-rep items), with leftovers REPORTED rather than silently
+*      dropped (a silently unbalanced pairing biases every paired statistic
+*      downstream). Pairing never keys on outcome content: matching reps by
+*      their outcomes deflates discordant-pair counts and makes McNemar
+*      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.
+*   2. Composition — feeding the matched pairs to the correct paired
+*      estimators that already live in `statistics`: `mcnemar` +
+*      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +
+*      `wilcoxonSignedRank` for continuous metrics. No statistic is
+*      re-implemented here.
+*
+* The row shape is deliberately structural — callers project a `RunRecord`
+* (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are
+* caller-supplied parameters; the module ships no domain literal.
+*/
+/**
+* Match rows across two arms into (baseline, treatment) pairs by `pairKey`.
+*
+* A `pairKey` with at most one row per arm pairs directly, no `repKey`
+* needed. A `pairKey` with multiple reps in either arm requires `repKey` on
+* every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)
+* match — pairing is keyed purely on row identity, never on outcome content
+* (outcome-keyed matching deflates discordant counts and biases McNemar), and
+* is therefore independent of input order. Reps whose `repKey` has no
+* counterpart in the other arm, and items present in only one arm, land in
+* the unpaired lists — reported, never truncated.
+*
+* Fail-loud: throws when either named arm has zero rows (an unknown arm
+* name would otherwise read as "everything unpaired"), when the two arm
+* names are equal, when a multi-rep `pairKey` has a row without `repKey`, or
+* when a (`pairKey`, arm) group repeats a `repKey` (the match would be
+* ambiguous).
+*/
+function pairArms(rows, opts) {
+	const { baselineArm, treatmentArm } = opts;
+	if (baselineArm === treatmentArm) throw new ValidationError(`pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`);
+	const byArm = /* @__PURE__ */ new Map();
+	const armsSeen = /* @__PURE__ */ new Set();
+	for (const row of rows) {
+		armsSeen.add(row.arm);
+		if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
+		const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
+		const group = byKey.get(row.pairKey) ?? [];
+		group.push(row);
+		byKey.set(row.pairKey, group);
+		byArm.set(row.arm, byKey);
+	}
+	for (const arm of [baselineArm, treatmentArm]) if (!byArm.has(arm)) throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${[...armsSeen].sort().join(", ") || "<none>"})`);
+	const baselineByKey = byArm.get(baselineArm);
+	const treatmentByKey = byArm.get(treatmentArm);
+	const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
+	const pairs = [];
+	const unpairedBaseline = [];
+	const unpairedTreatment = [];
+	for (const pairKey of allKeys) {
+		const b = baselineByKey.get(pairKey) ?? [];
+		const t = treatmentByKey.get(pairKey) ?? [];
+		if (b.length <= 1 && t.length <= 1) {
+			if (b.length === 1 && t.length === 1) {
+				const baseline = b[0];
+				const treatment = t[0];
+				if (baseline.repKey !== void 0 || treatment.repKey !== void 0) {
+					if (baseline.repKey === void 0 || treatment.repKey === void 0 || baseline.repKey !== treatment.repKey) {
+						unpairedBaseline.push(baseline);
+						unpairedTreatment.push(treatment);
+						continue;
+					}
+				}
+				pairs.push({
+					pairKey,
+					repIndex: 0,
+					baseline,
+					treatment
+				});
+			} else {
+				unpairedBaseline.push(...b);
+				unpairedTreatment.push(...t);
+			}
+			continue;
+		}
+		const bByRep = indexByRepKey(b, pairKey, baselineArm);
+		const tByRep = indexByRepKey(t, pairKey, treatmentArm);
+		const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
+		let repIndex = 0;
+		for (const repKey of repKeys) {
+			const baseline = bByRep.get(repKey);
+			const treatment = tByRep.get(repKey);
+			if (baseline !== void 0 && treatment !== void 0) pairs.push({
+				pairKey,
+				repIndex: repIndex++,
+				baseline,
+				treatment
+			});
+			else if (baseline !== void 0) unpairedBaseline.push(baseline);
+			else if (treatment !== void 0) unpairedTreatment.push(treatment);
+		}
+	}
+	return {
+		pairs,
+		unpairedBaseline,
+		unpairedTreatment
+	};
+}
+/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every
+*  row carries one and that no repKey repeats within the group. */
+function indexByRepKey(group, pairKey, arm) {
+	const byRep = /* @__PURE__ */ new Map();
+	for (const row of group) {
+		if (row.repKey === void 0) throw new ValidationError(`pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey — multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`);
+		if (byRep.has(row.repKey)) throw new ValidationError(`pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — (pairKey, repKey) must uniquely identify a rep within an arm`);
+		byRep.set(row.repKey, row);
+	}
+	return byRep;
+}
+/**
+* Full matched-pair arm comparison: pair via {@link pairArms}, then compose
+* the paired estimators from `statistics` over the matched pairs.
+*
+* Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`
+* is that subset's size); each metric uses only the pairs where both sides
+* carry a finite value for it, with the remainder counted in `nMissing`.
+* Deltas are treatment − baseline throughout.
+*
+* Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a
+* non-finite metric value — silently treating corrupt telemetry as "metric
+* absent" would misreport it as missing coverage.
+*/
+function comparePairedArms(rows, opts) {
+	const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
+	let correctness = null;
+	const baselinePass = [];
+	const treatmentPass = [];
+	for (const pair of pairs) {
+		if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
+		baselinePass.push(pair.baseline.pass ? 1 : 0);
+		treatmentPass.push(pair.treatment.pass ? 1 : 0);
+	}
+	if (baselinePass.length > 0) {
+		const mc = mcnemar(baselinePass, treatmentPass);
+		correctness = {
+			b10: mc.b,
+			b01: mc.c,
+			mcnemar: mc,
+			riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
+		};
+	}
+	const metricDeltas = (opts.metricNames ?? [...new Set(pairs.flatMap((p) => [...Object.keys(p.baseline.metrics ?? {}), ...Object.keys(p.treatment.metrics ?? {})]))].sort()).map((name) => {
+		const before = [];
+		const after = [];
+		let nMissing = 0;
+		for (const pair of pairs) {
+			const b = metricValue(pair.baseline, name);
+			const t = metricValue(pair.treatment, name);
+			if (b === void 0 || t === void 0) {
+				nMissing++;
+				continue;
+			}
+			before.push(b);
+			after.push(t);
+		}
+		const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
+		return {
+			name,
+			n: before.length,
+			nMissing,
+			medianDelta: bootstrapCi?.median ?? null,
+			meanDelta: bootstrapCi?.mean ?? null,
+			bootstrapCi,
+			wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
+		};
+	});
+	return {
+		nPairs: pairs.length,
+		nUnpairedBaseline: unpairedBaseline.length,
+		nUnpairedTreatment: unpairedTreatment.length,
+		correctness,
+		metricDeltas
+	};
+}
+/**
+* Pair two RunRecord arms by the identity of the evaluated work:
+* `(experimentId, scenarioId, seed)`.
+*
+* Falling back to array order, candidate id, or experiment id can compare
+* different tasks and fabricate lift. Duplicate identities throw.
+*/
+function pairRunRecords(baselineRuns, treatmentRuns) {
+	const baselineRows = runRecordArmRows(baselineRuns, "baseline");
+	const treatmentRows = runRecordArmRows(treatmentRuns, "treatment");
+	validateRunRecordArmRows(baselineRows, "baseline");
+	validateRunRecordArmRows(treatmentRows, "treatment");
+	if (baselineRows.length === 0 || treatmentRows.length === 0) return {
+		pairs: [],
+		unpairedBaseline: baselineRows.map((row) => row.run),
+		unpairedTreatment: treatmentRows.map((row) => row.run)
+	};
+	const result = pairArms([...baselineRows, ...treatmentRows], {
+		baselineArm: "baseline",
+		treatmentArm: "treatment"
+	});
+	return {
+		pairs: result.pairs.map((pair) => {
+			const baseline = pair.baseline;
+			const treatment = pair.treatment;
+			return {
+				pairKey: pair.pairKey,
+				repKey: baseline.repKey,
+				baseline: baseline.run,
+				treatment: treatment.run
+			};
+		}),
+		unpairedBaseline: result.unpairedBaseline.map((row) => row.run),
+		unpairedTreatment: result.unpairedTreatment.map((row) => row.run)
+	};
+}
+function runRecordArmRows(runs, arm) {
+	return runs.map((run) => {
+		const scenarioId = run.scenarioId.trim();
+		if (!scenarioId) throw new ValidationError(`pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`);
+		return {
+			pairKey: JSON.stringify([run.experimentId, scenarioId]),
+			repKey: String(run.seed),
+			arm,
+			run
+		};
+	});
+}
+function validateRunRecordArmRows(rows, arm) {
+	const byPairKey = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const group = byPairKey.get(row.pairKey) ?? [];
+		group.push(row);
+		byPairKey.set(row.pairKey, group);
+	}
+	for (const [pairKey, group] of byPairKey) if (group.length > 1) indexByRepKey(group, pairKey, arm);
+}
+function metricValue(row, name) {
+	const v = row.metrics?.[name];
+	if (v === void 0) return void 0;
+	if (!Number.isFinite(v)) throw new ValidationError(`comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`);
+	return v;
+}
+//#endregion
+export { passAtK as _, MANN_WHITNEY_EXACT_MAX_STATES as a, mannWhitneyU as c, isBinaryOutcomeVector as d, mcnemar as f, pairedRiskDifferenceScore as g, pairedRiskDifferenceExact as h, DEFAULT_PERMUTATIONS as i, wilcoxonSignedRank as l, pairedRiskDifference as m, pairArms as n, MANN_WHITNEY_EXACT_MAX_WORK as o, pairedBinaryScale as p, pairRunRecords as r, WILCOXON_EXACT_MAX_N as s, comparePairedArms as t, normalCdf as u, wilson as v };
+
+//# sourceMappingURL=paired-arms-D-XRF_fy.js.map