@tangle-network/agent-eval 0.135.0 → 0.135.2

package/dist/{skillopt-optimization-method-Cl4XPkLC.js → skillopt-optimization-method-0UmPD6aP.js}

@@ -1,11 +1,11 @@
 import { i as JudgeError, s as ValidationError, t as AgentEvalError } from "./errors-8YnH8WlF.js";
-import { c as costForTokenPricing, i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-ZAa_P4r0.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-BNcP4v08.js";
+import { c as costForTokenPricing, i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DHAjwNj7.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-DHx8pzyJ.js";
 import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-DCawte-y.js";
 import { n as mapConcurrent } from "./concurrency-MUjT7VjM.js";
-import { C as pairedBootstrap, D as pairedSignTest, I as weightedComposite, u as confidenceInterval } from "./statistics-RwRNu2__.js";
+import { E as pairedBootstrap, H as weightedComposite, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, T as pairedBinaryScale, d as confidenceInterval, j as pairedRiskDifferenceExact } from "./statistics-CnGCLLqc.js";
 import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
-import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-DCdRK9TY.js";
+import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-a-kYs0-i.js";
 import { a as appendLedgerLine, h as tryAcquireAtomicFileLock, m as probeAtomicFileLock, o as tryWithLedgerFileLock } from "./ledger-core-DAKFKRzi.js";
 import { createRequire } from "node:module";
 import { z } from "zod";
@@ -170,6 +170,32 @@ function minimumPairsForPairedDeltaTest(confidence = .95) {
 * switches to a pre-registered one-sided exact sign test. The exact path is
 * deliberately conservative: it requires both a point estimate above the
 * threshold and enough consistently positive paired differences.
+*
+* ## A zero-width interval is never significant
+*
+* When every paired delta is identical the resample distribution is a point
+* mass and the interval collapses: `[0, 0]` when all pairs tie, `[g, g]` on n
+* identical deltas of g. Neither says the effect is certain — both say the
+* sample carries no information about how far the estimate could be wrong, and
+* `low > threshold` then answers on the point estimate alone. It fails in both
+* directions: `[0, 0]` clears every NEGATIVE threshold, which is how a
+* tie-dominated pass/fail comparison laundered a regression into a
+* noninferiority pass, and `[g, g]` clears every threshold below g with no
+* spread behind it. Under a bounded asymmetric null whose true mean paired
+* delta is exactly 0 — 2 % of pairs dropping by 1.0, the rest gaining 0.0204 —
+* every sample that misses the drop is exactly that shape, and deciding on
+* `low > 0` promoted 65.65 % of samples at n = 20 against a nominal 5 %.
+*
+* So `indeterminate` is reported and `significant` is false whenever the
+* interval has zero width, on BOTH paths: at small n the exact sign test is a
+* test of the MEDIAN and a zero-spread sample is precisely where it stops
+* saying anything about the mean the caller is thresholding.
+*
+* `threshold` may be negative — that is a noninferiority margin, and it is the
+* regime the zero-width hole is worst in. For a two-point (pass/fail) outcome
+* the percentile bootstrap is not a valid interval at a nonzero margin at all;
+* use {@link decidePairedPromotion}, which routes those to Tango's score
+* interval, rather than thresholding this function's bootstrap directly.
 */
 function pairedDeltaTest(before, after, options = {}) {
 	const threshold = options.threshold ?? 0;
@@ -180,13 +206,15 @@ function pairedDeltaTest(before, after, options = {}) {
 	const minimumPairs = Math.max(requestedMinimum, exactMinimum);
 	const bootstrap = pairedBootstrap(before, after, options);
 	const sufficient = bootstrap.n >= minimumPairs;
+	const indeterminate = bootstrap.n > 0 && (!Number.isFinite(bootstrap.low) || !Number.isFinite(bootstrap.high) || bootstrap.low === bootstrap.high);
 	if (bootstrap.gateEligible) return {
 		bootstrap,
 		method: "bootstrap-ci",
 		pValue: null,
 		minimumPairs,
 		sufficient,
-		significant: sufficient && bootstrap.low > threshold
+		indeterminate,
+		significant: sufficient && !indeterminate && bootstrap.low > threshold
 	};
 	const exact = pairedSignTest(before.map((value, index) => after[index] - value - threshold), "greater");
 	const estimate = options.statistic === "mean" ? bootstrap.mean : bootstrap.median;
@@ -196,10 +224,174 @@ function pairedDeltaTest(before, after, options = {}) {
 		pValue: exact.pValue,
 		minimumPairs,
 		sufficient,
-		significant: sufficient && estimate > threshold && exact.pValue <= (1 - bootstrap.confidence) / 2
+		indeterminate,
+		significant: sufficient && !indeterminate && estimate > threshold && exact.pValue <= (1 - bootstrap.confidence) / 2
 	};
 }
 //#endregion
+//#region src/paired-promotion-decision.ts
+/**
+* @module
+* ONE rule for "does this paired interval clear a promotion threshold".
+*
+* The rule below was derived on `HeldOutGate` (#479) after the same estimator
+* bug shipped twice. It then turned out that a SECOND gate — the composable
+* `heldOutGate`, plus everything else routed through `heldoutSignificance` —
+* still carried the original defect, because the rule had been written into one
+* gate's method body rather than into a shared function. Two copies of a
+* statistical rule is how a defect survives in one of them, so there is now
+* exactly one copy and both gates call it.
+*
+* Three things the rule does that a bare `pairedBootstrap(...).low > threshold`
+* does not:
+*
+* 1. **Two-point (pass/fail) outcomes decide on Tango's SCORE interval.** On a
+*    pass/fail eval the paired delta vector is dominated by ties, so the
+*    bootstrap of the mean is a resample of a lattice with three atoms and its
+*    percentile interval is not valid at a nonzero margin. The score interval
+*    (`pairedRiskDifferenceScore`) re-estimates the nuisance loss rate under
+*    each hypothesised margin instead of fixing it at the observed value, which
+*    is the only construction that stays a confidence interval as the margin
+*    moves off zero — the regime every noninferiority threshold lives in.
+*    Measured on the composable gate before this change, at a true risk
+*    difference sitting exactly on the production caller's -0.05 margin and a
+*    nominal 5 %: 14.60 % false promotion at n = 40 and 10.10 % at n = 76.
+* 2. **McNemar's exact test holds a VETO at every non-negative threshold.**
+*    Redundant with the interval by construction and kept anyway, so that
+*    swapping the estimator for one without that duality cannot silently
+*    reintroduce "promotes what the exact test refuses". Witness: n = 6, b = 5,
+*    c = 0 — no exact argument reaches alpha = 0.05 with 5 discordant pairs
+*    (two-sided floor 2/2^5 = 0.0625), whatever an interval says. A NEGATIVE
+*    threshold is a noninferiority question, which McNemar's test of "no
+*    difference" is not the right test for, so the veto does not apply there.
+* 3. **A ZERO-WIDTH interval is refused, wherever it sits.** At [0, 0] it
+*    cannot tell a gain from a regression and clears every negative threshold.
+*    Away from zero it fails the opposite way: n identical positive deltas give
+*    [g, g], which clears threshold 0 on no spread at all. Both are an absence
+*    of evidence. Measured on the composable gate before this change, under a
+*    bounded asymmetric null whose true mean paired delta is exactly 0: 88.50 %
+*    false promotion at n = 6 and 65.65 % at n = 20 against a nominal 5 %.
+*
+* Orthogonal to the small-sample switch inside {@link pairedDeltaTest}: that
+* picks the TEST from the sample size (bootstrap CI at n >= 20, pre-registered
+* exact sign test below it), this picks the ESTIMATOR from the outcome's shape.
+* Both are needed — an exact sign test applied to a tie-pinned median is still
+* blind, and a mean bootstrap CI at n = 6 is still not a valid test.
+*/
+/**
+* Which estimator {@link decidePairedPromotion} would use on this data, and the
+* shape facts behind it — for callers that must report the shape on a path
+* where no interval is computed at all (an early rejection, or zero pairs).
+* Cheap: no bootstrap, no interval.
+*/
+function pairedDecisionShape(before, after, statistic = "mean") {
+	const tieFraction = before.length === 0 ? null : pairedDeltaTieFraction(before, after);
+	if (statistic === "median") return {
+		statistic: "median_bootstrap",
+		binaryScale: null,
+		tieFraction
+	};
+	const binaryScale = pairedBinaryScale(before, after);
+	if (binaryScale !== null) return {
+		statistic: "paired_risk_difference",
+		binaryScale,
+		tieFraction
+	};
+	return {
+		statistic: "mean_bootstrap",
+		binaryScale: null,
+		tieFraction
+	};
+}
+/**
+* Decide whether a paired candidate-minus-baseline delta clears a promotion
+* threshold. `before` is the baseline arm, `after` the candidate arm, paired by
+* position. Throws on unequal lengths.
+*/
+function decidePairedPromotion(before, after, options = {}) {
+	if (before.length !== after.length) throw new Error(`decidePairedPromotion: unequal sample sizes (${before.length} vs ${after.length})`);
+	const threshold = options.threshold ?? 0;
+	if (!Number.isFinite(threshold)) throw new Error(`decidePairedPromotion: threshold must be finite, got ${threshold}`);
+	const confidence = options.confidence ?? .95;
+	const exactMinimum = minimumPairsForPairedDeltaTest(confidence);
+	const requestedMinimum = options.minPairs ?? exactMinimum;
+	if (!Number.isInteger(requestedMinimum) || requestedMinimum < 1) throw new Error(`decidePairedPromotion: minPairs must be a positive integer, got ${requestedMinimum}`);
+	const minimumPairs = Math.max(requestedMinimum, exactMinimum);
+	const n = before.length;
+	const sufficient = n >= minimumPairs;
+	const { binaryScale, tieFraction } = pairedDecisionShape(before, after, options.statistic);
+	let core;
+	if (binaryScale !== null) {
+		const unitControl = before.map((v) => v / binaryScale);
+		const unitTreatment = after.map((v) => v / binaryScale);
+		const exact = pairedRiskDifferenceExact(unitControl, unitTreatment, confidence);
+		const score = pairedRiskDifferenceScore(unitControl, unitTreatment, confidence);
+		const low = score.lower * binaryScale;
+		core = {
+			statistic: "paired_risk_difference",
+			method: "score-interval",
+			delta: score.riskDifference * binaryScale,
+			low,
+			high: score.upper * binaryScale,
+			bootstrap: null,
+			mcnemar: {
+				b: exact.b,
+				c: exact.c,
+				nDiscordant: exact.nDiscordant,
+				pValue: exact.pValue
+			},
+			pValue: null,
+			clearsThreshold: low > threshold,
+			label: "success-rate",
+			methodDetail: ""
+		};
+	} else {
+		const bootstrapStatistic = options.statistic === "median" ? "median" : "mean";
+		const test = pairedDeltaTest(before, after, {
+			confidence,
+			resamples: options.resamples,
+			statistic: bootstrapStatistic,
+			seed: options.seed,
+			threshold,
+			minPairs: options.minPairs
+		});
+		const ci = test.bootstrap;
+		core = {
+			statistic: bootstrapStatistic === "mean" ? "mean_bootstrap" : "median_bootstrap",
+			method: test.method,
+			delta: bootstrapStatistic === "mean" ? ci.mean : ci.median,
+			low: ci.low,
+			high: ci.high,
+			bootstrap: ci,
+			mcnemar: null,
+			pValue: test.pValue,
+			clearsThreshold: test.significant,
+			label: bootstrapStatistic,
+			methodDetail: test.method === "exact-sign" ? ` Below ${test.minimumPairs} pairs the interval is descriptive only; the decision is the exact one-sided sign test, p=${fmt(test.pValue ?? 1)}.` : ""
+		};
+	}
+	const indeterminate = !Number.isFinite(core.low) || !Number.isFinite(core.high) || core.low === core.high;
+	const indeterminateCause = !indeterminate ? "" : tieFraction === 1 ? "every paired delta is an exact tie" : core.mcnemar !== null && core.mcnemar.nDiscordant === 0 ? "every pair is concordant (0 discordant pairs)" : `the ${core.label} CI collapsed to a point at ${fmt(core.low)}`;
+	const exactTestVetoes = core.mcnemar !== null && threshold >= 0 && !(core.mcnemar.pValue < 1 - confidence);
+	return {
+		n,
+		threshold,
+		confidence,
+		binaryScale,
+		tieFraction,
+		minimumPairs,
+		sufficient,
+		indeterminate,
+		indeterminateCause,
+		exactTestVetoes,
+		promote: sufficient && !indeterminate && core.clearsThreshold && !exactTestVetoes,
+		...core
+	};
+}
+function fmt(x) {
+	return x.toFixed(4);
+}
+//#endregion
 //#region src/json-recovery.ts
 /**
 * Truncation-tolerant JSON recovery — shared by every parser that reads JSON
@@ -4178,7 +4370,7 @@ async function compareOptimizationMethods(opts) {
 			confidence: intervalConfidence,
 			statistic: "mean"
 		});
-		const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
+		const favored = !Number.isFinite(boot.low) || !Number.isFinite(boot.high) || boot.low === boot.high ? "tie" : boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
 		return {
 			a: best.name,
 			b: other.name,
@@ -5294,18 +5486,39 @@ function pairHoldout(candidate, baseline, scenarioIds, select) {
 		cellIds
 	};
 }
-/** Significance of the held-out composite lift: ship only when the paired
-*  bootstrap CI lower bound on (candidate − baseline) exceeds `deltaThreshold`
-*  (default 0 ⇒ "confidently positive"). At small n, where the percentile
-*  bootstrap is descriptive only, a pre-registered exact sign test carries
-*  the decision. Interpret `deltaThreshold` in the judge's native scale. */
+/**
+* Significance of the held-out composite lift: ship only when the lower bound
+* of the interval the outcome's shape ADMITS exceeds `deltaThreshold` (default
+* 0 ⇒ "confidently positive"). Interpret `deltaThreshold` in the judge's native
+* scale.
+*
+* The decision is delegated whole to {@link decidePairedPromotion}, the one
+* copy of the rule (`src/paired-promotion-decision.ts`), which `HeldOutGate`
+* also calls. That module's header carries the measurements; the short version
+* is three guards a bare `bootstrap.low > threshold` does not have:
+*
+*   - a two-point (pass/fail) outcome decides on Tango's SCORE interval, the
+*     only paired-binary construction that stays valid at a nonzero margin;
+*   - McNemar's exact test VETOES at any non-negative threshold;
+*   - a ZERO-WIDTH interval is refused rather than promoted, in either
+*     direction — [0,0] clears every negative threshold and [g,g] clears every
+*     threshold below g, and both are an absence of evidence, not a result.
+*
+* Measured on this function before those guards landed, at a nominal 5 %:
+* 14.60 % false promotion at n = 40 on a paired-binary noninferiority boundary,
+* and 88.50 % at n = 6 under a bounded asymmetric null whose true mean paired
+* delta is exactly 0.
+*
+* At small n, where the percentile bootstrap is descriptive only, a
+* pre-registered exact sign test still carries the bootstrap path.
+*/
 function heldoutSignificance(paired, opts = {}) {
 	const deltaThreshold = opts.deltaThreshold ?? 0;
 	const confidence = opts.confidence ?? .95;
 	const resamples = opts.resamples ?? 2e3;
 	const seed = opts.seed ?? 1337;
 	const statistic = opts.statistic ?? "mean";
-	const decision = pairedDeltaTest(paired.before, paired.after, {
+	const decision = decidePairedPromotion(paired.before, paired.after, {
 		confidence,
 		resamples,
 		statistic,
@@ -5313,7 +5526,12 @@ function heldoutSignificance(paired, opts = {}) {
 		threshold: deltaThreshold,
 		minPairs: opts.minProductiveRuns
 	});
-	const bootstrap = decision.bootstrap;
+	const bootstrap = decision.bootstrap ?? pairedBootstrap(paired.before, paired.after, {
+		confidence,
+		resamples,
+		statistic,
+		seed
+	});
 	const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
 		confidence,
 		resamples,
@@ -5328,19 +5546,20 @@ function heldoutSignificance(paired, opts = {}) {
 		if (Math.abs(after - before) < 1e-9) ties += 1;
 	}
 	const tieFraction = n === 0 ? 0 : ties / n;
-	const fewRuns = !decision.sufficient;
-	const significant = decision.significant;
 	return {
 		paired,
 		bootstrap,
 		medianBootstrap,
+		decision,
+		decisionStatistic: decision.statistic,
+		mcnemar: decision.mcnemar,
 		tieFraction,
 		n,
 		minimumRequired: decision.minimumPairs,
 		decisionMethod: decision.method,
 		pValue: decision.pValue,
-		significant,
-		fewRuns
+		significant: decision.promote,
+		fewRuns: !decision.sufficient
 	};
 }
 /** Detect the native scale of a set of scores: 0-100 when any magnitude clears
@@ -5354,30 +5573,50 @@ function detectScale(values) {
 *  a dimension is "regressed" when the CI lower bound < −tolerance (conservative
 *  — blocks if the credible worst case exceeds tolerance, which is the right
 *  posture for safety dimensions like `hallucination_free`). When `tolerance`
-*  is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100. */
+*  is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100.
+*
+*  The interval comes from {@link decidePairedPromotion}, so a pass/fail
+*  dimension is judged on Tango's score interval rather than a percentile
+*  bootstrap of the mean — `tolerance` is a NONZERO margin, and the bootstrap
+*  is not a valid interval at one. That matters most here because this guard
+*  fails OPEN by construction: `tolerance` is positive, so an interval pinned at
+*  [0,0] never satisfies `low < −tolerance` and a real regression on a safety
+*  dimension would be reported as `regressed: false`. On the median it fails the
+*  same way for the same reason — when most pairs tie, which is automatic for a
+*  pass/fail dimension on {0,1} and on the 0-100 encoding `detectScale` exists
+*  to support, the median CI collapses to [0,0]. Pass `statistic: 'median'` to
+*  restore the pre-0.134 behaviour. */
 function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
 	const out = [];
 	for (const dim of criticalDimensions) {
 		const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
 		if (paired.before.length === 0) continue;
 		const tolerance = opts.tolerance ?? .05 * detectScale([...paired.before, ...paired.after]);
-		const bootstrap = pairedBootstrap(paired.before, paired.after, {
+		const bootstrapStatistic = opts.statistic ?? "mean";
+		const shared = {
 			confidence: opts.confidence ?? .95,
 			resamples: opts.resamples ?? 2e3,
-			statistic: "median",
+			statistic: bootstrapStatistic,
 			seed: opts.seed ?? 1337
-		});
-		const regression = pairedDeltaTest(paired.after, paired.before, {
-			confidence: opts.confidence ?? .95,
-			resamples: opts.resamples ?? 2e3,
-			statistic: "median",
-			seed: opts.seed ?? 1337,
+		};
+		const guard = decidePairedPromotion(paired.before, paired.after, shared);
+		const regression = decidePairedPromotion(paired.after, paired.before, {
+			...shared,
 			threshold: tolerance
 		});
+		const bootstrap = guard.bootstrap ?? pairedBootstrap(paired.before, paired.after, shared);
 		out.push({
 			dimension: dim,
 			bootstrap,
-			regressed: regression.significant,
+			bootstrapStatistic,
+			ci: {
+				low: guard.low,
+				high: guard.high
+			},
+			decisionStatistic: guard.statistic,
+			mcnemar: guard.mcnemar,
+			indeterminate: guard.indeterminate,
+			regressed: bootstrap.low < -tolerance || regression.promote,
 			tolerance,
 			n: paired.before.length
 		});
@@ -5430,7 +5669,8 @@ function defaultProductionGate(options) {
 					seed,
 					statistic: heldoutStatistic
 				});
-				delta = heldoutStatistic === "median" ? sig.bootstrap.median : sig.bootstrap.mean;
+				const dec = sig.decision;
+				delta = dec.delta;
 				const heldoutPass = sig.significant;
 				contributing.push({
 					name: "heldout-significance",
@@ -5438,13 +5678,20 @@ function defaultProductionGate(options) {
 					detail: {
 						n: sig.n,
 						delta,
+						decisionStatistic: sig.decisionStatistic,
+						decisionMethod: sig.decisionMethod,
+						binaryScale: dec.binaryScale,
+						mcnemar: dec.mcnemar,
+						indeterminate: dec.indeterminate,
 						deltaMean: sig.bootstrap.mean,
 						deltaMedianDiagnostic: sig.medianBootstrap.median,
 						deltaMedian: sig.medianBootstrap.median,
 						tieFraction: sig.tieFraction,
-						ciLow: sig.bootstrap.low,
-						ciHigh: sig.bootstrap.high,
-						confidence: sig.bootstrap.confidence,
+						ciLow: dec.low,
+						ciHigh: dec.high,
+						bootstrapCiLow: sig.bootstrap.low,
+						bootstrapCiHigh: sig.bootstrap.high,
+						confidence: dec.confidence,
 						deltaThreshold,
 						fewRuns: sig.fewRuns
 					}
@@ -5452,7 +5699,8 @@ function defaultProductionGate(options) {
 				if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
 				if (!heldoutPass) {
 					const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` : "";
-					reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${sig.minimumRequired}) — too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${heldoutStatistic} Δ ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`);
+					const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
+					reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`);
 				}
 			}
 			const dimensionsProvided = options.criticalDimensions !== void 0;
@@ -5480,7 +5728,11 @@ function defaultProductionGate(options) {
 						missingDimensions,
 						regressions: dimRegs.map((result) => ({
 							dimension: result.dimension,
-							ciLow: result.bootstrap.low,
+							ciLow: result.ci.low,
+							ciHigh: result.ci.high,
+							decisionStatistic: result.decisionStatistic,
+							indeterminate: result.indeterminate,
+							bootstrapCiLow: result.bootstrap.low,
 							median: result.bootstrap.median,
 							tolerance: result.tolerance,
 							n: result.n,
@@ -5492,7 +5744,7 @@ function defaultProductionGate(options) {
 					requiredUnavailable.add("dimension-regression");
 					reasons.push(`critical dimension(s) were not scored: ${missingDimensions.join(", ")}`);
 				}
-				if (regressed.length > 0) reasons.push(`critical dimension(s) regressed: ${regressed.map((result) => `${result.dimension} CI.low ${result.bootstrap.low.toFixed(3)} < -${result.tolerance}`).join("; ")}`);
+				if (regressed.length > 0) reasons.push(`critical dimension(s) regressed: ${regressed.map((result) => `${result.dimension} CI.low ${result.ci.low.toFixed(3)} < -${result.tolerance}`).join("; ")}`);
 			}
 			const budgetUsd = options.budgetUsd;
 			const budgetConfigured = budgetUsd !== void 0;
@@ -5646,8 +5898,10 @@ function extractText(artifact) {
 //#endregion
 //#region src/campaign/gates/heldout-gate.ts
 /**
-* Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound
-* of the candidate-minus-baseline composite delta clears `deltaThreshold`.
+* Composable held-out gate: ships only when the lower bound of the DECIDING
+* paired interval on the candidate-minus-baseline composite delta clears
+* `deltaThreshold` — Tango's score interval on a pass/fail holdout, the mean
+* bootstrap otherwise. See {@link decidePairedPromotion}.
 */
 function heldOutGate(options) {
 	const deltaThreshold = options.deltaThreshold ?? .5;
@@ -5667,24 +5921,35 @@ function heldOutGate(options) {
 				resamples,
 				seed
 			});
-			const delta = sig.bootstrap.mean;
+			const dec = sig.decision;
+			const delta = dec.delta;
 			const passed = sig.significant;
 			const status = sig.fewRuns ? "not_evaluated" : passed ? "pass" : "fail";
 			const tieNote = sig.tieFraction >= .4 ? `, ${(sig.tieFraction * 100).toFixed(0)}% tied` : "";
-			const ci = `${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]`;
+			const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
+			const held = `held-out ${dec.label} Δ ${delta.toFixed(3)}`;
+			const holdReason = sig.fewRuns ? `held-out: only ${sig.n} paired runs; ${sig.minimumRequired} required — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear ${deltaThreshold} on evidence (n=${sig.n}${tieNote})` : dec.exactTestVetoes ? `${held}, McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${ci}, n=${sig.n}${tieNote})` : `${held}, CI.low ${dec.low.toFixed(3)} ≤ ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`;
 			return {
 				decision: passed ? "ship" : "hold",
-				reasons: passed ? [`held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`] : [sig.fewRuns ? `held-out: only ${sig.n} paired runs; ${sig.minimumRequired} required — too few to claim significance` : `held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} ≤ ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`],
+				reasons: passed ? [`${held}, CI.low ${dec.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`] : [holdReason],
 				contributingGates: [{
 					name: "heldOutGate",
 					status,
 					detail: {
-						deltaMean: delta,
+						deltaMean: sig.bootstrap.mean,
+						decidingDelta: delta,
+						decisionStatistic: sig.decisionStatistic,
+						decisionMethod: sig.decisionMethod,
+						binaryScale: dec.binaryScale,
+						mcnemar: dec.mcnemar,
+						indeterminate: dec.indeterminate,
 						deltaMedianDiagnostic: sig.medianBootstrap.median,
 						tieFraction: sig.tieFraction,
-						ciLow: sig.bootstrap.low,
-						ciHigh: sig.bootstrap.high,
-						confidence: sig.bootstrap.confidence,
+						ciLow: dec.low,
+						ciHigh: dec.high,
+						bootstrapCiLow: sig.bootstrap.low,
+						bootstrapCiHigh: sig.bootstrap.high,
+						confidence: dec.confidence,
 						n: sig.n,
 						deltaThreshold,
 						fewRuns: sig.fewRuns,
@@ -5793,29 +6058,44 @@ function buildEvidenceVector(ctx, objectives, opts = {}) {
 		const n = paired.before.length;
 		const floorTolerance = obj.floorTolerance ?? .05 * detectScale([...paired.before, ...paired.after]);
 		const gainThreshold = obj.gainThreshold ?? 0;
-		const improvement = pairedDeltaTest(before, after, {
+		const bootstrapStatistic = opts.statistic ?? "mean";
+		const improvement = decidePairedPromotion(before, after, {
 			confidence,
 			resamples,
-			statistic: "median",
+			statistic: bootstrapStatistic,
 			seed,
 			threshold: gainThreshold,
 			minPairs: opts.minProductiveRuns
 		});
-		const regression = pairedDeltaTest(after, before, {
+		const regression = decidePairedPromotion(after, before, {
 			confidence,
 			resamples,
-			statistic: "median",
+			statistic: bootstrapStatistic,
 			seed,
 			threshold: floorTolerance,
 			minPairs: opts.minProductiveRuns
 		});
-		const bootstrap = improvement.bootstrap;
-		const verdict = !improvement.sufficient ? "few_runs" : regression.significant ? "regressed" : improvement.significant ? "improved" : "flat";
+		const bootstrap = improvement.bootstrap ?? pairedBootstrap(before, after, {
+			confidence,
+			resamples,
+			statistic: bootstrapStatistic,
+			seed
+		});
+		const floorBreached = bootstrap.low < -floorTolerance || regression.promote;
+		const verdict = !improvement.sufficient ? "few_runs" : floorBreached ? "regressed" : improvement.promote ? "improved" : "flat";
 		axes.push({
 			name: obj.name,
 			source: obj.source,
 			direction: obj.direction,
 			bootstrap,
+			bootstrapStatistic,
+			ci: {
+				low: improvement.low,
+				high: improvement.high
+			},
+			decisionStatistic: improvement.statistic,
+			mcnemar: improvement.mcnemar,
+			indeterminate: improvement.indeterminate,
 			n,
 			minimumRequired: improvement.minimumPairs,
 			decisionMethod: improvement.method,
@@ -5851,8 +6131,14 @@ const paretoPolicy = (ev) => {
 			verdict: ax.verdict,
 			n: ax.n,
 			deltaMedian: ax.bootstrap.median,
-			ciLow: ax.bootstrap.low,
-			ciHigh: ax.bootstrap.high,
+			ciLow: ax.ci.low,
+			ciHigh: ax.ci.high,
+			decisionStatistic: ax.decisionStatistic,
+			decisionMethod: ax.decisionMethod,
+			mcnemar: ax.mcnemar,
+			indeterminate: ax.indeterminate,
+			bootstrapCiLow: ax.bootstrap.low,
+			bootstrapCiHigh: ax.bootstrap.high,
 			confidence: ax.bootstrap.confidence,
 			gainThreshold: ax.gainThreshold,
 			floorTolerance: ax.floorTolerance
@@ -5865,13 +6151,13 @@ const paretoPolicy = (ev) => {
 	const reasons = [];
 	if (regressed.length > 0) {
 		decision = "hold";
-		for (const a of regressed) reasons.push(`objective '${a.name}' regressed: good-direction CI.low ${a.bootstrap.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`);
+		for (const a of regressed) reasons.push(`objective '${a.name}' regressed: good-direction CI.low ${a.ci.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`);
 	} else if (fewRuns.length > 0) {
 		decision = "need_more_work";
 		for (const a of fewRuns) reasons.push(`objective '${a.name}' has only n=${a.n} paired runs — insufficient evidence to claim significance`);
 	} else if (improved.length > 0) {
 		decision = "ship";
-		reasons.push(`Pareto improvement at the confidence level: ${improved.map((a) => `'${a.name}' +${a.bootstrap.median.toFixed(3)} (CI.low ${a.bootstrap.low.toFixed(3)})`).join(", ")}; no objective regressed`);
+		reasons.push(`Pareto improvement at the confidence level: ${improved.map((a) => `'${a.name}' +${a.ci.low > 0 ? a.ci.low.toFixed(3) : a.bootstrap.mean.toFixed(3)} (CI.low ${a.ci.low.toFixed(3)})`).join(", ")}; no objective regressed`);
 	} else {
 		decision = "hold";
 		reasons.push("no Pareto improvement: candidate statistically equivalent to baseline on every objective");
@@ -7833,6 +8119,6 @@ function skillOptOptimizationMethod(config) {
 	};
 }
 //#endregion
-export { SEARCH_LEDGER_FILE_CONTEXT as $, acquireSingleRunLock as A, dominates as At, surfaceContentHash as B, summarizeAgentReceiptIntegrity as Bt, detectScale as C, llmJudge as Ct, runCanaries as D, fileVerdictCache as Dt, pairHoldout as E, contentHash as Et, assertCodeSurfaceIdentity as F, minimumPairsForPairedDeltaTest as Ft, DEFAULT_MUTATION_PRIMITIVES as G, campaignBreakdown as H, JudgeParseError as Ht, assertComponentSurface as I, pairedDeltaTest as It, planCampaignRun as J, buildReflectionPrompt as K, codeSurfaceIdentityMaterial as L, BackendIntegrityError as Lt, compareOptimizationMethods as M, paretoFrontierWithCrowding as Mt, costFromLedgerSummary as N, scalarScore as Nt, composeGate as O, inMemoryVerdictCache as Ot, optimizationTokenUsageFromSummary as P, recoverTruncatedJson as Pt, inMemoryCampaignStorage as Q, componentSurfaceIdentityMaterial as R, assertRealAgentReceipts as Rt, defaultProductionGate as S, hashScenarios as St, heldoutSignificance as T, canonicalJson as Tt, campaignMeanComposite as U, surfaceHash as V, summarizeBackendIntegrity as Vt, compareRankKeys as W, createRunCostLedger as X, runCampaign as Y, fsCampaignStorage as Z, buildEvidenceVector as _, redTeamReport as _t, emitLoopProvenance as a, assertCampaignDesign as at, powerPreflight as b, Dataset as bt, provenanceRecordPath as c, campaignSplitDigest as ct, runImprovementLoop as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as dt, SearchLedgerConflictError as et, runOptimization as f, REFERENCE_EQUIVALENCE_JUDGE_VERSION as ft, gepaOptimizationMethod as g, redTeamDataset as gt, labelTrustRank as h, DEFAULT_RED_TEAM_CORPUS as ht, canonicalDigest as i, tangleTracesRoot as it, assertOptimizationResult as j, paretoFrontier as jt, externalTextOptimizationMethod as k, crowdingDistance as kt, provenanceSpansPath as l, campaignSplitDigestFromIdentities as lt, isProposedCandidate as m, runReferenceEquivalenceJudge as mt, buildLoopProvenanceRecord as n, SearchLedgerIntegrityError as nt, loopProvenanceArgsFromResult as o, assertCampaignSplitIdentity as ot, runEval as p, createReferenceEquivalenceJudge as pt, parseReflectionResponse as q, campaignMeasurementDigest as r, resolveRunDir as rt, loopProvenanceSpans as s, campaignScenarioIdentity as st, skillOptOptimizationMethod as t, SearchLedgerError as tt, verifyLoopProvenanceRecord as u, openAutoPr as ut, paretoPolicy as v, scoreRedTeamOutput as vt, dimensionRegressions as w, cachedJudge as wt, heldOutGate as x, HoldoutLockedError as xt, paretoSignificanceGate as y, toolNamesForRun as yt, renderSurfaceDiff as z, assertRealBackend as zt };
+export { SEARCH_LEDGER_FILE_CONTEXT as $, acquireSingleRunLock as A, dominates as At, surfaceContentHash as B, assertRealAgentReceipts as Bt, detectScale as C, llmJudge as Ct, runCanaries as D, fileVerdictCache as Dt, pairHoldout as E, contentHash as Et, assertCodeSurfaceIdentity as F, decidePairedPromotion as Ft, DEFAULT_MUTATION_PRIMITIVES as G, campaignBreakdown as H, summarizeAgentReceiptIntegrity as Ht, assertComponentSurface as I, pairedDecisionShape as It, planCampaignRun as J, buildReflectionPrompt as K, codeSurfaceIdentityMaterial as L, minimumPairsForPairedDeltaTest as Lt, compareOptimizationMethods as M, paretoFrontierWithCrowding as Mt, costFromLedgerSummary as N, scalarScore as Nt, composeGate as O, inMemoryVerdictCache as Ot, optimizationTokenUsageFromSummary as P, recoverTruncatedJson as Pt, inMemoryCampaignStorage as Q, componentSurfaceIdentityMaterial as R, pairedDeltaTest as Rt, defaultProductionGate as S, hashScenarios as St, heldoutSignificance as T, canonicalJson as Tt, campaignMeanComposite as U, summarizeBackendIntegrity as Ut, surfaceHash as V, assertRealBackend as Vt, compareRankKeys as W, JudgeParseError as Wt, createRunCostLedger as X, runCampaign as Y, fsCampaignStorage as Z, buildEvidenceVector as _, redTeamReport as _t, emitLoopProvenance as a, assertCampaignDesign as at, powerPreflight as b, Dataset as bt, provenanceRecordPath as c, campaignSplitDigest as ct, runImprovementLoop as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as dt, SearchLedgerConflictError as et, runOptimization as f, REFERENCE_EQUIVALENCE_JUDGE_VERSION as ft, gepaOptimizationMethod as g, redTeamDataset as gt, labelTrustRank as h, DEFAULT_RED_TEAM_CORPUS as ht, canonicalDigest as i, tangleTracesRoot as it, assertOptimizationResult as j, paretoFrontier as jt, externalTextOptimizationMethod as k, crowdingDistance as kt, provenanceSpansPath as l, campaignSplitDigestFromIdentities as lt, isProposedCandidate as m, runReferenceEquivalenceJudge as mt, buildLoopProvenanceRecord as n, SearchLedgerIntegrityError as nt, loopProvenanceArgsFromResult as o, assertCampaignSplitIdentity as ot, runEval as p, createReferenceEquivalenceJudge as pt, parseReflectionResponse as q, campaignMeasurementDigest as r, resolveRunDir as rt, loopProvenanceSpans as s, campaignScenarioIdentity as st, skillOptOptimizationMethod as t, SearchLedgerError as tt, verifyLoopProvenanceRecord as u, openAutoPr as ut, paretoPolicy as v, scoreRedTeamOutput as vt, dimensionRegressions as w, cachedJudge as wt, heldOutGate as x, HoldoutLockedError as xt, paretoSignificanceGate as y, toolNamesForRun as yt, renderSurfaceDiff as z, BackendIntegrityError as zt };
 
-//# sourceMappingURL=skillopt-optimization-method-Cl4XPkLC.js.map
+//# sourceMappingURL=skillopt-optimization-method-0UmPD6aP.js.map