@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/paired-tests-BHIhYVdu.js

@@ -0,0 +1,213 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { a as medianInPlace, i as makeRng, n as binomialHalfUpperTail, t as assertFiniteSample } from "./internal-BDHPCnjk.js";
+import { t as studentTCdf } from "./student-t-CvBq2mve.js";
+//#region src/statistics/paired-tests.ts
+/**
+* Paired significance tests on continuous scores: the paired t-test, the
+* promotion-gate paired bootstrap, the exact sign test, and the package-wide
+* paired-delta decision statistic.
+*/
+/**
+* Paired t-test — before/after measurements on the SAME items.
+* Pairing removes inter-item variance, giving tighter significance than
+* an unpaired test when comparing prompt v1 vs prompt v2 on identical
+* scenarios.
+*
+* Returns `t = p = null` where the statistic is undefined: fewer than two
+* pairs, or a non-zero constant delta whose observed variance is zero. A
+* constant shift carries no information about the variance it would have to
+* be compared against, so the honest answer is "undefined", not `p = 0` —
+* three observations cannot buy absolute certainty. This is the same contract
+* {@link pairedCohensDz} states for the same condition. An all-zero delta is
+* different: it is a measured null, and returns `t = 0, p = 1`.
+*/
+function pairedTTest(before, after) {
+	if (before.length !== after.length) throw new ValidationError(`pairedTTest: unequal sample sizes (${before.length} vs ${after.length})`);
+	assertFiniteSample("pairedTTest", "before", before);
+	assertFiniteSample("pairedTTest", "after", after);
+	const n = before.length;
+	if (n < 2) return {
+		t: null,
+		df: 0,
+		p: null
+	};
+	const diffs = before.map((b, i) => after[i] - b);
+	const mean = diffs.reduce((a, b) => a + b, 0) / n;
+	const variance = diffs.reduce((acc, d) => acc + (d - mean) ** 2, 0) / (n - 1);
+	const se = Math.sqrt(variance / n);
+	if (se === 0) return mean === 0 ? {
+		t: 0,
+		df: n - 1,
+		p: 1
+	} : {
+		t: null,
+		df: n - 1,
+		p: null
+	};
+	const t = mean / se;
+	const df = n - 1;
+	return {
+		t,
+		df,
+		p: 2 * (1 - studentTCdf(Math.abs(t), df))
+	};
+}
+/**
+* Pairs below which a percentile bootstrap interval is descriptive spread only.
+*
+* `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000
+* seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the
+* median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling
+* three points, not an implementation error — scipy's BCa gives 16.0 % on the
+* same n = 3 data — so no change to the estimator moves it. Below this floor
+* the decision belongs to the exact sign test or exact signed-rank test.
+*/
+const BOOTSTRAP_GATE_MIN_N = 20;
+/**
+* Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
+* statistic (median by default); pairs are resampled with replacement. Throws
+* on unequal sample sizes.
+*
+* `low > threshold` carries the stated confidence ONLY at `n ≥
+* {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the
+* check fires under a true null several times more often than nominal, so the
+* interval is descriptive spread and a promotion must not turn on it.
+*/
+function pairedBootstrap(before, after, opts = {}) {
+	if (before.length !== after.length) throw new Error(`pairedBootstrap: unequal sample sizes (${before.length} vs ${after.length})`);
+	const confidence = opts.confidence ?? .95;
+	const resamples = opts.resamples ?? 2e3;
+	const statistic = opts.statistic ?? "median";
+	if (confidence <= 0 || confidence >= 1) throw new Error(`pairedBootstrap: confidence must be in (0,1), got ${confidence}`);
+	const n = before.length;
+	const deltas = before.map((b, i) => after[i] - b);
+	const gateEligible = n >= 20;
+	if (n === 0) return {
+		n: 0,
+		median: 0,
+		mean: 0,
+		low: 0,
+		high: 0,
+		confidence,
+		resamples,
+		gateEligible
+	};
+	if (n === 1) {
+		const d = deltas[0];
+		return {
+			n: 1,
+			median: d,
+			mean: d,
+			low: d,
+			high: d,
+			confidence,
+			resamples,
+			gateEligible
+		};
+	}
+	const rng = makeRng(opts.seed, deltas);
+	const samples = new Array(resamples);
+	for (let b = 0; b < resamples; b++) if (statistic === "mean") {
+		let sum = 0;
+		for (let k = 0; k < n; k++) sum += deltas[Math.floor(rng() * n)];
+		samples[b] = sum / n;
+	} else {
+		const acc = new Array(n);
+		for (let k = 0; k < n; k++) acc[k] = deltas[Math.floor(rng() * n)];
+		samples[b] = medianInPlace(acc);
+	}
+	samples.sort((a, b) => a - b);
+	const alpha = 1 - confidence;
+	const lowIdx = Math.floor(alpha / 2 * resamples);
+	const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1);
+	return {
+		n,
+		median: medianInPlace([...deltas]),
+		mean: deltas.reduce((s, x) => s + x, 0) / n,
+		low: samples[lowIdx],
+		high: samples[Math.max(highIdx, lowIdx)],
+		confidence,
+		resamples,
+		gateEligible
+	};
+}
+/**
+* Exact one-sided sign test over paired differences.
+*
+* Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`
+* tests whether positive signs are more likely than negative signs and returns
+* `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats
+* negative signs as successes instead. With a continuous difference
+* distribution this is the usual directional median test. Exact zero
+* differences are ties and do not enter the binomial denominator. All-tie and
+* empty inputs return p = 1. Every input difference must be finite, and the
+* direction must be chosen explicitly so a caller cannot select it after
+* seeing the signs.
+*/
+function pairedSignTest(differences, alternative) {
+	if (alternative !== "greater" && alternative !== "less") throw new ValidationError(`pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`);
+	let positive = 0;
+	let negative = 0;
+	let ties = 0;
+	for (let i = 0; i < differences.length; i++) {
+		const difference = differences[i];
+		if (!Number.isFinite(difference)) throw new ValidationError(`pairedSignTest: difference at index ${i} must be finite, got ${difference}`);
+		if (difference > 0) positive++;
+		else if (difference < 0) negative++;
+		else ties++;
+	}
+	const nNonTies = positive + negative;
+	const successes = alternative === "greater" ? positive : negative;
+	return {
+		n: differences.length,
+		positive,
+		negative,
+		ties,
+		nNonTies,
+		alternative,
+		pValue: binomialHalfUpperTail(successes, nNonTies)
+	};
+}
+/** Fraction of paired observations whose delta is an exact tie (|after − before|
+*  < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */
+function pairedDeltaTieFraction(before, after) {
+	if (before.length !== after.length) throw new Error(`pairedDeltaTieFraction: unequal sample sizes (${before.length} vs ${after.length})`);
+	const n = before.length;
+	if (n === 0) return 0;
+	let ties = 0;
+	for (let i = 0; i < n; i++) if (Math.abs(after[i] - before[i]) < 1e-9) ties++;
+	return ties / n;
+}
+/**
+* The paired-delta statistic a DECISION is computed on, package-wide.
+*
+* The mean paired delta is the estimator that answers the question a promotion
+* gate asks — "by how much did the candidate move the score" — in the caller's
+* own units, and it equals the aggregate lift everyone quotes. The MEDIAN
+* answers a different question and loses the answer to this one in every regime
+* eval data actually lands in:
+*   - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in
+*     {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI
+*     are pinned at exactly 0 however large the shift. (Decide these on
+*     {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)
+*   - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by
+*     construction, and `ci.low > threshold` then answers "no" forever at a
+*     non-negative threshold and "yes" forever at a negative one.
+*   - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on
+*     integer 0-100, and block scores like {⅔, 1} from averaging pass/fail
+*     leaves, put the median on a coarse lattice whose bootstrap percentiles
+*     land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real
+*     +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —
+*     lower bound exactly 0, so a gate at threshold 0 refuses a real lift.
+* That last case is why there is no tie-fraction threshold here: any cutoff on
+* ties leaves the lattice case open on the other side of it.
+*
+* `heldoutSignificance` has defaulted to the mean since #316 for the same
+* reason. The median remains available per call site for callers who
+* specifically want outlier robustness and accept the blindness.
+*/
+const DECISION_PAIRED_DELTA_STATISTIC = "mean";
+//#endregion
+export { pairedSignTest as a, pairedDeltaTieFraction as i, DECISION_PAIRED_DELTA_STATISTIC as n, pairedTTest as o, pairedBootstrap as r, BOOTSTRAP_GATE_MIN_N as t };
+
+//# sourceMappingURL=paired-tests-BHIhYVdu.js.map

package/dist/paired-tests-BHIhYVdu.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paired-tests-BHIhYVdu.js","names":[],"sources":["../src/statistics/paired-tests.ts"],"sourcesContent":["/**\n * Paired significance tests on continuous scores: the paired t-test, the\n * promotion-gate paired bootstrap, the exact sign test, and the package-wide\n * paired-delta decision statistic.\n */\n\nimport { ValidationError } from '../errors'\nimport { studentTCdf } from '../math/student-t'\nimport { assertFiniteSample, binomialHalfUpperTail, makeRng, medianInPlace } from './internal'\n\nexport interface PairedTTestResult {\n  /** Null when the statistic is undefined — see {@link pairedTTest}. */\n  t: number | null\n  df: number\n  /** Null exactly when `t` is null. */\n  p: number | null\n}\n\n/**\n * Paired t-test — before/after measurements on the SAME items.\n * Pairing removes inter-item variance, giving tighter significance than\n * an unpaired test when comparing prompt v1 vs prompt v2 on identical\n * scenarios.\n *\n * Returns `t = p = null` where the statistic is undefined: fewer than two\n * pairs, or a non-zero constant delta whose observed variance is zero. A\n * constant shift carries no information about the variance it would have to\n * be compared against, so the honest answer is \"undefined\", not `p = 0` —\n * three observations cannot buy absolute certainty. This is the same contract\n * {@link pairedCohensDz} states for the same condition. An all-zero delta is\n * different: it is a measured null, and returns `t = 0, p = 1`.\n */\nexport function pairedTTest(before: number[], after: number[]): PairedTTestResult {\n  if (before.length !== after.length) {\n    throw new ValidationError(\n      `pairedTTest: unequal sample sizes (${before.length} vs ${after.length})`,\n    )\n  }\n  assertFiniteSample('pairedTTest', 'before', before)\n  assertFiniteSample('pairedTTest', 'after', after)\n  const n = before.length\n  if (n < 2) return { t: null, df: 0, p: null }\n\n  const diffs = before.map((b, i) => after[i]! - b)\n  const mean = diffs.reduce((a, b) => a + b, 0) / n\n  const variance = diffs.reduce((acc, d) => acc + (d - mean) ** 2, 0) / (n - 1)\n  const se = Math.sqrt(variance / n)\n  if (se === 0) {\n    return mean === 0 ? { t: 0, df: n - 1, p: 1 } : { t: null, df: n - 1, p: null }\n  }\n\n  const t = mean / se\n  const df = n - 1\n  const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n  return { t, df, p }\n}\n\n// ── Paired bootstrap (promotion-gate effect size) ────────────────────\n\nexport interface PairedBootstrapResult {\n  /** Number of paired observations. */\n  n: number\n  /** Median of paired deltas (after − before). */\n  median: number\n  /** Mean of paired deltas. */\n  mean: number\n  /** Lower bound of the bootstrap CI on the chosen statistic. */\n  low: number\n  /** Upper bound of the bootstrap CI on the chosen statistic. */\n  high: number\n  /** Confidence level used (e.g. 0.95). */\n  confidence: number\n  /** Number of bootstrap resamples used. */\n  resamples: number\n  /** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */\n  gateEligible: boolean\n}\n\n/**\n * Pairs below which a percentile bootstrap interval is descriptive spread only.\n *\n * `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000\n * seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the\n * median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling\n * three points, not an implementation error — scipy's BCa gives 16.0 % on the\n * same n = 3 data — so no change to the estimator moves it. Below this floor\n * the decision belongs to the exact sign test or exact signed-rank test.\n */\nexport const BOOTSTRAP_GATE_MIN_N = 20\n\nexport interface PairedBootstrapOptions {\n  /** Confidence level. Default 0.95. */\n  confidence?: number\n  /** Bootstrap resample count. Default 2000. */\n  resamples?: number\n  /** Statistic to bootstrap. Default 'median'. */\n  statistic?: 'median' | 'mean'\n  /** Deterministic seed. If omitted, derived from the deltas so the interval\n   *  is reproducible regardless. */\n  seed?: number\n}\n\n/**\n * Paired bootstrap on (after − before) deltas. Returns a CI on the chosen\n * statistic (median by default); pairs are resampled with replacement. Throws\n * on unequal sample sizes.\n *\n * `low > threshold` carries the stated confidence ONLY at `n ≥\n * {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the\n * check fires under a true null several times more often than nominal, so the\n * interval is descriptive spread and a promotion must not turn on it.\n */\nexport function pairedBootstrap(\n  before: number[],\n  after: number[],\n  opts: PairedBootstrapOptions = {},\n): PairedBootstrapResult {\n  if (before.length !== after.length) {\n    throw new Error(`pairedBootstrap: unequal sample sizes (${before.length} vs ${after.length})`)\n  }\n  const confidence = opts.confidence ?? 0.95\n  const resamples = opts.resamples ?? 2000\n  const statistic = opts.statistic ?? 'median'\n  if (confidence <= 0 || confidence >= 1) {\n    throw new Error(`pairedBootstrap: confidence must be in (0,1), got ${confidence}`)\n  }\n\n  const n = before.length\n  const deltas = before.map((b, i) => after[i]! - b)\n  const gateEligible = n >= BOOTSTRAP_GATE_MIN_N\n  if (n === 0) {\n    return { n: 0, median: 0, mean: 0, low: 0, high: 0, confidence, resamples, gateEligible }\n  }\n  if (n === 1) {\n    const d = deltas[0]!\n    return { n: 1, median: d, mean: d, low: d, high: d, confidence, resamples, gateEligible }\n  }\n\n  const rng = makeRng(opts.seed, deltas)\n  const samples = new Array<number>(resamples)\n  for (let b = 0; b < resamples; b++) {\n    if (statistic === 'mean') {\n      let sum = 0\n      for (let k = 0; k < n; k++) {\n        sum += deltas[Math.floor(rng() * n)]!\n      }\n      samples[b] = sum / n\n    } else {\n      const acc = new Array<number>(n)\n      for (let k = 0; k < n; k++) {\n        acc[k] = deltas[Math.floor(rng() * n)]!\n      }\n      samples[b] = medianInPlace(acc)\n    }\n  }\n  samples.sort((a, b) => a - b)\n\n  const alpha = 1 - confidence\n  const lowIdx = Math.floor((alpha / 2) * resamples)\n  const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)\n\n  return {\n    n,\n    median: medianInPlace([...deltas]),\n    mean: deltas.reduce((s, x) => s + x, 0) / n,\n    low: samples[lowIdx]!,\n    high: samples[Math.max(highIdx, lowIdx)]!,\n    confidence,\n    resamples,\n    gateEligible,\n  }\n}\n\n/** Pre-registered direction for a one-sided paired sign test. */\nexport type SignTestAlternative = 'greater' | 'less'\n\n/** Exact one-sided sign-test result for paired numeric differences. */\nexport interface PairedSignTestResult {\n  /** Total supplied differences, including zero ties. */\n  n: number\n  /** Strictly positive differences. */\n  positive: number\n  /** Strictly negative differences. */\n  negative: number\n  /** Zero differences excluded from the binomial test. */\n  ties: number\n  /** Non-zero differences used by the binomial test. */\n  nNonTies: number\n  /** Direction of the pre-registered alternative hypothesis. */\n  alternative: SignTestAlternative\n  /** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */\n  pValue: number\n}\n\n/**\n * Exact one-sided sign test over paired differences.\n *\n * Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`\n * tests whether positive signs are more likely than negative signs and returns\n * `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats\n * negative signs as successes instead. With a continuous difference\n * distribution this is the usual directional median test. Exact zero\n * differences are ties and do not enter the binomial denominator. All-tie and\n * empty inputs return p = 1. Every input difference must be finite, and the\n * direction must be chosen explicitly so a caller cannot select it after\n * seeing the signs.\n */\nexport function pairedSignTest(\n  differences: readonly number[],\n  alternative: SignTestAlternative,\n): PairedSignTestResult {\n  if (alternative !== 'greater' && alternative !== 'less') {\n    throw new ValidationError(\n      `pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`,\n    )\n  }\n\n  let positive = 0\n  let negative = 0\n  let ties = 0\n  for (let i = 0; i < differences.length; i++) {\n    const difference = differences[i]!\n    if (!Number.isFinite(difference)) {\n      throw new ValidationError(\n        `pairedSignTest: difference at index ${i} must be finite, got ${difference}`,\n      )\n    }\n    if (difference > 0) positive++\n    else if (difference < 0) negative++\n    else ties++\n  }\n\n  const nNonTies = positive + negative\n  const successes = alternative === 'greater' ? positive : negative\n  return {\n    n: differences.length,\n    positive,\n    negative,\n    ties,\n    nNonTies,\n    alternative,\n    pValue: binomialHalfUpperTail(successes, nNonTies),\n  }\n}\n\n/** Fraction of paired observations whose delta is an exact tie (|after − before|\n *  < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */\nexport function pairedDeltaTieFraction(\n  before: ArrayLike<number>,\n  after: ArrayLike<number>,\n): number {\n  if (before.length !== after.length) {\n    throw new Error(\n      `pairedDeltaTieFraction: unequal sample sizes (${before.length} vs ${after.length})`,\n    )\n  }\n  const n = before.length\n  if (n === 0) return 0\n  let ties = 0\n  for (let i = 0; i < n; i++) {\n    if (Math.abs(after[i]! - before[i]!) < 1e-9) ties++\n  }\n  return ties / n\n}\n\n/**\n * The paired-delta statistic a DECISION is computed on, package-wide.\n *\n * The mean paired delta is the estimator that answers the question a promotion\n * gate asks — \"by how much did the candidate move the score\" — in the caller's\n * own units, and it equals the aggregate lift everyone quotes. The MEDIAN\n * answers a different question and loses the answer to this one in every regime\n * eval data actually lands in:\n *   - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in\n *     {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI\n *     are pinned at exactly 0 however large the shift. (Decide these on\n *     {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)\n *   - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by\n *     construction, and `ci.low > threshold` then answers \"no\" forever at a\n *     non-negative threshold and \"yes\" forever at a negative one.\n *   - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on\n *     integer 0-100, and block scores like {⅔, 1} from averaging pass/fail\n *     leaves, put the median on a coarse lattice whose bootstrap percentiles\n *     land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real\n *     +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —\n *     lower bound exactly 0, so a gate at threshold 0 refuses a real lift.\n * That last case is why there is no tie-fraction threshold here: any cutoff on\n * ties leaves the lattice case open on the other side of it.\n *\n * `heldoutSignificance` has defaulted to the mean since #316 for the same\n * reason. The median remains available per call site for callers who\n * specifically want outlier robustness and accept the blindness.\n */\nexport const DECISION_PAIRED_DELTA_STATISTIC: 'mean' = 'mean'\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAgCA,SAAgB,YAAY,QAAkB,OAAoC;CAChF,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,gBACR,sCAAsC,OAAO,OAAO,MAAM,MAAM,OAAO,EACzE;CAEF,mBAAmB,eAAe,UAAU,MAAM;CAClD,mBAAmB,eAAe,SAAS,KAAK;CAChD,MAAM,IAAI,OAAO;CACjB,IAAI,IAAI,GAAG,OAAO;EAAE,GAAG;EAAM,IAAI;EAAG,GAAG;CAAK;CAE5C,MAAM,QAAQ,OAAO,KAAK,GAAG,MAAM,MAAM,KAAM,CAAC;CAChD,MAAM,OAAO,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAChD,MAAM,WAAW,MAAM,QAAQ,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC,KAAK,IAAI;CAC3E,MAAM,KAAK,KAAK,KAAK,WAAW,CAAC;CACjC,IAAI,OAAO,GACT,OAAO,SAAS,IAAI;EAAE,GAAG;EAAG,IAAI,IAAI;EAAG,GAAG;CAAE,IAAI;EAAE,GAAG;EAAM,IAAI,IAAI;EAAG,GAAG;CAAK;CAGhF,MAAM,IAAI,OAAO;CACjB,MAAM,KAAK,IAAI;CAEf,OAAO;EAAE;EAAG;EAAI,GADN,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;CAC5B;AACpB;;;;;;;;;;;AAiCA,MAAa,uBAAuB;;;;;;;;;;;AAwBpC,SAAgB,gBACd,QACA,OACA,OAA+B,CAAC,GACT;CACvB,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,MAAM,0CAA0C,OAAO,OAAO,MAAM,MAAM,OAAO,EAAE;CAE/F,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,YAAY,KAAK,aAAa;CACpC,IAAI,cAAc,KAAK,cAAc,GACnC,MAAM,IAAI,MAAM,qDAAqD,YAAY;CAGnF,MAAM,IAAI,OAAO;CACjB,MAAM,SAAS,OAAO,KAAK,GAAG,MAAM,MAAM,KAAM,CAAC;CACjD,MAAM,eAAe,KAAA;CACrB,IAAI,MAAM,GACR,OAAO;EAAE,GAAG;EAAG,QAAQ;EAAG,MAAM;EAAG,KAAK;EAAG,MAAM;EAAG;EAAY;EAAW;CAAa;CAE1F,IAAI,MAAM,GAAG;EACX,MAAM,IAAI,OAAO;EACjB,OAAO;GAAE,GAAG;GAAG,QAAQ;GAAG,MAAM;GAAG,KAAK;GAAG,MAAM;GAAG;GAAY;GAAW;EAAa;CAC1F;CAEA,MAAM,MAAM,QAAQ,KAAK,MAAM,MAAM;CACrC,MAAM,UAAU,IAAI,MAAc,SAAS;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAC7B,IAAI,cAAc,QAAQ;EACxB,IAAI,MAAM;EACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,OAAO,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC;EAEpC,QAAQ,KAAK,MAAM;CACrB,OAAO;EACL,MAAM,MAAM,IAAI,MAAc,CAAC;EAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,IAAI,KAAK,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC;EAEtC,QAAQ,KAAK,cAAc,GAAG;CAChC;CAEF,QAAQ,MAAM,GAAG,MAAM,IAAI,CAAC;CAE5B,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,KAAK,MAAO,QAAQ,IAAK,SAAS;CACjD,MAAM,UAAU,KAAK,IAAI,YAAY,GAAG,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC;CAElF,OAAO;EACL;EACA,QAAQ,cAAc,CAAC,GAAG,MAAM,CAAC;EACjC,MAAM,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;EAC1C,KAAK,QAAQ;EACb,MAAM,QAAQ,KAAK,IAAI,SAAS,MAAM;EACtC;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;AAoCA,SAAgB,eACd,aACA,aACsB;CACtB,IAAI,gBAAgB,aAAa,gBAAgB,QAC/C,MAAM,IAAI,gBACR,gEAAgE,aAClE;CAGF,IAAI,WAAW;CACf,IAAI,WAAW;CACf,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;EAC3C,MAAM,aAAa,YAAY;EAC/B,IAAI,CAAC,OAAO,SAAS,UAAU,GAC7B,MAAM,IAAI,gBACR,uCAAuC,EAAE,uBAAuB,YAClE;EAEF,IAAI,aAAa,GAAG;OACf,IAAI,aAAa,GAAG;OACpB;CACP;CAEA,MAAM,WAAW,WAAW;CAC5B,MAAM,YAAY,gBAAgB,YAAY,WAAW;CACzD,OAAO;EACL,GAAG,YAAY;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ,sBAAsB,WAAW,QAAQ;CACnD;AACF;;;AAIA,SAAgB,uBACd,QACA,OACQ;CACR,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,MACR,iDAAiD,OAAO,OAAO,MAAM,MAAM,OAAO,EACpF;CAEF,MAAM,IAAI,OAAO;CACjB,IAAI,MAAM,GAAG,OAAO;CACpB,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,IAAI,KAAK,IAAI,MAAM,KAAM,OAAO,EAAG,IAAI,MAAM;CAE/C,OAAO,OAAO;AAChB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAa,kCAA0C"}

package/dist/pareto-BqNW3LJR.d.ts

@@ -0,0 +1,117 @@
+//#region src/campaign/gates/power-preflight.d.ts
+/**
+ * Power preflight — "can this budget detect the effect you are hunting?"
+ *
+ * The failure it prevents (measured, twice): a live prompt-improvement campaign ran
+ * 333 sandbox cells over 5.6 hours and produced a +0.08 holdout lift the ship gate
+ * (paired bootstrap, CI.low > 0.05) could not distinguish from zero — because at
+ * that holdout size and worker variance the MINIMUM DETECTABLE lift was larger than
+ * any effect a prompt change plausibly produces. The budget was spent learning what
+ * a 30-second calculation on the baseline cells already knew. No eval framework we
+ * know of surfaces this; every underpowered improvement run everywhere ends in an
+ * uninformative "hold".
+ *
+ * Model: the ship rule is `CI.low(paired Δ) > deltaThreshold`. Approximating the
+ * bootstrap CI as normal, `CI.low ≈ effect − z·sd_Δ/√n`, so the smallest shippable
+ * true effect is `MDE = deltaThreshold + z·sd_Δ/√n`. The paired-delta SD is unknown
+ * before the candidate exists; we bound it by the zero-correlation case
+ * `sd_Δ ≤ √2·sd_baseline` — a CONSERVATIVE (upper) MDE, which is the correct
+ * direction for a warning. Pairing is per cell (`scenario:rep`), so reps multiply n.
+ *
+ * Standalone by design: feed it any baseline composites (a `gate:'none'` run, a
+ * live-proof table) BEFORE budgeting the real search; `selfImprove` also attaches
+ * it to every result and warns when the run was structurally unable to ship.
+ */
+interface PowerPreflightOptions {
+  /** Per-cell baseline composites on the HOLDOUT scenarios (one per scenario:rep cell). */
+  baselineComposites: number[];
+  /** Paired observations the budgeted comparison will produce
+   *  (holdout scenarios × reps). Defaults to `baselineComposites.length`. */
+  pairedN?: number;
+  /** The ship gate's effect-size threshold. Default 0.05 (defaultProductionGate). */
+  deltaThreshold?: number;
+  /** CI confidence the gate uses. Default 0.95. */
+  confidence?: number;
+  /** True when the holdout is scored by the SAME judge/scorer family as the gate
+   *  (selfImprove's default composition — one judge scores everything). Under a
+   *  shared channel, raising paired n reduces only the IDIOSYNCRATIC noise share;
+   *  systematic judge bias is untouched, so the MDE here is a lower bound and the
+   *  only full debiaser is an independent second scoring channel
+   *  (recursive-self-improvement S1c, closed form in EXP-023 P0). Default false. */
+  sharedScorerChannel?: boolean;
+}
+interface PowerPreflight {
+  /** Paired observations the comparison will have. */
+  n: number;
+  /** Baseline per-cell composite standard deviation (the variance the effect must beat). */
+  sd: number;
+  /** Minimum detectable lift: the smallest TRUE effect the gate could ship at this budget. */
+  mde: number;
+  /** Baseline holdout composite mean. */
+  baselineMean: number;
+  /** Headroom to a perfect 1.0 composite (the largest achievable lift on a [0,1] judge). */
+  headroom: number;
+  /** True when even the largest achievable effect (headroom) is below the MDE —
+   *  the run is structurally unable to ship regardless of proposal quality.
+   *  Only asserted for [0,1]-scaled judges (see `scaleAssumed`). */
+  underpowered: boolean;
+  /** True when composites look [0,1]-scaled; headroom/underpowered are only
+   *  meaningful under that convention (0-100 judges get mde/sd/n but no verdict). */
+  scaleAssumed: boolean;
+  deltaThreshold: number;
+  confidence: number;
+  /** Set when the holdout shares the gate's scoring channel: more cells cannot
+   *  buy back systematic judge bias — treat the MDE as a lower bound. */
+  sharedChannelCaveat?: string;
+  /** One actionable sentence for humans and logs. */
+  recommendation: string;
+}
+/** Estimate the minimum detectable lift a paired-holdout improvement run can
+ *  ship at a given budget, from the baseline holdout composites — call it BEFORE
+ *  spending a search to learn whether the effect you are hunting is even
+ *  observable at this holdout size and worker variance. */
+declare function powerPreflight(opts: PowerPreflightOptions): PowerPreflight;
+//#endregion
+//#region src/pareto.d.ts
+/**
+ * Pareto frontier — multi-objective optimization over candidate runs.
+ *
+ * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're
+ * trading off (cost, latency, quality) or (passRate, tokenBudget,
+ * ttfb), you rarely have a single "winner" — you have a set of
+ * non-dominated candidates. This module exposes:
+ *
+ *   - `paretoFrontier`: filter a set of candidates to the non-dominated ones
+ *   - `dominates`: does A dominate B across all objectives?
+ *
+ * Each objective is declared with a direction: 'maximize' (higher=better)
+ * or 'minimize' (lower=better). Candidates are any object; pass an
+ * `objective(candidate)` accessor.
+ */
+type Direction = 'maximize' | 'minimize';
+interface Objective<T> {
+  /** Stable label used in reports. */
+  name: string;
+  direction: Direction;
+  value: (candidate: T) => number;
+}
+interface ParetoResult<T> {
+  frontier: T[];
+  dominated: T[];
+  /** Index map: frontier[i] dominates each of dominatedBy[i]. */
+  dominanceMap: Array<{
+    dominator: T;
+    dominated: T[];
+  }>;
+}
+/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
+declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+/**
+ * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
+ * objective are excluded (can't rank them). A candidate enters the frontier
+ * iff no other candidate dominates it.
+ */
+declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+//#endregion
+export { paretoFrontier as a, powerPreflight as c, dominates as i, Objective as n, PowerPreflight as o, ParetoResult as r, PowerPreflightOptions as s, Direction as t };
+//# sourceMappingURL=pareto-BqNW3LJR.d.ts.map

package/dist/pareto-BqNW3LJR.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pareto-BqNW3LJR.d.ts","names":[],"sources":["../src/campaign/gates/power-preflight.ts","../src/pareto.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;UAwBiB;;EAEf;;;EAGA;;EAEA;;EAEA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;;EAGA;EACA;EACA;;;EAGA;;EAEA;;;;;;iBAec,eAAe,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;KClEjD;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa"}

package/dist/pipelines/index.d.ts

@@ -1,33 +1,8 @@
-import { S as ToolSpan, f as Run, r as BudgetSpec } from "../schema-BtVldJ3T.js";
+import { f as Run } from "../schema-BtVldJ3T.js";
 import { a as RunFilter, s as TraceStore } from "../store-CT9YIIve.js";
-import { o as llmSpans } from "../query-CJ_DX8vl.js";
-import { n as FailureClusterReport, r as failureClusterView, t as FailureCluster } from "../failure-cluster-CqcvCcdR.js";
-import { n as BaselineReport, p as computeToolUseMetrics, t as BaselineOptions } from "../baseline-CavEbRyH.js";
+import { n as FailureClusterReport, r as failureClusterView, t as FailureCluster } from "../failure-cluster-BLURuWG4.js";
 import { n as TrajectoryStep } from "../trajectory-YC15QDYQ.js";
-//#region src/pipelines/budget-breach.d.ts
-interface BudgetBreachFinding {
-  runId: string;
-  scenarioId: string;
-  variantId?: string;
-  dimension: keyof BudgetSpec;
-  limit: number;
-  consumed: number;
-  excessRatio: number;
-  timestamp: number;
-}
-interface BudgetBreachReport {
-  findings: BudgetBreachFinding[];
-  byDimension: Record<string, number>;
-  byScenario: Record<string, number>;
-  byVariant: Record<string, number>;
-  totalRuns: number;
-  breachedRunRatio: number;
-}
-declare function budgetBreachView(store: TraceStore, options?: {
-  scenarioId?: string;
-  variantId?: string;
-}): Promise<BudgetBreachReport>;
-//#endregion
+import { c as computeToolUseMetrics, d as judgeAgreementView, f as BudgetBreachFinding, g as BaselineReport, h as BaselineOptions, i as toolWasteView, l as JudgeAgreementReport, m as budgetBreachView, n as ToolWasteOptions, p as BudgetBreachReport, r as ToolWasteReport, t as ToolWasteFinding, u as JudgePair } from "../tool-waste-DjRDEsuI.js";
 //#region src/pipelines/first-divergence.d.ts
 interface DivergenceReport {
   runA: string;
@@ -45,23 +20,6 @@ interface DivergenceOptions {
 }
 declare function firstDivergenceView(store: TraceStore, runA: string, runB: string, options?: DivergenceOptions): Promise<DivergenceReport>;
 //#endregion
-//#region src/pipelines/judge-agreement.d.ts
-interface JudgePair {
-  judgeA: string;
-  judgeB: string;
-  dimension: string;
-  /** Number of (targetSpanId, dimension) tuples both judges scored. */
-  commonItems: number;
-  pearson: number;
-  krippendorff: number;
-}
-interface JudgeAgreementReport {
-  pairs: JudgePair[];
-  dimensions: string[];
-  judgeIds: string[];
-}
-declare function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport>;
-//#endregion
 //#region src/pipelines/regression.d.ts
 interface RegressionSpec {
   metric: string;
@@ -108,24 +66,5 @@ interface StuckLoopOptions {
 }
 declare function stuckLoopView(store: TraceStore, options?: StuckLoopOptions): Promise<StuckLoopReport>;
 //#endregion
-//#region src/pipelines/tool-waste.d.ts
-interface ToolWasteFinding {
-  runId: string;
-  wastedCalls: number;
-  totalCalls: number;
-  wasteRate: number;
-}
-interface ToolWasteReport {
-  byRun: ToolWasteFinding[];
-  overallWasteRate: number;
-}
-interface ToolWasteOptions {
-  runId?: string;
-  usageOracle?: (tool: ToolSpan, later: {
-    llm: Awaited<ReturnType<typeof llmSpans>>;
-  }) => boolean;
-}
-declare function toolWasteView(store: TraceStore, options?: ToolWasteOptions): Promise<ToolWasteReport>;
-//#endregion
 export { BudgetBreachFinding, BudgetBreachReport, DivergenceOptions, DivergenceReport, FailureCluster, FailureClusterReport, JudgeAgreementReport, JudgePair, RegressionOptions, RegressionSpec, StuckLoopFinding, StuckLoopOptions, StuckLoopReport, ToolWasteFinding, ToolWasteOptions, ToolWasteReport, budgetBreachView, computeToolUseMetrics, failureClusterView, firstDivergenceView, judgeAgreementView, regressionView, stuckLoopView, toolWasteView };
 //# sourceMappingURL=index.d.ts.map

package/dist/pipelines/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/pipelines/budget-breach.ts","../../src/pipelines/first-divergence.ts","../../src/pipelines/judge-agreement.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts","../../src/pipelines/tool-waste.ts"],"mappings":";;;;;;;UAUiB;EACf;EACA;EACA;EACA,iBAAiB;EACjB;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV,aAAa;EACb,YAAY;EACZ,WAAW;EACX;EACA;;iBAGoB,iBACpB,OAAO,YACP;EAAW;EAAqB;IAC/B,QAAQ;;;UCrBM;EACf;EACA;EACA;EACA,QAAQ;EACR,QAAQ;EACR;;EAEA;;UAGe;;EAEf,cAAc,GAAG,gBAAgB,GAAG;;iBAGhB,oBACpB,OAAO,YACP,cACA,cACA,UAAS,oBACR,QAAQ;;;UCnBM;EACf;EACA;EACA;;EAEA;EACA;EACA;;UAGe;EACf,OAAO;EACP;EACA;;iBAGoB,mBAAmB,OAAO,aAAa,QAAQ;;;UChBpD;EACf;EACA;;EAEA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B,0BAA0B;EACzC,UAAU;EACV,WAAW;;iBAGS,eACpB,OAAO,YACP,SAAS,kBACT,SAAS,oBACR,QAAQ;;;UCDM;EACf;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;;EAEf;;EAEA;;;;;EAKA;;EAEA;;iBAGoB,cACpB,OAAO,YACP,UAAS,mBACR,QAAQ;;;UChDM;EACf;EACA;EACA;EACA;;UAGe;EACf,OAAO;EACP;;UAGe;EACf;EACA,eAAe,MAAM,UAAU;IAAS,KAAK,QAAQ,kBAAkB;;;iBAGnD,cACpB,OAAO,YACP,UAAS,mBACR,QAAQ"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/pipelines/first-divergence.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts"],"mappings":";;;;;;UAYiB;EACf;EACA;EACA;EACA,QAAQ;EACR,QAAQ;EACR;;EAEA;;UAGe;;EAEf,cAAc,GAAG,gBAAgB,GAAG;;iBAGhB,oBACpB,OAAO,YACP,cACA,cACA,UAAS,oBACR,QAAQ;;;UCnBM;EACf;EACA;;EAEA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B,0BAA0B;EACzC,UAAU;EACV,WAAW;;iBAGS,eACpB,OAAO,YACP,SAAS,kBACT,SAAS,oBACR,QAAQ;;;UCDM;EACf;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;;EAEf;;EAEA;;;;;EAKA;;EAEA;;iBAGoB,cACpB,OAAO,YACP,UAAS,mBACR,QAAQ"}