@tangle-network/agent-eval 0.135.1 → 0.135.3

package/dist/rl.js

@@ -1,15 +1,15 @@
 import { s as ValidationError } from "./errors-8YnH8WlF.js";
-import { R as wilcoxonSignedRank, o as benjaminiHochberg } from "./statistics-RwRNu2__.js";
+import { W as wilcoxonSignedRank, s as benjaminiHochberg } from "./statistics-CnGCLLqc.js";
 import { r as observedSplitScore, s as trainingScore } from "./reward-nw2xZGZG.js";
 import { o as runTaskScore } from "./run-record-BIwU2wdV.js";
 import { l as assertRewardGate } from "./schema-C6DW4ZHR.js";
 import { t as isSplitEligible } from "./exporters-q9iL-2Jf.js";
 import { t as mintRolloutRows } from "./mint-DyRUc9k6.js";
 import { a as InMemoryTraceStore } from "./integrity-BzRbCHzi.js";
-import { c as campaignCellToRunRecord, i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-DCdRK9TY.js";
-import { t as runEvalCampaign } from "./eval-campaign-aMG73hvX.js";
+import { c as campaignCellToRunRecord, i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-a-kYs0-i.js";
+import { t as runEvalCampaign } from "./eval-campaign-Cc8WZJ6b.js";
 import { t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
-import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-D6Q6n9oq.js";
+import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-BJf-8ejY.js";
 import { n as thompsonCurriculum, r as varianceBasedCurriculum, t as observationsFromRunRecords } from "./active-curriculum-C4mk67HP.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-ChBKlTd_.js";
 import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
@@ -1558,8 +1558,13 @@ var PredictiveValidityResearcher = class {
 					unpairedCandidateRuns: 0,
 					unpairedBaselineRuns: 0,
 					medianPairedDelta: null,
+					deltaStatistic: "median_bootstrap",
+					decidingDelta: null,
 					pairedCI: null,
 					pairedPValue: null,
+					mcnemar: null,
+					binaryScale: null,
+					tieFraction: null,
 					searchScore: null,
 					holdoutScore: null,
 					overfitGap: null,
@@ -2038,11 +2043,18 @@ async function runRLCampaign(opts) {
 		split: opts.preferences?.split ?? splitTag
 	});
 	let interimConfidence = null;
+	let deltaCoverage = [];
+	const minDeltaCoverage = opts.sequential?.minDeltaCoverage ?? 1;
+	if (!(Number.isFinite(minDeltaCoverage) && minDeltaCoverage >= 0 && minDeltaCoverage <= 1)) throw new Error(`runRLCampaign: sequential.minDeltaCoverage must be a finite fraction in [0, 1], got ${minDeltaCoverage}`);
 	if (opts.report?.comparator) {
 		const comparator = opts.report.comparator;
-		const deltaSeries = collectPairedDeltaSeries(campaign.runs, comparator);
-		if (deltaSeries.some((s) => s.deltas.length > 0)) interimConfidence = evaluateInterimReleaseConfidence({
-			deltaSeries,
+		const series = collectPairedDeltaSeries(campaign.runs, campaign.failedRuns, comparator);
+		deltaCoverage = series.map((s) => s.coverage);
+		if (series.every((s) => s.coverage.coverage >= minDeltaCoverage) && series.some((s) => s.deltas.length > 0)) interimConfidence = evaluateInterimReleaseConfidence({
+			deltaSeries: series.map(({ candidateId, deltas }) => ({
+				candidateId,
+				deltas
+			})),
 			alpha: opts.sequential?.alpha,
 			bound: opts.sequential?.bound,
 			rope: opts.sequential?.rope ?? opts.report?.rope
@@ -2066,6 +2078,7 @@ async function runRLCampaign(opts) {
 		campaign,
 		preferences,
 		interimConfidence,
+		deltaCoverage,
 		rewardHacking,
 		predictiveValidity
 	});
@@ -2074,6 +2087,7 @@ async function runRLCampaign(opts) {
 		rewardSignals,
 		preferences,
 		interimConfidence,
+		deltaCoverage,
 		rewardHacking,
 		predictiveValidity,
 		trainerRows,
@@ -2081,36 +2095,108 @@ async function runRLCampaign(opts) {
 		kind: "agent-eval-rl-campaign"
 	};
 }
-function collectPairedDeltaSeries(runs, comparator) {
-	const baseline = /* @__PURE__ */ new Map();
+/**
+* Pair on (scenarioId, seed) and count what was DEALT, not only what paired.
+*
+* Every drop below used to be silent: a comparator run with no score never
+* entered the map, a candidate run with no score was skipped, and a candidate
+* cell with no comparator at the same identity was skipped. The surviving
+* deltas then flowed into `evaluateInterimReleaseConfidence`, whose
+* `recommendation.decision` can be `promote_now` — so a candidate that scored 6
+* of 26 cells could be recommended for promotion off a series of length 6, with
+* nothing in the result saying 20 cells went dark. Same defect as the promotion
+* gates, one call frame up.
+*
+* The denominator is MEASURED: a cell counts as dealt because a run for it
+* exists on either arm. Missing scores are not imputed — the caller who knows
+* the failure value of its metric writes it onto the record.
+*/
+function collectPairedDeltaSeries(runs, failedRuns, comparator) {
+	const cellKey = (r) => `${r.scenarioId}::${r.seed}`;
+	const dealtFromFailures = /* @__PURE__ */ new Map();
+	for (const f of failedRuns) {
+		let set = dealtFromFailures.get(f.variantId);
+		if (!set) {
+			set = /* @__PURE__ */ new Set();
+			dealtFromFailures.set(f.variantId, set);
+		}
+		set.add(cellKey(f));
+	}
+	const comparatorDealt = new Set(dealtFromFailures.get(comparator) ?? []);
+	const comparatorScore = /* @__PURE__ */ new Map();
 	for (const r of runs) {
 		if (r.candidateId !== comparator) continue;
-		const sid = r.scenarioId;
+		const key = cellKey(r);
+		comparatorDealt.add(key);
 		const score = runTaskScore(r);
 		if (score === void 0) continue;
-		baseline.set(`${sid}::${r.seed}`, score);
+		comparatorScore.set(key, score);
+	}
+	const dealtByCandidate = /* @__PURE__ */ new Map();
+	const scoreByCandidate = /* @__PURE__ */ new Map();
+	for (const [variantId, cells] of dealtFromFailures) {
+		if (variantId === comparator) continue;
+		dealtByCandidate.set(variantId, new Set(cells));
 	}
-	const byCandidate = /* @__PURE__ */ new Map();
 	for (const r of runs) {
 		if (r.candidateId === comparator) continue;
-		const sid = r.scenarioId;
+		const key = cellKey(r);
+		let dealt = dealtByCandidate.get(r.candidateId);
+		if (!dealt) {
+			dealt = /* @__PURE__ */ new Set();
+			dealtByCandidate.set(r.candidateId, dealt);
+		}
+		dealt.add(key);
 		const score = runTaskScore(r);
 		if (score === void 0) continue;
-		const baseScore = baseline.get(`${sid}::${r.seed}`);
-		if (typeof baseScore !== "number") continue;
-		const arr = byCandidate.get(r.candidateId) ?? [];
-		arr.push(score - baseScore);
-		byCandidate.set(r.candidateId, arr);
+		let scored = scoreByCandidate.get(r.candidateId);
+		if (!scored) {
+			scored = /* @__PURE__ */ new Map();
+			scoreByCandidate.set(r.candidateId, scored);
+		}
+		scored.set(key, score);
 	}
-	return [...byCandidate.entries()].map(([candidateId, deltas]) => ({
-		candidateId,
-		deltas
-	}));
+	return [...dealtByCandidate.entries()].map(([candidateId, candidateDealt]) => {
+		const scored = scoreByCandidate.get(candidateId) ?? /* @__PURE__ */ new Map();
+		const deltas = [];
+		let unscoredCandidate = 0;
+		let unscoredComparator = 0;
+		let unmatched = 0;
+		for (const key of /* @__PURE__ */ new Set([...candidateDealt, ...comparatorDealt])) {
+			const onCandidate = candidateDealt.has(key);
+			const onComparator = comparatorDealt.has(key);
+			if (!onCandidate || !onComparator) {
+				unmatched += 1;
+				continue;
+			}
+			const a = scored.get(key);
+			const b = comparatorScore.get(key);
+			if (a === void 0) unscoredCandidate += 1;
+			else if (b === void 0) unscoredComparator += 1;
+			else deltas.push(a - b);
+		}
+		const dealt = (/* @__PURE__ */ new Set([...candidateDealt, ...comparatorDealt])).size;
+		return {
+			candidateId,
+			deltas,
+			coverage: {
+				candidateId,
+				dealt,
+				answered: deltas.length,
+				unscoredCandidate,
+				unscoredComparator,
+				unmatched,
+				coverage: dealt === 0 ? 0 : deltas.length / dealt
+			}
+		};
+	});
 }
 function buildSummary(args) {
 	const c = args.campaign;
 	const lines = [`${c.campaignId}: ${c.runs.length} successful runs / ${c.failedRuns.length} failed (fingerprint ${c.campaignFingerprint.slice(0, 12)}…)`, `preferences: ${args.preferences.pairs.length} (${args.preferences.strategy}, ${args.preferences.pairsBelowMargin} below margin)`];
 	if (args.interimConfidence) lines.push(`sequential verdict: ${args.interimConfidence.recommendation.decision}` + (args.interimConfidence.recommendation.candidateId ? ` ${args.interimConfidence.recommendation.candidateId}` : ""));
+	const shortfall = args.deltaCoverage.filter((c) => c.answered < c.dealt);
+	if (shortfall.length > 0) lines.push(`paired-delta coverage: ${shortfall.map((c) => `${c.candidateId} ${c.answered}/${c.dealt}`).join(", ")}${args.interimConfidence ? "" : " (sequential verdict withheld)"}`);
 	lines.push(`reward-hacking: ${args.rewardHacking.verdict} (${args.rewardHacking.findings.length} signals checked)`);
 	if (args.predictiveValidity) {
 		const top = args.predictiveValidity.ranked[0];