@tangle-network/agent-eval 0.180.0 → 0.181.0

package/dist/rl.js

@@ -1,22 +1,22 @@
 import { s as ValidationError } from "./errors-Dngq5h35.js";
 import { i as compareCodeUnits } from "./canonical-DPyQ_rpt.js";
-import { i as makeRng } from "./internal-BMFSR8Ns.js";
+import { a as medianInPlace } from "./internal-BMFSR8Ns.js";
 import { t as mulberry32 } from "./random-Dn5fPWkt.js";
-import { o as benjaminiHochberg } from "./power-and-mde-B8F2RdcD.js";
 import { l as wilcoxonSignedRank } from "./paired-arms-D4aeIHUy.js";
+import { r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
+import { a as campaignCellToRunRecord, s as decidePairedPromotion } from "./run-record-Br-Yzt_k.js";
 import { r as observedSplitScore, s as trainingScore } from "./reward-nw2xZGZG.js";
-import { s as runTaskScore } from "./run-record-DQpSf7t-.js";
-import { a as campaignCellToRunRecord } from "./run-record-CR63CpHK.js";
+import { s as runTaskScore } from "./run-record-DualPTn2.js";
 import { i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
 import { n as InMemoryTraceStore } from "./store-DNe_Uv1Q.js";
-import { t as runEvalCampaign } from "./eval-campaign-Cs-7MiCs.js";
+import { t as runEvalCampaign } from "./eval-campaign-aYdtjtJR.js";
 import { s as assertRewardGate } from "./schema-C1aaAxTf.js";
 import { t as isSplitEligible } from "./exporters-Df7TgHFv.js";
-import { t as mintRolloutRows } from "./mint-Cc1_zwRQ.js";
+import { t as mintRolloutRows } from "./mint-ySIIkKlV.js";
 import { t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
-import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-2D5Gw9z9.js";
+import { n as assertUniqueObservationIds, s as validateOutcomeMetricSpecifications, t as rubricPredictiveValidity } from "./rubric-predictive-validity-CCK-1B7w.js";
 import { n as thompsonCurriculum, r as varianceBasedCurriculum, t as observationsFromRunRecords } from "./active-curriculum-OjIWgrUJ.js";
-import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-ChBKlTd_.js";
+import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-BXlkwMPR.js";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -24,29 +24,10 @@ import { dirname, join } from "node:path";
 /**
 * Sample-efficient adaptation evaluation.
 *
-* For foundation-model-based agents, the load-bearing capability isn't
-* raw end-state performance — it's *how fast the agent reaches that
-* performance from cold start*. The same model with a worse prompt that
-* adapts in 5 demonstrations beats the same model with a better prompt
-* that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit)
-* reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, …
-* in-context examples or fine-tune steps.
-*
-* This module ships:
-*
-*   1. `runAdaptationCurve` — given a runner that takes k demonstrations
-*      and returns a score, produce the (k, score) curve.
-*   2. `compareAdaptationCurves` — paired comparison across two policies.
-*      Returns per-k delta with bootstrap CIs and an "area-under-curve"
-*      summary statistic.
-*   3. `firstPassK` — for pass/fail evaluation, the minimum k at which
-*      the policy reliably passes (≥ pass-rate threshold over reps).
-*
-* Use cases:
-*   - Compare two prompt designs that have similar end-state performance
-*     but different in-context efficiency.
-*   - Decide between fine-tuning and prompting based on adaptation cost.
-*   - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts.
+* An adaptation curve records scores after k demonstrations or training steps.
+* Comparison pairs the same scenarios and resamples their whole curves.
+* The normalized area summarizes performance over the observed k range.
+* A first-pass k is descriptive and carries no separate reliability claim.
 */
 async function runAdaptationCurve(opts) {
 	const ks = opts.ks ?? [
@@ -59,6 +40,10 @@ async function runAdaptationCurve(opts) {
 	];
 	const reps = opts.reps ?? 3;
 	const passThreshold = opts.passThreshold ?? .5;
+	assertKs(ks, "runAdaptationCurve");
+	assertScenarioIds(opts.scenarios, "runAdaptationCurve");
+	if (!Number.isInteger(reps) || reps < 1) throw new ValidationError("runAdaptationCurve: reps must be a positive integer");
+	if (!Number.isFinite(passThreshold) || passThreshold < 0 || passThreshold > 1) throw new ValidationError("runAdaptationCurve: passThreshold must be in [0,1]");
 	const sortedKs = [...ks].sort((a, b) => a - b);
 	const points = [];
 	for (const k of sortedKs) {
@@ -67,7 +52,7 @@ async function runAdaptationCurve(opts) {
 		let totalPasses = 0;
 		let totalAttempts = 0;
 		for (const scenario of opts.scenarios) {
-			const sid = scenario.scenarioId ?? `scenario-${opts.scenarios.indexOf(scenario)}`;
+			const sid = scenario.scenarioId;
 			const scores = [];
 			let passes = 0;
 			for (let r = 0; r < reps; r++) {
@@ -76,6 +61,7 @@ async function runAdaptationCurve(opts) {
 					k,
 					rep: r
 				});
+				assertScore(score, `runAdaptationCurve: scenario '${sid}', k=${k}, rep=${r}`);
 				scores.push(score);
 				if (score >= passThreshold) passes++;
 				allScores.push(score);
@@ -118,69 +104,106 @@ async function runAdaptationCurve(opts) {
 	};
 }
 /**
-* Paired comparison of two adaptation curves. Per-k deltas with 95%
-* bootstrap CIs (constructed from each curve's `perScenario` per-k means
-* — the bootstrap unit is the scenario, not the rep).
+* Compare identical scenario cohorts on identical k grids, paired by scenarioId.
+* Missing pairs, duplicate identities, and cohort changes across k are refused.
+* The bootstrap resamples whole scenarios, preserving dependence across k.
+* Per-k intervals describe the curve; shared paired area decisions determine the verdict.
+* Bootstrap eligibility is necessary but does not establish scenario independence.
 */
 function compareAdaptationCurves(a, b, opts = {}) {
-	const conf = opts.confidence ?? .95;
-	const resamples = opts.bootstrapResamples ?? 500;
-	const rng = makeRng(opts.seed, a.points.flatMap((point) => point.perScenario.map((cell) => cell.meanScore)), b.points.flatMap((point) => point.perScenario.map((cell) => cell.meanScore)));
-	const perK = [];
-	for (const ap of a.points) {
-		const bp = b.points.find((p) => p.k === ap.k);
-		if (!bp) continue;
-		const aMeans = ap.perScenario.map((s) => s.meanScore);
-		const bMeans = bp.perScenario.map((s) => s.meanScore);
-		const aCi = bootstrapMeanCi(aMeans, resamples, conf, rng);
-		const bCi = bootstrapMeanCi(bMeans, resamples, conf, rng);
-		perK.push({
-			k: ap.k,
-			deltaMean: ap.meanScore - bp.meanScore,
-			aLow: aCi.low,
-			aHigh: aCi.high,
-			bLow: bCi.low,
-			bHigh: bCi.high
-		});
-	}
-	const areaDelta = a.adaptationArea - b.adaptationArea;
-	const firstPassKDelta = a.firstPassK !== null && b.firstPassK !== null ? b.firstPassK - a.firstPassK : null;
-	const meanDelta = perK.reduce((s, p) => s + p.deltaMean, 0) / Math.max(1, perK.length);
+	const confidence = opts.confidence ?? .95;
+	const resamples = opts.bootstrapResamples ?? 2e3;
+	const minimumEffect = opts.minimumEffect ?? 0;
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new ValidationError("compareAdaptationCurves: confidence must be in (0,1)");
+	if (!Number.isInteger(resamples) || resamples < 1) throw new ValidationError("compareAdaptationCurves: bootstrapResamples must be a positive integer");
+	if (!Number.isFinite(minimumEffect) || minimumEffect < 0 || minimumEffect > 1) throw new ValidationError("compareAdaptationCurves: minimumEffect must be in [0,1]");
+	const aPoints = indexCurve(a, "A");
+	const bPoints = indexCurve(b, "B");
+	const ks = [...aPoints.keys()].sort((x, y) => x - y);
+	const missingInA = [...bPoints.keys()].filter((k) => !aPoints.has(k));
+	const missingInB = ks.filter((k) => !bPoints.has(k));
+	if (missingInA.length > 0 || missingInB.length > 0) throw new ValidationError(`compareAdaptationCurves: k grids differ; missing in A=[${missingInA}], missing in B=[${missingInB}]`);
+	const scenarioIds = [...aPoints.get(ks[0]).keys()].sort();
+	const expectedIds = new Set(scenarioIds);
+	for (const [arm, points] of [["A", aPoints], ["B", bPoints]]) for (const [k, cells] of points) {
+		const missing = scenarioIds.filter((id) => !cells.has(id));
+		const extra = [...cells.keys()].filter((id) => !expectedIds.has(id));
+		if (missing.length > 0 || extra.length > 0) throw new ValidationError(`compareAdaptationCurves: scenario pairs differ in ${arm} at k=${k}; missing=[${missing}], unexpected=[${extra}]`);
+	}
+	const bootstrapOptions = {
+		confidence,
+		resamples,
+		statistic: "mean",
+		seed: opts.seed
+	};
+	const perK = ks.map((k) => ({
+		k,
+		delta: pairedBootstrap(scenarioIds.map((id) => bPoints.get(k).get(id)), scenarioIds.map((id) => aPoints.get(k).get(id)), bootstrapOptions)
+	}));
+	const aAreas = scenarioIds.map((id) => scenarioArea(ks, aPoints, id));
+	const bAreas = scenarioIds.map((id) => scenarioArea(ks, bPoints, id));
+	const decisionOptions = {
+		...bootstrapOptions,
+		threshold: minimumEffect
+	};
+	const aImprovement = decidePairedPromotion(bAreas, aAreas, decisionOptions);
+	const bImprovement = decidePairedPromotion(aAreas, bAreas, decisionOptions);
+	const areaDelta = aImprovement.bootstrap ?? pairedBootstrap(bAreas, aAreas, bootstrapOptions);
 	let verdict;
-	if (Math.abs(meanDelta) < .02 && Math.abs(areaDelta) < .02) verdict = "similar";
-	else if (meanDelta > 0 && areaDelta > 0) verdict = "a_better";
-	else if (meanDelta < 0 && areaDelta < 0) verdict = "b_better";
-	else verdict = "similar";
-	const rationale = `mean per-k delta=${meanDelta.toFixed(3)}, area delta=${areaDelta.toFixed(3)}` + (firstPassKDelta !== null ? `, first-pass-k delta=${firstPassKDelta}` : "");
+	if (!aImprovement.sufficient || ks.length < 2) verdict = "insufficient_evidence";
+	else if (aImprovement.promote) verdict = "a_better";
+	else if (bImprovement.promote) verdict = "b_better";
+	else verdict = "inconclusive";
+	const rationale = `paired scenarios=${scenarioIds.length}, area delta=${areaDelta.mean.toFixed(3)}, ${confidence * 100}% ${aImprovement.statistic} interval=[${aImprovement.low.toFixed(3)}, ${aImprovement.high.toFixed(3)}], minimum effect=${minimumEffect}; ${verdict}`;
 	return {
 		perK,
 		areaDelta,
-		firstPassKDelta,
+		aImprovement,
+		bImprovement,
+		scenarioIds,
 		verdict,
 		rationale
 	};
 }
-/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
+/** First observed k whose pass rate reaches the threshold; this is a descriptive summary. */
 function firstPassK(curve, threshold = .5) {
 	return curve.points.find((p) => p.passRate >= threshold)?.k ?? null;
 }
-function bootstrapMeanCi(xs, resamples, confidence, rng) {
-	if (xs.length < 2) return {
-		low: xs[0] ?? 0,
-		high: xs[0] ?? 0
-	};
-	const samples = new Array(resamples);
-	for (let b = 0; b < resamples; b++) {
-		let sum = 0;
-		for (let i = 0; i < xs.length; i++) sum += xs[Math.floor(rng() * xs.length)];
-		samples[b] = sum / xs.length;
+function assertKs(ks, where) {
+	if (ks.length === 0 || ks.some((k) => !Number.isInteger(k) || k < 0)) throw new ValidationError(`${where}: ks must contain nonnegative integers`);
+	if (new Set(ks).size !== ks.length) throw new ValidationError(`${where}: duplicate k values`);
+}
+function assertScenarioIds(cells, where) {
+	if (cells.length === 0 || cells.some((cell) => !cell.scenarioId?.trim())) throw new ValidationError(`${where}: scenarios must have explicit nonempty scenarioId values`);
+	const seen = /* @__PURE__ */ new Set();
+	for (const { scenarioId } of cells) {
+		if (seen.has(scenarioId)) throw new ValidationError(`${where}: duplicate scenarioId '${scenarioId}'`);
+		seen.add(scenarioId);
+	}
+}
+function assertScore(score, where) {
+	if (!Number.isFinite(score) || score < 0 || score > 1) throw new ValidationError(`${where}: score must be finite and in [0,1], got ${score}`);
+}
+function indexCurve(curve, arm) {
+	const where = `compareAdaptationCurves: ${arm}`;
+	assertKs(curve.points.map((point) => point.k), where);
+	return new Map(curve.points.map((point) => {
+		assertScenarioIds(point.perScenario, `${where} at k=${point.k}`);
+		return [point.k, new Map(point.perScenario.map((cell) => {
+			assertScore(cell.meanScore, `${where}: '${cell.scenarioId}' at k=${point.k}`);
+			return [cell.scenarioId, cell.meanScore];
+		}))];
+	}));
+}
+function scenarioArea(ks, points, id) {
+	let area = 0;
+	for (let i = 1; i < ks.length; i++) {
+		const left = ks[i - 1];
+		const right = ks[i];
+		area += (points.get(left).get(id) + points.get(right).get(id)) * (right - left) / 2;
 	}
-	samples.sort((a, b) => a - b);
-	const alpha = 1 - confidence;
-	return {
-		low: samples[Math.floor(alpha / 2 * resamples)],
-		high: samples[Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)]
-	};
+	const maxK = ks[ks.length - 1];
+	return maxK === 0 ? 0 : area / maxK;
 }
 //#endregion
 //#region src/rl/compute-curves.ts
@@ -311,81 +334,58 @@ function fitLogSlope(points) {
 /**
 * Contamination probe — held-out perturbation tests.
 *
-* The bug class: once a benchmark scenario set is published, models train
-* on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
-* MMLU-Pro all exist because their predecessors got contaminated within
-* months. The right defense is to keep a held-out *perturbed* version of
-* every scenario — same task, slightly different surface — and check
-* whether scores diverge significantly. Genuine capability transfers; rote
-* memorization doesn't.
-*
-* This module ships the probe contract:
-*
-*   1. A `ScenarioPerturbation` strategy type — function that produces a
-*      perturbed scenario from an original.
-*   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
-*      both halves and reports per-scenario score divergence + a global
-*      contamination verdict via paired Wilcoxon.
-*   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
-*      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
-*      task's structural difficulty while breaking surface memorization.
-*
-* The verdict is conservative: if the perturbed-vs-original score
-* difference is statistically significant (BH-adjusted p < 0.05) AND
-* the median drop is > 5 percentage points, we flag *contamination
-* suspected*. False positives are possible (the perturbation might
-* actually be harder); the default is to flag for review, not to
-* autoreject.
+* Score each scenario and its perturbation, then test the paired differences.
+* A significant global Wilcoxon result plus a worthwhile median drop flags
+* contamination for review. Perturbations may change difficulty, so the result
+* does not identify contamination as the cause. Per-item differences have no
+* calibrated sampling null and carry no p-values or q-values.
 */
 async function runContaminationProbe(input, opts = {}) {
-	const fdr = opts.fdr ?? .05;
+	const alpha = opts.alpha ?? .05;
 	const minMedianDrop = opts.minMedianDrop ?? .05;
 	const floor = opts.scoreFloor ?? 0;
+	if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) throw new ValidationError("runContaminationProbe: alpha must be in (0,1)");
+	for (const [name, value] of [["scoreFloor", floor], ["minMedianDrop", minMedianDrop]]) if (!Number.isFinite(value) || value < 0 || value > 1) throw new ValidationError(`runContaminationProbe: ${name} must be in [0,1]`);
+	const ids = input.originals.map(input.scenarioId);
+	if (ids.some((id) => !id?.trim()) || new Set(ids).size !== ids.length) throw new ValidationError("runContaminationProbe: original scenario IDs must be nonempty and unique");
 	if (!input.perturbed && !input.perturbation) throw new ValidationError("runContaminationProbe: must supply either `perturbed` or `perturbation`.");
 	const perturbed = input.perturbed ?? await Promise.all(input.originals.map((s) => input.perturbation.apply(s)));
 	if (perturbed.length !== input.originals.length) throw new ValidationError(`runContaminationProbe: perturbed length ${perturbed.length} ≠ originals ${input.originals.length}`);
 	const origScores = await Promise.all(input.originals.map((s) => input.scoreFn(s)));
 	const pertScores = await Promise.all(perturbed.map((s) => input.scoreFn(s)));
-	const perScenario = input.originals.map((s, i) => ({
-		scenarioId: input.scenarioId(s),
+	for (const score of [...origScores, ...pertScores]) if (!Number.isFinite(score) || score < 0 || score > 1) throw new ValidationError(`runContaminationProbe: scores must be finite and in [0,1], got ${score}`);
+	const perScenario = ids.map((scenarioId, i) => ({
+		scenarioId,
 		originalScore: origScores[i],
 		perturbedScore: pertScores[i],
-		delta: pertScores[i] - origScores[i],
-		qValue: NaN
+		delta: pertScores[i] - origScores[i]
 	}));
 	const valid = perScenario.filter((p) => p.originalScore >= floor && p.perturbedScore >= floor);
+	const excludedScenarioIds = perScenario.filter((p) => p.originalScore < floor || p.perturbedScore < floor).map((p) => p.scenarioId);
+	const deltas = valid.map((p) => p.delta);
+	const medianDelta = deltas.length === 0 ? null : medianInPlace(deltas);
+	const meanDelta = deltas.length === 0 ? null : deltas.reduce((sum, d) => sum + d, 0) / deltas.length;
 	if (valid.length < 4) return {
 		perScenario,
-		pairedTest: {
-			w: 0,
-			p: 1
-		},
-		medianDelta: 0,
-		meanDelta: 0,
+		pairedTest: null,
+		medianDelta,
+		meanDelta,
 		contaminationSuspected: false,
 		reason: `insufficient valid scenarios (n=${valid.length}, need ≥ 4)`,
-		n: valid.length
+		n: valid.length,
+		excludedScenarioIds
 	};
 	const pairedTest = wilcoxonSignedRank(valid.map((p) => p.originalScore), valid.map((p) => p.perturbedScore));
-	const deltas = valid.map((p) => p.delta);
-	const sortedDeltas = [...deltas].sort((a, b) => a - b);
-	const median = sortedDeltas[Math.floor(sortedDeltas.length / 2)];
-	const mean = deltas.reduce((s, d) => s + d, 0) / deltas.length;
-	const { qValues } = benjaminiHochberg(valid.map((p) => Math.min(1, Math.max(1e-6, 1 - Math.abs(p.delta) / 1))), fdr);
-	for (let i = 0; i < valid.length; i++) {
-		const v = valid[i];
-		const idx = perScenario.findIndex((p) => p.scenarioId === v.scenarioId);
-		if (idx >= 0) perScenario[idx].qValue = qValues[i];
-	}
-	const contaminationSuspected = pairedTest.p < fdr && median <= -minMedianDrop;
+	const contaminationSuspected = pairedTest.p < alpha && medianDelta <= -minMedianDrop;
 	return {
 		perScenario,
 		pairedTest,
-		medianDelta: median,
-		meanDelta: mean,
+		medianDelta,
+		meanDelta,
 		contaminationSuspected,
-		reason: contaminationSuspected ? `paired p=${pairedTest.p.toFixed(4)} < ${fdr} and median drop ${median.toFixed(4)} ≥ ${minMedianDrop}` : pairedTest.p >= fdr ? `no significant difference (paired p=${pairedTest.p.toFixed(4)})` : `significant but small effect (median delta ${median.toFixed(4)})`,
-		n: valid.length
+		reason: contaminationSuspected ? `paired p=${pairedTest.p.toFixed(4)} < ${alpha} and median drop ${(-medianDelta).toFixed(4)} ≥ ${minMedianDrop}` : pairedTest.p >= alpha ? `no significant difference (paired p=${pairedTest.p.toFixed(4)})` : `significant but no qualifying drop (median delta ${medianDelta.toFixed(4)})`,
+		n: valid.length,
+		excludedScenarioIds
 	};
 }
 /**
@@ -1474,14 +1474,21 @@ function clamp(x, lo, hi) {
 //#endregion
 //#region src/rl/predictive-validity-researcher.ts
 /**
-* Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain:
-* rubrics that don't predict deployment outcomes don't earn weight.
+* Proposes rubric experiments against one declared outcome.
+* A correlation supports a hypothesis; the caller must measure any resulting change.
 */
 var PredictiveValidityResearcher = class {
 	opts;
 	lastReport = null;
 	constructor(opts) {
-		this.opts = opts;
+		validateOutcomeMetricSpecifications([opts.targetOutcome]);
+		if (opts.rubrics !== void 0) assertUniqueObservationIds(opts.rubrics, "rubric");
+		if (opts.failureThreshold !== void 0 && !Number.isFinite(opts.failureThreshold)) throw new Error("failureThreshold must be finite");
+		this.opts = {
+			...opts,
+			targetOutcome: { ...opts.targetOutcome },
+			rubrics: opts.rubrics === void 0 ? void 0 : [...opts.rubrics]
+		};
 	}
 	async inspectFailures(runs) {
 		const threshold = this.opts.failureThreshold ?? .5;
@@ -1521,35 +1528,34 @@ var PredictiveValidityResearcher = class {
 			payload: { directive: "researcher.collect-more-outcomes" },
 			rationale: "predictive-validity researcher has no prior report; cannot recommend rubric reweighting until at least one report exists"
 		}];
-		const decorativeThreshold = this.opts.decorativeThreshold ?? .4;
 		const changes = [];
-		for (const ranking of this.lastReport.ranked) {
-			if (ranking.verdict === "load_bearing") continue;
-			if (Math.abs(ranking.spearman) >= decorativeThreshold) continue;
-			changes.push({
-				kind: "reviewer_prompt",
-				payload: {
-					rubric: ranking.rubric,
-					action: "down-weight",
-					spearman: ranking.spearman,
-					bestOutcome: ranking.bestOutcome
-				},
-				rationale: `predictive-validity Spearman=${ranking.spearman.toFixed(3)} vs ${ranking.bestOutcome} (decorative); recommend down-weighting`,
-				expectedDelta: -Math.max(0, .05 - Math.abs(ranking.spearman))
-			});
-		}
-		for (const ranking of this.lastReport.ranked.slice(0, 1)) {
-			if (ranking.verdict !== "load_bearing") continue;
+		const target = { ...this.opts.targetOutcome };
+		const pairs = this.lastReport.pairs.filter((pair) => pair.outcome === target.id && pair.outcomeDirection === target.direction && (this.opts.rubrics === void 0 || this.opts.rubrics.includes(pair.rubric)));
+		if (pairs.length === 0) return [{
+			kind: "threshold",
+			payload: {
+				directive: "researcher.collect-more-outcomes",
+				targetOutcome: target
+			},
+			rationale: `no estimable rubric association with ${target.id}; collect independent outcome observations before proposing weight changes`
+		}];
+		for (const pair of pairs) {
+			const interval = pair.alignedSpearmanCi95;
+			const aligned = pair.alignedSpearman >= .4 && interval !== null && interval.lower > 0;
+			const inverse = pair.alignedSpearman <= -.4 && interval !== null && interval.upper < 0;
+			const action = aligned ? "test-up-weight" : inverse ? "test-reverse-or-replace" : "collect-calibration-evidence";
 			changes.push({
 				kind: "reviewer_prompt",
 				payload: {
-					rubric: ranking.rubric,
-					action: "up-weight",
-					spearman: ranking.spearman,
-					bestOutcome: ranking.bestOutcome
+					rubric: pair.rubric,
+					action,
+					targetOutcome: target,
+					spearman: pair.spearman,
+					alignedSpearman: pair.alignedSpearman,
+					alignedSpearmanCi95: interval === null ? null : { ...interval },
+					samples: pair.n
 				},
-				rationale: `predictive-validity Spearman=${ranking.spearman.toFixed(3)} vs ${ranking.bestOutcome} (load-bearing); recommend up-weighting`,
-				expectedDelta: Math.max(0, Math.abs(ranking.spearman) - .5) * .1
+				rationale: aligned ? `higher ${pair.rubric} scores associate with better ${target.id}; test increased weight on fresh evidence before adopting it` : inverse ? `higher ${pair.rubric} scores associate with worse ${target.id}; test reversal or replacement on fresh evidence` : `the association of ${pair.rubric} with desired ${target.id} does not support a direction of change; collect calibration evidence`
 			});
 		}
 		return changes;
@@ -1604,11 +1610,11 @@ var PredictiveValidityResearcher = class {
 		const report = await rubricPredictiveValidity({
 			runs,
 			outcomes: this.opts.outcomes,
-			outcomeMetrics: this.opts.outcomeMetrics,
+			outcomeMetrics: [this.opts.targetOutcome],
 			rubrics: this.opts.rubrics
 		});
-		if (this.opts.onReport) await this.opts.onReport(report);
-		this.lastReport = report;
+		if (this.opts.onReport) await this.opts.onReport(structuredClone(report));
+		this.setReport(report);
 		return report;
 	}
 	/**
@@ -1617,10 +1623,11 @@ var PredictiveValidityResearcher = class {
 	* researcher's later proposals informed by it.
 	*/
 	setReport(report) {
-		this.lastReport = report;
+		if (report.outcomeMetrics.find((metric) => metric.id === this.opts.targetOutcome.id)?.direction !== this.opts.targetOutcome.direction) throw new Error("predictive validity report does not match the declared target outcome and direction");
+		this.lastReport = structuredClone(report);
 	}
 	getLastReport() {
-		return this.lastReport;
+		return this.lastReport === null ? null : structuredClone(this.lastReport);
 	}
 };
 /** Coverage of a split that was never dealt any work. */
@@ -2044,6 +2051,12 @@ function prmTrainingPairs(stepRewardsByRun, opts = {}) {
 * `result.rewardSignals` to a custom RL loop.
 */
 async function runRLCampaign(opts) {
+	const outcomeStore = opts.outcomeStore;
+	const outcomeMetrics = opts.outcomeMetrics?.map((metric) => ({ ...metric }));
+	if (outcomeStore !== void 0 || outcomeMetrics !== void 0) {
+		if (outcomeStore === void 0 || outcomeMetrics === void 0) throw new Error("runRLCampaign requires outcomeStore and outcomeMetrics together");
+		validateOutcomeMetricSpecifications(outcomeMetrics);
+	}
 	const splitTag = opts.splitTag ?? "search";
 	const campaign = await runEvalCampaign({
 		...opts,
@@ -2080,10 +2093,10 @@ async function runRLCampaign(opts) {
 		verifiableRewardOptions: opts.verifiableReward
 	});
 	let predictiveValidity = null;
-	if (opts.outcomeStore && opts.outcomeMetrics && opts.outcomeMetrics.length > 0) predictiveValidity = await rubricPredictiveValidity({
+	if (outcomeStore && outcomeMetrics) predictiveValidity = await rubricPredictiveValidity({
 		runs: campaign.runs,
-		outcomes: opts.outcomeStore,
-		outcomeMetrics: opts.outcomeMetrics
+		outcomes: outcomeStore,
+		outcomeMetrics
 	});
 	const trainerRows = {};
 	if (opts.trainerExport?.dpo) trainerRows.dpo = await toDpoRows(preferences.pairs, opts.trainerExport.dpo, { lines: rolloutLines });
@@ -2215,7 +2228,7 @@ function buildSummary(args) {
 	lines.push(`reward-hacking: ${args.rewardHacking.verdict} (${args.rewardHacking.findings.length} signals checked)`);
 	if (args.predictiveValidity) {
 		const top = args.predictiveValidity.ranked[0];
-		lines.push(`top-rubric: ${top?.rubric ?? "none"} ρ=${(top?.spearman ?? 0).toFixed(2)} (${top?.verdict ?? "no data"})`);
+		lines.push(top ? `top-rubric: ${top.rubric} aligned ρ=${top.alignedSpearman.toFixed(2)} vs ${top.bestOutcome} (${top.outcomeDirection}; ${top.verdict})` : "top-rubric: none (no estimable outcome associations)");
 	}
 	return lines.join(" | ");
 }