@tangle-network/agent-eval 0.216.1 → 0.217.1

package/dist/rl.js

@@ -1,27 +1,23 @@
-import { s as ValidationError } from "./errors-DVZiqS7K.js";
-import { a as hashCanonical, i as compareCodeUnits } from "./canonical-B2RqLofv.js";
-import { r as observedSplitScore, s as trainingScore } from "./reward-nw2xZGZG.js";
-import { o as runTaskScore } from "./run-record-Boc5IUTp.js";
-import { a as medianInPlace } from "./internal-Epl-YBZW.js";
-import { t as mulberry32 } from "./random-ByUSnV9w.js";
-import { i as pairedBootstrap } from "./mean-DHoMiudD.js";
-import { a as wilcoxonSignedRank } from "./paired-arms-CwMGJsrS.js";
-import { t as decidePairedPromotion } from "./paired-promotion-decision-DsAPYGCm.js";
-import { n as campaignCellToRunRecord } from "./run-record-C5e7FjhJ.js";
-import { i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-E0d-WVcP.js";
-import { n as InMemoryTraceStore } from "./store-BhNuLdJw.js";
-import { t as runEvalCampaign } from "./eval-campaign-90jeaCAI.js";
-import { s as assertRewardGate } from "./schema-BuiujZ-g.js";
-import { n as isTrainingLineEligible, s as toSftRows$1, t as isSplitEligible } from "./exporters-X4xwla8A.js";
-import { t as mintRolloutRows } from "./mint-QaD5cA0u.js";
-import { t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
-import { n as prepareSystemOneReview } from "./systemone-review-D7sXwsqq.js";
-import { n as assertUniqueObservationIds, s as validateOutcomeMetricSpecifications, t as rubricPredictiveValidity } from "./rubric-predictive-validity-DaFW70y9.js";
-import { t as varianceBasedCurriculum } from "./active-curriculum-B_Lc9kfO.js";
+import { s as ValidationError } from "./errors-BcOs5MKW.js";
+import { a as hashCanonical, i as compareCodeUnits } from "./canonical-Dq2hOQ8L.js";
+import { o as runTaskScore } from "./run-record-DhEJgoPz.js";
+import { a as medianInPlace } from "./internal-BTK91Bqt.js";
+import { a as wilcoxonSignedRank } from "./paired-arms-Ct4LTKc1.js";
+import { i as pairedBootstrap } from "./mean-BbA_JCzW.js";
+import { t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
+import { n as campaignCellToRunRecord } from "./run-record-CysKLB93.js";
+import { n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-ByuETaIb.js";
+import { t as runEvalCampaign } from "./eval-campaign-BLEfBGbA.js";
+import { s as assertRewardGate } from "./schema-DxcKTzbi.js";
+import { n as isTrainingLineEligible, o as toSftRows, t as isSplitEligible } from "./exporters-Cl6IeNAu.js";
+import { t as mintRolloutRows } from "./mint-zszyYlpA.js";
+import { t as evaluateInterimReleaseConfidence } from "./sequential-BVv1sjsw.js";
+import { n as prepareSystemOneReview } from "./systemone-review-DzdOJ8I6.js";
+import { n as assertUniqueObservationIds, s as validateOutcomeMetricSpecifications, t as rubricPredictiveValidity } from "./rubric-predictive-validity-D3_oI72N.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-DUcMj-3D.js";
 import { createHash } from "node:crypto";
-import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
 //#region src/rl/adaptation-eval.ts
 /**
 * Sample-efficient adaptation evaluation.
@@ -627,7 +623,7 @@ function claimIntegritySystemOneReview(claim, scope, options) {
 		}]
 	});
 	return prepareSystemOneReview({
-		version: options.version ?? `claim-integrity-systemone-2`,
+		version: options.version ?? `claim-integrity-systemone-${CLAIM_INTEGRITY_VERSION}`,
 		request: {
 			model: options.model,
 			state: claimBrief(claim, scope),
@@ -723,7 +719,7 @@ const CLAIM_INTEGRITY_VERSION = "2";
 /** The digest a verdict cites: detector version, phrase set and policy. */
 function claimIntegrityDigest(policy = {}) {
 	return hashCanonical({
-		version: "2",
+		version: CLAIM_INTEGRITY_VERSION,
 		phrases: GRADER_REFERENCE_PHRASES,
 		recordOnly: [...policy.recordOnly ?? []].sort()
 	});
@@ -807,29 +803,6 @@ function claimIntegrityVerdict(signals, policy = {}) {
 * or LLM-call count. We accept whatever the caller provides; the curve
 * is on whatever axis they pick.
 */
-async function runComputeCurve(opts) {
-	const points = [];
-	for (const budget of opts.budgets) {
-		const r = await opts.runAtBudget(budget);
-		points.push({
-			budgetId: budget.id,
-			cost: budget.cost,
-			score: r.score,
-			samples: r.samples,
-			std: r.std,
-			metrics: r.metrics
-		});
-	}
-	const sorted = [...points].sort((a, b) => a.cost - b.cost);
-	const logSlope = sorted.length >= 2 ? fitLogSlope(sorted) : null;
-	const best = points.reduce((a, b) => b.score > a.score ? b : a);
-	return {
-		candidateId: opts.candidateId,
-		points: sorted,
-		logSlope,
-		best
-	};
-}
 /** The simplest test-time scaling primitive. */
 async function bestOfN(opts) {
 	if (opts.n <= 0) throw new ValidationError("bestOfN: n must be > 0");
@@ -851,54 +824,11 @@ async function bestOfN(opts) {
 		bestIndex
 	};
 }
-/**
-* Self-consistency / majority-vote test-time scaling. For tasks with a
-* small categorical answer space (math problems, multiple choice).
-*/
-async function selfConsistency(opts) {
-	if (opts.n <= 0) throw new ValidationError("selfConsistency: n must be > 0");
-	const rollouts = [];
-	const histogram = {};
-	for (let i = 0; i < opts.n; i++) {
-		const r = await opts.sample(i);
-		rollouts.push(r);
-		const key = opts.answerKey(r);
-		histogram[key] = (histogram[key] ?? 0) + 1;
-	}
-	let answer = "";
-	let max = -1;
-	for (const [k, v] of Object.entries(histogram)) if (v > max) {
-		max = v;
-		answer = k;
-	}
-	const representative = rollouts.find((r) => opts.answerKey(r) === answer) ?? rollouts[0];
-	return {
-		answer,
-		agreement: max / opts.n,
-		histogram,
-		representative,
-		rollouts
-	};
-}
-function paretoFrontier(points) {
+function paretoCostScoreFrontier(points) {
 	const onFrontier = [];
 	for (const p of points) if (!points.some((q) => q !== p && q.cost <= p.cost && q.score >= p.score && (q.cost < p.cost || q.score > p.score))) onFrontier.push(p);
 	return onFrontier.sort((a, b) => a.cost - b.cost);
 }
-function fitLogSlope(points) {
-	const xs = points.map((p) => Math.log(Math.max(1e-12, p.cost)));
-	const ys = points.map((p) => p.score);
-	const n = xs.length;
-	const mx = xs.reduce((s, x) => s + x, 0) / n;
-	const my = ys.reduce((s, y) => s + y, 0) / n;
-	let num = 0;
-	let den = 0;
-	for (let i = 0; i < n; i++) {
-		num += (xs[i] - mx) * (ys[i] - my);
-		den += (xs[i] - mx) ** 2;
-	}
-	return den === 0 ? 0 : num / den;
-}
 //#endregion
 //#region src/rl/contamination.ts
 /**
@@ -960,67 +890,6 @@ async function runContaminationProbe(input, opts = {}) {
 		excludedScenarioIds
 	};
 }
-/**
-* Identifier-rename perturbation for code/text scenarios. Replaces every
-* occurrence of the listed identifiers with synthesized aliases. Use when
-* the scenario's structural difficulty is independent of variable names
-* (e.g. SWE-Bench-style coding tasks).
-*/
-function renameVariables(identifiers, rename = (n, i) => `${n}_${(i % 26 + 10).toString(36)}`) {
-	return {
-		kind: "rename_variables",
-		apply(scenario) {
-			let prompt = scenario.prompt;
-			identifiers.forEach((id, i) => {
-				const replacement = rename(id, i);
-				const re = new RegExp(`\\b${escapeRegex(id)}\\b`, "g");
-				prompt = prompt.replace(re, replacement);
-			});
-			return {
-				...scenario,
-				prompt
-			};
-		}
-	};
-}
-/**
-* Order-shuffle perturbation. Reshuffles a list-shaped section of the
-* prompt (for QA scenarios that present options A/B/C/D — answer depends
-* on the option labels, not order). Caller provides the section extractor.
-*/
-function shuffleOrder(shuffleSection, seed) {
-	const rng = mulberry32(seed);
-	return {
-		kind: "shuffle_order",
-		apply(scenario) {
-			const newPrompt = shuffleSection(scenario.prompt, rng);
-			return {
-				...scenario,
-				prompt: newPrompt
-			};
-		}
-	};
-}
-/**
-* Inject-irrelevant-clause perturbation. Adds a benign sentence that
-* shouldn't change the answer. Tests for "did the model just memorize
-* the input string."
-*/
-function injectIrrelevantClause(clause, position = "prefix") {
-	return {
-		kind: "inject_irrelevant_clause",
-		apply(scenario) {
-			const prompt = position === "prefix" ? `${clause} ${scenario.prompt}` : `${scenario.prompt} ${clause}`;
-			return {
-				...scenario,
-				prompt
-			};
-		}
-	};
-}
-function escapeRegex(s) {
-	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
 //#endregion
 //#region src/rl/rollout-input.ts
 /**
@@ -1315,7 +1184,7 @@ function toDpoJsonl(rows) {
 * On the line path a realness-gated line stays in its group at reward 0 rather
 * than being dropped. 0 is the honest label for a faked success and is usable
 * signal; removing the line would also move the group's baseline, just in the
-* other direction. (SFT differs — see `toSftRows`.)
+* other direction. (SFT differs — see `toSftRowsWithLookups`.)
 */
 async function toGrpoRows(lines, lookups) {
 	return grpoRowsFromLines(lines, lookups);
@@ -1381,7 +1250,7 @@ function toGrpoJsonl(rows) {
 * says which path a row took. A captured transcript whose every turn is copied
 * context yields no row rather than a lookup-built substitute.
 */
-async function toSftRows(lines, lookups) {
+async function toSftRowsWithLookups(lines, lookups) {
 	for (const line of lines) assertRewardGate(line, "SFT export");
 	const include = lookups.include ?? (() => true);
 	const rows = [];
@@ -1389,7 +1258,7 @@ async function toSftRows(lines, lookups) {
 		if (!isTrainingLineEligible(line, lookups) || !include(line)) continue;
 		const captured = line.messages.length > 0;
 		const messages = captured ? line.messages : await recoveredTranscript(line, lookups);
-		const [row] = toSftRows$1([{
+		const [row] = toSftRows([{
 			...line,
 			messages
 		}], lookups);
@@ -1511,35 +1380,6 @@ const PRM_CONTEXT_REQUIREMENT = {
 function admitPrmTriples(triples, context) {
 	return admitUngatedByInvocation(triples, (t) => [t.prefixRunId, t.rejectedRunId], context, PRM_CONTEXT_REQUIREMENT, (line) => assertPrmTrainableLine(line, context.mintedWithMaxSteps));
 }
-function toPrmJsonl(rows) {
-	return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
-}
-const STEP_REWARD_CONTEXT_REQUIREMENT = {
-	exporter: "step-reward export",
-	contextType: "RolloutLineContext",
-	because: "a StepReward carries a runId and a bare per-step reward, and nothing that says whether that run faked its success — so without the minted rollout lines this exporter ships the step-level components of a gamed run at full value while the run-level scalar sits at 0 elsewhere."
-};
-/**
-* Step-level reward rows as JSONL.
-*
-* `context` is REQUIRED for the same reason it is on `toDpoRows` and
-* `toPrmRows`: this is a line-less input carrying a reward number. Steps
-* belonging to a realness-gated run are dropped rather than zeroed — a
-* per-step reward of 0 across a whole trajectory is a claim that every step was
-* bad, which is a different (and false) statement from "this run's success was
-* fabricated, so its step-level credit assignment is meaningless".
-*/
-function stepRewardsToJsonl(stepRewards, context) {
-	const rows = admitUngatedByInvocation(stepRewards, (s) => [s.runId], context, STEP_REWARD_CONTEXT_REQUIREMENT).map((s) => ({
-		runId: s.runId,
-		spanId: s.spanId,
-		stepIndex: s.stepIndex,
-		reward: s.reward,
-		determinism: s.determinism,
-		weight: s.weight ?? 1
-	}));
-	return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
-}
 //#endregion
 //#region src/rl/dataset.ts
 const DATASET_FORMAT_SET = /* @__PURE__ */ new Set([
@@ -1644,7 +1484,7 @@ async function buildRlDataset(lines, lookups, config, preferences) {
 		rowCounts.grpo = rows.length;
 	}
 	if (formats.includes("sft")) {
-		const rows = await toSftRows(lines, lookups);
+		const rows = await toSftRowsWithLookups(lines, lookups);
 		requireRows("sft", rows.length);
 		files["train.sft.jsonl"] = toSftJsonl(rows);
 		rowCounts.sft = rows.length;
@@ -1753,35 +1593,6 @@ function datasheetToMarkdown(m) {
 * (what the eval harnesses capture; the RunRecord validator ignores the extra
 * keys). The harvest reads them directly — no trace store round-trip needed.
 */
-/**
-* Append graded records to the corpus (append-only JSONL). Deduplicates by
-* `runId` against what's already on disk so re-running the same harness is
-* idempotent. Creates the file and parent dir. This is the call every eval
-* harness makes by default after producing its records.
-*/
-function appendToCorpus(records, corpusPath) {
-	mkdirSync(dirname(corpusPath), { recursive: true });
-	const existing = existsSync(corpusPath) ? readCorpus(corpusPath) : [];
-	const seen = new Set(existing.map((r) => r.runId));
-	const lines = [];
-	let appended = 0;
-	let skipped = 0;
-	for (const r of records) {
-		if (seen.has(r.runId)) {
-			skipped++;
-			continue;
-		}
-		seen.add(r.runId);
-		lines.push(JSON.stringify(r));
-		appended++;
-	}
-	if (lines.length > 0) appendFileSync(corpusPath, `${lines.join("\n")}\n`);
-	return {
-		appended,
-		skipped,
-		total: existing.length + appended
-	};
-}
 /** Read the full corpus. Returns [] if the corpus does not exist yet. */
 function readCorpus(corpusPath) {
 	if (!existsSync(corpusPath)) return [];
@@ -1789,48 +1600,6 @@ function readCorpus(corpusPath) {
 	for (const line of readFileSync(corpusPath, "utf8").split("\n")) if (line.trim()) out.push(JSON.parse(line));
 	return out;
 }
-/**
-* The harvest's score reader is GATED: a gamed run reads 0, so it cannot buy
-* its way past `minScore` into the published bundle with its claimed score.
-* `null` = unscored (a labeled gap, dropped before packaging, never a 0).
-*/
-function rewardOf(r) {
-	const v = trainingScore(r);
-	return typeof v === "number" && Number.isFinite(v) ? v : null;
-}
-/**
-* Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
-* trajectory text from each record's top-level `prompt`/`completion`; records
-* missing either are excluded (a graded score with no trajectory can't train).
-* Optionally filters by score / split. Throws (via buildRlDataset) if nothing
-* survives — an empty dataset must never be published.
-*
-* `minScore` is applied to the GATED reward (`trainingScore`), so a gamed run
-* cannot buy its way into the published bundle with its claimed score —
-* `minScore` is exactly the door a reward-hacked run would otherwise clear for
-* SFT. Unscored records are dropped before packaging: a missing label is not a
-* zero, and it is not publishable either.
-*/
-async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
-	let records = readCorpus(corpusPath).filter((r) => typeof r.prompt === "string" && typeof r.completion === "string");
-	if (opts.splits) records = records.filter((r) => opts.splits.includes(r.splitTag));
-	records = records.filter((r) => rewardOf(r) !== null);
-	if (opts.minScore != null) records = records.filter((r) => {
-		const reward = rewardOf(r);
-		return reward !== null && reward >= opts.minScore;
-	});
-	const text = new Map(records.map((r) => [r.runId, {
-		prompt: r.prompt,
-		completion: r.completion
-	}]));
-	const lookups = {
-		promptOf: (id) => text.get(id)?.prompt ?? "",
-		completionOf: (id) => text.get(id)?.completion ?? "",
-		allowHeldOutTrainingData: opts.allowHeldOutTrainingData
-	};
-	const { rows } = await mintRolloutRows(records, new InMemoryTraceStore());
-	return buildRlDataset(rows, lookups, config);
-}
 //#endregion
 //#region src/rl/off-policy.ts
 /**
@@ -2022,18 +1791,6 @@ function doublyRobust(trajectories, opts = {}) {
 		contributionCounts
 	};
 }
-/**
-* Convenience: run all three estimators and return them side-by-side.
-* The recommended diagnostic — agreement across estimators is a much
-* stronger signal than any single one.
-*/
-function offPolicyEstimateAll(trajectories, opts = {}) {
-	return {
-		ips: inverseProbabilityWeighting(trajectories, opts),
-		snips: selfNormalizedImportanceWeighting(trajectories, opts),
-		dr: doublyRobust(trajectories, opts)
-	};
-}
 function zeroEstimate() {
 	return {
 		value: 0,
@@ -2471,78 +2228,6 @@ function makePair(a, b, scenarioId, minMargin) {
 }
 //#endregion
 //#region src/rl/process-reward.ts
-async function extractStepRewards(store, runId, opts) {
-	const ordered = [...await store.spans({ runId })].sort((a, b) => a.startedAt - b.startedAt);
-	const out = [];
-	let idx = 0;
-	for (const span of ordered) {
-		if (opts.preFilter && !opts.preFilter(span)) continue;
-		let scored = null;
-		for (const s of opts.scorers) {
-			if (!s.appliesTo.includes(span.kind)) continue;
-			const r = await s.score(span);
-			if (r) {
-				scored = r;
-				break;
-			}
-		}
-		if (!scored) continue;
-		out.push({
-			spanId: span.spanId,
-			runId,
-			stepIndex: idx++,
-			kind: span.kind,
-			name: span.name,
-			reward: scored.reward,
-			determinism: scored.determinism,
-			rationale: scored.rationale,
-			weight: scored.weight
-		});
-	}
-	return out;
-}
-function runwiseStepRewardSummary(stepRewards) {
-	if (stepRewards.length === 0) return {
-		runId: "",
-		totalSteps: 0,
-		meanReward: 0,
-		sumWeightedReward: 0,
-		failureFraction: 0,
-		worstStepDelta: 0,
-		worstStepIndex: null
-	};
-	const runId = stepRewards[0].runId;
-	let sumW = 0;
-	let sumWR = 0;
-	let failures = 0;
-	let worstDelta = 0;
-	let worstIdx = null;
-	let prev = stepRewards[0].reward;
-	for (let i = 0; i < stepRewards.length; i++) {
-		const s = stepRewards[i];
-		const w = s.weight ?? 1;
-		sumW += w;
-		sumWR += w * s.reward;
-		if (s.reward < .5) failures++;
-		if (i > 0) {
-			const delta = s.reward - prev;
-			if (delta < worstDelta) {
-				worstDelta = delta;
-				worstIdx = i;
-			}
-			prev = s.reward;
-		} else prev = s.reward;
-	}
-	return {
-		runId,
-		totalSteps: stepRewards.length,
-		meanReward: sumW === 0 ? 0 : sumWR / sumW,
-		sumWeightedReward: sumWR,
-		failureFraction: failures / stepRewards.length,
-		worstStepDelta: worstDelta,
-		worstStepIndex: worstIdx
-	};
-}
 /**
 * Build PRM training triples. The shape: pair runs that share an early
 * prefix (same scenario, same first N steps) and diverge later — at the
@@ -2690,7 +2375,7 @@ async function runRLCampaign(opts) {
 	const trainerRows = {};
 	if (opts.trainerExport?.dpo) trainerRows.dpo = await toDpoRows(preferences.pairs, opts.trainerExport.dpo, { lines: rolloutLines });
 	if (opts.trainerExport?.grpo) trainerRows.grpo = await toGrpoRows(rolloutLines, opts.trainerExport.grpo);
-	if (opts.trainerExport?.sft) trainerRows.sft = await toSftRows(rolloutLines, opts.trainerExport.sft);
+	if (opts.trainerExport?.sft) trainerRows.sft = await toSftRowsWithLookups(rolloutLines, opts.trainerExport.sft);
 	const summary = buildSummary({
 		campaign,
 		preferences,
@@ -2861,238 +2546,8 @@ function campaignToRunRecords(campaign, ctx) {
 		defaultCostUsd: ctx.defaultCostUsd
 	}));
 }
-/**
-* Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
-* A split score is emitted only when `report.taskScore` proves the configured
-* scoring panel completed. Partial scores remain in `outcome.raw` for
-* diagnosis. Layer errors and timeouts become judge or execution telemetry;
-* only a scored `fail` layer may produce task-failure detail.
-*/
-function verificationReportToRunRecord(report, ctx, opts = {}) {
-	const splitTag = ctx.splitTag ?? "search";
-	const runId = opts.runId ?? `run-${ctx.candidateId}-${ctx.experimentId}-${report.startedAt}`;
-	const taskScore = report.layers.some(hasValidTaskMeasurement) && isValidScore(report.taskScore) ? report.taskScore : void 0;
-	let executionErrorCount = 0;
-	let judgeErrorCount = 0;
-	let layerErrorCount = 0;
-	let layerTimeoutCount = 0;
-	let unscoredLayerCount = 0;
-	const raw = {
-		pass_count: report.passCount,
-		fail_count: report.failCount,
-		error_count: report.errorCount,
-		skipped_count: report.skippedCount,
-		duration_ms: report.durationMs,
-		execution_error_count: 0
-	};
-	for (const layer of report.layers) {
-		if (hasValidTaskMeasurement(layer)) raw[`layer.${layer.layer}`] = layer.score;
-		else unscoredLayerCount++;
-		raw[`layer_${layer.layer}_pass`] = layer.status === "pass" ? 1 : 0;
-		if (layer.status === "error" || layer.status === "timeout") {
-			if (layer.errorSource === "judge") judgeErrorCount++;
-			else executionErrorCount++;
-			if (layer.status === "error") layerErrorCount++;
-			else layerTimeoutCount++;
-		}
-		if (layer.diagnostics) {
-			for (const [k, v] of Object.entries(layer.diagnostics)) if (typeof v === "number" && Number.isFinite(v)) raw[`layer.${layer.layer}.${k}`] = v;
-		}
-	}
-	raw.execution_error_count = executionErrorCount;
-	if (judgeErrorCount > 0) raw.judge_error_count = judgeErrorCount;
-	if (layerErrorCount > 0) raw.layer_error_count = layerErrorCount;
-	if (layerTimeoutCount > 0) raw.layer_timeout_count = layerTimeoutCount;
-	if (unscoredLayerCount > 0) raw.unscored_layer_count = unscoredLayerCount;
-	if (taskScore !== void 0) raw.blended_score = taskScore;
-	const firstScoredFailure = report.layers.find((layer) => layer.status === "fail" && hasValidTaskMeasurement(layer));
-	const outcome = { raw };
-	if (taskScore !== void 0) {
-		if (splitTag === "holdout") outcome.holdoutScore = taskScore;
-		else outcome.searchScore = taskScore;
-	}
-	return {
-		runId,
-		experimentId: ctx.experimentId,
-		candidateId: ctx.candidateId,
-		seed: 0,
-		model: ctx.model,
-		promptHash: ctx.promptHash,
-		configHash: ctx.configHash,
-		commitSha: ctx.commitSha,
-		wallMs: report.durationMs,
-		costUsd: ctx.defaultCostUsd ?? null,
-		costProvenance: ctx.defaultCostUsd === void 0 ? {
-			kind: "uncaptured",
-			usd: null
-		} : {
-			kind: "estimated",
-			usd: ctx.defaultCostUsd
-		},
-		tokenUsage: {
-			input: 0,
-			output: 0
-		},
-		terminalOutcome: "succeeded",
-		outcome,
-		...firstScoredFailure ? {
-			failureClass: "unknown",
-			failureMode: `layer_${firstScoredFailure.layer}_fail`
-		} : {},
-		splitTag,
-		scenarioId: ctx.scenarioId
-	};
-}
-function hasValidTaskMeasurement(layer) {
-	return (layer.status === "pass" || layer.status === "fail") && isValidScore(layer.score);
-}
-function isValidScore(score) {
-	return typeof score === "number" && Number.isFinite(score) && score >= 0 && score <= 1;
-}
 //#endregion
 //#region src/rl/sim-fidelity.ts
-/**
-* Simulator fidelity — score a user SIMULATOR's realism against real-user
-* trace distributions.
-*
-* Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
-* user-simulator objectives) stand in for real users in most of the numbers
-* we publish. The standing threat is the Sim2Real gap: a simulator that is
-* distributionally unlike production creates "easy mode" and silently
-* inflates every score built on it. This module measures that gap from the
-* SAME artifact both sides already produce — `RunRecord`s — so no new
-* capture pipeline is needed:
-*
-*   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
-*     simulated and production record distributions, collapsed into a
-*     fidelity coefficient in [0,1].
-*   - `easyModeCheck` — the headline academic failure mode (sim inflates
-*     pass-rate over production) as its own named artifact.
-*
-* Every synthetic-persona eval result should publish its fidelity
-* coefficient alongside the score — a number from an unrepresentative
-* simulator is an unlabeled estimate. Wire-in points:
-*
-*   - canonical persona evals: pass the campaign's `RunRecord`s as
-*     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
-*     feedback tables, coding-agent sessions) as `production`
-*   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
-*     penalty when searching over generated personas
-*   - the durable corpus (`./corpus`): both sides read straight from
-*     `readCorpus` — tag sim vs production by `experimentId`
-*/
-/** Reserved histogram category for `null` feature values. A capture-rate
-*  difference (one side instruments a signal, the other does not) registers
-*  as divergence by design: a simulator that produces no tool traces is not
-*  representative of production that does. */
-const ABSENT_CATEGORY = "(absent)";
-/** Minimum non-null observations PER SIDE for a feature to enter the
-*  fidelity mean. Below this the JSD estimate is sampling noise. */
-const DEFAULT_MIN_N_PER_FEATURE = 20;
-/** Quantile buckets used to discretize numeric features. Quartiles balance
-*  resolution against per-bucket sample size at the default minN. */
-const DEFAULT_QUANTILE_BUCKETS = 4;
-/** Fidelity at or above this → 'representative'; below → 'skewed'.
-*  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
-*  clearly shifted mode — the point where per-feature shifts start changing
-*  which failure classes an eval can even observe. */
-const REPRESENTATIVE_MIN_FIDELITY = .8;
-const TOP_SHIFT_COUNT = 5;
-/**
-* Default feature set — ONLY fields verified present on both simulated and
-* production records:
-*
-*   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
-*     validator (non-finite values read as absent rather than poisoning a
-*     bucket).
-*   - `failure_class` — optional taxonomy field; absent counted explicitly.
-*   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
-*     `outcome.raw` counters the intake adapters and eval harnesses write
-*     (`turns_completed`, `assistant_messages`, `tool_errors`,
-*     `turns_aborted`); absent on records whose producer did not capture
-*     them, counted explicitly.
-*   - `completion_length` — from the optional `CorpusRecord` trajectory
-*     text; the message-length proxy when records come from the corpus.
-*
-* `RunRecord` carries event COUNTS, not event ordering, so
-* `tool_error_recovery` is a counts-only derivation: errors occurred and the
-* run still completed cleanly ('recovered') vs aborted or classified as a
-* failure ('unrecovered') — not a literal error→retry sequence check.
-*/
-const defaultBehaviorFeatures = (record) => {
-	const raw = record.outcome?.raw ?? {};
-	const toolErrors = finiteOrNull(raw.tool_errors);
-	const turnsAborted = finiteOrNull(raw.turns_aborted);
-	const completion = record.completion;
-	return {
-		score: finiteOrNull(observedSplitScore(record, "holdout")) ?? finiteOrNull(observedSplitScore(record, "search")),
-		failure_class: record.failureClass ?? null,
-		wall_ms: finiteOrNull(record.wallMs),
-		output_tokens: finiteOrNull(record.tokenUsage?.output),
-		turn_count: finiteOrNull(raw.turns_completed) ?? finiteOrNull(raw.assistant_messages),
-		tool_errors: toolErrors,
-		tool_error_recovery: toolErrorRecovery(toolErrors, turnsAborted, record.failureClass),
-		completion_length: typeof completion === "string" ? completion.length : null
-	};
-};
-function toolErrorRecovery(toolErrors, turnsAborted, failureClass) {
-	if (toolErrors === null) return null;
-	if (toolErrors === 0) return "no-tool-errors";
-	return (turnsAborted ?? 0) > 0 || failureClass !== void 0 && failureClass !== "success" ? "unrecovered" : "recovered";
-}
-function finiteOrNull(value) {
-	return typeof value === "number" && Number.isFinite(value) ? value : null;
-}
-/**
-* Jensen-Shannon divergence between two categorical histograms (raw counts;
-* normalized internally). Log base 2 → bounded [0,1]: 0 = identical
-* distributions, 1 = disjoint support. Symmetric, defined even where the
-* supports differ — exactly the regime sim-vs-production comparison lives in.
-* Throws on zero-mass or negative/non-finite counts: an empty histogram has
-* no distribution and a silent 0 would read as "perfectly representative".
-*/
-function jsDivergence(p, q) {
-	const keys = /* @__PURE__ */ new Set([...Object.keys(p), ...Object.keys(q)]);
-	if (keys.size === 0) throw new ValidationError("jsDivergence: both histograms are empty");
-	let pSum = 0;
-	let qSum = 0;
-	for (const key of keys) {
-		const pv = p[key] ?? 0;
-		const qv = q[key] ?? 0;
-		if (!Number.isFinite(pv) || !Number.isFinite(qv) || pv < 0 || qv < 0) throw new ValidationError(`jsDivergence: negative or non-finite count for category "${key}"`);
-		pSum += pv;
-		qSum += qv;
-	}
-	if (pSum === 0 || qSum === 0) throw new ValidationError("jsDivergence: a histogram with zero total mass has no distribution");
-	let divergence = 0;
-	for (const key of keys) {
-		const pp = (p[key] ?? 0) / pSum;
-		const qp = (q[key] ?? 0) / qSum;
-		const m = (pp + qp) / 2;
-		if (pp > 0) divergence += .5 * pp * Math.log2(pp / m);
-		if (qp > 0) divergence += .5 * qp * Math.log2(qp / m);
-	}
-	return Math.min(1, Math.max(0, divergence));
-}
-/**
-* Deterministic quantile edges over a value set (the UNION of both sides, so
-* sim and production land in the same buckets). Linear interpolation between
-* order statistics; duplicate edges from heavy ties collapse into fewer,
-* wider buckets. Returns `bucketCount - 1` edges before deduplication.
-*/
-function quantileEdges(values, bucketCount = DEFAULT_QUANTILE_BUCKETS) {
-	if (values.length === 0) throw new ValidationError("quantileEdges: requires at least one value");
-	if (!Number.isInteger(bucketCount) || bucketCount < 2) throw new ValidationError(`quantileEdges: bucketCount must be an integer >= 2, got ${bucketCount}`);
-	const sorted = [...values].sort((a, b) => a - b);
-	const edges = [];
-	for (let k = 1; k < bucketCount; k++) {
-		const pos = k / bucketCount * (sorted.length - 1);
-		const lo = sorted[Math.floor(pos)];
-		const hi = sorted[Math.ceil(pos)];
-		edges.push(lo + (pos - Math.floor(pos)) * (hi - lo));
-	}
-	return [...new Set(edges)];
-}
 /** Stable half-open bucket label for a value against quantile edges:
 *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
 function bucketLabel(value, edges) {
@@ -3100,126 +2555,6 @@ function bucketLabel(value, edges) {
 	while (i < edges.length && value >= edges[i]) i++;
 	return `[${i === 0 ? "-inf" : String(edges[i - 1])},${i === edges.length ? "+inf" : String(edges[i])})`;
 }
-/**
-* Compare a simulator's RunRecords against production RunRecords, feature by
-* feature. Numeric features are bucketed by deterministic quantiles of the
-* union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
-* inputs — "no records" is a wiring error, not a distribution.
-*/
-function simFidelityReport(simulated, production, opts = {}) {
-	if (simulated.length === 0) throw new ValidationError("simFidelityReport: simulated records are empty");
-	if (production.length === 0) throw new ValidationError("simFidelityReport: production records are empty");
-	const extract = opts.features ?? defaultBehaviorFeatures;
-	const minN = opts.minNPerFeature ?? DEFAULT_MIN_N_PER_FEATURE;
-	const simMaps = simulated.map(extract);
-	const prodMaps = production.map(extract);
-	const featureNames = [];
-	const seen = /* @__PURE__ */ new Set();
-	for (const map of [...simMaps, ...prodMaps]) for (const name of Object.keys(map)) if (!seen.has(name)) {
-		seen.add(name);
-		featureNames.push(name);
-	}
-	const perDimension = [];
-	const insufficientData = [];
-	for (const feature of featureNames) {
-		const simVals = simMaps.map((m) => m[feature] ?? null);
-		const prodVals = prodMaps.map((m) => m[feature] ?? null);
-		const nSim = simVals.filter((v) => v !== null).length;
-		const nProd = prodVals.filter((v) => v !== null).length;
-		if (nSim < minN || nProd < minN) {
-			insufficientData.push(feature);
-			continue;
-		}
-		const { sim, prod } = histograms(feature, simVals, prodVals);
-		perDimension.push({
-			feature,
-			divergence: jsDivergence(sim, prod),
-			topShifts: topShifts(sim, simVals.length, prod, prodVals.length),
-			nSim,
-			nProd
-		});
-	}
-	if (perDimension.length === 0) return {
-		perDimension,
-		fidelity: NaN,
-		insufficientData,
-		verdict: "insufficient-data"
-	};
-	const fidelity = 1 - perDimension.reduce((sum, d) => sum + d.divergence, 0) / perDimension.length;
-	return {
-		perDimension,
-		fidelity,
-		insufficientData,
-		verdict: fidelity >= REPRESENTATIVE_MIN_FIDELITY ? "representative" : "skewed"
-	};
-}
-function histograms(feature, simVals, prodVals) {
-	const kinds = /* @__PURE__ */ new Set();
-	for (const v of [...simVals, ...prodVals]) if (v !== null) kinds.add(typeof v);
-	if (kinds.size > 1) throw new ValidationError(`simFidelityReport: feature "${feature}" mixes string and number values — an extractor must return one kind per feature`);
-	let toCategory;
-	if (kinds.has("number")) {
-		const union = [];
-		for (const v of [...simVals, ...prodVals]) if (v !== null) union.push(v);
-		const edges = quantileEdges(union);
-		toCategory = (v) => bucketLabel(v, edges);
-	} else toCategory = (v) => v;
-	const count = (vals) => {
-		const hist = {};
-		for (const v of vals) {
-			const key = v === null ? ABSENT_CATEGORY : toCategory(v);
-			hist[key] = (hist[key] ?? 0) + 1;
-		}
-		return hist;
-	};
-	return {
-		sim: count(simVals),
-		prod: count(prodVals)
-	};
-}
-function topShifts(sim, simTotal, prod, prodTotal) {
-	const shifts = [.../* @__PURE__ */ new Set([...Object.keys(sim), ...Object.keys(prod)])].map((value) => ({
-		value,
-		pSim: (sim[value] ?? 0) / simTotal,
-		pProd: (prod[value] ?? 0) / prodTotal
-	}));
-	shifts.sort((a, b) => {
-		const delta = Math.abs(b.pSim - b.pProd) - Math.abs(a.pSim - a.pProd);
-		return delta !== 0 ? delta : a.value.localeCompare(b.value);
-	});
-	return shifts.slice(0, TOP_SHIFT_COUNT);
-}
-/**
-* The headline simulator failure mode as its own named artifact: a simulator
-* that creates "easy mode" inflates pass-rate relative to production, and
-* every score measured against it overstates reality. Throws on empty inputs
-* and on records carrying neither score — a silently-skipped record would
-* bias the very rate this check exists to keep honest.
-*/
-function easyModeCheck(simulated, production, opts = {}) {
-	if (simulated.length === 0) throw new ValidationError("easyModeCheck: simulated records are empty");
-	if (production.length === 0) throw new ValidationError("easyModeCheck: production records are empty");
-	const threshold = opts.passThreshold ?? .5;
-	const tolerance = opts.inflationTolerance ?? .1;
-	const passRate = (records, side) => {
-		let passes = 0;
-		for (const r of records) {
-			const score = finiteOrNull(observedSplitScore(r, "holdout")) ?? finiteOrNull(observedSplitScore(r, "search"));
-			if (score === null) throw new ValidationError(`easyModeCheck: ${side} run "${r.runId}" carries neither holdoutScore nor searchScore`);
-			if (score >= threshold) passes++;
-		}
-		return passes / records.length;
-	};
-	const simPassRate = passRate(simulated, "simulated");
-	const prodPassRate = passRate(production, "production");
-	const gap = simPassRate - prodPassRate;
-	return {
-		simPassRate,
-		prodPassRate,
-		gap,
-		inflated: gap > tolerance
-	};
-}
 //#endregion
 //#region src/rl/tournament.ts
 /**
@@ -3321,58 +2656,6 @@ function fitBradleyTerry(outcomes, opts = {}) {
 		converged: delta < tol
 	};
 }
-function applyEloUpdate(ratings, outcome, opts = {}) {
-	const defaultRating = opts.defaultRating ?? 1500;
-	const k = opts.kFactor ?? 32;
-	const rW = ratings.get(outcome.winner) ?? defaultRating;
-	const rL = ratings.get(outcome.loser) ?? defaultRating;
-	const expectedW = 1 / (1 + 10 ** ((rL - rW) / 400));
-	const scoreW = outcome.draw ? .5 : 1;
-	const scoreL = outcome.draw ? .5 : 0;
-	const w = outcome.weight ?? 1;
-	const winnerDelta = k * w * (scoreW - expectedW);
-	const loserDelta = k * w * (scoreL - (1 - expectedW));
-	ratings.set(outcome.winner, rW + winnerDelta);
-	ratings.set(outcome.loser, rL + loserDelta);
-	return {
-		winnerDelta,
-		loserDelta
-	};
-}
-function buildPairwiseFromCampaign(input) {
-	const drawMargin = input.drawMargin ?? 0;
-	const byKey = /* @__PURE__ */ new Map();
-	for (const r of input.runs) {
-		const arr = byKey.get(r.matchKey) ?? [];
-		arr.push({
-			candidateId: r.candidateId,
-			score: r.score
-		});
-		byKey.set(r.matchKey, arr);
-	}
-	const outcomes = [];
-	for (const arr of byKey.values()) for (let i = 0; i < arr.length; i++) for (let j = i + 1; j < arr.length; j++) {
-		const a = arr[i];
-		const b = arr[j];
-		if (a.candidateId === b.candidateId) continue;
-		const margin = Math.abs(a.score - b.score);
-		if (margin <= drawMargin) outcomes.push({
-			winner: a.candidateId,
-			loser: b.candidateId,
-			draw: true,
-			weight: 1
-		});
-		else {
-			const [winner, loser] = a.score > b.score ? [a, b] : [b, a];
-			outcomes.push({
-				winner: winner.candidateId,
-				loser: loser.candidateId,
-				weight: margin
-			});
-		}
-	}
-	return outcomes;
-}
 //#endregion
 //#region src/rl/verified-findings-dataset.ts
 /**
@@ -3590,9 +2873,6 @@ function summarizeVerifiedFindings(rows) {
 	}
 	return summary;
 }
-function verifiedFindingsToJsonl(rows) {
-	return rows.map((row) => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
-}
 function sha256(buffer) {
 	return createHash("sha256").update(buffer).digest("hex");
 }
@@ -3711,6 +2991,6 @@ function loadVerifiedFindingsDataset(source) {
 	};
 }
 //#endregion
-export { ABSENT_CATEGORY, CLAIM_INTEGRITY_VERSION, FileSystemOutcomeStore, GRADER_REFERENCE_PHRASES, InMemoryOutcomeStore, PredictiveValidityResearcher, VERIFIED_FINDING_SCHEMA, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, buildVerifiedFindingRow, campaignToRunRecords, checkerReadSignal, claimIntegrityDigest, claimIntegritySystemOneReview, claimIntegrityTraceQuestions, claimIntegrityVerdict, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, farmingSignal, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, graderReferenceSignal, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, loadVerifiedFindingsDataset, monitorSignal, offPolicyEstimateAll, paretoFrontier, parseSourceScopeReferee, prmTrainingPairs, quantileEdges, readCorpus, refereeSignal, renameVariables, ruleHolds, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, scopeSignals, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, sourceScopeRefereeMessages, stepRewardsToJsonl, summarizeVerifiedFindings, systemOneSignal, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, validateDatasetFormats, varianceBasedCurriculum, verificationReportToRunRecord, verifiedFindingsToJsonl };
+export { FileSystemOutcomeStore, GRADER_REFERENCE_PHRASES, InMemoryOutcomeStore, PredictiveValidityResearcher, bestOfN, bucketLabel, buildRlDataset, campaignToRunRecords, checkerReadSignal, claimIntegrityDigest, claimIntegritySystemOneReview, claimIntegrityTraceQuestions, claimIntegrityVerdict, compareAdaptationCurves, detectRewardHacking, doublyRobust, extractPreferences, extractVerifiableReward, extractVerifiableRewardsFromRecords, farmingSignal, firstPassK, fitBradleyTerry, graderReferenceSignal, inverseProbabilityWeighting, loadVerifiedFindingsDataset, monitorSignal, paretoCostScoreFrontier, parseSourceScopeReferee, prmTrainingPairs, readCorpus, refereeSignal, ruleHolds, runAdaptationCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, scopeSignals, selfNormalizedImportanceWeighting, sourceScopeRefereeMessages, systemOneSignal, toAnthropicFormat, toDpoRows, toGrpoRows, toPrmRows, toSftJsonl, toSftRowsWithLookups, toTRLFormat, validateDatasetFormats };
 
 //# sourceMappingURL=rl.js.map