@tangle-network/agent-eval 0.142.1 → 0.143.0

package/dist/rl.js

@@ -6,14 +6,15 @@ import { o as runTaskScore } from "./run-record-vRgqWmJw.js";
 import { t as isSplitEligible } from "./exporters-q9iL-2Jf.js";
 import { t as mintRolloutRows } from "./mint-Ctwk079K.js";
 import { a as InMemoryTraceStore } from "./integrity-fdt8XPAv.js";
-import { c as campaignCellToRunRecord, i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-GyN0kMd8.js";
+import { i as filterDeterministicallyRewarded, l as campaignCellToRunRecord, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-DjTi9HLb.js";
 import { t as runEvalCampaign } from "./eval-campaign-YdkpWWoT.js";
 import { t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
 import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-BRR632r1.js";
 import { n as thompsonCurriculum, r as varianceBasedCurriculum, t as observationsFromRunRecords } from "./active-curriculum-C4mk67HP.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-ChBKlTd_.js";
 import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
+import { createHash } from "node:crypto";
 //#region src/rl/adaptation-eval.ts
 async function runAdaptationCurve(opts) {
 	const ks = opts.ks ?? [
@@ -2753,6 +2754,334 @@ function buildPairwiseFromCampaign(input) {
 	return outcomes;
 }
 //#endregion
-export { ABSENT_CATEGORY, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, DPO_CONTEXT_REQUIREMENT, FileSystemOutcomeStore, InMemoryOutcomeStore, PRM_CONTEXT_REQUIREMENT, PredictiveValidityResearcher, REPRESENTATIVE_MIN_FIDELITY, STEP_REWARD_CONTEXT_REQUIREMENT, appendToCorpus, applyEloUpdate, assertPrmTrainableLine, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, observationsFromRunRecords, offPolicyEstimateAll, paretoFrontier, prmTrainingPairs, quantileEdges, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, stepRewardsToJsonl, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, validateDatasetFormats, varianceBasedCurriculum, verificationReportToRunRecord };
+//#region src/rl/verified-findings-dataset.ts
+/**
+* Verified-findings dataset — execution-verified gold labels as RL-ready rows.
+*
+* A replay-verify batch re-executes a labeled trajectory prefix inside the
+* original docker image and checks, at the gold "incorrect" step k, whether
+* the recorded failure reproduces (arm A) and whether a generated fix makes
+* it vanish (arm B). That turns an annotation into an *executed* label: the
+* verdict is a returncode/signature comparison, not a rater's opinion.
+*
+* This module joins three artifact families into one row per replayed case:
+*
+*   1. the batch report (`batch-report.json` — per-case verdicts, fix arms),
+*   2. the gold label corpus (`*-labels.json` — incorrect step annotations),
+*   3. the normalized trajectory (`normalized/<trajId>/steps.json` — the
+*      action/observation sequence the agent actually took).
+*
+* The emitted `VerifiedFindingRow` carries the trajectory prefix up to k,
+* the gold label, the execution verdict with its evidence (exit codes,
+* failure signature, prefix divergences), the fix arm when present, and
+* per-row provenance (label/steps/report sha256s, docker images, run ids).
+* Rows are trainer input for step-level localizer/critic models; the reward
+* is deterministic because execution decided it.
+*
+* Join discipline: every missing or inconsistent join throws — a dataset
+* built from partially joined artifacts would silently train on wrong
+* labels. The batch report is authoritative for fix outcomes (per-case
+* `replay-verdict.json` files are written before the fix arm completes);
+* per-case files contribute prefix-divergence detail and run ids only, and
+* are cross-checked against the report where they overlap.
+*/
+const VERIFIED_FINDING_SCHEMA = "agent-eval/verified-finding@0";
+const DEFAULT_MAX_OBSERVATION_CHARS = 4e3;
+function fail(caseId, message) {
+	throw new Error(`verified-findings: ${caseId}: ${message}`);
+}
+function deriveFixOutcome(caseId, batchCase) {
+	const fix = batchCase.fix;
+	if (fix === null) {
+		if (batchCase.replayed) fail(caseId, "replayed case has no fix record — the batch always records the fix arm for replayed cases");
+		return {
+			outcome: "not-attempted",
+			command: null,
+			llmError: null,
+			armBExit: null,
+			failureVanished: null
+		};
+	}
+	const base = {
+		command: fix.command,
+		llmError: fix.llmError,
+		armBExit: fix.armBExit,
+		failureVanished: fix.failureVanished
+	};
+	if (fix.command !== null) {
+		if (fix.failureVanished === null) fail(caseId, "fix command present but failureVanished missing — arm B verdict was never recorded");
+		return {
+			outcome: fix.failureVanished ? "flipped" : "not-flipped",
+			...base
+		};
+	}
+	if (fix.llmError !== null) return {
+		outcome: "generation-failed",
+		...base
+	};
+	if (!fix.attempted || fix.sampledOut) return {
+		outcome: "not-attempted",
+		...base
+	};
+	fail(caseId, "unrecognized fix record state (attempted, no command, no llmError)");
+}
+function truncateObservation(observation, maxChars) {
+	if (observation === null || observation === void 0) return {
+		observation: null,
+		observationTruncated: false,
+		observationChars: 0
+	};
+	if (observation.length <= maxChars) return {
+		observation,
+		observationTruncated: false,
+		observationChars: observation.length
+	};
+	return {
+		observation: observation.slice(0, maxChars),
+		observationTruncated: true,
+		observationChars: observation.length
+	};
+}
+/**
+* Join one batch case with its gold label and trajectory into a row.
+* Throws on any join inconsistency — never emits a partially joined row.
+*/
+function buildVerifiedFindingRow(args) {
+	const { batchCase, label, steps, detail } = args;
+	const caseId = `${args.runId}/${batchCase.corpus}/${batchCase.trajId}`;
+	const maxObservationChars = args.maxObservationChars ?? DEFAULT_MAX_OBSERVATION_CHARS;
+	if (batchCase.status !== "ok") fail(caseId, `case status is '${batchCase.status}' (error: ${batchCase.error ?? "none"}) — only ok cases join`);
+	if (label.traj_id !== batchCase.trajId) fail(caseId, `label traj_id '${label.traj_id}' does not match the case`);
+	if (label.step_count !== batchCase.stepCount) fail(caseId, `label step_count ${label.step_count} != case stepCount ${batchCase.stepCount}`);
+	if (steps.length !== batchCase.stepCount) fail(caseId, `steps.json has ${steps.length} steps, case expects ${batchCase.stepCount}`);
+	for (let i = 0; i < steps.length; i++) {
+		const step = steps[i];
+		if (step.step_id !== i + 1) fail(caseId, `steps.json is not contiguous 1..n: index ${i} has step_id ${step.step_id}`);
+	}
+	const k = batchCase.k;
+	if (k < 1 || k > batchCase.stepCount) fail(caseId, `gold step k=${k} is outside 1..${batchCase.stepCount}`);
+	if (!batchCase.goldIncorrectSteps.includes(k)) fail(caseId, `gold step k=${k} is not in goldIncorrectSteps [${batchCase.goldIncorrectSteps.join(", ")}]`);
+	const labelIncorrectSteps = [...new Set(label.incorrect_stages.flatMap((s) => s.incorrect_step_ids))].sort((a, b) => a - b);
+	for (const goldStep of batchCase.goldIncorrectSteps) if (!labelIncorrectSteps.includes(goldStep)) fail(caseId, `case gold step ${goldStep} is absent from the label's incorrect steps — label/report mismatch`);
+	if (detail !== void 0) {
+		if (detail.k !== k) fail(caseId, `per-case verdict k=${detail.k} != report k=${k}`);
+		if (detail.prefixExecuted !== batchCase.prefixExecuted) fail(caseId, `per-case verdict prefixExecuted=${detail.prefixExecuted} != report ${batchCase.prefixExecuted}`);
+		if (detail.recordedReturncode !== batchCase.recordedReturncodeAtK) fail(caseId, `per-case verdict recordedReturncode=${detail.recordedReturncode} != report ${batchCase.recordedReturncodeAtK}`);
+	}
+	const stepAtK = steps[k - 1];
+	const trajectorySteps = steps.slice(0, k).map((step) => ({
+		stepId: step.step_id,
+		action: step.action,
+		...truncateObservation(step.observation, maxObservationChars)
+	}));
+	return {
+		schema: VERIFIED_FINDING_SCHEMA,
+		caseId,
+		corpus: batchCase.corpus,
+		trajId: batchCase.trajId,
+		task: {
+			agent: label.agent ?? null,
+			model: label.model ?? null,
+			taskName: label.task_name ?? null,
+			difficulty: label.difficulty ?? null,
+			solved: label.solved,
+			stepCount: batchCase.stepCount
+		},
+		gold: {
+			stepK: k,
+			actionAtK: stepAtK.action,
+			goldIncorrectSteps: [...batchCase.goldIncorrectSteps].sort((a, b) => a - b),
+			labelIncorrectSteps,
+			recordedReturncodeAtK: batchCase.recordedReturncodeAtK
+		},
+		trajectory: {
+			window: {
+				start: 1,
+				end: k
+			},
+			steps: trajectorySteps
+		},
+		verification: {
+			reproduced: batchCase.replayed,
+			signatureStrict: batchCase.armASignatureMatch,
+			signatureBasis: detail?.signatureBasis ?? null,
+			signature: batchCase.signature,
+			prefixExecuted: batchCase.prefixExecuted,
+			prefixDivergences: batchCase.prefixDivergences,
+			prefixDivergencePct: batchCase.prefixDivergencePct,
+			prefixDivergenceDetail: detail?.prefixDivergences ?? null,
+			armAExit: batchCase.armAExit,
+			armAReturncodeMatch: batchCase.armAReturncodeMatch,
+			armACommand: detail?.armACommand ?? null,
+			wallMs: batchCase.wallMs
+		},
+		fix: deriveFixOutcome(caseId, batchCase),
+		provenance: {
+			runId: args.runId,
+			batchGeneratedAt: args.batchGeneratedAt,
+			batchReportSha256: args.batchReportSha256,
+			labelsPath: args.labelsPath,
+			labelsSha256: args.labelsSha256,
+			stepsPath: args.stepsPath,
+			stepsSha256: args.stepsSha256,
+			image: batchCase.image,
+			derivedImage: batchCase.derivedImage,
+			cwd: batchCase.cwd,
+			cwdSource: batchCase.cwdSource,
+			originalRunId: detail?.runIds.original ?? null,
+			armARunId: detail?.runIds.armA ?? null
+		}
+	};
+}
+function summarizeVerifiedFindings(rows) {
+	const summary = {
+		rows: rows.length,
+		reproduced: 0,
+		signatureStrict: 0,
+		fix: {
+			flipped: 0,
+			"not-flipped": 0,
+			"generation-failed": 0,
+			"not-attempted": 0
+		},
+		byCorpus: {}
+	};
+	for (const row of rows) {
+		if (row.verification.reproduced) summary.reproduced++;
+		if (row.verification.reproduced && row.verification.signatureStrict) summary.signatureStrict++;
+		summary.fix[row.fix.outcome]++;
+		let corpus = summary.byCorpus[row.corpus];
+		if (corpus === void 0) {
+			corpus = {
+				rows: 0,
+				reproduced: 0,
+				fixFlipped: 0
+			};
+			summary.byCorpus[row.corpus] = corpus;
+		}
+		corpus.rows++;
+		if (row.verification.reproduced) corpus.reproduced++;
+		if (row.fix.outcome === "flipped") corpus.fixFlipped++;
+	}
+	return summary;
+}
+function verifiedFindingsToJsonl(rows) {
+	return rows.map((row) => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
+}
+function sha256(buffer) {
+	return createHash("sha256").update(buffer).digest("hex");
+}
+function readJson(path, what) {
+	let buffer;
+	try {
+		buffer = readFileSync(path);
+	} catch (error) {
+		throw new Error(`verified-findings: cannot read ${what} at ${path}: ${error.message}`);
+	}
+	try {
+		return {
+			value: JSON.parse(buffer.toString("utf8")),
+			sha256: sha256(buffer)
+		};
+	} catch (error) {
+		throw new Error(`verified-findings: ${what} at ${path} is not valid JSON: ${error.message}`);
+	}
+}
+function loadCaseVerdictDetail(runDir, batchCase) {
+	const parsed = readJson(join(runDir, `${batchCase.corpus}--${batchCase.trajId}`, "replay-verdict.json"), `per-case verdict for ${batchCase.trajId}`).value;
+	return {
+		k: parsed.k,
+		prefixExecuted: parsed.prefixExecuted,
+		recordedReturncode: parsed.recordedReturncode,
+		signatureBasis: parsed.signatureBasis ?? null,
+		prefixDivergences: parsed.prefixDivergences ?? [],
+		armACommand: parsed.armA?.command ?? null,
+		runIds: {
+			original: parsed.runIds?.original ?? null,
+			armA: parsed.runIds?.armA ?? null
+		}
+	};
+}
+/**
+* Load a replay-verify batch and join it into verified-finding rows.
+* Every case in the report must join: an unresolvable corpus, a missing
+* label entry, or a missing trajectory throws instead of dropping the row.
+*/
+function loadVerifiedFindingsDataset(source) {
+	const report = readJson(source.batchReportPath, "batch report");
+	const parsedReport = report.value;
+	if (!Array.isArray(parsedReport.cases) || parsedReport.cases.length === 0) throw new Error(`verified-findings: batch report at ${source.batchReportPath} has no cases`);
+	if (typeof parsedReport.generatedAt !== "string" || parsedReport.generatedAt.length === 0) throw new Error(`verified-findings: batch report at ${source.batchReportPath} has no generatedAt`);
+	const labelCache = /* @__PURE__ */ new Map();
+	const corporaProvenance = {};
+	const resolveCorpus = (corpus) => {
+		const config = source.corpora[corpus];
+		if (config === void 0) throw new Error(`verified-findings: batch report references corpus '${corpus}' but no labels/preparedDir was configured for it`);
+		let cached = labelCache.get(corpus);
+		if (cached === void 0) {
+			const labels = readJson(config.labelsPath, `labels for corpus '${corpus}'`);
+			const entries = labels.value;
+			if (!Array.isArray(entries)) throw new Error(`verified-findings: labels for corpus '${corpus}' at ${config.labelsPath} are not an array`);
+			const byTrajId = /* @__PURE__ */ new Map();
+			for (const entry of entries) {
+				if (byTrajId.has(entry.traj_id)) throw new Error(`verified-findings: labels for corpus '${corpus}' contain duplicate traj_id '${entry.traj_id}'`);
+				byTrajId.set(entry.traj_id, entry);
+			}
+			cached = {
+				sha256: labels.sha256,
+				byTrajId
+			};
+			labelCache.set(corpus, cached);
+			corporaProvenance[corpus] = {
+				labelsPath: config.labelsPath,
+				labelsSha256: labels.sha256,
+				preparedDir: config.preparedDir
+			};
+		}
+		return {
+			config,
+			...cached
+		};
+	};
+	const rows = [];
+	for (const batchCase of parsedReport.cases) {
+		const { config, sha256: labelsSha256, byTrajId } = resolveCorpus(batchCase.corpus);
+		const label = byTrajId.get(batchCase.trajId);
+		if (label === void 0) throw new Error(`verified-findings: ${source.runId}/${batchCase.corpus}/${batchCase.trajId}: no label entry in ${config.labelsPath}`);
+		const stepsPath = join(config.preparedDir, "normalized", batchCase.trajId, "steps.json");
+		const stepsFile = readJson(stepsPath, `trajectory steps for ${batchCase.trajId}`);
+		const steps = stepsFile.value;
+		if (!Array.isArray(steps)) throw new Error(`verified-findings: trajectory steps at ${stepsPath} are not an array`);
+		const detail = source.runDir === void 0 ? void 0 : loadCaseVerdictDetail(source.runDir, batchCase);
+		rows.push(buildVerifiedFindingRow({
+			batchCase,
+			label,
+			steps,
+			runId: source.runId,
+			batchGeneratedAt: parsedReport.generatedAt,
+			batchReportSha256: report.sha256,
+			labelsPath: config.labelsPath,
+			labelsSha256,
+			stepsPath,
+			stepsSha256: stepsFile.sha256,
+			detail,
+			maxObservationChars: source.maxObservationChars
+		}));
+	}
+	rows.sort((a, b) => a.caseId.localeCompare(b.caseId));
+	return {
+		rows,
+		summary: summarizeVerifiedFindings(rows),
+		provenance: {
+			runId: source.runId,
+			batchReportPath: source.batchReportPath,
+			batchReportSha256: report.sha256,
+			batchGeneratedAt: parsedReport.generatedAt,
+			corpora: corporaProvenance
+		}
+	};
+}
+//#endregion
+export { ABSENT_CATEGORY, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, DPO_CONTEXT_REQUIREMENT, FileSystemOutcomeStore, InMemoryOutcomeStore, PRM_CONTEXT_REQUIREMENT, PredictiveValidityResearcher, REPRESENTATIVE_MIN_FIDELITY, STEP_REWARD_CONTEXT_REQUIREMENT, VERIFIED_FINDING_SCHEMA, appendToCorpus, applyEloUpdate, assertPrmTrainableLine, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, buildVerifiedFindingRow, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, loadVerifiedFindingsDataset, observationsFromRunRecords, offPolicyEstimateAll, paretoFrontier, prmTrainingPairs, quantileEdges, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, stepRewardsToJsonl, summarizeVerifiedFindings, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, validateDatasetFormats, varianceBasedCurriculum, verificationReportToRunRecord, verifiedFindingsToJsonl };
 
 //# sourceMappingURL=rl.js.map