@tangle-network/agent-eval 0.128.1 → 0.129.0

package/dist/rl.js

@@ -3,50 +3,66 @@ import {
   inverseProbabilityWeighting,
   offPolicyEstimateAll,
   selfNormalizedImportanceWeighting
-} from "./chunk-VGRCHJON.js";
+} from "./chunk-T6RLYGAD.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "./chunk-3RF76KTD.js";
 import {
   runEvalCampaign
-} from "./chunk-TBL77AUT.js";
+} from "./chunk-YQN4ICPP.js";
+import {
+  mintRolloutRows
+} from "./chunk-H23X7XKK.js";
+import {
+  isSplitEligible
+} from "./chunk-OWN5NPMC.js";
+import {
+  assertRewardGate
+} from "./chunk-PC5DOSM7.js";
+import "./chunk-RZTMDUO7.js";
 import {
   detectRewardHacking,
   extractVerifiableReward,
   extractVerifiableRewardsFromRecords,
   filterDeterministicallyRewarded
-} from "./chunk-NYLOYM6N.js";
+} from "./chunk-EG66UGL4.js";
 import {
   campaignCellToRunRecord
-} from "./chunk-2MKQIFS4.js";
-import "./chunk-PBE2LOSS.js";
+} from "./chunk-E7QXT7SX.js";
+import "./chunk-SFLLL76A.js";
 import {
   rubricPredictiveValidity
-} from "./chunk-S5YLIBFX.js";
+} from "./chunk-TQ7LNKZ3.js";
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
-import "./chunk-VLOATJQ2.js";
-import "./chunk-DPUHNQLN.js";
+import "./chunk-TJVT4QFF.js";
+import "./chunk-7FO3TNPI.js";
 import {
   benjaminiHochberg,
   wilcoxonSignedRank
-} from "./chunk-MHELPNRP.js";
+} from "./chunk-ZHTZ4EYI.js";
 import {
   observationsFromRunRecords,
   thompsonCurriculum,
   varianceBasedCurriculum
-} from "./chunk-VZSRQ272.js";
-import "./chunk-WS3NZZQQ.js";
+} from "./chunk-G7MGMCZD.js";
+import "./chunk-VCZ5FQYW.js";
 import "./chunk-VI2UW6B6.js";
-import "./chunk-TT4KNT67.js";
+import {
+  InMemoryTraceStore
+} from "./chunk-U4PHLT2N.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-VQMK5FMP.js";
 import {
   runTaskScore
-} from "./chunk-2JX3CFMB.js";
+} from "./chunk-56TAVBOK.js";
 import "./chunk-MA6HLL3S.js";
+import {
+  observedSplitScore,
+  trainingScore
+} from "./chunk-OIUOT4QD.js";
 import {
   ValidationError
 } from "./chunk-ONWEPEDO.js";
@@ -367,10 +383,119 @@ function escapeRegex(s) {
 import { appendFileSync, existsSync, mkdirSync, readFileSync } from "fs";
 import { dirname } from "path";
 
+// src/rl/rollout-input.ts
+function trainableLineReward(line) {
+  assertRewardGate(line, "trainable reward");
+  const { reward } = line.outcome;
+  if (reward === null || !Number.isFinite(reward)) return null;
+  return reward;
+}
+function isLineRealnessGated(line) {
+  return line.outcome.realness_gated === true;
+}
+function push(index, key, line) {
+  const existing = index.get(key);
+  if (existing === void 0) index.set(key, [line]);
+  else existing.push(line);
+}
+function invocationIndex(lines) {
+  const byRollout = /* @__PURE__ */ new Map();
+  const byRun = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    push(byRollout, line.rollout_id, line);
+    push(byRun, line.run_id, line);
+  }
+  return { byRollout, byRun };
+}
+function resolveInvocation(index, id) {
+  const rollouts = index.byRollout.get(id) ?? [];
+  const runs = index.byRun.get(id) ?? [];
+  if (rollouts.length > 1) return { kind: "ambiguous", count: rollouts.length };
+  const exact = rollouts[0];
+  if (exact !== void 0) {
+    if (runs.some((line) => line !== exact)) {
+      return { kind: "ambiguous", count: 1 + runs.filter((line) => line !== exact).length };
+    }
+    return { kind: "resolved", line: exact };
+  }
+  if (runs.length > 1) return { kind: "ambiguous", count: runs.length };
+  const only = runs[0];
+  return only === void 0 ? { kind: "missing" } : { kind: "resolved", line: only };
+}
+function auditInvocationAdmission(items, idsOf, context, requirement, inspect) {
+  if (context === void 0 || context === null) {
+    throw new Error(
+      `${requirement.exporter}: a ${requirement.contextType} is required \u2014 ${requirement.because} Pass \`{ lines: (await mintRolloutRows(...)).rows }\`.`
+    );
+  }
+  const index = invocationIndex(context.lines);
+  const audit = {
+    admitted: [],
+    gatedDrops: 0,
+    ambiguousDrops: 0,
+    ambiguous: []
+  };
+  for (const item of items) {
+    const lines = [];
+    let ambiguous = false;
+    for (const id of idsOf(item)) {
+      const resolution = resolveInvocation(index, id);
+      if (resolution.kind === "missing") {
+        throw new Error(
+          `${requirement.exporter}: no rollout line supplied for run ${id} \u2014 its realness gate and capture quality are unknown`
+        );
+      }
+      if (resolution.kind === "ambiguous") {
+        ambiguous = true;
+        if (!audit.ambiguous.some((entry) => entry.id === id)) {
+          audit.ambiguous.push({ id, invocations: resolution.count });
+        }
+        continue;
+      }
+      lines.push(resolution.line);
+    }
+    for (const line of lines) {
+      assertRewardGate(line, requirement.exporter);
+      inspect?.(line);
+    }
+    if (ambiguous) {
+      audit.ambiguousDrops++;
+      continue;
+    }
+    if (lines.some(isLineRealnessGated)) {
+      audit.gatedDrops++;
+      continue;
+    }
+    audit.admitted.push(item);
+  }
+  return audit;
+}
+function admitUngatedByInvocation(items, idsOf, context, requirement, inspect) {
+  const audit = auditInvocationAdmission(items, idsOf, context, requirement, inspect);
+  if (audit.ambiguousDrops > 0) {
+    const named = audit.ambiguous.map((e) => `${e.id} (${e.invocations} invocations)`).join(", ");
+    console.warn(
+      `[${requirement.exporter}] dropped ${audit.ambiguousDrops} item(s): ${named} name more than one invocation in the supplied lines, so the realness gate cannot be read for the invocation the artifact meant. Reference the \`rollout_id\` instead of the \`run_id\`, or supply a context holding one invocation per run.`
+    );
+  }
+  return audit.admitted;
+}
+
 // src/rl/exporters.ts
-async function toDpoRows(triples, lookups) {
+var DPO_CONTEXT_REQUIREMENT = {
+  exporter: "DPO export",
+  contextType: "DpoLineContext",
+  because: "a PreferenceTriple carries only run ids and a bare margin number, so without the minted rollout lines this exporter cannot see the realness gate and will write a run that faked its success onto the CHOSEN side of the pair \u2014 which is DPO trained to PREFER the gaming trajectory."
+};
+async function toDpoRows(triples, lookups, context) {
+  const admitted = admitUngatedByInvocation(
+    triples,
+    (t) => [t.chosenRunId, t.rejectedRunId],
+    context,
+    DPO_CONTEXT_REQUIREMENT
+  );
   const out = [];
-  for (const t of triples) {
+  for (const t of admitted) {
     const [chosenPrompt, rejectedPrompt, chosen, rejected] = await Promise.all([
       Promise.resolve(lookups.promptOf(t.chosenRunId)),
       Promise.resolve(lookups.promptOf(t.rejectedRunId)),
@@ -403,46 +528,41 @@ async function toDpoRows(triples, lookups) {
 function toDpoJsonl(rows) {
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
-async function toGrpoRows(runs, lookups) {
-  const rewardOf2 = lookups.rewardOf ?? defaultReward;
-  const promptHashByScenario = /* @__PURE__ */ new Map();
+async function toGrpoRows(lines, lookups) {
+  return grpoRowsFromLines(lines, lookups);
+}
+async function grpoRowsFromLines(lines, lookups) {
   const grouped = /* @__PURE__ */ new Map();
-  for (const r of runs) {
-    const reward2 = rewardOf2(r);
-    if (!isTrainingRunEligible(r, reward2, lookups)) continue;
-    const existingPromptHash = promptHashByScenario.get(r.scenarioId);
-    if (existingPromptHash !== void 0 && existingPromptHash !== r.promptHash) {
-      throw new Error(
-        `toGrpoRows: scenario "${r.scenarioId}" contains mixed prompt identities "${existingPromptHash}" and "${r.promptHash}"`
-      );
-    }
-    promptHashByScenario.set(r.scenarioId, r.promptHash);
-    const key = `${r.scenarioId}\0${r.promptHash}`;
-    const group = grouped.get(key) ?? {
-      scenarioId: r.scenarioId,
-      promptHash: r.promptHash,
-      scored: []
-    };
-    group.scored.push({ run: r, reward: reward2 });
-    grouped.set(key, group);
+  for (const line of lines) {
+    if (!isSelectedSplit(line, lookups)) continue;
+    const arr = grouped.get(line.task.instance_id) ?? [];
+    arr.push(line);
+    grouped.set(line.task.instance_id, arr);
   }
   const rows = [];
-  for (const { scenarioId, promptHash, scored } of grouped.values()) {
+  for (const [scenarioId, group] of grouped.entries()) {
+    if (group.length === 0) continue;
+    const scored = [];
+    for (const line of group) {
+      const reward = trainableLineReward(line);
+      if (reward === null) continue;
+      scored.push({ line, reward });
+    }
     if (scored.length < 2) continue;
     const prompts = await Promise.all(
-      scored.map(({ run }) => Promise.resolve(lookups.promptOf(run.runId)))
+      scored.map(({ line }) => Promise.resolve(lookups.promptOf(line.run_id)))
     );
     const prompt = prompts[0];
     if (prompts.some((value) => value !== prompt)) {
       throw new Error(
-        `toGrpoRows: prompt identity "${promptHash}" resolves to different text within scenario "${scenarioId}"`
+        `toGrpoRows: scenario "${scenarioId}" resolves to different prompt text within one group`
       );
     }
     const completions = await Promise.all(
-      scored.map(({ run }) => Promise.resolve(lookups.completionOf(run.runId)))
+      scored.map(({ line }) => Promise.resolve(lookups.completionOf(line.run_id)))
     );
-    const rewards = scored.map(({ reward: reward2 }) => reward2);
-    const runIds = scored.map(({ run }) => run.runId);
+    const rewards = scored.map(({ reward }) => reward);
+    const runIds = scored.map(({ line }) => line.run_id);
     rows.push({
       prompt,
       completions,
@@ -450,7 +570,6 @@ async function toGrpoRows(runs, lookups) {
       runIds,
       meta: {
         scenarioId,
-        promptHash,
         n: completions.length,
         meanReward: rewards.reduce((s, x) => s + x, 0) / rewards.length
       }
@@ -461,17 +580,30 @@ async function toGrpoRows(runs, lookups) {
 function toGrpoJsonl(rows) {
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
-async function toSftRows(runs, lookups) {
+async function toSftRows(lines, lookups) {
+  return sftRowsFromLines(lines, lookups);
+}
+async function sftRowsFromLines(lines, lookups) {
   const include = lookups.include ?? (() => true);
+  const minimumQualityExclusive = lookups.minimumQualityExclusive ?? 0;
+  if (!Number.isFinite(minimumQualityExclusive)) {
+    throw new Error("minimumQualityExclusive must be finite");
+  }
   const rows = [];
-  for (const r of runs) {
-    const score = runTaskScore(r);
-    if (!isTrainingRunEligible(r, score, lookups)) continue;
-    if (!include(r)) continue;
-    const system = lookups.systemOf?.(r);
+  for (const line of lines) {
+    assertRewardGate(line, "SFT export");
+    if (isLineRealnessGated(line)) continue;
+    if (!isSelectedSplit(line, lookups)) continue;
+    const score = trainableLineReward(line);
+    if (score === null || score <= minimumQualityExclusive) continue;
+    if (!line.outcome.is_completed || line.outcome.is_truncated || line.outcome.error !== null) {
+      continue;
+    }
+    if (!include(line)) continue;
+    const system = lookups.systemOf?.(line);
     const [prompt, completion] = await Promise.all([
-      Promise.resolve(lookups.promptOf(r.runId)),
-      Promise.resolve(lookups.completionOf(r.runId))
+      Promise.resolve(lookups.promptOf(line.run_id)),
+      Promise.resolve(lookups.completionOf(line.run_id))
     ]);
     const messages = [];
     if (system) messages.push({ role: "system", content: system });
@@ -480,11 +612,11 @@ async function toSftRows(runs, lookups) {
     rows.push({
       messages,
       meta: {
-        runId: r.runId,
-        candidateId: r.candidateId,
-        scenarioId: r.scenarioId,
+        runId: line.run_id,
+        candidateId: line.candidate_id ?? null,
+        scenarioId: line.task.instance_id,
         score,
-        model: r.model
+        model: line.policy.model
       }
     });
   }
@@ -493,9 +625,10 @@ async function toSftRows(runs, lookups) {
 function toSftJsonl(rows) {
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
-async function toPrmRows(triples, lookups) {
+async function toPrmRows(triples, lookups, context) {
+  const admitted = admitPrmTriples(triples, context);
   const rows = [];
-  for (const t of triples) {
+  for (const t of admitted) {
     const prompt = await Promise.resolve(lookups.promptOf(t.prefixRunId));
     const prefixSpanIds = lookups.prefixOf ? await Promise.resolve(lookups.prefixOf(t.prefixRunId, t.prefixStepIndex)) : [];
     const prefixStepText = [];
@@ -524,11 +657,59 @@ async function toPrmRows(triples, lookups) {
   }
   return rows;
 }
+function assertPrmTrainableLine(line, mintedWithMaxSteps) {
+  const id = line.rollout_id;
+  if (line.provenance.gap !== void 0) {
+    throw new Error(
+      `PRM export: rollout ${id} is a gap line (${line.provenance.gap}) \u2014 refusing to build a process-reward row from a trajectory that was never captured`
+    );
+  }
+  if (line.steps === void 0 || line.steps.length === 0) {
+    throw new Error(
+      `PRM export: rollout ${id} carries no steps \u2014 refusing to build a process-reward row with no trajectory`
+    );
+  }
+  if (line.outcome.is_truncated) {
+    throw new Error(
+      `PRM export: rollout ${id} is marked truncated \u2014 refusing to assign step-level credit over a partial trajectory`
+    );
+  }
+  if (mintedWithMaxSteps !== void 0 && line.steps.length >= mintedWithMaxSteps) {
+    throw new Error(
+      `PRM export: rollout ${id} has ${line.steps.length} steps at the mint cap of ${mintedWithMaxSteps} \u2014 its middle steps may have been dropped, and a capped trajectory carries no marker to prove otherwise`
+    );
+  }
+}
+var PRM_CONTEXT_REQUIREMENT = {
+  exporter: "PRM export",
+  contextType: "PrmLineContext",
+  because: "without the minted rollout lines this exporter cannot see the realness gate (a triple carries only a bare reward number) and cannot tell a fully-captured trajectory from a capped or empty one."
+};
+function admitPrmTriples(triples, context) {
+  return admitUngatedByInvocation(
+    triples,
+    (t) => [t.prefixRunId, t.rejectedRunId],
+    context,
+    PRM_CONTEXT_REQUIREMENT,
+    (line) => assertPrmTrainableLine(line, context.mintedWithMaxSteps)
+  );
+}
 function toPrmJsonl(rows) {
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
-function stepRewardsToJsonl(stepRewards) {
-  const rows = stepRewards.map((s) => ({
+var STEP_REWARD_CONTEXT_REQUIREMENT = {
+  exporter: "step-reward export",
+  contextType: "RolloutLineContext",
+  because: "a StepReward carries a runId and a bare per-step reward, and nothing that says whether that run faked its success \u2014 so without the minted rollout lines this exporter ships the step-level components of a gamed run at full value while the run-level scalar sits at 0 elsewhere."
+};
+function stepRewardsToJsonl(stepRewards, context) {
+  const admitted = admitUngatedByInvocation(
+    stepRewards,
+    (s) => [s.runId],
+    context,
+    STEP_REWARD_CONTEXT_REQUIREMENT
+  );
+  const rows = admitted.map((s) => ({
     runId: s.runId,
     spanId: s.spanId,
     stepIndex: s.stepIndex,
@@ -538,24 +719,9 @@ function stepRewardsToJsonl(stepRewards) {
   }));
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
-function defaultReward(run) {
-  return runTaskScore(run) ?? null;
-}
-function isTrainingRunEligible(run, quality, options = {}) {
-  const minimumQualityExclusive = options.minimumQualityExclusive ?? 0;
-  if (!Number.isFinite(minimumQualityExclusive)) {
-    throw new Error("minimumQualityExclusive must be finite");
-  }
-  if (quality === null || quality === void 0) return false;
-  if (!Number.isFinite(quality)) {
-    throw new Error(`training quality for run "${run.runId}" must be finite`);
-  }
-  if (quality <= minimumQualityExclusive) return false;
-  if (run.terminalOutcome !== "succeeded") return false;
-  if (run.failureClass !== void 0 || run.terminalFailureReason !== void 0) return false;
-  if (run.outcome.realness?.gated === true) return false;
-  if (run.splitTag === "search") return true;
-  return run.splitTag === "holdout" && options.allowHeldOutTrainingData === true;
+function isSelectedSplit(line, options) {
+  if (options.splitFilter !== void 0) return options.splitFilter.includes(line.task.split);
+  return isSplitEligible(line, options);
 }
 
 // src/rl/dataset.ts
@@ -584,15 +750,8 @@ function validateDatasetFormats(value) {
   }
   return formats;
 }
-function reward(r, rewardOf2) {
-  const value = rewardOf2 ? rewardOf2(r) : runTaskScore(r) ?? null;
-  if (value !== null && !Number.isFinite(value)) {
-    throw new Error(`buildRlDataset: reward for run "${r.runId}" must be finite`);
-  }
-  return value;
-}
 function distinct(xs) {
-  return [...new Set(xs)].sort();
+  return [...new Set(xs.filter((x) => typeof x === "string" && x.length > 0))].sort();
 }
 function computeRewardStats(values) {
   if (values.length === 0) {
@@ -606,47 +765,50 @@ function computeRewardStats(values) {
   const variance = sorted.reduce((s, x) => s + (x - mean) ** 2, 0) / n;
   return { n, mean, median, min: sorted[0], max: sorted[n - 1], std: Math.sqrt(variance) };
 }
-function computeStats(records, rewardOf2) {
-  const splits = { search: 0, dev: 0, holdout: 0 };
+function computeStatsFromLines(lines) {
+  const splits = { search: 0, dev: 0, holdout: 0, canary: 0 };
   let inTok = 0;
   let outTok = 0;
   let cost = 0;
+  let rolloutsWithoutCost = 0;
   const rewards = [];
-  for (const r of records) {
-    splits[r.splitTag] = (splits[r.splitTag] ?? 0) + 1;
-    inTok += r.tokenUsage.input;
-    outTok += r.tokenUsage.output;
-    cost += r.costUsd ?? 0;
-    const rw = reward(r, rewardOf2);
+  for (const line of lines) {
+    splits[line.task.split] += 1;
+    inTok += line.cost.tokens_in ?? 0;
+    outTok += line.cost.tokens_out ?? 0;
+    if (line.cost.usd === null) rolloutsWithoutCost++;
+    else cost += line.cost.usd;
+    const rw = trainableLineReward(line);
     if (rw !== null) rewards.push(rw);
   }
   return {
-    records: records.length,
+    records: lines.length,
     scoredRecords: rewards.length,
     splits,
     reward: computeRewardStats(rewards),
-    models: distinct(records.map((r) => r.model)),
-    promptHashes: distinct(records.map((r) => r.promptHash)),
-    commitShas: distinct(records.map((r) => r.commitSha)),
+    models: distinct(lines.map((l) => l.policy.model)),
+    promptHashes: distinct(lines.map((l) => l.policy.prompt_hash)),
+    commitShas: distinct(lines.map((l) => l.policy.profile_commit)),
     totalTokens: { input: inTok, output: outTok },
-    totalCostUsd: cost
+    totalCostUsd: cost,
+    rolloutsWithoutCost
   };
 }
-async function buildRlDataset(records, lookups, config, preferences) {
-  if (records.length === 0) {
-    throw new Error("buildRlDataset: no records \u2014 refusing to package an empty dataset");
+async function buildRlDataset(lines, lookups, config, preferences) {
+  if (lines.length === 0) {
+    throw new Error("buildRlDataset: no rollout lines \u2014 refusing to package an empty dataset");
   }
   const formats = validateDatasetFormats(config.formats === void 0 ? ["sft"] : config.formats);
   const files = {};
   const rowCounts = {};
   if (formats.includes("grpo")) {
-    const rows = await toGrpoRows(records, lookups);
+    const rows = await toGrpoRows(lines, lookups);
     requireRows("grpo", rows.length);
     files["train.grpo.jsonl"] = toGrpoJsonl(rows);
     rowCounts.grpo = rows.length;
   }
   if (formats.includes("sft")) {
-    const rows = await toSftRows(records, lookups);
+    const rows = await toSftRows(lines, lookups);
     requireRows("sft", rows.length);
     files["train.sft.jsonl"] = toSftJsonl(rows);
     rowCounts.sft = rows.length;
@@ -655,7 +817,7 @@ async function buildRlDataset(records, lookups, config, preferences) {
     if (!preferences) {
       throw new Error("buildRlDataset: format 'dpo' requires `preferences` (triples + lookups)");
     }
-    const rows = await toDpoRows(preferences.triples, preferences.lookups);
+    const rows = await toDpoRows(preferences.triples, preferences.lookups, { lines });
     requireRows("dpo", rows.length);
     files["train.dpo.jsonl"] = toDpoJsonl(rows);
     rowCounts.dpo = rows.length;
@@ -667,7 +829,7 @@ async function buildRlDataset(records, lookups, config, preferences) {
     ...config,
     formats,
     rowCounts,
-    stats: computeStats(records, lookups.rewardOf)
+    stats: computeStatsFromLines(lines)
   };
   files["manifest.json"] = `${JSON.stringify(manifest, null, 2)}
 `;
@@ -688,7 +850,8 @@ function stat(value) {
 function datasheetToMarkdown(m) {
   const s = m.stats;
   const total = s.records || 1;
-  const splitLines = ["search", "dev", "holdout"].map((k) => `  - \`${k}\`: ${s.splits[k]} (${pct(s.splits[k] / total)})`).join("\n");
+  const splitLines = ["search", "dev", "holdout", "canary"].map((k) => `  - \`${k}\`: ${s.splits[k]} (${pct(s.splits[k] / total)})`).join("\n");
+  const costNote = s.rolloutsWithoutCost > 0 ? ` (floor \u2014 ${s.rolloutsWithoutCost} rollout(s) never captured a cost)` : "";
   const deterministic = m.reward.kind === "deterministic";
   return [
     `# Dataset: ${m.name} \`v${m.version}\``,
@@ -714,7 +877,7 @@ function datasheetToMarkdown(m) {
     `- **Models:** ${s.models.join(", ")}`,
     `- **Prompt/agent versions (sha256):** ${s.promptHashes.length} distinct`,
     `- **Commits:** ${s.commitShas.join(", ")}`,
-    `- **Tokens:** ${s.totalTokens.input} in / ${s.totalTokens.output} out | **Cost:** $${s.totalCostUsd.toFixed(2)}`,
+    `- **Tokens:** ${s.totalTokens.input} in / ${s.totalTokens.output} out | **Cost:** $${s.totalCostUsd.toFixed(2)}${costNote}`,
     "",
     "## Quality gates",
     `- Contamination probe: ${m.qualityGates?.contaminationProbe ?? "not-run"}`,
@@ -765,7 +928,8 @@ function readCorpus(corpusPath) {
   return out;
 }
 function rewardOf(r) {
-  return runTaskScore(r) ?? null;
+  const v = trainingScore(r);
+  return typeof v === "number" && Number.isFinite(v) ? v : null;
 }
 async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
   let records = readCorpus(corpusPath).filter(
@@ -775,8 +939,8 @@ async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
   records = records.filter((r) => rewardOf(r) !== null);
   if (opts.minScore != null) {
     records = records.filter((r) => {
-      const reward2 = rewardOf(r);
-      return reward2 !== null && reward2 >= opts.minScore;
+      const reward = rewardOf(r);
+      return reward !== null && reward >= opts.minScore;
     });
   }
   const text = new Map(
@@ -787,7 +951,8 @@ async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
     completionOf: (id) => text.get(id)?.completion ?? "",
     allowHeldOutTrainingData: opts.allowHeldOutTrainingData
   };
-  return buildRlDataset(records, lookups, config);
+  const { rows } = await mintRolloutRows(records, new InMemoryTraceStore());
+  return buildRlDataset(rows, lookups, config);
 }
 
 // src/rl/predictive-validity-researcher.ts
@@ -897,7 +1062,8 @@ var PredictiveValidityResearcher = class {
         overfitGap: null,
         baselineOverfitGap: null,
         medianCandidateCost: null,
-        medianBaselineCost: null
+        medianBaselineCost: null,
+        realnessGatedRuns: 0
       },
       reason: "predictive-validity researcher does not execute plans; the caller is expected to run the sweep and call rubricPredictiveValidity directly with the resulting RunRecord[].",
       rejectionCode: "few_runs"
@@ -938,29 +1104,56 @@ var PredictiveValidityResearcher = class {
 };
 
 // src/rl/preferences.ts
-var DEFAULT_REWARD = (run) => {
-  return runTaskScore(run) ?? null;
-};
-function extractPreferences(runs, opts = {}) {
+var SPLIT_DEFAULT = "search";
+function extractPreferences(lines, opts = {}) {
   const strategy = opts.strategy ?? "paired-by-scenario-and-seed";
   const minMargin = opts.minMargin ?? 0.05;
-  const splitTag = opts.splitTag;
-  const rewardOf2 = opts.rewardOf ?? DEFAULT_REWARD;
-  if (splitTag === "holdout" && opts.allowHeldOutTrainingData !== true) {
+  const requestedSplit = opts.split;
+  if (requestedSplit === "holdout" && opts.allowHeldOutTrainingData !== true) {
+    throw new Error('extractPreferences: split "holdout" requires allowHeldOutTrainingData: true');
+  }
+  if (requestedSplit === "dev" || requestedSplit === "canary") {
     throw new Error(
-      'extractPreferences: splitTag "holdout" requires allowHeldOutTrainingData: true'
+      `extractPreferences: split "${requestedSplit}" is evaluation-only; train from "search"`
     );
   }
-  if (splitTag === "dev") {
-    throw new Error('extractPreferences: splitTag "dev" is evaluation-only; train from "search"');
-  }
-  const scoredEntries = [];
-  for (const run of runs) {
-    if (splitTag !== void 0 && run.splitTag !== splitTag) continue;
-    const s = rewardOf2(run);
-    if (!isTrainingRunEligible(run, s, opts)) continue;
-    scoredEntries.push({ run, score: s });
+  const candidates = candidatesFromLines(lines, opts);
+  const report = pairCandidates(candidates.rows, strategy, minMargin);
+  return { ...report, linesWithoutCandidateId: candidates.withoutCandidateId };
+}
+function candidatesFromLines(lines, opts) {
+  const split = opts.split ?? SPLIT_DEFAULT;
+  const rows = [];
+  let withoutCandidateId = 0;
+  for (const line of lines) {
+    if (line.task.split !== split) continue;
+    if (!line.outcome.is_completed || line.outcome.is_truncated || line.outcome.error !== null) {
+      continue;
+    }
+    const score = trainableLineReward(line);
+    if (score === null) continue;
+    const candidateId = line.candidate_id;
+    if (candidateId === null || candidateId === void 0 || candidateId.length === 0) {
+      withoutCandidateId++;
+      continue;
+    }
+    rows.push({
+      scenarioId: line.task.instance_id,
+      runId: line.run_id,
+      candidateId,
+      seed: line.task.seed,
+      score,
+      // `policy.*` is nullable on the wire; a minted line always carries these
+      // (RunRecord makes them mandatory). Empty string marks "not recorded" so
+      // `toTRLFormat`'s hash lookup fails visibly instead of silently matching.
+      promptHash: line.policy.prompt_hash ?? "",
+      configHash: line.policy.config_hash ?? "",
+      model: line.policy.model ?? ""
+    });
   }
+  return { rows, withoutCandidateId };
+}
+function pairCandidates(scoredEntries, strategy, minMargin) {
   const pairs = [];
   let pairsBelowMargin = 0;
   let cellsSingleton = 0;
@@ -968,13 +1161,12 @@ function extractPreferences(runs, opts = {}) {
   if (strategy === "paired-by-scenario-and-seed") {
     const groups = /* @__PURE__ */ new Map();
     for (const e of scoredEntries) {
-      const sid = scenarioOf(e.run);
-      const key = `${sid}::${e.run.seed}`;
+      const key = `${e.scenarioId}::${e.seed}`;
       const arr = groups.get(key) ?? [];
       arr.push(e);
       groups.set(key, arr);
     }
-    for (const [key, members] of groups.entries()) {
+    for (const members of groups.values()) {
       cellsInspected++;
       if (members.length < 2) {
         cellsSingleton++;
@@ -984,8 +1176,8 @@ function extractPreferences(runs, opts = {}) {
         for (let j = i + 1; j < members.length; j++) {
           const a = members[i];
           const b = members[j];
-          if (a.run.candidateId === b.run.candidateId) continue;
-          const result = makePair(a, b, key.split("::")[0], minMargin);
+          if (a.candidateId === b.candidateId) continue;
+          const result = makePair(a, b, a.scenarioId, minMargin);
           if (result.kind === "admit") pairs.push(result.pair);
           else pairsBelowMargin++;
         }
@@ -994,24 +1186,22 @@ function extractPreferences(runs, opts = {}) {
   } else if (strategy === "paired-by-scenario") {
     const byScenarioVariant = /* @__PURE__ */ new Map();
     for (const e of scoredEntries) {
-      const sid = scenarioOf(e.run);
-      let perScenario = byScenarioVariant.get(sid);
+      let perScenario = byScenarioVariant.get(e.scenarioId);
       if (!perScenario) {
         perScenario = /* @__PURE__ */ new Map();
-        byScenarioVariant.set(sid, perScenario);
+        byScenarioVariant.set(e.scenarioId, perScenario);
       }
-      const cur = perScenario.get(e.run.candidateId);
+      const cur = perScenario.get(e.candidateId);
       if (cur) {
         cur.sum += e.score;
         cur.n++;
-      } else perScenario.set(e.run.candidateId, { run: e.run, sum: e.score, n: 1 });
+      } else perScenario.set(e.candidateId, { entry: e, sum: e.score, n: 1 });
     }
     for (const [sid, perVariant] of byScenarioVariant.entries()) {
       cellsInspected++;
-      const arr = [...perVariant.entries()].map(([vid, agg]) => ({
-        run: agg.run,
-        score: agg.sum / agg.n,
-        variantId: vid
+      const arr = [...perVariant.values()].map((agg) => ({
+        ...agg.entry,
+        score: agg.sum / agg.n
       }));
       if (arr.length < 2) {
         cellsSingleton++;
@@ -1028,10 +1218,9 @@ function extractPreferences(runs, opts = {}) {
   } else {
     const byScenario = /* @__PURE__ */ new Map();
     for (const e of scoredEntries) {
-      const sid = scenarioOf(e.run);
-      const arr = byScenario.get(sid) ?? [];
+      const arr = byScenario.get(e.scenarioId) ?? [];
       arr.push(e);
-      byScenario.set(sid, arr);
+      byScenario.set(e.scenarioId, arr);
     }
     for (const [sid, arr] of byScenario.entries()) {
       cellsInspected++;
@@ -1042,7 +1231,7 @@ function extractPreferences(runs, opts = {}) {
       const sorted = [...arr].sort((a, b) => a.score - b.score);
       const top = sorted[sorted.length - 1];
       const bot = sorted[0];
-      if (top.run.candidateId === bot.run.candidateId) {
+      if (top.candidateId === bot.candidateId) {
         cellsSingleton++;
         continue;
       }
@@ -1053,8 +1242,51 @@ function extractPreferences(runs, opts = {}) {
   }
   return { pairs, cellsInspected, pairsBelowMargin, cellsSingleton, strategy };
 }
-function toAnthropicFormat(triples) {
-  return triples.map((t) => ({
+var PREFERENCE_RUN_IDS = (t) => [
+  t.chosenRunId,
+  t.rejectedRunId
+];
+var TRL_CONTEXT_REQUIREMENT = {
+  exporter: "TRL preference export",
+  contextType: "RolloutLineContext",
+  because: "a PreferenceTriple carries only run ids and hashes, so without the minted rollout lines this exporter cannot see the realness gate and will put a run that faked its success on the CHOSEN side of a DPO pair."
+};
+var ANTHROPIC_CONTEXT_REQUIREMENT = {
+  exporter: "Anthropic preference export",
+  contextType: "RolloutLineContext",
+  because: "a PreferenceTriple carries only run ids and a bare margin, so without the minted rollout lines this exporter cannot see the realness gate and will name a run that faked its success as the preferred one."
+};
+async function toTRLFormat(triples, lookups, context) {
+  const admitted = admitUngatedByInvocation(
+    triples,
+    PREFERENCE_RUN_IDS,
+    context,
+    TRL_CONTEXT_REQUIREMENT
+  );
+  const out = [];
+  for (const t of admitted) {
+    const [chosenPrompt, rejectedPrompt, chosen, rejected] = await Promise.all([
+      Promise.resolve(lookups.promptOf(t.chosenRunId)),
+      Promise.resolve(lookups.promptOf(t.rejectedRunId)),
+      Promise.resolve(lookups.completionOf(t.chosenRunId)),
+      Promise.resolve(lookups.completionOf(t.rejectedRunId))
+    ]);
+    if (chosenPrompt !== rejectedPrompt) {
+      throw new Error(
+        `toTRLFormat: preference "${t.chosenRunId}"/"${t.rejectedRunId}" resolves to different prompts`
+      );
+    }
+    out.push({ prompt: chosenPrompt, chosen, rejected });
+  }
+  return out;
+}
+function toAnthropicFormat(triples, context) {
+  return admitUngatedByInvocation(
+    triples,
+    PREFERENCE_RUN_IDS,
+    context,
+    ANTHROPIC_CONTEXT_REQUIREMENT
+  ).map((t) => ({
     scenarioId: t.scenarioId,
     chosenRunId: t.chosenRunId,
     rejectedRunId: t.rejectedRunId,
@@ -1065,31 +1297,29 @@ function makePair(a, b, scenarioId, minMargin) {
   const margin = Math.abs(a.score - b.score);
   if (margin < minMargin) return { kind: "reject" };
   const [chosen, rejected] = a.score > b.score ? [a, b] : [b, a];
+  const seed = chosen.seed !== null && chosen.seed === rejected.seed ? chosen.seed : void 0;
   return {
     kind: "admit",
     pair: {
       scenarioId,
-      chosenRunId: chosen.run.runId,
-      rejectedRunId: rejected.run.runId,
-      chosenVariantId: chosen.run.candidateId,
-      rejectedVariantId: rejected.run.candidateId,
+      chosenRunId: chosen.runId,
+      rejectedRunId: rejected.runId,
+      chosenVariantId: chosen.candidateId,
+      rejectedVariantId: rejected.candidateId,
       marginScore: chosen.score - rejected.score,
       scores: { chosen: chosen.score, rejected: rejected.score },
-      seed: chosen.run.seed === rejected.run.seed ? chosen.run.seed : void 0,
+      seed,
       meta: {
-        chosenPromptHash: chosen.run.promptHash,
-        rejectedPromptHash: rejected.run.promptHash,
-        chosenConfigHash: chosen.run.configHash,
-        rejectedConfigHash: rejected.run.configHash,
-        chosenModel: chosen.run.model,
-        rejectedModel: rejected.run.model
+        chosenPromptHash: chosen.promptHash,
+        rejectedPromptHash: rejected.promptHash,
+        chosenConfigHash: chosen.configHash,
+        rejectedConfigHash: rejected.configHash,
+        chosenModel: chosen.model,
+        rejectedModel: rejected.model
       }
     }
   };
 }
-function scenarioOf(run) {
-  return run.scenarioId;
-}
 
 // src/rl/process-reward.ts
 async function extractStepRewards(store, runId, opts) {
@@ -1224,11 +1454,13 @@ async function runRLCampaign(opts) {
     campaign.runs,
     opts.verifiableReward ?? {}
   );
-  const preferences = extractPreferences(campaign.runs, {
+  const scoredRuns = campaign.runs.filter((run) => runTaskScore(run) !== void 0);
+  const { rows: rolloutLines } = await mintRolloutRows(scoredRuns, new InMemoryTraceStore());
+  const preferences = extractPreferences(rolloutLines, {
     ...opts.preferences,
     strategy: opts.preferences?.strategy ?? "paired-by-scenario-and-seed",
     minMargin: opts.preferences?.minMargin ?? 0.05,
-    splitTag: opts.preferences?.splitTag ?? splitTag
+    split: opts.preferences?.split ?? splitTag
   });
   let interimConfidence = null;
   if (opts.report?.comparator) {
@@ -1257,13 +1489,15 @@ async function runRLCampaign(opts) {
   }
   const trainerRows = {};
   if (opts.trainerExport?.dpo) {
-    trainerRows.dpo = await toDpoRows(preferences.pairs, opts.trainerExport.dpo);
+    trainerRows.dpo = await toDpoRows(preferences.pairs, opts.trainerExport.dpo, {
+      lines: rolloutLines
+    });
   }
   if (opts.trainerExport?.grpo) {
-    trainerRows.grpo = await toGrpoRows(campaign.runs, opts.trainerExport.grpo);
+    trainerRows.grpo = await toGrpoRows(rolloutLines, opts.trainerExport.grpo);
   }
   if (opts.trainerExport?.sft) {
-    trainerRows.sft = await toSftRows(campaign.runs, opts.trainerExport.sft);
+    trainerRows.sft = await toSftRows(rolloutLines, opts.trainerExport.sft);
   }
   const summary = buildSummary({
     campaign,
@@ -1438,7 +1672,13 @@ var defaultBehaviorFeatures = (record) => {
   const turnsAborted = finiteOrNull(raw.turns_aborted);
   const completion = record.completion;
   return {
-    score: finiteOrNull(record.outcome?.holdoutScore) ?? finiteOrNull(record.outcome?.searchScore),
+    // RAW (`observedSplitScore`), deliberately: this feature vector is one
+    // half of a sim-vs-production divergence measurement. Gating a gamed run to
+    // 0 would move the simulated distribution toward production and report the
+    // simulator as MORE faithful precisely where it is being gamed. Each split
+    // is read separately rather than through `observedScore` so a non-finite
+    // holdout score falls back to search instead of poisoning the bucket.
+    score: finiteOrNull(observedSplitScore(record, "holdout")) ?? finiteOrNull(observedSplitScore(record, "search")),
     failure_class: record.failureClass ?? null,
     wall_ms: finiteOrNull(record.wallMs),
     output_tokens: finiteOrNull(record.tokenUsage?.output),
@@ -1620,7 +1860,7 @@ function easyModeCheck(simulated, production, opts = {}) {
   const passRate = (records, side) => {
     let passes = 0;
     for (const r of records) {
-      const score = finiteOrNull(r.outcome?.holdoutScore) ?? finiteOrNull(r.outcome?.searchScore);
+      const score = finiteOrNull(observedSplitScore(r, "holdout")) ?? finiteOrNull(observedSplitScore(r, "search"));
       if (score === null) {
         throw new ValidationError(
           `easyModeCheck: ${side} run "${r.runId}" carries neither holdoutScore nor searchScore`
@@ -1769,12 +2009,16 @@ export {
   ABSENT_CATEGORY,
   DEFAULT_MIN_N_PER_FEATURE,
   DEFAULT_QUANTILE_BUCKETS,
+  DPO_CONTEXT_REQUIREMENT,
   FileSystemOutcomeStore,
   InMemoryOutcomeStore,
+  PRM_CONTEXT_REQUIREMENT,
   PredictiveValidityResearcher,
   REPRESENTATIVE_MIN_FIDELITY,
+  STEP_REWARD_CONTEXT_REQUIREMENT,
   appendToCorpus,
   applyEloUpdate,
+  assertPrmTrainableLine,
   bestOfN,
   bucketLabel,
   buildDatasetFromCorpus,
@@ -1796,7 +2040,6 @@ export {
   fitBradleyTerry,
   injectIrrelevantClause,
   inverseProbabilityWeighting,
-  isTrainingRunEligible,
   jsDivergence,
   observationsFromRunRecords,
   offPolicyEstimateAll,
@@ -1826,6 +2069,7 @@ export {
   toPrmRows,
   toSftJsonl,
   toSftRows,
+  toTRLFormat,
   validateDatasetFormats,
   varianceBasedCurriculum,
   verificationReportToRunRecord