@tangle-network/agent-eval 0.126.7 → 0.128.0

package/dist/rl.js

@@ -10,25 +10,29 @@ import {
 } from "./chunk-3RF76KTD.js";
 import {
   runEvalCampaign
-} from "./chunk-P22LJ3Y2.js";
+} from "./chunk-TBL77AUT.js";
 import {
   detectRewardHacking,
   extractVerifiableReward,
   extractVerifiableRewardsFromRecords,
   filterDeterministicallyRewarded
-} from "./chunk-ARU2PZFM.js";
-import "./chunk-J5SQWP6Y.js";
+} from "./chunk-NYLOYM6N.js";
+import {
+  campaignCellToRunRecord
+} from "./chunk-2MKQIFS4.js";
+import "./chunk-PBE2LOSS.js";
 import {
   rubricPredictiveValidity
-} from "./chunk-X4UCIOTZ.js";
+} from "./chunk-S5YLIBFX.js";
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
-import "./chunk-DPZAEKA6.js";
+import "./chunk-VLOATJQ2.js";
+import "./chunk-DPUHNQLN.js";
 import {
   benjaminiHochberg,
   wilcoxonSignedRank
-} from "./chunk-PJQFMIOX.js";
+} from "./chunk-MHELPNRP.js";
 import {
   observationsFromRunRecords,
   thompsonCurriculum,
@@ -39,9 +43,10 @@ import "./chunk-VI2UW6B6.js";
 import "./chunk-TT4KNT67.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-VQMK5FMP.js";
-import "./chunk-IILEIWGW.js";
+import {
+  runTaskScore
+} from "./chunk-2JX3CFMB.js";
 import "./chunk-MA6HLL3S.js";
-import "./chunk-VSMTAMNK.js";
 import {
   ValidationError
 } from "./chunk-ONWEPEDO.js";
@@ -366,13 +371,19 @@ import { dirname } from "path";
 async function toDpoRows(triples, lookups) {
   const out = [];
   for (const t of triples) {
-    const [prompt, chosen, rejected] = await Promise.all([
+    const [chosenPrompt, rejectedPrompt, chosen, rejected] = await Promise.all([
       Promise.resolve(lookups.promptOf(t.chosenRunId)),
+      Promise.resolve(lookups.promptOf(t.rejectedRunId)),
       Promise.resolve(lookups.completionOf(t.chosenRunId)),
       Promise.resolve(lookups.completionOf(t.rejectedRunId))
     ]);
+    if (chosenPrompt !== rejectedPrompt) {
+      throw new Error(
+        `toDpoRows: preference "${t.chosenRunId}"/"${t.rejectedRunId}" resolves to different prompts`
+      );
+    }
     out.push({
-      prompt,
+      prompt: chosenPrompt,
       chosen,
       rejected,
       margin: t.marginScore,
@@ -394,29 +405,44 @@ function toDpoJsonl(rows) {
 }
 async function toGrpoRows(runs, lookups) {
   const rewardOf2 = lookups.rewardOf ?? defaultReward;
+  const promptHashByScenario = /* @__PURE__ */ new Map();
   const grouped = /* @__PURE__ */ new Map();
   for (const r of runs) {
-    const sid = r.scenarioId ?? r.experimentId;
-    const arr = grouped.get(sid) ?? [];
-    arr.push(r);
-    grouped.set(sid, arr);
+    const reward2 = rewardOf2(r);
+    if (!isTrainingRunEligible(r, reward2, lookups)) continue;
+    const existingPromptHash = promptHashByScenario.get(r.scenarioId);
+    if (existingPromptHash !== void 0 && existingPromptHash !== r.promptHash) {
+      throw new Error(
+        `toGrpoRows: scenario "${r.scenarioId}" contains mixed prompt identities "${existingPromptHash}" and "${r.promptHash}"`
+      );
+    }
+    promptHashByScenario.set(r.scenarioId, r.promptHash);
+    const key = `${r.scenarioId}\0${r.promptHash}`;
+    const group = grouped.get(key) ?? {
+      scenarioId: r.scenarioId,
+      promptHash: r.promptHash,
+      scored: []
+    };
+    group.scored.push({ run: r, reward: reward2 });
+    grouped.set(key, group);
   }
   const rows = [];
-  for (const [scenarioId, group] of grouped.entries()) {
-    if (group.length === 0) continue;
-    const prompt = await Promise.resolve(lookups.promptOf(group[0].runId));
-    const completions = [];
-    const rewards = [];
-    const runIds = [];
-    for (const r of group) {
-      const reward2 = rewardOf2(r);
-      if (reward2 === null) continue;
-      const completion = await Promise.resolve(lookups.completionOf(r.runId));
-      completions.push(completion);
-      rewards.push(reward2);
-      runIds.push(r.runId);
+  for (const { scenarioId, promptHash, scored } of grouped.values()) {
+    if (scored.length < 2) continue;
+    const prompts = await Promise.all(
+      scored.map(({ run }) => Promise.resolve(lookups.promptOf(run.runId)))
+    );
+    const prompt = prompts[0];
+    if (prompts.some((value) => value !== prompt)) {
+      throw new Error(
+        `toGrpoRows: prompt identity "${promptHash}" resolves to different text within scenario "${scenarioId}"`
+      );
     }
-    if (completions.length === 0) continue;
+    const completions = await Promise.all(
+      scored.map(({ run }) => Promise.resolve(lookups.completionOf(run.runId)))
+    );
+    const rewards = scored.map(({ reward: reward2 }) => reward2);
+    const runIds = scored.map(({ run }) => run.runId);
     rows.push({
       prompt,
       completions,
@@ -424,6 +450,7 @@ async function toGrpoRows(runs, lookups) {
       runIds,
       meta: {
         scenarioId,
+        promptHash,
         n: completions.length,
         meanReward: rewards.reduce((s, x) => s + x, 0) / rewards.length
       }
@@ -438,6 +465,8 @@ async function toSftRows(runs, lookups) {
   const include = lookups.include ?? (() => true);
   const rows = [];
   for (const r of runs) {
+    const score = runTaskScore(r);
+    if (!isTrainingRunEligible(r, score, lookups)) continue;
     if (!include(r)) continue;
     const system = lookups.systemOf?.(r);
     const [prompt, completion] = await Promise.all([
@@ -454,7 +483,7 @@ async function toSftRows(runs, lookups) {
         runId: r.runId,
         candidateId: r.candidateId,
         scenarioId: r.scenarioId,
-        score: r.outcome.holdoutScore ?? r.outcome.searchScore,
+        score,
         model: r.model
       }
     });
@@ -510,20 +539,65 @@ function stepRewardsToJsonl(stepRewards) {
   return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length > 0 ? "\n" : "");
 }
 function defaultReward(run) {
-  const v = run.outcome.holdoutScore ?? run.outcome.searchScore;
-  return typeof v === "number" && Number.isFinite(v) ? v : null;
+  return runTaskScore(run) ?? null;
+}
+function isTrainingRunEligible(run, quality, options = {}) {
+  const minimumQualityExclusive = options.minimumQualityExclusive ?? 0;
+  if (!Number.isFinite(minimumQualityExclusive)) {
+    throw new Error("minimumQualityExclusive must be finite");
+  }
+  if (quality === null || quality === void 0) return false;
+  if (!Number.isFinite(quality)) {
+    throw new Error(`training quality for run "${run.runId}" must be finite`);
+  }
+  if (quality <= minimumQualityExclusive) return false;
+  if (run.terminalOutcome !== "succeeded") return false;
+  if (run.failureClass !== void 0 || run.terminalFailureReason !== void 0) return false;
+  if (run.outcome.realness?.gated === true) return false;
+  if (run.splitTag === "search") return true;
+  return run.splitTag === "holdout" && options.allowHeldOutTrainingData === true;
 }
 
 // src/rl/dataset.ts
-function reward(r) {
-  const v = r.outcome.holdoutScore ?? r.outcome.searchScore;
-  return typeof v === "number" && Number.isFinite(v) ? v : null;
+var DATASET_FORMATS = ["grpo", "sft", "dpo"];
+var DATASET_FORMAT_SET = new Set(DATASET_FORMATS);
+function validateDatasetFormats(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("buildRlDataset: formats must contain at least one of: grpo, sft, dpo");
+  }
+  const formats = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const format of value) {
+    if (!DATASET_FORMAT_SET.has(format)) {
+      throw new Error(
+        `buildRlDataset: unsupported format ${JSON.stringify(format)}; expected exactly one of: grpo, sft, dpo`
+      );
+    }
+    const datasetFormat = format;
+    if (seen.has(datasetFormat)) {
+      throw new Error(
+        `buildRlDataset: duplicate format ${JSON.stringify(datasetFormat)}; each format may be requested once`
+      );
+    }
+    seen.add(datasetFormat);
+    formats.push(datasetFormat);
+  }
+  return formats;
+}
+function reward(r, rewardOf2) {
+  const value = rewardOf2 ? rewardOf2(r) : runTaskScore(r) ?? null;
+  if (value !== null && !Number.isFinite(value)) {
+    throw new Error(`buildRlDataset: reward for run "${r.runId}" must be finite`);
+  }
+  return value;
 }
 function distinct(xs) {
   return [...new Set(xs)].sort();
 }
 function computeRewardStats(values) {
-  if (values.length === 0) return { n: 0, mean: 0, median: 0, min: 0, max: 0, std: 0 };
+  if (values.length === 0) {
+    return { n: 0, mean: null, median: null, min: null, max: null, std: null };
+  }
   const sorted = [...values].sort((a, b) => a - b);
   const n = sorted.length;
   const mean = sorted.reduce((s, x) => s + x, 0) / n;
@@ -532,7 +606,7 @@ function computeRewardStats(values) {
   const variance = sorted.reduce((s, x) => s + (x - mean) ** 2, 0) / n;
   return { n, mean, median, min: sorted[0], max: sorted[n - 1], std: Math.sqrt(variance) };
 }
-function computeStats(records) {
+function computeStats(records, rewardOf2) {
   const splits = { search: 0, dev: 0, holdout: 0 };
   let inTok = 0;
   let outTok = 0;
@@ -542,12 +616,13 @@ function computeStats(records) {
     splits[r.splitTag] = (splits[r.splitTag] ?? 0) + 1;
     inTok += r.tokenUsage.input;
     outTok += r.tokenUsage.output;
-    cost += r.costUsd;
-    const rw = reward(r);
+    cost += r.costUsd ?? 0;
+    const rw = reward(r, rewardOf2);
     if (rw !== null) rewards.push(rw);
   }
   return {
     records: records.length,
+    scoredRecords: rewards.length,
     splits,
     reward: computeRewardStats(rewards),
     models: distinct(records.map((r) => r.model)),
@@ -561,16 +636,18 @@ async function buildRlDataset(records, lookups, config, preferences) {
   if (records.length === 0) {
     throw new Error("buildRlDataset: no records \u2014 refusing to package an empty dataset");
   }
-  const formats = config.formats ?? ["grpo", "sft"];
+  const formats = validateDatasetFormats(config.formats === void 0 ? ["sft"] : config.formats);
   const files = {};
   const rowCounts = {};
   if (formats.includes("grpo")) {
     const rows = await toGrpoRows(records, lookups);
+    requireRows("grpo", rows.length);
     files["train.grpo.jsonl"] = toGrpoJsonl(rows);
     rowCounts.grpo = rows.length;
   }
   if (formats.includes("sft")) {
     const rows = await toSftRows(records, lookups);
+    requireRows("sft", rows.length);
     files["train.sft.jsonl"] = toSftJsonl(rows);
     rowCounts.sft = rows.length;
   }
@@ -579,23 +656,35 @@ async function buildRlDataset(records, lookups, config, preferences) {
       throw new Error("buildRlDataset: format 'dpo' requires `preferences` (triples + lookups)");
     }
     const rows = await toDpoRows(preferences.triples, preferences.lookups);
+    requireRows("dpo", rows.length);
     files["train.dpo.jsonl"] = toDpoJsonl(rows);
     rowCounts.dpo = rows.length;
   }
+  if (!Object.keys(files).some((name) => name.startsWith("train.") && name.endsWith(".jsonl"))) {
+    throw new Error("buildRlDataset: no trainer file was emitted");
+  }
   const manifest = {
     ...config,
     formats,
     rowCounts,
-    stats: computeStats(records)
+    stats: computeStats(records, lookups.rewardOf)
   };
   files["manifest.json"] = `${JSON.stringify(manifest, null, 2)}
 `;
   files["DATASHEET.md"] = datasheetToMarkdown(manifest);
   return { manifest, files };
 }
+function requireRows(format, rows) {
+  if (rows === 0) {
+    throw new Error(`buildRlDataset: requested '${format}' format produced no trainable rows`);
+  }
+}
 function pct(x) {
   return `${(x * 100).toFixed(1)}%`;
 }
+function stat(value) {
+  return value === null ? "n/a" : value.toFixed(3);
+}
 function datasheetToMarkdown(m) {
   const s = m.stats;
   const total = s.records || 1;
@@ -604,31 +693,32 @@ function datasheetToMarkdown(m) {
   return [
     `# Dataset: ${m.name} \`v${m.version}\``,
     "",
-    `**Domain:** ${m.domain} \xB7 **Created:** ${m.createdAtIso} \xB7 **License:** ${m.license}`,
+    `**Domain:** ${m.domain} | **Created:** ${m.createdAtIso} | **License:** ${m.license}`,
     "",
     "## Reward provenance",
-    `- **Kind:** ${m.reward.kind}${deterministic ? " \u2705 (decidable \u2014 not judge-noise)" : ""}`,
+    `- **Kind:** ${m.reward.kind}${deterministic ? " (decidable, not judge noise)" : ""}`,
     `- **Source:** ${m.reward.source}`,
     `- **Description:** ${m.reward.description}`,
     "",
     "## Composition",
     `- **Records (trajectories):** ${s.records}`,
+    `- **Scored records:** ${s.scoredRecords}`,
     `- **Formats:** ${m.formats.map((f) => `${f} (${m.rowCounts[f] ?? 0} rows)`).join(", ")}`,
     "- **Splits:**",
     splitLines,
     "",
     "## Reward distribution",
-    `- n=${s.reward.n} \xB7 mean=${s.reward.mean.toFixed(3)} \xB7 median=${s.reward.median.toFixed(3)} \xB7 min=${s.reward.min.toFixed(3)} \xB7 max=${s.reward.max.toFixed(3)} \xB7 std=${s.reward.std.toFixed(3)}`,
+    `- n=${s.reward.n} | mean=${stat(s.reward.mean)} | median=${stat(s.reward.median)} | min=${stat(s.reward.min)} | max=${stat(s.reward.max)} | std=${stat(s.reward.std)}`,
     "",
     "## Provenance",
     `- **Models:** ${s.models.join(", ")}`,
     `- **Prompt/agent versions (sha256):** ${s.promptHashes.length} distinct`,
     `- **Commits:** ${s.commitShas.join(", ")}`,
-    `- **Tokens:** ${s.totalTokens.input} in / ${s.totalTokens.output} out \xB7 **Cost:** $${s.totalCostUsd.toFixed(2)}`,
+    `- **Tokens:** ${s.totalTokens.input} in / ${s.totalTokens.output} out | **Cost:** $${s.totalCostUsd.toFixed(2)}`,
     "",
     "## Quality gates",
     `- Contamination probe: ${m.qualityGates?.contaminationProbe ?? "not-run"}`,
-    `- Dedup: ${m.qualityGates?.dedup ? "yes" : "no"} \xB7 Verifiable-reward filter: ${m.qualityGates?.verifiableRewardFilter ? "yes" : "no"}`,
+    `- Dedup: ${m.qualityGates?.dedup ? "yes" : "no"} | Verifiable-reward filter: ${m.qualityGates?.verifiableRewardFilter ? "yes" : "no"}`,
     "",
     "## Recommended uses",
     m.intendedUse,
@@ -640,7 +730,7 @@ function datasheetToMarkdown(m) {
     m.limitations,
     "",
     "## Token rendering",
-    "For RL/SFT training, tokenize with the per-model renderer (DeepSeek-V3 / Kimi-K2 / Qwen3) to preserve token identity and per-token loss masks across tool-call turns \u2014 see `renderers` (PrimeIntellect). The `messages` / `completions` here are the renderer input.",
+    "For RL/SFT training, tokenize with the per-model renderer (DeepSeek-V3 / Kimi-K2 / Qwen3) to preserve token identity and per-token loss masks across tool-call turns. See `renderers` (PrimeIntellect). The `messages` / `completions` here are the renderer input.",
     ""
   ].join("\n");
 }
@@ -675,21 +765,27 @@ function readCorpus(corpusPath) {
   return out;
 }
 function rewardOf(r) {
-  const v = r.outcome.holdoutScore ?? r.outcome.searchScore;
-  return typeof v === "number" && Number.isFinite(v) ? v : 0;
+  return runTaskScore(r) ?? null;
 }
 async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
   let records = readCorpus(corpusPath).filter(
     (r) => typeof r.prompt === "string" && typeof r.completion === "string"
   );
   if (opts.splits) records = records.filter((r) => opts.splits.includes(r.splitTag));
-  if (opts.minScore != null) records = records.filter((r) => rewardOf(r) >= opts.minScore);
+  records = records.filter((r) => rewardOf(r) !== null);
+  if (opts.minScore != null) {
+    records = records.filter((r) => {
+      const reward2 = rewardOf(r);
+      return reward2 !== null && reward2 >= opts.minScore;
+    });
+  }
   const text = new Map(
     records.map((r) => [r.runId, { prompt: r.prompt, completion: r.completion }])
   );
   const lookups = {
     promptOf: (id) => text.get(id)?.prompt ?? "",
-    completionOf: (id) => text.get(id)?.completion ?? ""
+    completionOf: (id) => text.get(id)?.completion ?? "",
+    allowHeldOutTrainingData: opts.allowHeldOutTrainingData
   };
   return buildRlDataset(records, lookups, config);
 }
@@ -705,7 +801,7 @@ var PredictiveValidityResearcher = class {
     const threshold = this.opts.failureThreshold ?? 0.5;
     const failures = [];
     const failingRuns = runs.filter((r) => {
-      const score = r.outcome.holdoutScore ?? r.outcome.searchScore;
+      const score = runTaskScore(r);
       return typeof score === "number" && score < threshold;
     });
     if (failingRuns.length === 0) return failures;
@@ -717,8 +813,11 @@ var PredictiveValidityResearcher = class {
     }
     for (const [candidateId, group] of grouped.entries()) {
       const meanScore = group.reduce((s, r) => {
-        const x = r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
-        return s + x;
+        const score = runTaskScore(r);
+        if (score === void 0) {
+          throw new Error(`failing run ${r.runId} unexpectedly has no task score`);
+        }
+        return s + score;
       }, 0) / group.length;
       failures.push({
         code: `low-score-${candidateId}`,
@@ -788,15 +887,17 @@ var PredictiveValidityResearcher = class {
       baselineId: plan.baselineCandidateId,
       evidence: {
         productiveRuns: 0,
-        medianPairedDelta: 0,
-        pairedCI: { low: 0, high: 0 },
-        pairedPValue: 1,
-        searchScore: 0,
-        holdoutScore: 0,
-        overfitGap: 0,
-        baselineOverfitGap: 0,
-        medianCandidateCost: Number.NaN,
-        medianBaselineCost: Number.NaN
+        unpairedCandidateRuns: 0,
+        unpairedBaselineRuns: 0,
+        medianPairedDelta: null,
+        pairedCI: null,
+        pairedPValue: null,
+        searchScore: null,
+        holdoutScore: null,
+        overfitGap: null,
+        baselineOverfitGap: null,
+        medianCandidateCost: null,
+        medianBaselineCost: null
       },
       reason: "predictive-validity researcher does not execute plans; the caller is expected to run the sweep and call rubricPredictiveValidity directly with the resulting RunRecord[].",
       rejectionCode: "few_runs"
@@ -837,21 +938,27 @@ var PredictiveValidityResearcher = class {
 };
 
 // src/rl/preferences.ts
-var SPLIT_TAG_DEFAULT = "holdout";
 var DEFAULT_REWARD = (run) => {
-  const v = run.outcome.holdoutScore ?? run.outcome.searchScore;
-  return typeof v === "number" && Number.isFinite(v) ? v : null;
+  return runTaskScore(run) ?? null;
 };
 function extractPreferences(runs, opts = {}) {
   const strategy = opts.strategy ?? "paired-by-scenario-and-seed";
   const minMargin = opts.minMargin ?? 0.05;
-  const splitTag = opts.splitTag ?? SPLIT_TAG_DEFAULT;
+  const splitTag = opts.splitTag;
   const rewardOf2 = opts.rewardOf ?? DEFAULT_REWARD;
-  const filtered = runs.filter((r) => r.splitTag === splitTag);
+  if (splitTag === "holdout" && opts.allowHeldOutTrainingData !== true) {
+    throw new Error(
+      'extractPreferences: splitTag "holdout" requires allowHeldOutTrainingData: true'
+    );
+  }
+  if (splitTag === "dev") {
+    throw new Error('extractPreferences: splitTag "dev" is evaluation-only; train from "search"');
+  }
   const scoredEntries = [];
-  for (const run of filtered) {
+  for (const run of runs) {
+    if (splitTag !== void 0 && run.splitTag !== splitTag) continue;
     const s = rewardOf2(run);
-    if (s === null) continue;
+    if (!isTrainingRunEligible(run, s, opts)) continue;
     scoredEntries.push({ run, score: s });
   }
   const pairs = [];
@@ -946,14 +1053,6 @@ function extractPreferences(runs, opts = {}) {
   }
   return { pairs, cellsInspected, pairsBelowMargin, cellsSingleton, strategy };
 }
-function toTRLFormat(triples, promptOf) {
-  return triples.map((t) => ({
-    prompt: promptOf(t.meta.chosenPromptHash),
-    chosen: t.meta.chosenPromptHash,
-    // caller substitutes the model output via the runId map
-    rejected: t.meta.rejectedPromptHash
-  }));
-}
 function toAnthropicFormat(triples) {
   return triples.map((t) => ({
     scenarioId: t.scenarioId,
@@ -989,11 +1088,7 @@ function makePair(a, b, scenarioId, minMargin) {
   };
 }
 function scenarioOf(run) {
-  if (typeof run.scenarioId === "string" && run.scenarioId.length > 0) return run.scenarioId;
-  const fromRaw = run.outcome.raw.scenario_id;
-  if (typeof fromRaw === "number" && Number.isFinite(fromRaw)) return String(fromRaw);
-  if (typeof fromRaw === "string") return fromRaw;
-  return run.experimentId;
+  return run.scenarioId;
 }
 
 // src/rl/process-reward.ts
@@ -1123,16 +1218,17 @@ function prmTrainingPairs(stepRewardsByRun, opts = {}) {
 
 // src/rl/rl-campaign.ts
 async function runRLCampaign(opts) {
-  const campaign = await runEvalCampaign(opts);
+  const splitTag = opts.splitTag ?? "search";
+  const campaign = await runEvalCampaign({ ...opts, splitTag });
   const rewardSignals = extractVerifiableRewardsFromRecords(
     campaign.runs,
     opts.verifiableReward ?? {}
   );
   const preferences = extractPreferences(campaign.runs, {
+    ...opts.preferences,
     strategy: opts.preferences?.strategy ?? "paired-by-scenario-and-seed",
     minMargin: opts.preferences?.minMargin ?? 0.05,
-    splitTag: opts.preferences?.splitTag ?? opts.splitTag ?? "holdout",
-    rewardOf: opts.preferences?.rewardOf
+    splitTag: opts.preferences?.splitTag ?? splitTag
   });
   let interimConfidence = null;
   if (opts.report?.comparator) {
@@ -1192,17 +1288,17 @@ function collectPairedDeltaSeries(runs, comparator) {
   const baseline = /* @__PURE__ */ new Map();
   for (const r of runs) {
     if (r.candidateId !== comparator) continue;
-    const sid = r.scenarioId ?? r.experimentId;
-    const score = r.outcome.holdoutScore ?? r.outcome.searchScore;
-    if (typeof score !== "number" || !Number.isFinite(score)) continue;
+    const sid = r.scenarioId;
+    const score = runTaskScore(r);
+    if (score === void 0) continue;
     baseline.set(`${sid}::${r.seed}`, score);
   }
   const byCandidate = /* @__PURE__ */ new Map();
   for (const r of runs) {
     if (r.candidateId === comparator) continue;
-    const sid = r.scenarioId ?? r.experimentId;
-    const score = r.outcome.holdoutScore ?? r.outcome.searchScore;
-    if (typeof score !== "number" || !Number.isFinite(score)) continue;
+    const sid = r.scenarioId;
+    const score = runTaskScore(r);
+    if (score === void 0) continue;
     const baseScore = baseline.get(`${sid}::${r.seed}`);
     if (typeof baseScore !== "number") continue;
     const arr = byCandidate.get(r.candidateId) ?? [];
@@ -1238,62 +1334,68 @@ function buildSummary(args) {
 function campaignToRunRecords(campaign, ctx) {
   const splitTag = ctx.splitTag ?? "search";
   const candidateId = ctx.candidateId ?? campaign.manifestHash;
-  return campaign.cells.map((cell) => {
-    const composites = Object.values(cell.judgeScores).map((s) => s.composite);
-    const score = composites.length > 0 ? composites.reduce((a, b) => a + b, 0) / composites.length : 0;
-    const raw = { rep: cell.rep, duration_ms: cell.durationMs };
-    for (const judge of Object.values(cell.judgeScores)) {
-      for (const [dim, value] of Object.entries(judge.dimensions)) {
-        if (Number.isFinite(value)) raw[`dim.${dim}`] = value;
-      }
-    }
-    if (typeof cell.generation === "number") raw.generation = cell.generation;
-    const outcome = { raw };
-    if (splitTag === "holdout") outcome.holdoutScore = score;
-    else outcome.searchScore = score;
-    return {
+  return campaign.cells.map(
+    (cell) => campaignCellToRunRecord(cell, {
       runId: cell.cellId,
       experimentId: ctx.experimentId,
       candidateId,
-      seed: cell.seed,
       model: ctx.model,
       promptHash: ctx.promptHash,
       configHash: ctx.configHash,
       commitSha: ctx.commitSha,
-      wallMs: cell.durationMs,
-      costUsd: Number.isFinite(cell.costUsd) ? cell.costUsd : ctx.defaultCostUsd ?? 0,
-      tokenUsage: { input: 0, output: 0 },
-      outcome,
-      failureMode: cell.error ? "cell_error" : void 0,
       splitTag,
-      scenarioId: cell.scenarioId
-    };
-  });
+      defaultCostUsd: ctx.defaultCostUsd
+    })
+  );
 }
 function verificationReportToRunRecord(report, ctx, opts = {}) {
   const splitTag = ctx.splitTag ?? "search";
   const runId = opts.runId ?? `run-${ctx.candidateId}-${ctx.experimentId}-${report.startedAt}`;
+  const hasValidLayerMeasurement = report.layers.some(hasValidTaskMeasurement);
+  const taskScore = hasValidLayerMeasurement && isValidScore(report.taskScore) ? report.taskScore : void 0;
+  let executionErrorCount = 0;
+  let judgeErrorCount = 0;
+  let layerErrorCount = 0;
+  let layerTimeoutCount = 0;
+  let unscoredLayerCount = 0;
   const raw = {
     pass_count: report.passCount,
     fail_count: report.failCount,
     error_count: report.errorCount,
     skipped_count: report.skippedCount,
     duration_ms: report.durationMs,
-    blended_score: report.blendedScore
+    execution_error_count: 0
   };
   for (const layer of report.layers) {
-    if (typeof layer.score === "number") raw[`layer.${layer.layer}`] = layer.score;
+    if (hasValidTaskMeasurement(layer)) raw[`layer.${layer.layer}`] = layer.score;
+    else unscoredLayerCount++;
     raw[`layer_${layer.layer}_pass`] = layer.status === "pass" ? 1 : 0;
+    if (layer.status === "error" || layer.status === "timeout") {
+      if (layer.errorSource === "judge") judgeErrorCount++;
+      else executionErrorCount++;
+      if (layer.status === "error") layerErrorCount++;
+      else layerTimeoutCount++;
+    }
     if (layer.diagnostics) {
       for (const [k, v] of Object.entries(layer.diagnostics)) {
         if (typeof v === "number" && Number.isFinite(v)) raw[`layer.${layer.layer}.${k}`] = v;
       }
     }
   }
-  const firstFail = report.layers.find((l) => l.status === "fail" || l.status === "error");
+  raw.execution_error_count = executionErrorCount;
+  if (judgeErrorCount > 0) raw.judge_error_count = judgeErrorCount;
+  if (layerErrorCount > 0) raw.layer_error_count = layerErrorCount;
+  if (layerTimeoutCount > 0) raw.layer_timeout_count = layerTimeoutCount;
+  if (unscoredLayerCount > 0) raw.unscored_layer_count = unscoredLayerCount;
+  if (taskScore !== void 0) raw.blended_score = taskScore;
+  const firstScoredFailure = report.layers.find(
+    (layer) => layer.status === "fail" && hasValidTaskMeasurement(layer)
+  );
   const outcome = { raw };
-  if (splitTag === "holdout") outcome.holdoutScore = report.blendedScore;
-  else outcome.searchScore = report.blendedScore;
+  if (taskScore !== void 0) {
+    if (splitTag === "holdout") outcome.holdoutScore = taskScore;
+    else outcome.searchScore = taskScore;
+  }
   return {
     runId,
     experimentId: ctx.experimentId,
@@ -1304,19 +1406,24 @@ function verificationReportToRunRecord(report, ctx, opts = {}) {
     configHash: ctx.configHash,
     commitSha: ctx.commitSha,
     wallMs: report.durationMs,
-    costUsd: ctx.defaultCostUsd ?? 0,
+    costUsd: ctx.defaultCostUsd ?? null,
+    costProvenance: ctx.defaultCostUsd === void 0 ? { kind: "uncaptured", usd: null } : { kind: "estimated", usd: ctx.defaultCostUsd },
     tokenUsage: { input: 0, output: 0 },
+    terminalOutcome: "succeeded",
     outcome,
-    failureMode: firstFail ? failureModeFromLayer(firstFail) : void 0,
+    ...firstScoredFailure ? {
+      failureClass: "unknown",
+      failureMode: `layer_${firstScoredFailure.layer}_fail`
+    } : {},
     splitTag,
     scenarioId: ctx.scenarioId
   };
 }
-function failureModeFromLayer(layer) {
-  if (layer.status === "error") return `layer_${layer.layer}_error`;
-  if (layer.status === "fail") return `layer_${layer.layer}_fail`;
-  if (layer.status === "timeout") return `layer_${layer.layer}_timeout`;
-  return `layer_${layer.layer}_${layer.status}`;
+function hasValidTaskMeasurement(layer) {
+  return (layer.status === "pass" || layer.status === "fail") && isValidScore(layer.score);
+}
+function isValidScore(score) {
+  return typeof score === "number" && Number.isFinite(score) && score >= 0 && score <= 1;
 }
 
 // src/rl/sim-fidelity.ts
@@ -1689,6 +1796,7 @@ export {
   fitBradleyTerry,
   injectIrrelevantClause,
   inverseProbabilityWeighting,
+  isTrainingRunEligible,
   jsDivergence,
   observationsFromRunRecords,
   offPolicyEstimateAll,
@@ -1718,7 +1826,7 @@ export {
   toPrmRows,
   toSftJsonl,
   toSftRows,
-  toTRLFormat,
+  validateDatasetFormats,
   varianceBasedCurriculum,
   verificationReportToRunRecord
 };