@tangle-network/agent-eval 0.126.7 → 0.128.0

package/dist/index.js

@@ -9,11 +9,11 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-K6IAZZ6L.js";
+} from "./chunk-NACAGYSY.js";
 import {
   mintRolloutRows,
   rolloutReward
-} from "./chunk-M7AH34KV.js";
+} from "./chunk-IHQDPH7D.js";
 import {
   SUPERVISOR_RUN_SCHEMA,
   analyzeSupervisorRun,
@@ -27,26 +27,25 @@ import {
   showMeasured,
   supervisorRunRolloutLines,
   writeSupervisorRunReport
-} from "./chunk-LKKT3IVV.js";
+} from "./chunk-TSN7JT6D.js";
 import "./chunk-VBQ3CRKH.js";
 import {
   toJsonl,
   toRewardRows,
   toSftRows
-} from "./chunk-IPYXE555.js";
+} from "./chunk-EJGRPCO3.js";
 import {
-  ROLLOUT_FORMAT,
   ROLLOUT_SCHEMA,
   assertRolloutLine,
   isRolloutLine,
   isTrainableSplit,
   validateRolloutLine
-} from "./chunk-MAX3TN3C.js";
+} from "./chunk-UWZZKKU7.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
-} from "./chunk-CGG5SLH3.js";
+} from "./chunk-XPRT64IE.js";
 import {
   DEFAULT_RULES,
   classifyFailure,
@@ -54,7 +53,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-QOTFXW5L.js";
+} from "./chunk-P5W7RQKK.js";
 import {
   buildTrajectory
 } from "./chunk-RZTMDUO7.js";
@@ -77,17 +76,15 @@ import {
   agentProfileHash,
   agentProfileId,
   agentProfileModelId,
-  comparePairedArms,
   completionVerdict,
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
   expandProfileAxes,
   extractProducedState,
   harnessAxisOf,
-  pairArms,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-E3IADPLG.js";
+} from "./chunk-UB2LOJ6Q.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -121,7 +118,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-BTACVO2W.js";
+} from "./chunk-NKAGIDE2.js";
 import {
   BackendIntegrityError,
   assertRealAgentReceipts,
@@ -133,7 +130,7 @@ import {
   inMemoryVerdictCache,
   summarizeAgentReceiptIntegrity,
   summarizeBackendIntegrity
-} from "./chunk-ZVCHKKOP.js";
+} from "./chunk-EZJEIH2R.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
@@ -146,7 +143,7 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-KE2VWPZX.js";
+} from "./chunk-ZUUWPZCV.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -163,7 +160,7 @@ import {
   makeFinding,
   renderPriorFindings,
   renderUpstreamFindings
-} from "./chunk-LUNF2SEL.js";
+} from "./chunk-DJKY2TSY.js";
 import "./chunk-HHWE3POT.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -192,18 +189,19 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-474LBSOX.js";
+} from "./chunk-BYT7ELPS.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-MOXWMGPC.js";
+} from "./chunk-XDWDC2MP.js";
 import {
   runEvalCampaign
-} from "./chunk-P22LJ3Y2.js";
-import "./chunk-ARU2PZFM.js";
+} from "./chunk-TBL77AUT.js";
+import "./chunk-NYLOYM6N.js";
+import "./chunk-2MKQIFS4.js";
 import {
   LlmCallError,
   LlmClient,
@@ -219,7 +217,7 @@ import {
   maximumChargeForLlmRequest,
   probeLlm,
   stripFencedJson
-} from "./chunk-J5SQWP6Y.js";
+} from "./chunk-PBE2LOSS.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -230,7 +228,12 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-DPZAEKA6.js";
+} from "./chunk-VLOATJQ2.js";
+import {
+  comparePairedArms,
+  pairArms,
+  pairRunRecords
+} from "./chunk-DPUHNQLN.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -253,6 +256,7 @@ import {
   mulberry32,
   normalizeScores,
   pairedBootstrap,
+  pairedCohensDz,
   pairedMde,
   pairedRiskDifference,
   pairedSignTest,
@@ -262,6 +266,7 @@ import {
   pearsonR,
   positionalBias,
   ranks,
+  requiredPairedSampleSize,
   requiredSampleSize,
   selfPreference,
   spearmanR,
@@ -270,7 +275,7 @@ import {
   weightedMean,
   wilcoxonSignedRank,
   wilson
-} from "./chunk-PJQFMIOX.js";
+} from "./chunk-MHELPNRP.js";
 import {
   CostAccountingIncompleteError,
   CostCallConflictError,
@@ -321,8 +326,8 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-OCFJACJU.js";
-import "./chunk-H5UD2323.js";
+} from "./chunk-EOSZT7PL.js";
+import "./chunk-7ZZMD7UK.js";
 import {
   extractUsage,
   extractUsageFromResponse,
@@ -343,7 +348,7 @@ import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   analyzeTraces
-} from "./chunk-NUKSVU3W.js";
+} from "./chunk-5DTSBUL2.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -360,15 +365,17 @@ import {
   applyToolSpanOtlpAttributes,
   asString,
   buildTraceAnalystTools,
+  classifyOtlpSpanRole,
   extractOtlpAttributes,
   firstStringAttr,
   inferOtlpKind,
+  isOtlpModelCall,
   projectOtlpFlatLine,
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup,
   traceSpanKindToOpenInferenceKind
-} from "./chunk-IR3KBHOY.js";
+} from "./chunk-P6FYH6K4.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -394,18 +401,23 @@ import {
   assertRunAgentProfileCell,
   buildAgentInterfaceProfileCell,
   buildAgentProfileCell,
+  canonicalize,
+  evaluateHypothesis,
   groupRunsByAgentProfileCell,
+  hashJson,
   isRunRecord,
   modelHasSnapshot,
   parseRunRecordSafe,
   requireAgentProfileCell,
-  resolveRunCostProvenance,
   roundTripRunRecord,
+  runTaskScore,
+  signManifest,
   toAgentProfileJson,
   validateAgentProfileCell,
   validateRunRecord,
-  verifyAgentProfileCell
-} from "./chunk-IILEIWGW.js";
+  verifyAgentProfileCell,
+  verifyManifest
+} from "./chunk-2JX3CFMB.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -415,13 +427,6 @@ import {
   isSandboxSpan,
   isToolSpan
 } from "./chunk-MA6HLL3S.js";
-import {
-  canonicalize,
-  evaluateHypothesis,
-  hashJson,
-  signManifest,
-  verifyManifest
-} from "./chunk-VSMTAMNK.js";
 import {
   AgentEvalError,
   CaptureIntegrityError,
@@ -918,8 +923,8 @@ async function executeScenario(tc, scenario, config) {
           console.log(`    ${judgeName} retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
           await sleep2(wait);
         }
-        const scores2 = await judge(tc, judgeInput);
-        judgeResults.push(scores2);
+        const scores = await judge(tc, judgeInput);
+        judgeResults.push(scores);
         await sleep2(3e3);
         lastError = void 0;
         break;
@@ -1536,19 +1541,19 @@ function projectSelectedRows(rows, options) {
     const arm = options.arm(original);
     assertNonEmptyString("arm", arm);
     if (arm !== options.baselineArm && arm !== options.treatmentArm) continue;
-    const pairKey2 = options.pairKey(original);
+    const pairKey = options.pairKey(original);
     const clusterKey = options.clusterKey(original);
     const pass = options.pass(original);
     const repKey = options.repKey?.(original);
-    assertNonEmptyString("pairKey", pairKey2);
+    assertNonEmptyString("pairKey", pairKey);
     assertNonEmptyString("clusterKey", clusterKey);
     if (typeof pass !== "boolean") {
       throw new ValidationError(
-        `clusteredPairedBinary: pass accessor must return boolean for pairKey '${pairKey2}'`
+        `clusteredPairedBinary: pass accessor must return boolean for pairKey '${pairKey}'`
       );
     }
     if (repKey !== void 0) assertNonEmptyString("repKey", repKey);
-    projected.push({ pairKey: pairKey2, clusterKey, arm, pass, repKey, original });
+    projected.push({ pairKey, clusterKey, arm, pass, repKey, original });
   }
   return projected;
 }
@@ -2941,9 +2946,9 @@ function scorePrReviewComments(auditCase, comments, source, weights = {}) {
     })
   };
 }
-function summarizePrReviewBenchmark(scores2) {
+function summarizePrReviewBenchmark(scores) {
   const bySource = /* @__PURE__ */ new Map();
-  for (const score of scores2) {
+  for (const score of scores) {
     bySource.set(score.source, [...bySource.get(score.source) ?? [], score]);
   }
   return [...bySource.entries()].map(([source, sourceScores]) => ({
@@ -3688,12 +3693,12 @@ function runJudgesTool(judges) {
       }
       const signal = ctx?.signal ?? new AbortController().signal;
       const scenario = a.scenario;
-      const scores2 = {};
+      const scores = {};
       for (const judge of selected) {
         if (scenario !== void 0 && judge.appliesTo && !judge.appliesTo(scenario)) continue;
-        scores2[judge.name] = await judge.score({ artifact: a.artifact, scenario, signal });
+        scores[judge.name] = await judge.score({ artifact: a.artifact, scenario, signal });
       }
-      return { scores: scores2 };
+      return { scores };
     }
   };
 }
@@ -3807,14 +3812,14 @@ async function runHarnessExperiment(config) {
   const score = config.score ?? ((trace) => critic.scoreTrace(trace));
   const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
     const trace = await config.adapter.run(request);
-    const runScore4 = await score(trace, request);
+    const runScore3 = await score(trace, request);
     const result = {
       variant: request.variant,
       scenario: request.scenario,
       trialIndex: request.trialIndex,
       trace,
-      score: runScore4,
-      aggregate: aggregateRunScore(runScore4, config.weights)
+      score: runScore3,
+      aggregate: aggregateRunScore(runScore3, config.weights)
     };
     await config.onResult?.(result);
     return result;
@@ -3884,20 +3889,20 @@ async function mapLimit(items, limit, fn) {
 function mean3(values) {
   return values.length ? values.reduce((sum4, value) => sum4 + value, 0) / values.length : 0;
 }
-function meanRunScore(scores2) {
+function meanRunScore(scores) {
   return {
-    success: mean3(scores2.map((s) => s.success)),
-    goalProgress: mean3(scores2.map((s) => s.goalProgress)),
-    repoGroundedness: mean3(scores2.map((s) => s.repoGroundedness)),
-    driftPenalty: mean3(scores2.map((s) => s.driftPenalty)),
-    toolUseQuality: mean3(scores2.map((s) => s.toolUseQuality)),
-    patchQuality: mean3(scores2.map((s) => s.patchQuality)),
-    testReality: mean3(scores2.map((s) => s.testReality)),
-    finalGate: mean3(scores2.map((s) => s.finalGate)),
-    reviewerBlockers: mean3(scores2.map((s) => s.reviewerBlockers)),
-    costUsd: mean3(scores2.map((s) => s.costUsd)),
-    wallSeconds: mean3(scores2.map((s) => s.wallSeconds)),
-    notes: scores2.flatMap((s) => s.notes ?? [])
+    success: mean3(scores.map((s) => s.success)),
+    goalProgress: mean3(scores.map((s) => s.goalProgress)),
+    repoGroundedness: mean3(scores.map((s) => s.repoGroundedness)),
+    driftPenalty: mean3(scores.map((s) => s.driftPenalty)),
+    toolUseQuality: mean3(scores.map((s) => s.toolUseQuality)),
+    patchQuality: mean3(scores.map((s) => s.patchQuality)),
+    testReality: mean3(scores.map((s) => s.testReality)),
+    finalGate: mean3(scores.map((s) => s.finalGate)),
+    reviewerBlockers: mean3(scores.map((s) => s.reviewerBlockers)),
+    costUsd: mean3(scores.map((s) => s.costUsd)),
+    wallSeconds: mean3(scores.map((s) => s.wallSeconds)),
+    notes: scores.flatMap((s) => s.notes ?? [])
   };
 }
 
@@ -4837,12 +4842,7 @@ function assertNonNegative(n, name) {
 
 // src/eval-trace-store.ts
 function runScore(record) {
-  const { holdoutScore, searchScore } = record.outcome;
-  if (typeof holdoutScore === "number") return holdoutScore;
-  if (typeof searchScore === "number") return searchScore;
-  throw new ValidationError(
-    `EvalTraceStore: run ${record.runId} has neither holdoutScore nor searchScore`
-  );
+  return runTaskScore(record);
 }
 function matches(record, f) {
   if (f.experimentId && record.experimentId !== f.experimentId) return false;
@@ -4850,8 +4850,12 @@ function matches(record, f) {
   if (f.scenarioId && record.scenarioId !== f.scenarioId) return false;
   if (f.model && record.model !== f.model) return false;
   if (f.splitTag && record.splitTag !== f.splitTag) return false;
-  if (f.minScore !== void 0 && runScore(record) < f.minScore) return false;
-  if (f.maxScore !== void 0 && runScore(record) > f.maxScore) return false;
+  if (f.minScore !== void 0 || f.maxScore !== void 0) {
+    const score = runScore(record);
+    if (score === void 0) return false;
+    if (f.minScore !== void 0 && score < f.minScore) return false;
+    if (f.maxScore !== void 0 && score > f.maxScore) return false;
+  }
   if (f.rawEquals && record.outcome.raw[f.rawEquals.key] !== f.rawEquals.value) return false;
   if (f.where && !f.where(record)) return false;
   return true;
@@ -4941,13 +4945,17 @@ var EvalTraceStore = class {
       candidateId: opts.candidateId,
       splitTag: opts.splitTag
     });
-    if (rows.length === 0) return null;
-    let best = rows[0];
-    let bestScore = runScore(best);
-    for (let i = 1; i < rows.length; i++) {
-      const s = runScore(rows[i]);
+    const scored = rows.flatMap((record) => {
+      const score = runScore(record);
+      return score === void 0 ? [] : [{ record, score }];
+    });
+    if (scored.length === 0) return null;
+    let best = scored[0].record;
+    let bestScore = scored[0].score;
+    for (let i = 1; i < scored.length; i++) {
+      const s = scored[i].score;
       if (s > bestScore) {
-        best = rows[i];
+        best = scored[i].record;
         bestScore = s;
       }
     }
@@ -4973,6 +4981,7 @@ var EvalTraceStore = class {
         const sid = r.scenarioId;
         if (!sid) continue;
         const s = runScore(r);
+        if (s === void 0) continue;
         const prev = m.get(sid);
         if (prev === void 0 || s > prev) m.set(sid, s);
       }
@@ -5050,9 +5059,9 @@ function median(sorted) {
   const mid = Math.floor(n / 2);
   return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function stddev(values, mean7) {
+function stddev(values, mean6) {
   if (values.length < 2) return 0;
-  const variance = values.reduce((acc, v) => acc + (v - mean7) ** 2, 0) / values.length;
+  const variance = values.reduce((acc, v) => acc + (v - mean6) ** 2, 0) / values.length;
   return Math.sqrt(variance);
 }
 function computeExperimentStats(reps, thresholds) {
@@ -5071,22 +5080,22 @@ function computeExperimentStats(reps, thresholds) {
       stable: false
     };
   }
-  const scores2 = reps.map((r) => {
+  const scores = reps.map((r) => {
     if (!Number.isFinite(r.score)) {
       throw new ValidationError(`experiment-tracker: rep ${r.rep} has non-finite score ${r.score}`);
     }
     return r.score;
   });
-  const sorted = [...scores2].sort((a, b) => a - b);
-  const mean7 = scores2.reduce((s, v) => s + v, 0) / n;
-  const sd = stddev(scores2, mean7);
-  const spread = iqr(scores2);
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean6 = scores.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores, mean6);
+  const spread = iqr(scores);
   const rated = reps.filter((r) => typeof r.passed === "boolean");
   const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
   const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
   return {
     median: median(sorted),
-    mean: mean7,
+    mean: mean6,
     min: sorted[0],
     max: sorted[n - 1],
     iqr: spread,
@@ -5275,6 +5284,7 @@ function leaderboard(records, opts) {
     const profile = rs[0].agentProfile;
     const harness = profile?.harness?.id;
     const model = profile?.model;
+    const knownCosts = rs.map((r) => r.costUsd).filter(isFiniteNumber);
     return {
       key,
       label: harness && model ? `${harness} \xB7 ${model}` : key,
@@ -5284,7 +5294,7 @@ function leaderboard(records, opts) {
       n: rs.length,
       passRate: ci.estimate,
       passRateCi95: [ci.lower, ci.upper],
-      meanCostUsd: mean4(rs.map((r) => r.costUsd)),
+      meanCostUsd: knownCosts.length === rs.length ? mean4(knownCosts) : null,
       meanTokensIn: mean4(rs.map((r) => r.tokenUsage.input)),
       meanTokensOut: mean4(rs.map((r) => r.tokenUsage.output)),
       meanWallMs: mean4(rs.map((r) => r.wallMs))
@@ -5298,6 +5308,9 @@ function leaderboard(records, opts) {
   });
   return rows;
 }
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
 
 // src/muffled-gate-scanner.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
@@ -5687,9 +5700,6 @@ function median2(xs) {
   const mid = Math.floor(sorted.length / 2);
   return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function runScore2(run) {
-  return run.outcome.holdoutScore ?? run.outcome.searchScore;
-}
 function aggregatePerDimension(runs) {
   const sums = /* @__PURE__ */ new Map();
   for (const run of runs) {
@@ -5722,14 +5732,14 @@ function recordRuns(runs, opts) {
   }
   const lines = [];
   for (const [scenarioId, scenarioRuns] of byScenario) {
-    const scored = scenarioRuns.map((run) => ({ run, score: runScore2(run) })).filter((s) => s.score !== void 0);
+    const scored = scenarioRuns.map((run) => ({ run, score: runTaskScore(run) })).filter((s) => s.score !== void 0);
     if (scored.length === 0) continue;
-    const scores2 = scored.map((s) => s.score);
+    const scores = scored.map((s) => s.score);
     const entry = {
       commitSha: opts.commitSha,
       timestamp,
-      scores: scores2,
-      composite: median2(scores2),
+      scores,
+      composite: median2(scores),
       runIds: scored.map((s) => s.run.runId)
     };
     const perDimension = aggregatePerDimension(scenarioRuns);
@@ -6317,20 +6327,20 @@ function checkRule(rule, ordered) {
 function evaluateTraceContract(contract, spans) {
   assertContract(contract);
   const ordered = orderSpans(spans);
-  const scores2 = {};
+  const scores = {};
   const violations = [];
   for (const rule of contract.rules) {
     const ruleViolations = checkRule(rule, ordered);
-    scores2[rule.label] = ruleViolations.length === 0 ? 1 : 0;
+    scores[rule.label] = ruleViolations.length === 0 ? 1 : 0;
     violations.push(...ruleViolations);
   }
   const ruleCount = contract.rules.length;
-  const passCount = Object.values(scores2).filter((s) => s === 1).length;
+  const passCount = Object.values(scores).filter((s) => s === 1).length;
   return {
     contract: contract.name,
     valid: passCount === ruleCount,
     score: passCount / ruleCount,
-    scores: scores2,
+    scores,
     violations,
     notes: `${passCount}/${ruleCount} rules passed`
   };
@@ -6658,12 +6668,12 @@ function renderMarkdownReport(reports) {
 async function aggregateRunMetrics(runs, store) {
   if (runs.length === 0) return {};
   const durations = [];
-  const scores2 = [];
+  const scores = [];
   const passes = [];
   const costs = [];
   for (const r of runs) {
     if (r.endedAt) durations.push(r.endedAt - r.startedAt);
-    if (r.outcome?.score !== void 0) scores2.push(r.outcome.score);
+    if (r.outcome?.score !== void 0) scores.push(r.outcome.score);
     passes.push(r.outcome?.pass === true ? 1 : 0);
     const llm = await llmSpans(store, r.runId);
     costs.push(aggregateLlm(llm).costUsd);
@@ -6672,7 +6682,7 @@ async function aggregateRunMetrics(runs, store) {
     provisionMs: average(durations),
     firstTokenMs: average(durations),
     wallMs: average(durations),
-    overallScore: average(scores2),
+    overallScore: average(scores),
     passRate: average(passes),
     costUsd: average(costs)
   };
@@ -6735,7 +6745,7 @@ async function toLangfuseEnvelope(store, runId) {
     },
     metadata: { finishReason: s.finishReason, cachedTokens: s.cachedTokens }
   }));
-  const scores2 = judges.map((j) => ({
+  const scores = judges.map((j) => ({
     id: j.spanId,
     traceId: run.runId,
     observationId: j.targetSpanId,
@@ -6743,7 +6753,7 @@ async function toLangfuseEnvelope(store, runId) {
     value: j.score,
     comment: j.rationale
   }));
-  return { traceId: run.runId, generations, scores: scores2 };
+  return { traceId: run.runId, generations, scores };
 }
 async function toPrometheusText(store) {
   const runs = await store.listRuns();
@@ -6844,12 +6854,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean7 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance = all.reduce((a, v) => a + (v - mean7) ** 2, 0) / all.length;
+  const mean6 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance = all.reduce((a, v) => a + (v - mean6) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance);
-  const ref = Math.abs(mean7) > 1e-9 ? Math.abs(mean7) : 1;
+  const ref = Math.abs(mean6) > 1e-9 ? Math.abs(mean6) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean7, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean6, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6906,18 +6916,18 @@ async function paraphraseRobustnessScenarios(args) {
     const deltas = {};
     const paraphrasedAll = [];
     for (const m of args.mutators) {
-      const scores2 = [];
+      const scores = [];
       for (let r = 0; r < reps; r++) {
         const mutatedTurns = scenario.userTurns.map((t) => m.mutator(t));
         const out = await args.runScenario({
           id: scenario.id,
           userTurns: mutatedTurns
         });
-        scores2.push(out.score);
+        scores.push(out.score);
       }
-      const mean7 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-      deltas[m.name] = mean7 - originalScore;
-      paraphrasedAll.push(...scores2);
+      const mean6 = scores.reduce((a, b) => a + b, 0) / scores.length;
+      deltas[m.name] = mean6 - originalScore;
+      paraphrasedAll.push(...scores);
     }
     const paraphrasedMean = paraphrasedAll.length === 0 ? originalScore : paraphrasedAll.reduce((a, b) => a + b, 0) / paraphrasedAll.length;
     perScenario.push({ id: scenario.id, originalScore, paraphrasedMean, deltas });
@@ -7332,10 +7342,10 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
   }
   for (const s of scenarios) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
-    const scores2 = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
-    if (scores2.length < 3) continue;
-    const mean7 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-    const variance = scores2.reduce((a, b) => a + (b - mean7) ** 2, 0) / scores2.length;
+    const scores = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
+    if (scores.length < 3) continue;
+    const mean6 = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const variance = scores.reduce((a, b) => a + (b - mean6) ** 2, 0) / scores.length;
     if (variance > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -7570,15 +7580,15 @@ async function runSelfPlay(proposer, scorer, targets, options = {}) {
     const rejected = [];
     const surviving = [];
     for (const candidate of proposed) {
-      const scores2 = await scorer.scoreCandidate(candidate, targets);
-      if (scores2.length < 2) {
+      const scores = await scorer.scoreCandidate(candidate, targets);
+      if (scores.length < 2) {
         rejected.push({ candidate, reason: "scorer returned <2 results" });
         continue;
       }
-      const values = scores2.map((s) => s.score);
+      const values = scores.map((s) => s.score);
       const spread = Math.max(...values) - Math.min(...values);
       const maxScore = Math.max(...values);
-      scored.push({ candidate, scores: scores2, spread });
+      scored.push({ candidate, scores, spread });
       if (maxScore < floor) {
         rejected.push({
           candidate,
@@ -8406,13 +8416,32 @@ var MultiLayerVerifier = class {
         ) : null;
         const layerStart = Date.now();
         let result;
+        let rejectOnAbort;
+        const abortPromise = new Promise((_resolve, reject) => {
+          const rejectWithReason = () => {
+            const reason = mergedSignal.reason;
+            reject(reason instanceof Error ? reason : new Error(String(reason ?? "aborted")));
+          };
+          if (mergedSignal.aborted) {
+            rejectWithReason();
+            return;
+          }
+          mergedSignal.addEventListener("abort", rejectWithReason, { once: true });
+          rejectOnAbort = () => mergedSignal.removeEventListener("abort", rejectWithReason);
+        });
         try {
-          result = await layer.run({ env: opts.env, prior: { ...byName }, signal: mergedSignal });
+          result = await Promise.race([
+            Promise.resolve(
+              layer.run({ env: opts.env, prior: { ...byName }, signal: mergedSignal })
+            ),
+            abortPromise
+          ]);
         } catch (err) {
           const aborted = mergedSignal.aborted;
           result = {
             layer: layer.name,
             status: aborted ? "timeout" : "error",
+            errorSource: layer.errorSource ?? "execution",
             durationMs: Date.now() - layerStart,
             findings: [
               {
@@ -8424,8 +8453,12 @@ var MultiLayerVerifier = class {
             reason: err instanceof Error ? err.message : String(err)
           };
         } finally {
+          rejectOnAbort?.();
           if (layerTimer) clearTimeout(layerTimer);
         }
+        if (result.status === "error" || result.status === "timeout") {
+          result.errorSource ??= layer.errorSource ?? "execution";
+        }
         result.findings = result.findings.map((f) => ({ ...f, layer: f.layer ?? layer.name }));
         results.push(result);
         byName[layer.name] = result;
@@ -8452,8 +8485,6 @@ function aggregate(layers, results, startedAt, startedAtMs) {
   let errorCount = 0;
   let scoredWeightSum = 0;
   let scoredWeightedTotal = 0;
-  let ranAnyScoredLayer = false;
-  let anyScoredLayerFailed = false;
   for (const r of results) {
     const weight = weightByName.get(r.layer) ?? 1;
     const failContrib = failContribByName.get(r.layer) ?? false;
@@ -8461,26 +8492,32 @@ function aggregate(layers, results, startedAt, startedAtMs) {
     else if (r.status === "fail") failCount++;
     else if (r.status === "skipped") skippedCount++;
     else errorCount++;
-    if (r.score != null && weight > 0) {
+    if (isValidTaskMeasurement(r) && weight > 0) {
       if (r.status === "pass") {
-        ranAnyScoredLayer = true;
         scoredWeightSum += weight;
         scoredWeightedTotal += weight * r.score;
       } else if (r.status === "fail") {
         if (failContrib) {
-          ranAnyScoredLayer = true;
           scoredWeightSum += weight;
           scoredWeightedTotal += weight * r.score;
         }
-        anyScoredLayerFailed = true;
       }
-    } else if (r.status === "fail") {
-      anyScoredLayerFailed = true;
     }
   }
   const finishedAtMs = Date.now();
-  const allPass = ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0;
   const blendedScore = scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0;
+  const resultByName = new Map(results.map((result) => [result.layer, result]));
+  const completeTaskMeasurement = scoredWeightSum > 0 && results.length === layers.length && layers.every((layer) => {
+    const result = resultByName.get(layer.name);
+    if (!result) return false;
+    if ((layer.weight ?? 1) <= 0) return true;
+    if (result.status === "pass") {
+      return result.score === void 0 || isValidTaskMeasurement(result);
+    }
+    return result.status === "fail" && layer.failContributesToScore === true && isValidTaskMeasurement(result);
+  });
+  const taskScore = completeTaskMeasurement && scoredWeightSum > 0 && Number.isFinite(blendedScore) ? blendedScore : void 0;
+  const allPass = taskScore !== void 0 && results.length === layers.length && failCount === 0 && skippedCount === 0 && errorCount === 0;
   return {
     layers: results,
     passCount,
@@ -8489,13 +8526,17 @@ function aggregate(layers, results, startedAt, startedAtMs) {
     errorCount,
     allPass,
     blendedScore,
+    ...taskScore === void 0 ? {} : { taskScore },
     valid: allPass,
-    score: blendedScore,
+    score: taskScore ?? 0,
     durationMs: finishedAtMs - startedAtMs,
     startedAt,
     finishedAt: new Date(finishedAtMs).toISOString()
   };
 }
+function isValidTaskMeasurement(result) {
+  return (result.status === "pass" || result.status === "fail") && typeof result.score === "number" && Number.isFinite(result.score) && result.score >= 0 && result.score <= 1;
+}
 function mergeSignals(a, b) {
   if (a.aborted) return a;
   if (b.aborted) return b;
@@ -8797,15 +8838,15 @@ function scoreReferenceReplay(scenarios, options = {}) {
   const threshold = options.matchThreshold ?? DEFAULT_MATCH_THRESHOLD;
   const matchStrategy = options.matchStrategy ?? "reference-order";
   const allowedSplits = new Set(options.splits ?? ALL_SPLITS);
-  const scores2 = scenarios.filter((scenario) => {
+  const scores = scenarios.filter((scenario) => {
     const split = scenario.split ?? "train";
     if (split === "holdout" && !options.includeHoldout) return false;
     return allowedSplits.has(split);
   }).map((scenario) => scoreScenario(scenario, matcher, threshold, matchStrategy));
   return {
-    scenarios: scores2,
-    aggregate: aggregateScenarioScores(scores2),
-    bySplit: aggregateBySplit(scores2)
+    scenarios: scores,
+    aggregate: aggregateScenarioScores(scores),
+    bySplit: aggregateBySplit(scores)
   };
 }
 function compareReferenceReplay(baseline, candidate) {
@@ -9028,20 +9069,20 @@ function buildScenarioScore(scenario, matches2, falsePositives) {
     matches: matches2
   };
 }
-function aggregateBySplit(scores2) {
+function aggregateBySplit(scores) {
   const out = {};
   for (const split of ALL_SPLITS) {
-    const scoped = scores2.filter((score) => score.split === split);
+    const scoped = scores.filter((score) => score.split === split);
     if (scoped.length > 0) out[split] = aggregateScenarioScores(scoped);
   }
   return out;
 }
-function aggregateScenarioScores(scores2) {
-  const matched = sum2(scores2.map((score) => score.matched));
-  const total = sum2(scores2.map((score) => score.total));
-  const falsePositives = sum2(scores2.map((score) => score.falsePositives));
-  const matchedWeight = sum2(scores2.map((score) => score.matchedWeight));
-  const totalWeight = sum2(scores2.map((score) => score.totalWeight));
+function aggregateScenarioScores(scores) {
+  const matched = sum2(scores.map((score) => score.matched));
+  const total = sum2(scores.map((score) => score.total));
+  const falsePositives = sum2(scores.map((score) => score.falsePositives));
+  const matchedWeight = sum2(scores.map((score) => score.matchedWeight));
+  const totalWeight = sum2(scores.map((score) => score.totalWeight));
   const precision2 = ratio(matched, matched + falsePositives);
   const recall = ratio(matched, total);
   return {
@@ -9305,18 +9346,16 @@ function createDefaultReviewer(options) {
 
 // src/description-length-gate.ts
 import { gzipSync } from "zlib";
-function runScore3(run) {
-  const o = run.outcome;
-  const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
-  return typeof s === "number" && Number.isFinite(s) ? s : void 0;
+function runScore2(run) {
+  return runTaskScore(run);
 }
 function taskKey(run) {
-  return run.scenarioId ?? run.experimentId;
+  return run.scenarioId;
 }
 function perTaskMeanScore(runs) {
   const acc = /* @__PURE__ */ new Map();
   for (const run of runs) {
-    const s = runScore3(run);
+    const s = runScore2(run);
     if (s === void 0) continue;
     const key = taskKey(run);
     const cur = acc.get(key) ?? { sum: 0, n: 0 };
@@ -9564,55 +9603,82 @@ var HeldOutGate = class {
     }
     this.costPerTaskCeiling = config.costPerTaskCeiling;
   }
-  /** Decide whether `candidate` should replace `baseline`. Pairing
-   *  is by (experimentId, seed) — identical experiment + seed pairs
-   *  the candidate run with the matching baseline run. Pairs without
-   *  a holdout score on both sides are dropped. */
+  /** Decide whether `candidate` should replace `baseline`.
+   *  Pairing is by `(experimentId, scenarioId, seed)`.
+   *  Missing or duplicate identities throw instead of comparing by position. */
   evaluate(candidate, baseline) {
     const candidateId = inferCandidateId2(candidate, this.baselineKey);
     const baselineId = this.baselineKey;
-    const baselineHoldoutByKey = indexHoldoutByKey(baseline);
-    const beforeHoldout = [];
-    const afterHoldout = [];
-    for (const run of candidate) {
-      if (run.splitTag !== "holdout") continue;
-      if (run.outcome.holdoutScore === void 0) continue;
-      const key = pairKey(run);
-      const counterpart = baselineHoldoutByKey.get(key);
-      if (counterpart === void 0) continue;
-      beforeHoldout.push(counterpart);
-      afterHoldout.push(run.outcome.holdoutScore);
-    }
+    assertScenarioIdentities([...candidate, ...baseline]);
+    const candidateSearch = scoredRuns(candidate, "searchScore", "search");
+    const baselineSearch = scoredRuns(baseline, "searchScore", "search");
+    const candidateHoldout = scoredRuns(candidate, "holdoutScore", "holdout");
+    const baselineHoldout = scoredRuns(baseline, "holdoutScore", "holdout");
+    const searchPairing = pairRunRecords(baselineSearch, candidateSearch);
+    const holdoutPairing = pairRunRecords(baselineHoldout, candidateHoldout);
+    const beforeSearch = searchPairing.pairs.map((pair) => pair.baseline.outcome.searchScore);
+    const afterSearch = searchPairing.pairs.map((pair) => pair.treatment.outcome.searchScore);
+    const beforeHoldout = holdoutPairing.pairs.map((pair) => pair.baseline.outcome.holdoutScore);
+    const afterHoldout = holdoutPairing.pairs.map((pair) => pair.treatment.outcome.holdoutScore);
     const productiveRuns = beforeHoldout.length;
-    const candidateSearchMean = mean6(scores(candidate, "searchScore", "search"));
-    const candidateHoldoutMean = mean6(scores(candidate, "holdoutScore", "holdout"));
-    const baselineSearchMean = mean6(scores(baseline, "searchScore", "search"));
-    const baselineHoldoutMean = mean6(scores(baseline, "holdoutScore", "holdout"));
-    const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean);
-    const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean);
-    const medianCandidateCost = medianFinite(candidate.map((r) => r.costUsd));
-    const medianBaselineCost = medianFinite(baseline.map((r) => r.costUsd));
+    const candidateSearchMean = meanOrNull(afterSearch);
+    const candidateHoldoutMean = meanOrNull(afterHoldout);
+    const baselineSearchMean = meanOrNull(beforeSearch);
+    const baselineHoldoutMean = meanOrNull(beforeHoldout);
+    const overfitGap = diffOrNull(candidateSearchMean, candidateHoldoutMean);
+    const baselineOverfitGap = diffOrNull(baselineSearchMean, baselineHoldoutMean);
+    const medianCandidateCost = completeCostMedian(candidate);
+    const medianBaselineCost = completeCostMedian(baseline);
+    const commonEvidence = {
+      productiveRuns,
+      unpairedCandidateRuns: holdoutPairing.unpairedTreatment.length,
+      unpairedBaselineRuns: holdoutPairing.unpairedBaseline.length,
+      searchScore: candidateSearchMean,
+      holdoutScore: candidateHoldoutMean,
+      overfitGap,
+      baselineOverfitGap,
+      medianCandidateCost,
+      medianBaselineCost
+    };
+    const missingSplitScores = [
+      candidateSearch.length === 0 ? "candidate search" : null,
+      candidateHoldout.length === 0 ? "candidate holdout" : null,
+      baselineSearch.length === 0 ? "baseline search" : null,
+      baselineHoldout.length === 0 ? "baseline holdout" : null
+    ].filter((label) => label !== null);
+    if (missingSplitScores.length > 0) {
+      return {
+        promote: false,
+        candidateId,
+        baselineId,
+        evidence: {
+          ...commonEvidence,
+          medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : null,
+          pairedCI: null,
+          pairedPValue: null
+        },
+        reason: `missing_split_scores: ${missingSplitScores.join(", ")} score evidence is absent`,
+        rejectionCode: "missing_split_scores"
+      };
+    }
     if (productiveRuns < this.minProductiveRuns) {
       return {
         promote: false,
         candidateId,
         baselineId,
         evidence: {
-          productiveRuns,
-          medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : 0,
-          pairedCI: { low: 0, high: 0 },
-          pairedPValue: 1,
-          searchScore: candidateSearchMean,
-          holdoutScore: candidateHoldoutMean,
-          overfitGap,
-          baselineOverfitGap,
-          medianCandidateCost,
-          medianBaselineCost
+          ...commonEvidence,
+          medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : null,
+          pairedCI: null,
+          pairedPValue: null
         },
         reason: `few_runs: ${productiveRuns} paired holdout observation(s) < min ${this.minProductiveRuns}`,
         rejectionCode: "few_runs"
       };
     }
+    if (overfitGap === null || baselineOverfitGap === null) {
+      throw new Error("HeldOutGate: complete split scores did not produce overfit gaps");
+    }
     const ci = pairedBootstrap(beforeHoldout, afterHoldout, {
       confidence: this.confidence,
       resamples: this.resamples,
@@ -9621,16 +9687,10 @@ var HeldOutGate = class {
     });
     const wilcoxon = wilcoxonSignedRank(beforeHoldout, afterHoldout);
     const evidence = {
-      productiveRuns,
+      ...commonEvidence,
       medianPairedDelta: ci.median,
       pairedCI: { low: ci.low, high: ci.high },
-      pairedPValue: wilcoxon.p,
-      searchScore: candidateSearchMean,
-      holdoutScore: candidateHoldoutMean,
-      overfitGap,
-      baselineOverfitGap,
-      medianCandidateCost,
-      medianBaselineCost
+      pairedPValue: wilcoxon.p
     };
     if (!(ci.low > this.pairedDeltaThreshold)) {
       return {
@@ -9642,7 +9702,7 @@ var HeldOutGate = class {
         rejectionCode: "negative_delta"
       };
     }
-    if (Number.isFinite(overfitGap) && Number.isFinite(baselineOverfitGap) && overfitGap > baselineOverfitGap + this.overfitGapThreshold) {
+    if (overfitGap > baselineOverfitGap + this.overfitGapThreshold) {
       return {
         promote: false,
         candidateId,
@@ -9652,7 +9712,17 @@ var HeldOutGate = class {
         rejectionCode: "overfit_gap"
       };
     }
-    if (this.costPerTaskCeiling !== void 0 && Number.isFinite(medianCandidateCost) && medianCandidateCost > this.costPerTaskCeiling) {
+    if (this.costPerTaskCeiling !== void 0 && medianCandidateCost === null) {
+      return {
+        promote: false,
+        candidateId,
+        baselineId,
+        evidence,
+        reason: "missing_cost: candidate cost evidence is incomplete",
+        rejectionCode: "missing_cost"
+      };
+    }
+    if (this.costPerTaskCeiling !== void 0 && medianCandidateCost !== null && medianCandidateCost > this.costPerTaskCeiling) {
       return {
         promote: false,
         candidateId,
@@ -9678,49 +9748,50 @@ function inferCandidateId2(candidate, baselineKey) {
   }
   return candidate[0]?.candidateId ?? "(unknown candidate)";
 }
-function indexHoldoutByKey(runs) {
-  const out = /* @__PURE__ */ new Map();
-  for (const r of runs) {
-    if (r.splitTag !== "holdout") continue;
-    if (r.outcome.holdoutScore === void 0) continue;
-    out.set(pairKey(r), r.outcome.holdoutScore);
+function assertScenarioIdentities(runs) {
+  for (const run of runs) {
+    if (typeof run.scenarioId !== "string" || run.scenarioId.trim() === "") {
+      throw new Error(`HeldOutGate: run ${run.runId} is missing scenarioId`);
+    }
   }
-  return out;
-}
-function pairKey(r) {
-  return `${r.experimentId}::${r.seed}`;
 }
-function scores(runs, field, splitFilter) {
-  const out = [];
-  for (const r of runs) {
-    if (r.splitTag !== splitFilter) continue;
-    const v = r.outcome[field];
-    if (typeof v === "number" && Number.isFinite(v)) out.push(v);
-  }
-  return out;
+function scoredRuns(runs, field, splitFilter) {
+  return runs.filter(
+    (run) => run.splitTag === splitFilter && typeof run.outcome[field] === "number" && Number.isFinite(run.outcome[field])
+  );
 }
-function mean6(xs) {
-  if (xs.length === 0) return Number.NaN;
+function meanOrNull(xs) {
+  if (xs.length === 0) return null;
   return xs.reduce((s, x) => s + x, 0) / xs.length;
 }
-function safeDiff(a, b) {
-  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN;
+function diffOrNull(a, b) {
+  if (a === null || b === null) return null;
   return a - b;
 }
 function medianDelta(before, after) {
   const ds = before.map((b, i) => after[i] - b).sort((x, y) => x - y);
-  if (ds.length === 0) return 0;
+  if (ds.length === 0) throw new Error("HeldOutGate: median delta requires at least one pair");
   const mid = Math.floor(ds.length / 2);
   return ds.length % 2 === 0 ? (ds[mid - 1] + ds[mid]) / 2 : ds[mid];
 }
 function medianFinite(xs) {
   const ys = xs.filter((x) => Number.isFinite(x)).sort((x, y) => x - y);
-  if (ys.length === 0) return Number.NaN;
+  if (ys.length === 0) return null;
   const mid = Math.floor(ys.length / 2);
   return ys.length % 2 === 0 ? (ys[mid - 1] + ys[mid]) / 2 : ys[mid];
 }
+function completeCostMedian(runs) {
+  if (runs.length === 0) return null;
+  const costs = [];
+  for (const run of runs) {
+    const provenance = run.costProvenance;
+    if (provenance.kind === "uncaptured") return null;
+    costs.push(provenance.usd);
+  }
+  return medianFinite(costs);
+}
 function fmt(x) {
-  if (!Number.isFinite(x)) return String(x);
+  if (x === null) return "n/a";
   return x.toFixed(4);
 }
 
@@ -9739,9 +9810,9 @@ function passOrthogonality(input) {
       sims.push(cosineSimilarity(vectors[i], vectors[j]));
     }
   }
-  const mean7 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
+  const mean6 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
   return {
-    orthogonality: Math.max(0, Math.min(1, 1 - mean7)),
+    orthogonality: Math.max(0, Math.min(1, 1 - mean6)),
     passCount: passes.length,
     similarities: sims
   };
@@ -10255,17 +10326,17 @@ function traceJudge(judge, judgeName, opts) {
       }
     });
     try {
-      const scores2 = await judge(tc, input);
-      const composite = scores2.length > 0 ? scores2.reduce((sum4, s) => sum4 + s.score, 0) / scores2.length : 0;
+      const scores = await judge(tc, input);
+      const composite = scores.length > 0 ? scores.reduce((sum4, s) => sum4 + s.score, 0) / scores.length : 0;
       await span.end({
         attributes: {
           "judge.name": judgeName,
           "judge.composite_score": composite,
-          "judge.dimension_count": scores2.length,
+          "judge.dimension_count": scores.length,
           "eval.phase": "judge"
         }
       });
-      return scores2;
+      return scores;
     } catch (err) {
       await span.fail(err instanceof Error ? err : String(err));
       throw err;
@@ -10294,8 +10365,8 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           parentSpanId: ensembleSpan.span.spanId
         });
         try {
-          const scores2 = await tracedFn(tc, input);
-          allScores.push(...scores2);
+          const scores = await tracedFn(tc, input);
+          allScores.push(...scores);
         } catch (err) {
           if (!(err instanceof JudgeParseError)) throw err;
           failedJudges++;
@@ -10556,10 +10627,16 @@ function splitOf(record, opts) {
   return "practice";
 }
 function scoreOf(record) {
-  const score = record.outcome.holdoutScore ?? record.outcome.searchScore;
-  if (typeof score === "number" && Number.isFinite(score)) return clamp014(score);
-  const rawScore = record.outcome.raw.score ?? record.outcome.raw.composite;
-  return typeof rawScore === "number" && Number.isFinite(rawScore) ? clamp014(rawScore) : 0;
+  const score = runTaskScore(record);
+  if (score !== void 0) return clamp014(score);
+  if ((record.outcome.judgeScores?.failedJudges?.length ?? 0) > 0 || (record.outcome.raw.judge_error_count ?? 0) > 0) {
+    throw new ValidationError(
+      `Run ${record.runId} has incomplete judge evidence; product benchmark rows require a complete task score`
+    );
+  }
+  throw new ValidationError(
+    `Run ${record.runId} has no task score; product benchmark rows require an explicit score`
+  );
 }
 function rawPassOf(record) {
   const rawPass = record.outcome.raw.pass;
@@ -10569,17 +10646,28 @@ function rawPassOf(record) {
 }
 function passOf(record, score, threshold) {
   const rawPass = rawPassOf(record);
-  if (rawPass !== null) return rawPass && !record.failureMode;
-  return score >= threshold && !record.failureMode;
+  const hasTaskFailure = record.failureClass !== void 0 && record.failureClass !== "success";
+  if (rawPass !== null) return rawPass && !hasTaskFailure;
+  return score >= threshold && !hasTaskFailure;
 }
-function failureModeOf(record, score, threshold) {
-  if (record.failureMode) return record.failureMode;
+function failureOf(record, score, threshold) {
+  if (record.failureClass !== void 0 && record.failureClass !== "success") {
+    return {
+      failureClass: record.failureClass,
+      failureDetail: record.failureMode ?? null
+    };
+  }
   const belowThreshold = `quality-below-threshold: ${Math.round(score * 100)}% < ${Math.round(threshold * 100)}%`;
   if (rawPassOf(record) === false) {
-    return score < threshold ? belowThreshold : "product-pass-failed";
+    return {
+      failureClass: "unknown",
+      failureDetail: score < threshold ? belowThreshold : "product-pass-failed"
+    };
   }
-  if (passOf(record, score, threshold)) return null;
-  return belowThreshold;
+  if (passOf(record, score, threshold)) {
+    return { failureClass: null, failureDetail: null };
+  }
+  return { failureClass: "unknown", failureDetail: belowThreshold };
 }
 function numericDimensions(record) {
   const dimensions = {};
@@ -10696,18 +10784,25 @@ function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifac
   const opts = resolveOptions(options);
   const runtime = runtimeResolution(record, opts);
   const score = scoreOf(record);
+  const failure = failureOf(record, score, opts.passThreshold);
   const armId = armIdOf(record);
   const inputTokens = record.tokenUsage.input;
   const outputTokens = record.tokenUsage.output;
   const toolCallCount = toolCallsOf(record, runDir, opts);
   const dimensions = numericDimensions(record);
+  if (record.costUsd === null) {
+    throw new ValidationError(
+      `run '${record.runId}' has no USD cost; product benchmark exports require priced runs`
+    );
+  }
+  const costUsd = record.costUsd;
   if (!("tool_calls" in dimensions)) dimensions.tool_calls = toolCallCount;
   const product = {
     schemaVersion: 1,
     projectId: opts.projectId,
     benchmarkId: opts.benchmarkId,
     runId: record.runId,
-    scenarioId: record.scenarioId ?? record.experimentId,
+    scenarioId: record.scenarioId,
     split: splitOf(record, opts),
     armId,
     rep: Number(record.seed ?? 0) + 1,
@@ -10724,12 +10819,12 @@ function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifac
       pass: passOf(record, score, opts.passThreshold),
       score,
       dimensions,
-      failureMode: failureModeOf(record, score, opts.passThreshold)
+      ...failure
     },
     usage: {
       inputTokens,
       outputTokens,
-      costUsd: record.costUsd,
+      costUsd,
       // Rounded: the bundle contract requires integer milliseconds.
       wallMs: Math.round(record.wallMs),
       toolCalls: toolCallCount
@@ -10739,7 +10834,7 @@ function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifac
       rawCapture: existsArtifact(artifactRoot, artifacts.raws),
       traceCapture: existsArtifact(artifactRoot, artifacts.traces),
       noStubRows: inputTokens + outputTokens > 0,
-      priced: record.costUsd > 0,
+      priced: costUsd > 0,
       profileMaterialized: Boolean(record.agentProfile?.cellId)
     },
     artifacts
@@ -10939,6 +11034,13 @@ function expectString(value, path) {
     fail(path, "must be a non-empty string");
   return value;
 }
+function expectFailureClass(value, path) {
+  const parsed = expectString(value, path);
+  if (parsed === "success" || !FAILURE_CLASSES.includes(parsed)) {
+    fail(path, `expected a non-success FailureClass, received ${JSON.stringify(parsed)}`);
+  }
+  return parsed;
+}
 function expectBoolean(value, path) {
   if (typeof value !== "boolean") fail(path, "must be a boolean");
   return value;
@@ -11081,6 +11183,18 @@ function validateProductBenchmarkRecord(value) {
   const usage = expectObject(obj.usage, "record.usage");
   const integrity = expectObject(obj.integrity, "record.integrity");
   const artifacts = expectObject(obj.artifacts, "record.artifacts");
+  const outcomePass = expectBoolean(outcome.pass, "record.outcome.pass");
+  const failureClass = outcome.failureClass === null ? null : expectFailureClass(outcome.failureClass, "record.outcome.failureClass");
+  const failureDetail = outcome.failureDetail === null ? null : expectString(outcome.failureDetail, "record.outcome.failureDetail");
+  if (outcomePass && (failureClass !== null || failureDetail !== null)) {
+    fail("record.outcome", "a passing row cannot carry failure evidence");
+  }
+  if (!outcomePass && failureClass === null) {
+    fail("record.outcome.failureClass", "a failed row requires a failure class");
+  }
+  if (failureClass === null && failureDetail !== null) {
+    fail("record.outcome.failureDetail", "failure detail requires a failure class");
+  }
   const record = {
     schemaVersion: 1,
     projectId: expectString(obj.projectId, "record.projectId"),
@@ -11106,10 +11220,11 @@ function validateProductBenchmarkRecord(value) {
       version: expectString(backend.version, "record.backend.version")
     },
     outcome: {
-      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
+      pass: outcomePass,
       score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
       dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
-      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
+      failureClass,
+      failureDetail
     },
     usage: {
       inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {
@@ -11379,7 +11494,6 @@ export {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
-  ROLLOUT_FORMAT,
   ROLLOUT_SCHEMA,
   RUN_COST_ATTR_KEYS,
   ReplayCache,
@@ -11498,6 +11612,7 @@ export {
   checkTraceContracts,
   clamp01,
   classifyFailure,
+  classifyOtlpSpanRole,
   classifyTreatment,
   claudeCodeSupervisorRunReader,
   cliffsDelta,
@@ -11661,6 +11776,7 @@ export {
   isLlmSpan,
   isModelPriced,
   isOtelConfigured,
+  isOtlpModelCall,
   isRetrievalSpan,
   isRolloutLine,
   isRunRecord,
@@ -11719,7 +11835,9 @@ export {
   otlpToRunRecords,
   otlpToTraceRunRecords,
   pairArms,
+  pairRunRecords,
   pairedBootstrap,
+  pairedCohensDz,
   pairedEvalueSequence,
   pairedMde,
   pairedRiskDifference,
@@ -11788,10 +11906,10 @@ export {
   replayScorerOverCorpus,
   replayTraceThroughJudge,
   requireAgentProfileCell,
+  requiredPairedSampleSize,
   requiredSampleSize,
   researchReport,
   resolveModelPricing,
-  resolveRunCostProvenance,
   resolveSeat,
   rolloutReward,
   rollupSupervisorRuns,
@@ -11822,6 +11940,7 @@ export {
   runScore,
   runSelfPlay,
   runSemanticConceptJudge,
+  runTaskScore,
   runTestGradedScenario,
   runsForScenario,
   scalarScore,