@tangle-network/agent-eval 0.126.6 → 0.127.0

package/dist/contract/index.js

@@ -7,14 +7,14 @@ import {
   fromPigraphSession,
   observeCodeAgentSession,
   parseCodeAgentJsonl
-} from "../chunk-7VYEGHWF.js";
+} from "../chunk-SZLVEKMJ.js";
 import {
   createHostedClient
-} from "../chunk-ZZUXHH3R.js";
+} from "../chunk-IG35YDN3.js";
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-K6IAZZ6L.js";
+} from "../chunk-FA3MFYNA.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -40,7 +40,7 @@ import {
   skillOptOptimizationMethod,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-BTACVO2W.js";
+} from "../chunk-65QETEKG.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -48,47 +48,60 @@ import {
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-ZVCHKKOP.js";
+} from "../chunk-WED5KLZQ.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-LUNF2SEL.js";
+} from "../chunk-DJKY2TSY.js";
 import "../chunk-HHWE3POT.js";
 import "../chunk-WGXIEX7P.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "../chunk-3RF76KTD.js";
-import "../chunk-ARU2PZFM.js";
-import "../chunk-J5SQWP6Y.js";
-import "../chunk-DPZAEKA6.js";
+import "../chunk-424XHVKV.js";
+import {
+  campaignCellExecutionEvidence,
+  campaignCellJudgeDimensions,
+  campaignCellTaskScore,
+  campaignCellToRunRecord
+} from "../chunk-ZPUEEM5U.js";
+import "../chunk-PBE2LOSS.js";
+import "../chunk-IMGJZBFG.js";
+import "../chunk-DPUHNQLN.js";
 import {
   pairedBootstrap
-} from "../chunk-PJQFMIOX.js";
+} from "../chunk-MHELPNRP.js";
 import "../chunk-WS3NZZQQ.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
-  summarizeExecutionMeasurements
-} from "../chunk-H5UD2323.js";
+  summarizeExecutionMeasurements,
+  summarizeTraceErrors
+} from "../chunk-IN455XKS.js";
 import "../chunk-PXE2VKMX.js";
 import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-IR3KBHOY.js";
+import {
+  classifyOtlpSpanRole,
+  isOtlpModelCall
+} from "../chunk-P6FYH6K4.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
-} from "../chunk-IILEIWGW.js";
+} from "../chunk-NL33OXEP.js";
 import "../chunk-MA6HLL3S.js";
 import "../chunk-VSMTAMNK.js";
-import "../chunk-ONWEPEDO.js";
 import {
-  LLM_MODEL_ATTR_KEYS
+  ValidationError
+} from "../chunk-ONWEPEDO.js";
+import {
+  LLM_MODEL_ATTR_KEYS,
+  SPAN_KIND_ATTR_KEYS
 } from "../chunk-K4DBDHLK.js";
 import "../chunk-PZ5AY32C.js";
 
 // src/contract/self-improve.ts
-import { createHash } from "crypto";
 var SelfImproveRunError = class extends Error {
   cost;
   receipts;
@@ -343,11 +356,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     findings: opts.findings,
     selectionRankKey: opts.selectionRankKey
   });
-  const winnerSearch = holdoutDeferred ? winnerSearchCampaign(result) : void 0;
-  const baseline = meanComposite(
-    (holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout).aggregates.byScenario
-  );
-  const winnerStats = meanComposite((winnerSearch ?? result.winnerOnHoldout).aggregates.byScenario);
+  const reportSplit = holdoutDeferred ? "search" : "holdout";
+  const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
+  const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
+  const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
+  const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
   let power;
   const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
     const scores = Object.values(cell.judgeScores);
@@ -390,12 +403,19 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const totalCost = cost.totalCostUsd;
   const insight = await analyzeRuns({
     runs: [
-      ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
       ...cellsToRunRecords(
-        (winnerSearch ?? result.winnerOnHoldout).cells,
+        reportBaselineCampaign.cells,
+        "baseline",
+        runDir,
+        opts.baselineSurface,
+        reportSplit
+      ),
+      ...cellsToRunRecords(
+        reportWinnerCampaign.cells,
         "winner",
         runDir,
-        result.winnerSurface
+        result.winnerSurface,
+        reportSplit
       )
     ],
     baselineCandidateId: "baseline",
@@ -468,19 +488,21 @@ async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
   const client = createHostedClient(tenant);
   function snapshotFromCampaign(index, surface, campaign, durationMs) {
     const cells = campaign.cells.map((cell) => {
-      const judgeScores = Object.values(cell.judgeScores);
-      const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
+      const execution = campaignCellExecutionEvidence(cell);
       return {
         scenarioId: cell.scenarioId,
         rep: cell.rep,
-        compositeMean: composite,
-        dimensions: Object.fromEntries(
-          Object.entries(cell.judgeScores).map(([name, score]) => [name, score.dimensions])
-        ),
+        compositeMean: campaignCellTaskScore(cell) ?? null,
+        dimensions: campaignCellJudgeDimensions(cell),
+        terminalOutcome: execution.terminalOutcome,
+        executionErrorCount: execution.executionErrorCount ?? null,
         errorMessage: cell.error ?? void 0
       };
     });
-    const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
+    const scoredCells = cells.flatMap(
+      (cell) => cell.compositeMean === null ? [] : [cell.compositeMean]
+    );
+    const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
     return {
       index,
       surfaceHash: surfaceHash(surface),
@@ -531,67 +553,23 @@ function hashString(s) {
   }
   return h.toString(16).padStart(8, "0");
 }
-function cellsToRunRecords(cells, candidateId, runId, surface) {
+function cellsToRunRecords(cells, candidateId, runId, surface, splitTag) {
   const promptHash = surfaceContentHash(surface);
-  const configHash = `sha256:${createHash("sha256").update(candidateId).digest("hex")}`;
-  return cells.map((cell) => {
-    const perJudge = {};
-    const perDimMeanAccum = {};
-    let compositeSum = 0;
-    let compositeCount = 0;
-    for (const [judgeId, score] of Object.entries(cell.judgeScores)) {
-      perJudge[judgeId] = { ...score.dimensions };
-      for (const [dim, value] of Object.entries(score.dimensions)) {
-        if (!Number.isFinite(value)) continue;
-        const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
-        accum.sum += value;
-        accum.n += 1;
-        perDimMeanAccum[dim] = accum;
-      }
-      if (Number.isFinite(score.composite)) {
-        compositeSum += score.composite;
-        compositeCount += 1;
-      }
-    }
-    const perDimMean = {};
-    for (const [dim, { sum, n }] of Object.entries(perDimMeanAccum)) {
-      perDimMean[dim] = n === 0 ? 0 : sum / n;
-    }
-    const composite = compositeCount === 0 ? 0 : compositeSum / compositeCount;
-    const judgeScores = {
-      perJudge,
-      perDimMean,
-      composite
-    };
-    return {
+  const configHash = surfaceContentHash(candidateId);
+  return cells.map(
+    (cell) => campaignCellToRunRecord(cell, {
       runId: `${runId}::${candidateId}::${cell.cellId}`,
       experimentId: runId,
       candidateId,
-      // Pair on (scenarioId, rep) — analyzeRuns pairs on (experimentId, seed).
-      // Synthesize a stable seed for that pairing.
+      // scenarioId is explicit; seed keeps repeated runs distinct.
       seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
       model: "campaign-cell",
       promptHash,
       configHash,
       commitSha: "cell",
-      wallMs: cell.durationMs,
-      costUsd: cell.costUsd,
-      tokenUsage: {
-        input: cell.tokenUsage.input,
-        output: cell.tokenUsage.output,
-        ...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
-        ...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
-        ...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
-      },
-      outcome: {
-        holdoutScore: composite,
-        raw: {},
-        judgeScores
-      },
-      splitTag: "holdout",
-      ...cell.error ? { failureMode: cell.error } : {}
-    };
-  });
+      splitTag
+    })
+  );
 }
 
 // src/contract/define-agent-eval.ts
@@ -698,6 +676,7 @@ import {
   agentCandidateBenchmarkSuiteSchema,
   agentCandidateBenchmarkTaskSchema,
   agentCandidateBundleSchema,
+  agentCandidateEvaluationPolicySchema,
   agentCandidateExperimentSchema,
   agentImprovementMeasuredComparisonSchema,
   candidateExecutionEvidenceSchema,
@@ -800,19 +779,28 @@ async function runCandidateExperiment(options) {
   await Promise.all(lanes);
   return measurements;
 }
-function measuredComparisonFromCandidateExperiment(options) {
-  const experiment = verifyCandidateExperiment(options.experiment);
+function evaluatePairedMeasurements(options) {
+  if (options.measurements.length === 0) {
+    throw new Error("paired measurement evaluation requires at least one paired cell");
+  }
+  const additionalCostUsd = options.additionalCostUsd ?? 0;
+  if (!Number.isFinite(additionalCostUsd) || additionalCostUsd < 0) {
+    throw new Error("paired measurement evaluation additionalCostUsd must be a non-negative number");
+  }
+  if (typeof options.sharedScorerChannel !== "boolean") {
+    throw new Error("paired measurement evaluation sharedScorerChannel must be a boolean");
+  }
+  const policy = agentCandidateEvaluationPolicySchema.parse(options.policy);
   const measurements = options.measurements.map(
-    (measurement, index) => verifyMeasurement(experiment, measurement, index)
+    (measurement, index) => projectPairedMeasurement(measurement, index, options.adapter)
   );
-  const expectedN = experiment.benchmark.suite.taskDigests.length * experiment.benchmark.suite.reps;
-  if (measurements.length !== expectedN) {
-    throw new Error(
-      `candidate experiment is incomplete (${measurements.length}/${expectedN} paired cells)`
-    );
+  const cellIds2 = measurements.map((measurement) => measurement.cellId);
+  if (new Set(cellIds2).size !== cellIds2.length) {
+    throw new Error("paired measurement evaluation cell ids must be unique");
   }
-  verifyStableProfileMaterialization(measurements);
-  if (!options.runId.trim()) throw new Error("candidate experiment runId is required");
+  const dimensions = sharedProjectedDimensions(measurements);
+  const baselineScores = measurements.map((measurement) => measurement.baseline.score);
+  const candidateScores = measurements.map((measurement) => measurement.candidate.score);
   const {
     confidenceLevel: confidence,
     resamples,
@@ -822,15 +810,23 @@ function measuredComparisonFromCandidateExperiment(options) {
     budgetUsd,
     criticalDimensions,
     regressionTolerance
-  } = experiment.policy;
-  const baselineScores = measurements.map(scoreOf("baseline"));
-  const candidateScores = measurements.map(scoreOf("candidate"));
+  } = policy;
+  const significance = heldoutSignificance(
+    { before: baselineScores, after: candidateScores, cellIds: cellIds2 },
+    {
+      confidence,
+      resamples,
+      seed: bootstrapSeed,
+      statistic: "mean",
+      deltaThreshold,
+      minProductiveRuns
+    }
+  );
   const overall = measuredEstimate(baselineScores, candidateScores, {
     confidence,
     resamples,
     seed: bootstrapSeed
   });
-  const dimensions = sharedDimensions(measurements);
   const objectives = [
     {
       kind: "objective",
@@ -848,21 +844,33 @@ function measuredComparisonFromCandidateExperiment(options) {
       unit: "score",
       availability: "measured",
       ...measuredEstimate(
-        measurements.map(dimensionOf("baseline", name)),
-        measurements.map(dimensionOf("candidate", name)),
-        { confidence, resamples, seed: bootstrapSeed + index + 1 }
+        measurements.map((measurement) => dimensionScore(measurement.baseline, name)),
+        measurements.map((measurement) => dimensionScore(measurement.candidate, name)),
+        {
+          confidence,
+          resamples,
+          seed: bootstrapSeed + index + 1
+        }
       )
     }))
   ];
   const cost = measuredEstimate(
-    measurements.map(costOf("baseline")),
-    measurements.map(costOf("candidate")),
-    { confidence, resamples, seed: bootstrapSeed + dimensions.length + 1 }
+    measurements.map((measurement) => measurement.baseline.costUsd),
+    measurements.map((measurement) => measurement.candidate.costUsd),
+    {
+      confidence,
+      resamples,
+      seed: bootstrapSeed + dimensions.length + 1
+    }
   );
   const latency = measuredEstimate(
-    measurements.map(latencyOf("baseline")),
-    measurements.map(latencyOf("candidate")),
-    { confidence, resamples, seed: bootstrapSeed + dimensions.length + 2 }
+    measurements.map((measurement) => measurement.baseline.latencyMs),
+    measurements.map((measurement) => measurement.candidate.latencyMs),
+    {
+      confidence,
+      resamples,
+      seed: bootstrapSeed + dimensions.length + 2
+    }
   );
   objectives.push(
     {
@@ -882,23 +890,12 @@ function measuredComparisonFromCandidateExperiment(options) {
       ...latency
     }
   );
-  const significance = heldoutSignificance(
-    { before: baselineScores, after: candidateScores, cellIds: cellIds(experiment) },
-    {
-      confidence,
-      resamples,
-      seed: bootstrapSeed,
-      statistic: "mean",
-      deltaThreshold,
-      minProductiveRuns
-    }
-  );
   const power = baselineScores.length >= 3 ? powerPreflight({
     baselineComposites: baselineScores,
     pairedN: baselineScores.length,
     deltaThreshold,
     confidence,
-    sharedScorerChannel: true
+    sharedScorerChannel: options.sharedScorerChannel
   }) : void 0;
   const powerSufficient = baselineScores.length >= minProductiveRuns && power !== void 0 && !power.underpowered;
   const guardedDimensions = new Set(criticalDimensions);
@@ -909,20 +906,21 @@ function measuredComparisonFromCandidateExperiment(options) {
     (objective) => objective.kind === "dimension" && guardedDimensions.has(objective.name) && objective.availability === "measured" && objective.confidenceInterval.lower < -regressionTolerance
   );
   const executionCostUsd = measurements.reduce(
-    (sum, measurement) => sum + costFromEvidence(measurement.baseline) + costFromEvidence(measurement.candidate),
+    (sum, measurement) => sum + measurement.baseline.costUsd + measurement.candidate.costUsd,
+    0
+  );
+  const executionDurationMs = measurements.reduce(
+    (sum, measurement) => sum + measurement.baseline.latencyMs + measurement.candidate.latencyMs,
     0
   );
-  const searchCostUsd = options.searchCostUsd ?? 0;
-  const totalCostUsd = executionCostUsd + searchCostUsd;
-  const budgetPassed = budgetUsd === void 0 || totalCostUsd <= budgetUsd;
   const completedRuns = measurements.flatMap((measurement) => [
     measurement.baseline,
     measurement.candidate
   ]);
-  const incompleteRuns = completedRuns.filter((evidence) => !completedSuccessfully(evidence));
-  const failedCandidateResults = measurements.filter(
-    (measurement) => !measurement.candidate.receipt.benchmarkResult.material.passed
-  );
+  const incompleteRuns = completedRuns.filter((run) => !run.completed);
+  const failedCandidateResults = measurements.filter((measurement) => !measurement.candidate.passed);
+  const totalCostUsd = executionCostUsd + additionalCostUsd;
+  const budgetPassed = budgetUsd === void 0 || totalCostUsd <= budgetUsd;
   const checks = [
     { name: "paired-significance", passed: significance.significant },
     { name: "statistical-power", passed: powerSufficient },
@@ -946,17 +944,7 @@ function measuredComparisonFromCandidateExperiment(options) {
     ...failedCandidateResults.length === 0 ? [] : [`candidate failed ${failedCandidateResults.length} benchmark tasks`],
     ...budgetPassed ? [] : [`total cost ${totalCostUsd} exceeded budget ${budgetUsd}`]
   ];
-  const diff = deriveCandidateBundleDiff(experiment);
-  const executionDurationMs = measurements.reduce(
-    (sum, measurement) => sum + latencyFromEvidence(measurement.baseline) + latencyFromEvidence(measurement.candidate),
-    0
-  );
-  const searchDurationMs = options.searchDurationMs ?? 0;
-  const durationMs = executionDurationMs + searchDurationMs;
-  const provisional = agentImprovementMeasuredComparisonSchema.parse({
-    kind: "agent-improvement-measured-comparison",
-    experiment,
-    measurements,
+  return {
     overall: {
       name: "composite",
       direction: "higher-is-better",
@@ -964,7 +952,6 @@ function measuredComparisonFromCandidateExperiment(options) {
       ...overall
     },
     objectives,
-    ...options.candidate ? { candidate: options.candidate } : {},
     decision: {
       outcome: shipped ? "ship" : significance.fewRuns || !powerSufficient ? "need_more_work" : "hold",
       reasons: reasons.length > 0 ? reasons : ["all measured checks passed"],
@@ -976,9 +963,51 @@ function measuredComparisonFromCandidateExperiment(options) {
       minimumDetectableDelta: power?.mde ?? 1,
       confidenceLevel: confidence,
       scaleAssumed: power?.scaleAssumed ?? true,
-      sharedScorerChannel: true,
+      sharedScorerChannel: options.sharedScorerChannel,
       reason: power?.recommendation ?? `need at least ${Math.max(3, minProductiveRuns)} paired runs`
     },
+    executionCostUsd,
+    totalCostUsd,
+    executionDurationMs
+  };
+}
+function measuredComparisonFromCandidateExperiment(options) {
+  const experiment = verifyCandidateExperiment(options.experiment);
+  const measurements = options.measurements.map(
+    (measurement, index) => verifyMeasurement(experiment, measurement, index)
+  );
+  const expectedN = experiment.benchmark.suite.taskDigests.length * experiment.benchmark.suite.reps;
+  if (measurements.length !== expectedN) {
+    throw new Error(
+      `candidate experiment is incomplete (${measurements.length}/${expectedN} paired cells)`
+    );
+  }
+  verifyStableProfileMaterialization(measurements);
+  if (!options.runId.trim()) throw new Error("candidate experiment runId is required");
+  const searchCostUsd = options.searchCostUsd ?? 0;
+  const evaluation = evaluatePairedMeasurements({
+    measurements: measurements.map((measurement, index) => ({
+      cellId: cellIds(experiment)[index],
+      ...measurement
+    })),
+    policy: experiment.policy,
+    adapter: candidateExecutionEvidenceAdapter,
+    sharedScorerChannel: true,
+    additionalCostUsd: searchCostUsd
+  });
+  const diff = deriveCandidateBundleDiff(experiment);
+  const searchDurationMs = options.searchDurationMs ?? 0;
+  const totalCostUsd = evaluation.totalCostUsd;
+  const durationMs = evaluation.executionDurationMs + searchDurationMs;
+  const provisional = agentImprovementMeasuredComparisonSchema.parse({
+    kind: "agent-improvement-measured-comparison",
+    experiment,
+    measurements,
+    overall: evaluation.overall,
+    objectives: evaluation.objectives,
+    ...options.candidate ? { candidate: options.candidate } : {},
+    decision: evaluation.decision,
+    power: evaluation.power,
     provenance: {
       kind: "agent-eval-loop",
       schema: "agent-candidate-experiment",
@@ -991,10 +1020,10 @@ function measuredComparisonFromCandidateExperiment(options) {
     evaluation: {
       generationsExplored: options.generationsExplored ?? 0,
       searchDurationMs,
-      executionDurationMs,
+      executionDurationMs: evaluation.executionDurationMs,
       durationMs,
       searchCostUsd,
-      executionCostUsd,
+      executionCostUsd: evaluation.executionCostUsd,
       totalCostUsd
     },
     ...options.metadata ? { metadata: options.metadata } : {}
@@ -1197,6 +1226,65 @@ function verifyMaterialAddressed(evidence, label) {
     throw new Error(`${label} digest is invalid`);
   }
 }
+function projectPairedMeasurement(measurement, index, adapter) {
+  if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) {
+    throw new Error(`paired measurement ${index} requires a cell id`);
+  }
+  return {
+    cellId: measurement.cellId,
+    baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
+    candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
+  };
+}
+function projectRun(run, adapter, label) {
+  const suppliedDimensions = adapter.dimensions(run);
+  if (!Array.isArray(suppliedDimensions)) {
+    throw new Error(`${label} dimensions must be an array`);
+  }
+  const dimensions = /* @__PURE__ */ new Map();
+  for (const dimension of suppliedDimensions) {
+    if (typeof dimension.name !== "string" || !dimension.name.trim()) {
+      throw new Error(`${label} contains an unnamed dimension`);
+    }
+    if (dimensions.has(dimension.name)) {
+      throw new Error(`${label} repeats dimension '${dimension.name}'`);
+    }
+    dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
+  }
+  const completed = adapter.completed(run);
+  const passed = adapter.passed(run);
+  if (typeof completed !== "boolean" || typeof passed !== "boolean") {
+    throw new Error(`${label} completion and pass values must be booleans`);
+  }
+  return {
+    score: finiteMeasurement(adapter.score(run), `${label} score`),
+    dimensions,
+    costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
+    latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
+    completed,
+    passed
+  };
+}
+function sharedProjectedDimensions(measurements) {
+  const expected = [...measurements[0].baseline.dimensions.keys()];
+  for (const [index, measurement] of measurements.entries()) {
+    for (const [arm, run] of [
+      ["baseline", measurement.baseline],
+      ["candidate", measurement.candidate]
+    ]) {
+      const actual = [...run.dimensions.keys()];
+      if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+        throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
+      }
+    }
+  }
+  return expected;
+}
+function dimensionScore(run, name) {
+  const value = run.dimensions.get(name);
+  if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
+  return value;
+}
 function measuredEstimate(baseline, candidate, options) {
   const bootstrap = pairedBootstrap(baseline, candidate, {
     confidence: options.confidence,
@@ -1222,46 +1310,22 @@ function measuredEstimate(baseline, candidate, options) {
     n: bootstrap.n
   };
 }
-function sharedDimensions(measurements) {
-  const expected = dimensionNames(measurements[0]?.baseline);
-  for (const [index, measurement] of measurements.entries()) {
-    for (const [arm, evidence] of [
-      ["baseline", measurement.baseline],
-      ["candidate", measurement.candidate]
-    ]) {
-      const actual = dimensionNames(evidence);
-      if (JSON.stringify(actual) !== JSON.stringify(expected)) {
-        throw new Error(
-          `candidate experiment measurement ${index} ${arm} dimensions do not match the suite`
-        );
-      }
-    }
-  }
-  return expected;
-}
-function dimensionNames(evidence) {
-  return (evidence?.receipt.benchmarkResult.material.dimensions ?? []).map(
-    (dimension) => dimension.name
-  );
-}
-function scoreOf(arm) {
-  return (measurement) => measurement[arm].receipt.benchmarkResult.material.score;
-}
-function dimensionOf(arm, name) {
-  return (measurement) => {
-    const dimension = measurement[arm].receipt.benchmarkResult.material.dimensions.find(
-      (entry) => entry.name === name
-    );
-    if (!dimension) throw new Error(`candidate experiment is missing dimension '${name}'`);
-    return dimension.score;
-  };
-}
-function costOf(arm) {
-  return (measurement) => costFromEvidence(measurement[arm]);
+function finiteMeasurement(value, label) {
+  if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
+  return value;
 }
-function latencyOf(arm) {
-  return (measurement) => latencyFromEvidence(measurement[arm]);
+function nonNegativeMeasurement(value, label) {
+  if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
+  return value;
 }
+var candidateExecutionEvidenceAdapter = {
+  score: (evidence) => evidence.receipt.benchmarkResult.material.score,
+  dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
+  costUsd: costFromEvidence,
+  latencyMs: latencyFromEvidence,
+  completed: completedSuccessfully,
+  passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
+};
 function costFromEvidence(evidence) {
   return combinedUsage(evidence).costUsdNanos / 1e9;
 }
@@ -1479,7 +1543,7 @@ function diffGenerations(before, after) {
       rep: beforeCell.rep,
       compositeBefore: beforeCell.compositeMean,
       compositeAfter: afterCell.compositeMean,
-      compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
+      compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
       dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
     });
   }
@@ -1497,7 +1561,7 @@ function diffGenerations(before, after) {
     added,
     compositeBefore: before.compositeMean,
     compositeAfter: after.compositeMean,
-    compositeDelta: after.compositeMean - before.compositeMean,
+    compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
     costUsdBefore: before.costUsd,
     costUsdAfter: after.costUsd,
     costUsdDelta: after.costUsd - before.costUsd,
@@ -1650,14 +1714,13 @@ function fromFeedbackTable(opts) {
       perDimMean: { rating: meanScore },
       composite: meanScore
     };
+    const splitTag = runMeta.splitTag ?? "holdout";
     const outcome = {
-      // Feedback corpora ARE the holdout signal — score lands on
-      // `holdoutScore` so downstream substrate primitives (`paretoChart`,
-      // promotion gates) read it correctly by default.
-      holdoutScore: meanScore,
+      ...splitTag === "holdout" ? { holdoutScore: meanScore } : { searchScore: meanScore },
       raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
       judgeScores
     };
+    const costUsd = runMeta.costUsd ?? null;
     runs.push({
       runId,
       experimentId: runMeta.experimentId ?? "feedback-corpus",
@@ -1668,10 +1731,13 @@ function fromFeedbackTable(opts) {
       configHash: runMeta.configHash ?? "sha256:unknown",
       commitSha: runMeta.commitSha ?? "unknown",
       wallMs: runMeta.wallMs ?? 0,
-      costUsd: runMeta.costUsd ?? 0,
+      costUsd,
+      costProvenance: costUsd === null ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: costUsd },
       tokenUsage: { input: 0, output: 0 },
+      terminalOutcome: "unknown",
       outcome,
-      splitTag: runMeta.splitTag ?? "holdout"
+      splitTag,
+      scenarioId: runMeta.scenarioId ?? runId
     });
     if (emitRaterScores) {
       for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
@@ -1681,7 +1747,12 @@ function fromFeedbackTable(opts) {
 }
 
 // src/contract/intake/otel-spans.ts
-var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
+var TASK_SCORE_ATTR_KEYS = [
+  "gen_ai.evaluation.score.value",
+  "tangle.task.score",
+  "eval.score",
+  "tangle.score"
+];
 var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
 var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
 var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
@@ -1703,15 +1774,31 @@ function fromOtelSpans(opts) {
     );
     const callSpanIds = new Set(measurements.callSpanIds);
     const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
-    const wallMs = Math.max(0, (root.endTimeUnixNano - root.startTimeUnixNano) / 1e6);
+    const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
     const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
     const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
-    const costUsd = capturedCost ?? 0;
+    const costUsd = capturedCost ?? null;
+    const scenarioId = readConsistentScenarioId(groupKey, groupSpans2) ?? groupKey;
     const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
     const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
-    const score = readAttrNumber(groupSpans2, SCORE_KEYS);
+    const score = resolveTaskScore(groupKey, groupSpans2, opts.scoreForRun);
     const rawNumeric = collectNumericAttrs(groupSpans2);
-    rawNumeric.error_span_count = groupSpans2.filter((span) => span.status?.code === "ERROR").length;
+    const errorSummary = summarizeTraceErrors(
+      groupSpans2.map((span) => ({
+        id: spanIdentity(span),
+        ...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
+        role: errorRoleForSpan(span),
+        error: span.status?.code === "ERROR",
+        processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
+      }))
+    );
+    rawNumeric.error_span_count = errorSummary.total;
+    rawNumeric.execution_error_count = errorSummary.execution;
+    rawNumeric.process_error_count = errorSummary.process;
+    rawNumeric.guardrail_error_count = errorSummary.guardrail;
+    rawNumeric.judge_error_count = errorSummary.evaluation;
+    rawNumeric.propagated_error_count = errorSummary.propagated;
+    rawNumeric.unclassified_error_count = errorSummary.unclassified;
     rawNumeric.llm_span_count = measurements.modelCallCount;
     if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
       rawNumeric.partial_observed_cost_usd = measurements.cost.value;
@@ -1722,12 +1809,18 @@ function fromOtelSpans(opts) {
       perDimMean: { score },
       composite: score
     } : void 0;
-    const errorSpan = groupSpans2.find((s) => s.status?.code === "ERROR");
+    const terminalOutcome = terminalOutcomeFromRoots(groupSpans2);
+    const failedRoot = terminalOutcome === "failed" ? groupSpans2.find(
+      (span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR"
+    ) : void 0;
     const outcome = {
-      ...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
       raw: rawNumeric,
       ...judgeScores ? { judgeScores } : {}
     };
+    if (score !== void 0) {
+      if (defaultSplit === "holdout") outcome.holdoutScore = score;
+      else outcome.searchScore = score;
+    }
     runs.push({
       runId: groupKey,
       experimentId,
@@ -1741,22 +1834,55 @@ function fromOtelSpans(opts) {
       costUsd,
       costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
       tokenUsage: measurements.tokenUsage,
+      terminalOutcome,
+      ...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
       outcome,
       splitTag: defaultSplit,
-      ...errorSpan ? { failureMode: errorSpan.name } : {}
+      scenarioId
     });
   }
   return runs;
 }
+function terminalOutcomeFromRoots(spans) {
+  const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
+  if (roots.length !== 1) return "unknown";
+  if (roots[0].status?.code === "OK") return "succeeded";
+  if (roots[0].status?.code === "ERROR") return "failed";
+  return "unknown";
+}
+function isTerminalRootCandidate(span) {
+  const role = errorRoleForSpan(span);
+  if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") {
+    return false;
+  }
+  return true;
+}
+function readSpanKind(span) {
+  return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
+}
+function errorRoleForSpan(span) {
+  return classifyOtlpSpanRole({
+    kind: readSpanKind(span),
+    name: span.name,
+    attributes: span.attributes
+  });
+}
+function spanIdentity(span) {
+  return `${span.traceId}:${span.spanId}`;
+}
+function parentIdentity(span) {
+  return `${span.traceId}:${span.parentSpanId}`;
+}
 function isExplicitModelCall(span) {
-  const kind = span.attributes["openinference.span.kind"];
-  if (typeof kind === "string") return kind.toUpperCase() === "LLM";
-  const spanType = span.attributes["span.type"];
-  return typeof spanType === "string" && spanType.toLowerCase() === "llm_request" || span.name.toLowerCase().includes("llm") || readAttrString([span], MODEL_KEYS) !== void 0 || typeof span.attributes["gen_ai.operation.name"] === "string";
+  return isOtlpModelCall({
+    kind: readSpanKind(span),
+    name: span.name,
+    attributes: span.attributes
+  });
 }
 function isExplicitAggregate(span) {
-  const kind = span.attributes["openinference.span.kind"];
-  return typeof kind === "string" && kind.toUpperCase() !== "LLM";
+  const kind = readSpanKind(span);
+  return kind !== void 0 && kind !== "LLM";
 }
 function groupSpans(spans) {
   const m = /* @__PURE__ */ new Map();
@@ -1769,7 +1895,10 @@ function groupSpans(spans) {
   return m;
 }
 function findRoot(group) {
-  return group.find((s) => !s.parentSpanId) ?? group[0];
+  const structuralRoots = group.filter((span) => !span.parentSpanId);
+  const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
+  const pool = terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group;
+  return orderSpans(pool)[0];
 }
 function readAttrString(spans, keys) {
   for (const span of spans) {
@@ -1780,18 +1909,102 @@ function readAttrString(spans, keys) {
   }
   return void 0;
 }
-function readAttrNumber(spans, keys) {
+function readConsistentScenarioId(runId, spans) {
+  const values = /* @__PURE__ */ new Set();
   for (const span of spans) {
-    for (const key of keys) {
-      const v = span.attributes[key];
-      if (typeof v === "number" && Number.isFinite(v)) return v;
-      if (typeof v === "string") {
-        const parsed = Number(v);
-        if (Number.isFinite(parsed)) return parsed;
+    const topLevel = span["tangle.scenarioId"];
+    if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
+    const attribute = span.attributes["tangle.scenarioId"];
+    if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
+  }
+  if (values.size > 1) {
+    throw new ValidationError(
+      `fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`
+    );
+  }
+  return values.values().next().value;
+}
+function resolveTaskScore(runId, spans, scoreForRun) {
+  const orderedSpans = orderSpans(spans);
+  const sources = [];
+  if (scoreForRun) {
+    const supplied = scoreForRun(runId, orderedSpans);
+    if (supplied !== void 0) {
+      if (typeof supplied !== "number" || !Number.isFinite(supplied)) {
+        throw new ValidationError(
+          `fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`
+        );
       }
+      sources.push({ label: "scoreForRun", value: supplied });
     }
   }
-  return void 0;
+  for (const span of orderedSpans) {
+    const role = errorRoleForSpan(span);
+    if (span.parentSpanId && role !== "EVALUATOR") continue;
+    if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
+    for (const key of TASK_SCORE_ATTR_KEYS) {
+      if (!Object.hasOwn(span.attributes, key)) continue;
+      sources.push({
+        label: `span '${span.spanId}' attribute '${key}'`,
+        value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
+      });
+    }
+  }
+  if (sources.length === 0) return void 0;
+  sources.sort((left, right) => left.label.localeCompare(right.label));
+  const score = sources[0].value;
+  if (sources.some((source) => source.value !== score)) {
+    const details = sources.map((source) => `${source.label}=${source.value}`).join(", ");
+    throw new ValidationError(
+      `fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`
+    );
+  }
+  return score;
+}
+function parseTaskScoreAttribute(runId, spanId, key, value) {
+  const source = `span '${spanId}' attribute '${key}'`;
+  if (typeof value === "string") {
+    if (value.trim().length === 0) {
+      throw new ValidationError(
+        `fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`
+      );
+    }
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  } else if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  throw new ValidationError(
+    `fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`
+  );
+}
+function orderSpans(spans) {
+  return [...spans].sort(
+    (left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId)
+  );
+}
+function parseUnixNano(value) {
+  try {
+    return BigInt(value);
+  } catch {
+    throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
+  }
+}
+function compareUnixNano(left, right) {
+  const leftValue = parseUnixNano(left);
+  const rightValue = parseUnixNano(right);
+  return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
+}
+function unixNanoDurationMs(start, end) {
+  const delta = parseUnixNano(end) - parseUnixNano(start);
+  if (delta <= 0n) return 0;
+  const wholeMs = delta / 1000000n;
+  const fractionalMs = delta % 1000000n;
+  const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
+  if (!Number.isSafeInteger(Number(wholeMs))) {
+    throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
+  }
+  return value;
 }
 function collectNumericAttrs(spans) {
   const raw = {};
@@ -1822,6 +2035,7 @@ export {
   diffRunBaselineToWinner,
   diffRuns,
   evalReportingSuite,
+  evaluatePairedMeasurements,
   externalTextOptimizationMethod,
   fromClaudeCodeSession,
   fromCodexSession,