@tangle-network/agent-eval 0.119.0 → 0.120.0

package/dist/contract/index.js

@@ -13,12 +13,15 @@ import {
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-XJ7JVCHB.js";
+} from "../chunk-3A246TSA.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   buildEvidenceVector,
+  buildLoopProvenanceRecord,
+  campaignBreakdown,
   campaignMeanComposite,
+  canonicalDigest,
   composeGate,
   createReferenceEquivalenceJudge,
   defaultProductionGate,
@@ -26,28 +29,36 @@ import {
   evolutionaryProposer,
   gepaProposer,
   heldOutGate,
+  loopProvenanceArgsFromResult,
+  pairArms,
   paretoPolicy,
   paretoSignificanceGate,
   powerPreflight,
+  renderSurfaceDiff,
   runEval,
   runImprovementLoop,
   runReferenceEquivalenceJudge,
   surfaceContentHash,
-  surfaceHash
-} from "../chunk-RLCJQ6VZ.js";
+  surfaceHash,
+  verifyLoopProvenanceRecord
+} from "../chunk-32BZXMSO.js";
 import {
+  assertCampaignSplitIdentity,
+  assertRealAgentReceipts,
+  campaignCoverage,
+  campaignSplitDigest,
   createRunCostLedger,
   fsCampaignStorage,
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-PAHNGS65.js";
+} from "../chunk-XDIRG3TO.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-GERDAIAL.js";
+} from "../chunk-QWMPPZ3X.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-7A4LIMMY.js";
+import "../chunk-S5TT5R3L.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
@@ -55,8 +66,12 @@ import {
 import "../chunk-ARU2PZFM.js";
 import "../chunk-NJC7U437.js";
 import "../chunk-DPZAEKA6.js";
-import "../chunk-PJQFMIOX.js";
-import "../chunk-JHOJHHU7.js";
+import {
+  pairedBootstrap
+} from "../chunk-PJQFMIOX.js";
+import {
+  CostLedger
+} from "../chunk-JHOJHHU7.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
@@ -65,7 +80,7 @@ import {
 import "../chunk-PXE2VKMX.js";
 import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-6QIM2EAP.js";
+import "../chunk-PXD6ZFNY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
@@ -249,29 +264,17 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     candidateCandidateId: "winner"
   });
   const durationMs = Date.now() - startedAt;
-  const workerRecords = opts.collectWorkerRecords?.() ?? cellsToRunRecords(result.winnerOnHoldout.cells, "winner", runDir, result.winnerSurface);
   const { record: provenance } = await emitLoopProvenance({
-    runId: `${runDir}#${startedAt}`,
-    runDir,
-    timestamp: new Date(startedAt).toISOString(),
-    baselineSurface: opts.baselineSurface,
-    winnerSurface: result.winnerSurface,
-    winnerLabel: result.winnerLabel,
-    winnerRationale: result.winnerRationale,
-    diff: result.promotedDiff,
-    baselineSearchComposite: campaignMeanComposite(result.baselineCampaign),
-    generations: result.generations.map((g) => ({
-      generationIndex: g.record.generationIndex,
-      candidates: g.record.candidates,
-      promoted: g.record.promoted,
-      surfaces: g.surfaces.map((s) => ({ surfaceHash: s.surfaceHash, surface: s.surface }))
-    })),
-    gate: result.gateResult,
-    baselineOnHoldout: result.baselineOnHoldout,
-    winnerOnHoldout: result.winnerOnHoldout,
-    workerRecords,
-    totalCostUsd: totalCost,
-    totalDurationMs: durationMs,
+    ...loopProvenanceArgsFromResult({
+      runId: `${runDir}#${startedAt}`,
+      runDir,
+      timestamp: new Date(startedAt).toISOString(),
+      baselineSurface: opts.baselineSurface,
+      result,
+      costReceipts: costLedger.list(),
+      totalCostUsd: totalCost,
+      totalDurationMs: durationMs
+    }),
     storage,
     hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
   });
@@ -419,7 +422,13 @@ function cellsToRunRecords(cells, candidateId, runId, surface) {
       commitSha: "cell",
       wallMs: cell.durationMs,
       costUsd: cell.costUsd,
-      tokenUsage: { input: 0, output: 0 },
+      tokenUsage: {
+        input: cell.tokenUsage.input,
+        output: cell.tokenUsage.output,
+        ...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
+        ...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
+        ...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
+      },
       outcome: {
         holdoutScore: composite,
         raw: {},
@@ -536,6 +545,559 @@ function requirePositiveInteger(value, field) {
   return value;
 }
 
+// src/contract/measured-comparison.ts
+import {
+  agentImprovementMeasuredComparisonSchema
+} from "@tangle-network/agent-interface";
+function measuredComparisonFromSelfImproveResult(options) {
+  const { result } = options;
+  verifyLoopProvenanceRecord(result.provenance);
+  const power = result.power;
+  if (!power) throw new Error("agent improvement comparison requires heldout power analysis");
+  if (result.provenance.gate.reasons.length === 0) {
+    throw new Error("agent improvement comparison requires measured decision reasons");
+  }
+  const receiptLedger = new CostLedger({ receipts: result.receipts });
+  const receiptCost = receiptLedger.summary();
+  const receipts = receiptLedger.list();
+  assertRealAgentReceipts(receipts, { allowMixed: false });
+  const receiptsById = new Map(receipts.map((receipt) => [receipt.callId, receipt]));
+  const receiptIdsByCell = indexReceiptIdsByCell(receipts);
+  assertCompleteMeasuredCampaign(
+    result.raw.baselineOnHoldout,
+    "heldout baseline",
+    receiptsById,
+    receiptIdsByCell
+  );
+  assertCompleteMeasuredCampaign(
+    result.raw.winnerOnHoldout,
+    "heldout candidate",
+    receiptsById,
+    receiptIdsByCell
+  );
+  const pairs = pairMeasuredCells(
+    result.raw.baselineOnHoldout.cells,
+    result.raw.winnerOnHoldout.cells
+  );
+  const composite = measuredObjective(
+    {
+      kind: "objective",
+      name: "composite",
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    pairs,
+    measuredComposite
+  );
+  assertMeasuredNumber(result.lift, composite.delta, "heldout lift");
+  assertMeasuredNumber(result.baseline.compositeMean, composite.baseline, "heldout baseline");
+  assertMeasuredNumber(result.winner.compositeMean, composite.candidate, "heldout candidate");
+  const baselineContentHash = surfaceContentHash(options.baselineSurface);
+  const candidateContentHash = surfaceContentHash(result.winner.surface);
+  const canonicalDiff = baselineContentHash === candidateContentHash ? "" : renderSurfaceDiff(result.raw.winnerSurface, options.baselineSurface);
+  assertMeasuredIdentity(result.gateDecision, result.raw.gateResult.decision, "raw decision");
+  assertMeasuredIdentity(
+    candidateContentHash,
+    surfaceContentHash(result.raw.winnerSurface),
+    "raw winner surface"
+  );
+  assertMeasuredIdentity(result.diff, canonicalDiff, "surface diff");
+  assertMeasuredIdentity(result.raw.promotedDiff, canonicalDiff, "raw surface diff");
+  if (result.gateDecision === "ship" && (baselineContentHash === candidateContentHash || result.diff.trim().length === 0)) {
+    throw new Error("a shipped improvement requires a changed surface and non-empty diff");
+  }
+  assertMeasuredIdentity(
+    canonicalDigest(power),
+    canonicalDigest(
+      powerPreflight({
+        baselineComposites: pairs.map(([cell]) => measuredComposite(cell)),
+        sharedScorerChannel: true
+      })
+    ),
+    "power analysis"
+  );
+  assertMeasuredNumber(power.n, composite.n, "power sample size");
+  assertMeasuredNumber(power.confidence, 0.95, "power confidence");
+  assertCompleteMeasuredCampaign(
+    result.raw.baselineCampaign,
+    "search baseline",
+    receiptsById,
+    receiptIdsByCell
+  );
+  assertGenerationMeasurements(result.raw.generations, receiptsById, receiptIdsByCell);
+  if (result.raw.neutralizedOnHoldout === void 0 !== (result.raw.neutralizedSurface === void 0)) {
+    throw new Error("neutralized surface and campaign must be supplied together");
+  }
+  if (result.raw.neutralizedOnHoldout) {
+    assertCompleteMeasuredCampaign(
+      result.raw.neutralizedOnHoldout,
+      "heldout neutralized",
+      receiptsById,
+      receiptIdsByCell
+    );
+    pairMeasuredCells(result.raw.baselineOnHoldout.cells, result.raw.neutralizedOnHoldout.cells);
+  }
+  const rebuiltProvenance = buildLoopProvenanceRecord(
+    loopProvenanceArgsFromResult({
+      runId: result.provenance.runId,
+      runDir: result.provenance.runDir,
+      timestamp: result.provenance.timestamp,
+      baselineSurface: options.baselineSurface,
+      result: result.raw,
+      costReceipts: receipts,
+      totalCostUsd: result.totalCostUsd,
+      totalDurationMs: result.durationMs
+    })
+  );
+  assertMeasuredIdentity(
+    result.provenance.recordDigest,
+    rebuiltProvenance.recordDigest,
+    "provenance record"
+  );
+  assertMeasuredIdentity(
+    options.benchmark.splitDigest,
+    rebuiltProvenance.evidence.holdout.splitDigest,
+    "benchmark heldout split"
+  );
+  const generationsExplored = result.raw.generations.length;
+  assertMeasuredNumber(result.generationsExplored, generationsExplored, "generation count");
+  assertMeasuredNumber(result.totalCostUsd, result.cost.totalCostUsd, "cost summary");
+  assertMeasuredIdentity(
+    canonicalDigest(result.cost),
+    canonicalDigest(result.raw.cost),
+    "raw cost summary"
+  );
+  if (!result.cost.accountingComplete) {
+    throw new Error(
+      `cost accounting is incomplete: ${result.cost.incompleteReasons.join("; ") || "unknown reason"}`
+    );
+  }
+  if (!receiptCost.accountingComplete) {
+    throw new Error(
+      `cost accounting is incomplete: ${receiptCost.incompleteReasons.join("; ") || "unknown reason"}`
+    );
+  }
+  assertMeasuredIdentity(
+    canonicalDigest(result.cost),
+    canonicalDigest(receiptCost),
+    "cost receipt summary"
+  );
+  assertMeasuredNumber(result.totalCostUsd, receiptCost.totalCostUsd, "cost receipts");
+  assertMeasuredOptional(result.winner.label, result.raw.winnerLabel, "raw winner label");
+  assertMeasuredOptional(
+    result.winner.rationale,
+    result.raw.winnerRationale,
+    "raw winner rationale"
+  );
+  return agentImprovementMeasuredComparisonSchema.parse({
+    schemaVersion: 1,
+    kind: "agent-improvement-measured-comparison",
+    benchmark: options.benchmark,
+    baselineProfileDigest: options.baselineProfileDigest,
+    candidateBundleDigest: options.candidateBundleDigest,
+    overall: {
+      name: "composite",
+      baseline: composite.baseline,
+      candidate: composite.candidate,
+      delta: composite.delta,
+      confidenceInterval: composite.confidenceInterval,
+      n: composite.n,
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    objectives: measuredObjectives(pairs),
+    ...result.winner.label || result.winner.rationale ? {
+      candidate: {
+        ...result.winner.label ? { label: result.winner.label } : {},
+        ...result.winner.rationale ? { rationale: result.winner.rationale } : {}
+      }
+    } : {},
+    decision: {
+      outcome: result.gateDecision,
+      reasons: rebuiltProvenance.gate.reasons,
+      contributingChecks: rebuiltProvenance.gate.contributingGates.map((check) => ({
+        name: check.name,
+        passed: check.passed
+      }))
+    },
+    power: {
+      sufficient: power.scaleAssumed && !power.underpowered,
+      n: power.n,
+      minimumDetectableDelta: power.mde,
+      confidenceLevel: power.confidence,
+      scaleAssumed: power.scaleAssumed,
+      sharedScorerChannel: power.sharedChannelCaveat !== void 0,
+      reason: power.recommendation
+    },
+    provenance: {
+      kind: "agent-eval-loop",
+      schema: rebuiltProvenance.schema,
+      runId: rebuiltProvenance.runId,
+      recordDigest: rebuiltProvenance.recordDigest,
+      baselineContentHash,
+      candidateContentHash
+    },
+    diff: canonicalDiff,
+    evaluation: {
+      generationsExplored,
+      durationMs: result.durationMs,
+      totalCostUsd: result.totalCostUsd
+    }
+  });
+}
+function assertGenerationMeasurements(generations, receiptsById, receiptIdsByCell) {
+  for (const generation of generations) {
+    for (const measured of generation.surfaces) {
+      const candidate = generation.record.candidates.find(
+        (entry) => entry.surfaceHash === measured.surfaceHash
+      );
+      if (!candidate) {
+        throw new Error(
+          `generation ${generation.record.generationIndex} is missing candidate ${measured.surfaceHash}`
+        );
+      }
+      const composite = campaignMeanComposite(measured.campaign);
+      const breakdown = campaignBreakdown(measured.campaign);
+      const coverage = campaignCoverage(
+        measured.campaign.cells,
+        measured.campaign.scenarios,
+        measured.campaign.reps,
+        true
+      );
+      assertScorableMeasuredCells(
+        measured.campaign,
+        coverage.scorableCellIds,
+        receiptsById,
+        receiptIdsByCell
+      );
+      assertMeasuredNumber(
+        candidate.composite,
+        composite,
+        `candidate ${measured.surfaceHash} composite`
+      );
+      if (!candidate.coverage) {
+        throw new Error(`candidate ${measured.surfaceHash} does not report campaign coverage`);
+      }
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.coverage),
+        canonicalDigest({
+          expectedCells: coverage.expectedCellIds.length,
+          scorableCells: coverage.scorableCellIds.length,
+          unscorableCells: coverage.unscorableCells
+        }),
+        `candidate ${measured.surfaceHash} coverage`
+      );
+      assertMeasuredIdentity(
+        String(candidate.eligibleForPromotion),
+        String(coverage.complete),
+        `candidate ${measured.surfaceHash} eligibility`
+      );
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.dimensions),
+        canonicalDigest(breakdown.dimensions),
+        `candidate ${measured.surfaceHash} dimensions`
+      );
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.scenarios),
+        canonicalDigest(breakdown.scenarios),
+        `candidate ${measured.surfaceHash} scenarios`
+      );
+    }
+  }
+}
+function assertCompleteMeasuredCampaign(campaign, name, receiptsById, receiptIdsByCell) {
+  assertCampaignSplitIdentity(campaign.scenarios, campaign.reps, campaign.splitDigest);
+  const coverage = campaignCoverage(campaign.cells, campaign.scenarios, campaign.reps, true);
+  if (!coverage.complete) {
+    throw new Error(
+      `${name} is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable)`
+    );
+  }
+  assertScorableMeasuredCells(campaign, coverage.scorableCellIds, receiptsById, receiptIdsByCell);
+}
+function assertScorableMeasuredCells(campaign, scorableCellIds, receiptsById, receiptIdsByCell) {
+  const cellsById = new Map(campaign.cells.map((cell) => [cell.cellId, cell]));
+  for (const cellId of scorableCellIds) {
+    const cell = cellsById.get(cellId);
+    if (!cell) throw new Error(`measured campaign is missing scorable cell ${cellId}`);
+    assertMeasuredCell(cell, receiptsById, receiptIdsByCell, campaign.runDir);
+  }
+}
+function measuredObjectives(pairs) {
+  const qualityColumns = /* @__PURE__ */ new Map();
+  for (const [baseline, candidate] of pairs) {
+    for (const cell of [baseline, candidate]) {
+      for (const [objective, score] of Object.entries(cell.judgeScores)) {
+        if (score.failed) continue;
+        qualityColumns.set(`objective:${objective}`, {
+          kind: "objective",
+          name: objective,
+          direction: "higher-is-better",
+          unit: "score"
+        });
+        for (const name of Object.keys(score.dimensions)) {
+          qualityColumns.set(`dimension:${objective}:${name}`, {
+            kind: "dimension",
+            objective,
+            name,
+            direction: "higher-is-better",
+            unit: "score"
+          });
+        }
+      }
+    }
+  }
+  return [
+    ...[...qualityColumns.entries()].sort(([left], [right]) => left.localeCompare(right)).map(
+      ([, column]) => measuredObjective(column, pairs, (cell) => measuredQuality(cell, column))
+    ),
+    measuredCostObjective(pairs),
+    measuredObjective(
+      {
+        kind: "latency",
+        name: "latency",
+        direction: "lower-is-better",
+        unit: "milliseconds"
+      },
+      pairs,
+      (cell) => cell.durationMs
+    )
+  ];
+}
+function measuredCostObjective(pairs) {
+  const cells = pairs.flat();
+  for (const cell of cells) finiteMeasuredValue(cell.costUsd, "cost:cost");
+  return measuredObjective(
+    {
+      kind: "cost",
+      name: "cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    },
+    pairs,
+    (cell) => cell.costUsd
+  );
+}
+function measuredComposite(cell) {
+  const values = Object.values(cell.judgeScores).filter((score) => !score.failed).map((score) => score.composite).filter(Number.isFinite);
+  if (values.length === 0) {
+    throw new Error(`heldout cell ${measuredCellKey(cell)} has no successful composite score`);
+  }
+  return measuredMean(values);
+}
+function pairMeasuredCells(baselineCells, candidateCells) {
+  const cells = [...baselineCells, ...candidateCells];
+  const errors = cells.filter((cell) => cell.error);
+  if (errors.length > 0) {
+    throw new Error(
+      `measured objectives cannot publish ${errors.length} errored heldout cells: ${errors.map((cell) => measuredCellKey(cell)).join(", ")}`
+    );
+  }
+  const rows = [
+    ...baselineCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "baseline",
+      cell
+    })),
+    ...candidateCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "candidate",
+      cell
+    }))
+  ];
+  const paired = pairArms(rows, { baselineArm: "baseline", treatmentArm: "candidate" });
+  if (paired.pairs.length === 0 || paired.unpairedBaseline.length > 0 || paired.unpairedTreatment.length > 0) {
+    throw new Error("measured objectives require the same non-empty paired heldout cells");
+  }
+  return paired.pairs.map((pair) => {
+    const baseline = pair.baseline.cell;
+    const candidate = pair.treatment.cell;
+    if (baseline.seed !== candidate.seed) {
+      throw new Error(`heldout cell ${baseline.cellId} does not share one paired seed`);
+    }
+    return [baseline, candidate];
+  });
+}
+function assertMeasuredCell(cell, receiptsById, receiptIdsByCell, runDir) {
+  if (!cell.scenarioId.trim() || !Number.isSafeInteger(cell.rep) || cell.rep < 0 || !Number.isSafeInteger(cell.seed) || cell.cellId !== measuredCellKey(cell)) {
+    throw new Error(`heldout cell '${cell.cellId}' has an invalid scenario/rep identity`);
+  }
+  nonnegativeMeasuredValue(cell.costUsd, `heldout cell ${cell.cellId} cost`);
+  nonnegativeMeasuredValue(cell.durationMs, `heldout cell ${cell.cellId} latency`);
+  if (!cell.tokenUsage) {
+    throw new Error(`heldout cell ${cell.cellId} does not report token usage`);
+  }
+  nonnegativeMeasuredInteger(cell.tokenUsage.input, `heldout cell ${cell.cellId} input tokens`);
+  nonnegativeMeasuredInteger(cell.tokenUsage.output, `heldout cell ${cell.cellId} output tokens`);
+  if (cell.tokenUsage.cached !== void 0) {
+    nonnegativeMeasuredInteger(cell.tokenUsage.cached, `heldout cell ${cell.cellId} cached tokens`);
+  }
+  if (!Array.isArray(cell.costCallIds)) {
+    throw new Error(`heldout cell ${cell.cellId} does not identify its cost receipts`);
+  }
+  if (new Set(cell.costCallIds).size !== cell.costCallIds.length) {
+    throw new Error(`heldout cell ${cell.cellId} repeats a cost receipt`);
+  }
+  const linkedReceipts = cell.costCallIds.map((callId) => {
+    const receipt = receiptsById.get(callId);
+    if (!receipt)
+      throw new Error(`heldout cell ${cell.cellId} references missing cost receipt ${callId}`);
+    if (receipt.tags?.runDir !== runDir || receipt.tags.cellId !== cell.cellId || receipt.tags.scenarioId !== cell.scenarioId || receipt.tags.rep !== String(cell.rep)) {
+      throw new Error(`heldout cell ${cell.cellId} references a cost receipt from another cell`);
+    }
+    return receipt;
+  });
+  assertMeasuredIdentity(
+    canonicalDigest([...cell.costCallIds].sort()),
+    canonicalDigest(
+      [
+        ...receiptIdsByCell.get(receiptCellKey(runDir, cell.cellId, cell.scenarioId, cell.rep)) ?? []
+      ].sort()
+    ),
+    `heldout cell ${cell.cellId} cost receipt IDs`
+  );
+  const agentReceipts = linkedReceipts.filter((receipt) => receipt.channel === "agent");
+  if (agentReceipts.some((receipt) => receipt.error)) {
+    throw new Error(`measured cell ${cell.cellId} links a failed agent receipt`);
+  }
+  assertRealAgentReceipts(agentReceipts, { allowMixed: false });
+  assertMeasuredNumber(
+    cell.costUsd,
+    agentReceipts.reduce((total, receipt) => total + receipt.costUsd, 0),
+    `heldout cell ${cell.cellId} cost receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.input,
+    agentReceipts.reduce((total, receipt) => total + receipt.inputTokens, 0),
+    `heldout cell ${cell.cellId} input receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.output,
+    agentReceipts.reduce((total, receipt) => total + receipt.outputTokens, 0),
+    `heldout cell ${cell.cellId} output receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.cached ?? 0,
+    agentReceipts.reduce((total, receipt) => total + (receipt.cachedTokens ?? 0), 0),
+    `heldout cell ${cell.cellId} cached receipts`
+  );
+  for (const [judge, score] of Object.entries(cell.judgeScores)) {
+    if (score.failed) {
+      throw new Error(`heldout cell ${cell.cellId} contains failed judge '${judge}'`);
+    }
+    finiteMeasuredValue(score.composite, `objective:${judge}`);
+    for (const [dimension, value] of Object.entries(score.dimensions)) {
+      finiteMeasuredValue(value, `dimension:${judge}:${dimension}`);
+    }
+  }
+  measuredComposite(cell);
+}
+function measuredObjective(identity, pairs, value) {
+  const key = measuredObjectiveKey(identity);
+  const baseline = pairs.map(([cell]) => finiteMeasuredValue(value(cell), key));
+  const candidate = pairs.map(([, cell]) => finiteMeasuredValue(value(cell), key));
+  const interval = pairedBootstrap(baseline, candidate, {
+    confidence: 0.95,
+    resamples: 2e3,
+    statistic: "mean",
+    seed: measuredSeed(key)
+  });
+  const baselineMean = measuredMean(baseline);
+  const candidateMean = measuredMean(candidate);
+  return {
+    ...identity,
+    availability: "measured",
+    baseline: baselineMean,
+    candidate: candidateMean,
+    delta: candidateMean - baselineMean,
+    confidenceInterval: {
+      level: interval.confidence,
+      lower: interval.low,
+      upper: interval.high,
+      method: "paired-bootstrap",
+      statistic: "mean",
+      resamples: interval.resamples
+    },
+    n: interval.n
+  };
+}
+function measuredQuality(cell, column) {
+  const objective = column.kind === "objective" ? column.name : column.objective;
+  const score = cell.judgeScores[objective];
+  if (!score || score.failed) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing measured objective '${objective}'`
+    );
+  }
+  const value = column.kind === "objective" ? score.composite : score.dimensions[column.name];
+  if (value === void 0) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing '${objective}' dimension '${column.name}'`
+    );
+  }
+  return finiteMeasuredValue(value, measuredObjectiveKey(column));
+}
+function measuredObjectiveKey(identity) {
+  return identity.kind === "dimension" ? `${identity.kind}:${identity.objective}:${identity.name}` : `${identity.kind}:${identity.name}`;
+}
+function measuredCellKey(cell) {
+  return `${cell.scenarioId}:${cell.rep}`;
+}
+function finiteMeasuredValue(value, name) {
+  if (!Number.isFinite(value)) throw new Error(`measured objective '${name}' is not finite`);
+  return value;
+}
+function nonnegativeMeasuredValue(value, name) {
+  if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be non-negative`);
+  return value;
+}
+function nonnegativeMeasuredInteger(value, name) {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${name} must be a non-negative safe integer`);
+  }
+  return value;
+}
+function measuredMean(values) {
+  if (values.length === 0) throw new Error("measured objective has no paired values");
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function measuredSeed(value) {
+  let seed = 2166136261;
+  for (const byte of Buffer.from(value, "utf8")) {
+    seed = Math.imul(seed ^ byte, 16777619) >>> 0;
+  }
+  return seed;
+}
+function assertMeasuredNumber(actual, expected, name) {
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)) * 8;
+  if (!Number.isFinite(actual) || !Number.isFinite(expected) || Math.abs(actual - expected) > tolerance) {
+    throw new Error(`${name} does not agree across the measured comparison`);
+  }
+}
+function assertMeasuredIdentity(actual, expected, name) {
+  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
+}
+function assertMeasuredOptional(actual, expected, name) {
+  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
+}
+function indexReceiptIdsByCell(receipts) {
+  const idsByCell = /* @__PURE__ */ new Map();
+  for (const receipt of receipts) {
+    const tags = receipt.tags;
+    if (!tags?.runDir || !tags.cellId || !tags.scenarioId || tags.rep === void 0) continue;
+    const key = receiptCellKey(tags.runDir, tags.cellId, tags.scenarioId, tags.rep);
+    const ids = idsByCell.get(key) ?? [];
+    ids.push(receipt.callId);
+    idsByCell.set(key, ids);
+  }
+  return idsByCell;
+}
+function receiptCellKey(runDir, cellId, scenarioId, rep) {
+  return JSON.stringify([runDir, cellId, scenarioId, String(rep)]);
+}
+
 // src/contract/eval-reporting-suite.ts
 import { mkdir, writeFile } from "fs/promises";
 import { dirname, join as join2 } from "path";
@@ -1051,6 +1613,7 @@ export {
   analyzeRuns,
   buildDefaultAnalystRegistry,
   buildEvidenceVector,
+  campaignSplitDigest,
   composeGate,
   createChatClient,
   createReferenceEquivalenceJudge,
@@ -1074,6 +1637,7 @@ export {
   gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
+  measuredComparisonFromSelfImproveResult,
   paretoPolicy,
   paretoSignificanceGate,
   parseAgentTrace,