@tangle-network/agent-eval 0.120.4 → 0.122.0

package/dist/contract/index.js

@@ -5,23 +5,20 @@ import {
   fromOpenCodeSession,
   fromPiSession,
   fromPigraphSession,
+  observeCodeAgentSession,
   parseCodeAgentJsonl
-} from "../chunk-HKUCJ437.js";
+} from "../chunk-7VYEGHWF.js";
 import {
   createHostedClient
 } from "../chunk-ZZUXHH3R.js";
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-SQQED7ZH.js";
+} from "../chunk-3A246TSA.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   buildEvidenceVector,
-  buildLoopProvenanceRecord,
-  campaignBreakdown,
-  campaignMeanComposite,
-  canonicalDigest,
   composeGate,
   createReferenceEquivalenceJudge,
   defaultProductionGate,
@@ -29,36 +26,31 @@ import {
   evolutionaryProposer,
   gepaProposer,
   heldOutGate,
+  heldoutSignificance,
   loopProvenanceArgsFromResult,
-  pairArms,
   paretoPolicy,
   paretoSignificanceGate,
   powerPreflight,
-  renderSurfaceDiff,
   runEval,
   runImprovementLoop,
   runReferenceEquivalenceJudge,
   surfaceContentHash,
-  surfaceHash,
-  verifyLoopProvenanceRecord
-} from "../chunk-7QBFOJ3E.js";
+  surfaceHash
+} from "../chunk-VZ6VKOJT.js";
 import {
-  assertCampaignSplitIdentity,
-  assertRealAgentReceipts,
-  campaignCoverage,
   campaignSplitDigest,
   createRunCostLedger,
   fsCampaignStorage,
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-SYV364BL.js";
+} from "../chunk-XDIRG3TO.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-5BYTIDZ7.js";
+} from "../chunk-QWMPPZ3X.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-5CVUPHJ4.js";
+import "../chunk-S5TT5R3L.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
@@ -69,9 +61,7 @@ import "../chunk-DPZAEKA6.js";
 import {
   pairedBootstrap
 } from "../chunk-PJQFMIOX.js";
-import {
-  CostLedger
-} from "../chunk-ZMXDQ4K7.js";
+import "../chunk-JHOJHHU7.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
@@ -548,555 +538,605 @@ function requirePositiveInteger(value, field) {
 
 // src/contract/measured-comparison.ts
 import {
-  agentImprovementMeasuredComparisonSchema
+  agentCandidateBenchmarkSuiteSchema,
+  agentCandidateBenchmarkTaskSchema,
+  agentCandidateBundleSchema,
+  agentCandidateExperimentSchema,
+  agentImprovementMeasuredComparisonSchema,
+  candidateExecutionEvidenceSchema,
+  canonicalCandidateDigest,
+  omitTopLevelDigest
 } from "@tangle-network/agent-interface";
-function measuredComparisonFromSelfImproveResult(options) {
-  const { result } = options;
-  verifyLoopProvenanceRecord(result.provenance);
-  const power = result.power;
-  if (!power) throw new Error("agent improvement comparison requires heldout power analysis");
-  if (result.provenance.gate.reasons.length === 0) {
-    throw new Error("agent improvement comparison requires measured decision reasons");
-  }
-  const receiptLedger = new CostLedger({ receipts: result.receipts });
-  const receiptCost = receiptLedger.summary();
-  const receipts = receiptLedger.list();
-  assertRealAgentReceipts(receipts, { allowMixed: false });
-  const receiptsById = new Map(receipts.map((receipt) => [receipt.callId, receipt]));
-  const receiptIdsByCell = indexReceiptIdsByCell(receipts);
-  assertCompleteMeasuredCampaign(
-    result.raw.baselineOnHoldout,
-    "heldout baseline",
-    receiptsById,
-    receiptIdsByCell
-  );
-  assertCompleteMeasuredCampaign(
-    result.raw.winnerOnHoldout,
-    "heldout candidate",
-    receiptsById,
-    receiptIdsByCell
+function sealCandidateBenchmarkTask(material) {
+  return agentCandidateBenchmarkTaskSchema.parse({
+    ...material,
+    digest: canonicalCandidateDigest(material)
+  });
+}
+function sealCandidateBenchmarkSuite(options) {
+  for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
+  const material = {
+    kind: "agent-candidate-benchmark-suite",
+    digestAlgorithm: "rfc8785-sha256",
+    taskDigests: options.tasks.map((task) => task.digest),
+    reps: options.reps,
+    seeds: options.seeds
+  };
+  const suite = agentCandidateBenchmarkSuiteSchema.parse({
+    ...material,
+    digest: canonicalCandidateDigest(material)
+  });
+  return { suite, tasks: options.tasks };
+}
+function sealCandidateExperiment(material) {
+  const parsed = agentCandidateExperimentSchema.parse({
+    ...material,
+    digest: canonicalCandidateDigest(material)
+  });
+  return verifyCandidateExperiment(parsed);
+}
+function verifyCandidateExperiment(input) {
+  const experiment = agentCandidateExperimentSchema.parse(input);
+  verifySelfAddressed(experiment, "candidate experiment");
+  verifyBundle(experiment.baseline, "baseline bundle");
+  verifyBundle(experiment.candidate, "candidate bundle");
+  if (experiment.baseline.digest === experiment.candidate.digest) {
+    throw new Error("candidate experiment baseline and candidate bundles are identical");
+  }
+  verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
+  return experiment;
+}
+async function runCandidateExperiment(options) {
+  const experiment = verifyCandidateExperiment(options.experiment);
+  const { suite, tasks } = experiment.benchmark;
+  const maxConcurrency = options.maxConcurrency ?? 2;
+  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1) {
+    throw new Error("candidate experiment maxConcurrency must be a positive integer");
+  }
+  const measurements = new Array(
+    suite.taskDigests.length * suite.reps
   );
-  const pairs = pairMeasuredCells(
-    result.raw.baselineOnHoldout.cells,
-    result.raw.winnerOnHoldout.cells
+  let nextIndex = 0;
+  const lanes = Array.from({ length: Math.min(maxConcurrency, measurements.length) }, async () => {
+    while (true) {
+      if (options.signal?.aborted) throw abortError(options.signal);
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= measurements.length) return;
+      const taskIndex = Math.floor(index / suite.reps);
+      const repetition = index % suite.reps;
+      const task = tasks[taskIndex];
+      const seed = suite.seeds[index];
+      if (!task || seed === void 0) {
+        throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
+      }
+      const benchmarkCell = {
+        suiteDigest: suite.digest,
+        taskIndex,
+        repetition
+      };
+      const [baseline, candidate] = await Promise.all([
+        options.execute({
+          experiment,
+          arm: "baseline",
+          bundle: experiment.baseline,
+          task,
+          benchmarkCell,
+          seed,
+          ...options.signal ? { signal: options.signal } : {}
+        }),
+        options.execute({
+          experiment,
+          arm: "candidate",
+          bundle: experiment.candidate,
+          task,
+          benchmarkCell,
+          seed,
+          ...options.signal ? { signal: options.signal } : {}
+        })
+      ]);
+      const measurement = { baseline, candidate };
+      verifyMeasurement(experiment, measurement, index);
+      measurements[index] = measurement;
+    }
+  });
+  await Promise.all(lanes);
+  return measurements;
+}
+function measuredComparisonFromCandidateExperiment(options) {
+  const experiment = verifyCandidateExperiment(options.experiment);
+  const measurements = options.measurements.map(
+    (measurement, index) => verifyMeasurement(experiment, measurement, index)
   );
-  const composite = measuredObjective(
+  const expectedN = experiment.benchmark.suite.taskDigests.length * experiment.benchmark.suite.reps;
+  if (measurements.length !== expectedN) {
+    throw new Error(
+      `candidate experiment is incomplete (${measurements.length}/${expectedN} paired cells)`
+    );
+  }
+  verifyStableProfileMaterialization(measurements);
+  if (!options.runId.trim()) throw new Error("candidate experiment runId is required");
+  const {
+    confidenceLevel: confidence,
+    resamples,
+    bootstrapSeed,
+    deltaThreshold,
+    minProductiveRuns,
+    budgetUsd,
+    criticalDimensions,
+    regressionTolerance
+  } = experiment.policy;
+  const baselineScores = measurements.map(scoreOf("baseline"));
+  const candidateScores = measurements.map(scoreOf("candidate"));
+  const overall = measuredEstimate(baselineScores, candidateScores, {
+    confidence,
+    resamples,
+    seed: bootstrapSeed
+  });
+  const dimensions = sharedDimensions(measurements);
+  const objectives = [
     {
       kind: "objective",
-      name: "composite",
+      name: "benchmark-score",
       direction: "higher-is-better",
-      unit: "score"
+      unit: "score",
+      availability: "measured",
+      ...overall
     },
-    pairs,
-    measuredComposite
+    ...dimensions.map((name, index) => ({
+      kind: "dimension",
+      objective: "benchmark-score",
+      name,
+      direction: "higher-is-better",
+      unit: "score",
+      availability: "measured",
+      ...measuredEstimate(
+        measurements.map(dimensionOf("baseline", name)),
+        measurements.map(dimensionOf("candidate", name)),
+        { confidence, resamples, seed: bootstrapSeed + index + 1 }
+      )
+    }))
+  ];
+  const cost = measuredEstimate(
+    measurements.map(costOf("baseline")),
+    measurements.map(costOf("candidate")),
+    { confidence, resamples, seed: bootstrapSeed + dimensions.length + 1 }
   );
-  assertMeasuredNumber(result.lift, composite.delta, "heldout lift");
-  assertMeasuredNumber(result.baseline.compositeMean, composite.baseline, "heldout baseline");
-  assertMeasuredNumber(result.winner.compositeMean, composite.candidate, "heldout candidate");
-  const baselineContentHash = surfaceContentHash(options.baselineSurface);
-  const candidateContentHash = surfaceContentHash(result.winner.surface);
-  const canonicalDiff = baselineContentHash === candidateContentHash ? "" : renderSurfaceDiff(result.raw.winnerSurface, options.baselineSurface);
-  assertMeasuredIdentity(result.gateDecision, result.raw.gateResult.decision, "raw decision");
-  assertMeasuredIdentity(
-    candidateContentHash,
-    surfaceContentHash(result.raw.winnerSurface),
-    "raw winner surface"
+  const latency = measuredEstimate(
+    measurements.map(latencyOf("baseline")),
+    measurements.map(latencyOf("candidate")),
+    { confidence, resamples, seed: bootstrapSeed + dimensions.length + 2 }
   );
-  assertMeasuredIdentity(result.diff, canonicalDiff, "surface diff");
-  assertMeasuredIdentity(result.raw.promotedDiff, canonicalDiff, "raw surface diff");
-  if (result.gateDecision === "ship" && (baselineContentHash === candidateContentHash || result.diff.trim().length === 0)) {
-    throw new Error("a shipped improvement requires a changed surface and non-empty diff");
-  }
-  assertMeasuredIdentity(
-    canonicalDigest(power),
-    canonicalDigest(
-      powerPreflight({
-        baselineComposites: pairs.map(([cell]) => measuredComposite(cell)),
-        sharedScorerChannel: true
-      })
-    ),
-    "power analysis"
-  );
-  assertMeasuredNumber(power.n, composite.n, "power sample size");
-  assertMeasuredNumber(power.confidence, 0.95, "power confidence");
-  assertCompleteMeasuredCampaign(
-    result.raw.baselineCampaign,
-    "search baseline",
-    receiptsById,
-    receiptIdsByCell
+  objectives.push(
+    {
+      kind: "cost",
+      name: "cost",
+      direction: "lower-is-better",
+      unit: "usd",
+      availability: "measured",
+      ...cost
+    },
+    {
+      kind: "latency",
+      name: "latency",
+      direction: "lower-is-better",
+      unit: "milliseconds",
+      availability: "measured",
+      ...latency
+    }
   );
-  assertGenerationMeasurements(result.raw.generations, receiptsById, receiptIdsByCell);
-  if (result.raw.neutralizedOnHoldout === void 0 !== (result.raw.neutralizedSurface === void 0)) {
-    throw new Error("neutralized surface and campaign must be supplied together");
-  }
-  if (result.raw.neutralizedOnHoldout) {
-    assertCompleteMeasuredCampaign(
-      result.raw.neutralizedOnHoldout,
-      "heldout neutralized",
-      receiptsById,
-      receiptIdsByCell
-    );
-    pairMeasuredCells(result.raw.baselineOnHoldout.cells, result.raw.neutralizedOnHoldout.cells);
-  }
-  const rebuiltProvenance = buildLoopProvenanceRecord(
-    loopProvenanceArgsFromResult({
-      runId: result.provenance.runId,
-      runDir: result.provenance.runDir,
-      timestamp: result.provenance.timestamp,
-      baselineSurface: options.baselineSurface,
-      result: result.raw,
-      costReceipts: receipts,
-      totalCostUsd: result.totalCostUsd,
-      totalDurationMs: result.durationMs
-    })
+  const significance = heldoutSignificance(
+    { before: baselineScores, after: candidateScores, cellIds: cellIds(experiment) },
+    {
+      confidence,
+      resamples,
+      seed: bootstrapSeed,
+      statistic: "mean",
+      deltaThreshold,
+      minProductiveRuns
+    }
   );
-  assertMeasuredIdentity(
-    result.provenance.recordDigest,
-    rebuiltProvenance.recordDigest,
-    "provenance record"
+  const power = baselineScores.length >= 3 ? powerPreflight({
+    baselineComposites: baselineScores,
+    pairedN: baselineScores.length,
+    deltaThreshold,
+    confidence,
+    sharedScorerChannel: true
+  }) : void 0;
+  const powerSufficient = baselineScores.length >= minProductiveRuns && power !== void 0 && !power.underpowered;
+  const guardedDimensions = new Set(criticalDimensions);
+  const missingCriticalDimensions = criticalDimensions.filter(
+    (dimension) => !dimensions.includes(dimension)
   );
-  assertMeasuredIdentity(
-    options.benchmark.splitDigest,
-    rebuiltProvenance.evidence.holdout.splitDigest,
-    "benchmark heldout split"
+  const regressions = objectives.filter(
+    (objective) => objective.kind === "dimension" && guardedDimensions.has(objective.name) && objective.availability === "measured" && objective.confidenceInterval.lower < -regressionTolerance
   );
-  const generationsExplored = result.raw.generations.length;
-  assertMeasuredNumber(result.generationsExplored, generationsExplored, "generation count");
-  assertMeasuredNumber(result.totalCostUsd, result.cost.totalCostUsd, "cost summary");
-  assertMeasuredIdentity(
-    canonicalDigest(result.cost),
-    canonicalDigest(result.raw.cost),
-    "raw cost summary"
+  const executionCostUsd = measurements.reduce(
+    (sum, measurement) => sum + costFromEvidence(measurement.baseline) + costFromEvidence(measurement.candidate),
+    0
   );
-  if (!result.cost.accountingComplete) {
-    throw new Error(
-      `cost accounting is incomplete: ${result.cost.incompleteReasons.join("; ") || "unknown reason"}`
-    );
-  }
-  if (!receiptCost.accountingComplete) {
-    throw new Error(
-      `cost accounting is incomplete: ${receiptCost.incompleteReasons.join("; ") || "unknown reason"}`
-    );
-  }
-  assertMeasuredIdentity(
-    canonicalDigest(result.cost),
-    canonicalDigest(receiptCost),
-    "cost receipt summary"
+  const searchCostUsd = options.searchCostUsd ?? 0;
+  const totalCostUsd = executionCostUsd + searchCostUsd;
+  const budgetPassed = budgetUsd === void 0 || totalCostUsd <= budgetUsd;
+  const completedRuns = measurements.flatMap((measurement) => [
+    measurement.baseline,
+    measurement.candidate
+  ]);
+  const incompleteRuns = completedRuns.filter((evidence) => !completedSuccessfully(evidence));
+  const failedCandidateResults = measurements.filter(
+    (measurement) => !measurement.candidate.receipt.benchmarkResult.material.passed
   );
-  assertMeasuredNumber(result.totalCostUsd, receiptCost.totalCostUsd, "cost receipts");
-  assertMeasuredOptional(result.winner.label, result.raw.winnerLabel, "raw winner label");
-  assertMeasuredOptional(
-    result.winner.rationale,
-    result.raw.winnerRationale,
-    "raw winner rationale"
+  const checks = [
+    { name: "paired-significance", passed: significance.significant },
+    { name: "statistical-power", passed: powerSufficient },
+    { name: "all-runs-completed", passed: incompleteRuns.length === 0 },
+    { name: "candidate-task-pass", passed: failedCandidateResults.length === 0 },
+    {
+      name: "critical-dimensions",
+      passed: regressions.length === 0 && missingCriticalDimensions.length === 0
+    },
+    { name: "budget", passed: budgetPassed }
+  ];
+  const shipped = checks.every((check) => check.passed);
+  const reasons = [
+    ...significance.significant ? [] : [
+      significance.fewRuns ? `only ${significance.n} paired runs; ${minProductiveRuns} required` : `paired interval lower bound ${significance.bootstrap.low} did not clear ${deltaThreshold}`
+    ],
+    ...powerSufficient ? [] : [power?.recommendation ?? `need at least ${Math.max(3, minProductiveRuns)} paired runs`],
+    ...regressions.length === 0 ? [] : [`critical dimensions regressed: ${regressions.map((entry) => entry.name).join(", ")}`],
+    ...missingCriticalDimensions.length === 0 ? [] : [`critical dimensions missing: ${missingCriticalDimensions.join(", ")}`],
+    ...incompleteRuns.length === 0 ? [] : [`${incompleteRuns.length} benchmark executions did not exit successfully`],
+    ...failedCandidateResults.length === 0 ? [] : [`candidate failed ${failedCandidateResults.length} benchmark tasks`],
+    ...budgetPassed ? [] : [`total cost ${totalCostUsd} exceeded budget ${budgetUsd}`]
+  ];
+  const diff = deriveCandidateBundleDiff(experiment);
+  const executionDurationMs = measurements.reduce(
+    (sum, measurement) => sum + latencyFromEvidence(measurement.baseline) + latencyFromEvidence(measurement.candidate),
+    0
   );
-  return agentImprovementMeasuredComparisonSchema.parse({
-    schemaVersion: 1,
+  const searchDurationMs = options.searchDurationMs ?? 0;
+  const durationMs = executionDurationMs + searchDurationMs;
+  const provisional = agentImprovementMeasuredComparisonSchema.parse({
     kind: "agent-improvement-measured-comparison",
-    benchmark: options.benchmark,
-    baselineProfileDigest: options.baselineProfileDigest,
-    candidateBundleDigest: options.candidateBundleDigest,
+    experiment,
+    measurements,
     overall: {
       name: "composite",
-      baseline: composite.baseline,
-      candidate: composite.candidate,
-      delta: composite.delta,
-      confidenceInterval: composite.confidenceInterval,
-      n: composite.n,
       direction: "higher-is-better",
-      unit: "score"
+      unit: "score",
+      ...overall
     },
-    objectives: measuredObjectives(pairs),
-    ...result.winner.label || result.winner.rationale ? {
-      candidate: {
-        ...result.winner.label ? { label: result.winner.label } : {},
-        ...result.winner.rationale ? { rationale: result.winner.rationale } : {}
-      }
-    } : {},
+    objectives,
+    ...options.candidate ? { candidate: options.candidate } : {},
     decision: {
-      outcome: result.gateDecision,
-      reasons: rebuiltProvenance.gate.reasons,
-      contributingChecks: rebuiltProvenance.gate.contributingGates.map((check) => ({
-        name: check.name,
-        passed: check.passed
-      }))
+      outcome: shipped ? "ship" : significance.fewRuns || !powerSufficient ? "need_more_work" : "hold",
+      reasons: reasons.length > 0 ? reasons : ["all measured checks passed"],
+      contributingChecks: checks
     },
     power: {
-      sufficient: power.scaleAssumed && !power.underpowered,
-      n: power.n,
-      minimumDetectableDelta: power.mde,
-      confidenceLevel: power.confidence,
-      scaleAssumed: power.scaleAssumed,
-      sharedScorerChannel: power.sharedChannelCaveat !== void 0,
-      reason: power.recommendation
+      sufficient: powerSufficient,
+      n: baselineScores.length,
+      minimumDetectableDelta: power?.mde ?? 1,
+      confidenceLevel: confidence,
+      scaleAssumed: power?.scaleAssumed ?? true,
+      sharedScorerChannel: true,
+      reason: power?.recommendation ?? `need at least ${Math.max(3, minProductiveRuns)} paired runs`
     },
     provenance: {
       kind: "agent-eval-loop",
-      schema: rebuiltProvenance.schema,
-      runId: rebuiltProvenance.runId,
-      recordDigest: rebuiltProvenance.recordDigest,
-      baselineContentHash,
-      candidateContentHash
+      schema: "agent-candidate-experiment",
+      runId: options.runId,
+      recordDigest: canonicalCandidateDigest({}),
+      baselineContentHash: experiment.baseline.digest,
+      candidateContentHash: experiment.candidate.digest
     },
-    diff: canonicalDiff,
+    diff,
     evaluation: {
-      generationsExplored,
-      durationMs: result.durationMs,
-      totalCostUsd: result.totalCostUsd
+      generationsExplored: options.generationsExplored ?? 0,
+      searchDurationMs,
+      executionDurationMs,
+      durationMs,
+      searchCostUsd,
+      executionCostUsd,
+      totalCostUsd
+    },
+    ...options.metadata ? { metadata: options.metadata } : {}
+  });
+  const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
+  return agentImprovementMeasuredComparisonSchema.parse({
+    ...provisional,
+    provenance: {
+      ...provenance,
+      recordDigest: canonicalCandidateDigest({ ...provisional, provenance })
     }
   });
 }
-function assertGenerationMeasurements(generations, receiptsById, receiptIdsByCell) {
-  for (const generation of generations) {
-    for (const measured of generation.surfaces) {
-      const candidate = generation.record.candidates.find(
-        (entry) => entry.surfaceHash === measured.surfaceHash
-      );
-      if (!candidate) {
-        throw new Error(
-          `generation ${generation.record.generationIndex} is missing candidate ${measured.surfaceHash}`
-        );
-      }
-      const composite = campaignMeanComposite(measured.campaign);
-      const breakdown = campaignBreakdown(measured.campaign);
-      const coverage = campaignCoverage(
-        measured.campaign.cells,
-        measured.campaign.scenarios,
-        measured.campaign.reps,
-        true
-      );
-      assertScorableMeasuredCells(
-        measured.campaign,
-        coverage.scorableCellIds,
-        receiptsById,
-        receiptIdsByCell
-      );
-      assertMeasuredNumber(
-        candidate.composite,
-        composite,
-        `candidate ${measured.surfaceHash} composite`
-      );
-      if (!candidate.coverage) {
-        throw new Error(`candidate ${measured.surfaceHash} does not report campaign coverage`);
-      }
-      assertMeasuredIdentity(
-        canonicalDigest(candidate.coverage),
-        canonicalDigest({
-          expectedCells: coverage.expectedCellIds.length,
-          scorableCells: coverage.scorableCellIds.length,
-          unscorableCells: coverage.unscorableCells
-        }),
-        `candidate ${measured.surfaceHash} coverage`
-      );
-      assertMeasuredIdentity(
-        String(candidate.eligibleForPromotion),
-        String(coverage.complete),
-        `candidate ${measured.surfaceHash} eligibility`
-      );
-      assertMeasuredIdentity(
-        canonicalDigest(candidate.dimensions),
-        canonicalDigest(breakdown.dimensions),
-        `candidate ${measured.surfaceHash} dimensions`
-      );
-      assertMeasuredIdentity(
-        canonicalDigest(candidate.scenarios),
-        canonicalDigest(breakdown.scenarios),
-        `candidate ${measured.surfaceHash} scenarios`
-      );
-    }
+function verifyCandidateExperimentComparison(input) {
+  const comparison = agentImprovementMeasuredComparisonSchema.parse(input);
+  const recomputed = measuredComparisonFromCandidateExperiment({
+    experiment: comparison.experiment,
+    measurements: comparison.measurements,
+    runId: comparison.provenance.runId,
+    ...comparison.candidate ? { candidate: comparison.candidate } : {},
+    generationsExplored: comparison.evaluation.generationsExplored,
+    searchDurationMs: comparison.evaluation.searchDurationMs,
+    searchCostUsd: comparison.evaluation.searchCostUsd,
+    ...comparison.metadata ? { metadata: comparison.metadata } : {}
+  });
+  if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) {
+    throw new Error("candidate experiment comparison does not match its Runtime receipts");
+  }
+  return comparison;
+}
+function deriveCandidateBundleDiff(experiment) {
+  const surfaces = ["profile", "code", "execution", "knowledge", "memory"];
+  const changed = surfaces.flatMap((surface) => {
+    const baseline = experiment.baseline[surface] ?? null;
+    const candidate = experiment.candidate[surface] ?? null;
+    const baselineDigest = canonicalCandidateDigest(baseline);
+    const candidateDigest = canonicalCandidateDigest(candidate);
+    if (baselineDigest === candidateDigest) return [];
+    return [
+      [
+        `--- baseline/${surface} (${baselineDigest})`,
+        `+++ candidate/${surface} (${candidateDigest})`,
+        JSON.stringify({ baseline, candidate }, null, 2)
+      ].join("\n")
+    ];
+  });
+  if (changed.length === 0) {
+    throw new Error("candidate experiment has no changed candidate surface");
   }
+  return changed.join("\n\n");
 }
-function assertCompleteMeasuredCampaign(campaign, name, receiptsById, receiptIdsByCell) {
-  assertCampaignSplitIdentity(campaign.scenarios, campaign.reps, campaign.splitDigest);
-  const coverage = campaignCoverage(campaign.cells, campaign.scenarios, campaign.reps, true);
-  if (!coverage.complete) {
-    throw new Error(
-      `${name} is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable)`
-    );
+function verifyCandidateBenchmarkTask(input) {
+  const task = agentCandidateBenchmarkTaskSchema.parse(input);
+  verifySelfAddressed(task, "candidate benchmark task");
+  return task;
+}
+function verifyCandidateBenchmarkSuiteInputs(input) {
+  if (input === null || typeof input !== "object" || Array.isArray(input)) {
+    throw new Error("candidate benchmark suite inputs must be an object");
   }
-  assertScorableMeasuredCells(campaign, coverage.scorableCellIds, receiptsById, receiptIdsByCell);
-}
-function assertScorableMeasuredCells(campaign, scorableCellIds, receiptsById, receiptIdsByCell) {
-  const cellsById = new Map(campaign.cells.map((cell) => [cell.cellId, cell]));
-  for (const cellId of scorableCellIds) {
-    const cell = cellsById.get(cellId);
-    if (!cell) throw new Error(`measured campaign is missing scorable cell ${cellId}`);
-    assertMeasuredCell(cell, receiptsById, receiptIdsByCell, campaign.runDir);
+  const candidate = input;
+  const suite = verifyCandidateBenchmarkSuite(candidate.suite);
+  if (!Array.isArray(candidate.tasks) || candidate.tasks.length !== suite.taskDigests.length) {
+    throw new Error("candidate benchmark suite task count does not match its signed digests");
   }
-}
-function measuredObjectives(pairs) {
-  const qualityColumns = /* @__PURE__ */ new Map();
-  for (const [baseline, candidate] of pairs) {
-    for (const cell of [baseline, candidate]) {
-      for (const [objective, score] of Object.entries(cell.judgeScores)) {
-        if (score.failed) continue;
-        qualityColumns.set(`objective:${objective}`, {
-          kind: "objective",
-          name: objective,
-          direction: "higher-is-better",
-          unit: "score"
-        });
-        for (const name of Object.keys(score.dimensions)) {
-          qualityColumns.set(`dimension:${objective}:${name}`, {
-            kind: "dimension",
-            objective,
-            name,
-            direction: "higher-is-better",
-            unit: "score"
-          });
-        }
-      }
+  candidate.tasks.forEach((task, index) => {
+    const verified = verifyCandidateBenchmarkTask(task);
+    if (verified.digest !== suite.taskDigests[index]) {
+      throw new Error(`candidate benchmark task ${index} does not match the signed suite`);
     }
-  }
-  return [
-    ...[...qualityColumns.entries()].sort(([left], [right]) => left.localeCompare(right)).map(
-      ([, column]) => measuredObjective(column, pairs, (cell) => measuredQuality(cell, column))
-    ),
-    measuredCostObjective(pairs),
-    measuredObjective(
-      {
-        kind: "latency",
-        name: "latency",
-        direction: "lower-is-better",
-        unit: "milliseconds"
-      },
-      pairs,
-      (cell) => cell.durationMs
-    )
-  ];
-}
-function measuredCostObjective(pairs) {
-  const cells = pairs.flat();
-  for (const cell of cells) finiteMeasuredValue(cell.costUsd, "cost:cost");
-  return measuredObjective(
-    {
-      kind: "cost",
-      name: "cost",
-      direction: "lower-is-better",
-      unit: "usd"
-    },
-    pairs,
-    (cell) => cell.costUsd
+  });
+  return { suite, tasks: candidate.tasks };
+}
+function verifyCandidateBenchmarkSuite(input) {
+  const suite = agentCandidateBenchmarkSuiteSchema.parse(input);
+  verifySelfAddressed(suite, "candidate benchmark suite");
+  return suite;
+}
+function verifyBundle(input, label) {
+  const bundle = agentCandidateBundleSchema.parse(input);
+  verifySelfAddressed(bundle, label);
+  return bundle;
+}
+function verifyMeasurement(experiment, input, index) {
+  const suite = experiment.benchmark.suite;
+  const taskIndex = Math.floor(index / suite.reps);
+  const repetition = index % suite.reps;
+  const task = experiment.benchmark.tasks[taskIndex];
+  const seed = suite.seeds[index];
+  if (!task || seed === void 0) {
+    throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
+  }
+  const baseline = verifyExecutionEvidence(input.baseline);
+  const candidate = verifyExecutionEvidence(input.candidate);
+  for (const [arm, evidence] of [
+    ["baseline", baseline],
+    ["candidate", candidate]
+  ]) {
+    const bundle = experiment[arm];
+    const materialization = evidence.materializationReceipt;
+    const plan = materialization.executionPlan.material;
+    const runCell = plan.runCell;
+    verifySelfAddressed(runCell, "candidate run cell");
+    if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) {
+      throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
+    }
+    verifyTaskOutcome(task, evidence, index, arm);
+  }
+  const baselinePlan = baseline.materializationReceipt.executionPlan.material;
+  const candidatePlan = candidate.materializationReceipt.executionPlan.material;
+  if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) {
+    throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
+  }
+  return { baseline, candidate };
+}
+function verifyExecutionEvidence(input) {
+  const evidence = candidateExecutionEvidenceSchema.parse(input);
+  verifySelfAddressed(evidence, "candidate execution evidence");
+  verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
+  verifySelfAddressed(
+    evidence.materializationReceipt.profileActivation,
+    "candidate profile activation"
   );
-}
-function measuredComposite(cell) {
-  const values = Object.values(cell.judgeScores).filter((score) => !score.failed).map((score) => score.composite).filter(Number.isFinite);
-  if (values.length === 0) {
-    throw new Error(`heldout cell ${measuredCellKey(cell)} has no successful composite score`);
-  }
-  return measuredMean(values);
-}
-function pairMeasuredCells(baselineCells, candidateCells) {
-  const cells = [...baselineCells, ...candidateCells];
-  const errors = cells.filter((cell) => cell.error);
-  if (errors.length > 0) {
-    throw new Error(
-      `measured objectives cannot publish ${errors.length} errored heldout cells: ${errors.map((cell) => measuredCellKey(cell)).join(", ")}`
-    );
-  }
-  const rows = [
-    ...baselineCells.map((cell) => ({
-      pairKey: cell.scenarioId,
-      repKey: String(cell.rep),
-      arm: "baseline",
-      cell
-    })),
-    ...candidateCells.map((cell) => ({
-      pairKey: cell.scenarioId,
-      repKey: String(cell.rep),
-      arm: "candidate",
-      cell
-    }))
+  verifyMaterialAddressed(
+    evidence.materializationReceipt.profileActivation.profilePlan,
+    "candidate profile plan"
+  );
+  verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
+  verifySelfAddressed(evidence.receipt, "candidate run receipt");
+  verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
+  verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
+  verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
+  return evidence;
+}
+function verifySelfAddressed(document, label) {
+  if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) {
+    throw new Error(`${label} digest is invalid`);
+  }
+}
+function verifyTaskOutcome(task, evidence, index, arm) {
+  const outcome = evidence.receipt.taskOutcome.material.outcome;
+  const result = evidence.receipt.benchmarkResult.material;
+  const prefix = `candidate experiment measurement ${index} ${arm}`;
+  if (result.evidence.sha256 === task.grader.artifact.sha256) {
+    throw new Error(`${prefix} reused grader bytes as grading evidence`);
+  }
+  const usage = combinedUsage(evidence);
+  const usageChecks = [
+    [usage.modelCalls, task.limits.maxModelCalls, "model calls"],
+    [usage.inputTokens, task.limits.maxInputTokens, "input tokens"],
+    [usage.outputTokens, task.limits.maxOutputTokens, "output tokens"],
+    [usage.costUsdNanos, Math.round(task.limits.maxCostUsd * 1e9), "cost"]
   ];
-  const paired = pairArms(rows, { baselineArm: "baseline", treatmentArm: "candidate" });
-  if (paired.pairs.length === 0 || paired.unpairedBaseline.length > 0 || paired.unpairedTreatment.length > 0) {
-    throw new Error("measured objectives require the same non-empty paired heldout cells");
-  }
-  return paired.pairs.map((pair) => {
-    const baseline = pair.baseline.cell;
-    const candidate = pair.treatment.cell;
-    if (baseline.seed !== candidate.seed) {
-      throw new Error(`heldout cell ${baseline.cellId} does not share one paired seed`);
+  for (const [actual, maximum, label] of usageChecks) {
+    if (actual > maximum) {
+      throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
     }
-    return [baseline, candidate];
-  });
-}
-function assertMeasuredCell(cell, receiptsById, receiptIdsByCell, runDir) {
-  if (!cell.scenarioId.trim() || !Number.isSafeInteger(cell.rep) || cell.rep < 0 || !Number.isSafeInteger(cell.seed) || cell.cellId !== measuredCellKey(cell)) {
-    throw new Error(`heldout cell '${cell.cellId}' has an invalid scenario/rep identity`);
   }
-  nonnegativeMeasuredValue(cell.costUsd, `heldout cell ${cell.cellId} cost`);
-  nonnegativeMeasuredValue(cell.durationMs, `heldout cell ${cell.cellId} latency`);
-  if (!cell.tokenUsage) {
-    throw new Error(`heldout cell ${cell.cellId} does not report token usage`);
+  if (outcome.kind !== task.outcome.kind) {
+    throw new Error(`${prefix} returned an outcome outside the signed task contract`);
   }
-  nonnegativeMeasuredInteger(cell.tokenUsage.input, `heldout cell ${cell.cellId} input tokens`);
-  nonnegativeMeasuredInteger(cell.tokenUsage.output, `heldout cell ${cell.cellId} output tokens`);
-  if (cell.tokenUsage.cached !== void 0) {
-    nonnegativeMeasuredInteger(cell.tokenUsage.cached, `heldout cell ${cell.cellId} cached tokens`);
-  }
-  if (!Array.isArray(cell.costCallIds)) {
-    throw new Error(`heldout cell ${cell.cellId} does not identify its cost receipts`);
+  if (task.outcome.kind === "output") {
+    if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) {
+      throw new Error(`${prefix} changed the signed output contract`);
+    }
+    return;
   }
-  if (new Set(cell.costCallIds).size !== cell.costCallIds.length) {
-    throw new Error(`heldout cell ${cell.cellId} repeats a cost receipt`);
+  const repository = task.repository;
+  if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) {
+    throw new Error(`${prefix} did not start from the signed repository state`);
   }
-  const linkedReceipts = cell.costCallIds.map((callId) => {
-    const receipt = receiptsById.get(callId);
-    if (!receipt)
-      throw new Error(`heldout cell ${cell.cellId} references missing cost receipt ${callId}`);
-    if (receipt.tags?.runDir !== runDir || receipt.tags.cellId !== cell.cellId || receipt.tags.scenarioId !== cell.scenarioId || receipt.tags.rep !== String(cell.rep)) {
-      throw new Error(`heldout cell ${cell.cellId} references a cost receipt from another cell`);
+}
+function verifyStableProfileMaterialization(measurements) {
+  for (const arm of ["baseline", "candidate"]) {
+    const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
+    if (!expected) throw new Error("candidate experiment contains no profile materialization");
+    const expectedDigest = canonicalCandidateDigest({
+      profilePlanDigest: expected.profilePlan.digest,
+      files: expected.files
+    });
+    for (const [index, measurement] of measurements.entries()) {
+      const activation = measurement[arm].materializationReceipt.profileActivation;
+      if (canonicalCandidateDigest({
+        profilePlanDigest: activation.profilePlan.digest,
+        files: activation.files
+      }) !== expectedDigest) {
+        throw new Error(
+          `candidate experiment measurement ${index} ${arm} materialized a different profile`
+        );
+      }
     }
-    return receipt;
-  });
-  assertMeasuredIdentity(
-    canonicalDigest([...cell.costCallIds].sort()),
-    canonicalDigest(
-      [
-        ...receiptIdsByCell.get(receiptCellKey(runDir, cell.cellId, cell.scenarioId, cell.rep)) ?? []
-      ].sort()
-    ),
-    `heldout cell ${cell.cellId} cost receipt IDs`
-  );
-  const agentReceipts = linkedReceipts.filter((receipt) => receipt.channel === "agent");
-  if (agentReceipts.some((receipt) => receipt.error)) {
-    throw new Error(`measured cell ${cell.cellId} links a failed agent receipt`);
   }
-  assertRealAgentReceipts(agentReceipts, { allowMixed: false });
-  assertMeasuredNumber(
-    cell.costUsd,
-    agentReceipts.reduce((total, receipt) => total + receipt.costUsd, 0),
-    `heldout cell ${cell.cellId} cost receipts`
-  );
-  assertMeasuredNumber(
-    cell.tokenUsage.input,
-    agentReceipts.reduce((total, receipt) => total + receipt.inputTokens, 0),
-    `heldout cell ${cell.cellId} input receipts`
-  );
-  assertMeasuredNumber(
-    cell.tokenUsage.output,
-    agentReceipts.reduce((total, receipt) => total + receipt.outputTokens, 0),
-    `heldout cell ${cell.cellId} output receipts`
-  );
-  assertMeasuredNumber(
-    cell.tokenUsage.cached ?? 0,
-    agentReceipts.reduce((total, receipt) => total + (receipt.cachedTokens ?? 0), 0),
-    `heldout cell ${cell.cellId} cached receipts`
-  );
-  for (const [judge, score] of Object.entries(cell.judgeScores)) {
-    if (score.failed) {
-      throw new Error(`heldout cell ${cell.cellId} contains failed judge '${judge}'`);
-    }
-    finiteMeasuredValue(score.composite, `objective:${judge}`);
-    for (const [dimension, value] of Object.entries(score.dimensions)) {
-      finiteMeasuredValue(value, `dimension:${judge}:${dimension}`);
-    }
+}
+function completedSuccessfully(evidence) {
+  const termination = evidence.receipt.termination;
+  return termination.kind === "exit" && termination.exitCode === 0;
+}
+function verifyMaterialAddressed(evidence, label) {
+  if (canonicalCandidateDigest(evidence.material) !== evidence.digest) {
+    throw new Error(`${label} digest is invalid`);
   }
-  measuredComposite(cell);
-}
-function measuredObjective(identity, pairs, value) {
-  const key = measuredObjectiveKey(identity);
-  const baseline = pairs.map(([cell]) => finiteMeasuredValue(value(cell), key));
-  const candidate = pairs.map(([, cell]) => finiteMeasuredValue(value(cell), key));
-  const interval = pairedBootstrap(baseline, candidate, {
-    confidence: 0.95,
-    resamples: 2e3,
+}
+function measuredEstimate(baseline, candidate, options) {
+  const bootstrap = pairedBootstrap(baseline, candidate, {
+    confidence: options.confidence,
+    resamples: options.resamples,
     statistic: "mean",
-    seed: measuredSeed(key)
+    seed: options.seed
   });
-  const baselineMean = measuredMean(baseline);
-  const candidateMean = measuredMean(candidate);
+  const baselineMean = mean(baseline);
+  const candidateMean = mean(candidate);
+  const delta = candidateMean - baselineMean;
   return {
-    ...identity,
-    availability: "measured",
     baseline: baselineMean,
     candidate: candidateMean,
-    delta: candidateMean - baselineMean,
+    delta,
     confidenceInterval: {
-      level: interval.confidence,
-      lower: interval.low,
-      upper: interval.high,
+      level: bootstrap.confidence,
+      lower: Math.min(bootstrap.low, delta),
+      upper: Math.max(bootstrap.high, delta),
       method: "paired-bootstrap",
       statistic: "mean",
-      resamples: interval.resamples
+      resamples: bootstrap.resamples
     },
-    n: interval.n
+    n: bootstrap.n
   };
 }
-function measuredQuality(cell, column) {
-  const objective = column.kind === "objective" ? column.name : column.objective;
-  const score = cell.judgeScores[objective];
-  if (!score || score.failed) {
-    throw new Error(
-      `heldout cell ${measuredCellKey(cell)} is missing measured objective '${objective}'`
-    );
-  }
-  const value = column.kind === "objective" ? score.composite : score.dimensions[column.name];
-  if (value === void 0) {
-    throw new Error(
-      `heldout cell ${measuredCellKey(cell)} is missing '${objective}' dimension '${column.name}'`
-    );
+function sharedDimensions(measurements) {
+  const expected = dimensionNames(measurements[0]?.baseline);
+  for (const [index, measurement] of measurements.entries()) {
+    for (const [arm, evidence] of [
+      ["baseline", measurement.baseline],
+      ["candidate", measurement.candidate]
+    ]) {
+      const actual = dimensionNames(evidence);
+      if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+        throw new Error(
+          `candidate experiment measurement ${index} ${arm} dimensions do not match the suite`
+        );
+      }
+    }
   }
-  return finiteMeasuredValue(value, measuredObjectiveKey(column));
+  return expected;
 }
-function measuredObjectiveKey(identity) {
-  return identity.kind === "dimension" ? `${identity.kind}:${identity.objective}:${identity.name}` : `${identity.kind}:${identity.name}`;
+function dimensionNames(evidence) {
+  return (evidence?.receipt.benchmarkResult.material.dimensions ?? []).map(
+    (dimension) => dimension.name
+  );
 }
-function measuredCellKey(cell) {
-  return `${cell.scenarioId}:${cell.rep}`;
+function scoreOf(arm) {
+  return (measurement) => measurement[arm].receipt.benchmarkResult.material.score;
 }
-function finiteMeasuredValue(value, name) {
-  if (!Number.isFinite(value)) throw new Error(`measured objective '${name}' is not finite`);
-  return value;
+function dimensionOf(arm, name) {
+  return (measurement) => {
+    const dimension = measurement[arm].receipt.benchmarkResult.material.dimensions.find(
+      (entry) => entry.name === name
+    );
+    if (!dimension) throw new Error(`candidate experiment is missing dimension '${name}'`);
+    return dimension.score;
+  };
 }
-function nonnegativeMeasuredValue(value, name) {
-  if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be non-negative`);
-  return value;
+function costOf(arm) {
+  return (measurement) => costFromEvidence(measurement[arm]);
 }
-function nonnegativeMeasuredInteger(value, name) {
-  if (!Number.isSafeInteger(value) || value < 0) {
-    throw new Error(`${name} must be a non-negative safe integer`);
-  }
-  return value;
+function latencyOf(arm) {
+  return (measurement) => latencyFromEvidence(measurement[arm]);
 }
-function measuredMean(values) {
-  if (values.length === 0) throw new Error("measured objective has no paired values");
-  return values.reduce((sum, value) => sum + value, 0) / values.length;
+function costFromEvidence(evidence) {
+  return combinedUsage(evidence).costUsdNanos / 1e9;
 }
-function measuredSeed(value) {
-  let seed = 2166136261;
-  for (const byte of Buffer.from(value, "utf8")) {
-    seed = Math.imul(seed ^ byte, 16777619) >>> 0;
-  }
-  return seed;
+function latencyFromEvidence(evidence) {
+  return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
 }
-function assertMeasuredNumber(actual, expected, name) {
-  const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)) * 8;
-  if (!Number.isFinite(actual) || !Number.isFinite(expected) || Math.abs(actual - expected) > tolerance) {
-    throw new Error(`${name} does not agree across the measured comparison`);
-  }
+function combinedUsage(evidence) {
+  const candidate = evidence.receipt.modelSettlement.material.usage;
+  const grader = evidence.receipt.benchmarkResult.material.grading.usage;
+  return {
+    inputTokens: candidate.inputTokens + grader.inputTokens,
+    outputTokens: candidate.outputTokens + grader.outputTokens,
+    cachedInputTokens: candidate.cachedInputTokens + grader.cachedInputTokens,
+    reasoningTokens: candidate.reasoningTokens + grader.reasoningTokens,
+    modelCalls: candidate.modelCalls + grader.modelCalls,
+    costUsdNanos: candidate.costUsdNanos + grader.costUsdNanos
+  };
 }
-function assertMeasuredIdentity(actual, expected, name) {
-  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
-}
-function assertMeasuredOptional(actual, expected, name) {
-  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
-}
-function indexReceiptIdsByCell(receipts) {
-  const idsByCell = /* @__PURE__ */ new Map();
-  for (const receipt of receipts) {
-    const tags = receipt.tags;
-    if (!tags?.runDir || !tags.cellId || !tags.scenarioId || tags.rep === void 0) continue;
-    const key = receiptCellKey(tags.runDir, tags.cellId, tags.scenarioId, tags.rep);
-    const ids = idsByCell.get(key) ?? [];
-    ids.push(receipt.callId);
-    idsByCell.set(key, ids);
-  }
-  return idsByCell;
+function cellIds(experiment) {
+  const { suite, tasks } = experiment.benchmark;
+  return suite.seeds.map((_, index) => {
+    const taskIndex = Math.floor(index / suite.reps);
+    const repetition = index % suite.reps;
+    return `${tasks[taskIndex]?.scenario.id ?? taskIndex}:${repetition}`;
+  });
+}
+function mean(values) {
+  if (values.length === 0) throw new Error("candidate experiment requires measured values");
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
 }
-function receiptCellKey(runDir, cellId, scenarioId, rep) {
-  return JSON.stringify([runDir, cellId, scenarioId, String(rep)]);
+function abortError(signal) {
+  return signal.reason instanceof Error ? signal.reason : new Error("candidate experiment aborted");
 }
 
 // src/contract/eval-reporting-suite.ts
@@ -1638,17 +1678,27 @@ export {
   gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
-  measuredComparisonFromSelfImproveResult,
+  measuredComparisonFromCandidateExperiment,
+  observeCodeAgentSession,
   paretoPolicy,
   paretoSignificanceGate,
   parseAgentTrace,
   parseCodeAgentJsonl,
   partitionRunsByAuthoringModel,
   runCampaign,
+  runCandidateExperiment,
   runEval,
   runImprovementLoop,
   runReferenceEquivalenceJudge,
+  sealCandidateBenchmarkSuite,
+  sealCandidateBenchmarkTask,
+  sealCandidateExperiment,
   selfImprove,
-  summarizeExecution
+  summarizeExecution,
+  verifyCandidateBenchmarkSuite,
+  verifyCandidateBenchmarkSuiteInputs,
+  verifyCandidateBenchmarkTask,
+  verifyCandidateExperiment,
+  verifyCandidateExperimentComparison
 };
 //# sourceMappingURL=index.js.map