@tangle-network/agent-eval 0.119.0 → 0.119.1

package/dist/contract/index.d.ts

@@ -1,3 +1,4 @@
+import { AgentImprovementMeasuredComparison, Sha256Digest } from '@tangle-network/agent-interface';
 import { AxFunction, AxAIService } from '@ax-llm/ax';
 import { z } from 'zod';
 
@@ -1122,6 +1123,8 @@ interface AnalystCost {
     est_usd_per_run?: number;
     /** Models the analyst expects to use (informational). */
     models?: string[];
+    /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
+    settlement_timeout_ms?: number;
 }
 interface AnalystRequirements {
     /** Min number of shots / samples the analyst needs to produce signal. */
@@ -1145,7 +1148,7 @@ interface AnalystContext {
     runId: string;
     /** Stable correlation id so logs from a single registry.run() share a tag. */
     correlationId: string;
-    /** Wall-clock deadline (epoch ms). Analysts SHOULD honor for graceful cancel. */
+    /** Enforced wall-clock deadline (epoch ms). */
     deadlineMs?: number;
     /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
     budgetUsd?: number;
@@ -4000,6 +4003,15 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
  */
 declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
 
+interface MeasuredComparisonFromSelfImproveResultOptions<TScenario extends Scenario, TArtifact> {
+    result: SelfImproveResult<TScenario, TArtifact>;
+    benchmark: AgentImprovementMeasuredComparison['benchmark'];
+    baselineProfileDigest: Sha256Digest;
+    candidateBundleDigest: Sha256Digest;
+}
+/** Convert one paired self-improvement result into the portable Interface evidence record. */
+declare function measuredComparisonFromSelfImproveResult<TScenario extends Scenario, TArtifact>(options: MeasuredComparisonFromSelfImproveResultOptions<TScenario, TArtifact>): AgentImprovementMeasuredComparison;
+
 /**
  * Typed Ax output for analyst findings.
  *
@@ -4065,8 +4077,8 @@ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchem
  * Analyst-kind factory — the typed way to define trace analysts.
  *
  * A "kind" is a specialized analyst whose actor prompt, tool subset,
- * and Ax recursion config target one failure-mode lens (failure-mode
- * classification, knowledge gap discovery, knowledge poisoning, recursive
+ * and bounded Ax subqueries target one failure-mode lens (failure-mode
+ * classification, knowledge gap discovery, knowledge poisoning,
  * self-improvement, ...). Kinds emit findings in the typed
  * `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
  * validates each row with Zod and lifts it into `AnalystFinding[]`.
@@ -4077,12 +4089,11 @@ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchem
  *   - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
  *     A kind that never needs full-trace dumps can drop `viewTrace` /
  *     `viewSpans` and stay cheap.
- *   - Each kind declares its recursion + parallelism budget. Discovery-
- *     heavy kinds (failure-mode) get higher `maxDepth`; lens kinds
- *     (poisoning) usually stay at 0 since they have a tighter brief.
+ *   - Each kind declares its subquery + parallelism budget. Discovery-heavy
+ *     kinds can fan out more bounded semantic questions than narrow lenses.
  *
  * Optimizer hook: kinds may declare `goldens` — labeled examples used
- * by `AxMiPRO` / `AxBootstrapFewShot` / `AxGEPA` to fit the actor
+ * by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
  * description programmatically. Stored on the kind, not the registry,
  * because the right metric is kind-specific.
  */
@@ -4102,20 +4113,18 @@ interface TraceAnalystKindSpec {
     version: string;
     /** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
     actorDescription: string;
-    /** Responder system prompt; falls back to a minimal "format the findings" instruction. */
-    responderDescription?: string;
     /** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
     buildTools: (store: TraceAnalysisStore) => AxFunction[];
-    /** Recursion budget. `maxDepth: 0` disables subagents. */
-    recursion?: {
-        maxDepth: number;
-        maxParallelSubagents?: number;
+    /** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
+    subqueries?: {
+        maxCalls: number;
+        maxParallel?: number;
     };
     /** Actor turn cap. Default 12. */
     maxTurns?: number;
     /** Runtime char cap. Default 6000. */
     maxRuntimeChars?: number;
-    /** Maximum output tokens for every actor, responder, and recursive model call. Default 4096. */
+    /** Maximum output tokens for every actor and subquery model call. Default 4096. */
     maxOutputTokens?: number;
     /** Cost classification surfaced in `registry.list()` and budget enforcement. */
     cost: AnalystCost;
@@ -4219,7 +4228,7 @@ interface RegistryRunOpts {
     skip?: string[];
     /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
     budget?: BudgetPolicy;
-    /** Wall-clock cap. Analysts SHOULD honor `ctx.deadlineMs`. */
+    /** Active-work cap for the complete registry run. Model receipt settlement may follow. */
     timeoutMs?: number;
     /** Abort signal — forwarded into every analyst's context. */
     signal?: AbortSignal;
@@ -4933,4 +4942,4 @@ interface FromOtelSpansOptions {
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
+export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };

package/dist/contract/index.js

@@ -26,6 +26,7 @@ import {
   evolutionaryProposer,
   gepaProposer,
   heldOutGate,
+  pairArms,
   paretoPolicy,
   paretoSignificanceGate,
   powerPreflight,
@@ -34,8 +35,9 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-RLCJQ6VZ.js";
+} from "../chunk-PSXJ32SR.js";
 import {
+  canonicalJson,
   createRunCostLedger,
   fsCampaignStorage,
   inMemoryCampaignStorage,
@@ -45,9 +47,9 @@ import {
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-GERDAIAL.js";
+} from "../chunk-QWMPPZ3X.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-7A4LIMMY.js";
+import "../chunk-S5TT5R3L.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
@@ -55,7 +57,9 @@ import {
 import "../chunk-ARU2PZFM.js";
 import "../chunk-NJC7U437.js";
 import "../chunk-DPZAEKA6.js";
-import "../chunk-PJQFMIOX.js";
+import {
+  pairedBootstrap
+} from "../chunk-PJQFMIOX.js";
 import "../chunk-JHOJHHU7.js";
 import "../chunk-VI2UW6B6.js";
 import {
@@ -65,7 +69,7 @@ import {
 import "../chunk-PXE2VKMX.js";
 import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-6QIM2EAP.js";
+import "../chunk-PXD6ZFNY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
@@ -536,6 +540,277 @@ function requirePositiveInteger(value, field) {
   return value;
 }
 
+// src/contract/measured-comparison.ts
+import { createHash as createHash2 } from "crypto";
+import {
+  agentImprovementMeasuredComparisonSchema
+} from "@tangle-network/agent-interface";
+function measuredComparisonFromSelfImproveResult(options) {
+  const { result } = options;
+  const power = result.power;
+  if (!power) throw new Error("agent improvement comparison requires heldout power analysis");
+  if (result.provenance.gate.reasons.length === 0) {
+    throw new Error("agent improvement comparison requires measured decision reasons");
+  }
+  const pairs = pairMeasuredCells(
+    result.raw.baselineOnHoldout.cells,
+    result.raw.winnerOnHoldout.cells
+  );
+  const composite = measuredObjective(
+    {
+      kind: "objective",
+      name: "composite",
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    pairs,
+    measuredComposite
+  );
+  assertMeasuredNumber(result.lift, composite.delta, "heldout lift");
+  assertMeasuredNumber(result.baseline.compositeMean, composite.baseline, "heldout baseline");
+  assertMeasuredNumber(result.winner.compositeMean, composite.candidate, "heldout candidate");
+  assertMeasuredNumber(result.provenance.heldOutLift, composite.delta, "provenance heldout lift");
+  if (result.gateDecision !== result.provenance.gate.decision || power.n !== composite.n || power.confidence !== 0.95) {
+    throw new Error("agent improvement measurement sources do not agree");
+  }
+  return agentImprovementMeasuredComparisonSchema.parse({
+    schemaVersion: 1,
+    kind: "agent-improvement-measured-comparison",
+    benchmark: options.benchmark,
+    baselineProfileDigest: options.baselineProfileDigest,
+    candidateBundleDigest: options.candidateBundleDigest,
+    overall: {
+      name: "composite",
+      baseline: composite.baseline,
+      candidate: composite.candidate,
+      delta: composite.delta,
+      confidenceInterval: composite.confidenceInterval,
+      n: composite.n,
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    objectives: measuredObjectives(pairs),
+    ...result.winner.label || result.winner.rationale ? {
+      candidate: {
+        ...result.winner.label ? { label: result.winner.label } : {},
+        ...result.winner.rationale ? { rationale: result.winner.rationale } : {}
+      }
+    } : {},
+    decision: {
+      outcome: result.gateDecision,
+      reasons: result.provenance.gate.reasons,
+      contributingChecks: result.provenance.gate.contributingGates.map((check) => ({
+        name: check.name,
+        passed: check.passed
+      }))
+    },
+    power: {
+      sufficient: power.scaleAssumed && !power.underpowered,
+      n: power.n,
+      minimumDetectableDelta: power.mde,
+      confidenceLevel: power.confidence,
+      scaleAssumed: power.scaleAssumed,
+      sharedScorerChannel: power.sharedChannelCaveat !== void 0,
+      reason: power.recommendation
+    },
+    provenance: {
+      kind: "agent-eval-loop",
+      schema: result.provenance.schema,
+      runId: result.provenance.runId,
+      recordDigest: digest(result.provenance),
+      baselineContentHash: result.provenance.baselineContentHash,
+      candidateContentHash: result.provenance.winnerContentHash
+    },
+    diff: result.diff,
+    evaluation: {
+      generationsExplored: result.generationsExplored,
+      durationMs: result.durationMs,
+      totalCostUsd: result.totalCostUsd
+    }
+  });
+}
+function measuredObjectives(pairs) {
+  const qualityColumns = /* @__PURE__ */ new Map();
+  for (const [baseline, candidate] of pairs) {
+    for (const cell of [baseline, candidate]) {
+      for (const [objective, score] of Object.entries(cell.judgeScores)) {
+        if (score.failed) continue;
+        qualityColumns.set(`objective:${objective}`, {
+          kind: "objective",
+          name: objective,
+          direction: "higher-is-better",
+          unit: "score"
+        });
+        for (const name of Object.keys(score.dimensions)) {
+          qualityColumns.set(`dimension:${objective}:${name}`, {
+            kind: "dimension",
+            objective,
+            name,
+            direction: "higher-is-better",
+            unit: "score"
+          });
+        }
+      }
+    }
+  }
+  return [
+    ...[...qualityColumns.entries()].sort(([left], [right]) => left.localeCompare(right)).map(
+      ([, column]) => measuredObjective(column, pairs, (cell) => measuredQuality(cell, column))
+    ),
+    measuredCostObjective(pairs),
+    measuredObjective(
+      {
+        kind: "latency",
+        name: "latency",
+        direction: "lower-is-better",
+        unit: "milliseconds"
+      },
+      pairs,
+      (cell) => cell.durationMs
+    )
+  ];
+}
+function measuredCostObjective(pairs) {
+  const cells = pairs.flat();
+  for (const cell of cells) finiteMeasuredValue(cell.costUsd, "cost:cost");
+  const reported = cells.some(
+    (cell) => cell.costUsd !== 0 || cell.tokenUsage !== void 0 && (cell.tokenUsage.input > 0 || cell.tokenUsage.output > 0)
+  );
+  if (!reported) {
+    return {
+      kind: "cost",
+      name: "cost",
+      availability: "unavailable",
+      reason: "heldout cells did not report model usage or cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    };
+  }
+  return measuredObjective(
+    {
+      kind: "cost",
+      name: "cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    },
+    pairs,
+    (cell) => cell.costUsd
+  );
+}
+function measuredComposite(cell) {
+  const values = Object.values(cell.judgeScores).filter((score) => !score.failed).map((score) => score.composite).filter(Number.isFinite);
+  if (values.length === 0) {
+    throw new Error(`heldout cell ${measuredCellKey(cell)} has no successful composite score`);
+  }
+  return measuredMean(values);
+}
+function pairMeasuredCells(baselineCells, candidateCells) {
+  const errors = [...baselineCells, ...candidateCells].filter((cell) => cell.error);
+  if (errors.length > 0) {
+    throw new Error(
+      `measured objectives cannot publish ${errors.length} errored heldout cells: ${errors.map((cell) => measuredCellKey(cell)).join(", ")}`
+    );
+  }
+  const rows = [
+    ...baselineCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "baseline",
+      cell
+    })),
+    ...candidateCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "candidate",
+      cell
+    }))
+  ];
+  const paired = pairArms(rows, { baselineArm: "baseline", treatmentArm: "candidate" });
+  if (paired.pairs.length === 0 || paired.unpairedBaseline.length > 0 || paired.unpairedTreatment.length > 0) {
+    throw new Error("measured objectives require the same non-empty paired heldout cells");
+  }
+  return paired.pairs.map(
+    (pair) => [pair.baseline.cell, pair.treatment.cell]
+  );
+}
+function measuredObjective(identity, pairs, value) {
+  const key = measuredObjectiveKey(identity);
+  const baseline = pairs.map(([cell]) => finiteMeasuredValue(value(cell), key));
+  const candidate = pairs.map(([, cell]) => finiteMeasuredValue(value(cell), key));
+  const interval = pairedBootstrap(baseline, candidate, {
+    confidence: 0.95,
+    resamples: 2e3,
+    statistic: "mean",
+    seed: measuredSeed(key)
+  });
+  const baselineMean = measuredMean(baseline);
+  const candidateMean = measuredMean(candidate);
+  return {
+    ...identity,
+    availability: "measured",
+    baseline: baselineMean,
+    candidate: candidateMean,
+    delta: candidateMean - baselineMean,
+    confidenceInterval: {
+      level: interval.confidence,
+      lower: interval.low,
+      upper: interval.high,
+      method: "paired-bootstrap",
+      statistic: "mean",
+      resamples: interval.resamples
+    },
+    n: interval.n
+  };
+}
+function measuredQuality(cell, column) {
+  const objective = column.kind === "objective" ? column.name : column.objective;
+  const score = cell.judgeScores[objective];
+  if (!score || score.failed) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing measured objective '${objective}'`
+    );
+  }
+  const value = column.kind === "objective" ? score.composite : score.dimensions[column.name];
+  if (value === void 0) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing '${objective}' dimension '${column.name}'`
+    );
+  }
+  return finiteMeasuredValue(value, measuredObjectiveKey(column));
+}
+function measuredObjectiveKey(identity) {
+  return identity.kind === "dimension" ? `${identity.kind}:${identity.objective}:${identity.name}` : `${identity.kind}:${identity.name}`;
+}
+function measuredCellKey(cell) {
+  return `${cell.scenarioId}:${cell.rep}`;
+}
+function finiteMeasuredValue(value, name) {
+  if (!Number.isFinite(value)) throw new Error(`measured objective '${name}' is not finite`);
+  return value;
+}
+function measuredMean(values) {
+  if (values.length === 0) throw new Error("measured objective has no paired values");
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function measuredSeed(value) {
+  let seed = 2166136261;
+  for (const byte of Buffer.from(value, "utf8")) {
+    seed = Math.imul(seed ^ byte, 16777619) >>> 0;
+  }
+  return seed;
+}
+function assertMeasuredNumber(actual, expected, name) {
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)) * 8;
+  if (!Number.isFinite(actual) || !Number.isFinite(expected) || Math.abs(actual - expected) > tolerance) {
+    throw new Error(`${name} does not agree across the measured comparison`);
+  }
+}
+function digest(value) {
+  const json = JSON.stringify(value);
+  if (json === void 0) throw new Error("agent improvement provenance is not serializable");
+  return `sha256:${createHash2("sha256").update(canonicalJson(JSON.parse(json))).digest("hex")}`;
+}
+
 // src/contract/eval-reporting-suite.ts
 import { mkdir, writeFile } from "fs/promises";
 import { dirname, join as join2 } from "path";
@@ -1074,6 +1349,7 @@ export {
   gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
+  measuredComparisonFromSelfImproveResult,
   paretoPolicy,
   paretoSignificanceGate,
   parseAgentTrace,