@tangle-network/agent-eval 0.120.2 → 0.120.3

package/dist/contract/index.js

@@ -0,0 +1,1654 @@
+import {
+  fromClaudeCodeSession,
+  fromCodexSession,
+  fromKimiCodeSession,
+  fromOpenCodeSession,
+  fromPiSession,
+  fromPigraphSession,
+  parseCodeAgentJsonl
+} from "../chunk-HKUCJ437.js";
+import {
+  createHostedClient
+} from "../chunk-ZZUXHH3R.js";
+import {
+  analyzeRuns,
+  summarizeExecution
+} from "../chunk-SQQED7ZH.js";
+import {
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  buildEvidenceVector,
+  buildLoopProvenanceRecord,
+  campaignBreakdown,
+  campaignMeanComposite,
+  canonicalDigest,
+  composeGate,
+  createReferenceEquivalenceJudge,
+  defaultProductionGate,
+  emitLoopProvenance,
+  evolutionaryProposer,
+  gepaProposer,
+  heldOutGate,
+  loopProvenanceArgsFromResult,
+  pairArms,
+  paretoPolicy,
+  paretoSignificanceGate,
+  powerPreflight,
+  renderSurfaceDiff,
+  runEval,
+  runImprovementLoop,
+  runReferenceEquivalenceJudge,
+  surfaceContentHash,
+  surfaceHash,
+  verifyLoopProvenanceRecord
+} from "../chunk-3CDFMEMO.js";
+import {
+  assertCampaignSplitIdentity,
+  assertRealAgentReceipts,
+  campaignCoverage,
+  campaignSplitDigest,
+  createRunCostLedger,
+  fsCampaignStorage,
+  inMemoryCampaignStorage,
+  resolveRunDir,
+  runCampaign
+} from "../chunk-SYV364BL.js";
+import {
+  buildDefaultAnalystRegistry,
+  createChatClient
+} from "../chunk-5BYTIDZ7.js";
+import "../chunk-HHWE3POT.js";
+import "../chunk-5CVUPHJ4.js";
+import {
+  FileSystemOutcomeStore,
+  InMemoryOutcomeStore
+} from "../chunk-3RF76KTD.js";
+import "../chunk-ARU2PZFM.js";
+import "../chunk-NJC7U437.js";
+import "../chunk-DPZAEKA6.js";
+import {
+  pairedBootstrap
+} from "../chunk-PJQFMIOX.js";
+import {
+  CostLedger
+} from "../chunk-ZMXDQ4K7.js";
+import "../chunk-VI2UW6B6.js";
+import {
+  recordAggregateMeasurements,
+  summarizeExecutionMeasurements
+} from "../chunk-H5UD2323.js";
+import "../chunk-PXE2VKMX.js";
+import "../chunk-ZET2UAYW.js";
+import "../chunk-GGE4NNQT.js";
+import "../chunk-PXD6ZFNY.js";
+import "../chunk-PC4UYEBM.js";
+import {
+  parseRunRecordSafe
+} from "../chunk-S3UZOQ5Y.js";
+import "../chunk-MA6HLL3S.js";
+import "../chunk-XJYR7XFV.js";
+import "../chunk-VSMTAMNK.js";
+import "../chunk-ONWEPEDO.js";
+import {
+  LLM_MODEL_ATTR_KEYS
+} from "../chunk-K4DBDHLK.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/contract/self-improve.ts
+import { createHash } from "crypto";
+var SelfImproveRunError = class extends Error {
+  cost;
+  receipts;
+  constructor(cause, ledger) {
+    const original = cause instanceof Error ? cause : new Error(String(cause));
+    super(original.message, { cause: original });
+    this.name = "SelfImproveRunError";
+    this.cost = ledger.summary();
+    this.receipts = ledger.list();
+  }
+};
+function splitTrainHoldout(scenarios, fraction) {
+  function hash(s) {
+    let h = 2166136261 >>> 0;
+    for (let i = 0; i < s.length; i++) {
+      h ^= s.charCodeAt(i);
+      h = Math.imul(h, 16777619) >>> 0;
+    }
+    return h;
+  }
+  const sorted = [...scenarios].sort((a, b) => hash(a.id) - hash(b.id));
+  const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+  return {
+    holdout: sorted.slice(0, nHoldout),
+    train: sorted.slice(nHoldout)
+  };
+}
+function meanComposite(byScenario) {
+  const perScenario = {};
+  const values = [];
+  for (const [id, agg] of Object.entries(byScenario)) {
+    perScenario[id] = agg.meanComposite;
+    values.push(agg.meanComposite);
+  }
+  return {
+    compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length,
+    perScenario
+  };
+}
+async function selfImprove(opts) {
+  const startedAt = Date.now();
+  const requestedRunDir = opts.runDir ?? `mem://selfImprove-${startedAt}`;
+  const runDir = resolveRunDir(requestedRunDir);
+  const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
+  const costLedger = createRunCostLedger({
+    storage,
+    runDir,
+    costCeilingUsd: opts.budget?.dollars
+  });
+  try {
+    return await runSelfImprove(opts, costLedger, startedAt, runDir, storage);
+  } catch (error) {
+    throw new SelfImproveRunError(error, costLedger);
+  }
+}
+async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
+  const budget = opts.budget ?? {};
+  const generations = budget.generations ?? 3;
+  const populationSize = budget.populationSize ?? 2;
+  const maxConcurrency = budget.maxConcurrency ?? 2;
+  const holdoutFraction = budget.holdoutFraction ?? 0.25;
+  const expectUsage = opts.expectUsage ?? "assert";
+  const explicitHoldout = budget.holdoutScenarios;
+  const { train, holdout } = explicitHoldout ? {
+    train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
+    holdout: explicitHoldout
+  } : splitTrainHoldout(opts.scenarios, holdoutFraction);
+  if (train.length === 0) {
+    throw new Error(
+      "selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios."
+    );
+  }
+  if (holdout.length === 0) {
+    throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
+  }
+  const proposer = opts.proposer ?? gepaProposer({
+    llm: {
+      baseUrl: opts.llm?.baseUrl ?? "https://router.tangle.tools/v1",
+      apiKey: opts.llm?.apiKey ?? process.env.OPENAI_API_KEY ?? ""
+    },
+    model: opts.llm?.model ?? "anthropic/claude-sonnet-4.6",
+    target: opts.proposerTarget ?? "agent surface (system prompt or config) being optimized by selfImprove",
+    // Pass-through: when unset, gepaProposer falls back to its own
+    // domain-neutral engineering primitives.
+    mutationPrimitives: opts.mutationPrimitives
+  });
+  const gate = opts.gate ?? defaultProductionGate({
+    holdoutScenarios: holdout,
+    deltaThreshold: 0.05
+  });
+  if (opts.onProgress) {
+    opts.onProgress({ kind: "baseline.started", scenarios: opts.scenarios.length });
+  }
+  const result = await runImprovementLoop({
+    scenarios: train,
+    baselineSurface: opts.baselineSurface,
+    dispatchWithSurface: opts.agent,
+    proposer,
+    judges: [opts.judge],
+    populationSize,
+    maxGenerations: generations,
+    promoteTopK: budget.promoteTopK,
+    reps: budget.reps,
+    holdoutScenarios: holdout,
+    gate,
+    neutralize: opts.neutralize,
+    autoOnPromote: opts.autoOnPromote ?? "none",
+    ghOwner: opts.ghOwner,
+    ghRepo: opts.ghRepo,
+    storage,
+    runDir,
+    maxConcurrency,
+    cellPlacement: opts.cellPlacement,
+    dispatchTimeoutMs: opts.dispatchTimeoutMs,
+    costLedger,
+    expectUsage,
+    labeledStore: opts.labeledStore,
+    captureSource: opts.captureSource,
+    analyzeGeneration: opts.analyzeGeneration,
+    findings: opts.findings
+  });
+  const baseline = meanComposite(result.baselineOnHoldout.aggregates.byScenario);
+  const winnerStats = meanComposite(result.winnerOnHoldout.aggregates.byScenario);
+  let power;
+  const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
+    const scores = Object.values(cell.judgeScores);
+    return scores.length === 0 ? Number.NaN : scores.reduce((sum, s) => sum + s.composite, 0) / scores.length;
+  }).filter((v) => Number.isFinite(v));
+  if (baselineHoldoutComposites.length >= 3) {
+    power = powerPreflight({
+      baselineComposites: baselineHoldoutComposites,
+      sharedScorerChannel: true
+    });
+    if (opts.onProgress) {
+      opts.onProgress({
+        kind: "power.estimated",
+        n: power.n,
+        sd: power.sd,
+        mde: power.mde,
+        underpowered: power.underpowered
+      });
+    }
+    if (power.underpowered && generations > 0) {
+      console.warn(`[selfImprove] ${power.recommendation}`);
+    }
+  }
+  if (opts.onProgress) {
+    opts.onProgress({
+      kind: "baseline.completed",
+      compositeMean: baseline.compositeMean,
+      durationMs: Date.now() - startedAt
+    });
+    opts.onProgress({
+      kind: "gate.decided",
+      decision: result.gateResult.decision,
+      lift: winnerStats.compositeMean - baseline.compositeMean
+    });
+  }
+  const cost = result.cost;
+  const totalCost = cost.totalCostUsd;
+  const insight = await analyzeRuns({
+    runs: [
+      ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
+      ...cellsToRunRecords(result.winnerOnHoldout.cells, "winner", runDir, result.winnerSurface)
+    ],
+    baselineCandidateId: "baseline",
+    candidateCandidateId: "winner"
+  });
+  const durationMs = Date.now() - startedAt;
+  const { record: provenance } = await emitLoopProvenance({
+    ...loopProvenanceArgsFromResult({
+      runId: `${runDir}#${startedAt}`,
+      runDir,
+      timestamp: new Date(startedAt).toISOString(),
+      baselineSurface: opts.baselineSurface,
+      result,
+      costReceipts: costLedger.list(),
+      totalCostUsd: totalCost,
+      totalDurationMs: durationMs
+    }),
+    storage,
+    hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
+  });
+  if (opts.onProvenance) opts.onProvenance(provenance);
+  const summary = {
+    baseline,
+    winner: {
+      ...winnerStats,
+      surface: result.winnerSurface,
+      ...result.winnerLabel ? { label: result.winnerLabel } : {},
+      ...result.winnerRationale ? { rationale: result.winnerRationale } : {}
+    },
+    lift: winnerStats.compositeMean - baseline.compositeMean,
+    diff: result.promotedDiff,
+    provenance,
+    gateDecision: result.gateResult.decision,
+    generationsExplored: result.generations.length,
+    durationMs,
+    totalCostUsd: totalCost,
+    cost,
+    receipts: costLedger.list(),
+    insight,
+    ...power ? { power } : {},
+    raw: result
+  };
+  if (opts.hostedTenant) {
+    try {
+      await shipEvalRunToHosted(opts.hostedTenant, opts, summary, result, runDir);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[agent-eval] hosted ingest failed (continuing): ${msg}`);
+    }
+  }
+  return summary;
+}
+async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
+  const client = createHostedClient(tenant);
+  function snapshotFromCampaign(index, surface, campaign, durationMs) {
+    const cells = campaign.cells.map((cell) => {
+      const judgeScores = Object.values(cell.judgeScores);
+      const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
+      return {
+        scenarioId: cell.scenarioId,
+        rep: cell.rep,
+        compositeMean: composite,
+        dimensions: Object.fromEntries(
+          Object.entries(cell.judgeScores).map(([name, score]) => [name, score.dimensions])
+        ),
+        errorMessage: cell.error ?? void 0
+      };
+    });
+    const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
+    return {
+      index,
+      surfaceHash: surfaceHash(surface),
+      surface,
+      cells,
+      compositeMean,
+      costUsd: campaign.aggregates.totalCostUsd,
+      durationMs
+    };
+  }
+  const generations = [];
+  generations.push(snapshotFromCampaign(0, opts.baselineSurface, raw.baselineCampaign, 0));
+  for (const gen of raw.generations) {
+    const winner = gen.surfaces.reduce(
+      (best, s) => s.campaign.aggregates.cellsExecuted > 0 && (best === void 0 || averageComposite(s.campaign) > averageComposite(best.campaign)) ? s : best,
+      gen.surfaces[0]
+    );
+    if (!winner) continue;
+    generations.push(
+      snapshotFromCampaign(gen.record.generationIndex + 1, winner.surface, winner.campaign, 0)
+    );
+  }
+  const event = {
+    runId: `${runDir}#${Date.now()}`,
+    runDir,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    status: "finished",
+    labels: opts.hostedLabels ?? {},
+    baseline: generations[0],
+    generations,
+    gateDecision: summary.gateDecision,
+    holdoutLift: summary.lift,
+    totalCostUsd: summary.totalCostUsd,
+    totalDurationMs: summary.durationMs,
+    insightReport: summary.insight
+  };
+  await client.ingestEvalRun(event);
+}
+function averageComposite(campaign) {
+  const aggs = Object.values(campaign.aggregates.byScenario);
+  return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length;
+}
+function hashString(s) {
+  let h = 2166136261 >>> 0;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+function cellsToRunRecords(cells, candidateId, runId, surface) {
+  const promptHash = surfaceContentHash(surface);
+  const configHash = `sha256:${createHash("sha256").update(candidateId).digest("hex")}`;
+  return cells.map((cell) => {
+    const perJudge = {};
+    const perDimMeanAccum = {};
+    let compositeSum = 0;
+    let compositeCount = 0;
+    for (const [judgeId, score] of Object.entries(cell.judgeScores)) {
+      perJudge[judgeId] = { ...score.dimensions };
+      for (const [dim, value] of Object.entries(score.dimensions)) {
+        if (!Number.isFinite(value)) continue;
+        const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
+        accum.sum += value;
+        accum.n += 1;
+        perDimMeanAccum[dim] = accum;
+      }
+      if (Number.isFinite(score.composite)) {
+        compositeSum += score.composite;
+        compositeCount += 1;
+      }
+    }
+    const perDimMean = {};
+    for (const [dim, { sum, n }] of Object.entries(perDimMeanAccum)) {
+      perDimMean[dim] = n === 0 ? 0 : sum / n;
+    }
+    const composite = compositeCount === 0 ? 0 : compositeSum / compositeCount;
+    const judgeScores = {
+      perJudge,
+      perDimMean,
+      composite
+    };
+    return {
+      runId: `${runId}::${candidateId}::${cell.cellId}`,
+      experimentId: runId,
+      candidateId,
+      // Pair on (scenarioId, rep) — analyzeRuns pairs on (experimentId, seed).
+      // Synthesize a stable seed for that pairing.
+      seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
+      model: "campaign-cell",
+      promptHash,
+      configHash,
+      commitSha: "cell",
+      wallMs: cell.durationMs,
+      costUsd: cell.costUsd,
+      tokenUsage: {
+        input: cell.tokenUsage.input,
+        output: cell.tokenUsage.output,
+        ...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
+        ...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
+        ...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
+      },
+      outcome: {
+        holdoutScore: composite,
+        raw: {},
+        judgeScores
+      },
+      splitTag: "holdout",
+      ...cell.error ? { failureMode: cell.error } : {}
+    };
+  });
+}
+
+// src/contract/define-agent-eval.ts
+function defineAgentEval(defaults) {
+  const defaultEvaluateOptions = evaluateDefaults(defaults);
+  return {
+    scenarios: defaults.scenarios,
+    baselineSurface: defaults.baselineSurface,
+    async evaluate(opts = {}) {
+      const { agent, judge, judges, runDir, scenarios, surface, ...campaignOpts } = opts;
+      const selectedAgent = agent ?? defaults.agent;
+      const selectedSurface = surface ?? defaults.baselineSurface;
+      const selectedRunDir = runDir ?? defaults.runDir ?? `mem://defineAgentEval-${Date.now()}`;
+      const selectedStorage = campaignOpts.storage ?? defaultEvaluateOptions.storage ?? (selectedRunDir.startsWith("mem://") ? inMemoryCampaignStorage() : void 0);
+      const evalOptions = {
+        ...defaultEvaluateOptions,
+        ...campaignOpts,
+        ...selectedStorage ? { storage: selectedStorage } : {},
+        runDir: selectedRunDir,
+        scenarios: scenarios ?? defaults.scenarios,
+        dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx),
+        judges: evaluateJudges(judges, judge ?? defaults.judge)
+      };
+      if (evalOptions.reps !== void 0)
+        evalOptions.reps = requirePositiveInteger(evalOptions.reps, "reps");
+      return runEval(evalOptions);
+    },
+    async improve(opts = {}) {
+      const {
+        budget: budgetOverride,
+        hostedTenant: hostedTenantOverride,
+        llm: llmOverride,
+        ...topLevelOverrides
+      } = opts;
+      const merged = mergeDefined(defaults, topLevelOverrides);
+      const budget = mergeBudget(defaults.budget, budgetOverride);
+      const llm = mergeLlm(defaults.llm, llmOverride);
+      const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
+      return selfImprove({
+        ...merged,
+        ...budget ? { budget } : {},
+        ...llm ? { llm } : {},
+        ...hostedTenant ? { hostedTenant } : {}
+      });
+    }
+  };
+}
+function evaluateDefaults(defaults) {
+  const out = {};
+  if (defaults.storage) out.storage = defaults.storage;
+  if (defaults.labeledStore) out.labeledStore = defaults.labeledStore;
+  if (defaults.captureSource) out.captureSource = defaults.captureSource;
+  if (defaults.cellPlacement) out.cellPlacement = defaults.cellPlacement;
+  if (defaults.expectUsage) out.expectUsage = defaults.expectUsage;
+  if (defaults.budget?.dollars !== void 0) out.costCeiling = defaults.budget.dollars;
+  if (defaults.budget?.maxConcurrency !== void 0)
+    out.maxConcurrency = defaults.budget.maxConcurrency;
+  if (defaults.budget?.reps !== void 0)
+    out.reps = requirePositiveInteger(defaults.budget.reps, "budget.reps");
+  return out;
+}
+function mergeBudget(defaults, overrides) {
+  const merged = mergeOptionalObject(defaults, overrides);
+  if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
+  return merged;
+}
+function mergeLlm(defaults, overrides) {
+  return mergeOptionalObject(defaults, overrides);
+}
+function mergeHostedTenant(defaults, overrides) {
+  const merged = mergeOptionalObject(defaults, overrides);
+  if (!merged) return void 0;
+  if (!merged.endpoint?.trim() || !merged.apiKey?.trim() || !merged.tenantId?.trim()) {
+    throw new Error(
+      "defineAgentEval.improve: hostedTenant requires endpoint, apiKey, and tenantId after merging defaults and overrides"
+    );
+  }
+  return merged;
+}
+function mergeDefined(defaults, overrides) {
+  if (!overrides) return defaults;
+  const merged = { ...defaults };
+  for (const [key, value] of Object.entries(overrides)) {
+    if (value !== void 0) merged[key] = value;
+  }
+  return merged;
+}
+function mergeOptionalObject(defaults, overrides) {
+  if (!defaults && !overrides) return void 0;
+  return mergeDefined(defaults ?? {}, overrides);
+}
+function evaluateJudges(judges, defaultJudge) {
+  if (judges !== void 0) {
+    if (judges.length === 0) {
+      throw new Error("defineAgentEval.evaluate: judges must not be empty");
+    }
+    return judges;
+  }
+  return [defaultJudge];
+}
+function requirePositiveInteger(value, field) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`defineAgentEval: ${field} must be a positive integer`);
+  }
+  return value;
+}
+
+// src/contract/measured-comparison.ts
+import {
+  agentImprovementMeasuredComparisonSchema
+} from "@tangle-network/agent-interface";
+function measuredComparisonFromSelfImproveResult(options) {
+  const { result } = options;
+  verifyLoopProvenanceRecord(result.provenance);
+  const power = result.power;
+  if (!power) throw new Error("agent improvement comparison requires heldout power analysis");
+  if (result.provenance.gate.reasons.length === 0) {
+    throw new Error("agent improvement comparison requires measured decision reasons");
+  }
+  const receiptLedger = new CostLedger({ receipts: result.receipts });
+  const receiptCost = receiptLedger.summary();
+  const receipts = receiptLedger.list();
+  assertRealAgentReceipts(receipts, { allowMixed: false });
+  const receiptsById = new Map(receipts.map((receipt) => [receipt.callId, receipt]));
+  const receiptIdsByCell = indexReceiptIdsByCell(receipts);
+  assertCompleteMeasuredCampaign(
+    result.raw.baselineOnHoldout,
+    "heldout baseline",
+    receiptsById,
+    receiptIdsByCell
+  );
+  assertCompleteMeasuredCampaign(
+    result.raw.winnerOnHoldout,
+    "heldout candidate",
+    receiptsById,
+    receiptIdsByCell
+  );
+  const pairs = pairMeasuredCells(
+    result.raw.baselineOnHoldout.cells,
+    result.raw.winnerOnHoldout.cells
+  );
+  const composite = measuredObjective(
+    {
+      kind: "objective",
+      name: "composite",
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    pairs,
+    measuredComposite
+  );
+  assertMeasuredNumber(result.lift, composite.delta, "heldout lift");
+  assertMeasuredNumber(result.baseline.compositeMean, composite.baseline, "heldout baseline");
+  assertMeasuredNumber(result.winner.compositeMean, composite.candidate, "heldout candidate");
+  const baselineContentHash = surfaceContentHash(options.baselineSurface);
+  const candidateContentHash = surfaceContentHash(result.winner.surface);
+  const canonicalDiff = baselineContentHash === candidateContentHash ? "" : renderSurfaceDiff(result.raw.winnerSurface, options.baselineSurface);
+  assertMeasuredIdentity(result.gateDecision, result.raw.gateResult.decision, "raw decision");
+  assertMeasuredIdentity(
+    candidateContentHash,
+    surfaceContentHash(result.raw.winnerSurface),
+    "raw winner surface"
+  );
+  assertMeasuredIdentity(result.diff, canonicalDiff, "surface diff");
+  assertMeasuredIdentity(result.raw.promotedDiff, canonicalDiff, "raw surface diff");
+  if (result.gateDecision === "ship" && (baselineContentHash === candidateContentHash || result.diff.trim().length === 0)) {
+    throw new Error("a shipped improvement requires a changed surface and non-empty diff");
+  }
+  assertMeasuredIdentity(
+    canonicalDigest(power),
+    canonicalDigest(
+      powerPreflight({
+        baselineComposites: pairs.map(([cell]) => measuredComposite(cell)),
+        sharedScorerChannel: true
+      })
+    ),
+    "power analysis"
+  );
+  assertMeasuredNumber(power.n, composite.n, "power sample size");
+  assertMeasuredNumber(power.confidence, 0.95, "power confidence");
+  assertCompleteMeasuredCampaign(
+    result.raw.baselineCampaign,
+    "search baseline",
+    receiptsById,
+    receiptIdsByCell
+  );
+  assertGenerationMeasurements(result.raw.generations, receiptsById, receiptIdsByCell);
+  if (result.raw.neutralizedOnHoldout === void 0 !== (result.raw.neutralizedSurface === void 0)) {
+    throw new Error("neutralized surface and campaign must be supplied together");
+  }
+  if (result.raw.neutralizedOnHoldout) {
+    assertCompleteMeasuredCampaign(
+      result.raw.neutralizedOnHoldout,
+      "heldout neutralized",
+      receiptsById,
+      receiptIdsByCell
+    );
+    pairMeasuredCells(result.raw.baselineOnHoldout.cells, result.raw.neutralizedOnHoldout.cells);
+  }
+  const rebuiltProvenance = buildLoopProvenanceRecord(
+    loopProvenanceArgsFromResult({
+      runId: result.provenance.runId,
+      runDir: result.provenance.runDir,
+      timestamp: result.provenance.timestamp,
+      baselineSurface: options.baselineSurface,
+      result: result.raw,
+      costReceipts: receipts,
+      totalCostUsd: result.totalCostUsd,
+      totalDurationMs: result.durationMs
+    })
+  );
+  assertMeasuredIdentity(
+    result.provenance.recordDigest,
+    rebuiltProvenance.recordDigest,
+    "provenance record"
+  );
+  assertMeasuredIdentity(
+    options.benchmark.splitDigest,
+    rebuiltProvenance.evidence.holdout.splitDigest,
+    "benchmark heldout split"
+  );
+  const generationsExplored = result.raw.generations.length;
+  assertMeasuredNumber(result.generationsExplored, generationsExplored, "generation count");
+  assertMeasuredNumber(result.totalCostUsd, result.cost.totalCostUsd, "cost summary");
+  assertMeasuredIdentity(
+    canonicalDigest(result.cost),
+    canonicalDigest(result.raw.cost),
+    "raw cost summary"
+  );
+  if (!result.cost.accountingComplete) {
+    throw new Error(
+      `cost accounting is incomplete: ${result.cost.incompleteReasons.join("; ") || "unknown reason"}`
+    );
+  }
+  if (!receiptCost.accountingComplete) {
+    throw new Error(
+      `cost accounting is incomplete: ${receiptCost.incompleteReasons.join("; ") || "unknown reason"}`
+    );
+  }
+  assertMeasuredIdentity(
+    canonicalDigest(result.cost),
+    canonicalDigest(receiptCost),
+    "cost receipt summary"
+  );
+  assertMeasuredNumber(result.totalCostUsd, receiptCost.totalCostUsd, "cost receipts");
+  assertMeasuredOptional(result.winner.label, result.raw.winnerLabel, "raw winner label");
+  assertMeasuredOptional(
+    result.winner.rationale,
+    result.raw.winnerRationale,
+    "raw winner rationale"
+  );
+  return agentImprovementMeasuredComparisonSchema.parse({
+    schemaVersion: 1,
+    kind: "agent-improvement-measured-comparison",
+    benchmark: options.benchmark,
+    baselineProfileDigest: options.baselineProfileDigest,
+    candidateBundleDigest: options.candidateBundleDigest,
+    overall: {
+      name: "composite",
+      baseline: composite.baseline,
+      candidate: composite.candidate,
+      delta: composite.delta,
+      confidenceInterval: composite.confidenceInterval,
+      n: composite.n,
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    objectives: measuredObjectives(pairs),
+    ...result.winner.label || result.winner.rationale ? {
+      candidate: {
+        ...result.winner.label ? { label: result.winner.label } : {},
+        ...result.winner.rationale ? { rationale: result.winner.rationale } : {}
+      }
+    } : {},
+    decision: {
+      outcome: result.gateDecision,
+      reasons: rebuiltProvenance.gate.reasons,
+      contributingChecks: rebuiltProvenance.gate.contributingGates.map((check) => ({
+        name: check.name,
+        passed: check.passed
+      }))
+    },
+    power: {
+      sufficient: power.scaleAssumed && !power.underpowered,
+      n: power.n,
+      minimumDetectableDelta: power.mde,
+      confidenceLevel: power.confidence,
+      scaleAssumed: power.scaleAssumed,
+      sharedScorerChannel: power.sharedChannelCaveat !== void 0,
+      reason: power.recommendation
+    },
+    provenance: {
+      kind: "agent-eval-loop",
+      schema: rebuiltProvenance.schema,
+      runId: rebuiltProvenance.runId,
+      recordDigest: rebuiltProvenance.recordDigest,
+      baselineContentHash,
+      candidateContentHash
+    },
+    diff: canonicalDiff,
+    evaluation: {
+      generationsExplored,
+      durationMs: result.durationMs,
+      totalCostUsd: result.totalCostUsd
+    }
+  });
+}
+function assertGenerationMeasurements(generations, receiptsById, receiptIdsByCell) {
+  for (const generation of generations) {
+    for (const measured of generation.surfaces) {
+      const candidate = generation.record.candidates.find(
+        (entry) => entry.surfaceHash === measured.surfaceHash
+      );
+      if (!candidate) {
+        throw new Error(
+          `generation ${generation.record.generationIndex} is missing candidate ${measured.surfaceHash}`
+        );
+      }
+      const composite = campaignMeanComposite(measured.campaign);
+      const breakdown = campaignBreakdown(measured.campaign);
+      const coverage = campaignCoverage(
+        measured.campaign.cells,
+        measured.campaign.scenarios,
+        measured.campaign.reps,
+        true
+      );
+      assertScorableMeasuredCells(
+        measured.campaign,
+        coverage.scorableCellIds,
+        receiptsById,
+        receiptIdsByCell
+      );
+      assertMeasuredNumber(
+        candidate.composite,
+        composite,
+        `candidate ${measured.surfaceHash} composite`
+      );
+      if (!candidate.coverage) {
+        throw new Error(`candidate ${measured.surfaceHash} does not report campaign coverage`);
+      }
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.coverage),
+        canonicalDigest({
+          expectedCells: coverage.expectedCellIds.length,
+          scorableCells: coverage.scorableCellIds.length,
+          unscorableCells: coverage.unscorableCells
+        }),
+        `candidate ${measured.surfaceHash} coverage`
+      );
+      assertMeasuredIdentity(
+        String(candidate.eligibleForPromotion),
+        String(coverage.complete),
+        `candidate ${measured.surfaceHash} eligibility`
+      );
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.dimensions),
+        canonicalDigest(breakdown.dimensions),
+        `candidate ${measured.surfaceHash} dimensions`
+      );
+      assertMeasuredIdentity(
+        canonicalDigest(candidate.scenarios),
+        canonicalDigest(breakdown.scenarios),
+        `candidate ${measured.surfaceHash} scenarios`
+      );
+    }
+  }
+}
+function assertCompleteMeasuredCampaign(campaign, name, receiptsById, receiptIdsByCell) {
+  assertCampaignSplitIdentity(campaign.scenarios, campaign.reps, campaign.splitDigest);
+  const coverage = campaignCoverage(campaign.cells, campaign.scenarios, campaign.reps, true);
+  if (!coverage.complete) {
+    throw new Error(
+      `${name} is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable)`
+    );
+  }
+  assertScorableMeasuredCells(campaign, coverage.scorableCellIds, receiptsById, receiptIdsByCell);
+}
+function assertScorableMeasuredCells(campaign, scorableCellIds, receiptsById, receiptIdsByCell) {
+  const cellsById = new Map(campaign.cells.map((cell) => [cell.cellId, cell]));
+  for (const cellId of scorableCellIds) {
+    const cell = cellsById.get(cellId);
+    if (!cell) throw new Error(`measured campaign is missing scorable cell ${cellId}`);
+    assertMeasuredCell(cell, receiptsById, receiptIdsByCell, campaign.runDir);
+  }
+}
+function measuredObjectives(pairs) {
+  const qualityColumns = /* @__PURE__ */ new Map();
+  for (const [baseline, candidate] of pairs) {
+    for (const cell of [baseline, candidate]) {
+      for (const [objective, score] of Object.entries(cell.judgeScores)) {
+        if (score.failed) continue;
+        qualityColumns.set(`objective:${objective}`, {
+          kind: "objective",
+          name: objective,
+          direction: "higher-is-better",
+          unit: "score"
+        });
+        for (const name of Object.keys(score.dimensions)) {
+          qualityColumns.set(`dimension:${objective}:${name}`, {
+            kind: "dimension",
+            objective,
+            name,
+            direction: "higher-is-better",
+            unit: "score"
+          });
+        }
+      }
+    }
+  }
+  return [
+    ...[...qualityColumns.entries()].sort(([left], [right]) => left.localeCompare(right)).map(
+      ([, column]) => measuredObjective(column, pairs, (cell) => measuredQuality(cell, column))
+    ),
+    measuredCostObjective(pairs),
+    measuredObjective(
+      {
+        kind: "latency",
+        name: "latency",
+        direction: "lower-is-better",
+        unit: "milliseconds"
+      },
+      pairs,
+      (cell) => cell.durationMs
+    )
+  ];
+}
+function measuredCostObjective(pairs) {
+  const cells = pairs.flat();
+  for (const cell of cells) finiteMeasuredValue(cell.costUsd, "cost:cost");
+  return measuredObjective(
+    {
+      kind: "cost",
+      name: "cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    },
+    pairs,
+    (cell) => cell.costUsd
+  );
+}
+function measuredComposite(cell) {
+  const values = Object.values(cell.judgeScores).filter((score) => !score.failed).map((score) => score.composite).filter(Number.isFinite);
+  if (values.length === 0) {
+    throw new Error(`heldout cell ${measuredCellKey(cell)} has no successful composite score`);
+  }
+  return measuredMean(values);
+}
+function pairMeasuredCells(baselineCells, candidateCells) {
+  const cells = [...baselineCells, ...candidateCells];
+  const errors = cells.filter((cell) => cell.error);
+  if (errors.length > 0) {
+    throw new Error(
+      `measured objectives cannot publish ${errors.length} errored heldout cells: ${errors.map((cell) => measuredCellKey(cell)).join(", ")}`
+    );
+  }
+  const rows = [
+    ...baselineCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "baseline",
+      cell
+    })),
+    ...candidateCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "candidate",
+      cell
+    }))
+  ];
+  const paired = pairArms(rows, { baselineArm: "baseline", treatmentArm: "candidate" });
+  if (paired.pairs.length === 0 || paired.unpairedBaseline.length > 0 || paired.unpairedTreatment.length > 0) {
+    throw new Error("measured objectives require the same non-empty paired heldout cells");
+  }
+  return paired.pairs.map((pair) => {
+    const baseline = pair.baseline.cell;
+    const candidate = pair.treatment.cell;
+    if (baseline.seed !== candidate.seed) {
+      throw new Error(`heldout cell ${baseline.cellId} does not share one paired seed`);
+    }
+    return [baseline, candidate];
+  });
+}
+function assertMeasuredCell(cell, receiptsById, receiptIdsByCell, runDir) {
+  if (!cell.scenarioId.trim() || !Number.isSafeInteger(cell.rep) || cell.rep < 0 || !Number.isSafeInteger(cell.seed) || cell.cellId !== measuredCellKey(cell)) {
+    throw new Error(`heldout cell '${cell.cellId}' has an invalid scenario/rep identity`);
+  }
+  nonnegativeMeasuredValue(cell.costUsd, `heldout cell ${cell.cellId} cost`);
+  nonnegativeMeasuredValue(cell.durationMs, `heldout cell ${cell.cellId} latency`);
+  if (!cell.tokenUsage) {
+    throw new Error(`heldout cell ${cell.cellId} does not report token usage`);
+  }
+  nonnegativeMeasuredInteger(cell.tokenUsage.input, `heldout cell ${cell.cellId} input tokens`);
+  nonnegativeMeasuredInteger(cell.tokenUsage.output, `heldout cell ${cell.cellId} output tokens`);
+  if (cell.tokenUsage.cached !== void 0) {
+    nonnegativeMeasuredInteger(cell.tokenUsage.cached, `heldout cell ${cell.cellId} cached tokens`);
+  }
+  if (!Array.isArray(cell.costCallIds)) {
+    throw new Error(`heldout cell ${cell.cellId} does not identify its cost receipts`);
+  }
+  if (new Set(cell.costCallIds).size !== cell.costCallIds.length) {
+    throw new Error(`heldout cell ${cell.cellId} repeats a cost receipt`);
+  }
+  const linkedReceipts = cell.costCallIds.map((callId) => {
+    const receipt = receiptsById.get(callId);
+    if (!receipt)
+      throw new Error(`heldout cell ${cell.cellId} references missing cost receipt ${callId}`);
+    if (receipt.tags?.runDir !== runDir || receipt.tags.cellId !== cell.cellId || receipt.tags.scenarioId !== cell.scenarioId || receipt.tags.rep !== String(cell.rep)) {
+      throw new Error(`heldout cell ${cell.cellId} references a cost receipt from another cell`);
+    }
+    return receipt;
+  });
+  assertMeasuredIdentity(
+    canonicalDigest([...cell.costCallIds].sort()),
+    canonicalDigest(
+      [
+        ...receiptIdsByCell.get(receiptCellKey(runDir, cell.cellId, cell.scenarioId, cell.rep)) ?? []
+      ].sort()
+    ),
+    `heldout cell ${cell.cellId} cost receipt IDs`
+  );
+  const agentReceipts = linkedReceipts.filter((receipt) => receipt.channel === "agent");
+  if (agentReceipts.some((receipt) => receipt.error)) {
+    throw new Error(`measured cell ${cell.cellId} links a failed agent receipt`);
+  }
+  assertRealAgentReceipts(agentReceipts, { allowMixed: false });
+  assertMeasuredNumber(
+    cell.costUsd,
+    agentReceipts.reduce((total, receipt) => total + receipt.costUsd, 0),
+    `heldout cell ${cell.cellId} cost receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.input,
+    agentReceipts.reduce((total, receipt) => total + receipt.inputTokens, 0),
+    `heldout cell ${cell.cellId} input receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.output,
+    agentReceipts.reduce((total, receipt) => total + receipt.outputTokens, 0),
+    `heldout cell ${cell.cellId} output receipts`
+  );
+  assertMeasuredNumber(
+    cell.tokenUsage.cached ?? 0,
+    agentReceipts.reduce((total, receipt) => total + (receipt.cachedTokens ?? 0), 0),
+    `heldout cell ${cell.cellId} cached receipts`
+  );
+  for (const [judge, score] of Object.entries(cell.judgeScores)) {
+    if (score.failed) {
+      throw new Error(`heldout cell ${cell.cellId} contains failed judge '${judge}'`);
+    }
+    finiteMeasuredValue(score.composite, `objective:${judge}`);
+    for (const [dimension, value] of Object.entries(score.dimensions)) {
+      finiteMeasuredValue(value, `dimension:${judge}:${dimension}`);
+    }
+  }
+  measuredComposite(cell);
+}
+function measuredObjective(identity, pairs, value) {
+  const key = measuredObjectiveKey(identity);
+  const baseline = pairs.map(([cell]) => finiteMeasuredValue(value(cell), key));
+  const candidate = pairs.map(([, cell]) => finiteMeasuredValue(value(cell), key));
+  const interval = pairedBootstrap(baseline, candidate, {
+    confidence: 0.95,
+    resamples: 2e3,
+    statistic: "mean",
+    seed: measuredSeed(key)
+  });
+  const baselineMean = measuredMean(baseline);
+  const candidateMean = measuredMean(candidate);
+  return {
+    ...identity,
+    availability: "measured",
+    baseline: baselineMean,
+    candidate: candidateMean,
+    delta: candidateMean - baselineMean,
+    confidenceInterval: {
+      level: interval.confidence,
+      lower: interval.low,
+      upper: interval.high,
+      method: "paired-bootstrap",
+      statistic: "mean",
+      resamples: interval.resamples
+    },
+    n: interval.n
+  };
+}
+function measuredQuality(cell, column) {
+  const objective = column.kind === "objective" ? column.name : column.objective;
+  const score = cell.judgeScores[objective];
+  if (!score || score.failed) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing measured objective '${objective}'`
+    );
+  }
+  const value = column.kind === "objective" ? score.composite : score.dimensions[column.name];
+  if (value === void 0) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing '${objective}' dimension '${column.name}'`
+    );
+  }
+  return finiteMeasuredValue(value, measuredObjectiveKey(column));
+}
+function measuredObjectiveKey(identity) {
+  return identity.kind === "dimension" ? `${identity.kind}:${identity.objective}:${identity.name}` : `${identity.kind}:${identity.name}`;
+}
+function measuredCellKey(cell) {
+  return `${cell.scenarioId}:${cell.rep}`;
+}
+function finiteMeasuredValue(value, name) {
+  if (!Number.isFinite(value)) throw new Error(`measured objective '${name}' is not finite`);
+  return value;
+}
+function nonnegativeMeasuredValue(value, name) {
+  if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be non-negative`);
+  return value;
+}
+function nonnegativeMeasuredInteger(value, name) {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${name} must be a non-negative safe integer`);
+  }
+  return value;
+}
+function measuredMean(values) {
+  if (values.length === 0) throw new Error("measured objective has no paired values");
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function measuredSeed(value) {
+  let seed = 2166136261;
+  for (const byte of Buffer.from(value, "utf8")) {
+    seed = Math.imul(seed ^ byte, 16777619) >>> 0;
+  }
+  return seed;
+}
+function assertMeasuredNumber(actual, expected, name) {
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)) * 8;
+  if (!Number.isFinite(actual) || !Number.isFinite(expected) || Math.abs(actual - expected) > tolerance) {
+    throw new Error(`${name} does not agree across the measured comparison`);
+  }
+}
+function assertMeasuredIdentity(actual, expected, name) {
+  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
+}
+function assertMeasuredOptional(actual, expected, name) {
+  if (actual !== expected) throw new Error(`${name} does not agree across the measured comparison`);
+}
+function indexReceiptIdsByCell(receipts) {
+  const idsByCell = /* @__PURE__ */ new Map();
+  for (const receipt of receipts) {
+    const tags = receipt.tags;
+    if (!tags?.runDir || !tags.cellId || !tags.scenarioId || tags.rep === void 0) continue;
+    const key = receiptCellKey(tags.runDir, tags.cellId, tags.scenarioId, tags.rep);
+    const ids = idsByCell.get(key) ?? [];
+    ids.push(receipt.callId);
+    idsByCell.set(key, ids);
+  }
+  return idsByCell;
+}
+function receiptCellKey(runDir, cellId, scenarioId, rep) {
+  return JSON.stringify([runDir, cellId, scenarioId, String(rep)]);
+}
+
+// src/contract/eval-reporting-suite.ts
+import { mkdir, writeFile } from "fs/promises";
+import { dirname, join as join2 } from "path";
+
+// src/contract/intake/run-record-dir.ts
+import { readdir, readFile, stat } from "fs/promises";
+import { join } from "path";
+var ANALYSIS_ARTIFACT = "analysis.json";
+function defaultInclude(fileName) {
+  if (fileName === ANALYSIS_ARTIFACT) return false;
+  return fileName.endsWith(".json") || fileName.endsWith(".jsonl");
+}
+async function fromRunRecordDir(path, options = {}) {
+  const onInvalid = options.onInvalid ?? "throw";
+  const include = options.include ?? defaultInclude;
+  const stats = await stat(path);
+  const filePaths = stats.isDirectory() ? await collectFiles(path, include, options.recursive ?? false) : [path];
+  const runs = [];
+  const rejected = [];
+  for (const file of filePaths) {
+    const raw = await parseRecordFile(file);
+    for (const { index, value } of raw) {
+      const parsed = parseRunRecordSafe(value);
+      if (parsed.ok) {
+        runs.push(parsed.value);
+        continue;
+      }
+      const rejection = { file, index, reason: parsed.error.message };
+      if (onInvalid === "throw") {
+        throw new Error(
+          `fromRunRecordDir: invalid RunRecord in '${file}' at index ${index}: ${parsed.error.message}`
+        );
+      }
+      rejected.push(rejection);
+    }
+  }
+  return { runs, rejected, files: filePaths };
+}
+async function parseRecordFile(file) {
+  const text = await readFile(file, "utf8");
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return [];
+  if (trimmed.startsWith("[")) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error(`fromRunRecordDir: file '${file}' did not parse to an array`);
+    }
+    return parsed.map((value, index) => ({ index, value }));
+  }
+  const out = [];
+  const lines = trimmed.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line.length === 0) continue;
+    try {
+      out.push({ index: i, value: JSON.parse(line) });
+    } catch (err) {
+      throw new Error(
+        `fromRunRecordDir: file '${file}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+  return out;
+}
+async function collectFiles(dir, include, recursive) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  const subdirs = [];
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      if (recursive) subdirs.push(join(dir, entry.name));
+      continue;
+    }
+    if (include(entry.name)) files.push(join(dir, entry.name));
+  }
+  files.sort();
+  subdirs.sort();
+  for (const sub of subdirs) {
+    files.push(...await collectFiles(sub, include, recursive));
+  }
+  return files;
+}
+
+// src/contract/eval-reporting-suite.ts
+var ANALYSIS_ARTIFACT2 = "analysis.json";
+async function evalReportingSuite(input, options = {}) {
+  const fromPath = typeof input === "string";
+  let runs;
+  let files = [];
+  let rejected = [];
+  if (fromPath) {
+    const loaded = await fromRunRecordDir(input, options.load);
+    runs = loaded.runs;
+    files = loaded.files;
+    rejected = loaded.rejected;
+  } else {
+    runs = input;
+  }
+  if (runs.length === 0) {
+    throw new Error(
+      fromPath ? `evalReportingSuite: no RunRecords found at '${input}'` : "evalReportingSuite: no RunRecords to analyze"
+    );
+  }
+  const report = await analyzeRuns({ ...options.analyze, runs });
+  const result = {
+    report,
+    provenance: {
+      generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      runCount: runs.length,
+      sourcePath: fromPath ? input : null,
+      files,
+      rejected
+    },
+    writtenTo: null
+  };
+  const target = resolveWriteTarget(options.write, fromPath ? input : null);
+  if (target) {
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, `${JSON.stringify(result, null, 2)}
+`, "utf8");
+    result.writtenTo = target;
+  }
+  return result;
+}
+function resolveWriteTarget(write, sourcePath) {
+  if (!write) return null;
+  if (typeof write === "string") {
+    const looksLikeDir = write.endsWith("/") || !write.endsWith(".json") && !write.endsWith(".jsonl");
+    return looksLikeDir ? join2(write, ANALYSIS_ARTIFACT2) : write;
+  }
+  if (sourcePath === null) {
+    throw new Error(
+      "evalReportingSuite: write:true needs a source path to anchor analysis.json \u2014 pass an explicit output path when analyzing in-memory records"
+    );
+  }
+  const isFile = sourcePath.endsWith(".json") || sourcePath.endsWith(".jsonl");
+  return isFile ? join2(dirname(sourcePath), ANALYSIS_ARTIFACT2) : join2(sourcePath, ANALYSIS_ARTIFACT2);
+}
+
+// src/contract/diff.ts
+function keyForCell(cell) {
+  return JSON.stringify([cell.scenarioId, cell.rep]);
+}
+function diffDimensions(before, after) {
+  const out = {};
+  const judges = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
+  for (const judge of judges) {
+    const beforeDims = before[judge] ?? {};
+    const afterDims = after[judge] ?? {};
+    const dims = /* @__PURE__ */ new Set([...Object.keys(beforeDims), ...Object.keys(afterDims)]);
+    const judgeOut = {};
+    for (const dim of dims) {
+      const rawBefore = beforeDims[dim];
+      const rawAfter = afterDims[dim];
+      const b = typeof rawBefore === "number" && Number.isFinite(rawBefore) ? rawBefore : null;
+      const a = typeof rawAfter === "number" && Number.isFinite(rawAfter) ? rawAfter : null;
+      judgeOut[dim] = {
+        before: b,
+        after: a,
+        delta: b !== null && a !== null ? a - b : null
+      };
+    }
+    out[judge] = judgeOut;
+  }
+  return out;
+}
+function diffGenerations(before, after) {
+  const beforeMap = new Map(before.cells.map((c) => [keyForCell(c), c]));
+  const afterMap = new Map(after.cells.map((c) => [keyForCell(c), c]));
+  const matched = [];
+  const removed = [];
+  const added = [];
+  for (const [key, beforeCell] of beforeMap) {
+    const afterCell = afterMap.get(key);
+    if (!afterCell) {
+      removed.push(beforeCell);
+      continue;
+    }
+    matched.push({
+      scenarioId: beforeCell.scenarioId,
+      rep: beforeCell.rep,
+      compositeBefore: beforeCell.compositeMean,
+      compositeAfter: afterCell.compositeMean,
+      compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
+      dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
+    });
+  }
+  for (const [key, afterCell] of afterMap) {
+    if (!beforeMap.has(key)) added.push(afterCell);
+  }
+  return {
+    beforeIndex: before.index,
+    afterIndex: after.index,
+    beforeSurfaceHash: before.surfaceHash,
+    afterSurfaceHash: after.surfaceHash,
+    surfaceChanged: before.surfaceHash !== after.surfaceHash,
+    matched,
+    removed,
+    added,
+    compositeBefore: before.compositeMean,
+    compositeAfter: after.compositeMean,
+    compositeDelta: after.compositeMean - before.compositeMean,
+    costUsdBefore: before.costUsd,
+    costUsdAfter: after.costUsd,
+    costUsdDelta: after.costUsd - before.costUsd,
+    durationMsBefore: before.durationMs,
+    durationMsAfter: after.durationMs,
+    durationMsDelta: after.durationMs - before.durationMs
+  };
+}
+function winnerOf(run) {
+  if (run.generations.length === 0) return null;
+  let winner = run.generations[0];
+  for (const gen of run.generations) {
+    if (gen.index > winner.index) winner = gen;
+  }
+  return winner;
+}
+function diffRuns(before, after) {
+  const beforeWinner = winnerOf(before);
+  const afterWinner = winnerOf(after);
+  const baselineDiff = before.baseline && after.baseline ? diffGenerations(before.baseline, after.baseline) : null;
+  const winnersDiff = beforeWinner && afterWinner ? diffGenerations(beforeWinner, afterWinner) : null;
+  const beforeLift = before.holdoutLift ?? null;
+  const afterLift = after.holdoutLift ?? null;
+  return {
+    beforeRunId: before.runId,
+    afterRunId: after.runId,
+    beforeTimestamp: before.timestamp,
+    afterTimestamp: after.timestamp,
+    beforeGateDecision: before.gateDecision ?? null,
+    afterGateDecision: after.gateDecision ?? null,
+    beforeHoldoutLift: beforeLift,
+    afterHoldoutLift: afterLift,
+    holdoutLiftDelta: beforeLift !== null && afterLift !== null ? afterLift - beforeLift : null,
+    beforeTotalCostUsd: before.totalCostUsd,
+    afterTotalCostUsd: after.totalCostUsd,
+    totalCostUsdDelta: after.totalCostUsd - before.totalCostUsd,
+    beforeTotalDurationMs: before.totalDurationMs,
+    afterTotalDurationMs: after.totalDurationMs,
+    totalDurationMsDelta: after.totalDurationMs - before.totalDurationMs,
+    baselineDiff,
+    winnersDiff
+  };
+}
+function diffRunBaselineToWinner(run) {
+  if (!run.baseline) return null;
+  const winner = winnerOf(run);
+  if (!winner || winner.index === run.baseline.index) return null;
+  return diffGenerations(run.baseline, winner);
+}
+
+// src/contract/intake/agent-trace.ts
+function rangeLines(r) {
+  return Math.max(0, r.end_line - r.start_line + 1);
+}
+function parseAgentTrace(records) {
+  const acc = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const sha = record.vcs?.revision;
+    if (!sha) continue;
+    let a = acc.get(sha);
+    if (!a) {
+      a = {
+        models: /* @__PURE__ */ new Set(),
+        tools: /* @__PURE__ */ new Set(),
+        files: /* @__PURE__ */ new Set(),
+        conversationCount: 0,
+        lineCount: 0,
+        humanInvolved: false
+      };
+      acc.set(sha, a);
+    }
+    if (record.tool?.name) a.tools.add(record.tool.name);
+    for (const file of record.files ?? []) {
+      a.files.add(file.path);
+      for (const conv of file.conversations ?? []) {
+        a.conversationCount += 1;
+        for (const range of conv.ranges ?? []) {
+          const contributor = range.contributor ?? conv.contributor;
+          a.lineCount += rangeLines(range);
+          if (!contributor) continue;
+          if (contributor.type === "human" || contributor.type === "mixed") {
+            a.humanInvolved = true;
+          }
+          if ((contributor.type === "ai" || contributor.type === "mixed") && contributor.model_id) {
+            a.models.add(contributor.model_id);
+          }
+        }
+      }
+    }
+  }
+  const index = /* @__PURE__ */ new Map();
+  for (const [sha, a] of acc) {
+    index.set(sha, {
+      commitSha: sha,
+      aiModels: [...a.models].sort(),
+      tools: [...a.tools].sort(),
+      conversationCount: a.conversationCount,
+      fileCount: a.files.size,
+      lineCount: a.lineCount,
+      humanInvolved: a.humanInvolved
+    });
+  }
+  return index;
+}
+function partitionRunsByAuthoringModel(runs, index) {
+  const byModel = /* @__PURE__ */ new Map();
+  const unattributed = [];
+  for (const run of runs) {
+    const provenance = index.get(run.commitSha);
+    if (!provenance || provenance.aiModels.length === 0) {
+      unattributed.push(run);
+      continue;
+    }
+    for (const model of provenance.aiModels) {
+      const cohort = byModel.get(model) ?? [];
+      cohort.push(run);
+      byModel.set(model, cohort);
+    }
+  }
+  return { byModel, unattributed };
+}
+
+// src/contract/intake/feedback-table.ts
+function fromFeedbackTable(opts) {
+  const { ratings, meta = [], scale, emitRaterScores = true } = opts;
+  const metaByRun = new Map(meta.map((m) => [m.runId, m]));
+  const normalise = (rating) => {
+    if (typeof rating === "boolean") return rating ? 1 : 0;
+    if (!Number.isFinite(rating)) return Number.NaN;
+    if (!scale) return rating;
+    const { min, max } = scale;
+    if (max === min) return rating;
+    return (rating - min) / (max - min);
+  };
+  const byRun = /* @__PURE__ */ new Map();
+  for (const row of ratings) {
+    const list = byRun.get(row.runId) ?? [];
+    list.push(row);
+    byRun.set(row.runId, list);
+  }
+  const runs = [];
+  const raterScores = [];
+  for (const [runId, rowsForRun] of byRun) {
+    const normalised = rowsForRun.map((r) => ({ rater: r.rater, score: normalise(r.rating) })).filter((r) => Number.isFinite(r.score));
+    if (normalised.length === 0) continue;
+    const meanScore = normalised.reduce((s, r) => s + r.score, 0) / normalised.length;
+    const runMeta = metaByRun.get(runId) ?? { runId };
+    const judgeScores = {
+      perJudge: Object.fromEntries(normalised.map((r) => [r.rater, { rating: r.score }])),
+      perDimMean: { rating: meanScore },
+      composite: meanScore
+    };
+    const outcome = {
+      // Feedback corpora ARE the holdout signal — score lands on
+      // `holdoutScore` so downstream substrate primitives (`paretoChart`,
+      // promotion gates) read it correctly by default.
+      holdoutScore: meanScore,
+      raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
+      judgeScores
+    };
+    runs.push({
+      runId,
+      experimentId: runMeta.experimentId ?? "feedback-corpus",
+      candidateId: runMeta.candidateId ?? runId,
+      seed: 0,
+      model: runMeta.model ?? "unknown@unknown",
+      promptHash: runMeta.promptHash ?? "sha256:unknown",
+      configHash: runMeta.configHash ?? "sha256:unknown",
+      commitSha: runMeta.commitSha ?? "unknown",
+      wallMs: runMeta.wallMs ?? 0,
+      costUsd: runMeta.costUsd ?? 0,
+      tokenUsage: { input: 0, output: 0 },
+      outcome,
+      splitTag: runMeta.splitTag ?? "holdout"
+    });
+    if (emitRaterScores) {
+      for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
+    }
+  }
+  return { runs, raterScores };
+}
+
+// src/contract/intake/otel-spans.ts
+var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
+var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
+var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
+var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
+function fromOtelSpans(opts) {
+  const { spans, defaultSplit = "holdout", experimentId = "otel-corpus" } = opts;
+  const grouped = groupSpans(spans);
+  const runs = [];
+  for (const [groupKey, groupSpans2] of grouped) {
+    const root = findRoot(groupSpans2);
+    if (!root) continue;
+    const measurements = summarizeExecutionMeasurements(
+      groupSpans2.map((span) => ({
+        id: span.spanId,
+        ...span.parentSpanId ? { parentId: span.parentSpanId } : {},
+        attributes: span.attributes,
+        modelCall: isExplicitModelCall(span),
+        aggregate: isExplicitAggregate(span)
+      }))
+    );
+    const callSpanIds = new Set(measurements.callSpanIds);
+    const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
+    const wallMs = Math.max(0, (root.endTimeUnixNano - root.startTimeUnixNano) / 1e6);
+    const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
+    const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
+    const costUsd = capturedCost ?? 0;
+    const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
+    const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
+    const score = readAttrNumber(groupSpans2, SCORE_KEYS);
+    const rawNumeric = collectNumericAttrs(groupSpans2);
+    rawNumeric.error_span_count = groupSpans2.filter((span) => span.status?.code === "ERROR").length;
+    rawNumeric.llm_span_count = measurements.modelCallCount;
+    if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
+      rawNumeric.partial_observed_cost_usd = measurements.cost.value;
+    }
+    recordAggregateMeasurements(rawNumeric, measurements.aggregate);
+    const judgeScores = score !== void 0 ? {
+      perJudge: { "otel-derived": { score } },
+      perDimMean: { score },
+      composite: score
+    } : void 0;
+    const errorSpan = groupSpans2.find((s) => s.status?.code === "ERROR");
+    const outcome = {
+      ...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
+      raw: rawNumeric,
+      ...judgeScores ? { judgeScores } : {}
+    };
+    runs.push({
+      runId: groupKey,
+      experimentId,
+      candidateId: root.attributes["tangle.candidateId"] ?? "otel-default",
+      seed: 0,
+      model,
+      promptHash,
+      configHash,
+      commitSha: root.attributes["tangle.commit_sha"] ?? "unknown",
+      wallMs,
+      costUsd,
+      costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
+      tokenUsage: measurements.tokenUsage,
+      outcome,
+      splitTag: defaultSplit,
+      ...errorSpan ? { failureMode: errorSpan.name } : {}
+    });
+  }
+  return runs;
+}
+function isExplicitModelCall(span) {
+  const kind = span.attributes["openinference.span.kind"];
+  if (typeof kind === "string") return kind.toUpperCase() === "LLM";
+  const spanType = span.attributes["span.type"];
+  return typeof spanType === "string" && spanType.toLowerCase() === "llm_request" || span.name.toLowerCase().includes("llm") || readAttrString([span], MODEL_KEYS) !== void 0 || typeof span.attributes["gen_ai.operation.name"] === "string";
+}
+function isExplicitAggregate(span) {
+  const kind = span.attributes["openinference.span.kind"];
+  return typeof kind === "string" && kind.toUpperCase() !== "LLM";
+}
+function groupSpans(spans) {
+  const m = /* @__PURE__ */ new Map();
+  for (const span of spans) {
+    const key = span["tangle.runId"] ?? span.traceId;
+    const list = m.get(key) ?? [];
+    list.push(span);
+    m.set(key, list);
+  }
+  return m;
+}
+function findRoot(group) {
+  return group.find((s) => !s.parentSpanId) ?? group[0];
+}
+function readAttrString(spans, keys) {
+  for (const span of spans) {
+    for (const key of keys) {
+      const v = span.attributes[key];
+      if (typeof v === "string" && v.length > 0) return v;
+    }
+  }
+  return void 0;
+}
+function readAttrNumber(spans, keys) {
+  for (const span of spans) {
+    for (const key of keys) {
+      const v = span.attributes[key];
+      if (typeof v === "number" && Number.isFinite(v)) return v;
+      if (typeof v === "string") {
+        const parsed = Number(v);
+        if (Number.isFinite(parsed)) return parsed;
+      }
+    }
+  }
+  return void 0;
+}
+function collectNumericAttrs(spans) {
+  const raw = {};
+  for (const span of spans) {
+    for (const [k, v] of Object.entries(span.attributes)) {
+      if (typeof v === "number" && Number.isFinite(v)) raw[k] = v;
+    }
+  }
+  return raw;
+}
+export {
+  FileSystemOutcomeStore,
+  InMemoryOutcomeStore,
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  SelfImproveRunError,
+  analyzeRuns,
+  buildDefaultAnalystRegistry,
+  buildEvidenceVector,
+  campaignSplitDigest,
+  composeGate,
+  createChatClient,
+  createReferenceEquivalenceJudge,
+  defaultProductionGate,
+  defineAgentEval,
+  diffGenerations,
+  diffRunBaselineToWinner,
+  diffRuns,
+  evalReportingSuite,
+  evolutionaryProposer,
+  fromClaudeCodeSession,
+  fromCodexSession,
+  fromFeedbackTable,
+  fromKimiCodeSession,
+  fromOpenCodeSession,
+  fromOtelSpans,
+  fromPiSession,
+  fromPigraphSession,
+  fromRunRecordDir,
+  fsCampaignStorage,
+  gepaProposer,
+  heldOutGate,
+  inMemoryCampaignStorage,
+  measuredComparisonFromSelfImproveResult,
+  paretoPolicy,
+  paretoSignificanceGate,
+  parseAgentTrace,
+  parseCodeAgentJsonl,
+  partitionRunsByAuthoringModel,
+  runCampaign,
+  runEval,
+  runImprovementLoop,
+  runReferenceEquivalenceJudge,
+  selfImprove,
+  summarizeExecution
+};
+//# sourceMappingURL=index.js.map