@tangle-network/agent-eval 0.184.0 → 0.185.0

package/dist/{define-agent-eval-RTekxhUE.js → analyze-runs-sTygaDyD.js}

@@ -1,4 +1,3 @@
-import { s as ValidationError } from "./errors-Dngq5h35.js";
 import { a as spearmanR, r as pearsonR } from "./descriptive-1V17A-qa.js";
 import { r as continuousAgreement } from "./judge-calibration-DYmaBtJr.js";
 import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
@@ -6,15 +5,11 @@ import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-B
 import { r as pairRunRecords } from "./paired-arms-JSDcs2zx.js";
 import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
 import { r as welchsTTest } from "./baseline-BC-eBZ7U.js";
-import { a as campaignCellToRunRecord, i as campaignCellTaskScore, n as campaignCellExecutionEvidence, r as campaignCellJudgeDimensions } from "./run-record-BhMVsHR1.js";
-import { B as surfaceDispatchRef, N as defineEvaluationClaim, T as pairedCampaignComposites, V as surfaceHash, Z as campaignSplitDigest, f as campaignMeasurementDigest, h as loopProvenanceArgsFromResult, k as powerPreflight, m as emitLoopProvenance, p as canonicalDigest, t as createCampaignEvidenceReceipt, tt as aggregatePairedHoldout, z as surfaceContentHash } from "./campaign-evidence-aWEmfROB.js";
+import { n as aggregatePairedHoldout } from "./statistical-heldout-CeWT_W7y.js";
 import "./query-DlV0pdx4.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
-import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-tg_5H2IK.js";
+import { c as validateRunRecord } from "./run-record-tg_5H2IK.js";
 import { r as paretoChart } from "./summary-report-BMvou2oZ.js";
-import { C as assertSearchHistoryAdmissionOptions, G as assertIndependentEvaluationSplit, I as runFinalComparison, J as reserveFinalEvidence, K as captureFinalEvidencePolicy, L as openAutoPr, Q as resolveRunDir, R as captureJudge, Y as runEval, d as combineComparisonCosts, f as costFromLedgerSummary, h as runImprovementLoop, q as evaluationUnitMap, u as executeOptimizationMethod, z as defaultProductionGate } from "./llm-judge-C5wbADA5.js";
-import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-DMgmqTB6.js";
-import { t as createHostedClient } from "./client-CXE-U1SA.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
 	const leaks = [];
@@ -977,808 +972,6 @@ function buildRecommendations(ctx) {
 	return out;
 }
 //#endregion
-//#region src/contract/self-improve-reporting.ts
-function pairedCompositeSummary(baseline, winner, independentUnitByScenarioId) {
-	const scores = pairedCampaignComposites(baseline, winner, independentUnitByScenarioId);
-	const perScenario = (campaign) => Object.fromEntries(Object.entries(campaign.aggregates.byScenario).map(([id, aggregate]) => [id, aggregate.meanComposite]));
-	return {
-		baseline: {
-			compositeMean: scores.beforeMean,
-			perScenario: perScenario(baseline)
-		},
-		winner: {
-			compositeMean: scores.afterMean,
-			perScenario: perScenario(winner)
-		},
-		baselineComposites: scores.before,
-		observations: scores.observations
-	};
-}
-/** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key.
-*
-*  Frozen: the key names a persisted cell, so a change orphans every cell
-*  already written. This is a cell-key function, not a general-purpose hash. */
-function hashString(s) {
-	let h = 2166136261;
-	for (let i = 0; i < s.length; i++) {
-		h ^= s.charCodeAt(i);
-		h = Math.imul(h, 16777619) >>> 0;
-	}
-	return h.toString(16).padStart(8, "0");
-}
-/**
-* Adapt campaign cells into the `RunRecord` shape `analyzeRuns()` consumes.
-* Each cell becomes one run; `candidateId` is the caller-supplied label so
-* baseline + winner pair cleanly on `(experimentId, scenarioId, seed)`.
-*
-* `promptHash` identifies the executed surface; `configHash` identifies the candidate label.
-*/
-function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbackModel) {
-	const promptHash = surfaceContentHash(surface);
-	const configHash = surfaceContentHash(candidateId);
-	return cells.map((cell) => {
-		const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []);
-		if (receiptModels.length > 1) throw new ValidationError(`selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(", ")}`);
-		const model = receiptModels[0] ?? fallbackModel;
-		if (!model) throw new ValidationError(`selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`);
-		if (!modelHasSnapshot(model)) throw new ValidationError(`selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`);
-		return campaignCellToRunRecord(cell, {
-			runId: `${runId}::${candidateId}::${cell.cellId}`,
-			experimentId: runId,
-			candidateId,
-			seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
-			model,
-			promptHash,
-			configHash,
-			commitSha: "cell",
-			splitTag
-		});
-	});
-}
-//#endregion
-//#region src/contract/self-improve-method.ts
-/** Complete methods own search and selection; this path only measures their final choice. */
-async function runSelfImproveMethod(args) {
-	const { opts, train, selection, holdout, costLedger, storage, runDir, startedAt } = args;
-	const budget = opts.budget ?? {};
-	if (opts.autoOnPromote === "pr" && (!opts.ghOwner || !opts.ghRepo)) throw new Error("selfImprove: autoOnPromote='pr' requires ghOwner + ghRepo");
-	const evidenceContext = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
-	if (evidenceContext && budget.holdout === "deferred") throw new Error("selfImprove: evidence receipts require measured holdout");
-	const baselineSurface = structuredClone(opts.baselineSurface);
-	const judge = opts.judge;
-	const { selected, cost: methodCost, history } = await executeOptimizationMethod({
-		method: opts.method,
-		storage,
-		searchHistoryPolicy: opts.searchHistoryPolicy,
-		searchHistoryVerification: opts.searchHistoryVerification,
-		input: {
-			baselineSurface,
-			trainScenarios: train,
-			selectionScenarios: selection,
-			dispatchWithSurface: opts.agent,
-			judges: [judge],
-			runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, "_")}`,
-			seed: 42,
-			runOptions: {
-				storage,
-				maxConcurrency: budget.maxConcurrency ?? 2,
-				reps: budget.reps,
-				dispatchRef: opts.dispatchRef,
-				dispatchTimeoutMs: opts.dispatchTimeoutMs,
-				cellRetry: opts.cellRetry,
-				cellPlacement: opts.cellPlacement,
-				labeledStore: opts.labeledStore,
-				captureSource: opts.captureSource,
-				expectUsage: opts.expectUsage ?? "assert"
-			},
-			costLedger
-		}
-	});
-	const finalPhase = "selfImprove.method-final";
-	const finalCost = () => combineComparisonCosts([
-		"holdout.baseline",
-		"holdout.winner",
-		"holdout.neutralized",
-		"promotion.gate"
-	].map((phase) => ({
-		label: phase,
-		cost: costFromLedgerSummary(costLedger.summary({ phase: `${finalPhase}.${phase}` }))
-	})));
-	const totalCost = () => combineComparisonCosts([{
-		label: opts.method.name,
-		cost: methodCost
-	}, {
-		label: "final comparison",
-		cost: finalCost()
-	}]);
-	const gate = opts.gate ?? defaultProductionGate({
-		holdoutScenarios: holdout,
-		deltaThreshold: opts.claim?.minimumEffect ?? .05,
-		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
-	});
-	const comparison = await runFinalComparison({
-		claim: opts.claim,
-		finalEvidence: opts.finalEvidence,
-		baselineSurface,
-		winnerSurface: selected.winnerSurface,
-		scenarios: holdout,
-		dispatchWithSurface: opts.agent,
-		dispatchRef: opts.dispatchRef,
-		judges: [judge],
-		holdout: budget.holdout,
-		neutralize: opts.neutralize,
-		gate: {
-			...gate,
-			async decide(context) {
-				const cost = totalCost();
-				if (budget.dollars !== void 0 && (!cost.accountingComplete || cost.totalCostUsd > budget.dollars)) return {
-					decision: "hold",
-					reasons: ["complete method spend cannot satisfy the declared run budget"],
-					contributingGates: [{
-						name: "budget",
-						status: "fail",
-						detail: {
-							cost,
-							budgetUsd: budget.dollars
-						}
-					}]
-				};
-				return gate.decide(context);
-			}
-		},
-		runDir,
-		storage,
-		costLedger,
-		costPhase: finalPhase,
-		reps: budget.reps,
-		maxConcurrency: budget.maxConcurrency ?? 2,
-		dispatchTimeoutMs: opts.dispatchTimeoutMs,
-		cellRetry: opts.cellRetry,
-		cellPlacement: opts.cellPlacement,
-		expectUsage: opts.expectUsage ?? "assert",
-		label: "selfImprove"
-	});
-	const evidence = evidenceContext ? {
-		baseline: createCampaignEvidenceReceipt({
-			campaign: comparison.baselineOnHoldout,
-			surface: baselineSurface,
-			context: evidenceContext
-		}),
-		winner: createCampaignEvidenceReceipt({
-			campaign: comparison.winnerOnHoldout,
-			surface: selected.winnerSurface,
-			context: evidenceContext
-		})
-	} : void 0;
-	const deferred = comparison.holdout === "deferred";
-	const report = deferred ? null : pairedCompositeSummary(comparison.baselineOnHoldout, comparison.winnerOnHoldout, opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0);
-	const baseline = report?.baseline ?? null;
-	const winner = report?.winner ?? null;
-	const lift = baseline && winner ? winner.compositeMean - baseline.compositeMean : void 0;
-	const insight = deferred ? void 0 : await analyzeRuns({
-		runs: [...cellsToRunRecords(comparison.baselineOnHoldout.cells, "baseline", runDir, baselineSurface, "holdout", opts.model), ...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? [] : cellsToRunRecords(comparison.winnerOnHoldout.cells, "winner", runDir, selected.winnerSurface, "holdout", opts.model)],
-		baselineCandidateId: "baseline",
-		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0,
-		decisionThreshold: opts.claim?.minimumEffect ?? .05,
-		...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? {} : { candidateCandidateId: "winner" }
-	});
-	const cost = totalCost();
-	const optimization = {
-		name: opts.method.name,
-		cost: methodCost,
-		...selected.durationMs === void 0 ? {} : { durationMs: selected.durationMs },
-		...selected.provenance === void 0 ? {} : { provenance: selected.provenance }
-	};
-	const durationMs = Date.now() - startedAt;
-	const receipts = costLedger.list();
-	const record = {
-		schema: "tangle.method-improvement",
-		runId: `${runDir}#${startedAt}`,
-		runDir,
-		timestamp: new Date(startedAt).toISOString(),
-		baselineContentHash: surfaceContentHash(baselineSurface),
-		winnerContentHash: surfaceContentHash(selected.winnerSurface),
-		diff: comparison.promotedDiff,
-		optimizationMethod: optimization,
-		...opts.claim ? { claim: opts.claim } : {},
-		evidence: {
-			trainSplitDigest: campaignSplitDigest(train, budget.reps ?? 1),
-			selectionSplitDigest: campaignSplitDigest(selection, budget.reps ?? 1),
-			holdoutSplitDigest: campaignSplitDigest(holdout, budget.reps ?? 1),
-			baselineCampaignDigest: campaignMeasurementDigest(comparison.baselineOnHoldout),
-			winnerCampaignDigest: campaignMeasurementDigest(comparison.winnerOnHoldout),
-			costReceiptsDigest: canonicalDigest(receipts),
-			...report ? { holdoutObservations: report.observations } : {},
-			...comparison.neutralizedOnHoldout ? { neutralizedCampaignDigest: campaignMeasurementDigest(comparison.neutralizedOnHoldout) } : {}
-		},
-		gate: comparison.gateResult,
-		...deferred ? { holdout: "deferred" } : {},
-		...baseline && winner ? {
-			baselineHoldoutComposite: baseline.compositeMean,
-			winnerHoldoutComposite: winner.compositeMean,
-			heldOutLift: lift
-		} : {},
-		cost,
-		totalCostUsd: cost.totalCostUsd,
-		totalDurationMs: durationMs
-	};
-	const serialized = JSON.parse(JSON.stringify(record));
-	const provenance = {
-		...serialized,
-		recordDigest: canonicalDigest(serialized)
-	};
-	storage.ensureDir(runDir);
-	storage.write(`${runDir}/method-provenance.json`, JSON.stringify(provenance, null, 2));
-	opts.onProvenance?.(provenance);
-	opts.onProgress?.({
-		kind: "gate.decided",
-		decision: comparison.gateResult.decision,
-		...lift === void 0 ? {} : { lift }
-	});
-	const prResult = opts.autoOnPromote === "pr" && comparison.gateResult.decision === "ship" ? openAutoPr({
-		result: comparison.winnerOnHoldout,
-		gate: comparison.gateResult,
-		promotedDiff: comparison.promotedDiff,
-		ghOwner: opts.ghOwner,
-		ghRepo: opts.ghRepo
-	}) : void 0;
-	const result = {
-		...evidence ? { evidence } : {},
-		mode: "method",
-		...opts.claim ? { claim: opts.claim } : {},
-		...comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {},
-		searchHistoryCoverage: history,
-		baseline,
-		winner: {
-			surface: selected.winnerSurface,
-			compositeMean: winner?.compositeMean ?? null,
-			perScenario: winner?.perScenario ?? {},
-			label: opts.method.name
-		},
-		...lift === void 0 ? {} : { lift },
-		diff: comparison.promotedDiff,
-		gateDecision: comparison.gateResult.decision,
-		provenance,
-		optimization,
-		cost,
-		ledgerCost: costLedger.summary(),
-		totalCostUsd: cost.totalCostUsd,
-		durationMs,
-		receipts,
-		...insight ? { insight } : {},
-		...selected.searchHistory ? { searchHistory: selected.searchHistory } : {},
-		raw: {
-			...comparison,
-			kind: "method",
-			baselineSurface,
-			winnerSurface: selected.winnerSurface,
-			winnerSurfaceHash: surfaceHash(selected.winnerSurface),
-			method: selected,
-			cost,
-			...prResult ? { prResult } : {}
-		}
-	};
-	if (opts.hostedTenant) {
-		const snapshot = (index, surface, campaign, compositeMean) => ({
-			index,
-			surfaceHash: surfaceHash(surface),
-			surface,
-			cells: campaign.cells.map((cell) => {
-				const execution = campaignCellExecutionEvidence(cell);
-				return {
-					scenarioId: cell.scenarioId,
-					rep: cell.rep,
-					compositeMean: campaignCellTaskScore(cell) ?? null,
-					dimensions: campaignCellJudgeDimensions(cell),
-					terminalOutcome: execution.terminalOutcome,
-					executionErrorCount: execution.executionErrorCount ?? null,
-					...cell.error ? { errorMessage: cell.error } : {}
-				};
-			}),
-			compositeMean,
-			costUsd: campaign.aggregates.cost.totalCostUsd,
-			durationMs: campaign.durationMs
-		});
-		try {
-			await createHostedClient(opts.hostedTenant).ingestEvalRun({
-				runId: provenance.runId,
-				runDir,
-				timestamp: provenance.timestamp,
-				status: "finished",
-				labels: {
-					...opts.hostedLabels,
-					mode: "method"
-				},
-				baseline: snapshot(0, baselineSurface, comparison.baselineOnHoldout, baseline?.compositeMean ?? null),
-				generations: [snapshot(1, selected.winnerSurface, comparison.winnerOnHoldout, winner?.compositeMean ?? null)],
-				gateDecision: result.gateDecision,
-				...lift === void 0 ? {} : { holdoutLift: lift },
-				totalCostUsd: result.totalCostUsd,
-				totalDurationMs: durationMs,
-				...insight ? { insightReport: insight } : {}
-			});
-		} catch (error) {
-			console.warn(`[agent-eval] hosted ingest failed (continuing): ${error instanceof Error ? error.message : String(error)}`);
-		}
-	}
-	return result;
-}
-//#endregion
-//#region src/contract/self-improve.ts
-/** Failed self-improvement run with an immutable receipt snapshot. */
-var SelfImproveRunError = class extends Error {
-	cost;
-	receipts;
-	constructor(cause, ledger) {
-		const original = cause instanceof Error ? cause : new Error(String(cause));
-		super(original.message, { cause: original });
-		this.name = "SelfImproveRunError";
-		this.cost = ledger.summary();
-		this.receipts = ledger.list();
-	}
-};
-function assertSelfImproveSearchMode(opts) {
-	assertSearchHistoryAdmissionOptions(opts);
-	if (!opts.method && (opts.searchHistoryPolicy !== void 0 || opts.searchHistoryVerification !== void 0 || opts.evidence !== void 0)) throw new Error("selfImprove: search history admission requires method");
-	if (opts.method && opts.proposer) throw new Error("selfImprove: method and proposer are mutually exclusive");
-	if (!opts.method) {
-		if (opts.selectionScenarios !== void 0) throw new Error("selfImprove: selectionScenarios requires method");
-		return;
-	}
-	if (typeof opts.method.name !== "string" || !opts.method.name.trim() || opts.method.name.trim() !== opts.method.name || typeof opts.method.optimize !== "function") throw new Error("selfImprove: method must have a trimmed name and optimize(input)");
-	const budget = opts.budget;
-	if (budget?.generations !== void 0) throw new Error("selfImprove: method owns its rounds; budget.generations applies only to proposer mode");
-	if (budget?.populationSize !== void 0) throw new Error("selfImprove: method owns its candidates; budget.populationSize applies only to proposer mode");
-	if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0 || opts.selectParent !== void 0 || opts.premeasuredBaseline !== void 0 || opts.selectionRankKey !== void 0 || opts.searchLedger !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, selectParent, premeasuredBaseline, selectionRankKey, and searchLedger apply only to proposer mode");
-}
-function splitMethodPartitions(searchScenarios, explicitSelection, fraction, unitByScenario) {
-	if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
-	const byId = /* @__PURE__ */ new Map();
-	for (const scenario of searchScenarios) {
-		if (byId.has(scenario.id)) throw new Error(`selfImprove: duplicate scenario id '${scenario.id}'`);
-		byId.set(scenario.id, scenario);
-	}
-	if (explicitSelection) {
-		if (explicitSelection.length === 0) throw new Error("selfImprove: selectionScenarios must not be empty");
-		const selectionIds = /* @__PURE__ */ new Set();
-		for (const scenario of explicitSelection) {
-			if (!byId.has(scenario.id)) throw new Error(`selfImprove: selection scenario '${scenario.id}' is absent from the non-final cases`);
-			if (selectionIds.has(scenario.id)) throw new Error(`selfImprove: duplicate selection scenario id '${scenario.id}'`);
-			selectionIds.add(scenario.id);
-		}
-		const train = searchScenarios.filter((scenario) => !selectionIds.has(scenario.id));
-		if (train.length === 0) throw new Error("selfImprove: method train split is empty");
-		if (unitByScenario) {
-			const selectedUnits = new Set([...selectionIds].map((id) => unitByScenario.get(id)));
-			if (train.some((scenario) => selectedUnits.has(unitByScenario.get(scenario.id)))) throw new Error("selfImprove: training and selection share independent units");
-		}
-		return {
-			train,
-			selection: explicitSelection.map((scenario) => byId.get(scenario.id))
-		};
-	}
-	if (searchScenarios.length < 2) throw new Error("selfImprove: method requires at least two non-final scenarios");
-	if (unitByScenario) {
-		const split = splitTrainHoldout(searchScenarios, fraction, unitByScenario);
-		return {
-			train: split.train,
-			selection: split.holdout
-		};
-	}
-	const sorted = [...searchScenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
-	const count = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
-	return {
-		selection: sorted.slice(0, count),
-		train: sorted.slice(count)
-	};
-}
-/** 32-bit FNV-1a over raw UTF-16 code units, read as an unsigned int and used
-*  only to order scenarios deterministically.
-*
-*  Frozen: the order it produces assigns scenarios to partitions, so a change
-*  re-partitions decisions already recorded. It masks no byte, unlike
-*  `fnv1a32` in `src/partition-held-out.ts`, so the two disagree on any
-*  non-ASCII id and cannot be shared. */
-function stableScenarioHash(value) {
-	let hash = 2166136261;
-	for (let index = 0; index < value.length; index++) {
-		hash ^= value.charCodeAt(index);
-		hash = Math.imul(hash, 16777619) >>> 0;
-	}
-	return hash;
-}
-/** Deterministic split by scenario identity, or by source identity for new-unit claims. */
-function splitTrainHoldout(scenarios, fraction, unitByScenario) {
-	if (unitByScenario) {
-		const units = [...new Set(scenarios.map((scenario) => unitByScenario.get(scenario.id)))].sort((a, b) => stableScenarioHash(a) - stableScenarioHash(b) || (a < b ? -1 : a > b ? 1 : 0));
-		if (units.length < 2) throw new Error("selfImprove: splitting needs at least two independent units");
-		const count = Math.max(1, Math.min(units.length - 1, Math.round(units.length * fraction)));
-		const finalUnits = new Set(units.slice(0, count));
-		return {
-			holdout: scenarios.filter((scenario) => finalUnits.has(unitByScenario.get(scenario.id))),
-			train: scenarios.filter((scenario) => !finalUnits.has(unitByScenario.get(scenario.id)))
-		};
-	}
-	const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
-	const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
-	return {
-		holdout: sorted.slice(0, nHoldout),
-		train: sorted.slice(nHoldout)
-	};
-}
-/**
-* Latest search campaign measured for the winner surface; the baseline search
-* campaign when the winner IS the baseline. Used by the deferred-holdout
-* summary, where no holdout campaign exists to summarize.
-*/
-function winnerSearchCampaign(result) {
-	for (let i = result.generations.length - 1; i >= 0; i--) {
-		const measured = result.generations[i]?.surfaces.find((s) => s.surfaceHash === result.winnerSurfaceHash);
-		if (measured) return measured.campaign;
-	}
-	return result.baselineCampaign;
-}
-async function selfImprove(opts) {
-	opts = {
-		...opts,
-		judge: captureJudge(opts.judge),
-		...opts.claim || opts.finalEvidence ? {
-			claim: opts.claim && defineEvaluationClaim(opts.claim),
-			finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence),
-			scenarios: structuredClone(opts.scenarios),
-			selectionScenarios: opts.selectionScenarios && structuredClone(opts.selectionScenarios),
-			budget: opts.budget && structuredClone(opts.budget)
-		} : {}
-	};
-	const startedAt = Date.now();
-	const runDir = resolveRunDir(opts.runDir ?? (opts.method ? `.agent-eval/runs/self-improve-${startedAt}` : `mem://selfImprove-${startedAt}`));
-	const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
-	const costLedger = createRunCostLedger({
-		storage,
-		runDir,
-		costCeilingUsd: opts.budget?.dollars
-	});
-	try {
-		return await runSelfImprove(opts, costLedger, startedAt, runDir, storage);
-	} catch (error) {
-		throw new SelfImproveRunError(error, costLedger);
-	}
-}
-async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
-	const budget = opts.budget ?? {};
-	assertSelfImproveSearchMode(opts);
-	const maxConcurrency = budget.maxConcurrency ?? 2;
-	const holdoutFraction = budget.holdoutFraction ?? .25;
-	const holdoutMode = budget.holdout ?? "measured";
-	const holdoutDeferred = holdoutMode === "deferred";
-	const expectUsage = opts.expectUsage ?? "assert";
-	const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.scenarios) : void 0;
-	const splitUnitByScenario = opts.claim?.generalization === "new-units" ? unitByScenario : void 0;
-	const explicitHoldout = budget.holdoutScenarios;
-	const { train, holdout } = explicitHoldout ? {
-		train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
-		holdout: explicitHoldout
-	} : holdoutDeferred ? {
-		train: opts.scenarios,
-		holdout: []
-	} : splitTrainHoldout(opts.scenarios, holdoutFraction, splitUnitByScenario);
-	if (train.length === 0) throw new Error("selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios.");
-	if (holdout.length === 0 && !holdoutDeferred) throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
-	if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, holdout, train);
-	if (opts.finalEvidence) {
-		if (holdoutDeferred) throw new Error("final evidence requires measured holdout");
-		if (opts.method) await reserveFinalEvidence(opts.finalEvidence, opts.claim, holdout, train);
-	}
-	if (opts.method) {
-		const partitions = splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? .25, splitUnitByScenario);
-		return runSelfImproveMethod({
-			opts: {
-				...opts,
-				method: opts.method
-			},
-			train: partitions.train,
-			selection: partitions.selection,
-			holdout,
-			costLedger,
-			storage,
-			runDir,
-			startedAt
-		});
-	}
-	const generations = budget.generations ?? 3;
-	const populationSize = budget.populationSize ?? 2;
-	if (generations > 0 && !opts.proposer) throw new Error("selfImprove: method or proposer is required when budget.generations is greater than zero");
-	const proposer = opts.proposer ?? {
-		kind: "baseline-only",
-		propose: async () => []
-	};
-	const gate = opts.gate ?? defaultProductionGate({
-		holdoutScenarios: holdout,
-		deltaThreshold: opts.claim?.minimumEffect ?? .05,
-		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
-	});
-	if (opts.onProgress) opts.onProgress({
-		kind: "baseline.started",
-		scenarios: opts.scenarios.length
-	});
-	const result = await runImprovementLoop({
-		scenarios: train,
-		baselineSurface: opts.baselineSurface,
-		premeasuredBaseline: opts.premeasuredBaseline,
-		dispatchWithSurface: opts.agent,
-		dispatchRef: opts.dispatchRef,
-		proposer,
-		judges: [opts.judge],
-		populationSize,
-		maxGenerations: generations,
-		candidateConcurrency: budget.candidateConcurrency,
-		reps: budget.reps,
-		maxImprovementShots: budget.maxImprovementShots,
-		holdoutScenarios: holdout,
-		claim: opts.claim,
-		finalEvidence: opts.finalEvidence,
-		holdout: holdoutMode,
-		gate,
-		neutralize: opts.neutralize,
-		autoOnPromote: opts.autoOnPromote ?? "none",
-		ghOwner: opts.ghOwner,
-		ghRepo: opts.ghRepo,
-		storage,
-		runDir,
-		maxConcurrency,
-		cellPlacement: opts.cellPlacement,
-		dispatchTimeoutMs: opts.dispatchTimeoutMs,
-		cellRetry: opts.cellRetry,
-		costLedger,
-		expectUsage,
-		labeledStore: opts.labeledStore,
-		captureSource: opts.captureSource,
-		analyzeGeneration: opts.analyzeGeneration,
-		findings: opts.findings,
-		selectionRankKey: opts.selectionRankKey,
-		selectParent: opts.selectParent,
-		searchLedger: opts.searchLedger
-	});
-	const reportSplit = holdoutDeferred ? "search" : "holdout";
-	const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
-	const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
-	const reportUnitMap = opts.claim ? evaluationUnitMap(opts.claim, holdoutDeferred ? train : holdout) : void 0;
-	const report = pairedCompositeSummary(reportBaselineCampaign, reportWinnerCampaign, reportUnitMap);
-	const baseline = report.baseline;
-	const winnerStats = report.winner;
-	let power;
-	const baselineHoldoutComposites = holdoutDeferred ? [] : report.baselineComposites;
-	if (baselineHoldoutComposites.length >= 3) {
-		power = powerPreflight({
-			baselineComposites: baselineHoldoutComposites,
-			deltaThreshold: opts.claim?.minimumEffect ?? .05,
-			sharedScorerChannel: true
-		});
-		if (opts.onProgress) opts.onProgress({
-			kind: "power.estimated",
-			n: power.n,
-			sd: power.sd,
-			mde: power.mde,
-			underpowered: power.underpowered
-		});
-		if (power.underpowered && generations > 0) console.warn(`[selfImprove] ${power.recommendation}`);
-	}
-	if (opts.onProgress) {
-		opts.onProgress({
-			kind: "baseline.completed",
-			compositeMean: baseline.compositeMean,
-			durationMs: Date.now() - startedAt
-		});
-		opts.onProgress({
-			kind: "gate.decided",
-			decision: result.gateResult.decision,
-			...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean }
-		});
-	}
-	const cost = result.cost;
-	const totalCost = cost.totalCostUsd;
-	const insight = await analyzeRuns({
-		runs: [...cellsToRunRecords(reportBaselineCampaign.cells, "baseline", runDir, opts.baselineSurface, reportSplit, opts.model), ...reportWinnerCampaign === reportBaselineCampaign ? [] : cellsToRunRecords(reportWinnerCampaign.cells, "winner", runDir, result.winnerSurface, reportSplit, opts.model)],
-		baselineCandidateId: "baseline",
-		independentUnitByScenarioId: reportUnitMap,
-		decisionThreshold: opts.claim?.minimumEffect ?? .05,
-		...reportWinnerCampaign === reportBaselineCampaign ? {} : { candidateCandidateId: "winner" }
-	});
-	const durationMs = Date.now() - startedAt;
-	const { record: provenance } = await emitLoopProvenance({
-		...loopProvenanceArgsFromResult({
-			runId: `${runDir}#${startedAt}`,
-			runDir,
-			timestamp: new Date(startedAt).toISOString(),
-			baselineSurface: opts.baselineSurface,
-			result,
-			costReceipts: costLedger.list(),
-			totalCostUsd: totalCost,
-			totalDurationMs: durationMs,
-			independentUnitByScenarioId: holdoutDeferred ? void 0 : reportUnitMap
-		}),
-		storage,
-		hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
-	});
-	if (opts.onProvenance) opts.onProvenance(provenance);
-	const summary = {
-		...opts.claim ? { claim: opts.claim } : {},
-		...result.finalEvidence ? { finalEvidence: result.finalEvidence } : {},
-		mode: "proposer",
-		baseline,
-		winner: {
-			...winnerStats,
-			surface: result.winnerSurface,
-			...result.winnerLabel ? { label: result.winnerLabel } : {},
-			...result.winnerRationale ? { rationale: result.winnerRationale } : {}
-		},
-		...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean },
-		diff: result.promotedDiff,
-		provenance,
-		gateDecision: result.gateResult.decision,
-		generationsExplored: result.generations.length,
-		durationMs,
-		totalCostUsd: totalCost,
-		cost,
-		receipts: costLedger.list(),
-		...result.searchHistory ? { searchHistory: result.searchHistory } : {},
-		insight,
-		...power ? { power } : {},
-		raw: result
-	};
-	if (opts.hostedTenant) try {
-		await shipEvalRunToHosted(opts.hostedTenant, opts, summary, result, runDir);
-	} catch (err) {
-		const msg = err instanceof Error ? err.message : String(err);
-		console.warn(`[agent-eval] hosted ingest failed (continuing): ${msg}`);
-	}
-	return summary;
-}
-async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
-	const client = createHostedClient(tenant);
-	function snapshotFromCampaign(index, surface, campaign, durationMs) {
-		const cells = campaign.cells.map((cell) => {
-			const execution = campaignCellExecutionEvidence(cell);
-			return {
-				scenarioId: cell.scenarioId,
-				rep: cell.rep,
-				compositeMean: campaignCellTaskScore(cell) ?? null,
-				dimensions: campaignCellJudgeDimensions(cell),
-				terminalOutcome: execution.terminalOutcome,
-				executionErrorCount: execution.executionErrorCount ?? null,
-				errorMessage: cell.error ?? void 0
-			};
-		});
-		const scoredCells = cells.flatMap((cell) => cell.compositeMean === null ? [] : [cell.compositeMean]);
-		const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
-		return {
-			index,
-			surfaceHash: surfaceHash(surface),
-			surface,
-			cells,
-			compositeMean,
-			costUsd: campaign.aggregates.cost.totalCostUsd,
-			durationMs
-		};
-	}
-	const generations = [];
-	generations.push(snapshotFromCampaign(0, opts.baselineSurface, raw.baselineCampaign, 0));
-	for (const gen of raw.generations) {
-		const winner = gen.surfaces.reduce((best, s) => s.campaign.aggregates.cellsExecuted > 0 && (best === void 0 || averageComposite(s.campaign) > averageComposite(best.campaign)) ? s : best, gen.surfaces[0]);
-		if (!winner) continue;
-		generations.push(snapshotFromCampaign(gen.record.generationIndex + 1, winner.surface, winner.campaign, 0));
-	}
-	const event = {
-		runId: `${runDir}#${Date.now()}`,
-		runDir,
-		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-		status: "finished",
-		labels: opts.hostedLabels ?? {},
-		baseline: generations[0],
-		generations,
-		gateDecision: summary.gateDecision,
-		holdoutLift: summary.lift,
-		totalCostUsd: summary.totalCostUsd,
-		totalDurationMs: summary.durationMs,
-		insightReport: summary.insight
-	};
-	await client.ingestEvalRun(event);
-}
-function averageComposite(campaign) {
-	const aggs = Object.values(campaign.aggregates.byScenario);
-	return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length;
-}
-//#endregion
-//#region src/contract/define-agent-eval.ts
-/**
-* Define an agent eval once, then either score a surface with `evaluate()` or
-* run the closed loop with `improve()`.
-*
-* This is a DX wrapper only: it delegates to `runEval()` and `selfImprove()` and
-* returns their native result shapes.
-*/
-function defineAgentEval(defaults) {
-	const defaultEvaluateOptions = evaluateDefaults(defaults);
-	return {
-		scenarios: defaults.scenarios,
-		baselineSurface: defaults.baselineSurface,
-		async evaluate(opts = {}) {
-			const { agent, judge, judges, runDir, scenarios, surface, ...campaignOpts } = opts;
-			const selectedAgent = agent ?? defaults.agent;
-			const selectedSurface = surface ?? defaults.baselineSurface;
-			const selectedRunDir = runDir ?? defaults.runDir ?? `mem://defineAgentEval-${Date.now()}`;
-			const selectedStorage = campaignOpts.storage ?? defaultEvaluateOptions.storage ?? (selectedRunDir.startsWith("mem://") ? inMemoryCampaignStorage() : void 0);
-			const evalOptions = {
-				...defaultEvaluateOptions,
-				...campaignOpts,
-				...selectedStorage ? { storage: selectedStorage } : {},
-				runDir: selectedRunDir,
-				scenarios: scenarios ?? defaults.scenarios,
-				dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx),
-				dispatchRef: surfaceDispatchRef(selectedSurface, campaignOpts.dispatchRef ?? defaults.dispatchRef),
-				judges: evaluateJudges(judges, judge ?? defaults.judge)
-			};
-			if (evalOptions.reps !== void 0) evalOptions.reps = requirePositiveInteger(evalOptions.reps, "reps");
-			return runEval(evalOptions);
-		},
-		async improve(opts = {}) {
-			const { budget: budgetOverride, hostedTenant: hostedTenantOverride, ...topLevelOverrides } = opts;
-			const merged = mergeDefined(defaults, topLevelOverrides);
-			const budget = mergeBudget(defaults.budget, budgetOverride);
-			const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
-			return selfImprove({
-				...merged,
-				...budget ? { budget } : {},
-				...hostedTenant ? { hostedTenant } : {}
-			});
-		}
-	};
-}
-function evaluateDefaults(defaults) {
-	const out = {};
-	if (defaults.storage) out.storage = defaults.storage;
-	if (defaults.labeledStore) out.labeledStore = defaults.labeledStore;
-	if (defaults.captureSource) out.captureSource = defaults.captureSource;
-	if (defaults.cellPlacement) out.cellPlacement = defaults.cellPlacement;
-	if (defaults.expectUsage) out.expectUsage = defaults.expectUsage;
-	if (defaults.budget?.dollars !== void 0) out.costCeiling = defaults.budget.dollars;
-	if (defaults.budget?.maxConcurrency !== void 0) out.maxConcurrency = defaults.budget.maxConcurrency;
-	if (defaults.budget?.reps !== void 0) out.reps = requirePositiveInteger(defaults.budget.reps, "budget.reps");
-	return out;
-}
-function mergeBudget(defaults, overrides) {
-	const merged = mergeOptionalObject(defaults, overrides);
-	if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
-	return merged;
-}
-function mergeHostedTenant(defaults, overrides) {
-	const merged = mergeOptionalObject(defaults, overrides);
-	if (!merged) return void 0;
-	if (!merged.endpoint?.trim() || !merged.apiKey?.trim() || !merged.tenantId?.trim()) throw new Error("defineAgentEval.improve: hostedTenant requires endpoint, apiKey, and tenantId after merging defaults and overrides");
-	return merged;
-}
-function mergeDefined(defaults, overrides) {
-	if (!overrides) return defaults;
-	const merged = { ...defaults };
-	for (const [key, value] of Object.entries(overrides)) if (value !== void 0) merged[key] = value;
-	return merged;
-}
-function mergeOptionalObject(defaults, overrides) {
-	if (!defaults && !overrides) return void 0;
-	return mergeDefined(defaults ?? {}, overrides);
-}
-function evaluateJudges(judges, defaultJudge) {
-	if (judges !== void 0) {
-		if (judges.length === 0) throw new Error("defineAgentEval.evaluate: judges must not be empty");
-		return judges;
-	}
-	return [defaultJudge];
-}
-function requirePositiveInteger(value, field) {
-	if (!Number.isInteger(value) || value < 1) throw new Error(`defineAgentEval: ${field} must be a positive integer`);
-	return value;
-}
-//#endregion
-export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
+export { summarizeExecution as n, checkCanaries as r, analyzeRuns as t };
 
-//# sourceMappingURL=define-agent-eval-RTekxhUE.js.map
+//# sourceMappingURL=analyze-runs-sTygaDyD.js.map