@tangle-network/agent-eval 0.183.0 → 0.185.0

package/dist/define-agent-eval-D4uumjl3.js

@@ -0,0 +1,813 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { t as analyzeRuns } from "./analyze-runs-sTygaDyD.js";
+import { i as modelHasSnapshot } from "./run-record-tg_5H2IK.js";
+import { C as assertSearchHistoryAdmissionOptions, G as assertIndependentEvaluationSplit, I as runFinalComparison, J as reserveFinalEvidence, K as captureFinalEvidencePolicy, L as openAutoPr, Q as resolveRunDir, R as captureJudge, Y as runEval, d as combineComparisonCosts, f as costFromLedgerSummary, h as runImprovementLoop, q as evaluationUnitMap, u as executeOptimizationMethod, z as defaultProductionGate } from "./llm-judge-FWWhEb31.js";
+import { a as campaignCellToRunRecord, i as campaignCellTaskScore, n as campaignCellExecutionEvidence, r as campaignCellJudgeDimensions } from "./run-record-CNNzWGpB.js";
+import { B as surfaceDispatchRef, N as defineEvaluationClaim, T as pairedCampaignComposites, V as surfaceHash, Z as campaignSplitDigest, f as campaignMeasurementDigest, h as loopProvenanceArgsFromResult, k as powerPreflight, m as emitLoopProvenance, p as canonicalDigest, t as createCampaignEvidenceReceipt, z as surfaceContentHash } from "./campaign-evidence-BsrCmBSE.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-DMgmqTB6.js";
+import { t as createHostedClient } from "./client-CXE-U1SA.js";
+//#region src/contract/self-improve-reporting.ts
+function pairedCompositeSummary(baseline, winner, independentUnitByScenarioId) {
+	const scores = pairedCampaignComposites(baseline, winner, independentUnitByScenarioId);
+	const perScenario = (campaign) => Object.fromEntries(Object.entries(campaign.aggregates.byScenario).map(([id, aggregate]) => [id, aggregate.meanComposite]));
+	return {
+		baseline: {
+			compositeMean: scores.beforeMean,
+			perScenario: perScenario(baseline)
+		},
+		winner: {
+			compositeMean: scores.afterMean,
+			perScenario: perScenario(winner)
+		},
+		baselineComposites: scores.before,
+		observations: scores.observations
+	};
+}
+/** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key.
+*
+*  Frozen: the key names a persisted cell, so a change orphans every cell
+*  already written. This is a cell-key function, not a general-purpose hash. */
+function hashString(s) {
+	let h = 2166136261;
+	for (let i = 0; i < s.length; i++) {
+		h ^= s.charCodeAt(i);
+		h = Math.imul(h, 16777619) >>> 0;
+	}
+	return h.toString(16).padStart(8, "0");
+}
+/**
+* Adapt campaign cells into the `RunRecord` shape `analyzeRuns()` consumes.
+* Each cell becomes one run; `candidateId` is the caller-supplied label so
+* baseline + winner pair cleanly on `(experimentId, scenarioId, seed)`.
+*
+* `promptHash` identifies the executed surface; `configHash` identifies the candidate label.
+*/
+function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbackModel) {
+	const promptHash = surfaceContentHash(surface);
+	const configHash = surfaceContentHash(candidateId);
+	return cells.map((cell) => {
+		const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []);
+		if (receiptModels.length > 1) throw new ValidationError(`selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(", ")}`);
+		const model = receiptModels[0] ?? fallbackModel;
+		if (!model) throw new ValidationError(`selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`);
+		if (!modelHasSnapshot(model)) throw new ValidationError(`selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`);
+		return campaignCellToRunRecord(cell, {
+			runId: `${runId}::${candidateId}::${cell.cellId}`,
+			experimentId: runId,
+			candidateId,
+			seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
+			model,
+			promptHash,
+			configHash,
+			commitSha: "cell",
+			splitTag
+		});
+	});
+}
+//#endregion
+//#region src/contract/self-improve-method.ts
+/** Complete methods own search and selection; this path only measures their final choice. */
+async function runSelfImproveMethod(args) {
+	const { opts, train, selection, holdout, costLedger, storage, runDir, startedAt } = args;
+	const budget = opts.budget ?? {};
+	if (opts.autoOnPromote === "pr" && (!opts.ghOwner || !opts.ghRepo)) throw new Error("selfImprove: autoOnPromote='pr' requires ghOwner + ghRepo");
+	const evidenceContext = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
+	if (evidenceContext && budget.holdout === "deferred") throw new Error("selfImprove: evidence receipts require measured holdout");
+	const baselineSurface = structuredClone(opts.baselineSurface);
+	const judge = opts.judge;
+	const { selected, cost: methodCost, history } = await executeOptimizationMethod({
+		method: opts.method,
+		storage,
+		searchHistoryPolicy: opts.searchHistoryPolicy,
+		searchHistoryVerification: opts.searchHistoryVerification,
+		input: {
+			baselineSurface,
+			trainScenarios: train,
+			selectionScenarios: selection,
+			dispatchWithSurface: opts.agent,
+			judges: [judge],
+			runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, "_")}`,
+			seed: 42,
+			runOptions: {
+				storage,
+				maxConcurrency: budget.maxConcurrency ?? 2,
+				reps: budget.reps,
+				dispatchRef: opts.dispatchRef,
+				dispatchTimeoutMs: opts.dispatchTimeoutMs,
+				cellRetry: opts.cellRetry,
+				cellPlacement: opts.cellPlacement,
+				labeledStore: opts.labeledStore,
+				captureSource: opts.captureSource,
+				expectUsage: opts.expectUsage ?? "assert"
+			},
+			costLedger
+		}
+	});
+	const finalPhase = "selfImprove.method-final";
+	const finalCost = () => combineComparisonCosts([
+		"holdout.baseline",
+		"holdout.winner",
+		"holdout.neutralized",
+		"promotion.gate"
+	].map((phase) => ({
+		label: phase,
+		cost: costFromLedgerSummary(costLedger.summary({ phase: `${finalPhase}.${phase}` }))
+	})));
+	const totalCost = () => combineComparisonCosts([{
+		label: opts.method.name,
+		cost: methodCost
+	}, {
+		label: "final comparison",
+		cost: finalCost()
+	}]);
+	const gate = opts.gate ?? defaultProductionGate({
+		holdoutScenarios: holdout,
+		deltaThreshold: opts.claim?.minimumEffect ?? .05,
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
+	});
+	const comparison = await runFinalComparison({
+		claim: opts.claim,
+		finalEvidence: opts.finalEvidence,
+		baselineSurface,
+		winnerSurface: selected.winnerSurface,
+		scenarios: holdout,
+		dispatchWithSurface: opts.agent,
+		dispatchRef: opts.dispatchRef,
+		judges: [judge],
+		holdout: budget.holdout,
+		neutralize: opts.neutralize,
+		gate: {
+			...gate,
+			async decide(context) {
+				const cost = totalCost();
+				if (budget.dollars !== void 0 && (!cost.accountingComplete || cost.totalCostUsd > budget.dollars)) return {
+					decision: "hold",
+					reasons: ["complete method spend cannot satisfy the declared run budget"],
+					contributingGates: [{
+						name: "budget",
+						status: "fail",
+						detail: {
+							cost,
+							budgetUsd: budget.dollars
+						}
+					}]
+				};
+				return gate.decide(context);
+			}
+		},
+		runDir,
+		storage,
+		costLedger,
+		costPhase: finalPhase,
+		reps: budget.reps,
+		maxConcurrency: budget.maxConcurrency ?? 2,
+		dispatchTimeoutMs: opts.dispatchTimeoutMs,
+		cellRetry: opts.cellRetry,
+		cellPlacement: opts.cellPlacement,
+		expectUsage: opts.expectUsage ?? "assert",
+		label: "selfImprove"
+	});
+	const evidence = evidenceContext ? {
+		baseline: createCampaignEvidenceReceipt({
+			campaign: comparison.baselineOnHoldout,
+			surface: baselineSurface,
+			context: evidenceContext
+		}),
+		winner: createCampaignEvidenceReceipt({
+			campaign: comparison.winnerOnHoldout,
+			surface: selected.winnerSurface,
+			context: evidenceContext
+		})
+	} : void 0;
+	const deferred = comparison.holdout === "deferred";
+	const report = deferred ? null : pairedCompositeSummary(comparison.baselineOnHoldout, comparison.winnerOnHoldout, opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0);
+	const baseline = report?.baseline ?? null;
+	const winner = report?.winner ?? null;
+	const lift = baseline && winner ? winner.compositeMean - baseline.compositeMean : void 0;
+	const insight = deferred ? void 0 : await analyzeRuns({
+		runs: [...cellsToRunRecords(comparison.baselineOnHoldout.cells, "baseline", runDir, baselineSurface, "holdout", opts.model), ...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? [] : cellsToRunRecords(comparison.winnerOnHoldout.cells, "winner", runDir, selected.winnerSurface, "holdout", opts.model)],
+		baselineCandidateId: "baseline",
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0,
+		decisionThreshold: opts.claim?.minimumEffect ?? .05,
+		...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? {} : { candidateCandidateId: "winner" }
+	});
+	const cost = totalCost();
+	const optimization = {
+		name: opts.method.name,
+		cost: methodCost,
+		...selected.durationMs === void 0 ? {} : { durationMs: selected.durationMs },
+		...selected.provenance === void 0 ? {} : { provenance: selected.provenance }
+	};
+	const durationMs = Date.now() - startedAt;
+	const receipts = costLedger.list();
+	const record = {
+		schema: "tangle.method-improvement",
+		runId: `${runDir}#${startedAt}`,
+		runDir,
+		timestamp: new Date(startedAt).toISOString(),
+		baselineContentHash: surfaceContentHash(baselineSurface),
+		winnerContentHash: surfaceContentHash(selected.winnerSurface),
+		diff: comparison.promotedDiff,
+		optimizationMethod: optimization,
+		...opts.claim ? { claim: opts.claim } : {},
+		evidence: {
+			trainSplitDigest: campaignSplitDigest(train, budget.reps ?? 1),
+			selectionSplitDigest: campaignSplitDigest(selection, budget.reps ?? 1),
+			holdoutSplitDigest: campaignSplitDigest(holdout, budget.reps ?? 1),
+			baselineCampaignDigest: campaignMeasurementDigest(comparison.baselineOnHoldout),
+			winnerCampaignDigest: campaignMeasurementDigest(comparison.winnerOnHoldout),
+			costReceiptsDigest: canonicalDigest(receipts),
+			...report ? { holdoutObservations: report.observations } : {},
+			...comparison.neutralizedOnHoldout ? { neutralizedCampaignDigest: campaignMeasurementDigest(comparison.neutralizedOnHoldout) } : {}
+		},
+		gate: comparison.gateResult,
+		...deferred ? { holdout: "deferred" } : {},
+		...baseline && winner ? {
+			baselineHoldoutComposite: baseline.compositeMean,
+			winnerHoldoutComposite: winner.compositeMean,
+			heldOutLift: lift
+		} : {},
+		cost,
+		totalCostUsd: cost.totalCostUsd,
+		totalDurationMs: durationMs
+	};
+	const serialized = JSON.parse(JSON.stringify(record));
+	const provenance = {
+		...serialized,
+		recordDigest: canonicalDigest(serialized)
+	};
+	storage.ensureDir(runDir);
+	storage.write(`${runDir}/method-provenance.json`, JSON.stringify(provenance, null, 2));
+	opts.onProvenance?.(provenance);
+	opts.onProgress?.({
+		kind: "gate.decided",
+		decision: comparison.gateResult.decision,
+		...lift === void 0 ? {} : { lift }
+	});
+	const prResult = opts.autoOnPromote === "pr" && comparison.gateResult.decision === "ship" ? openAutoPr({
+		result: comparison.winnerOnHoldout,
+		gate: comparison.gateResult,
+		promotedDiff: comparison.promotedDiff,
+		ghOwner: opts.ghOwner,
+		ghRepo: opts.ghRepo
+	}) : void 0;
+	const result = {
+		...evidence ? { evidence } : {},
+		mode: "method",
+		...opts.claim ? { claim: opts.claim } : {},
+		...comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {},
+		searchHistoryCoverage: history,
+		baseline,
+		winner: {
+			surface: selected.winnerSurface,
+			compositeMean: winner?.compositeMean ?? null,
+			perScenario: winner?.perScenario ?? {},
+			label: opts.method.name
+		},
+		...lift === void 0 ? {} : { lift },
+		diff: comparison.promotedDiff,
+		gateDecision: comparison.gateResult.decision,
+		provenance,
+		optimization,
+		cost,
+		ledgerCost: costLedger.summary(),
+		totalCostUsd: cost.totalCostUsd,
+		durationMs,
+		receipts,
+		...insight ? { insight } : {},
+		...selected.searchHistory ? { searchHistory: selected.searchHistory } : {},
+		raw: {
+			...comparison,
+			kind: "method",
+			baselineSurface,
+			winnerSurface: selected.winnerSurface,
+			winnerSurfaceHash: surfaceHash(selected.winnerSurface),
+			method: selected,
+			cost,
+			...prResult ? { prResult } : {}
+		}
+	};
+	if (opts.hostedTenant) {
+		const snapshot = (index, surface, campaign, compositeMean) => ({
+			index,
+			surfaceHash: surfaceHash(surface),
+			surface,
+			cells: campaign.cells.map((cell) => {
+				const execution = campaignCellExecutionEvidence(cell);
+				return {
+					scenarioId: cell.scenarioId,
+					rep: cell.rep,
+					compositeMean: campaignCellTaskScore(cell) ?? null,
+					dimensions: campaignCellJudgeDimensions(cell),
+					terminalOutcome: execution.terminalOutcome,
+					executionErrorCount: execution.executionErrorCount ?? null,
+					...cell.error ? { errorMessage: cell.error } : {}
+				};
+			}),
+			compositeMean,
+			costUsd: campaign.aggregates.cost.totalCostUsd,
+			durationMs: campaign.durationMs
+		});
+		try {
+			await createHostedClient(opts.hostedTenant).ingestEvalRun({
+				runId: provenance.runId,
+				runDir,
+				timestamp: provenance.timestamp,
+				status: "finished",
+				labels: {
+					...opts.hostedLabels,
+					mode: "method"
+				},
+				baseline: snapshot(0, baselineSurface, comparison.baselineOnHoldout, baseline?.compositeMean ?? null),
+				generations: [snapshot(1, selected.winnerSurface, comparison.winnerOnHoldout, winner?.compositeMean ?? null)],
+				gateDecision: result.gateDecision,
+				...lift === void 0 ? {} : { holdoutLift: lift },
+				totalCostUsd: result.totalCostUsd,
+				totalDurationMs: durationMs,
+				...insight ? { insightReport: insight } : {}
+			});
+		} catch (error) {
+			console.warn(`[agent-eval] hosted ingest failed (continuing): ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+	return result;
+}
+//#endregion
+//#region src/contract/self-improve.ts
+/** Failed self-improvement run with an immutable receipt snapshot. */
+var SelfImproveRunError = class extends Error {
+	cost;
+	receipts;
+	constructor(cause, ledger) {
+		const original = cause instanceof Error ? cause : new Error(String(cause));
+		super(original.message, { cause: original });
+		this.name = "SelfImproveRunError";
+		this.cost = ledger.summary();
+		this.receipts = ledger.list();
+	}
+};
+function assertSelfImproveSearchMode(opts) {
+	assertSearchHistoryAdmissionOptions(opts);
+	if (!opts.method && (opts.searchHistoryPolicy !== void 0 || opts.searchHistoryVerification !== void 0 || opts.evidence !== void 0)) throw new Error("selfImprove: search history admission requires method");
+	if (opts.method && opts.proposer) throw new Error("selfImprove: method and proposer are mutually exclusive");
+	if (!opts.method) {
+		if (opts.selectionScenarios !== void 0) throw new Error("selfImprove: selectionScenarios requires method");
+		return;
+	}
+	if (typeof opts.method.name !== "string" || !opts.method.name.trim() || opts.method.name.trim() !== opts.method.name || typeof opts.method.optimize !== "function") throw new Error("selfImprove: method must have a trimmed name and optimize(input)");
+	const budget = opts.budget;
+	if (budget?.generations !== void 0) throw new Error("selfImprove: method owns its rounds; budget.generations applies only to proposer mode");
+	if (budget?.populationSize !== void 0) throw new Error("selfImprove: method owns its candidates; budget.populationSize applies only to proposer mode");
+	if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0 || opts.selectParent !== void 0 || opts.premeasuredBaseline !== void 0 || opts.selectionRankKey !== void 0 || opts.searchLedger !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, selectParent, premeasuredBaseline, selectionRankKey, and searchLedger apply only to proposer mode");
+}
+function splitMethodPartitions(searchScenarios, explicitSelection, fraction, unitByScenario) {
+	if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
+	const byId = /* @__PURE__ */ new Map();
+	for (const scenario of searchScenarios) {
+		if (byId.has(scenario.id)) throw new Error(`selfImprove: duplicate scenario id '${scenario.id}'`);
+		byId.set(scenario.id, scenario);
+	}
+	if (explicitSelection) {
+		if (explicitSelection.length === 0) throw new Error("selfImprove: selectionScenarios must not be empty");
+		const selectionIds = /* @__PURE__ */ new Set();
+		for (const scenario of explicitSelection) {
+			if (!byId.has(scenario.id)) throw new Error(`selfImprove: selection scenario '${scenario.id}' is absent from the non-final cases`);
+			if (selectionIds.has(scenario.id)) throw new Error(`selfImprove: duplicate selection scenario id '${scenario.id}'`);
+			selectionIds.add(scenario.id);
+		}
+		const train = searchScenarios.filter((scenario) => !selectionIds.has(scenario.id));
+		if (train.length === 0) throw new Error("selfImprove: method train split is empty");
+		if (unitByScenario) {
+			const selectedUnits = new Set([...selectionIds].map((id) => unitByScenario.get(id)));
+			if (train.some((scenario) => selectedUnits.has(unitByScenario.get(scenario.id)))) throw new Error("selfImprove: training and selection share independent units");
+		}
+		return {
+			train,
+			selection: explicitSelection.map((scenario) => byId.get(scenario.id))
+		};
+	}
+	if (searchScenarios.length < 2) throw new Error("selfImprove: method requires at least two non-final scenarios");
+	if (unitByScenario) {
+		const split = splitTrainHoldout(searchScenarios, fraction, unitByScenario);
+		return {
+			train: split.train,
+			selection: split.holdout
+		};
+	}
+	const sorted = [...searchScenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
+	const count = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+	return {
+		selection: sorted.slice(0, count),
+		train: sorted.slice(count)
+	};
+}
+/** 32-bit FNV-1a over raw UTF-16 code units, read as an unsigned int and used
+*  only to order scenarios deterministically.
+*
+*  Frozen: the order it produces assigns scenarios to partitions, so a change
+*  re-partitions decisions already recorded. It masks no byte, unlike
+*  `fnv1a32` in `src/partition-held-out.ts`, so the two disagree on any
+*  non-ASCII id and cannot be shared. */
+function stableScenarioHash(value) {
+	let hash = 2166136261;
+	for (let index = 0; index < value.length; index++) {
+		hash ^= value.charCodeAt(index);
+		hash = Math.imul(hash, 16777619) >>> 0;
+	}
+	return hash;
+}
+/** Deterministic split by scenario identity, or by source identity for new-unit claims. */
+function splitTrainHoldout(scenarios, fraction, unitByScenario) {
+	if (unitByScenario) {
+		const units = [...new Set(scenarios.map((scenario) => unitByScenario.get(scenario.id)))].sort((a, b) => stableScenarioHash(a) - stableScenarioHash(b) || (a < b ? -1 : a > b ? 1 : 0));
+		if (units.length < 2) throw new Error("selfImprove: splitting needs at least two independent units");
+		const count = Math.max(1, Math.min(units.length - 1, Math.round(units.length * fraction)));
+		const finalUnits = new Set(units.slice(0, count));
+		return {
+			holdout: scenarios.filter((scenario) => finalUnits.has(unitByScenario.get(scenario.id))),
+			train: scenarios.filter((scenario) => !finalUnits.has(unitByScenario.get(scenario.id)))
+		};
+	}
+	const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
+	const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+	return {
+		holdout: sorted.slice(0, nHoldout),
+		train: sorted.slice(nHoldout)
+	};
+}
+/**
+* Latest search campaign measured for the winner surface; the baseline search
+* campaign when the winner IS the baseline. Used by the deferred-holdout
+* summary, where no holdout campaign exists to summarize.
+*/
+function winnerSearchCampaign(result) {
+	for (let i = result.generations.length - 1; i >= 0; i--) {
+		const measured = result.generations[i]?.surfaces.find((s) => s.surfaceHash === result.winnerSurfaceHash);
+		if (measured) return measured.campaign;
+	}
+	return result.baselineCampaign;
+}
+async function selfImprove(opts) {
+	opts = {
+		...opts,
+		judge: captureJudge(opts.judge),
+		...opts.claim || opts.finalEvidence ? {
+			claim: opts.claim && defineEvaluationClaim(opts.claim),
+			finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence),
+			scenarios: structuredClone(opts.scenarios),
+			selectionScenarios: opts.selectionScenarios && structuredClone(opts.selectionScenarios),
+			budget: opts.budget && structuredClone(opts.budget)
+		} : {}
+	};
+	const startedAt = Date.now();
+	const runDir = resolveRunDir(opts.runDir ?? (opts.method ? `.agent-eval/runs/self-improve-${startedAt}` : `mem://selfImprove-${startedAt}`));
+	const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
+	const costLedger = createRunCostLedger({
+		storage,
+		runDir,
+		costCeilingUsd: opts.budget?.dollars
+	});
+	try {
+		return await runSelfImprove(opts, costLedger, startedAt, runDir, storage);
+	} catch (error) {
+		throw new SelfImproveRunError(error, costLedger);
+	}
+}
+async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
+	const budget = opts.budget ?? {};
+	assertSelfImproveSearchMode(opts);
+	const maxConcurrency = budget.maxConcurrency ?? 2;
+	const holdoutFraction = budget.holdoutFraction ?? .25;
+	const holdoutMode = budget.holdout ?? "measured";
+	const holdoutDeferred = holdoutMode === "deferred";
+	const expectUsage = opts.expectUsage ?? "assert";
+	const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.scenarios) : void 0;
+	const splitUnitByScenario = opts.claim?.generalization === "new-units" ? unitByScenario : void 0;
+	const explicitHoldout = budget.holdoutScenarios;
+	const { train, holdout } = explicitHoldout ? {
+		train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
+		holdout: explicitHoldout
+	} : holdoutDeferred ? {
+		train: opts.scenarios,
+		holdout: []
+	} : splitTrainHoldout(opts.scenarios, holdoutFraction, splitUnitByScenario);
+	if (train.length === 0) throw new Error("selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios.");
+	if (holdout.length === 0 && !holdoutDeferred) throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
+	if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, holdout, train);
+	if (opts.finalEvidence) {
+		if (holdoutDeferred) throw new Error("final evidence requires measured holdout");
+		if (opts.method) await reserveFinalEvidence(opts.finalEvidence, opts.claim, holdout, train);
+	}
+	if (opts.method) {
+		const partitions = splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? .25, splitUnitByScenario);
+		return runSelfImproveMethod({
+			opts: {
+				...opts,
+				method: opts.method
+			},
+			train: partitions.train,
+			selection: partitions.selection,
+			holdout,
+			costLedger,
+			storage,
+			runDir,
+			startedAt
+		});
+	}
+	const generations = budget.generations ?? 3;
+	const populationSize = budget.populationSize ?? 2;
+	if (generations > 0 && !opts.proposer) throw new Error("selfImprove: method or proposer is required when budget.generations is greater than zero");
+	const proposer = opts.proposer ?? {
+		kind: "baseline-only",
+		propose: async () => []
+	};
+	const gate = opts.gate ?? defaultProductionGate({
+		holdoutScenarios: holdout,
+		deltaThreshold: opts.claim?.minimumEffect ?? .05,
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
+	});
+	if (opts.onProgress) opts.onProgress({
+		kind: "baseline.started",
+		scenarios: opts.scenarios.length
+	});
+	const result = await runImprovementLoop({
+		scenarios: train,
+		baselineSurface: opts.baselineSurface,
+		premeasuredBaseline: opts.premeasuredBaseline,
+		dispatchWithSurface: opts.agent,
+		dispatchRef: opts.dispatchRef,
+		proposer,
+		judges: [opts.judge],
+		populationSize,
+		maxGenerations: generations,
+		candidateConcurrency: budget.candidateConcurrency,
+		reps: budget.reps,
+		maxImprovementShots: budget.maxImprovementShots,
+		holdoutScenarios: holdout,
+		claim: opts.claim,
+		finalEvidence: opts.finalEvidence,
+		holdout: holdoutMode,
+		gate,
+		neutralize: opts.neutralize,
+		autoOnPromote: opts.autoOnPromote ?? "none",
+		ghOwner: opts.ghOwner,
+		ghRepo: opts.ghRepo,
+		storage,
+		runDir,
+		maxConcurrency,
+		cellPlacement: opts.cellPlacement,
+		dispatchTimeoutMs: opts.dispatchTimeoutMs,
+		cellRetry: opts.cellRetry,
+		costLedger,
+		expectUsage,
+		labeledStore: opts.labeledStore,
+		captureSource: opts.captureSource,
+		analyzeGeneration: opts.analyzeGeneration,
+		findings: opts.findings,
+		selectionRankKey: opts.selectionRankKey,
+		selectParent: opts.selectParent,
+		searchLedger: opts.searchLedger
+	});
+	const reportSplit = holdoutDeferred ? "search" : "holdout";
+	const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
+	const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
+	const reportUnitMap = opts.claim ? evaluationUnitMap(opts.claim, holdoutDeferred ? train : holdout) : void 0;
+	const report = pairedCompositeSummary(reportBaselineCampaign, reportWinnerCampaign, reportUnitMap);
+	const baseline = report.baseline;
+	const winnerStats = report.winner;
+	let power;
+	const baselineHoldoutComposites = holdoutDeferred ? [] : report.baselineComposites;
+	if (baselineHoldoutComposites.length >= 3) {
+		power = powerPreflight({
+			baselineComposites: baselineHoldoutComposites,
+			deltaThreshold: opts.claim?.minimumEffect ?? .05,
+			sharedScorerChannel: true
+		});
+		if (opts.onProgress) opts.onProgress({
+			kind: "power.estimated",
+			n: power.n,
+			sd: power.sd,
+			mde: power.mde,
+			underpowered: power.underpowered
+		});
+		if (power.underpowered && generations > 0) console.warn(`[selfImprove] ${power.recommendation}`);
+	}
+	if (opts.onProgress) {
+		opts.onProgress({
+			kind: "baseline.completed",
+			compositeMean: baseline.compositeMean,
+			durationMs: Date.now() - startedAt
+		});
+		opts.onProgress({
+			kind: "gate.decided",
+			decision: result.gateResult.decision,
+			...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean }
+		});
+	}
+	const cost = result.cost;
+	const totalCost = cost.totalCostUsd;
+	const insight = await analyzeRuns({
+		runs: [...cellsToRunRecords(reportBaselineCampaign.cells, "baseline", runDir, opts.baselineSurface, reportSplit, opts.model), ...reportWinnerCampaign === reportBaselineCampaign ? [] : cellsToRunRecords(reportWinnerCampaign.cells, "winner", runDir, result.winnerSurface, reportSplit, opts.model)],
+		baselineCandidateId: "baseline",
+		independentUnitByScenarioId: reportUnitMap,
+		decisionThreshold: opts.claim?.minimumEffect ?? .05,
+		...reportWinnerCampaign === reportBaselineCampaign ? {} : { candidateCandidateId: "winner" }
+	});
+	const durationMs = Date.now() - startedAt;
+	const { record: provenance } = await emitLoopProvenance({
+		...loopProvenanceArgsFromResult({
+			runId: `${runDir}#${startedAt}`,
+			runDir,
+			timestamp: new Date(startedAt).toISOString(),
+			baselineSurface: opts.baselineSurface,
+			result,
+			costReceipts: costLedger.list(),
+			totalCostUsd: totalCost,
+			totalDurationMs: durationMs,
+			independentUnitByScenarioId: holdoutDeferred ? void 0 : reportUnitMap
+		}),
+		storage,
+		hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
+	});
+	if (opts.onProvenance) opts.onProvenance(provenance);
+	const summary = {
+		...opts.claim ? { claim: opts.claim } : {},
+		...result.finalEvidence ? { finalEvidence: result.finalEvidence } : {},
+		mode: "proposer",
+		baseline,
+		winner: {
+			...winnerStats,
+			surface: result.winnerSurface,
+			...result.winnerLabel ? { label: result.winnerLabel } : {},
+			...result.winnerRationale ? { rationale: result.winnerRationale } : {}
+		},
+		...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean },
+		diff: result.promotedDiff,
+		provenance,
+		gateDecision: result.gateResult.decision,
+		generationsExplored: result.generations.length,
+		durationMs,
+		totalCostUsd: totalCost,
+		cost,
+		receipts: costLedger.list(),
+		...result.searchHistory ? { searchHistory: result.searchHistory } : {},
+		insight,
+		...power ? { power } : {},
+		raw: result
+	};
+	if (opts.hostedTenant) try {
+		await shipEvalRunToHosted(opts.hostedTenant, opts, summary, result, runDir);
+	} catch (err) {
+		const msg = err instanceof Error ? err.message : String(err);
+		console.warn(`[agent-eval] hosted ingest failed (continuing): ${msg}`);
+	}
+	return summary;
+}
+async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
+	const client = createHostedClient(tenant);
+	function snapshotFromCampaign(index, surface, campaign, durationMs) {
+		const cells = campaign.cells.map((cell) => {
+			const execution = campaignCellExecutionEvidence(cell);
+			return {
+				scenarioId: cell.scenarioId,
+				rep: cell.rep,
+				compositeMean: campaignCellTaskScore(cell) ?? null,
+				dimensions: campaignCellJudgeDimensions(cell),
+				terminalOutcome: execution.terminalOutcome,
+				executionErrorCount: execution.executionErrorCount ?? null,
+				errorMessage: cell.error ?? void 0
+			};
+		});
+		const scoredCells = cells.flatMap((cell) => cell.compositeMean === null ? [] : [cell.compositeMean]);
+		const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
+		return {
+			index,
+			surfaceHash: surfaceHash(surface),
+			surface,
+			cells,
+			compositeMean,
+			costUsd: campaign.aggregates.cost.totalCostUsd,
+			durationMs
+		};
+	}
+	const generations = [];
+	generations.push(snapshotFromCampaign(0, opts.baselineSurface, raw.baselineCampaign, 0));
+	for (const gen of raw.generations) {
+		const winner = gen.surfaces.reduce((best, s) => s.campaign.aggregates.cellsExecuted > 0 && (best === void 0 || averageComposite(s.campaign) > averageComposite(best.campaign)) ? s : best, gen.surfaces[0]);
+		if (!winner) continue;
+		generations.push(snapshotFromCampaign(gen.record.generationIndex + 1, winner.surface, winner.campaign, 0));
+	}
+	const event = {
+		runId: `${runDir}#${Date.now()}`,
+		runDir,
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		status: "finished",
+		labels: opts.hostedLabels ?? {},
+		baseline: generations[0],
+		generations,
+		gateDecision: summary.gateDecision,
+		holdoutLift: summary.lift,
+		totalCostUsd: summary.totalCostUsd,
+		totalDurationMs: summary.durationMs,
+		insightReport: summary.insight
+	};
+	await client.ingestEvalRun(event);
+}
+function averageComposite(campaign) {
+	const aggs = Object.values(campaign.aggregates.byScenario);
+	return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length;
+}
+//#endregion
+//#region src/contract/define-agent-eval.ts
+/**
+* Define an agent eval once, then either score a surface with `evaluate()` or
+* run the closed loop with `improve()`.
+*
+* This is a DX wrapper only: it delegates to `runEval()` and `selfImprove()` and
+* returns their native result shapes.
+*/
+function defineAgentEval(defaults) {
+	const defaultEvaluateOptions = evaluateDefaults(defaults);
+	return {
+		scenarios: defaults.scenarios,
+		baselineSurface: defaults.baselineSurface,
+		async evaluate(opts = {}) {
+			const { agent, judge, judges, runDir, scenarios, surface, ...campaignOpts } = opts;
+			const selectedAgent = agent ?? defaults.agent;
+			const selectedSurface = surface ?? defaults.baselineSurface;
+			const selectedRunDir = runDir ?? defaults.runDir ?? `mem://defineAgentEval-${Date.now()}`;
+			const selectedStorage = campaignOpts.storage ?? defaultEvaluateOptions.storage ?? (selectedRunDir.startsWith("mem://") ? inMemoryCampaignStorage() : void 0);
+			const evalOptions = {
+				...defaultEvaluateOptions,
+				...campaignOpts,
+				...selectedStorage ? { storage: selectedStorage } : {},
+				runDir: selectedRunDir,
+				scenarios: scenarios ?? defaults.scenarios,
+				dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx),
+				dispatchRef: surfaceDispatchRef(selectedSurface, campaignOpts.dispatchRef ?? defaults.dispatchRef),
+				judges: evaluateJudges(judges, judge ?? defaults.judge)
+			};
+			if (evalOptions.reps !== void 0) evalOptions.reps = requirePositiveInteger(evalOptions.reps, "reps");
+			return runEval(evalOptions);
+		},
+		async improve(opts = {}) {
+			const { budget: budgetOverride, hostedTenant: hostedTenantOverride, ...topLevelOverrides } = opts;
+			const merged = mergeDefined(defaults, topLevelOverrides);
+			const budget = mergeBudget(defaults.budget, budgetOverride);
+			const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
+			return selfImprove({
+				...merged,
+				...budget ? { budget } : {},
+				...hostedTenant ? { hostedTenant } : {}
+			});
+		}
+	};
+}
+function evaluateDefaults(defaults) {
+	const out = {};
+	if (defaults.storage) out.storage = defaults.storage;
+	if (defaults.labeledStore) out.labeledStore = defaults.labeledStore;
+	if (defaults.captureSource) out.captureSource = defaults.captureSource;
+	if (defaults.cellPlacement) out.cellPlacement = defaults.cellPlacement;
+	if (defaults.expectUsage) out.expectUsage = defaults.expectUsage;
+	if (defaults.budget?.dollars !== void 0) out.costCeiling = defaults.budget.dollars;
+	if (defaults.budget?.maxConcurrency !== void 0) out.maxConcurrency = defaults.budget.maxConcurrency;
+	if (defaults.budget?.reps !== void 0) out.reps = requirePositiveInteger(defaults.budget.reps, "budget.reps");
+	return out;
+}
+function mergeBudget(defaults, overrides) {
+	const merged = mergeOptionalObject(defaults, overrides);
+	if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
+	return merged;
+}
+function mergeHostedTenant(defaults, overrides) {
+	const merged = mergeOptionalObject(defaults, overrides);
+	if (!merged) return void 0;
+	if (!merged.endpoint?.trim() || !merged.apiKey?.trim() || !merged.tenantId?.trim()) throw new Error("defineAgentEval.improve: hostedTenant requires endpoint, apiKey, and tenantId after merging defaults and overrides");
+	return merged;
+}
+function mergeDefined(defaults, overrides) {
+	if (!overrides) return defaults;
+	const merged = { ...defaults };
+	for (const [key, value] of Object.entries(overrides)) if (value !== void 0) merged[key] = value;
+	return merged;
+}
+function mergeOptionalObject(defaults, overrides) {
+	if (!defaults && !overrides) return void 0;
+	return mergeDefined(defaults ?? {}, overrides);
+}
+function evaluateJudges(judges, defaultJudge) {
+	if (judges !== void 0) {
+		if (judges.length === 0) throw new Error("defineAgentEval.evaluate: judges must not be empty");
+		return judges;
+	}
+	return [defaultJudge];
+}
+function requirePositiveInteger(value, field) {
+	if (!Number.isInteger(value) || value < 1) throw new Error(`defineAgentEval: ${field} must be a positive integer`);
+	return value;
+}
+//#endregion
+export { SelfImproveRunError as n, selfImprove as r, defineAgentEval as t };
+
+//# sourceMappingURL=define-agent-eval-D4uumjl3.js.map