@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js}

@@ -1,10 +1,19 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { a as spearmanR, r as pearsonR } from "./descriptive-B5MwKfbf.js";
+import { r as continuousAgreement } from "./judge-calibration-DZkWrm5H.js";
+import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
+import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-CHIrXJll.js";
+import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
+import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
+import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
-import { D as pairedCohensDz, E as pairedBootstrap, L as pearsonR, P as pairedTTest, V as spearmanR, Z as continuousAgreement, k as pairedMde, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
-import { r as pairRunRecords } from "./paired-arms-iZ08VFMN.js";
-import { s as validateRunRecord } from "./run-record-BmSPWXJR.js";
-import { r as welchsTTest } from "./baseline-C-GocmIW.js";
-import { o as llmSpans } from "./query-Di7eEQ79.js";
-import { r as paretoChart } from "./summary-report-Lf-5I7xh.js";
+import { r as modelHasSnapshot, s as validateRunRecord } from "./run-record-BvHPVS-i.js";
+import { r as paretoChart } from "./summary-report-Blysd6Z2.js";
+import { A as surfaceHash, G as runEval, J as resolveRunDir, f as runImprovementLoop, k as surfaceContentHash, o as emitLoopProvenance, s as loopProvenanceArgsFromResult, y as assertOptimizationResult, z as defaultProductionGate } from "./llm-judge-dZ8P6nGI.js";
+import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-DNgjilrV.js";
+import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-DrJ9hR8u.js";
+import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
+import { t as createHostedClient } from "./client-LIuo-KPv.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
 	const leaks = [];
@@ -18,128 +27,6 @@ function checkCanaries(output, scenarios) {
 	}
 	return leaks;
 }
-/**
-* Behavioral canary: tests the AGENT, not the eval grep.
-*
-*   - `forbiddenPattern` PRESENT in output ⇒ AGENT EMITTED BAD CONTENT ⇒ LEAK.
-*   - `forbiddenPattern` ABSENT from output ⇒ AGENT HELD THE LINE       ⇒ PASS.
-*
-* Use when running known-bad-prompt scenarios against the agent under
-* test and you want to know if the agent misbehaved. The classical
-* {@link checkCanaries} / {@link import('./canary').runCanaries | runCanaries}
-* test whether the eval check fires when the bad output is forced
-* into the eval flow — different question, different answer.
-*
-* Pattern resolution order (first match wins):
-*   1. `scenario.forbiddenPattern` — if it parses as `/body/flags`,
-*      treated as a regex; otherwise a literal substring.
-*   2. `scenario.canary` — literal substring fallback so the helper
-*      works on existing scenario fixtures.
-*
-* Returns `null` when nothing forbidden was found OR the scenario
-* declared no pattern.
-*/
-function checkBehavioralCanary(output, scenario) {
-	const pattern = scenario.forbiddenPattern ?? scenario.canary;
-	if (!pattern) return null;
-	const hit = matchForbidden(output, pattern);
-	if (!hit) return null;
-	return {
-		scenarioId: scenario.id,
-		canary: pattern,
-		evidence: excerpt(output, hit)
-	};
-}
-/**
-* Behavioral canary over many (scenario, output) pairs. Sibling to
-* {@link import('./canary').runCanaries | runCanaries} — same idea
-* (run-many → report) but the question being answered is "did the
-* AGENT misbehave?" rather than "did the EVAL grep fire?".
-*
-* Returns one `CanaryLeak` per pair where the agent's output
-* contained its scenario's `forbiddenPattern` (or `canary` fallback).
-*/
-function runBehavioralCanaries(cases) {
-	const leaks = [];
-	for (const c of cases) {
-		const leak = checkBehavioralCanary(c.output, c.scenario);
-		if (leak) leaks.push({
-			...leak,
-			runId: c.runId ?? leak.runId
-		});
-	}
-	return leaks;
-}
-/**
-* Resolve a forbidden-pattern string to the matched substring inside
-* `output`. `/body/flags` notation is interpreted as a regex; anything
-* else is a literal substring.
-*/
-function matchForbidden(output, pattern) {
-	const re = tryParseRegex(pattern);
-	if (re) {
-		const m = output.match(re);
-		return m && m[0].length > 0 ? m[0] : null;
-	}
-	return output.includes(pattern) ? pattern : null;
-}
-function tryParseRegex(pattern) {
-	if (pattern.length < 2 || pattern[0] !== "/") return null;
-	const last = pattern.lastIndexOf("/");
-	if (last <= 0) return null;
-	const body = pattern.slice(1, last);
-	const flags = pattern.slice(last + 1);
-	if (!/^[gimsuy]*$/.test(flags)) return null;
-	try {
-		return new RegExp(body, flags);
-	} catch {
-		return null;
-	}
-}
-/**
-* Scan the LLM-output history in a corpus; returns every case where a
-* canary from a known scenario appeared in agent output. Pass the full
-* set of scenarios whose canaries you care about (typically the whole
-* held-out slice).
-*/
-async function canaryLeakView(store, scenarios) {
-	const targets = scenarios.filter((s) => !!s.canary);
-	if (targets.length === 0) return [];
-	const spans = await llmSpans(store);
-	const leaks = [];
-	for (const span of spans) {
-		const output = span.output ?? "";
-		for (const s of targets) if (s.canary && output.includes(s.canary)) leaks.push({
-			scenarioId: s.id,
-			canary: s.canary,
-			runId: span.runId,
-			evidence: excerpt(output, s.canary)
-		});
-	}
-	return leaks;
-}
-var HoldoutAuditor = class {
-	scenarios;
-	accessLog = [];
-	constructor(scenarios) {
-		this.scenarios = scenarios;
-	}
-	/** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
-	get(scenarioId, purpose) {
-		if (purpose !== "evaluation" && purpose !== "debugging") throw new Error(`HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`);
-		const s = this.scenarios.find((x) => x.id === scenarioId);
-		if (!s) throw new Error(`holdout scenario "${scenarioId}" not found`);
-		this.accessLog.push({
-			scenarioId,
-			purpose,
-			at: Date.now()
-		});
-		return s;
-	}
-	getAccessLog() {
-		return this.accessLog;
-	}
-};
 function excerpt(source, needle) {
 	const at = source.indexOf(needle);
 	if (at < 0) return "";
@@ -1077,6 +964,531 @@ function buildRecommendations(ctx) {
 	return out;
 }
 //#endregion
-export { checkBehavioralCanary as a, canaryLeakView as i, summarizeExecution as n, checkCanaries as o, HoldoutAuditor as r, runBehavioralCanaries as s, analyzeRuns as t };
+//#region src/contract/self-improve.ts
+/** Failed self-improvement run with an immutable receipt snapshot. */
+var SelfImproveRunError = class extends Error {
+	cost;
+	receipts;
+	constructor(cause, ledger) {
+		const original = cause instanceof Error ? cause : new Error(String(cause));
+		super(original.message, { cause: original });
+		this.name = "SelfImproveRunError";
+		this.cost = ledger.summary();
+		this.receipts = ledger.list();
+	}
+};
+function assertSelfImproveSearchMode(opts) {
+	if (opts.method && opts.proposer) throw new Error("selfImprove: method and proposer are mutually exclusive");
+	if (!opts.method) {
+		if (opts.selectionScenarios !== void 0) throw new Error("selfImprove: selectionScenarios requires method");
+		return;
+	}
+	if (typeof opts.method.name !== "string" || !opts.method.name.trim() || opts.method.name.trim() !== opts.method.name || typeof opts.method.optimize !== "function") throw new Error("selfImprove: method must have a trimmed name and optimize(input)");
+	const budget = opts.budget;
+	if (budget?.generations !== void 0 && budget.generations !== 1) throw new Error("selfImprove: method owns its rounds; budget.generations must be 1 when set");
+	if (budget?.populationSize !== void 0 && budget.populationSize !== 1) throw new Error("selfImprove: method owns its candidates; budget.populationSize must be 1 when set");
+	if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, and findings apply only to proposer mode");
+}
+function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
+	if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
+	const byId = /* @__PURE__ */ new Map();
+	for (const scenario of searchScenarios) {
+		if (byId.has(scenario.id)) throw new Error(`selfImprove: duplicate scenario id '${scenario.id}'`);
+		byId.set(scenario.id, scenario);
+	}
+	if (explicitSelection) {
+		if (explicitSelection.length === 0) throw new Error("selfImprove: selectionScenarios must not be empty");
+		const selectionIds = /* @__PURE__ */ new Set();
+		for (const scenario of explicitSelection) {
+			if (!byId.has(scenario.id)) throw new Error(`selfImprove: selection scenario '${scenario.id}' is absent from the non-final cases`);
+			if (selectionIds.has(scenario.id)) throw new Error(`selfImprove: duplicate selection scenario id '${scenario.id}'`);
+			selectionIds.add(scenario.id);
+		}
+		const train = searchScenarios.filter((scenario) => !selectionIds.has(scenario.id));
+		if (train.length === 0) throw new Error("selfImprove: method train split is empty");
+		return {
+			train,
+			selection: explicitSelection.map((scenario) => byId.get(scenario.id))
+		};
+	}
+	if (searchScenarios.length < 2) throw new Error("selfImprove: method requires at least two non-final scenarios");
+	const sorted = [...searchScenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
+	const count = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+	return {
+		selection: sorted.slice(0, count),
+		train: sorted.slice(count)
+	};
+}
+function safeRunComponent(value) {
+	return value.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+function stableScenarioHash(value) {
+	let hash = 2166136261;
+	for (let index = 0; index < value.length; index++) {
+		hash ^= value.charCodeAt(index);
+		hash = Math.imul(hash, 16777619) >>> 0;
+	}
+	return hash;
+}
+/**
+* Deterministic train/holdout split by a stable hash of `scenario.id`,
+* so the same scenario set always splits the same way across runs.
+*/
+function splitTrainHoldout(scenarios, fraction) {
+	const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
+	const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+	return {
+		holdout: sorted.slice(0, nHoldout),
+		train: sorted.slice(nHoldout)
+	};
+}
+function meanComposite(byScenario) {
+	const perScenario = {};
+	const values = [];
+	for (const [id, agg] of Object.entries(byScenario)) {
+		perScenario[id] = agg.meanComposite;
+		values.push(agg.meanComposite);
+	}
+	return {
+		compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length,
+		perScenario
+	};
+}
+/**
+* Latest search campaign measured for the winner surface; the baseline search
+* campaign when the winner IS the baseline. Used by the deferred-holdout
+* summary, where no holdout campaign exists to summarize.
+*/
+function winnerSearchCampaign(result) {
+	for (let i = result.generations.length - 1; i >= 0; i--) {
+		const measured = result.generations[i]?.surfaces.find((s) => s.surfaceHash === result.winnerSurfaceHash);
+		if (measured) return measured.campaign;
+	}
+	return result.baselineCampaign;
+}
+/**
+* One-shot self-improvement loop. See module docstring for defaults +
+* extension points.
+*
+* @example Minimum:
+*
+*   const result = await selfImprove({
+*     agent: (surface, scenario, ctx) => myAgent(surface, scenario, ctx.signal),
+*     scenarios,
+*     judge,
+*     baselineSurface: DEFAULT_PROMPT,
+*     proposer,
+*   })
+*   console.log(`lift: ${result.lift.toFixed(3)} (${result.gateDecision})`)
+*
+* @example Distributed (workers in three regions):
+*
+*   await selfImprove({
+*     agent: httpDispatch({ resolveUrl: ({ placement }) => REGION_URLS[placement!] }),
+*     scenarios,
+*     judge,
+*     baselineSurface: DEFAULT_PROMPT,
+*     cellPlacement: ({ scenario }) => scenario.region,
+*     budget: { maxConcurrency: 12 },
+*   })
+*/
+async function selfImprove(opts) {
+	const startedAt = Date.now();
+	const runDir = resolveRunDir(opts.runDir ?? (opts.method ? `.agent-eval/runs/self-improve-${startedAt}` : `mem://selfImprove-${startedAt}`));
+	const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
+	const costLedger = createRunCostLedger({
+		storage,
+		runDir,
+		costCeilingUsd: opts.budget?.dollars
+	});
+	try {
+		return await runSelfImprove(opts, costLedger, startedAt, runDir, storage);
+	} catch (error) {
+		throw new SelfImproveRunError(error, costLedger);
+	}
+}
+async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
+	const budget = opts.budget ?? {};
+	assertSelfImproveSearchMode(opts);
+	const generations = opts.method ? 1 : budget.generations ?? 3;
+	const populationSize = opts.method ? 1 : budget.populationSize ?? 2;
+	const maxConcurrency = budget.maxConcurrency ?? 2;
+	const holdoutFraction = budget.holdoutFraction ?? .25;
+	const holdoutMode = budget.holdout ?? "measured";
+	const holdoutDeferred = holdoutMode === "deferred";
+	const expectUsage = opts.expectUsage ?? "assert";
+	const explicitHoldout = budget.holdoutScenarios;
+	const { train, holdout } = explicitHoldout ? {
+		train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
+		holdout: explicitHoldout
+	} : holdoutDeferred ? {
+		train: opts.scenarios,
+		holdout: []
+	} : splitTrainHoldout(opts.scenarios, holdoutFraction);
+	if (train.length === 0) throw new Error("selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios.");
+	if (holdout.length === 0 && !holdoutDeferred) throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
+	if (generations > 0 && !opts.proposer && !opts.method) throw new Error("selfImprove: method or proposer is required when budget.generations is greater than zero");
+	let optimizationResult;
+	const methodPartitions = opts.method ? splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? .25) : void 0;
+	const proposer = opts.method ? {
+		kind: `method:${opts.method.name}`,
+		propose: async (context) => {
+			if (context.generation > 0) return [];
+			const result = await opts.method.optimize(Object.freeze({
+				baselineSurface: structuredClone(context.currentSurface),
+				trainScenarios: Object.freeze(methodPartitions.train.map((scenario) => structuredClone(scenario))),
+				selectionScenarios: Object.freeze(methodPartitions.selection.map((scenario) => structuredClone(scenario))),
+				dispatchWithSurface: opts.agent,
+				judges: Object.freeze([opts.judge]),
+				runDir: `${runDir}/optimization/${safeRunComponent(opts.method.name)}`,
+				seed: 42,
+				runOptions: Object.freeze({
+					storage,
+					maxConcurrency,
+					reps: budget.reps,
+					dispatchTimeoutMs: opts.dispatchTimeoutMs,
+					expectUsage,
+					costCeiling: budget.dollars
+				}),
+				costLedger
+			}));
+			assertOptimizationResult(opts.method.name, result);
+			optimizationResult = structuredClone(result);
+			return [{
+				surface: structuredClone(result.winnerSurface),
+				label: opts.method.name,
+				rationale: `${opts.method.name} selected this surface without final cases.`
+			}];
+		}
+	} : opts.proposer ?? {
+		kind: "baseline-only",
+		propose: async () => []
+	};
+	const gate = opts.gate ?? defaultProductionGate({
+		holdoutScenarios: holdout,
+		deltaThreshold: .05
+	});
+	if (opts.onProgress) opts.onProgress({
+		kind: "baseline.started",
+		scenarios: opts.scenarios.length
+	});
+	const result = await runImprovementLoop({
+		scenarios: train,
+		baselineSurface: opts.baselineSurface,
+		premeasuredBaseline: opts.premeasuredBaseline,
+		dispatchWithSurface: opts.agent,
+		proposer,
+		judges: [opts.judge],
+		populationSize,
+		maxGenerations: generations,
+		candidateConcurrency: budget.candidateConcurrency,
+		reps: budget.reps,
+		maxImprovementShots: budget.maxImprovementShots,
+		holdoutScenarios: holdout,
+		holdout: holdoutMode,
+		gate,
+		neutralize: opts.neutralize,
+		autoOnPromote: opts.autoOnPromote ?? "none",
+		ghOwner: opts.ghOwner,
+		ghRepo: opts.ghRepo,
+		storage,
+		runDir,
+		maxConcurrency,
+		cellPlacement: opts.cellPlacement,
+		dispatchTimeoutMs: opts.dispatchTimeoutMs,
+		costLedger,
+		expectUsage,
+		labeledStore: opts.labeledStore,
+		captureSource: opts.captureSource,
+		analyzeGeneration: opts.analyzeGeneration,
+		findings: opts.findings,
+		selectionRankKey: opts.selectionRankKey
+	});
+	const reportSplit = holdoutDeferred ? "search" : "holdout";
+	const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
+	const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
+	const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
+	const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
+	let power;
+	const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
+		const scores = Object.values(cell.judgeScores);
+		return scores.length === 0 ? NaN : scores.reduce((sum, s) => sum + s.composite, 0) / scores.length;
+	}).filter((v) => Number.isFinite(v));
+	if (baselineHoldoutComposites.length >= 3) {
+		power = powerPreflight({
+			baselineComposites: baselineHoldoutComposites,
+			sharedScorerChannel: true
+		});
+		if (opts.onProgress) opts.onProgress({
+			kind: "power.estimated",
+			n: power.n,
+			sd: power.sd,
+			mde: power.mde,
+			underpowered: power.underpowered
+		});
+		if (power.underpowered && generations > 0) console.warn(`[selfImprove] ${power.recommendation}`);
+	}
+	if (opts.onProgress) {
+		opts.onProgress({
+			kind: "baseline.completed",
+			compositeMean: baseline.compositeMean,
+			durationMs: Date.now() - startedAt
+		});
+		opts.onProgress({
+			kind: "gate.decided",
+			decision: result.gateResult.decision,
+			...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean }
+		});
+	}
+	const cost = result.cost;
+	const totalCost = cost.totalCostUsd;
+	const insight = await analyzeRuns({
+		runs: [...cellsToRunRecords(reportBaselineCampaign.cells, "baseline", runDir, opts.baselineSurface, reportSplit, opts.model), ...cellsToRunRecords(reportWinnerCampaign.cells, "winner", runDir, result.winnerSurface, reportSplit, opts.model)],
+		baselineCandidateId: "baseline",
+		candidateCandidateId: "winner"
+	});
+	const durationMs = Date.now() - startedAt;
+	const { record: provenance } = await emitLoopProvenance({
+		...loopProvenanceArgsFromResult({
+			runId: `${runDir}#${startedAt}`,
+			runDir,
+			timestamp: new Date(startedAt).toISOString(),
+			baselineSurface: opts.baselineSurface,
+			result,
+			costReceipts: costLedger.list(),
+			totalCostUsd: totalCost,
+			totalDurationMs: durationMs
+		}),
+		...optimizationResult ? { optimizationMethod: {
+			name: opts.method.name,
+			cost: structuredClone(optimizationResult.cost),
+			...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
+			...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
+		} } : {},
+		storage,
+		hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
+	});
+	if (opts.onProvenance) opts.onProvenance(provenance);
+	const summary = {
+		baseline,
+		winner: {
+			...winnerStats,
+			surface: result.winnerSurface,
+			...result.winnerLabel ? { label: result.winnerLabel } : {},
+			...result.winnerRationale ? { rationale: result.winnerRationale } : {}
+		},
+		...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean },
+		diff: result.promotedDiff,
+		provenance,
+		gateDecision: result.gateResult.decision,
+		generationsExplored: result.generations.length,
+		durationMs,
+		totalCostUsd: totalCost,
+		cost,
+		receipts: costLedger.list(),
+		...optimizationResult ? { optimization: {
+			name: opts.method.name,
+			cost: structuredClone(optimizationResult.cost),
+			...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
+			...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
+		} } : {},
+		insight,
+		...power ? { power } : {},
+		raw: result
+	};
+	if (opts.hostedTenant) try {
+		await shipEvalRunToHosted(opts.hostedTenant, opts, summary, result, runDir);
+	} catch (err) {
+		const msg = err instanceof Error ? err.message : String(err);
+		console.warn(`[agent-eval] hosted ingest failed (continuing): ${msg}`);
+	}
+	return summary;
+}
+async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
+	const client = createHostedClient(tenant);
+	function snapshotFromCampaign(index, surface, campaign, durationMs) {
+		const cells = campaign.cells.map((cell) => {
+			const execution = campaignCellExecutionEvidence(cell);
+			return {
+				scenarioId: cell.scenarioId,
+				rep: cell.rep,
+				compositeMean: campaignCellTaskScore(cell) ?? null,
+				dimensions: campaignCellJudgeDimensions(cell),
+				terminalOutcome: execution.terminalOutcome,
+				executionErrorCount: execution.executionErrorCount ?? null,
+				errorMessage: cell.error ?? void 0
+			};
+		});
+		const scoredCells = cells.flatMap((cell) => cell.compositeMean === null ? [] : [cell.compositeMean]);
+		const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
+		return {
+			index,
+			surfaceHash: surfaceHash(surface),
+			surface,
+			cells,
+			compositeMean,
+			costUsd: campaign.aggregates.cost.totalCostUsd,
+			durationMs
+		};
+	}
+	const generations = [];
+	generations.push(snapshotFromCampaign(0, opts.baselineSurface, raw.baselineCampaign, 0));
+	for (const gen of raw.generations) {
+		const winner = gen.surfaces.reduce((best, s) => s.campaign.aggregates.cellsExecuted > 0 && (best === void 0 || averageComposite(s.campaign) > averageComposite(best.campaign)) ? s : best, gen.surfaces[0]);
+		if (!winner) continue;
+		generations.push(snapshotFromCampaign(gen.record.generationIndex + 1, winner.surface, winner.campaign, 0));
+	}
+	const event = {
+		runId: `${runDir}#${Date.now()}`,
+		runDir,
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		status: "finished",
+		labels: opts.hostedLabels ?? {},
+		baseline: generations[0],
+		generations,
+		gateDecision: summary.gateDecision,
+		holdoutLift: summary.lift,
+		totalCostUsd: summary.totalCostUsd,
+		totalDurationMs: summary.durationMs,
+		insightReport: summary.insight
+	};
+	await client.ingestEvalRun(event);
+}
+function averageComposite(campaign) {
+	const aggs = Object.values(campaign.aggregates.byScenario);
+	return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length;
+}
+function hashString(s) {
+	let h = 2166136261;
+	for (let i = 0; i < s.length; i++) {
+		h ^= s.charCodeAt(i);
+		h = Math.imul(h, 16777619) >>> 0;
+	}
+	return h.toString(16).padStart(8, "0");
+}
+/**
+* Adapt campaign cells into the `RunRecord` shape `analyzeRuns()` consumes.
+* Each cell becomes one run; `candidateId` is the caller-supplied label so
+* baseline + winner pair cleanly on `(experimentId, scenarioId, seed)`.
+*
+* `promptHash` is the REAL sha256 content hash of the surface this cell ran
+* (baseline vs winner are byte-distinguishable + byte-identical-verifiable);
+* `configHash` is the sha256 of the candidate label so the two candidates'
+* config rows differ. Both were previously the literal `'sha256:cell'`, which
+* made baseline and winner indistinguishable in every downstream record.
+*/
+function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbackModel) {
+	const promptHash = surfaceContentHash(surface);
+	const configHash = surfaceContentHash(candidateId);
+	return cells.map((cell) => {
+		const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []);
+		if (receiptModels.length > 1) throw new ValidationError(`selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(", ")}`);
+		const model = receiptModels[0] ?? fallbackModel;
+		if (!model) throw new ValidationError(`selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`);
+		if (!modelHasSnapshot(model)) throw new ValidationError(`selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`);
+		return campaignCellToRunRecord(cell, {
+			runId: `${runId}::${candidateId}::${cell.cellId}`,
+			experimentId: runId,
+			candidateId,
+			seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
+			model,
+			promptHash,
+			configHash,
+			commitSha: "cell",
+			splitTag
+		});
+	});
+}
+//#endregion
+//#region src/contract/define-agent-eval.ts
+/**
+* Define an agent eval once, then either score a surface with `evaluate()` or
+* run the closed loop with `improve()`.
+*
+* This is a DX wrapper only: it delegates to `runEval()` and `selfImprove()` and
+* returns their native result shapes.
+*/
+function defineAgentEval(defaults) {
+	const defaultEvaluateOptions = evaluateDefaults(defaults);
+	return {
+		scenarios: defaults.scenarios,
+		baselineSurface: defaults.baselineSurface,
+		async evaluate(opts = {}) {
+			const { agent, judge, judges, runDir, scenarios, surface, ...campaignOpts } = opts;
+			const selectedAgent = agent ?? defaults.agent;
+			const selectedSurface = surface ?? defaults.baselineSurface;
+			const selectedRunDir = runDir ?? defaults.runDir ?? `mem://defineAgentEval-${Date.now()}`;
+			const selectedStorage = campaignOpts.storage ?? defaultEvaluateOptions.storage ?? (selectedRunDir.startsWith("mem://") ? inMemoryCampaignStorage() : void 0);
+			const evalOptions = {
+				...defaultEvaluateOptions,
+				...campaignOpts,
+				...selectedStorage ? { storage: selectedStorage } : {},
+				runDir: selectedRunDir,
+				scenarios: scenarios ?? defaults.scenarios,
+				dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx),
+				judges: evaluateJudges(judges, judge ?? defaults.judge)
+			};
+			if (evalOptions.reps !== void 0) evalOptions.reps = requirePositiveInteger(evalOptions.reps, "reps");
+			return runEval(evalOptions);
+		},
+		async improve(opts = {}) {
+			const { budget: budgetOverride, hostedTenant: hostedTenantOverride, ...topLevelOverrides } = opts;
+			const merged = mergeDefined(defaults, topLevelOverrides);
+			const budget = mergeBudget(defaults.budget, budgetOverride);
+			const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
+			return selfImprove({
+				...merged,
+				...budget ? { budget } : {},
+				...hostedTenant ? { hostedTenant } : {}
+			});
+		}
+	};
+}
+function evaluateDefaults(defaults) {
+	const out = {};
+	if (defaults.storage) out.storage = defaults.storage;
+	if (defaults.labeledStore) out.labeledStore = defaults.labeledStore;
+	if (defaults.captureSource) out.captureSource = defaults.captureSource;
+	if (defaults.cellPlacement) out.cellPlacement = defaults.cellPlacement;
+	if (defaults.expectUsage) out.expectUsage = defaults.expectUsage;
+	if (defaults.budget?.dollars !== void 0) out.costCeiling = defaults.budget.dollars;
+	if (defaults.budget?.maxConcurrency !== void 0) out.maxConcurrency = defaults.budget.maxConcurrency;
+	if (defaults.budget?.reps !== void 0) out.reps = requirePositiveInteger(defaults.budget.reps, "budget.reps");
+	return out;
+}
+function mergeBudget(defaults, overrides) {
+	const merged = mergeOptionalObject(defaults, overrides);
+	if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
+	return merged;
+}
+function mergeHostedTenant(defaults, overrides) {
+	const merged = mergeOptionalObject(defaults, overrides);
+	if (!merged) return void 0;
+	if (!merged.endpoint?.trim() || !merged.apiKey?.trim() || !merged.tenantId?.trim()) throw new Error("defineAgentEval.improve: hostedTenant requires endpoint, apiKey, and tenantId after merging defaults and overrides");
+	return merged;
+}
+function mergeDefined(defaults, overrides) {
+	if (!overrides) return defaults;
+	const merged = { ...defaults };
+	for (const [key, value] of Object.entries(overrides)) if (value !== void 0) merged[key] = value;
+	return merged;
+}
+function mergeOptionalObject(defaults, overrides) {
+	if (!defaults && !overrides) return void 0;
+	return mergeDefined(defaults ?? {}, overrides);
+}
+function evaluateJudges(judges, defaultJudge) {
+	if (judges !== void 0) {
+		if (judges.length === 0) throw new Error("defineAgentEval.evaluate: judges must not be empty");
+		return judges;
+	}
+	return [defaultJudge];
+}
+function requirePositiveInteger(value, field) {
+	if (!Number.isInteger(value) || value < 1) throw new Error(`defineAgentEval: ${field} must be a positive integer`);
+	return value;
+}
+//#endregion
+export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
 
-//# sourceMappingURL=analyze-runs-C30yljDJ.js.map
+//# sourceMappingURL=define-agent-eval-iqjT--ZZ.js.map