@tangle-network/agent-eval 0.216.0 → 0.216.1

package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js}

@@ -1,14 +1,18 @@
-import { a as hashCanonical } from "./canonical-BsZVgBhW.js";
-import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-1Gbmex7v.js";
-import { n as contentHash, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
-import { D as surfaceDiff, O as surfaceNode, T as developmentClaim, d as combineComparisonCosts, f as costFromLedgerSummary, l as optimizationTokenUsageFromSummary, m as readGepaCandidatePopulationArtifact, p as assertGepaCandidatePopulationSummary, rt as runCampaign, t as llmJudge, w as SearchRecorder } from "./llm-judge-PXV7Xljl.js";
-import { N as surfaceContentHash, v as campaignBreakdown } from "./campaign-evidence-OcUciD-N.js";
+import { a as hashCanonical } from "./canonical-B2RqLofv.js";
+import { t as mean$1 } from "./mean-DHoMiudD.js";
+import { t as decidePairedPromotion } from "./paired-promotion-decision-DsAPYGCm.js";
+import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-42NlCNDe.js";
+import { n as contentHash, t as canonicalJson } from "./verdict-cache-Bm8EKL2s.js";
+import { $ as evaluationUnitMap, A as assertSearchHistoryAdmissionOptions, C as SearchRecorder, D as surfaceNode, E as surfaceDiff, Q as captureFinalEvidencePolicy, W as captureJudge, Z as assertIndependentEvaluationSplit, at as resolveRunDir, c as executeOptimizationMethod, d as combineComparisonCosts, et as exposeFinalEvidence, f as costFromLedgerSummary, l as assertGepaCandidatePopulationSummary, p as optimizationTokenUsageFromSummary, rt as runCampaign, t as llmJudge, tt as reserveFinalEvidence, u as readGepaCandidatePopulationArtifact, w as developmentClaim } from "./llm-judge-CG12SQJq.js";
+import { B as assertCampaignDesign, H as assertCompleteCampaign, N as surfaceContentHash, t as createCampaignEvidenceReceipt, v as campaignBreakdown } from "./campaign-evidence-Clrn4hBV.js";
+import { m as mapConcurrent } from "./journal-DfRz9kps.js";
 import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
-import { N as createRunCostLedger, P as fsCampaignStorage, _ as removeCredentialEnvironment, b as safePathComponent, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as isRecord, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerCallbackLimits, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-DcerlZSG.js";
-import { n as openSearchLedger } from "./search-ledger-d7V8o9Lk.js";
-import { t as acquireSingleRunLock } from "./single-run-lock-fVu3MZWH.js";
-import { g as assertServedModelPolicy } from "./llm-client-DQveoxSM.js";
-import { t as startExternalOptimizerCallback } from "./external-optimizer-process-nnsY5D_4.js";
+import { N as createRunCostLedger, P as fsCampaignStorage, _ as removeCredentialEnvironment, b as safePathComponent, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as isRecord, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerCallbackLimits, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-BqW0hHcI.js";
+import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-q4HQhK1G.js";
+import { n as openSearchLedger } from "./search-ledger-CVUGLDZp.js";
+import { t as acquireSingleRunLock } from "./single-run-lock-CV9Eyd_q.js";
+import { f as assertServedModelPolicy, y as combineAbortSignals } from "./llm-client-CyBAoYiX.js";
+import { t as startExternalOptimizerCallback } from "./external-optimizer-process-BjMGUuds.js";
 import { createHash, randomBytes } from "node:crypto";
 import { z } from "zod";
 import { mkdir } from "node:fs/promises";
@@ -2200,6 +2204,417 @@ function optimizerAccounting(tokenUsage, proposerCostUsd) {
 		}
 	};
 }
+//#endregion
+//#region src/campaign/presets/compare-optimization-methods.ts
+/**
+* Compare optimization methods on shared train, selection, and test data.
+* Optimizers receive only train and selection data. After every optimizer
+* finishes, their selected surfaces are measured on the same untouched test
+* data and compared with paired confidence intervals.
+*/
+/** Optimization method inputs, provenance, results, and comparison contracts. */
+/** Equal-budget optimization followed by final measurements and paired method comparisons. */
+/** Comparison admission, disjoint partitions, and isolated method/final measurement inputs. */
+/**
+* Compare complete optimization methods on disjoint train, selection, and final test data.
+*/
+async function compareOptimizationMethods(opts) {
+	opts = {
+		...opts,
+		methods: opts.methods.map((method) => ({ ...method })),
+		trainScenarios: structuredClone(opts.trainScenarios),
+		selectionScenarios: structuredClone(opts.selectionScenarios),
+		testScenarios: structuredClone(opts.testScenarios),
+		judges: opts.judges.map(captureJudge),
+		claim: opts.claim && defineEvaluationClaim(opts.claim),
+		finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
+	};
+	assertOptimizationMethods(opts.methods);
+	assertComparisonPartitions(opts);
+	const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
+	assertSearchHistoryAdmissionOptions(opts);
+	const seed = opts.seed ?? 42;
+	const confidence = opts.confidence ?? .95;
+	assertConfidence(confidence);
+	const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
+	const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
+	const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
+	const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
+	const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
+	assertComparisonControls(opts, seed, resamples, confidence);
+	const evidence = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
+	const evidenceBySurface = /* @__PURE__ */ new Map();
+	const storage = opts.storage ?? fsCampaignStorage();
+	const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
+	const baselineSurface = structuredClone(opts.baselineSurface);
+	const costLedger = opts.costLedger ?? createRunCostLedger({
+		storage,
+		runDir: `${resolvedRunDir}/cost`,
+		costCeilingUsd: opts.costCeiling
+	});
+	const scoreOnTest = async (surface, tag, costPhase) => {
+		const measuredSurface = structuredClone(surface);
+		const campaign = await runCampaign({
+			...opts,
+			storage,
+			costLedger,
+			costPhase,
+			scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
+			dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
+			dispatchRef: `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(measuredSurface)}`,
+			runDir: `${resolvedRunDir}/${tag}`
+		});
+		assertCompleteCampaign(campaign, opts.testScenarios, opts.reps ?? 1, true, `compareOptimizationMethods: ${tag} final comparison`);
+		if (evidence) evidenceBySurface.set(surfaceContentHash(measuredSurface), createCampaignEvidenceReceipt({
+			campaign,
+			surface: measuredSurface,
+			context: evidence
+		}));
+		const byScenario = {};
+		for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) byScenario[scenarioId] = composite;
+		return byScenario;
+	};
+	const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
+	const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.testScenarios) : new Map(scenarioIds.map((id) => [id, id]));
+	const units = opts.claim ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) : {
+		observations: scenarioIds.length,
+		independentUnits: scenarioIds.length,
+		units: scenarioIds.map((id) => ({
+			id,
+			observations: 1
+		}))
+	};
+	const aggregateUnits = (arr) => {
+		const values = /* @__PURE__ */ new Map();
+		scenarioIds.forEach((id, index) => {
+			const unitId = unitByScenario.get(id);
+			const bucket = values.get(unitId) ?? [];
+			bucket.push(arr[index]);
+			values.set(unitId, bucket);
+		});
+		return units.units.map((unit) => mean(values.get(unit.id)));
+	};
+	const decisionOptions = {
+		seed,
+		resamples,
+		confidence: intervalConfidence,
+		threshold: opts.claim?.minimumEffect ?? 0
+	};
+	const align = (byScenario, label) => {
+		const missing = scenarioIds.filter((id) => !(id in byScenario));
+		if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
+		return scenarioIds.map((id) => byScenario[id]);
+	};
+	if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
+	if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
+	const optimizationOwner = new AbortController();
+	const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
+		try {
+			const { selected: out, cost, history: searchHistoryCoverage } = await executeOptimizationMethod({
+				method,
+				input: createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal),
+				storage,
+				searchHistoryPolicy: opts.searchHistoryPolicy,
+				searchHistoryVerification: opts.searchHistoryVerification
+			});
+			const winnerSurface = out.winnerSurface;
+			return {
+				name: method.name,
+				winnerSurface,
+				cost,
+				...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
+				...out.composition === void 0 ? {} : { composition: out.composition },
+				...out.provenance === void 0 ? {} : { provenance: out.provenance },
+				searchHistoryCoverage
+			};
+		} catch (error) {
+			if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
+			throw error;
+		}
+	});
+	assertReportedCostWithinCeiling(combineComparisonCosts(optimized.map((result) => ({
+		label: `method '${result.name}'`,
+		cost: result.cost
+	}))).totalCostUsd, opts.costCeiling, "optimization");
+	const testCostPhase = `compareOptimizationMethods.test:${contentHash({
+		baseline: surfaceContentHash(baselineSurface),
+		winners: optimized.map((winner) => ({
+			name: winner.name,
+			surface: surfaceContentHash(winner.winnerSurface)
+		})),
+		testScenarios: opts.testScenarios,
+		judges: opts.judges.map((judge) => ({
+			name: judge.name,
+			dimensions: judge.dimensions,
+			version: judge.judgeVersion ?? contentHash({
+				score: judge.score.toString(),
+				appliesTo: judge.appliesTo?.toString() ?? null
+			})
+		})),
+		dispatch: callerDispatchRef(opts),
+		seed,
+		reps: opts.reps ?? 1
+	})}`;
+	const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
+	const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
+	const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
+	const winners = [];
+	for (const winner of optimized) {
+		const surfaceKey = surfaceContentHash(winner.winnerSurface);
+		let arr = testScoresBySurface.get(surfaceKey);
+		if (!arr) {
+			arr = align(await scoreOnTest(winner.winnerSurface, `test/methods/${slug(winner.name)}`, testCostPhase), `method "${winner.name}"`);
+			testScoresBySurface.set(surfaceKey, arr);
+		}
+		winners.push({
+			...winner,
+			arr
+		});
+	}
+	const scores = winners.map((w) => {
+		const baselineUnits = aggregateUnits(baselineArr);
+		const winnerUnits = aggregateUnits(w.arr);
+		const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions);
+		const score = {
+			name: w.name,
+			baselineComposite: mean(baselineUnits),
+			winnerComposite: mean(winnerUnits),
+			lift: decision.delta,
+			liftCi: {
+				low: decision.low,
+				high: decision.high
+			},
+			decision,
+			optimizationCost: w.cost,
+			scenarioScores: scenarioIds.map((scenarioId, index) => ({
+				scenarioId,
+				baselineComposite: baselineArr[index],
+				winnerComposite: w.arr[index],
+				lift: w.arr[index] - baselineArr[index]
+			})),
+			unitScores: units.units.map((unit, index) => ({
+				unitId: unit.id,
+				scenarios: unit.observations,
+				baselineComposite: baselineUnits[index],
+				winnerComposite: winnerUnits[index],
+				lift: winnerUnits[index] - baselineUnits[index]
+			})),
+			winnerSurface: structuredClone(w.winnerSurface),
+			rank: 0
+		};
+		if (evidence) {
+			const baseline = evidenceBySurface.get(surfaceContentHash(baselineSurface));
+			const winner = evidenceBySurface.get(surfaceContentHash(w.winnerSurface));
+			if (!baseline || !winner) throw new Error("final measurement evidence is missing");
+			score.evidence = {
+				baseline,
+				winner
+			};
+		}
+		if (w.composition !== void 0) score.composition = structuredClone(w.composition);
+		if (w.durationMs !== void 0) score.durationMs = w.durationMs;
+		if (w.provenance !== void 0) score.provenance = structuredClone(w.provenance);
+		return score;
+	});
+	scores.sort((a, b) => b.lift - a.lift);
+	for (let start = 0; start < scores.length;) {
+		let end = start + 1;
+		while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
+		const tied = scores.slice(start, end);
+		if (tied.every((score) => score.optimizationCost.accountingComplete)) {
+			tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
+			scores.splice(start, tied.length, ...tied);
+		}
+		start = end;
+	}
+	scores.forEach((s, i) => {
+		s.rank = i + 1;
+	});
+	const best = scores[0];
+	const byName = new Map(winners.map((w) => [w.name, w]));
+	const bestArr = aggregateUnits(byName.get(best.name).arr);
+	const pairwise = scores.slice(1).map((other) => {
+		const otherArr = aggregateUnits(byName.get(other.name).arr);
+		const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions);
+		const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions);
+		const favored = decision.promote ? best.name : reverse.promote ? other.name : null;
+		return {
+			a: best.name,
+			b: other.name,
+			deltaMean: decision.delta,
+			low: decision.low,
+			high: decision.high,
+			favored,
+			decision
+		};
+	});
+	const optimizationCost = combineComparisonCosts(scores.map((score) => ({
+		label: `method '${score.name}'`,
+		cost: score.optimizationCost
+	})));
+	const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
+	const totalCost = combineComparisonCosts([{
+		label: "optimization",
+		cost: optimizationCost
+	}, {
+		label: "final test",
+		cost: testCost
+	}]);
+	assertReportedCostWithinCeiling(totalCost.totalCostUsd, opts.costCeiling, "total");
+	return {
+		scores,
+		best,
+		pairwise,
+		testScenarioIds: scenarioIds,
+		units,
+		observationUnit: opts.claim ? "registered" : "scenario",
+		pairedCellN: scenarioIds.length * (opts.reps ?? 1),
+		...opts.claim ? { claim: opts.claim } : {},
+		...finalEvidence ? { finalEvidence } : {},
+		optimizationCost,
+		testCost,
+		totalCost,
+		confidence,
+		intervalConfidence,
+		comparisonCount,
+		seed,
+		resamples,
+		reps: opts.reps ?? 1,
+		searchHistory: Object.freeze({
+			policy: searchHistoryPolicy,
+			allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
+			producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
+		})
+	};
+}
+function mean(xs) {
+	return xs.length === 0 ? 0 : mean$1(xs);
+}
+function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
+	const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling ?? 0)) * 8;
+	if (costCeiling !== void 0 && totalCostUsd > costCeiling + tolerance) throw new Error(`compareOptimizationMethods: reported ${phase} cost ${totalCostUsd} exceeds costCeiling ${costCeiling}`);
+}
+function assertOptimizationMethods(methods) {
+	if (!Array.isArray(methods) || methods.length === 0) throw new Error("compareOptimizationMethods: no methods to compare");
+	const names = /* @__PURE__ */ new Set();
+	const pathOwners = /* @__PURE__ */ new Map();
+	for (const method of methods) {
+		if (!method || typeof method !== "object" || typeof method.optimize !== "function") throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
+		if (!method.name || method.name.trim() !== method.name) throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
+		if (names.has(method.name)) throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
+		names.add(method.name);
+		const pathKey = slug(method.name);
+		const prior = pathOwners.get(pathKey);
+		if (prior) throw new Error(`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`);
+		pathOwners.set(pathKey, method.name);
+	}
+}
+function assertComparisonControls(opts, seed, resamples, confidence) {
+	if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
+	if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
+	if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
+	if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
+	const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
+	if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
+	if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
+	try {
+		surfaceContentHash(opts.baselineSurface);
+	} catch (cause) {
+		throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
+	}
+	const judgeNames = /* @__PURE__ */ new Set();
+	for (const judge of opts.judges) {
+		if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
+		if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
+		judgeNames.add(judge.name);
+		const dimensionKeys = /* @__PURE__ */ new Set();
+		for (const dimension of judge.dimensions) {
+			if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
+			if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
+			dimensionKeys.add(dimension.key);
+		}
+	}
+	if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
+	if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
+	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
+	assertConfidence(confidence);
+	const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
+	if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
+	if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
+	if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
+	if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
+	if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
+	if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
+}
+function assertComparisonPartitions(opts) {
+	const partitions = [
+		{
+			name: "trainScenarios",
+			scenarios: opts.trainScenarios
+		},
+		{
+			name: "selectionScenarios",
+			scenarios: opts.selectionScenarios
+		},
+		{
+			name: "testScenarios",
+			scenarios: opts.testScenarios
+		}
+	];
+	const owner = /* @__PURE__ */ new Map();
+	for (const partition of partitions) {
+		if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
+		if (partition.name === "testScenarios" && partition.scenarios.length < 2) throw new Error("compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty");
+		const local = /* @__PURE__ */ new Set();
+		const duplicates = /* @__PURE__ */ new Set();
+		const overlaps = /* @__PURE__ */ new Map();
+		for (const scenario of partition.scenarios) {
+			if (local.has(scenario.id)) duplicates.add(scenario.id);
+			local.add(scenario.id);
+			const prior = owner.get(scenario.id);
+			if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
+		}
+		if (duplicates.size > 0) throw new Error(`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[...duplicates].join(", ")}]`);
+		if (overlaps.size > 0) {
+			const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} ∩ ${partition.name})`).join(", ");
+			throw new Error(`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`);
+		}
+		assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
+		for (const id of local) owner.set(id, partition.name);
+	}
+}
+function slug(name) {
+	return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
+}
+function assertConfidence(confidence) {
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
+}
+function minimumBootstrapResamples(confidence, comparisonCount) {
+	const exact = 2 * comparisonCount / (1 - confidence);
+	return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
+}
+function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
+	const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
+	const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
+	return Object.freeze({
+		baselineSurface: structuredClone(baselineSurface),
+		trainScenarios: opts.trainScenarios,
+		selectionScenarios: opts.selectionScenarios,
+		dispatchWithSurface: opts.dispatchWithSurface,
+		judges: opts.judges,
+		runDir: methodRunDir,
+		seed,
+		runOptions: Object.freeze({
+			...opts.optimizationRunOptions ?? {},
+			...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
+			...signal ? { signal } : {}
+		}),
+		costLedger
+	});
+}
+function callerDispatchRef(opts) {
+	return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
+}
 const MAX_TIMER_DELAY_MS = 2147483647;
 function snapshotSkillOptOptimizationConfig(config) {
 	const runner = snapshotExternalOptimizerRunner(config.runner);
@@ -2581,6 +2996,6 @@ function skillOptOptimizationMethod(config) {
 	};
 }
 //#endregion
-export { importGepaPopulation as a, heldOutGate as c, decodeExternalTextCandidate as d, readExternalOptimizerObservationArtifact as f, runReferenceEquivalenceJudge as g, createReferenceEquivalenceJudge as h, importExternalEvaluations as i, composeGate as l, REFERENCE_EQUIVALENCE_JUDGE_VERSION as m, gepaOptimizationMethod as n, recordGepaSearch as o, REFERENCE_EQUIVALENCE_INPUT_LIMITS as p, externalSurface as r, recordSkillOptSearch as s, skillOptOptimizationMethod as t, externalTextOptimizationMethod as u };
+export { runReferenceEquivalenceJudge as _, importExternalEvaluations as a, recordSkillOptSearch as c, externalTextOptimizationMethod as d, decodeExternalTextCandidate as f, createReferenceEquivalenceJudge as g, REFERENCE_EQUIVALENCE_JUDGE_VERSION as h, externalSurface as i, heldOutGate as l, REFERENCE_EQUIVALENCE_INPUT_LIMITS as m, compareOptimizationMethods as n, importGepaPopulation as o, readExternalOptimizerObservationArtifact as p, gepaOptimizationMethod as r, recordGepaSearch as s, skillOptOptimizationMethod as t, composeGate as u };
 
-//# sourceMappingURL=skillopt-optimization-method-CCtzralq.js.map
+//# sourceMappingURL=skillopt-optimization-method-BiwA8nfX.js.map