@tangle-network/agent-eval 0.218.1 → 0.219.0

package/dist/{skillopt-optimization-method-5dlex7MJ.js → skillopt-optimization-method-DLKqdhcR.js}

@@ -3,17 +3,17 @@ import { t as mean$1 } from "./mean-BbA_JCzW.js";
 import { t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
 import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
 import { n as contentHash, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
-import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-Djxh-Uhs.js";
+import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-Cc8vDJh_.js";
 import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-DunzcJGv.js";
 import { a as mapConcurrent } from "./journal-C7om8Gum.js";
 import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
-import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-B5gdLRZv.js";
+import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-D670Oxf2.js";
 import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
-import { n as openSearchLedger } from "./search-ledger-ClpN4I3e.js";
+import { n as openSearchLedger } from "./search-ledger-CZRqtFbW.js";
 import { t as acquireSingleRunLock } from "./single-run-lock-Dekd8_Wv.js";
 import { n as isRecord } from "./json-BhdsJaOQ.js";
-import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-_Y6ic-tf.js";
-import { t as startExternalOptimizerCallback } from "./external-optimizer-process-C3Wm7Vhn.js";
+import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-DXuOTseU.js";
+import { t as startExternalOptimizerCallback } from "./external-optimizer-process-wOLfzsCC.js";
 import { createHash, randomBytes } from "node:crypto";
 import { mkdir } from "node:fs/promises";
 import { isDeepStrictEqual } from "node:util";
@@ -368,6 +368,7 @@ async function scoreOneScenario(args) {
 		info: {
 			scenarioId: row.scenarioId,
 			dimensions: breakdown.dimensions,
+			scores: { ...breakdown.dimensions },
 			status: "scored",
 			...row.notes ? { notes: row.notes } : {},
 			...artifact !== void 0 ? { artifact } : {}
@@ -1841,6 +1842,70 @@ function optimizerAccounting(tokenUsage, proposerCostUsd) {
 		}
 	};
 }
+/** What the GEPA run key is hashed over; fields added later join only when enabled so earlier run keys keep their identity. */
+function gepaRunMaterial(config, input, parts) {
+	const { name, runtimeIdentity, bridgeRunner, seedCandidate, trainSet, selectionSet, maxCandidateChars, maxPopulationCandidates, maxEvidenceChars } = parts;
+	return {
+		optimizer: "gepa",
+		runtime: runtimeIdentity,
+		method: name,
+		evaluationId: config.evaluationId,
+		dispatchRef: input.runOptions.dispatchRef ?? null,
+		seed: input.seed,
+		recipe: snapshotJson(config.recipe, "GEPA run settings"),
+		engineModules: config.engineModules ?? [],
+		objective: config.objective,
+		background: config.background ?? "",
+		seedCandidate,
+		trainSet,
+		selectionSet,
+		maxCandidateChars,
+		maxPopulationCandidates,
+		maxEvidenceChars,
+		evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
+		optimizerModel: config.optimizer ? {
+			model: config.optimizer.model,
+			callRef: config.optimizer.callRef,
+			budget: config.optimizer.budget,
+			...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
+		} : null,
+		runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
+		trustResumeState: config.trustResumeState === true
+	};
+}
+/** Starts the callback server the GEPA bridge calls back into: evaluations are counted against the run budget and observed into the log. */
+async function startGepaCallback(config, input, parts) {
+	const { runDir, runId, runBudget, observationLog, evaluationLimit, maxCandidateChars, maxEvidenceChars } = parts;
+	const signal = input.runOptions.signal;
+	const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
+	const evaluate = createExternalTextEvaluator({
+		input,
+		label: "GEPA bridge",
+		runDir,
+		compatibleRunId: runId,
+		costPhase: "gepa.external-evaluation",
+		costTags: runBudget.attemptTags,
+		costLedger: input.costLedger,
+		scenarioById,
+		maxCandidateChars,
+		maxEvidenceChars,
+		describeArtifact: config.describeArtifact
+	});
+	let attemptEvaluationCount = 0;
+	return startExternalOptimizerCallback({
+		token: randomBytes(32).toString("hex"),
+		maxEvaluations: evaluationLimit,
+		acceptEvaluation: () => {
+			if (runBudget.acceptEvaluation() === void 0) return void 0;
+			attemptEvaluationCount += 1;
+			return attemptEvaluationCount;
+		},
+		evaluate,
+		observe: observationLog.observe,
+		...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
+		...signal ? { signal } : {}
+	});
+}
 /** The settled inputs of one GEPA run: its identity, budgets, evaluation callback and logs. */
 async function prepareGepaRun(config, input, name) {
 	const signal = input.runOptions.signal;
@@ -1877,33 +1942,17 @@ async function prepareGepaRun(config, input, name) {
 	const evaluationLimit = gepaRecipeEvaluationLimit(config.recipe, input.selectionScenarios.length);
 	const maxPopulationCandidates = Math.min(Number.MAX_SAFE_INTEGER, evaluationLimit + 1);
 	const populationScenarioIds = (selectionSet.length > 0 ? selectionSet : trainSet).map((scenario) => scenario.id);
-	const runMaterial = {
-		optimizer: "gepa",
-		runtime: runtimeIdentity,
-		method: name,
-		evaluationId: config.evaluationId,
-		dispatchRef: input.runOptions.dispatchRef ?? null,
-		seed: input.seed,
-		recipe: snapshotJson(config.recipe, "GEPA run settings"),
-		engineModules: config.engineModules ?? [],
-		objective: config.objective,
-		background: config.background ?? "",
+	const runMaterial = gepaRunMaterial(config, input, {
+		name,
+		runtimeIdentity,
+		bridgeRunner,
 		seedCandidate,
 		trainSet,
 		selectionSet,
 		maxCandidateChars,
 		maxPopulationCandidates,
-		maxEvidenceChars,
-		evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
-		optimizerModel: config.optimizer ? {
-			model: config.optimizer.model,
-			callRef: config.optimizer.callRef,
-			budget: config.optimizer.budget,
-			...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
-		} : null,
-		runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
-		trustResumeState: config.trustResumeState === true
-	};
+		maxEvidenceChars
+	});
 	const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
 	const runId = externalOptimizerRunKey({
 		material: runMaterial,
@@ -1921,25 +1970,6 @@ async function prepareGepaRun(config, input, name) {
 		storage,
 		path: `${runDir}/observations-${attemptId}.jsonl`
 	});
-	const executionLog = config.optimizer ? openExternalOptimizerExecutionLog({
-		storage,
-		path: `${runDir}/model-executions-${attemptId}.jsonl`
-	}) : void 0;
-	const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
-	const evaluate = createExternalTextEvaluator({
-		input,
-		label: "GEPA bridge",
-		runDir,
-		compatibleRunId: runId,
-		costPhase: "gepa.external-evaluation",
-		costTags: runBudget.attemptTags,
-		costLedger,
-		scenarioById,
-		maxCandidateChars,
-		maxEvidenceChars,
-		describeArtifact: config.describeArtifact
-	});
-	let attemptEvaluationCount = 0;
 	return {
 		signal,
 		expectsComponents,
@@ -1964,19 +1994,18 @@ async function prepareGepaRun(config, input, name) {
 		runId,
 		runBudget,
 		observationLog,
-		executionLog,
-		callback: await startExternalOptimizerCallback({
-			token: randomBytes(32).toString("hex"),
-			maxEvaluations: evaluationLimit,
-			acceptEvaluation: () => {
-				if (runBudget.acceptEvaluation() === void 0) return void 0;
-				attemptEvaluationCount += 1;
-				return attemptEvaluationCount;
-			},
-			evaluate,
-			observe: observationLog.observe,
-			...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
-			...signal ? { signal } : {}
+		executionLog: config.optimizer ? openExternalOptimizerExecutionLog({
+			storage,
+			path: `${runDir}/model-executions-${attemptId}.jsonl`
+		}) : void 0,
+		callback: await startGepaCallback(config, input, {
+			runDir,
+			runId,
+			runBudget,
+			observationLog,
+			evaluationLimit,
+			maxCandidateChars,
+			maxEvidenceChars
 		}),
 		name,
 		config,
@@ -2267,7 +2296,8 @@ function gepaOptimizationMethod(config) {
 //#endregion
 //#region src/campaign/presets/comparison-controls.ts
 /** Admission checks for the shared controls of a method comparison. */
-function assertComparisonControls(opts, seed, resamples, confidence) {
+/** The caller-supplied judges, dispatch function and references, and the baseline surface. */
+function assertCallerInputs(opts) {
 	if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
 	if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
 	if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
@@ -2280,6 +2310,9 @@ function assertComparisonControls(opts, seed, resamples, confidence) {
 	} catch (cause) {
 		throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
 	}
+}
+/** Every judge has a unique trimmed name, a score function and uniquely keyed dimensions. */
+function assertJudges(opts) {
 	const judgeNames = /* @__PURE__ */ new Set();
 	for (const judge of opts.judges) {
 		if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
@@ -2292,18 +2325,30 @@ function assertComparisonControls(opts, seed, resamples, confidence) {
 			dimensionKeys.add(dimension.key);
 		}
 	}
+}
+/** The run directory and the seed, resamples and confidence that fix the paired intervals. */
+function assertStatistics(opts, seed, resamples, confidence) {
 	if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
 	if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
 	if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
 	assertConfidence(confidence);
 	const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
 	if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
+}
+/** Concurrency, timeout and cost limits. */
+function assertLimits(opts) {
 	if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
 	if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
 	if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
 	if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
 	if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
 }
+function assertComparisonControls(opts, seed, resamples, confidence) {
+	assertCallerInputs(opts);
+	assertJudges(opts);
+	assertStatistics(opts, seed, resamples, confidence);
+	assertLimits(opts);
+}
 function assertConfidence(confidence) {
 	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
 }
@@ -3200,4 +3245,4 @@ async function finishSkillOptRun(context, result, outputDir, modelProxy) {
 //#endregion
 export { importExternalEvaluations as a, composeGate as c, readExternalOptimizerObservationArtifact as d, externalSurface as i, externalTextOptimizationMethod as l, compareOptimizationMethods as n, importGepaPopulation as o, gepaOptimizationMethod as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
 
-//# sourceMappingURL=skillopt-optimization-method-5dlex7MJ.js.map
+//# sourceMappingURL=skillopt-optimization-method-DLKqdhcR.js.map