@tangle-network/agent-eval 0.148.0 → 0.150.0

package/dist/{skillopt-optimization-method-Donmq4sq.js → skillopt-optimization-method-L8vi4h4m.js}

@@ -1,9 +1,10 @@
 import { n as contentHash, t as canonicalJson } from "./verdict-cache-mZf5FEiY.js";
-import { C as optimizationTokenUsageFromSummary, G as assertGepaCandidatePopulationSummary, K as readGepaCandidatePopulationArtifact, S as costFromLedgerSummary, b as combineComparisonCosts, et as runCampaign, k as surfaceContentHash, t as llmJudge, z as campaignBreakdown } from "./llm-judge-DWq1Ptco.js";
-import { _ as resolveExternalOptimizerCallbackLimits, b as createRunCostLedger, d as assertJsonValue, f as assertNoCredentialValues, g as removeCredentialEnvironment, h as isRecord, i as startExternalOptimizerModelProxy, m as isExternalTextCandidate, n as closeExternalOptimizerResources, p as isCandidateText, r as runWithCleanup, t as runExternalOptimizerProcess, u as assertExternalOptimizerModelBudget, v as resolveExternalOptimizerProcessLimits, x as fsCampaignStorage, y as safePathComponent } from "./external-optimizer-subprocess-DU1KM8yV.js";
+import { C as optimizationTokenUsageFromSummary, G as assertGepaCandidatePopulationSummary, K as readGepaCandidatePopulationArtifact, S as costFromLedgerSummary, b as combineComparisonCosts, et as runCampaign, k as surfaceContentHash, t as llmJudge, z as campaignBreakdown } from "./llm-judge-D7UuMI2t.js";
+import { _ as resolveExternalOptimizerCallbackLimits, b as createRunCostLedger, d as assertJsonValue, f as assertNoCredentialValues, g as removeCredentialEnvironment, h as isRecord, i as startExternalOptimizerModelProxy, m as isExternalTextCandidate, n as closeExternalOptimizerResources, p as isCandidateText, r as runWithCleanup, t as runExternalOptimizerProcess, u as assertExternalOptimizerModelBudget, v as resolveExternalOptimizerProcessLimits, x as fsCampaignStorage, y as safePathComponent } from "./external-optimizer-subprocess-BepyNwtj.js";
 import { a as heldoutSignificance, o as pairHoldout } from "./power-preflight-DEw-uC7q.js";
 import { c as deepFreezeCanonicalJson } from "./types-BI4fT3HN.js";
-import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-C4AyCcQd.js";
+import { y as assertServedModelPolicy } from "./llm-client-CxyJ8Oft.js";
+import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-tDe72VRO.js";
 import { createHash, randomBytes } from "node:crypto";
 import { z } from "zod";
 import { mkdir } from "node:fs/promises";
@@ -1188,6 +1189,8 @@ function assertOptimizerModel(value, label) {
 	if (!value || typeof value !== "object") throw new Error(`${label} is required`);
 	for (const [field, item] of [["model", value.model], ["callRef", value.callRef]]) if (typeof item !== "string" || !item.trim() || item.trim() !== item) throw new Error(`${label}.${field} must be trimmed and non-empty`);
 	if (typeof value.call !== "function") throw new Error(`${label}.call must be a function`);
+	assertServedModelPolicy(value.servedModelPolicy, `${label}.servedModelPolicy`);
+	if (value.anthropicEndpoint !== void 0 && typeof value.anthropicEndpoint !== "boolean") throw new Error(`${label}.anthropicEndpoint must be a boolean`);
 	assertExternalOptimizerModelBudget(value.budget, `${label}.budget`);
 }
 function snapshotOptimizerModel(value) {
@@ -1195,9 +1198,20 @@ function snapshotOptimizerModel(value) {
 		model: value.model,
 		budget: structuredClone(value.budget),
 		call: value.call,
-		callRef: value.callRef
+		callRef: value.callRef,
+		...value.servedModelPolicy ? { servedModelPolicy: value.servedModelPolicy } : {},
+		...value.anthropicEndpoint === void 0 ? {} : { anthropicEndpoint: value.anthropicEndpoint }
 	};
 }
+/**
+* GEPA engines that drive a `claude` CLI subprocess. With
+* `optimizer.anthropicEndpoint: true` their model traffic is metered through
+* the loopback proxy's Anthropic route instead of being refused.
+*/
+const GEPA_AGENT_CLI_ENGINES = /* @__PURE__ */ new Set(["autoresearch", "meta_harness"]);
+function gepaRecipeHasAgentCliEngine(recipe) {
+	return recipeEngineOptions(recipe).some((run) => GEPA_AGENT_CLI_ENGINES.has(run.engine));
+}
 const MAX_TIMER_DELAY_MS$1 = 2147483647;
 function snapshotGepaOptimizationConfig(config) {
 	const runner = snapshotExternalOptimizerRunner(config.runner);
@@ -1229,7 +1243,7 @@ function assertGepaOptimizationConfig(config) {
 	if (config.optimizer !== void 0) {
 		assertOptimizerModel(config.optimizer, "gepaOptimizationMethod: optimizer");
 		if (config.engineModules?.length) throw new Error("gepaOptimizationMethod: optimizer cannot be combined with engineModules because proxied reflection requires the built-in GEPA engine");
-		assertProxiedGepaRecipe(config.recipe);
+		assertProxiedGepaRecipe(config.recipe, config.optimizer);
 	}
 }
 function gepaRecipeSupportsResume(recipe) {
@@ -1318,9 +1332,17 @@ function assertEngineOptions(run, label) {
 	if (run.sandbox !== void 0 && typeof run.sandbox !== "boolean") throw new Error(`gepaOptimizationMethod: ${label}.sandbox must be a boolean`);
 	assertJsonValue(run.engineConfig ?? {}, `gepaOptimizationMethod: ${label}.engineConfig`);
 	assertNoCredentialValues(run.engineConfig ?? {}, `gepaOptimizationMethod: ${label}.engineConfig`);
+	if (run.engine === "gepa") {
+		const nestedEngine = isRecord(run.engineConfig?.engine) ? run.engineConfig.engine : void 0;
+		if (nestedEngine && Object.hasOwn(nestedEngine, "seed")) throw new Error(`gepaOptimizationMethod: ${label}.engineConfig.engine.seed conflicts with the comparison seed; the bridge forwards the run seed`);
+	}
 }
-function assertProxiedGepaRecipe(recipe) {
+function assertProxiedGepaRecipe(recipe, optimizer) {
 	for (const [index, run] of recipeEngineOptions(recipe).entries()) {
+		if (GEPA_AGENT_CLI_ENGINES.has(run.engine)) {
+			assertProxiedAgentCliEngine(run, index, optimizer);
+			continue;
+		}
 		if (run.engine !== "gepa") throw new Error(`gepaOptimizationMethod: optimizer requires GEPA's 'gepa' engine; recipe engine ${index} is '${run.engine}'`);
 		const reflection = isRecord(run.engineConfig?.reflection) ? run.engineConfig.reflection : void 0;
 		if (reflection && Object.hasOwn(reflection, "reflection_lm")) throw new Error("gepaOptimizationMethod: optimizer replaces engineConfig.reflection.reflection_lm; remove the duplicate setting");
@@ -1337,6 +1359,11 @@ function assertProxiedGepaRecipe(recipe) {
 		].some((key) => Object.hasOwn(reflectionOptions, key))) throw new Error("gepaOptimizationMethod: proxied reflection transport settings belong in optimizer");
 	}
 }
+function assertProxiedAgentCliEngine(run, index, optimizer) {
+	if (optimizer.anthropicEndpoint !== true) throw new Error(`gepaOptimizationMethod: agent engine '${run.engine}' (recipe engine ${index}) requires optimizer.anthropicEndpoint: true so the claude CLI meters through the loopback proxy`);
+	if (run.engineConfig?.model !== optimizer.model) throw new Error(`gepaOptimizationMethod: recipe engine ${index} ('${run.engine}') must set engineConfig.model to the optimizer model '${optimizer.model}'`);
+	if (run.engineConfig && Object.hasOwn(run.engineConfig, "max_thinking_tokens")) throw new Error(`gepaOptimizationMethod: recipe engine ${index} ('${run.engine}') cannot set engineConfig.max_thinking_tokens through the Anthropic endpoint`);
+}
 function assertParallelControls(recipe) {
 	if (recipe.maxWorkers !== void 0) assertPositiveSafeInteger$1(recipe.maxWorkers, "recipe.maxWorkers");
 }
@@ -1360,6 +1387,7 @@ function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, max
 	if (!isGepaCandidate(result.bestCandidate, maxCandidateChars) || expectsComponents !== (typeof result.bestCandidate !== "string")) throw new Error(`${name}: GEPA bridge returned an invalid candidate`);
 	if (!Number.isFinite(result.bestScore)) throw new Error(`${name}: GEPA bridge returned an invalid bestScore`);
 	if (!Number.isSafeInteger(result.totalEvaluations) || result.totalEvaluations < 0 || result.totalEvaluations > maxEvaluations) throw new Error(`${name}: GEPA bridge returned an invalid totalEvaluations`);
+	if (result.upstreamReportedEvaluations !== void 0 && (!Number.isSafeInteger(result.upstreamReportedEvaluations) || result.upstreamReportedEvaluations < 0)) throw new Error(`${name}: GEPA bridge returned an invalid upstreamReportedEvaluations`);
 	if (result.proposerCostUsd !== void 0 && (!Number.isFinite(result.proposerCostUsd) || result.proposerCostUsd < 0)) throw new Error(`${name}: GEPA bridge returned an invalid proposerCostUsd`);
 	if (result.proposerCostAccounting !== "metered" && result.proposerCostAccounting !== "reported" && result.proposerCostAccounting !== "unavailable") throw new Error(`${name}: GEPA bridge returned invalid proposerCostAccounting`);
 	if (result.proposerCostAccounting !== "unavailable" !== (result.proposerCostUsd !== void 0)) throw new Error(`${name}: GEPA bridge returned inconsistent proposer cost accounting`);
@@ -1368,6 +1396,7 @@ function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, max
 	assertExternalOptimizerPackageSource(result.upstream, "gepa", name, "GEPA");
 	if (typeof result.runId !== "string" || result.runId.length === 0 || result.runId !== result.runId.trim()) throw new Error(`${name}: GEPA bridge returned an invalid runId`);
 	if (typeof result.resumed !== "boolean") throw new Error(`${name}: GEPA bridge returned an invalid resumed flag`);
+	if (typeof result.seedApplied !== "boolean") throw new Error(`${name}: GEPA bridge returned an invalid seedApplied flag; upgrade agent-eval-rpc to a release that forwards the run seed`);
 	if (result.candidatePopulation !== void 0) {
 		assertGepaCandidatePopulationSummary(result.candidatePopulation);
 		if (result.candidatePopulation.runId !== result.runId) throw new Error(`${name}: GEPA candidate population has a different run ID`);
@@ -1388,6 +1417,27 @@ function isGepaCandidate(value, maxChars) {
 * method without a test-set field. The local callback routes every candidate
 * evaluation through the same dispatch and judges used by other methods.
 */
+/**
+* Environment for a bridge child that spawns `claude` CLI subprocesses.
+*
+* Both agent CLI engines start the CLI with the bridge's own environment, so
+* these variables travel bridge → engine → CLI. They merge AFTER
+* `removeCredentialEnvironment`: the ephemeral loopback token is the only
+* credential the child may see, and a caller-supplied ANTHROPIC_AUTH_TOKEN is
+* stripped like any other credential.
+*/
+function anthropicShimEnvironment(proxy, optimizer) {
+	return {
+		ANTHROPIC_BASE_URL: new URL(proxy.baseUrl).origin,
+		ANTHROPIC_AUTH_TOKEN: proxy.apiKey,
+		ANTHROPIC_MODEL: optimizer.model,
+		ANTHROPIC_SMALL_FAST_MODEL: optimizer.model,
+		CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
+		CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING: "1",
+		CLAUDE_CODE_MAX_OUTPUT_TOKENS: String(Math.min(optimizer.budget.maxOutputTokensPerRequest, 64e3)),
+		API_TIMEOUT_MS: String((optimizer.budget.requestTimeoutMs ?? 3e5) + 6e4)
+	};
+}
 function gepaOptimizationMethod(config) {
 	assertGepaOptimizationConfig(config);
 	config = snapshotGepaOptimizationConfig(config);
@@ -1450,7 +1500,8 @@ function gepaOptimizationMethod(config) {
 				optimizerModel: config.optimizer ? {
 					model: config.optimizer.model,
 					callRef: config.optimizer.callRef,
-					budget: config.optimizer.budget
+					budget: config.optimizer.budget,
+					...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
 				} : null,
 				runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
 				trustResumeState: config.trustResumeState === true
@@ -1506,6 +1557,18 @@ function gepaOptimizationMethod(config) {
 			});
 			const runnerEnv = bridgeRunner?.env ?? {};
 			let modelProxy;
+			const bridgeRunnerForSpawn = () => {
+				if (!modelProxy) return bridgeRunner;
+				const anthropicChildEnv = config.optimizer?.anthropicEndpoint === true && gepaRecipeHasAgentCliEngine(config.recipe) ? anthropicShimEnvironment(modelProxy, config.optimizer) : void 0;
+				if (!bridgeRunner && !anthropicChildEnv) return bridgeRunner;
+				return {
+					...bridgeRunner ?? {},
+					env: {
+						...removeCredentialEnvironment(runnerEnv),
+						...anthropicChildEnv ?? {}
+					}
+				};
+			};
 			const closeResources = () => closeExternalOptimizerResources({
 				label: name,
 				callback,
@@ -1526,6 +1589,8 @@ function gepaOptimizationMethod(config) {
 							recordExecution: executionLog.observe,
 							model: config.optimizer.model,
 							budget: config.optimizer.budget,
+							...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
+							...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {},
 							costLedger,
 							phase: "gepa.optimizer-model",
 							actor: name,
@@ -1571,13 +1636,11 @@ function gepaOptimizationMethod(config) {
 									baseUrl: modelProxy.baseUrl,
 									apiKey: modelProxy.apiKey,
 									model: config.optimizer.model,
-									budget: config.optimizer.budget
+									budget: config.optimizer.budget,
+									...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
 								} } : {}
 							},
-							runner: modelProxy && bridgeRunner ? {
-								...bridgeRunner,
-								env: removeCredentialEnvironment(runnerEnv)
-							} : bridgeRunner,
+							runner: bridgeRunnerForSpawn(),
 							timeoutMs,
 							...signal ? { signal } : {}
 						}),
@@ -1624,7 +1687,18 @@ function gepaOptimizationMethod(config) {
 			const reportedProposerCost = result.proposerCostUsd ?? 0;
 			if (modelProxy) {
 				modelProxy.assertExecutionComplete();
-				assertExternalOptimizerCompletionCount(result.tokenUsage, modelProxy.requestAttempts(), modelProxy.successfulCompletions(), name, "GEPA");
+				const wires = modelProxy.wireUsage();
+				assertExternalOptimizerCompletionCount(result.tokenUsage, wires.openai.requestAttempts, wires.openai.successfulCompletions, name, "GEPA");
+				if (config.optimizer?.anthropicEndpoint === true) {
+					const anthropicReceipts = costLedger.list({
+						phase: "gepa.optimizer-model",
+						tags: {
+							...runBudget.attemptTags,
+							wire: "anthropic"
+						}
+					});
+					if (anthropicReceipts.length !== wires.anthropic.requestAttempts) throw new Error(`${name}: the Anthropic endpoint admitted ${wires.anthropic.requestAttempts} model calls but the ledger holds ${anthropicReceipts.length} receipts`);
+				}
 			}
 			const tokenUsage = modelProxy ? optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts) : void 0;
 			const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
@@ -1661,10 +1735,13 @@ function gepaOptimizationMethod(config) {
 						optimizerModel: config.optimizer.model,
 						optimizerCallRef: config.optimizer.callRef
 					} : {},
+					...modelProxy && config.optimizer?.anthropicEndpoint === true ? { anthropicEndpoint: modelProxy.wireUsage().anthropic } : {},
 					compatibleRunId,
 					runId,
 					resumed: result.resumed,
+					seedApplied: result.seedApplied,
 					evaluationCount: runBudget.acceptedEvaluations(),
+					...result.upstreamReportedEvaluations !== void 0 ? { upstreamReportedEvaluations: result.upstreamReportedEvaluations } : {},
 					artifactDir: outputDir,
 					...tokenUsage ? { tokenUsage } : {},
 					observations: observationLog.summary(),
@@ -1889,6 +1966,7 @@ function skillOptOptimizationMethod(config) {
 						recordExecution: executionLog.observe,
 						model: config.optimizer.model,
 						budget: config.optimizer.budget,
+						...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
 						costLedger,
 						phase: "skillopt.optimizer-model",
 						actor: name,
@@ -2019,4 +2097,4 @@ function skillOptOptimizationMethod(config) {
 //#endregion
 export { externalTextOptimizationMethod as a, REFERENCE_EQUIVALENCE_INPUT_LIMITS as c, runReferenceEquivalenceJudge as d, composeGate as i, REFERENCE_EQUIVALENCE_JUDGE_VERSION as l, gepaOptimizationMethod as n, decodeExternalTextCandidate as o, heldOutGate as r, readExternalOptimizerObservationArtifact as s, skillOptOptimizationMethod as t, createReferenceEquivalenceJudge as u };
 
-//# sourceMappingURL=skillopt-optimization-method-Donmq4sq.js.map
+//# sourceMappingURL=skillopt-optimization-method-L8vi4h4m.js.map