@tangle-network/agent-eval 0.144.0 → 0.144.3

package/dist/{skillopt-optimization-method-Ds8J1_K8.js → skillopt-optimization-method-C4FX42dy.js}

@@ -1,17 +1,16 @@
 import { c as ValidationError, i as JudgeError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
 import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-D3EoChAU.js";
 import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-2GIUo1et.js";
-import { C as assertNoCredentialValues, D as removeCredentialEnvironment, E as isRecord, O as safePathComponent, S as assertJsonValue, T as isExternalTextCandidate, _ as canonicalJson, d as runWithCleanup, f as startExternalOptimizerModelProxy, h as sendJson, l as runExternalOptimizerProcess, m as listenLocal, n as createRunCostLedger, p as closeServer, r as fsCampaignStorage, t as acquireSingleRunLock, u as closeExternalOptimizerResources, v as contentHash, w as isCandidateText, x as assertExternalOptimizerModelBudget } from "./single-run-lock-t1si1ob7.js";
-import { p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
+import { A as removeCredentialEnvironment, D as isCandidateText, E as assertNoCredentialValues, M as resolveExternalOptimizerProcessLimits, N as safePathComponent, O as isExternalTextCandidate, T as assertJsonValue, d as closeExternalOptimizerResources, f as runWithCleanup, j as resolveExternalOptimizerCallbackLimits, k as isRecord, l as startExternalOptimizerCallback, n as createRunCostLedger, p as startExternalOptimizerModelProxy, r as fsCampaignStorage, t as acquireSingleRunLock, u as runExternalOptimizerProcess, v as canonicalJson, w as assertExternalOptimizerModelBudget, y as contentHash } from "./single-run-lock-D5iN0Xzb.js";
+import { p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
 import { E as pairedBootstrap, H as weightedComposite, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, T as pairedBinaryScale, d as confidenceInterval, j as pairedRiskDifferenceExact } from "./statistics-ByxzSiOM.js";
 import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
-import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-DjTi9HLb.js";
+import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-CyuzxKly.js";
 import { z } from "zod";
 import { writeFileSync } from "node:fs";
 import { basename, isAbsolute, join } from "node:path";
 import { createHash, randomBytes } from "node:crypto";
-import { createServer } from "node:http";
 import { execSync } from "node:child_process";
 import { mkdir } from "node:fs/promises";
 import { homedir, tmpdir } from "node:os";
@@ -1649,204 +1648,6 @@ function designedCellIds(scenarios, reps) {
 	return ids;
 }
 //#endregion
-//#region src/campaign/external-optimizer-callback.ts
-const MAX_CALLBACK_BODY_BYTES = 1e6;
-async function startExternalOptimizerCallback(args) {
-	assertCallbackConfig(args);
-	args.signal?.throwIfAborted();
-	let evaluations = 0;
-	let accepting = true;
-	let closePromise;
-	const activeControllers = /* @__PURE__ */ new Set();
-	const activeHandlers = /* @__PURE__ */ new Set();
-	const proposedCandidateHashes = /* @__PURE__ */ new Set();
-	let observationSequence = 0;
-	const server = createServer((request, response) => {
-		if (!accepting) {
-			sendJsonIfOpen(response, 503, { error: "external optimizer callback is closing" });
-			return;
-		}
-		const controller = new AbortController();
-		const abortRequest = () => {
-			request.destroy();
-			response.destroy();
-		};
-		activeControllers.add(controller);
-		controller.signal.addEventListener("abort", abortRequest, { once: true });
-		let handler;
-		handler = handleCallback(request, response, controller.signal, args, () => {
-			const accepted = args.acceptEvaluation ? args.acceptEvaluation() : evaluations + 1;
-			if (accepted === void 0) return void 0;
-			if (!Number.isSafeInteger(accepted) || accepted <= 0) throw new Error("external optimizer callback: invalid accepted evaluation count");
-			if (accepted > args.maxEvaluations) return void 0;
-			evaluations += 1;
-			return accepted;
-		}, (observation) => {
-			if (!args.observe) return;
-			args.observe({
-				...observation,
-				sequence: ++observationSequence
-			});
-		}, proposedCandidateHashes).finally(() => {
-			controller.signal.removeEventListener("abort", abortRequest);
-			activeControllers.delete(controller);
-			activeHandlers.delete(handler);
-		});
-		activeHandlers.add(handler);
-		handler.catch(() => void 0);
-	});
-	const port = await listenLocal(server);
-	const close = () => {
-		closePromise ??= closeCallbackServer();
-		return closePromise;
-	};
-	const onAbort = () => {
-		close().catch(() => void 0);
-	};
-	args.signal?.addEventListener("abort", onAbort, { once: true });
-	if (args.signal?.aborted) onAbort();
-	return {
-		url: `http://127.0.0.1:${port}/evaluate`,
-		token: args.token,
-		evaluations: () => evaluations,
-		close
-	};
-	async function closeCallbackServer() {
-		args.signal?.removeEventListener("abort", onAbort);
-		accepting = false;
-		const closingServer = closeServer(server);
-		server.closeIdleConnections?.();
-		for (const controller of activeControllers) controller.abort();
-		const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers(activeHandlers)]);
-		if (activeControllers.size !== 0 || activeHandlers.size !== 0) throw new Error("external optimizer callback closed with active request work");
-		if (serverResult?.status === "rejected") throw serverResult.reason;
-	}
-}
-async function handleCallback(request, response, signal, args, nextEvaluation, observe, proposedCandidateHashes) {
-	try {
-		if (request.method !== "POST" || request.url !== "/evaluate") {
-			sendJsonIfOpen(response, 404, { error: "not found" });
-			return;
-		}
-		if (request.headers.authorization !== `Bearer ${args.token}`) {
-			sendJsonIfOpen(response, 401, { error: "unauthorized" });
-			return;
-		}
-		let body;
-		try {
-			body = await readJson(request);
-		} catch {
-			observe({
-				kind: "refusal",
-				reason: "invalid-request"
-			});
-			sendJsonIfOpen(response, 400, { error: "request body must be bounded valid JSON" });
-			return;
-		}
-		if (!isRecord(body) || !isExternalTextCandidate(body.candidate) || typeof body.exampleId !== "string") {
-			observe({
-				kind: "refusal",
-				reason: "invalid-request"
-			});
-			sendJsonIfOpen(response, 400, { error: "candidate and exampleId are required strings" });
-			return;
-		}
-		const candidate = cloneCandidate(body.candidate);
-		const candidateHash = contentHash({
-			kind: "external-text-candidate",
-			candidate
-		});
-		if (!proposedCandidateHashes.has(candidateHash)) {
-			observe({
-				kind: "proposal",
-				candidate,
-				candidateHash
-			});
-			proposedCandidateHashes.add(candidateHash);
-		}
-		const count = nextEvaluation();
-		if (count === void 0) {
-			observe({
-				kind: "refusal",
-				reason: "evaluation-limit",
-				candidate,
-				candidateHash,
-				exampleId: body.exampleId
-			});
-			sendJsonIfOpen(response, 429, { error: "evaluation limit reached" });
-			return;
-		}
-		let result;
-		try {
-			result = await args.evaluate({
-				candidate,
-				exampleId: body.exampleId
-			}, signal);
-		} catch {
-			observe({
-				kind: "refusal",
-				reason: "evaluation-failed",
-				candidate,
-				candidateHash,
-				exampleId: body.exampleId
-			});
-			sendJsonIfOpen(response, 500, { error: "evaluation failed" });
-			return;
-		}
-		observe({
-			kind: "evaluation",
-			candidate,
-			candidateHash,
-			exampleId: body.exampleId,
-			evaluationNumber: count,
-			response: structuredClone(result)
-		});
-		sendJsonIfOpen(response, 200, result);
-	} catch {
-		sendJsonIfOpen(response, 500, { error: "evaluation failed" });
-	}
-}
-async function waitForActiveHandlers(activeHandlers) {
-	while (activeHandlers.size > 0) await Promise.allSettled([...activeHandlers]);
-}
-function sendJsonIfOpen(response, status, body) {
-	if (response.destroyed || response.writableEnded) return;
-	sendJson(response, status, body);
-}
-function readJson(request) {
-	return new Promise((resolvePromise, reject) => {
-		let size = 0;
-		const chunks = [];
-		request.on("data", (chunk) => {
-			size += chunk.length;
-			if (size > MAX_CALLBACK_BODY_BYTES) {
-				reject(/* @__PURE__ */ new Error("callback body too large"));
-				request.destroy();
-				return;
-			}
-			chunks.push(chunk);
-		});
-		request.on("error", reject);
-		request.on("end", () => {
-			try {
-				resolvePromise(JSON.parse(Buffer.concat(chunks).toString("utf8")));
-			} catch (error) {
-				reject(error);
-			}
-		});
-	});
-}
-function assertCallbackConfig(args) {
-	if (typeof args.token !== "string" || !args.token.trim()) throw new Error("external optimizer callback: token must be non-empty");
-	if (!Number.isSafeInteger(args.maxEvaluations) || args.maxEvaluations <= 0) throw new Error("external optimizer callback: maxEvaluations must be a positive safe integer");
-	if (args.acceptEvaluation !== void 0 && typeof args.acceptEvaluation !== "function") throw new Error("external optimizer callback: acceptEvaluation must be a function");
-	if (typeof args.evaluate !== "function") throw new Error("external optimizer callback: evaluate must be a function");
-	if (args.observe !== void 0 && typeof args.observe !== "function") throw new Error("external optimizer callback: observe must be a function");
-}
-function cloneCandidate(candidate) {
-	return typeof candidate === "string" ? candidate : { ...candidate };
-}
-//#endregion
 //#region src/campaign/external-optimizer-run-budget.ts
 function externalOptimizerRunKey(input) {
 	if (!input.attemptId.trim() || input.attemptId.trim() !== input.attemptId) throw new Error("external optimizer attemptId must be trimmed and non-empty");
@@ -2099,10 +1900,11 @@ async function executeCell(args) {
 		if (cached.status === "hit") {
 			const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
 			if (receiptProblem === void 0) {
-				enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? "warn");
+				const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags);
+				enforceDispatchUsage(cell, args.opts.expectUsage ?? "warn");
 				return {
 					cell: {
-						...cached.cell,
+						...cell,
 						cached: true
 					},
 					artifactsByPath: {}
@@ -2246,7 +2048,7 @@ async function executeCell(args) {
 		...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
 		...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
 	};
-	const resolvedModel = agentReceipts.at(-1)?.model;
+	const agentModelEvidence = modelEvidenceFromReceipts(agentReceipts);
 	const dispatchResult = {
 		cellId: args.slot.cellId,
 		artifact,
@@ -2303,7 +2105,7 @@ async function executeCell(args) {
 		costProvenance: agentCost.costProvenance,
 		costCallIds,
 		tokenUsage,
-		...resolvedModel ? { resolvedModel } : {},
+		...agentModelEvidence,
 		durationMs: Date.now() - startMs,
 		seed: args.slot.cellSeed,
 		cached: false,
@@ -2619,6 +2421,25 @@ function assertScheduleCachesReusable(args) {
 	}
 	if (blocked.length > 0) throw invalidCachedCellsError(blocked);
 }
+function modelEvidenceFromReceipts(receipts) {
+	const models = [...new Set(receipts.map((receipt) => receipt.model))];
+	if (models.length === 0) return {};
+	return {
+		resolvedModels: models,
+		...models.length === 1 ? { resolvedModel: models[0] } : {}
+	};
+}
+function withCurrentAgentModelEvidence(cached, costLedger, stableCostTags) {
+	const cachedCallIds = new Set(cached.costCallIds ?? []);
+	const receipts = costLedger.list({
+		channel: "agent",
+		tags: stableCostTags
+	}).filter((receipt) => cachedCallIds.has(receipt.callId));
+	const cell = { ...cached };
+	delete cell.resolvedModel;
+	delete cell.resolvedModels;
+	return Object.assign(cell, modelEvidenceFromReceipts(receipts));
+}
 function cachedCellReceiptProblem(cached, costLedger, stableCostTags) {
 	const reportsPaidActivity = cached.costProvenance.kind !== "observed" || cached.costUsd > 0 || cached.tokenUsage.input > 0 || cached.tokenUsage.output > 0;
 	if (cached.costCallIds === void 0) {
@@ -5392,7 +5213,8 @@ function externalOptimizerRunnerIdentity(runner, module) {
 	return {
 		command: runner?.command ?? "python",
 		args: [...runner?.args ?? ["-m", module]],
-		environment: removeCredentialEnvironment(runner?.env ?? {})
+		environment: removeCredentialEnvironment(runner?.env ?? {}),
+		limits: resolveExternalOptimizerProcessLimits(runner?.limits)
 	};
 }
 function snapshotExternalOptimizerRunner(runner) {
@@ -5400,7 +5222,8 @@ function snapshotExternalOptimizerRunner(runner) {
 	return {
 		...runner,
 		...runner.args ? { args: [...runner.args] } : {},
-		...runner.env ? { env: { ...runner.env } } : {}
+		...runner.env ? { env: { ...runner.env } } : {},
+		...runner.limits ? { limits: { ...runner.limits } } : {}
 	};
 }
 function snapshotJson(value, label) {
@@ -5498,6 +5321,7 @@ function snapshotGepaOptimizationConfig(config) {
 		recipe: structuredClone(config.recipe),
 		...config.engineModules ? { engineModules: [...config.engineModules] } : {},
 		...config.optimizer ? { optimizer: snapshotOptimizerModel(config.optimizer) } : {},
+		...config.evaluationCallbackLimits ? { evaluationCallbackLimits: { ...config.evaluationCallbackLimits } } : {},
 		...runner ? { runner } : {}
 	};
 }
@@ -5516,6 +5340,7 @@ function assertGepaOptimizationConfig(config) {
 	const evidenceLimit = config.maxEvidenceChars ?? 1e5;
 	if (JSON.stringify(config.objective).length > evidenceLimit || JSON.stringify(config.background ?? "").length > evidenceLimit) throw new Error("gepaOptimizationMethod: objective and background must each fit maxEvidenceChars");
 	assertEngineModules(config.engineModules);
+	resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits, "gepaOptimizationMethod: evaluationCallbackLimits");
 	if (config.optimizer !== void 0) {
 		assertOptimizerModel(config.optimizer, "gepaOptimizationMethod: optimizer");
 		if (config.engineModules?.length) throw new Error("gepaOptimizationMethod: optimizer cannot be combined with engineModules because proxied reflection requires the built-in GEPA engine");
@@ -5727,6 +5552,7 @@ function gepaOptimizationMethod(config) {
 				selectionSet,
 				maxCandidateChars,
 				maxEvidenceChars,
+				evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
 				optimizerModel: config.optimizer ? {
 					model: config.optimizer.model,
 					callRef: config.optimizer.callRef,
@@ -5776,6 +5602,7 @@ function gepaOptimizationMethod(config) {
 				acceptEvaluation: () => runBudget.acceptEvaluation(),
 				evaluate,
 				observe: observationLog.observe,
+				...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
 				...signal ? { signal } : {}
 			});
 			const runnerEnv = bridgeRunner?.env ?? {};
@@ -6680,6 +6507,7 @@ function campaignMeasurementDigest(campaign) {
 			costProvenance: cell.costProvenance,
 			costCallIds: [...cell.costCallIds ?? []].sort(),
 			tokenUsage: cell.tokenUsage,
+			resolvedModels: [...cell.resolvedModels ?? []].sort(),
 			resolvedModel: cell.resolvedModel ?? null,
 			durationMs: cell.durationMs,
 			seed: cell.seed,
@@ -7004,6 +6832,7 @@ function snapshotSkillOptOptimizationConfig(config) {
 		...config,
 		trainer: structuredClone(config.trainer),
 		optimizer: snapshotOptimizerModel(config.optimizer),
+		...config.evaluationCallbackLimits ? { evaluationCallbackLimits: { ...config.evaluationCallbackLimits } } : {},
 		...runner ? { runner } : {}
 	};
 }
@@ -7056,6 +6885,7 @@ function assertSkillOptOptimizationConfig(config) {
 	assertJsonValue(config.trainer.overrides ?? {}, "skillOptOptimizationMethod: trainer.overrides");
 	assertNoCredentialValues(config.trainer.overrides ?? {}, "skillOptOptimizationMethod: trainer.overrides", "optimizer");
 	assertOptimizerModel(config.optimizer, "skillOptOptimizationMethod: optimizer");
+	resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits, "skillOptOptimizationMethod: evaluationCallbackLimits");
 	const evidenceLimit = config.maxEvidenceChars ?? 1e5;
 	if (JSON.stringify(config.objective).length > evidenceLimit || JSON.stringify(config.background ?? "").length > evidenceLimit) throw new Error("skillOptOptimizationMethod: objective and background must each fit maxEvidenceChars");
 }
@@ -7136,6 +6966,7 @@ function skillOptOptimizationMethod(config) {
 				hardScoreThreshold: config.hardScoreThreshold ?? 1,
 				maxCandidateChars,
 				maxEvidenceChars,
+				evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
 				runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.skillopt_bridge")
 			};
 			const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
@@ -7179,6 +7010,7 @@ function skillOptOptimizationMethod(config) {
 				acceptEvaluation: () => runBudget.acceptEvaluation(),
 				evaluate,
 				observe: observationLog.observe,
+				...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
 				...signal ? { signal } : {}
 			});
 			const runnerEnv = bridgeRunner?.env ?? {};
@@ -7332,4 +7164,4 @@ function skillOptOptimizationMethod(config) {
 //#endregion
 export { assertCampaignSplitIdentity as $, assertOptimizationResult as A, summarizeAgentReceiptIntegrity as At, surfaceContentHash as B, detectScale as C, decidePairedPromotion as Ct, runCanaries as D, BackendIntegrityError as Dt, pairHoldout as E, pairedDeltaTest as Et, assertCodeSurfaceIdentity as F, DEFAULT_MUTATION_PRIMITIVES as G, campaignBreakdown as H, assertComponentSurface as I, planCampaignRun as J, buildReflectionPrompt as K, codeSurfaceIdentityMaterial as L, compareOptimizationMethods as M, JudgeParseError as Mt, costFromLedgerSummary as N, composeGate as O, assertRealAgentReceipts as Ot, optimizationTokenUsageFromSummary as P, assertCampaignDesign as Q, componentSurfaceIdentityMaterial as R, defaultProductionGate as S, recoverTruncatedJson as St, heldoutSignificance as T, minimumPairsForPairedDeltaTest as Tt, campaignMeanComposite as U, surfaceHash as V, compareRankKeys as W, resolveRunDir as X, runCampaign as Y, tangleTracesRoot as Z, buildEvidenceVector as _, crowdingDistance as _t, emitLoopProvenance as a, REFERENCE_EQUIVALENCE_JUDGE_VERSION as at, powerPreflight as b, paretoFrontierWithCrowding as bt, provenanceRecordPath as c, DEFAULT_RED_TEAM_CORPUS as ct, runImprovementLoop as d, scoreRedTeamOutput as dt, campaignScenarioIdentity as et, runOptimization as f, toolNamesForRun as ft, gepaOptimizationMethod as g, llmJudge as gt, labelTrustRank as h, hashScenarios as ht, canonicalDigest as i, REFERENCE_EQUIVALENCE_INPUT_LIMITS as it, combineComparisonCosts as j, summarizeBackendIntegrity as jt, externalTextOptimizationMethod as k, assertRealBackend as kt, provenanceSpansPath as l, redTeamDataset as lt, isProposedCandidate as m, HoldoutLockedError as mt, buildLoopProvenanceRecord as n, campaignSplitDigestFromIdentities as nt, loopProvenanceArgsFromResult as o, createReferenceEquivalenceJudge as ot, runEval as p, Dataset as pt, parseReflectionResponse as q, campaignMeasurementDigest as r, openAutoPr as rt, loopProvenanceSpans as s, runReferenceEquivalenceJudge as st, skillOptOptimizationMethod as t, campaignSplitDigest as tt, verifyLoopProvenanceRecord as u, redTeamReport as ut, paretoPolicy as v, dominates as vt, dimensionRegressions as w, pairedDecisionShape as wt, heldOutGate as x, scalarScore as xt, paretoSignificanceGate as y, paretoFrontier as yt, renderSurfaceDiff as z };
 
-//# sourceMappingURL=skillopt-optimization-method-Ds8J1_K8.js.map
+//# sourceMappingURL=skillopt-optimization-method-C4FX42dy.js.map