@tangle-network/agent-eval 0.189.0 → 0.190.1

package/dist/{skillopt-optimization-method-BVSB676o.js → skillopt-optimization-method-l2vDBtKp.js}

@@ -1,11 +1,12 @@
+import { a as hashCanonical } from "./canonical-BsZVgBhW.js";
 import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-GcLIPxk4.js";
 import { n as contentHash, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
-import { A as openSearchLedger, X as runCampaign, b as recordCandidatePopulationSearch, d as combineComparisonCosts, f as costFromLedgerSummary, l as optimizationTokenUsageFromSummary, m as readGepaCandidatePopulationArtifact, p as assertGepaCandidatePopulationSummary, t as llmJudge } from "./llm-judge-M_l_WXw2.js";
-import { x as campaignBreakdown, z as surfaceContentHash } from "./campaign-evidence-CBlGJ3kC.js";
+import { _ as SearchRecorder, b as surfaceDiff, d as combineComparisonCosts, f as costFromLedgerSummary, j as openSearchLedger, l as optimizationTokenUsageFromSummary, m as readGepaCandidatePopulationArtifact, ot as runCampaign, p as assertGepaCandidatePopulationSummary, t as llmJudge, v as developmentClaim, x as surfaceNode } from "./llm-judge-BnxAv7ed.js";
+import { B as surfaceContentHash, x as campaignBreakdown } from "./campaign-evidence-D40-LHIL.js";
 import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
-import { S as fsCampaignStorage, _ as removeCredentialEnvironment, b as safePathComponent, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as isRecord, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerCallbackLimits, x as createRunCostLedger, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-BDwgKLO1.js";
-import { g as assertServedModelPolicy } from "./llm-client-DF9bPNYA.js";
-import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-C3quNeKV.js";
+import { S as fsCampaignStorage, _ as removeCredentialEnvironment, b as safePathComponent, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as isRecord, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerCallbackLimits, x as createRunCostLedger, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-kbcPu5po.js";
+import { g as assertServedModelPolicy } from "./llm-client-KYQBh9C8.js";
+import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-BnSmrkgL.js";
 import { createHash, randomBytes } from "node:crypto";
 import { z } from "zod";
 import { mkdir } from "node:fs/promises";
@@ -1409,6 +1410,346 @@ function isGepaCandidate(value, maxChars) {
 	return (typeof value === "string" ? value.length : JSON.stringify(value).length) <= maxChars;
 }
 //#endregion
+//#region src/campaign/gepa-search-import.ts
+/**
+* Import an optimizer's own record into a search ledger.
+*
+* GEPA searches inside its own process. Its candidate population names each
+* candidate's parents by index, and the evaluation callback's observation log
+* holds every candidate it evaluated. The importer turns both into ledger
+* events: one node per distinct candidate, one `correlated` edge per population
+* entry (joined by content digest from GEPA's own parent record), and one cell
+* per evaluation. Nothing is inferred from timing or order: a candidate GEPA
+* evaluated but kept out of its population has no parent record, so its edge
+* is `unknown` and has no parents.
+*/
+const NO_GEPA_RATIONALE = { unknown: "GEPA records no rationale per candidate; its reflection attaches to the proposer operation" };
+/**
+* Register GEPA's population as nodes and its parent indices as `correlated`
+* edges. A parentless entry (GEPA's seed) gets a seed edge unless its node is
+* already in the tree, for example the root the caller seeded.
+*/
+async function importGepaPopulation(input) {
+	const { recorder, population } = input;
+	const node = input.node ?? ((candidate) => surfaceNode(recorder, externalSurface(candidate.candidate)));
+	const diff = input.diff ?? ((parent, child) => surfaceDiff(recorder, externalSurface(parent.candidate), externalSurface(child.candidate)));
+	const nodeIds = /* @__PURE__ */ new Map();
+	const byIndex = new Map(population.candidates.map((candidate) => [candidate.index, candidate]));
+	const seenNodes = /* @__PURE__ */ new Set();
+	let edges = 0;
+	let collapsedDuplicates = 0;
+	let unchangedProposals = 0;
+	for (const candidate of population.candidates) {
+		const { nodeId } = await recorder.registerNode(node(candidate));
+		if (seenNodes.has(nodeId)) collapsedDuplicates += 1;
+		seenNodes.add(nodeId);
+		nodeIds.set(candidate.index, nodeId);
+		const parentIndices = [...new Set(candidate.parentIndices.filter((index) => index !== null))];
+		const parents = [];
+		for (const index of parentIndices) {
+			const parentNodeId = nodeIds.get(index);
+			const parent = byIndex.get(index);
+			if (parentNodeId === void 0 || parent === void 0) throw new Error(`GEPA candidate ${candidate.index} names parent ${index}, which precedes no population entry`);
+			if (parentNodeId === nodeId) continue;
+			if (!parents.some((known) => known.nodeId === parentNodeId)) parents.push({
+				nodeId: parentNodeId,
+				candidate: parent
+			});
+		}
+		if (parentIndices.length > 0 && parents.length === 0) {
+			unchangedProposals += 1;
+			continue;
+		}
+		if (parents.length === 0) {
+			if (((await recorder.state()).node(nodeId)?.edgeIds.length ?? 0) > 0) continue;
+			await recorder.recordEdge({
+				childNodeId: nodeId,
+				parents: [],
+				operator: "seed",
+				attribution: "correlated",
+				proposer: null,
+				proposalKey: `gepa:${population.runId}:${candidate.index}`,
+				rationale: NO_GEPA_RATIONALE,
+				diffs: [],
+				label: `GEPA seed candidate ${candidate.index}`
+			});
+			edges += 1;
+			continue;
+		}
+		await recorder.recordEdge({
+			childNodeId: nodeId,
+			parents: parents.map((parent) => parent.nodeId),
+			operator: parents.length > 1 ? "merge" : "improve",
+			attribution: "correlated",
+			proposer: input.proposer,
+			proposalKey: `gepa:${population.runId}:${candidate.index}`,
+			rationale: NO_GEPA_RATIONALE,
+			diffs: parents.map((parent) => diff(parent.candidate, candidate)),
+			label: `GEPA candidate ${candidate.index}`
+		});
+		edges += 1;
+	}
+	return {
+		nodeIds,
+		nodes: seenNodes.size,
+		edges,
+		collapsedDuplicates,
+		unchangedProposals
+	};
+}
+/**
+* Record every evaluation in the callback's observation log as an `external`
+* cell attempt. A candidate that is not yet a node (GEPA evaluated it on a
+* minibatch and discarded it) is registered with an `unknown` edge.
+*/
+async function importExternalEvaluations(input) {
+	const { recorder } = input;
+	const reps = /* @__PURE__ */ new Map();
+	const unrecorded = [];
+	let cells = 0;
+	for (const observation of input.observations.observations) {
+		if (!(observation.kind === "evaluation" || observation.kind === "refusal" && observation.reason === "evaluation-failed") || observation.candidate === void 0 || observation.exampleId === void 0) continue;
+		const candidateHash = observation.candidateHash;
+		const surface = externalSurface(observation.candidate);
+		const registered = await recorder.registerNode(surfaceNode(recorder, surface));
+		if (((await recorder.state()).node(registered.nodeId)?.edgeIds.length ?? 0) === 0) {
+			await recorder.recordEdge({
+				childNodeId: registered.nodeId,
+				parents: [],
+				operator: "improve",
+				attribution: "unknown",
+				proposer: input.proposer,
+				proposalKey: `gepa-evaluated:${candidateHash}`,
+				rationale: NO_GEPA_RATIONALE,
+				diffs: [],
+				label: "GEPA proposal outside its population"
+			});
+			unrecorded.push(registered.nodeId);
+		}
+		const split = input.splitOf(observation.exampleId);
+		const repKey = `${registered.nodeId}\u0000${observation.exampleId}`;
+		const rep = reps.get(repKey) ?? 0;
+		reps.set(repKey, rep + 1);
+		const cellId = await recorder.allocateCell({
+			nodeId: registered.nodeId,
+			taskId: observation.exampleId,
+			split,
+			rep,
+			stage: "external"
+		});
+		await recorder.settleCell({
+			cellId,
+			attempt: 1,
+			outcome: observation.kind === "evaluation" ? evaluationOutcome(observation.response) : {
+				status: "errored",
+				metrics: {},
+				error: {
+					code: "evaluation-failed",
+					message: "the evaluation callback failed before a score",
+					retryable: false
+				}
+			},
+			accounting: {
+				tokens: {
+					status: "unknown",
+					reason: "the evaluation callback meters tokens per run"
+				},
+				cost: {
+					status: "unknown",
+					knownLowerBoundUsd: 0,
+					reason: "the evaluation callback meters cost per run, not per evaluation"
+				}
+			},
+			identity: input.identity
+		});
+		cells += 1;
+	}
+	return {
+		cells,
+		unrecordedParentNodeIds: unrecorded
+	};
+}
+/** GEPA's callback response. A `failed` evaluation carries a penalty score for
+* the optimizer that is not a measurement, so the cell is unscored. */
+function evaluationOutcome(response) {
+	const record = response ?? {};
+	if (record.info?.status === "scored" && typeof record.score === "number" && Number.isFinite(record.score)) return {
+		status: "passed",
+		score: record.score,
+		metrics: { score: record.score }
+	};
+	return {
+		status: "errored",
+		metrics: {},
+		error: {
+			code: "evaluation-failed",
+			message: typeof record.info?.notes === "string" && record.info.notes.trim().length > 0 ? record.info.notes.trim() : "the evaluation produced no score",
+			retryable: false
+		}
+	};
+}
+/** An optimizer candidate as the mutable surface it represents. */
+function externalSurface(candidate) {
+	return typeof candidate === "string" ? candidate : {
+		kind: "components",
+		components: candidate
+	};
+}
+/**
+* Write GEPA's finished search into its ledger: the baseline as the seeded
+* root, the population as nodes with `correlated` edges, every callback
+* evaluation as an `external` cell, GEPA's proposer spend as one operation,
+* GEPA's choice as the selected node, and the close. GEPA's own selection is a
+* budget decision, so the ledger carries no claim; the final comparison runs
+* outside this search.
+*/
+async function recordGepaSearch(input) {
+	const { identity, population } = input;
+	const tasks = (scenarios) => scenarios.map((scenario) => ({
+		taskId: scenario.id,
+		unitId: scenario.id,
+		source: {
+			uri: `scenario://${scenario.id}`,
+			revision: hashCanonical(scenario)
+		}
+	}));
+	const train = tasks(input.trainScenarios);
+	const selection = tasks(input.selectionScenarios);
+	const taskSet = hashCanonical({
+		train,
+		selection
+	});
+	const recorder = await SearchRecorder.open({
+		ledger: openSearchLedger({
+			path: input.path,
+			searchId: input.searchId
+		}),
+		storage: input.storage
+	}, {
+		subject: identity.subject ?? input.name,
+		process: {
+			name: input.name,
+			executionRef: identity.search
+		},
+		artifactKind: "prompt",
+		objective: {
+			metric: "score",
+			direction: "maximize",
+			judge: identity.judge ?? { unknown: "the evaluation callback judges through caller functions without a pinned source" },
+			claim: identity.claim ?? developmentClaim(taskSet)
+		},
+		splits: {
+			train,
+			selection,
+			test: [],
+			heldOutUnits: true
+		},
+		policy: {
+			expansion: "gepa",
+			allocation: "gepa",
+			seed: input.seed
+		},
+		budget: {
+			maxUsd: null,
+			maxCells: input.evaluationLimit,
+			maxNodes: null,
+			deadline: null,
+			maxConcurrency: null,
+			reservedClaimUsd: 0
+		},
+		containment: null,
+		derivedFrom: null,
+		identity: {
+			model: identity.model,
+			agent: identity.agent,
+			benchmark: {
+				uri: `optimizer://${input.name}`,
+				revision: taskSet
+			}
+		}
+	});
+	const root = await recorder.registerNode(surfaceNode(recorder, input.baselineSurface));
+	await recorder.recordEdge({
+		childNodeId: root.nodeId,
+		parents: [],
+		operator: "seed",
+		attribution: "explicit",
+		proposer: null,
+		proposalKey: "baseline",
+		rationale: { unknown: "the baseline is the caller-supplied starting surface" },
+		diffs: [],
+		label: "baseline"
+	});
+	const operationId = "gepa-proposals";
+	await recorder.startOperation({
+		operationId,
+		operationKind: "candidate-generation"
+	});
+	const proposer = {
+		kind: "optimizer",
+		name: input.name,
+		operationId,
+		source: identity.proposer.source
+	};
+	const imported = await importGepaPopulation({
+		recorder,
+		population,
+		proposer
+	});
+	const trainIds = new Set(train.map((task) => task.taskId));
+	const evaluations = await importExternalEvaluations({
+		recorder,
+		observations: input.observations,
+		proposer,
+		identity: {
+			model: identity.model,
+			agent: identity.agent,
+			benchmark: {
+				uri: `optimizer://${input.name}`,
+				revision: taskSet
+			}
+		},
+		splitOf: (exampleId) => trainIds.has(exampleId) ? "train" : "selection"
+	});
+	await recorder.recordOperation({
+		operationId,
+		operationKind: "candidate-generation",
+		execution: identity.proposer,
+		outcome: { status: "completed" },
+		accounting: input.generationAccounting
+	});
+	const best = imported.nodeIds.get(population.bestIndex);
+	const state = await recorder.state();
+	const nodes = state.nodes();
+	const bestScored = best !== void 0 && state.cells({ nodeId: best }).some((cell) => cell.score !== null);
+	for (const node of nodes) if (node.nodeId === best && bestScored) await recorder.decideNode({
+		nodeId: node.nodeId,
+		decision: { status: "selected" },
+		rule: "gepa-best-aggregate",
+		reason: "GEPA chose this candidate by its aggregate selection score"
+	});
+	else if (evaluations.unrecordedParentNodeIds.includes(node.nodeId)) await recorder.decideNode({
+		nodeId: node.nodeId,
+		decision: { status: "pruned" },
+		rule: "gepa-population",
+		reason: "GEPA evaluated the candidate and kept it out of its population"
+	});
+	else await recorder.decideNode({
+		nodeId: node.nodeId,
+		decision: node.nodeId === best ? { status: "invalid" } : { status: "rejected" },
+		rule: "gepa-best-aggregate",
+		reason: node.nodeId === best ? "GEPA chose this candidate, but no evaluation of it was scored" : "GEPA chose another candidate"
+	});
+	await recorder.close({
+		reason: "budget",
+		claim: null
+	});
+	return recorder.receipt({
+		producerId: input.name,
+		runId: input.searchId
+	});
+}
+//#endregion
 //#region src/campaign/gepa-optimization-method.ts
 /**
 * Turn an optional GEPA installation into an `OptimizationMethod`.
@@ -1675,19 +2016,23 @@ function gepaOptimizationMethod(config) {
 					candidate: result.bestCandidate
 				});
 				if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
-				if (config.searchLedger) searchHistory = await recordCandidatePopulationSearch({
-					ledger: openSearchLedger({
-						path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
-						campaignId: runId
-					}),
-					storage,
-					runDir,
+				if (config.searchLedger) searchHistory = await recordGepaSearch({
+					name,
+					path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
+					searchId: runId,
 					identity: config.searchLedger.identity,
+					storage,
+					seed: input.seed,
+					baselineSurface: input.baselineSurface,
+					trainScenarios: input.trainScenarios,
+					selectionScenarios: input.selectionScenarios,
+					evaluationLimit,
 					population,
-					scenarios: input.selectionScenarios,
-					generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd),
-					producerId: name,
-					runId
+					observations: readExternalOptimizerObservationArtifact({
+						summary: observationLog.summary(),
+						storage
+					}),
+					generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd)
 				});
 			}
 			const evaluationCost = costFromLedgerSummary(costLedger.summary({
@@ -2139,6 +2484,6 @@ function skillOptOptimizationMethod(config) {
 	};
 }
 //#endregion
-export { externalTextOptimizationMethod as a, REFERENCE_EQUIVALENCE_INPUT_LIMITS as c, runReferenceEquivalenceJudge as d, composeGate as i, REFERENCE_EQUIVALENCE_JUDGE_VERSION as l, gepaOptimizationMethod as n, decodeExternalTextCandidate as o, heldOutGate as r, readExternalOptimizerObservationArtifact as s, skillOptOptimizationMethod as t, createReferenceEquivalenceJudge as u };
+export { importGepaPopulation as a, composeGate as c, readExternalOptimizerObservationArtifact as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as f, runReferenceEquivalenceJudge as h, importExternalEvaluations as i, externalTextOptimizationMethod as l, createReferenceEquivalenceJudge as m, gepaOptimizationMethod as n, recordGepaSearch as o, REFERENCE_EQUIVALENCE_JUDGE_VERSION as p, externalSurface as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
 
-//# sourceMappingURL=skillopt-optimization-method-BVSB676o.js.map
+//# sourceMappingURL=skillopt-optimization-method-l2vDBtKp.js.map