@tangle-network/agent-eval 0.195.0 → 0.196.0

package/dist/{skillopt-optimization-method-CIJqP8oS.js → skillopt-optimization-method-IbEhlA1V.js}

@@ -1,11 +1,11 @@
 import { a as hashCanonical } from "./canonical-BsZVgBhW.js";
 import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
 import { n as contentHash, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
-import { S as developmentClaim, T as surfaceNode, d as combineComparisonCosts, f as costFromLedgerSummary, l as optimizationTokenUsageFromSummary, m as readGepaCandidatePopulationArtifact, p as assertGepaCandidatePopulationSummary, t as llmJudge, ut as runCampaign, w as surfaceDiff, x as SearchRecorder } from "./llm-judge-CvKpaRKY.js";
+import { S as developmentClaim, T as surfaceNode, d as combineComparisonCosts, f as costFromLedgerSummary, l as optimizationTokenUsageFromSummary, m as readGepaCandidatePopulationArtifact, p as assertGepaCandidatePopulationSummary, t as llmJudge, ut as runCampaign, w as surfaceDiff, x as SearchRecorder } from "./llm-judge-CxfoYqdk.js";
 import { F as surfaceContentHash, b as campaignBreakdown } from "./campaign-evidence-DJc60F9C.js";
 import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
 import { C as fsCampaignStorage, S as createRunCostLedger, _ as removeCredentialEnvironment, b as safePathComponent, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as isRecord, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerCallbackLimits, x as acquireSingleRunLock, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-BrghTvEp.js";
-import { n as openSearchLedger } from "./search-ledger-BbKtG_WF.js";
+import { n as openSearchLedger } from "./search-ledger-9W0g5FP6.js";
 import { g as assertServedModelPolicy } from "./llm-client-c_bKHTXz.js";
 import { t as startExternalOptimizerCallback } from "./external-optimizer-process-_YIECYJ7.js";
 import { createHash, randomBytes } from "node:crypto";
@@ -1413,18 +1413,22 @@ function isGepaCandidate(value, maxChars) {
 //#endregion
 //#region src/campaign/gepa-search-import.ts
 /**
-* Import an optimizer's own record into a search ledger.
+* Import an external optimizer's own record into a search ledger.
 *
-* GEPA searches inside its own process. Its candidate population names each
-* candidate's parents by index, and the evaluation callback's observation log
-* holds every candidate it evaluated. The importer turns both into ledger
-* events: one node per distinct candidate, one `correlated` edge per population
-* entry (joined by content digest from GEPA's own parent record), and one cell
-* per evaluation. Nothing is inferred from timing or order: a candidate GEPA
-* evaluated but kept out of its population has no parent record, so its edge
-* is `unknown` and has no parents.
+* GEPA and SkillOpt search inside their own processes. The evaluation
+* callback's observation log holds every candidate either one evaluated, and
+* GEPA's candidate population also names each candidate's parents by index.
+* The importer turns both into ledger events: one node per distinct candidate,
+* one `correlated` edge per population entry (joined by content digest from
+* GEPA's own parent record), and one cell per evaluation. Nothing is inferred
+* from timing or order: a candidate with no parent record (every SkillOpt
+* candidate, and a candidate GEPA kept out of its population) has an `unknown`
+* edge and no parents.
 */
-const NO_GEPA_RATIONALE = { unknown: "GEPA records no rationale per candidate; its reflection attaches to the proposer operation" };
+function noRationale(optimizer) {
+	return { unknown: `${optimizer} records no rationale per candidate; its reflection attaches to the proposer operation` };
+}
+const NO_GEPA_RATIONALE = noRationale("GEPA");
 /**
 * Register GEPA's population as nodes and its parent indices as `correlated`
 * edges. A parentless entry (GEPA's seed) gets a seed edge unless its node is
@@ -1501,10 +1505,12 @@ async function importGepaPopulation(input) {
 /**
 * Record every evaluation in the callback's observation log as an `external`
 * cell attempt. A candidate that is not yet a node (GEPA evaluated it on a
-* minibatch and discarded it) is registered with an `unknown` edge.
+* minibatch and discarded it, or SkillOpt, which reports no parents, proposed
+* it) is registered with an `unknown` edge.
 */
 async function importExternalEvaluations(input) {
 	const { recorder } = input;
+	const optimizer = input.optimizer ?? "GEPA";
 	const reps = /* @__PURE__ */ new Map();
 	const unrecorded = [];
 	let cells = 0;
@@ -1520,10 +1526,10 @@ async function importExternalEvaluations(input) {
 				operator: "improve",
 				attribution: "unknown",
 				proposer: input.proposer,
-				proposalKey: `gepa-evaluated:${candidateHash}`,
-				rationale: NO_GEPA_RATIONALE,
+				proposalKey: `${optimizer.toLowerCase()}-evaluated:${candidateHash}`,
+				rationale: noRationale(optimizer),
 				diffs: [],
-				label: "GEPA proposal outside its population"
+				label: optimizer === "GEPA" ? "GEPA proposal outside its population" : `${optimizer} proposal without a parent record`
 			});
 			unrecorded.push(registered.nodeId);
 		}
@@ -1605,7 +1611,31 @@ function externalSurface(candidate) {
 * outside this search.
 */
 async function recordGepaSearch(input) {
-	const { identity, population } = input;
+	const best = input.population.candidates.find((candidate) => candidate.index === input.population.bestIndex);
+	if (!best) throw new Error(`GEPA candidate population has no bestIndex entry ${input.population.bestIndex}`);
+	return recordExternalSearch({
+		...input,
+		optimizer: "GEPA",
+		selected: externalSurface(best.candidate)
+	});
+}
+/**
+* Write SkillOpt's finished search into its ledger: the baseline as the seeded
+* root, every candidate SkillOpt evaluated as a node with an `unknown` edge
+* (SkillOpt reports no parents), every callback evaluation as an `external`
+* cell, SkillOpt's optimizer spend as one operation, SkillOpt's choice as the
+* selected node, and the close. Like GEPA's, the ledger carries no claim.
+*/
+async function recordSkillOptSearch(input) {
+	return recordExternalSearch({
+		...input,
+		optimizer: "SkillOpt",
+		population: null
+	});
+}
+async function recordExternalSearch(input) {
+	const { identity, population, optimizer } = input;
+	const policyName = optimizer.toLowerCase();
 	const tasks = (scenarios) => scenarios.map((scenario) => ({
 		taskId: scenario.id,
 		unitId: scenario.id,
@@ -1646,8 +1676,8 @@ async function recordGepaSearch(input) {
 			heldOutUnits: true
 		},
 		policy: {
-			expansion: "gepa",
-			allocation: "gepa",
+			expansion: policyName,
+			allocation: policyName,
 			seed: input.seed
 		},
 		budget: {
@@ -1681,7 +1711,7 @@ async function recordGepaSearch(input) {
 		diffs: [],
 		label: "baseline"
 	});
-	const operationId = "gepa-proposals";
+	const operationId = `${policyName}-proposals`;
 	await recorder.startOperation({
 		operationId,
 		operationKind: "candidate-generation"
@@ -1692,7 +1722,7 @@ async function recordGepaSearch(input) {
 		operationId,
 		source: identity.proposer.source
 	};
-	const imported = await importGepaPopulation({
+	if (population) await importGepaPopulation({
 		recorder,
 		population,
 		proposer
@@ -1710,7 +1740,20 @@ async function recordGepaSearch(input) {
 				revision: taskSet
 			}
 		},
-		splitOf: (exampleId) => trainIds.has(exampleId) ? "train" : "selection"
+		splitOf: (exampleId) => trainIds.has(exampleId) ? "train" : "selection",
+		optimizer
+	});
+	const chosen = await recorder.registerNode(surfaceNode(recorder, input.selected));
+	if (((await recorder.state()).node(chosen.nodeId)?.edgeIds.length ?? 0) === 0) await recorder.recordEdge({
+		childNodeId: chosen.nodeId,
+		parents: [],
+		operator: "improve",
+		attribution: "unknown",
+		proposer,
+		proposalKey: `${policyName}-selected`,
+		rationale: noRationale(optimizer),
+		diffs: [],
+		label: `${optimizer} choice without an evaluation`
 	});
 	await recorder.recordOperation({
 		operationId,
@@ -1719,17 +1762,19 @@ async function recordGepaSearch(input) {
 		outcome: { status: "completed" },
 		accounting: input.generationAccounting
 	});
-	const best = imported.nodeIds.get(population.bestIndex);
 	const state = await recorder.state();
 	const nodes = state.nodes();
-	const bestScored = best !== void 0 && state.cells({ nodeId: best }).some((cell) => cell.score !== null);
+	const best = chosen.nodeId;
+	const bestScored = state.cells({ nodeId: best }).some((cell) => cell.score !== null);
+	const rule = population ? "gepa-best-aggregate" : `${policyName}-choice`;
+	const basis = population ? "its aggregate selection score" : "its own validation";
 	for (const node of nodes) if (node.nodeId === best && bestScored) await recorder.decideNode({
 		nodeId: node.nodeId,
 		decision: { status: "selected" },
-		rule: "gepa-best-aggregate",
-		reason: "GEPA chose this candidate by its aggregate selection score"
+		rule,
+		reason: `${optimizer} chose this candidate by ${basis}`
 	});
-	else if (evaluations.unrecordedParentNodeIds.includes(node.nodeId)) await recorder.decideNode({
+	else if (population && evaluations.unrecordedParentNodeIds.includes(node.nodeId)) await recorder.decideNode({
 		nodeId: node.nodeId,
 		decision: { status: "pruned" },
 		rule: "gepa-population",
@@ -1738,8 +1783,8 @@ async function recordGepaSearch(input) {
 	else await recorder.decideNode({
 		nodeId: node.nodeId,
 		decision: node.nodeId === best ? { status: "invalid" } : { status: "rejected" },
-		rule: "gepa-best-aggregate",
-		reason: node.nodeId === best ? "GEPA chose this candidate, but no evaluation of it was scored" : "GEPA chose another candidate"
+		rule,
+		reason: node.nodeId === best ? `${optimizer} chose this candidate, but no evaluation of it was scored` : `${optimizer} chose another candidate`
 	});
 	await recorder.close({
 		reason: "budget",
@@ -2463,8 +2508,46 @@ function skillOptOptimizationMethod(config) {
 				label: "optimizer model",
 				cost: optimizerCost
 			}]);
+			const searchHistory = config.searchLedger ? await recordSkillOptSearch({
+				name,
+				path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
+				searchId: runId,
+				identity: config.searchLedger.identity,
+				storage,
+				seed: input.seed,
+				baselineSurface: input.baselineSurface,
+				trainScenarios: input.trainScenarios,
+				selectionScenarios: input.selectionScenarios,
+				evaluationLimit: config.maxEvaluations,
+				observations: readExternalOptimizerObservationArtifact({
+					summary: observationLog.summary(),
+					storage
+				}),
+				selected: result.bestCandidate,
+				generationAccounting: {
+					tokens: optimizerUsage.usageComplete ? {
+						status: "known",
+						inputTokens: optimizerUsage.inputTokens,
+						outputTokens: optimizerUsage.outputTokens,
+						cachedTokens: optimizerUsage.cachedTokens
+					} : {
+						status: "unknown",
+						reason: "an optimizer call reported no token usage"
+					},
+					cost: optimizerCost.accountingComplete ? {
+						status: "known",
+						usd: optimizerCost.totalCostUsd,
+						source: "provider"
+					} : {
+						status: "unknown",
+						knownLowerBoundUsd: optimizerCost.totalCostUsd,
+						reason: optimizerCost.incompleteReasons.join("; ") || "optimizer spend incomplete"
+					}
+				}
+			}) : void 0;
 			return {
 				winnerSurface: result.bestCandidate,
+				...searchHistory ? { searchHistory } : {},
 				cost: combinedCost,
 				durationMs: Date.now() - started,
 				provenance: {
@@ -2485,6 +2568,6 @@ function skillOptOptimizationMethod(config) {
 	};
 }
 //#endregion
-export { importGepaPopulation as a, composeGate as c, readExternalOptimizerObservationArtifact as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as f, runReferenceEquivalenceJudge as h, importExternalEvaluations as i, externalTextOptimizationMethod as l, createReferenceEquivalenceJudge as m, gepaOptimizationMethod as n, recordGepaSearch as o, REFERENCE_EQUIVALENCE_JUDGE_VERSION as p, externalSurface as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
+export { importGepaPopulation as a, heldOutGate as c, decodeExternalTextCandidate as d, readExternalOptimizerObservationArtifact as f, runReferenceEquivalenceJudge as g, createReferenceEquivalenceJudge as h, importExternalEvaluations as i, composeGate as l, REFERENCE_EQUIVALENCE_JUDGE_VERSION as m, gepaOptimizationMethod as n, recordGepaSearch as o, REFERENCE_EQUIVALENCE_INPUT_LIMITS as p, externalSurface as r, recordSkillOptSearch as s, skillOptOptimizationMethod as t, externalTextOptimizationMethod as u };
 
-//# sourceMappingURL=skillopt-optimization-method-CIJqP8oS.js.map
+//# sourceMappingURL=skillopt-optimization-method-IbEhlA1V.js.map