@tangle-network/agent-eval 0.146.0 → 0.148.0

package/dist/{llm-judge-B7ffRF8w.js → llm-judge-DWq1Ptco.js}

@@ -3,14 +3,15 @@ import { o as weightedComposite, t as confidenceInterval } from "./descriptive-B
 import { r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
 import { n as contentHash, t as canonicalJson } from "./verdict-cache-mZf5FEiY.js";
 import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-DFo2FU5J.js";
-import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-BSe92yAV.js";
+import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-B1qx30B4.js";
 import { u as mapConcurrent } from "./ledger-core-DTae9rv_.js";
-import { b as createRunCostLedger, h as isRecord, m as isExternalTextCandidate, x as fsCampaignStorage } from "./external-optimizer-subprocess-r6dP9DAj.js";
+import { i as hashCanonical } from "./canonical-D-XsTQ6_.js";
+import { b as createRunCostLedger, h as isRecord, m as isExternalTextCandidate, x as fsCampaignStorage } from "./external-optimizer-subprocess-DU1KM8yV.js";
 import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
 import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./power-preflight-DEw-uC7q.js";
 import { a as combineAbortSignals, n as clamp01, r as assertProposalFindings } from "./run-score-lDzV0X8j.js";
 import { c as deepFreezeCanonicalJson } from "./types-BI4fT3HN.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-_UE4fU7Z.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-BMuxYoZy.js";
 import { createHash } from "node:crypto";
 import { writeFileSync } from "node:fs";
 import { z } from "zod";
@@ -2848,6 +2849,249 @@ function campaignBreakdown(campaign) {
 	};
 }
 //#endregion
+//#region src/campaign/search-history-receipt.ts
+const SEARCH_HISTORY_RECEIPT_SCHEMA_VERSION = "1.0.0";
+const SEARCH_HISTORY_RECEIPT_DIGEST_ALGORITHM = "rfc8785-sha256";
+var SearchHistoryRequiredError = class extends Error {
+	producerId;
+	reasons;
+	constructor(producerId, reasons) {
+		const normalized = reasons.length > 0 ? reasons : ["search history receipt is missing"];
+		super(`search producer '${producerId}' lacks complete history: ${normalized.join("; ")}`);
+		this.name = "SearchHistoryRequiredError";
+		this.producerId = producerId;
+		this.reasons = Object.freeze([...normalized]);
+	}
+};
+/** Build a bounded receipt from the projection returned by canonical ledger replay. */
+function createSearchHistoryReceipt(input) {
+	if (!input || typeof input !== "object") throw new TypeError("search history input is required");
+	const producerId = nonEmpty(input.producerId, "search history producerId");
+	const runId = nonEmpty(input.runId, "search history runId");
+	const ledger = normalizeArtifactRef(input.ledger);
+	const replay = validateReplayProjection(input.replay);
+	const auditDigest = ledgerHash(hashCanonical(replay.audit), "search history auditDigest");
+	const summary = summarizeReplay(replay);
+	const incompleteReasons = Object.freeze(historyIncompleteReasons(summary));
+	const material = Object.freeze({
+		schemaVersion: SEARCH_HISTORY_RECEIPT_SCHEMA_VERSION,
+		kind: "search-history-receipt",
+		digestAlgorithm: SEARCH_HISTORY_RECEIPT_DIGEST_ALGORITHM,
+		producerId,
+		runId,
+		ledger,
+		auditDigest,
+		summary,
+		complete: incompleteReasons.length === 0,
+		incompleteReasons
+	});
+	return Object.freeze({
+		...material,
+		receiptDigest: hashCanonical(material)
+	});
+}
+/** Verify the bounded receipt. Full ledger bytes are verified by SearchLedger. */
+function verifySearchHistoryReceipt(receipt) {
+	if (!receipt || typeof receipt !== "object") throw new TypeError("search history receipt is required");
+	if (receipt.schemaVersion !== "1.0.0") throw new Error(`unsupported search history schemaVersion '${receipt.schemaVersion}'`);
+	if (receipt.kind !== "search-history-receipt") throw new Error(`search history kind must be 'search-history-receipt'`);
+	if (receipt.digestAlgorithm !== "rfc8785-sha256") throw new Error(`unsupported search history digestAlgorithm '${receipt.digestAlgorithm}'`);
+	const producerId = nonEmpty(receipt.producerId, "search history producerId");
+	const runId = nonEmpty(receipt.runId, "search history runId");
+	const ledger = normalizeArtifactRef(receipt.ledger);
+	const auditDigest = ledgerHash(receipt.auditDigest, "search history auditDigest");
+	const summary = normalizeSummary(receipt.summary);
+	const incompleteReasons = normalizeReasons(receipt.incompleteReasons);
+	const expectedReasons = historyIncompleteReasons(summary);
+	if (!sameStrings(incompleteReasons, expectedReasons)) throw new Error("search history incompleteReasons disagree with its summary");
+	if (receipt.complete !== (expectedReasons.length === 0)) throw new Error("search history complete disagrees with its summary");
+	if (hashCanonical({
+		schemaVersion: receipt.schemaVersion,
+		kind: receipt.kind,
+		digestAlgorithm: receipt.digestAlgorithm,
+		producerId,
+		runId,
+		ledger,
+		auditDigest,
+		summary,
+		complete: receipt.complete,
+		incompleteReasons
+	}) !== receipt.receiptDigest) throw new Error("search history receipt digest mismatch");
+	return receipt;
+}
+/** Prove that a receipt still describes the exact canonical replay supplied. */
+function assertSearchHistoryMatchesReplay(receipt, replay) {
+	verifySearchHistoryReceipt(receipt);
+	if (createSearchHistoryReceipt({
+		producerId: receipt.producerId,
+		runId: receipt.runId,
+		ledger: receipt.ledger,
+		replay
+	}).receiptDigest !== receipt.receiptDigest) throw new Error("search history receipt does not match the supplied SearchLedger replay");
+}
+/** Require a receipt owned by this producer and a terminal, denominator-complete history. */
+function assertCompleteSearchHistory(producerId, receipt) {
+	if (receipt === void 0) throw new SearchHistoryRequiredError(producerId, []);
+	verifySearchHistoryReceipt(receipt);
+	if (receipt.producerId !== producerId) throw new SearchHistoryRequiredError(producerId, [`receipt belongs to producer '${receipt.producerId}'`]);
+	if (!receipt.complete) throw new SearchHistoryRequiredError(producerId, receipt.incompleteReasons);
+}
+/** Classify one producer's history without treating malformed evidence as absence. */
+function searchHistoryCoverageRow(producerId, receipt) {
+	if (receipt === void 0) return Object.freeze({
+		producerId,
+		status: "missing",
+		reasons: Object.freeze(["search history receipt is missing"])
+	});
+	verifySearchHistoryReceipt(receipt);
+	if (receipt.producerId !== producerId) throw new SearchHistoryRequiredError(producerId, [`receipt belongs to producer '${receipt.producerId}'`]);
+	return Object.freeze({
+		producerId,
+		status: receipt.complete ? "complete" : "incomplete",
+		reasons: Object.freeze([...receipt.incompleteReasons]),
+		receipt
+	});
+}
+function validateReplayProjection(replay) {
+	if (!replay || typeof replay !== "object") throw new TypeError("search history replay is required");
+	if (!Array.isArray(replay.entries)) throw new TypeError("search history replay.entries must be an array");
+	if (!replay.audit || typeof replay.audit !== "object") throw new TypeError("search history replay.audit is required");
+	if (replay.plan !== null && (!replay.plan || typeof replay.plan !== "object")) throw new TypeError("search history replay.plan must be an event or null");
+	if (replay.completion !== null && (!replay.completion || typeof replay.completion !== "object")) throw new TypeError("search history replay.completion must be an event or null");
+	if (replay.audit.eventCount !== replay.entries.length) throw new Error(`search history audit eventCount ${replay.audit.eventCount} does not match ${replay.entries.length} entries`);
+	const observedHead = replay.entries.at(-1)?.entryHash ?? null;
+	if (replay.audit.headHash !== observedHead) throw new Error("search history audit headHash does not match the final ledger entry");
+	return replay;
+}
+function summarizeReplay(replay) {
+	const audit = replay.audit;
+	return normalizeSummary({
+		campaignId: audit.campaignId,
+		headHash: audit.headHash,
+		status: audit.status,
+		selectedCandidateId: audit.selectedCandidateId,
+		eventCount: audit.eventCount,
+		candidateCount: audit.candidateCount,
+		closedCandidateSlotCount: audit.closedCandidateSlotCount,
+		attemptCount: audit.attemptCount,
+		operationCount: audit.operationCount,
+		expectedCandidateSlots: audit.expected.candidateSlots,
+		expectedTaskOutcomes: audit.expected.taskOutcomes,
+		expectedOperations: audit.expected.operations,
+		missingCandidateSlots: audit.expected.missingCandidateSlots.length,
+		missingTaskOutcomes: audit.expected.missingTaskOutcomes.length,
+		missingOperations: audit.expected.missingOperations.length,
+		pendingDecisions: audit.decisions.pending,
+		hasPlan: replay.plan !== null,
+		hasCompletion: replay.completion !== null
+	});
+}
+function normalizeSummary(value) {
+	if (!value || typeof value !== "object") throw new TypeError("search history summary is required");
+	const campaignId = nonEmpty(value.campaignId, "search history summary.campaignId");
+	const headHash = nullableLedgerHash(value.headHash, "search history summary.headHash");
+	const status = searchStatus(value.status);
+	const selectedCandidateId = optionalNullableText(value.selectedCandidateId, "search history summary.selectedCandidateId");
+	const eventCount = nonNegativeSafeInteger(value.eventCount, "search history summary.eventCount");
+	const candidateCount = nonNegativeSafeInteger(value.candidateCount, "search history summary.candidateCount");
+	const closedCandidateSlotCount = nonNegativeSafeInteger(value.closedCandidateSlotCount, "search history summary.closedCandidateSlotCount");
+	const attemptCount = nonNegativeSafeInteger(value.attemptCount, "search history summary.attemptCount");
+	const operationCount = nonNegativeSafeInteger(value.operationCount, "search history summary.operationCount");
+	const expectedCandidateSlots = nonNegativeSafeInteger(value.expectedCandidateSlots, "search history summary.expectedCandidateSlots");
+	const expectedTaskOutcomes = nonNegativeSafeInteger(value.expectedTaskOutcomes, "search history summary.expectedTaskOutcomes");
+	const expectedOperations = nonNegativeSafeInteger(value.expectedOperations, "search history summary.expectedOperations");
+	const missingCandidateSlots = nonNegativeSafeInteger(value.missingCandidateSlots, "search history summary.missingCandidateSlots");
+	const missingTaskOutcomes = nonNegativeSafeInteger(value.missingTaskOutcomes, "search history summary.missingTaskOutcomes");
+	const missingOperations = nonNegativeSafeInteger(value.missingOperations, "search history summary.missingOperations");
+	const pendingDecisions = nonNegativeSafeInteger(value.pendingDecisions, "search history summary.pendingDecisions");
+	const hasPlan = boolean(value.hasPlan, "search history summary.hasPlan");
+	const hasCompletion = boolean(value.hasCompletion, "search history summary.hasCompletion");
+	if (missingCandidateSlots > expectedCandidateSlots) throw new Error("search history missingCandidateSlots exceeds expectedCandidateSlots");
+	if (missingTaskOutcomes > expectedTaskOutcomes) throw new Error("search history missingTaskOutcomes exceeds expectedTaskOutcomes");
+	if (missingOperations > expectedOperations) throw new Error("search history missingOperations exceeds expectedOperations");
+	if (pendingDecisions > candidateCount) throw new Error("search history pendingDecisions exceeds candidateCount");
+	if (eventCount === 0 !== (headHash === null)) throw new Error("search history eventCount and headHash disagree");
+	if (status === "in-progress" === hasCompletion) throw new Error("search history status and hasCompletion disagree");
+	if (status === "selected" && selectedCandidateId === null) throw new Error("selected search history requires selectedCandidateId");
+	if (status !== "selected" && selectedCandidateId !== null) throw new Error("non-selected search history cannot name selectedCandidateId");
+	return Object.freeze({
+		campaignId,
+		headHash,
+		status,
+		selectedCandidateId,
+		eventCount,
+		candidateCount,
+		closedCandidateSlotCount,
+		attemptCount,
+		operationCount,
+		expectedCandidateSlots,
+		expectedTaskOutcomes,
+		expectedOperations,
+		missingCandidateSlots,
+		missingTaskOutcomes,
+		missingOperations,
+		pendingDecisions,
+		hasPlan,
+		hasCompletion
+	});
+}
+function historyIncompleteReasons(summary) {
+	const reasons = [];
+	if (!summary.hasPlan) reasons.push("search plan is missing");
+	if (!summary.hasCompletion) reasons.push("terminal search-completed event is missing");
+	if (summary.status === "in-progress") reasons.push("search status is in-progress");
+	if (summary.missingCandidateSlots > 0) reasons.push(countReason(summary.missingCandidateSlots, "candidate slot", "candidate slots"));
+	if (summary.missingTaskOutcomes > 0) reasons.push(countReason(summary.missingTaskOutcomes, "task outcome", "task outcomes"));
+	if (summary.missingOperations > 0) reasons.push(countReason(summary.missingOperations, "operation", "operations"));
+	if (summary.pendingDecisions > 0) reasons.push(countReason(summary.pendingDecisions, "candidate decision", "candidate decisions", "pending"));
+	return reasons;
+}
+function countReason(count, singular, plural, adjective = "unresolved") {
+	return `${count} ${count === 1 ? singular : plural} ${count === 1 ? "is" : "are"} ${adjective}`;
+}
+function normalizeArtifactRef(value) {
+	if (!value || typeof value !== "object") throw new TypeError("search history ledger artifact is required");
+	return Object.freeze({
+		role: nonEmpty(value.role, "search history ledger role"),
+		uri: nonEmpty(value.uri, "search history ledger uri"),
+		sha256: ledgerHash(value.sha256, "search history ledger sha256"),
+		byteLength: nonNegativeSafeInteger(value.byteLength, "search history ledger byteLength")
+	});
+}
+function normalizeReasons(value) {
+	if (!Array.isArray(value)) throw new TypeError("search history incompleteReasons must be an array");
+	return Object.freeze(value.map((reason, index) => nonEmpty(reason, `search history incompleteReasons[${index}]`)));
+}
+function sameStrings(left, right) {
+	return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+function searchStatus(value) {
+	if (value !== "in-progress" && value !== "selected" && value !== "all-rejected") throw new TypeError(`search history summary.status is invalid: ${String(value)}`);
+	return value;
+}
+function nullableLedgerHash(value, label) {
+	return value === null ? null : ledgerHash(value, label);
+}
+function optionalNullableText(value, label) {
+	return value === null ? null : nonEmpty(value, label);
+}
+function ledgerHash(value, label) {
+	if (typeof value !== "string" || !/^sha256:[a-f0-9]{64}$/.test(value)) throw new TypeError(`${label} must be a lowercase sha256 digest`);
+	return value;
+}
+function nonEmpty(value, label) {
+	if (typeof value !== "string" || value.length === 0 || value.trim() !== value) throw new TypeError(`${label} must be a trimmed non-empty string`);
+	return value;
+}
+function nonNegativeSafeInteger(value, label) {
+	if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${label} must be a non-negative safe integer`);
+	return value;
+}
+function boolean(value, label) {
+	if (typeof value !== "boolean") throw new TypeError(`${label} must be a boolean`);
+	return value;
+}
+//#endregion
 //#region src/campaign/surface-identity.ts
 const GIT_OBJECT_ID = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
 const SHA256 = /^sha256:[a-f0-9]{64}$/;
@@ -2957,6 +3201,8 @@ function renderSurfaceDiff(winnerSurface, baselineSurface) {
 async function compareOptimizationMethods(opts) {
 	assertOptimizationMethods(opts.methods);
 	assertComparisonPartitions(opts);
+	const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
+	if (searchHistoryPolicy !== "allow-missing" && searchHistoryPolicy !== "require-complete") throw new TypeError(`compareOptimizationMethods: unknown searchHistoryPolicy '${String(searchHistoryPolicy)}'`);
 	const seed = opts.seed ?? 42;
 	const confidence = opts.confidence ?? .95;
 	assertConfidence(confidence);
@@ -3001,13 +3247,16 @@ async function compareOptimizationMethods(opts) {
 		try {
 			const out = await method.optimize(createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal));
 			assertOptimizationResult(method.name, out);
+			const searchHistoryCoverage = searchHistoryCoverageRow(method.name, out.searchHistory);
+			if (searchHistoryPolicy === "require-complete") assertCompleteSearchHistory(method.name, out.searchHistory);
 			const winnerSurface = structuredClone(out.winnerSurface);
 			return {
 				name: method.name,
 				winnerSurface,
 				cost: out.cost,
-				durationMs: out.durationMs,
-				provenance: out.provenance
+				...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
+				...out.provenance === void 0 ? {} : { provenance: out.provenance },
+				searchHistoryCoverage
 			};
 		} catch (error) {
 			if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
@@ -3125,7 +3374,12 @@ async function compareOptimizationMethods(opts) {
 		comparisonCount,
 		seed,
 		resamples,
-		reps: opts.reps ?? 1
+		reps: opts.reps ?? 1,
+		searchHistory: Object.freeze({
+			policy: searchHistoryPolicy,
+			allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
+			producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
+		})
 	};
 }
 function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
@@ -4860,6 +5114,6 @@ function firstString(value) {
 	return typeof value === "string" && value.trim() ? value : void 0;
 }
 //#endregion
-export { assertCampaignDesign as $, surfaceHash as A, DEFAULT_RED_TEAM_CORPUS as B, optimizationTokenUsageFromSummary as C, componentSurfaceIdentityMaterial as D, codeSurfaceIdentityMaterial as E, parseReflectionResponse as F, runEval as G, redTeamReport as H, recoverTruncatedJson as I, resolveRunDir as J, runCampaign as K, assertGepaCandidatePopulationSummary as L, campaignMeanComposite as M, compareRankKeys as N, renderSurfaceDiff as O, buildReflectionPrompt as P, summarizeBackendIntegrity as Q, readGepaCandidatePopulationArtifact as R, costFromLedgerSummary as S, assertComponentSurface as T, scoreRedTeamOutput as U, redTeamDataset as V, runCanaries as W, BackendIntegrityError as X, tangleTracesRoot as Y, assertRealBackend as Z, paretoFrontier as _, canonicalDigest as a, cellCachePath as at, combineComparisonCosts as b, loopProvenanceSpans as c, verifyLoopProvenanceRecord as d, assertCampaignSplitIdentity as et, runImprovementLoop as f, dominates as g, labelTrustRank as h, campaignMeasurementDigest as i, readCachedCell as it, campaignBreakdown as j, surfaceContentHash as k, provenanceRecordPath as l, isProposedCandidate as m, JudgeParseError as n, campaignSplitDigest as nt, emitLoopProvenance as o, computeManifestHash as ot, runOptimization as p, planCampaignRun as q, buildLoopProvenanceRecord as r, campaignSplitDigestFromIdentities as rt, loopProvenanceArgsFromResult as s, llmJudge as t, campaignScenarioIdentity as tt, provenanceSpansPath as u, openAutoPr as v, assertCodeSurfaceIdentity as w, compareOptimizationMethods as x, assertOptimizationResult as y, defaultProductionGate as z };
+export { runEval as $, surfaceHash as A, campaignMeanComposite as B, optimizationTokenUsageFromSummary as C, componentSurfaceIdentityMaterial as D, codeSurfaceIdentityMaterial as E, assertSearchHistoryMatchesReplay as F, assertGepaCandidatePopulationSummary as G, buildReflectionPrompt as H, createSearchHistoryReceipt as I, DEFAULT_RED_TEAM_CORPUS as J, readGepaCandidatePopulationArtifact as K, searchHistoryCoverageRow as L, SEARCH_HISTORY_RECEIPT_SCHEMA_VERSION as M, SearchHistoryRequiredError as N, renderSurfaceDiff as O, assertCompleteSearchHistory as P, runCanaries as Q, verifySearchHistoryReceipt as R, costFromLedgerSummary as S, assertComponentSurface as T, parseReflectionResponse as U, compareRankKeys as V, recoverTruncatedJson as W, redTeamReport as X, redTeamDataset as Y, scoreRedTeamOutput as Z, paretoFrontier as _, canonicalDigest as a, assertRealBackend as at, combineComparisonCosts as b, loopProvenanceSpans as c, assertCampaignSplitIdentity as ct, verifyLoopProvenanceRecord as d, campaignSplitDigestFromIdentities as dt, runCampaign as et, runImprovementLoop as f, readCachedCell as ft, dominates as g, labelTrustRank as h, campaignMeasurementDigest as i, BackendIntegrityError as it, SEARCH_HISTORY_RECEIPT_DIGEST_ALGORITHM as j, surfaceContentHash as k, provenanceRecordPath as l, campaignScenarioIdentity as lt, isProposedCandidate as m, computeManifestHash as mt, JudgeParseError as n, resolveRunDir as nt, emitLoopProvenance as o, summarizeBackendIntegrity as ot, runOptimization as p, cellCachePath as pt, defaultProductionGate as q, buildLoopProvenanceRecord as r, tangleTracesRoot as rt, loopProvenanceArgsFromResult as s, assertCampaignDesign as st, llmJudge as t, planCampaignRun as tt, provenanceSpansPath as u, campaignSplitDigest as ut, openAutoPr as v, assertCodeSurfaceIdentity as w, compareOptimizationMethods as x, assertOptimizationResult as y, campaignBreakdown as z };
 
-//# sourceMappingURL=llm-judge-B7ffRF8w.js.map
+//# sourceMappingURL=llm-judge-DWq1Ptco.js.map