@tangle-network/agent-eval 0.142.1 → 0.143.0

package/dist/{skillopt-optimization-method-DV6c4ooy.js → skillopt-optimization-method-Di8Xkalt.js}

@@ -2,11 +2,11 @@ import { c as ValidationError, i as JudgeError, t as AgentEvalError } from "./er
 import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
 import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
 import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-2GIUo1et.js";
-import { C as safePathComponent, S as removeCredentialEnvironment, _ as assertJsonValue, b as isExternalTextCandidate, d as runWithCleanup, f as startExternalOptimizerModelProxy, g as assertExternalOptimizerModelBudget, h as sendJson, l as runExternalOptimizerProcess, m as listenLocal, n as createRunCostLedger, p as closeServer, r as fsCampaignStorage, t as acquireSingleRunLock, u as closeExternalOptimizerResources, v as assertNoCredentialValues, x as isRecord, y as isCandidateText } from "./single-run-lock-BcvPQY57.js";
+import { C as safePathComponent, S as removeCredentialEnvironment, _ as assertJsonValue, b as isExternalTextCandidate, d as runWithCleanup, f as startExternalOptimizerModelProxy, g as assertExternalOptimizerModelBudget, h as sendJson, l as runExternalOptimizerProcess, m as listenLocal, n as createRunCostLedger, p as closeServer, r as fsCampaignStorage, t as acquireSingleRunLock, u as closeExternalOptimizerResources, v as assertNoCredentialValues, x as isRecord, y as isCandidateText } from "./single-run-lock-Do-8vBIJ.js";
 import { p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
 import { E as pairedBootstrap, H as weightedComposite, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, T as pairedBinaryScale, d as confidenceInterval, j as pairedRiskDifferenceExact } from "./statistics-ByxzSiOM.js";
 import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
-import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-GyN0kMd8.js";
+import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-DjTi9HLb.js";
 import { z } from "zod";
 import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, isAbsolute, join } from "node:path";
@@ -2069,6 +2069,15 @@ async function runCampaign(opts) {
 	const cells = [];
 	const artifactsByPath = {};
 	const schedule = buildCellSchedule(opts.scenarios, seed, reps);
+	if (resumable) assertScheduleCachesReusable({
+		schedule,
+		runDir: opts.runDir,
+		manifestHash,
+		storage,
+		costLedger,
+		costTags: opts.costTags,
+		rerunInvalidCachedCells: opts.rerunInvalidCachedCells ?? false
+	});
 	const campaignAbort = new AbortController();
 	const onOwnerAbort = () => campaignAbort.abort(opts.signal?.reason);
 	if (opts.signal?.aborted) campaignAbort.abort(opts.signal.reason);
@@ -2153,15 +2162,9 @@ async function runCampaign(opts) {
 }
 async function executeCell(args) {
 	const storage = args.storage;
-	const cellDir = join(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
+	const cellDir = cellDirectory(args.opts.runDir, args.slot.cellId);
 	storage.ensureDir(cellDir);
-	const stableCostTags = {
-		...args.opts.costTags ?? {},
-		runDir: args.opts.runDir,
-		cellId: args.slot.cellId,
-		scenarioId: args.slot.scenario.id,
-		rep: String(args.slot.rep)
-	};
+	const stableCostTags = stableCostTagsFor(args.opts, args.slot);
 	const costTags = {
 		...stableCostTags,
 		runAttemptId: args.runAttemptId
@@ -2174,24 +2177,27 @@ async function executeCell(args) {
 			cellId: args.slot.cellId,
 			manifestHash: args.manifestHash
 		});
+		if (cached.status === "miss" && cacheIssueRequiresExplicitRerun(cached.reason) && !args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
+			cellId: args.slot.cellId,
+			reason: cached.reason
+		}]);
 		if (cached.status === "hit") {
-			enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? "warn");
-			const cachedHasUsage = cached.cell.costUsd > 0 || cached.cell.tokenUsage.input > 0 || cached.cell.tokenUsage.output > 0;
-			if (cached.cell.costCallIds === void 0) {
-				if (cachedHasUsage || Object.keys(cached.cell.judgeScores).length > 0) throw new CostAccountingIncompleteError(`runCampaign: cached cell '${args.slot.cellId}' does not identify its ledger receipts`);
-			} else if (!Array.isArray(cached.cell.costCallIds) || cached.cell.costCallIds.some((callId) => typeof callId !== "string" || callId.trim().length === 0) || new Set(cached.cell.costCallIds).size !== cached.cell.costCallIds.length) throw new CostAccountingIncompleteError(`runCampaign: cached cell '${args.slot.cellId}' has invalid ledger receipt IDs`);
-			else {
-				const restoredCallIds = new Set(args.costLedger.list({ tags: stableCostTags }).map((receipt) => receipt.callId));
-				const missingCallIds = cached.cell.costCallIds.filter((callId) => !restoredCallIds.has(callId));
-				if (missingCallIds.length > 0) throw new CostAccountingIncompleteError(`runCampaign: cached cell '${args.slot.cellId}' is missing ledger receipt(s): ${missingCallIds.join(", ")}`);
+			const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
+			if (receiptProblem === void 0) {
+				enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? "warn");
+				return {
+					cell: {
+						...cached.cell,
+						cached: true
+					},
+					artifactsByPath: {}
+				};
 			}
-			return {
-				cell: {
-					...cached.cell,
-					cached: true
-				},
-				artifactsByPath: {}
-			};
+			if (!args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
+				cellId: args.slot.cellId,
+				reason: "invalid-cost-receipts",
+				detail: receiptProblem
+			}]);
 		}
 	}
 	const startMs = Date.now();
@@ -2321,7 +2327,9 @@ async function executeCell(args) {
 	const tokenUsage = {
 		input: agentCost.inputTokens,
 		output: agentCost.outputTokens,
-		...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {}
+		...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
+		...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
+		...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
 	};
 	const resolvedModel = agentReceipts.at(-1)?.model;
 	const dispatchResult = {
@@ -2377,7 +2385,7 @@ async function executeCell(args) {
 		artifact: artifact ?? null,
 		judgeScores,
 		costUsd: agentCost.totalCostUsd,
-		costEstimated: agentReceipts.some((receipt) => receipt.actualCostUsd === void 0 && !receipt.costUnknown),
+		costProvenance: agentCost.costProvenance,
 		costCallIds,
 		tokenUsage,
 		...resolvedModel ? { resolvedModel } : {},
@@ -2467,6 +2475,11 @@ function planCampaignRun(opts) {
 	assertCampaignDesign(opts.scenarios, reps);
 	if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("planCampaignRun: runDir is required and must be a non-empty string");
 	opts.runDir = resolveRunDir(opts.runDir, opts.repo);
+	const costLedger = opts.costLedger ?? createRunCostLedger({
+		storage,
+		runDir: opts.runDir,
+		ensureRunDir: false
+	});
 	const manifestHash = computeManifestHash({
 		scenarios: opts.scenarios,
 		judges: opts.judges ?? [],
@@ -2476,7 +2489,7 @@ function planCampaignRun(opts) {
 	});
 	const splitDigest = campaignSplitDigest(opts.scenarios, reps);
 	const cells = buildCellSchedule(opts.scenarios, seed, reps).map((slot) => {
-		const cachePath = join(opts.runDir, slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"), "cached-result.json");
+		const cachePath = cellCachePath(opts.runDir, slot.cellId);
 		if (!resumable) return {
 			cellId: slot.cellId,
 			scenarioId: slot.scenario.id,
@@ -2492,31 +2505,48 @@ function planCampaignRun(opts) {
 			cellId: slot.cellId,
 			manifestHash
 		});
-		if (cached.status === "hit") return {
-			cellId: slot.cellId,
-			scenarioId: slot.scenario.id,
-			rep: slot.rep,
-			seed: slot.cellSeed,
-			cachePath,
-			status: "cached"
-		};
+		if (cached.status === "hit") {
+			if (cachedCellReceiptProblem(cached.cell, costLedger, stableCostTagsFor({
+				runDir: opts.runDir,
+				costTags: opts.costTags
+			}, slot)) !== void 0) return {
+				cellId: slot.cellId,
+				scenarioId: slot.scenario.id,
+				rep: slot.rep,
+				seed: slot.cellSeed,
+				cachePath,
+				status: opts.rerunInvalidCachedCells ? "run" : "blocked",
+				reason: "invalid-cost-receipts"
+			};
+			return {
+				cellId: slot.cellId,
+				scenarioId: slot.scenario.id,
+				rep: slot.rep,
+				seed: slot.cellSeed,
+				cachePath,
+				status: "cached"
+			};
+		}
+		const blocked = cacheIssueRequiresExplicitRerun(cached.reason) && !opts.rerunInvalidCachedCells;
 		return {
 			cellId: slot.cellId,
 			scenarioId: slot.scenario.id,
 			rep: slot.rep,
 			seed: slot.cellSeed,
 			cachePath,
-			status: "run",
+			status: blocked ? "blocked" : "run",
 			reason: cached.reason
 		};
 	});
 	const cellsCached = cells.filter((cell) => cell.status === "cached").length;
+	const cellsBlocked = cells.filter((cell) => cell.status === "blocked").length;
 	return {
 		manifestHash,
 		splitDigest,
 		totalCells: cells.length,
 		cellsCached,
-		cellsToRun: cells.length - cellsCached,
+		cellsBlocked,
+		cellsToRun: cells.filter((cell) => cell.status === "run").length,
 		cells
 	};
 }
@@ -2633,6 +2663,59 @@ function buildCellSchedule(scenarios, seed, reps) {
 	}
 	return schedule;
 }
+function cellDirectory(runDir, cellId) {
+	return join(runDir, cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
+}
+function cellCachePath(runDir, cellId) {
+	return join(cellDirectory(runDir, cellId), "cached-result.json");
+}
+function stableCostTagsFor(opts, slot) {
+	return {
+		...opts.costTags ?? {},
+		runDir: opts.runDir,
+		cellId: slot.cellId,
+		scenarioId: slot.scenario.id,
+		rep: String(slot.rep)
+	};
+}
+function assertScheduleCachesReusable(args) {
+	const blocked = [];
+	for (const slot of args.schedule) {
+		const cached = readCachedCell({
+			storage: args.storage,
+			cachePath: cellCachePath(args.runDir, slot.cellId),
+			cellId: slot.cellId,
+			manifestHash: args.manifestHash
+		});
+		if (cached.status === "hit") {
+			const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTagsFor({
+				runDir: args.runDir,
+				costTags: args.costTags
+			}, slot));
+			if (receiptProblem !== void 0 && !args.rerunInvalidCachedCells) blocked.push({
+				cellId: slot.cellId,
+				reason: "invalid-cost-receipts",
+				detail: receiptProblem
+			});
+		} else if (cacheIssueRequiresExplicitRerun(cached.reason) && !args.rerunInvalidCachedCells) blocked.push({
+			cellId: slot.cellId,
+			reason: cached.reason
+		});
+	}
+	if (blocked.length > 0) throw invalidCachedCellsError(blocked);
+}
+function cachedCellReceiptProblem(cached, costLedger, stableCostTags) {
+	const reportsPaidActivity = cached.costProvenance.kind !== "observed" || cached.costUsd > 0 || cached.tokenUsage.input > 0 || cached.tokenUsage.output > 0;
+	if (cached.costCallIds === void 0) {
+		if (reportsPaidActivity || Object.keys(cached.judgeScores).length > 0) return "does not identify its ledger receipts";
+		return;
+	}
+	if (!Array.isArray(cached.costCallIds) || cached.costCallIds.some((callId) => typeof callId !== "string" || callId.trim().length === 0) || new Set(cached.costCallIds).size !== cached.costCallIds.length) return "has invalid ledger receipt IDs";
+	if (cached.costCallIds.length === 0 && reportsPaidActivity) return "does not identify its ledger receipts";
+	const restoredCallIds = new Set(costLedger.list({ tags: stableCostTags }).map((receipt) => receipt.callId));
+	const missingCallIds = cached.costCallIds.filter((callId) => !restoredCallIds.has(callId));
+	if (missingCallIds.length > 0) return `is missing ledger receipt(s): ${missingCallIds.join(", ")}`;
+}
 function dispatchRefFor(dispatch, override) {
 	const ref = override ?? dispatch?.name ?? "anonymous";
 	if (typeof ref !== "string" || ref.trim().length === 0) throw new Error("runCampaign: dispatchRef must be a non-empty string when provided");
@@ -2642,18 +2725,32 @@ function readCachedCell(args) {
 	const raw = args.storage.read(args.cachePath);
 	if (raw === void 0) return {
 		status: "miss",
-		reason: "missing"
+		reason: args.storage.exists(args.cachePath) ? "corrupt" : "missing"
 	};
+	let parsed;
 	try {
-		const cached = JSON.parse(raw);
-		if (cached.cellId !== args.cellId) return {
-			status: "miss",
-			reason: "cell-mismatch"
-		};
-		if (cached.manifestHash !== args.manifestHash) return {
+		parsed = JSON.parse(raw);
+	} catch {
+		return {
 			status: "miss",
-			reason: "manifest-mismatch"
+			reason: "corrupt"
 		};
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {
+		status: "miss",
+		reason: "corrupt"
+	};
+	const cached = parsed;
+	if (cached.cellId !== args.cellId) return {
+		status: "miss",
+		reason: "cell-mismatch"
+	};
+	if (cached.manifestHash !== args.manifestHash) return {
+		status: "miss",
+		reason: "manifest-mismatch"
+	};
+	try {
+		campaignCellCostProvenance(cached);
 		return {
 			status: "hit",
 			cell: cached
@@ -2661,10 +2758,16 @@ function readCachedCell(args) {
 	} catch {
 		return {
 			status: "miss",
-			reason: "corrupt"
+			reason: cached.costProvenance === void 0 ? "missing-cost-provenance" : "invalid-cost-provenance"
 		};
 	}
 }
+function cacheIssueRequiresExplicitRerun(reason) {
+	return reason === "cell-mismatch" || reason === "missing-cost-provenance" || reason === "invalid-cost-provenance" || reason === "invalid-cost-receipts" || reason === "corrupt";
+}
+function invalidCachedCellsError(cells) {
+	return new CostAccountingIncompleteError(`runCampaign: cached cell(s) require explicit paid re-dispatch: ${cells.map((cell) => `${cell.cellId} (${cell.reason}${cell.detail ? `: ${cell.detail}` : ""})`).join(", ")}; refusing to begin campaign. Inspect planCampaignRun, then set rerunInvalidCachedCells: true to rerun only these cells while retaining valid caches, or resumable: false when a full rerun is intended.`);
+}
 async function captureToStore(args) {
 	await args.store.observe({
 		scenario: args.scenario,
@@ -6524,7 +6627,7 @@ function campaignMeasurementDigest(campaign) {
 			generation: cell.generation ?? null,
 			judgeScores: cell.judgeScores,
 			costUsd: cell.costUsd,
-			costEstimated: cell.costEstimated ?? null,
+			costProvenance: cell.costProvenance,
 			costCallIds: [...cell.costCallIds ?? []].sort(),
 			tokenUsage: cell.tokenUsage,
 			resolvedModel: cell.resolvedModel ?? null,
@@ -7162,4 +7265,4 @@ function skillOptOptimizationMethod(config) {
 //#endregion
 export { campaignScenarioIdentity as $, assertOptimizationResult as A, pairedDeltaTest as At, surfaceHash as B, detectScale as C, paretoFrontier as Ct, runCanaries as D, decidePairedPromotion as Dt, pairHoldout as E, recoverTruncatedJson as Et, assertComponentSurface as F, summarizeBackendIntegrity as Ft, buildReflectionPrompt as G, campaignMeanComposite as H, codeSurfaceIdentityMaterial as I, JudgeParseError as It, runCampaign as J, parseReflectionResponse as K, componentSurfaceIdentityMaterial as L, costFromLedgerSummary as M, assertRealAgentReceipts as Mt, optimizationTokenUsageFromSummary as N, assertRealBackend as Nt, composeGate as O, pairedDecisionShape as Ot, assertCodeSurfaceIdentity as P, summarizeAgentReceiptIntegrity as Pt, assertCampaignSplitIdentity as Q, renderSurfaceDiff as R, defaultProductionGate as S, dominates as St, heldoutSignificance as T, scalarScore as Tt, compareRankKeys as U, campaignBreakdown as V, DEFAULT_MUTATION_PRIMITIVES as W, tangleTracesRoot as X, resolveRunDir as Y, assertCampaignDesign as Z, buildEvidenceVector as _, canonicalJson as _t, emitLoopProvenance as a, createReferenceEquivalenceJudge as at, powerPreflight as b, inMemoryVerdictCache as bt, provenanceRecordPath as c, redTeamDataset as ct, runImprovementLoop as d, toolNamesForRun as dt, campaignSplitDigest as et, runOptimization as f, Dataset as ft, gepaOptimizationMethod as g, cachedJudge as gt, labelTrustRank as h, llmJudge as ht, canonicalDigest as i, REFERENCE_EQUIVALENCE_JUDGE_VERSION as it, compareOptimizationMethods as j, BackendIntegrityError as jt, externalTextOptimizationMethod as k, minimumPairsForPairedDeltaTest as kt, provenanceSpansPath as l, redTeamReport as lt, isProposedCandidate as m, hashScenarios as mt, buildLoopProvenanceRecord as n, openAutoPr as nt, loopProvenanceArgsFromResult as o, runReferenceEquivalenceJudge as ot, runEval as p, HoldoutLockedError as pt, planCampaignRun as q, campaignMeasurementDigest as r, REFERENCE_EQUIVALENCE_INPUT_LIMITS as rt, loopProvenanceSpans as s, DEFAULT_RED_TEAM_CORPUS as st, skillOptOptimizationMethod as t, campaignSplitDigestFromIdentities as tt, verifyLoopProvenanceRecord as u, scoreRedTeamOutput as ut, paretoPolicy as v, contentHash as vt, dimensionRegressions as w, paretoFrontierWithCrowding as wt, heldOutGate as x, crowdingDistance as xt, paretoSignificanceGate as y, fileVerdictCache as yt, surfaceContentHash as z };
 
-//# sourceMappingURL=skillopt-optimization-method-DV6c4ooy.js.map
+//# sourceMappingURL=skillopt-optimization-method-Di8Xkalt.js.map