@tangle-network/agent-eval 0.203.1 → 0.203.4

package/dist/{llm-judge-EUMHOZOQ.js → llm-judge-DyZdrp6I.js}

@@ -23,7 +23,7 @@ import { C as combineAbortSignals, d as stripFencedJson, o as costReceiptFromLlm
 import { createHash } from "node:crypto";
 import { writeFileSync } from "node:fs";
 import { z } from "zod";
-import { basename, dirname, isAbsolute, join } from "node:path";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { homedir, tmpdir } from "node:os";
 import { execSync } from "node:child_process";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -175,6 +175,10 @@ function buildCellSchedule(scenarios, seed, reps) {
 function cellDirectory(runDir, cellId) {
 	return join(runDir, cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
 }
+function cellAttemptDirectory(cellDir, attempt) {
+	if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(attempt.runAttemptId) || !Number.isSafeInteger(attempt.number) || attempt.number < 1) throw new Error("Invalid campaign attempt identity");
+	return join(cellDir, "attempts", attempt.runAttemptId, `attempt-${attempt.number}`);
+}
 function cellCachePath(runDir, cellId) {
 	return join(cellDirectory(runDir, cellId), "cached-result.json");
 }
@@ -342,6 +346,10 @@ async function executeCell(args) {
 		...stableCostTags,
 		runAttemptId: args.runAttemptId
 	};
+	const attemptCostTags = {
+		...costTags,
+		attemptNumber: String(args.attempt)
+	};
 	const cachePath = join(cellDir, "cached-result.json");
 	if (args.resumable) {
 		const cached = readCachedCell({
@@ -375,15 +383,39 @@ async function executeCell(args) {
 		}
 	}
 	const startMs = Date.now();
-	const trace = args.buildTraceWriter(args.slot.cellId, cellDir);
+	const attempt = {
+		runAttemptId: args.runAttemptId,
+		number: args.attempt
+	};
+	const attemptDir = cellAttemptDirectory(cellDir, attempt);
+	const artifactDir = join(attemptDir, "artifacts");
+	const traceDir = join(attemptDir, "trace");
+	storage.ensureDir(artifactDir);
+	storage.ensureDir(traceDir);
+	const attemptPrefix = relative(cellDir, attemptDir).split(sep).join("/");
 	const artifactsByPath = {};
+	const identityPath = join(attemptDir, "identity.json");
+	storage.write(identityPath, JSON.stringify({
+		...attempt,
+		manifestHash: args.manifestHash,
+		cellId: args.slot.cellId,
+		scenarioId: args.slot.scenario.id,
+		rep: args.slot.rep,
+		seed: args.slot.cellSeed,
+		startedAt: args.now().toISOString()
+	}, null, 2));
+	artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/identity.json`] = identityPath;
+	storage.write(join(cellDir, "latest-attempt.json"), JSON.stringify(attempt));
+	const trace = args.buildTraceWriter(args.slot.cellId, traceDir);
 	let paidCallStarted = false;
 	const artifacts = {
 		async write(path, content) {
-			const fullPath = join(cellDir, path);
+			const fullPath = resolve(artifactDir, path);
+			const artifactPath = relative(artifactDir, fullPath);
+			if (!artifactPath || artifactPath === ".." || artifactPath.startsWith(".." + sep) || isAbsolute(artifactPath)) throw new Error("Artifact path must remain inside this execution attempt");
 			storage.ensureDir(join(fullPath, ".."));
 			storage.write(fullPath, content);
-			artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath;
+			artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/artifacts/${artifactPath.split(sep).join("/")}`] = fullPath;
 			return fullPath;
 		},
 		async writeJson(path, value) {
@@ -397,7 +429,7 @@ async function executeCell(args) {
 			channel: input.channel ?? "agent",
 			phase: args.costPhase,
 			actor: input.actor,
-			tags: costTags,
+			tags: attemptCostTags,
 			signal: cellAbort.signal
 		});
 		if (result.receipt) trace.span(`cost.${result.receipt.actor}`, { amountUsd: result.receipt.costUsd }).end();
@@ -531,7 +563,7 @@ async function executeCell(args) {
 				signal: args.signal,
 				costLedger: args.costLedger,
 				costPhase: args.costPhase,
-				costTags
+				costTags: attemptCostTags
 			});
 			judgeScores[judge.name] = score;
 		} catch (err) {
@@ -555,6 +587,7 @@ async function executeCell(args) {
 	const costCallIds = args.costLedger.list({ tags: costTags }).map((receipt) => receipt.callId).sort();
 	const cell = {
 		manifestHash: args.manifestHash,
+		attempt,
 		cellId: args.slot.cellId,
 		scenarioId: args.slot.scenario.id,
 		rep: args.slot.rep,
@@ -591,13 +624,15 @@ async function executeCell(args) {
 			})
 		};
 		retry = args.cellRetry !== void 0 && args.attempt < args.cellRetry.attempts && !args.signal.aborted && fatalCellError === void 0 && args.cellRetry.retryable(receipt.failure);
-		const receiptName = retry ? `failure-receipt.attempt-${args.attempt}.json` : "failure-receipt.json";
-		const failurePath = join(cellDir, receiptName);
+		const failurePath = join(attemptDir, "failure-receipt.json");
 		storage.write(failurePath, JSON.stringify(receipt, null, 2));
-		artifactsByPath[`${args.slot.cellId}/${receiptName}`] = failurePath;
+		artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/failure-receipt.json`] = failurePath;
 		if (!retry) args.onFailure?.(failure);
 	}
 	await trace.flush();
+	const resultPath = join(attemptDir, "result.json");
+	storage.write(resultPath, JSON.stringify(cell));
+	artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/result.json`] = resultPath;
 	if (!errorMessage && args.resumable) storage.write(cachePath, JSON.stringify(cell));
 	if (fatalCellError !== void 0) throw fatalCellError;
 	return {
@@ -4556,14 +4591,21 @@ var SurfaceNodes = class {
 		if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
 		return this.cellResult(cell, work);
 	}
-	/** A cell's result as its campaign stored it: the cache of a scored cell,
-	* the final failure receipt of a failed one. */
 	/** A cell's result as its campaign stored it for this surface: the cache of
 	* a scored cell, the final failure receipt of a failed one. A result another
 	* surface left in the same directory does not count. */
 	storedCell(dir, surface, taskId, rep) {
 		const cellId = `${taskId}:${rep}`;
 		const manifestHash = this.manifestFor(surface);
+		const cellDir = cellDirectory(dir, cellId);
+		const latest = this.storage.read(join(cellDir, "latest-attempt.json"));
+		if (latest !== void 0) {
+			const attemptDir = cellAttemptDirectory(cellDir, JSON.parse(latest));
+			const result = this.storage.read(join(attemptDir, "result.json"));
+			const receipt = this.storage.read(join(attemptDir, "failure-receipt.json"));
+			const cell = result !== void 0 ? JSON.parse(result) : receipt !== void 0 ? JSON.parse(receipt).cell : void 0;
+			return cell?.manifestHash === manifestHash ? cell : void 0;
+		}
 		const cached = this.storage.read(cellCachePath(dir, cellId));
 		const scored = cached === void 0 ? void 0 : JSON.parse(cached);
 		if (scored?.manifestHash === manifestHash) return scored;
@@ -6132,4 +6174,4 @@ function firstString(value) {
 //#endregion
 export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
 
-//# sourceMappingURL=llm-judge-EUMHOZOQ.js.map
+//# sourceMappingURL=llm-judge-DyZdrp6I.js.map