@tangle-network/agent-eval 0.145.19 → 0.145.20

package/dist/multishot/index.js

@@ -1,5 +1,5 @@
 import { i as isModelPriced, n as estimateCost } from "../metrics-Cl0L1KUy.js";
-import { t as runAgentMatrix } from "../matrix-BzQnu2S6.js";
+import { r as withCellSpend, t as runAgentMatrix } from "../matrix-pWinRMPl.js";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 //#region src/multishot/router.ts
@@ -398,10 +398,11 @@ const MULTISHOT_ROLES = /* @__PURE__ */ new Set([
 *  scores as though the artifact was never produced; a non-finite `costUsd`
 *  reaches `summary.totalCostUsd` and makes every cost number NaN.
 *
-*  It does NOT protect spend. A rejected cell records `costUsd: 0`, so a shot
-*  that spends before it returns a malformed result leaves that spend out of
-*  the cumulative sum the cost ceiling reads. That is how the matrix records
-*  every failed cell, not something this guard changes.
+*  A rejected cell is still billed: the matrix cell reads the shot's own
+*  `costUsd` when it is a usable amount and declares that spend on the throw,
+*  so money the shot spent before returning a malformed result stays in the
+*  cumulative sum the cost ceiling reads. A result whose `costUsd` is itself
+*  malformed carries no usable amount, and the cell records as `uncaptured`.
 *
 *  Every required field of `MultishotMessage` and `MultishotArtifact` is
 *  checked, including `toolCalls` elements and `invocation.args`. Optional
@@ -416,6 +417,16 @@ function assertMultishotShotResult(value) {
 	assertFiniteCount(result.toolCalls, "toolCalls");
 	assertFiniteCount(result.durationMs, "durationMs");
 	assertFiniteCount(result.costUsd, "costUsd");
+	if (result.costProvenance !== void 0) assertCostProvenance(result.costProvenance);
+}
+function assertCostProvenance(value) {
+	const row = requireRow(value, "costProvenance");
+	if (row.kind !== "observed" && row.kind !== "estimated" && row.kind !== "uncaptured") throw new MultishotShotResultError(`costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`);
+	if (row.kind === "uncaptured") {
+		if (row.usd !== null) throw new MultishotShotResultError(`uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`);
+		return;
+	}
+	assertFiniteCount(row.usd, "costProvenance.usd");
 }
 function assertMessage(value, index) {
 	const row = requireRow(value, `transcript[${index}]`);
@@ -460,6 +471,22 @@ function describeValue(value) {
 //#endregion
 //#region src/multishot/multishot.ts
 async function runMultishot(opts) {
+	const meter = {
+		costUsd: 0,
+		startedAt: Date.now(),
+		uncaptured: false
+	};
+	try {
+		return await runShotTurns(opts, meter);
+	} catch (err) {
+		throw withCellSpend(err, {
+			costUsd: meter.costUsd,
+			durationMs: Date.now() - meter.startedAt,
+			kind: meter.uncaptured ? "uncaptured" : "estimated"
+		});
+	}
+}
+async function runShotTurns(opts, meter) {
 	const apiKey = opts.apiKey ?? requireRouterApiKey();
 	const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
 	const maxTurns = opts.maxTurns ?? 10;
@@ -485,11 +512,9 @@ async function runMultishot(opts) {
 	const agentTransport = opts.agentTransport ?? routerTransport;
 	const driverTransport = opts.driverTransport ?? routerTransport;
 	const shape = defaultShapeFromProfile(opts.profile, opts.shape);
-	const start = Date.now();
 	const transcript = [];
 	const artifacts = [];
 	let toolCalls = 0;
-	let totalCostUsd = 0;
 	const opener = shape.buildOpener(opts.persona);
 	transcript.push({
 		role: "user",
@@ -514,7 +539,8 @@ async function runMultishot(opts) {
 				maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
 				signal: opts.signal
 			});
-			totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
+			meter.costUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
+			if (agentCostUsd === void 0 && agentUsage === void 0) meter.uncaptured = true;
 			const agentText = (agentMsg.content ?? "").trim();
 			const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
 				id: tc.id,
@@ -553,7 +579,7 @@ async function runMultishot(opts) {
 							signal: opts.signal
 						});
 						toolResult = r.content;
-						totalCostUsd += r.costUsd;
+						meter.costUsd += r.costUsd;
 						const artifactType = artifactTypeFor(tc.name);
 						if (artifactType) artifacts.push({
 							type: artifactType,
@@ -590,9 +616,9 @@ async function runMultishot(opts) {
 				turn,
 				models: driverModels,
 				maxTokens: driverMaxTokens,
-				signal: opts.signal
+				signal: opts.signal,
+				meter
 			});
-			totalCostUsd += driver.costUsd;
 			agentMessages.push({
 				role: "user",
 				content: driver.content
@@ -607,8 +633,15 @@ async function runMultishot(opts) {
 		transcript,
 		artifacts,
 		toolCalls,
-		durationMs: Date.now() - start,
-		costUsd: totalCostUsd
+		durationMs: Date.now() - meter.startedAt,
+		costUsd: meter.costUsd,
+		costProvenance: meter.uncaptured ? {
+			kind: "uncaptured",
+			usd: null
+		} : {
+			kind: "estimated",
+			usd: meter.costUsd
+		}
 	};
 }
 async function driverTurn(opts) {
@@ -637,11 +670,10 @@ async function driverTurn(opts) {
 			maxTokens: opts.maxTokens,
 			signal: opts.signal
 		});
+		opts.meter.costUsd += costUsd ?? estimateRouterCost(model, usage);
+		if (costUsd === void 0 && usage === void 0) opts.meter.uncaptured = true;
 		const content = (message.content ?? "").trim();
-		if (content.length > 0) return {
-			content,
-			costUsd: costUsd ?? estimateRouterCost(model, usage)
-		};
+		if (content.length > 0) return { content };
 	}
 	throw new MultishotDriverEmptyError(opts.turn);
 }
@@ -704,7 +736,9 @@ async function runMultishotMatrix(opts) {
 		reps: opts.reps ?? 1,
 		maxConcurrency: opts.maxConcurrency ?? 2,
 		costCeiling: opts.costCeiling,
+		maxCellCostUsd: opts.maxCellCostUsd,
 		async runCell(cell) {
+			const cellStartedAt = Date.now();
 			const profile = cell.axes.profile?.value;
 			const persona = cell.axes.persona?.value;
 			const profileId = String(cell.axes.profile?.id ?? "unknown");
@@ -729,87 +763,111 @@ async function runMultishotMatrix(opts) {
 				apiKey: opts.apiKey,
 				baseUrl: opts.baseUrl
 			});
-			assertMultishotShotResult(sim);
-			const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
-			const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
-			const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([
-				runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
-					transcript: sim.transcript,
-					persona
-				}),
-				opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
-					artifact,
-					persona
-				}).then((result) => ({
-					score: {
-						...result.score,
-						turn: artifact.turn,
-						type: artifact.type
+			const shotCostUsd = shotCostSubtotal(sim);
+			const shotCostComplete = sim?.costProvenance?.kind !== "uncaptured";
+			let judgeCostUsd = 0;
+			let judgeCostComplete = false;
+			let phase = "validate";
+			try {
+				assertMultishotShotResult(sim);
+				const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
+				const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
+				phase = "judging";
+				const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([
+					runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
+						transcript: sim.transcript,
+						persona
+					}),
+					opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
+						artifact,
+						persona
+					}).then((result) => ({
+						score: {
+							...result.score,
+							turn: artifact.turn,
+							type: artifact.type
+						},
+						cost: result.cost
+					})))) : Promise.resolve([]),
+					opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
+						artifact,
+						persona
+					}).then((result) => ({
+						score: {
+							...result.score,
+							turn: artifact.turn,
+							type: artifact.type
+						},
+						cost: result.cost
+					})))) : Promise.resolve([])
+				]);
+				const judgeRuns = [
+					conversationRun,
+					...codeReviewRuns,
+					...contentReviewRuns
+				];
+				judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0);
+				judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== "uncaptured");
+				phase = "scoring";
+				const conversation = conversationRun.score;
+				const codeReviews = codeReviewRuns.map((run) => run.score);
+				const contentReviews = contentReviewRuns.map((run) => run.score);
+				const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
+					conversation,
+					codeReviews: opts.judges.codeReview ? codeReviews : void 0,
+					contentReviews: opts.judges.contentQuality ? contentReviews : void 0
+				});
+				const cellScore = {
+					composite,
+					conversation
+				};
+				if (opts.judges.codeReview) cellScore.codeReview = {
+					perArtifact: codeReviews,
+					composite: codeComposite
+				};
+				if (opts.judges.contentQuality) cellScore.contentQuality = {
+					perArtifact: contentReviews,
+					composite: contentComposite
+				};
+				const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
+				mkdirSync(cellDir, { recursive: true });
+				writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
+				writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
+				writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
+				const notes = [`convo=${conversation.composite.toFixed(1)}`];
+				if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
+				if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
+				if (allJudgesFailed) notes.push("all-judges-failed");
+				if (!judgeCostComplete) notes.push("judge-cost-incomplete");
+				return {
+					output: {
+						turns: sim.transcript.length,
+						toolCalls: sim.toolCalls,
+						artifactCount: sim.artifacts.length
 					},
-					cost: result.cost
-				})))) : Promise.resolve([]),
-				opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
-					artifact,
-					persona
-				}).then((result) => ({
-					score: {
-						...result.score,
-						turn: artifact.turn,
-						type: artifact.type
+					verdict: {
+						valid: composite >= 5,
+						score: composite,
+						notes: notes.join(" ")
 					},
-					cost: result.cost
-				})))) : Promise.resolve([])
-			]);
-			const conversation = conversationRun.score;
-			const codeReviews = codeReviewRuns.map((run) => run.score);
-			const contentReviews = contentReviewRuns.map((run) => run.score);
-			const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
-				conversation,
-				codeReviews: opts.judges.codeReview ? codeReviews : void 0,
-				contentReviews: opts.judges.contentQuality ? contentReviews : void 0
-			});
-			const cellScore = {
-				composite,
-				conversation
-			};
-			if (opts.judges.codeReview) cellScore.codeReview = {
-				perArtifact: codeReviews,
-				composite: codeComposite
-			};
-			if (opts.judges.contentQuality) cellScore.contentQuality = {
-				perArtifact: contentReviews,
-				composite: contentComposite
-			};
-			const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
-			mkdirSync(cellDir, { recursive: true });
-			writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
-			writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
-			writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
-			const notes = [`convo=${conversation.composite.toFixed(1)}`];
-			if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
-			if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
-			if (allJudgesFailed) notes.push("all-judges-failed");
-			const judgeRuns = [
-				conversationRun,
-				...codeReviewRuns,
-				...contentReviewRuns
-			];
-			const judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0);
-			if (judgeRuns.some((run) => run.cost.kind === "uncaptured")) notes.push("judge-cost-incomplete");
-			return {
-				output: {
-					turns: sim.transcript.length,
-					toolCalls: sim.toolCalls,
-					artifactCount: sim.artifacts.length
-				},
-				verdict: {
-					valid: composite >= 5,
-					score: composite,
-					notes: notes.join(" ")
-				},
-				costUsd: sim.costUsd + judgeCostUsd,
-				durationMs: sim.durationMs
-			};
+					costUsd: sim.costUsd + judgeCostUsd,
+					costProvenance: judgeCostComplete && shotCostComplete ? {
+						kind: "estimated",
+						usd: sim.costUsd + judgeCostUsd
+					} : {
+						kind: "uncaptured",
+						usd: null
+					},
+					durationMs: sim.durationMs
+				};
+			} catch (err) {
+				if (shotCostUsd === void 0) throw err;
+				throw withCellSpend(err, {
+					costUsd: shotCostUsd + judgeCostUsd,
+					durationMs: Date.now() - cellStartedAt,
+					kind: shotCostComplete && (phase === "validate" || phase === "scoring" && judgeCostComplete) ? "estimated" : "uncaptured"
+				});
+			}
 		}
 	});
 	const summary = {
@@ -817,6 +875,8 @@ async function runMultishotMatrix(opts) {
 		passRate: matrix.summary.overallPassRate,
 		meanScore: matrix.summary.overallMeanScore,
 		totalCostUsd: matrix.summary.totalCostUsd,
+		costUncapturedCells: matrix.summary.costUncapturedCells,
+		ceilingChargedUsd: matrix.summary.ceilingChargedUsd,
 		durationMs: matrix.summary.durationMs,
 		runsExecuted: matrix.summary.runsExecuted,
 		cellsSkipped: matrix.summary.cellsSkipped,
@@ -824,11 +884,14 @@ async function runMultishotMatrix(opts) {
 		byPersona: matrix.byAxis.persona
 	};
 	writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
+	const uncaptured = matrix.summary.costUncapturedCells;
+	const costLabel = uncaptured > 0 ? "Cost (at least)" : "Cost";
 	const md = [
 		`# Multishot matrix`,
 		``,
-		`**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
+		`**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
 		``,
+		...uncaptured > 0 ? [`> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`, ``] : [],
 		`## By profile`,
 		``,
 		"| profile | pass | mean | cost |",
@@ -845,6 +908,14 @@ async function runMultishotMatrix(opts) {
 	writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
 	return { matrix };
 }
+/** The shot's own spend, when the value it resolved with reports a usable
+*  amount. `undefined` when it does not — a malformed shot result is exactly
+*  the case where the number cannot be trusted, and billing a wrong figure is
+*  worse than recording the cell as uncaptured. */
+function shotCostSubtotal(sim) {
+	const value = sim?.costUsd;
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
+}
 function withJudgeMaxTokens(judge, maxTokens) {
 	if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
 	return {