@tangle-network/agent-eval 0.145.18 → 0.145.20

package/dist/multishot/index.js

@@ -1,5 +1,5 @@
 import { i as isModelPriced, n as estimateCost } from "../metrics-Cl0L1KUy.js";
-import { t as runAgentMatrix } from "../matrix-BzQnu2S6.js";
+import { r as withCellSpend, t as runAgentMatrix } from "../matrix-pWinRMPl.js";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 //#region src/multishot/router.ts
@@ -379,9 +379,114 @@ var MultishotFatalToolError = class extends Error {
 		this.name = "MultishotFatalToolError";
 	}
 };
+var MultishotShotResultError = class extends Error {
+	constructor(reason) {
+		super(`multishot: shot returned an invalid MultishotResult — ${reason}`);
+		this.name = "MultishotShotResultError";
+	}
+};
+const MULTISHOT_ROLES = /* @__PURE__ */ new Set([
+	"user",
+	"assistant",
+	"tool"
+]);
+/** Contract guard for the value a caller-supplied shot resolves with. The
+*  matrix writes per-cell artifacts, builds judge inputs, and meters cost from
+*  this value, so a malformed result must stop the cell instead of scoring a
+*  degraded one. Two silent degradations this closes: an artifact with no
+*  `type` matches neither the code nor the content artifact set, so the cell
+*  scores as though the artifact was never produced; a non-finite `costUsd`
+*  reaches `summary.totalCostUsd` and makes every cost number NaN.
+*
+*  A rejected cell is still billed: the matrix cell reads the shot's own
+*  `costUsd` when it is a usable amount and declares that spend on the throw,
+*  so money the shot spent before returning a malformed result stays in the
+*  cumulative sum the cost ceiling reads. A result whose `costUsd` is itself
+*  malformed carries no usable amount, and the cell records as `uncaptured`.
+*
+*  Every required field of `MultishotMessage` and `MultishotArtifact` is
+*  checked, including `toolCalls` elements and `invocation.args`. Optional
+*  fields are checked only when present. */
+function assertMultishotShotResult(value) {
+	if (typeof value !== "object" || value === null) throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`);
+	const result = value;
+	if (!Array.isArray(result.transcript)) throw new MultishotShotResultError(`transcript must be an array, received ${describeValue(result.transcript)}`);
+	if (!Array.isArray(result.artifacts)) throw new MultishotShotResultError(`artifacts must be an array, received ${describeValue(result.artifacts)}`);
+	result.transcript.forEach(assertMessage);
+	result.artifacts.forEach(assertArtifact);
+	assertFiniteCount(result.toolCalls, "toolCalls");
+	assertFiniteCount(result.durationMs, "durationMs");
+	assertFiniteCount(result.costUsd, "costUsd");
+	if (result.costProvenance !== void 0) assertCostProvenance(result.costProvenance);
+}
+function assertCostProvenance(value) {
+	const row = requireRow(value, "costProvenance");
+	if (row.kind !== "observed" && row.kind !== "estimated" && row.kind !== "uncaptured") throw new MultishotShotResultError(`costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`);
+	if (row.kind === "uncaptured") {
+		if (row.usd !== null) throw new MultishotShotResultError(`uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`);
+		return;
+	}
+	assertFiniteCount(row.usd, "costProvenance.usd");
+}
+function assertMessage(value, index) {
+	const row = requireRow(value, `transcript[${index}]`);
+	if (typeof row.role !== "string" || !MULTISHOT_ROLES.has(row.role)) throw new MultishotShotResultError(`transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`);
+	assertString(row.content, `transcript[${index}].content`);
+	if (row.toolCallId !== void 0) assertString(row.toolCallId, `transcript[${index}].toolCallId`);
+	if (row.toolCalls === void 0) return;
+	if (!Array.isArray(row.toolCalls)) throw new MultishotShotResultError(`transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`);
+	row.toolCalls.forEach((call, callIndex) => {
+		const field = `transcript[${index}].toolCalls[${callIndex}]`;
+		const row = requireRow(call, field);
+		assertString(row.id, `${field}.id`);
+		assertString(row.name, `${field}.name`);
+		requireRow(row.args, `${field}.args`);
+	});
+}
+function assertArtifact(value, index) {
+	const row = requireRow(value, `artifacts[${index}]`);
+	assertString(row.type, `artifacts[${index}].type`);
+	assertString(row.content, `artifacts[${index}].content`);
+	assertFiniteCount(row.turn, `artifacts[${index}].turn`);
+	const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`);
+	assertString(invocation.name, `artifacts[${index}].invocation.name`);
+	requireRow(invocation.args, `artifacts[${index}].invocation.args`);
+}
+function requireRow(value, field) {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new MultishotShotResultError(`${field} must be an object, received ${describeValue(value)}`);
+	return value;
+}
+function assertString(value, field) {
+	if (typeof value !== "string") throw new MultishotShotResultError(`${field} must be a string, received ${describeValue(value)}`);
+}
+function assertFiniteCount(value, field) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new MultishotShotResultError(`${field} must be a finite number >= 0, received ${describeValue(value)}`);
+}
+function describeValue(value) {
+	if (value === null) return "null";
+	if (Array.isArray(value)) return "an array";
+	if (typeof value === "object") return "an object";
+	return `${typeof value} ${String(value)}`;
+}
 //#endregion
 //#region src/multishot/multishot.ts
 async function runMultishot(opts) {
+	const meter = {
+		costUsd: 0,
+		startedAt: Date.now(),
+		uncaptured: false
+	};
+	try {
+		return await runShotTurns(opts, meter);
+	} catch (err) {
+		throw withCellSpend(err, {
+			costUsd: meter.costUsd,
+			durationMs: Date.now() - meter.startedAt,
+			kind: meter.uncaptured ? "uncaptured" : "estimated"
+		});
+	}
+}
+async function runShotTurns(opts, meter) {
 	const apiKey = opts.apiKey ?? requireRouterApiKey();
 	const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
 	const maxTurns = opts.maxTurns ?? 10;
@@ -407,11 +512,9 @@ async function runMultishot(opts) {
 	const agentTransport = opts.agentTransport ?? routerTransport;
 	const driverTransport = opts.driverTransport ?? routerTransport;
 	const shape = defaultShapeFromProfile(opts.profile, opts.shape);
-	const start = Date.now();
 	const transcript = [];
 	const artifacts = [];
 	let toolCalls = 0;
-	let totalCostUsd = 0;
 	const opener = shape.buildOpener(opts.persona);
 	transcript.push({
 		role: "user",
@@ -436,7 +539,8 @@ async function runMultishot(opts) {
 				maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
 				signal: opts.signal
 			});
-			totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
+			meter.costUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
+			if (agentCostUsd === void 0 && agentUsage === void 0) meter.uncaptured = true;
 			const agentText = (agentMsg.content ?? "").trim();
 			const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
 				id: tc.id,
@@ -475,7 +579,7 @@ async function runMultishot(opts) {
 							signal: opts.signal
 						});
 						toolResult = r.content;
-						totalCostUsd += r.costUsd;
+						meter.costUsd += r.costUsd;
 						const artifactType = artifactTypeFor(tc.name);
 						if (artifactType) artifacts.push({
 							type: artifactType,
@@ -512,9 +616,9 @@ async function runMultishot(opts) {
 				turn,
 				models: driverModels,
 				maxTokens: driverMaxTokens,
-				signal: opts.signal
+				signal: opts.signal,
+				meter
 			});
-			totalCostUsd += driver.costUsd;
 			agentMessages.push({
 				role: "user",
 				content: driver.content
@@ -529,8 +633,15 @@ async function runMultishot(opts) {
 		transcript,
 		artifacts,
 		toolCalls,
-		durationMs: Date.now() - start,
-		costUsd: totalCostUsd
+		durationMs: Date.now() - meter.startedAt,
+		costUsd: meter.costUsd,
+		costProvenance: meter.uncaptured ? {
+			kind: "uncaptured",
+			usd: null
+		} : {
+			kind: "estimated",
+			usd: meter.costUsd
+		}
 	};
 }
 async function driverTurn(opts) {
@@ -559,11 +670,10 @@ async function driverTurn(opts) {
 			maxTokens: opts.maxTokens,
 			signal: opts.signal
 		});
+		opts.meter.costUsd += costUsd ?? estimateRouterCost(model, usage);
+		if (costUsd === void 0 && usage === void 0) opts.meter.uncaptured = true;
 		const content = (message.content ?? "").trim();
-		if (content.length > 0) return {
-			content,
-			costUsd: costUsd ?? estimateRouterCost(model, usage)
-		};
+		if (content.length > 0) return { content };
 	}
 	throw new MultishotDriverEmptyError(opts.turn);
 }
@@ -610,6 +720,7 @@ function computeCellComposite(input) {
 async function runMultishotMatrix(opts) {
 	const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
 	const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
+	const runShot = opts.runShot ?? runMultishot;
 	mkdirSync(opts.runDir, { recursive: true });
 	const matrix = await runAgentMatrix({
 		axes: [{
@@ -625,12 +736,14 @@ async function runMultishotMatrix(opts) {
 		reps: opts.reps ?? 1,
 		maxConcurrency: opts.maxConcurrency ?? 2,
 		costCeiling: opts.costCeiling,
+		maxCellCostUsd: opts.maxCellCostUsd,
 		async runCell(cell) {
+			const cellStartedAt = Date.now();
 			const profile = cell.axes.profile?.value;
 			const persona = cell.axes.persona?.value;
 			const profileId = String(cell.axes.profile?.id ?? "unknown");
 			const personaId = String(cell.axes.persona?.id ?? "unknown");
-			const sim = await runMultishot({
+			const sim = await runShot({
 				profile,
 				persona,
 				shape: opts.shape,
@@ -650,86 +763,111 @@ async function runMultishotMatrix(opts) {
 				apiKey: opts.apiKey,
 				baseUrl: opts.baseUrl
 			});
-			const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
-			const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
-			const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([
-				runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
-					transcript: sim.transcript,
-					persona
-				}),
-				opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
-					artifact,
-					persona
-				}).then((result) => ({
-					score: {
-						...result.score,
-						turn: artifact.turn,
-						type: artifact.type
+			const shotCostUsd = shotCostSubtotal(sim);
+			const shotCostComplete = sim?.costProvenance?.kind !== "uncaptured";
+			let judgeCostUsd = 0;
+			let judgeCostComplete = false;
+			let phase = "validate";
+			try {
+				assertMultishotShotResult(sim);
+				const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
+				const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
+				phase = "judging";
+				const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([
+					runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
+						transcript: sim.transcript,
+						persona
+					}),
+					opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
+						artifact,
+						persona
+					}).then((result) => ({
+						score: {
+							...result.score,
+							turn: artifact.turn,
+							type: artifact.type
+						},
+						cost: result.cost
+					})))) : Promise.resolve([]),
+					opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
+						artifact,
+						persona
+					}).then((result) => ({
+						score: {
+							...result.score,
+							turn: artifact.turn,
+							type: artifact.type
+						},
+						cost: result.cost
+					})))) : Promise.resolve([])
+				]);
+				const judgeRuns = [
+					conversationRun,
+					...codeReviewRuns,
+					...contentReviewRuns
+				];
+				judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0);
+				judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== "uncaptured");
+				phase = "scoring";
+				const conversation = conversationRun.score;
+				const codeReviews = codeReviewRuns.map((run) => run.score);
+				const contentReviews = contentReviewRuns.map((run) => run.score);
+				const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
+					conversation,
+					codeReviews: opts.judges.codeReview ? codeReviews : void 0,
+					contentReviews: opts.judges.contentQuality ? contentReviews : void 0
+				});
+				const cellScore = {
+					composite,
+					conversation
+				};
+				if (opts.judges.codeReview) cellScore.codeReview = {
+					perArtifact: codeReviews,
+					composite: codeComposite
+				};
+				if (opts.judges.contentQuality) cellScore.contentQuality = {
+					perArtifact: contentReviews,
+					composite: contentComposite
+				};
+				const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
+				mkdirSync(cellDir, { recursive: true });
+				writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
+				writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
+				writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
+				const notes = [`convo=${conversation.composite.toFixed(1)}`];
+				if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
+				if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
+				if (allJudgesFailed) notes.push("all-judges-failed");
+				if (!judgeCostComplete) notes.push("judge-cost-incomplete");
+				return {
+					output: {
+						turns: sim.transcript.length,
+						toolCalls: sim.toolCalls,
+						artifactCount: sim.artifacts.length
 					},
-					cost: result.cost
-				})))) : Promise.resolve([]),
-				opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
-					artifact,
-					persona
-				}).then((result) => ({
-					score: {
-						...result.score,
-						turn: artifact.turn,
-						type: artifact.type
+					verdict: {
+						valid: composite >= 5,
+						score: composite,
+						notes: notes.join(" ")
 					},
-					cost: result.cost
-				})))) : Promise.resolve([])
-			]);
-			const conversation = conversationRun.score;
-			const codeReviews = codeReviewRuns.map((run) => run.score);
-			const contentReviews = contentReviewRuns.map((run) => run.score);
-			const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
-				conversation,
-				codeReviews: opts.judges.codeReview ? codeReviews : void 0,
-				contentReviews: opts.judges.contentQuality ? contentReviews : void 0
-			});
-			const cellScore = {
-				composite,
-				conversation
-			};
-			if (opts.judges.codeReview) cellScore.codeReview = {
-				perArtifact: codeReviews,
-				composite: codeComposite
-			};
-			if (opts.judges.contentQuality) cellScore.contentQuality = {
-				perArtifact: contentReviews,
-				composite: contentComposite
-			};
-			const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
-			mkdirSync(cellDir, { recursive: true });
-			writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
-			writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
-			writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
-			const notes = [`convo=${conversation.composite.toFixed(1)}`];
-			if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
-			if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
-			if (allJudgesFailed) notes.push("all-judges-failed");
-			const judgeRuns = [
-				conversationRun,
-				...codeReviewRuns,
-				...contentReviewRuns
-			];
-			const judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0);
-			if (judgeRuns.some((run) => run.cost.kind === "uncaptured")) notes.push("judge-cost-incomplete");
-			return {
-				output: {
-					turns: sim.transcript.length,
-					toolCalls: sim.toolCalls,
-					artifactCount: sim.artifacts.length
-				},
-				verdict: {
-					valid: composite >= 5,
-					score: composite,
-					notes: notes.join(" ")
-				},
-				costUsd: sim.costUsd + judgeCostUsd,
-				durationMs: sim.durationMs
-			};
+					costUsd: sim.costUsd + judgeCostUsd,
+					costProvenance: judgeCostComplete && shotCostComplete ? {
+						kind: "estimated",
+						usd: sim.costUsd + judgeCostUsd
+					} : {
+						kind: "uncaptured",
+						usd: null
+					},
+					durationMs: sim.durationMs
+				};
+			} catch (err) {
+				if (shotCostUsd === void 0) throw err;
+				throw withCellSpend(err, {
+					costUsd: shotCostUsd + judgeCostUsd,
+					durationMs: Date.now() - cellStartedAt,
+					kind: shotCostComplete && (phase === "validate" || phase === "scoring" && judgeCostComplete) ? "estimated" : "uncaptured"
+				});
+			}
 		}
 	});
 	const summary = {
@@ -737,6 +875,8 @@ async function runMultishotMatrix(opts) {
 		passRate: matrix.summary.overallPassRate,
 		meanScore: matrix.summary.overallMeanScore,
 		totalCostUsd: matrix.summary.totalCostUsd,
+		costUncapturedCells: matrix.summary.costUncapturedCells,
+		ceilingChargedUsd: matrix.summary.ceilingChargedUsd,
 		durationMs: matrix.summary.durationMs,
 		runsExecuted: matrix.summary.runsExecuted,
 		cellsSkipped: matrix.summary.cellsSkipped,
@@ -744,11 +884,14 @@ async function runMultishotMatrix(opts) {
 		byPersona: matrix.byAxis.persona
 	};
 	writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
+	const uncaptured = matrix.summary.costUncapturedCells;
+	const costLabel = uncaptured > 0 ? "Cost (at least)" : "Cost";
 	const md = [
 		`# Multishot matrix`,
 		``,
-		`**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
+		`**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
 		``,
+		...uncaptured > 0 ? [`> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`, ``] : [],
 		`## By profile`,
 		``,
 		"| profile | pass | mean | cost |",
@@ -765,6 +908,14 @@ async function runMultishotMatrix(opts) {
 	writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
 	return { matrix };
 }
+/** The shot's own spend, when the value it resolved with reports a usable
+*  amount. `undefined` when it does not — a malformed shot result is exactly
+*  the case where the number cannot be trusted, and billing a wrong figure is
+*  worse than recording the cell as uncaptured. */
+function shotCostSubtotal(sim) {
+	const value = sim?.costUsd;
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
+}
 function withJudgeMaxTokens(judge, maxTokens) {
 	if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
 	return {
@@ -773,6 +924,6 @@ function withJudgeMaxTokens(judge, maxTokens) {
 	};
 }
 //#endregion
-export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
+export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, MultishotShotResultError, assertMultishotShotResult, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
 
 //# sourceMappingURL=index.js.map