@tangle-network/agent-eval 0.221.3 → 0.221.6

package/dist/multishot/index.js

@@ -138,140 +138,6 @@ function defaultDelegationTools(config = {}) {
 	};
 }
 //#endregion
-//#region src/multishot/judges.ts
-const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
-async function runJudge(judge, input) {
-	const model = judge.model ?? "openai/gpt-4o-mini";
-	const prompt = judge.buildPrompt(input);
-	let raw = "";
-	let llmCall;
-	let cost;
-	const startedAt = Date.now();
-	try {
-		const response = await judge.transport({
-			model,
-			temperature: 0,
-			maxTokens: judge.maxTokens ?? 1500,
-			messages: [{
-				role: "system",
-				content: judge.systemPrompt
-			}, {
-				role: "user",
-				content: prompt
-			}]
-		});
-		const call = judgeCallMetadata(response, model, Date.now() - startedAt);
-		llmCall = call.llmCall;
-		cost = call.cost;
-		raw = (response.message.content ?? "").trim();
-	} catch (err) {
-		return {
-			score: {
-				dimensions: {},
-				composite: 0,
-				failed: true,
-				notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,
-				llmCall: unknownJudgeCall(model, Date.now() - startedAt)
-			},
-			cost: {
-				kind: "uncaptured",
-				usd: null
-			}
-		};
-	}
-	let parsed = null;
-	try {
-		const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
-		parsed = JSON.parse(cleaned);
-	} catch {
-		return {
-			score: {
-				dimensions: {},
-				composite: 0,
-				failed: true,
-				notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,
-				llmCall
-			},
-			cost
-		};
-	}
-	const dimensions = {};
-	let sum = 0;
-	for (const dim of judge.dimensions) {
-		const v = Number(parsed[dim.key] ?? 0);
-		const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
-		dimensions[dim.key] = clamped;
-		sum += clamped;
-	}
-	return {
-		score: {
-			dimensions,
-			composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
-			notes: typeof parsed.notes === "string" ? parsed.notes : "",
-			llmCall
-		},
-		cost
-	};
-}
-function judgeCallMetadata(response, requestedModel, durationMs) {
-	const usage = canonicalUsage(response.usage);
-	const model = response.model ?? requestedModel;
-	return {
-		llmCall: {
-			usage,
-			costUsd: response.costUsd ?? null,
-			model,
-			durationMs
-		},
-		cost: response.costUsd !== void 0 ? {
-			kind: "observed",
-			usd: response.costUsd
-		} : usage.captured === false || !isModelPriced(model) ? {
-			kind: "uncaptured",
-			usd: null
-		} : {
-			kind: "estimated",
-			usd: estimateCost(usage.promptTokens, usage.completionTokens, model)
-		}
-	};
-}
-function canonicalUsage(usage) {
-	const promptTokens = tokenCount(usage?.prompt_tokens);
-	const completionTokens = tokenCount(usage?.completion_tokens);
-	const captured = promptTokens !== void 0 && completionTokens !== void 0;
-	return {
-		promptTokens: promptTokens ?? 0,
-		completionTokens: completionTokens ?? 0,
-		totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),
-		captured
-	};
-}
-function unknownJudgeCall(model, durationMs) {
-	return {
-		usage: {
-			promptTokens: 0,
-			completionTokens: 0,
-			totalTokens: 0,
-			captured: false
-		},
-		costUsd: null,
-		model,
-		durationMs
-	};
-}
-function tokenCount(value) {
-	return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
-}
-/** Convenience: stringified dimension list for inclusion in a judge prompt.
-*  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
-function renderDimensions(dims) {
-	return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
-}
-/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
-function renderJsonFooter(dims) {
-	return `Respond with ONLY this JSON (no markdown, no preamble):\n{${dims.map((d) => `"${d.key}":N`).join(",")},"notes":"1-2 sentence critique"}`;
-}
-//#endregion
 //#region src/multishot/shape-defaults.ts
 /** Persona payload rendered as stable `- key: value` lines. `id` is identity,
 *  not voice, and structured values are serialized so nothing is dropped. */
@@ -321,31 +187,37 @@ function defaultShapeFromProfile(profile, shape) {
 	};
 }
 //#endregion
-//#region src/multishot/multishot.ts
-/** The package's reference conversation engine, and the referent of three
-*  things that stop working without it.
-*
-*  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names
-*     no `runShot`, so the matrix always has an engine to run.
-*  2. `golden.test.ts` proves the frozen records against it on every run, and
-*     perturbs it to prove the records detect a change. Records checked only
-*     against a replay of themselves prove nothing.
-*  3. The repo's recording command names it as `--engine`. The script defaults
-*     no engine, so a version records whichever one the caller names — and a
-*     record set minted from a consumer's engine would let a product's wiring
-*     define this package's frozen contract.
+//#region src/multishot/conversation.ts
+/**
+* Run one multishot conversation whose leg order `drive` decides.
 *
-*  An alternative engine reaches the matrix through `runShot` and reaches the
-*  records through `assertMultishotGoldenScenario`. Neither seam removes the
-*  three roles above. */
-async function runMultishot(opts) {
+* Returns the conversation's `MultishotResult`. When any leg or `drive` throws,
+* the error is rethrown carrying the spend so far (`withCellSpend`), so a
+* matrix bills the failed cell for the turns it already paid for.
+*/
+async function runMultishotConversation(opts, drive) {
 	const meter = {
 		costUsd: 0,
 		startedAt: Date.now(),
 		uncaptured: false
 	};
 	try {
-		return await runShotTurns(opts, meter);
+		const state = openConversation(opts, meter);
+		await drive(state.conversation);
+		return {
+			transcript: state.transcript,
+			artifacts: state.artifacts,
+			toolCalls: state.toolCalls(),
+			durationMs: Date.now() - meter.startedAt,
+			costUsd: meter.costUsd,
+			costProvenance: meter.uncaptured ? {
+				kind: "uncaptured",
+				usd: null
+			} : {
+				kind: "estimated",
+				usd: meter.costUsd
+			}
+		};
 	} catch (err) {
 		throw withCellSpend(err, {
 			costUsd: meter.costUsd,
@@ -354,11 +226,12 @@ async function runMultishot(opts) {
 		});
 	}
 }
-async function runShotTurns(opts, meter) {
+function openConversation(opts, meter) {
 	const maxTurns = opts.maxTurns ?? 10;
 	const maxToolDispatches = opts.maxToolDispatches ?? 4;
 	const agentModel = opts.agentModel ?? "openai/gpt-5.4";
-	const driverModels = [opts.driverModel ?? "openai/gpt-4o-mini", ...opts.driverFallbackModels ?? []];
+	const driverModel = opts.driverModel ?? "openai/gpt-4o-mini";
+	const driverModels = [driverModel, ...opts.driverFallbackModels ?? []];
 	const agentMaxTokens = opts.agentMaxTokens ?? 2500;
 	const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;
 	const driverMaxTokens = opts.driverMaxTokens ?? 600;
@@ -389,8 +262,13 @@ async function runShotTurns(opts, meter) {
 		role: "user",
 		content: opener
 	}];
-	for (let turn = 0; turn < maxTurns; turn++) {
+	const driverSystemPrompt = shape.buildDriverSystemPrompt(opts.persona);
+	async function agentTurn(turn) {
 		if (opts.signal?.aborted) throw new Error("multishot aborted");
+		const tokens = {
+			input: 0,
+			output: 0
+		};
 		let dispatchesThisTurn = 0;
 		while (true) {
 			const { message: agentMsg, usage: agentUsage, costUsd: agentCostUsd } = await agentTransport({
@@ -403,6 +281,8 @@ async function runShotTurns(opts, meter) {
 			});
 			meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage);
 			if (agentCostUsd === void 0 && agentUsage === void 0) meter.uncaptured = true;
+			tokens.input += agentUsage?.prompt_tokens ?? 0;
+			tokens.output += agentUsage?.completion_tokens ?? 0;
 			const agentText = (agentMsg.content ?? "").trim();
 			const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
 				id: tc.id,
@@ -468,76 +348,77 @@ async function runShotTurns(opts, meter) {
 				});
 			}
 		}
-		if (turn < maxTurns - 1) {
-			const driver = await driverTurn({
-				transport: driverTransport,
-				persona: opts.persona,
-				shape,
-				transcript,
-				turn,
-				models: driverModels,
-				maxTokens: driverMaxTokens,
-				signal: opts.signal,
-				meter
-			});
-			agentMessages.push({
+		return { tokens };
+	}
+	async function driverTurn(turn) {
+		const driverMessages = [{
+			role: "system",
+			content: driverSystemPrompt
+		}];
+		for (const msg of transcript) {
+			if (msg.role === "tool") continue;
+			const content = driverVisibleContent(msg);
+			if (!content) continue;
+			if (msg.role === "assistant") driverMessages.push({
 				role: "user",
-				content: driver.content
+				content
 			});
-			transcript.push({
-				role: "user",
-				content: driver.content
+			else if (msg.role === "user") driverMessages.push({
+				role: "assistant",
+				content
 			});
 		}
+		const tokens = {
+			input: 0,
+			output: 0
+		};
+		for (const model of driverModels) for (let attempt = 0; attempt < 2; attempt++) {
+			const { message, usage, costUsd } = await driverTransport({
+				model,
+				messages: driverMessages,
+				temperature: .9,
+				maxTokens: driverMaxTokens,
+				signal: opts.signal
+			});
+			meter.costUsd += costUsd ?? estimateMultishotCost(model, usage);
+			if (costUsd === void 0 && usage === void 0) meter.uncaptured = true;
+			tokens.input += usage?.prompt_tokens ?? 0;
+			tokens.output += usage?.completion_tokens ?? 0;
+			const content = (message.content ?? "").trim();
+			if (content.length > 0) {
+				agentMessages.push({
+					role: "user",
+					content
+				});
+				transcript.push({
+					role: "user",
+					content
+				});
+				return {
+					content,
+					tokens
+				};
+			}
+		}
+		throw new MultishotDriverEmptyError(turn);
 	}
 	return {
+		conversation: {
+			maxTurns,
+			agentModel,
+			driverModel,
+			opener,
+			driverSystemPrompt,
+			agentTurn,
+			driverTurn
+		},
 		transcript,
 		artifacts,
-		toolCalls,
-		durationMs: Date.now() - meter.startedAt,
-		costUsd: meter.costUsd,
-		costProvenance: meter.uncaptured ? {
-			kind: "uncaptured",
-			usd: null
-		} : {
-			kind: "estimated",
-			usd: meter.costUsd
-		}
+		toolCalls: () => toolCalls
 	};
 }
-async function driverTurn(opts) {
-	const driverMessages = [{
-		role: "system",
-		content: opts.shape.buildDriverSystemPrompt(opts.persona)
-	}];
-	for (const msg of opts.transcript) {
-		if (msg.role === "tool") continue;
-		const content = driverVisibleContent(msg);
-		if (!content) continue;
-		if (msg.role === "assistant") driverMessages.push({
-			role: "user",
-			content
-		});
-		else if (msg.role === "user") driverMessages.push({
-			role: "assistant",
-			content
-		});
-	}
-	for (const model of opts.models) for (let attempt = 0; attempt < 2; attempt++) {
-		const { message, usage, costUsd } = await opts.transport({
-			model,
-			messages: driverMessages,
-			temperature: .9,
-			maxTokens: opts.maxTokens,
-			signal: opts.signal
-		});
-		opts.meter.costUsd += costUsd ?? estimateMultishotCost(model, usage);
-		if (costUsd === void 0 && usage === void 0) opts.meter.uncaptured = true;
-		const content = (message.content ?? "").trim();
-		if (content.length > 0) return { content };
-	}
-	throw new MultishotDriverEmptyError(opts.turn);
-}
+/** Silent tool-calling agent turns surface to the driver as
+*  "Agent called tools: …" so the simulated user can react to them. */
 function driverVisibleContent(msg) {
 	const text = msg.content.trim();
 	if (text.length > 0) return text;
@@ -547,6 +428,166 @@ function driverVisibleContent(msg) {
 	return `Agent called ${toolNames.length === 1 ? "tool" : "tools"}: ${toolNames.join(", ")}.`;
 }
 //#endregion
+//#region src/multishot/judges.ts
+const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+async function runJudge(judge, input) {
+	const model = judge.model ?? "openai/gpt-4o-mini";
+	const prompt = judge.buildPrompt(input);
+	let raw = "";
+	let llmCall;
+	let cost;
+	const startedAt = Date.now();
+	try {
+		const response = await judge.transport({
+			model,
+			temperature: 0,
+			maxTokens: judge.maxTokens ?? 1500,
+			messages: [{
+				role: "system",
+				content: judge.systemPrompt
+			}, {
+				role: "user",
+				content: prompt
+			}]
+		});
+		const call = judgeCallMetadata(response, model, Date.now() - startedAt);
+		llmCall = call.llmCall;
+		cost = call.cost;
+		raw = (response.message.content ?? "").trim();
+	} catch (err) {
+		return {
+			score: {
+				dimensions: {},
+				composite: 0,
+				failed: true,
+				notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,
+				llmCall: unknownJudgeCall(model, Date.now() - startedAt)
+			},
+			cost: {
+				kind: "uncaptured",
+				usd: null
+			}
+		};
+	}
+	let parsed = null;
+	try {
+		const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
+		parsed = JSON.parse(cleaned);
+	} catch {
+		return {
+			score: {
+				dimensions: {},
+				composite: 0,
+				failed: true,
+				notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,
+				llmCall
+			},
+			cost
+		};
+	}
+	const dimensions = {};
+	let sum = 0;
+	for (const dim of judge.dimensions) {
+		const v = Number(parsed[dim.key] ?? 0);
+		const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
+		dimensions[dim.key] = clamped;
+		sum += clamped;
+	}
+	return {
+		score: {
+			dimensions,
+			composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
+			notes: typeof parsed.notes === "string" ? parsed.notes : "",
+			llmCall
+		},
+		cost
+	};
+}
+function judgeCallMetadata(response, requestedModel, durationMs) {
+	const usage = canonicalUsage(response.usage);
+	const model = response.model ?? requestedModel;
+	return {
+		llmCall: {
+			usage,
+			costUsd: response.costUsd ?? null,
+			model,
+			durationMs
+		},
+		cost: response.costUsd !== void 0 ? {
+			kind: "observed",
+			usd: response.costUsd
+		} : usage.captured === false || !isModelPriced(model) ? {
+			kind: "uncaptured",
+			usd: null
+		} : {
+			kind: "estimated",
+			usd: estimateCost(usage.promptTokens, usage.completionTokens, model)
+		}
+	};
+}
+function canonicalUsage(usage) {
+	const promptTokens = tokenCount(usage?.prompt_tokens);
+	const completionTokens = tokenCount(usage?.completion_tokens);
+	const captured = promptTokens !== void 0 && completionTokens !== void 0;
+	return {
+		promptTokens: promptTokens ?? 0,
+		completionTokens: completionTokens ?? 0,
+		totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),
+		captured
+	};
+}
+function unknownJudgeCall(model, durationMs) {
+	return {
+		usage: {
+			promptTokens: 0,
+			completionTokens: 0,
+			totalTokens: 0,
+			captured: false
+		},
+		costUsd: null,
+		model,
+		durationMs
+	};
+}
+function tokenCount(value) {
+	return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
+}
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+*  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+function renderDimensions(dims) {
+	return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
+}
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+function renderJsonFooter(dims) {
+	return `Respond with ONLY this JSON (no markdown, no preamble):\n{${dims.map((d) => `"${d.key}":N`).join(",")},"notes":"1-2 sentence critique"}`;
+}
+//#endregion
+//#region src/multishot/multishot.ts
+/** The package's reference conversation engine, and the referent of three
+*  things that stop working without it.
+*
+*  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names
+*     no `runShot`, so the matrix always has an engine to run.
+*  2. `golden.test.ts` proves the frozen records against it on every run, and
+*     perturbs it to prove the records detect a change. Records checked only
+*     against a replay of themselves prove nothing.
+*  3. The repo's recording command names it as `--engine`. The script defaults
+*     no engine, so a version records whichever one the caller names — and a
+*     record set minted from a consumer's engine would let a product's wiring
+*     define this package's frozen contract.
+*
+*  An alternative engine reaches the matrix through `runShot` and reaches the
+*  records through `assertMultishotGoldenScenario`. Neither seam removes the
+*  three roles above. */
+async function runMultishot(opts) {
+	return runMultishotConversation(opts, async (conversation) => {
+		for (let turn = 0; turn < conversation.maxTurns; turn++) {
+			await conversation.agentTurn(turn);
+			if (turn < conversation.maxTurns - 1) await conversation.driverTurn(turn);
+		}
+	});
+}
+//#endregion
 //#region src/multishot/matrix.ts
 /** Mean composite over non-failed scores. `0` when the list is empty (a
 *  configured judge with nothing to score contributes 0, matching the cell
@@ -784,6 +825,6 @@ function withJudgeMaxTokens(judge, maxTokens) {
 	};
 }
 //#endregion
-export { DEFAULT_CODER_MODEL, DEFAULT_JUDGE_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, assertMultishotShotResult, computeCellComposite, defaultDelegationTools, defaultShapeFromProfile, estimateMultishotCost, renderDimensions, renderJsonFooter, runJudge, runMultishot, runMultishotMatrix };
+export { DEFAULT_CODER_MODEL, DEFAULT_JUDGE_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, assertMultishotShotResult, computeCellComposite, defaultDelegationTools, defaultShapeFromProfile, estimateMultishotCost, renderDimensions, renderJsonFooter, runJudge, runMultishot, runMultishotConversation, runMultishotMatrix };
 
 //# sourceMappingURL=index.js.map