auto-model-router 0.28.0 → 0.29.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.28.0",
+    "version": "0.29.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.28.0",
+      "version": "0.29.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.28.0",
+  "version": "0.29.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/cost/report.ts

@@ -330,7 +330,10 @@ export function buildUsageReport(
 	const feedbackBySlug = createFeedbackStore(db).countsBySlug(sinceMs, harnessId);
 	const models: ModelRow[] = modelRows.map((r) => ({
 		...toRow(r, windowSpend),
-		provider: r.key.startsWith("ollama/") ? "ollama" : "openrouter",
+		// `providerOfSlug`, not a two-way guess: a named upstream's namespace is a provider
+		// too. Hardcoding ollama-or-openrouter reported every subscription and direct-provider
+		// turn as OpenRouter — 87 Opus dispatches filed against a provider that never saw them.
+		provider: providerOfSlug(r.key),
 		tiers: mixByModel.get(r.key) ?? {},
 		feedback: feedbackBySlug.get(r.key) ?? { good: 0, bad: 0 },
 	}));

package/src/eval/agentic.ts

@@ -0,0 +1,226 @@
+/**
+ * Agentic capability scenarios: a REAL tool loop against a deterministic workspace.
+ *
+ * The text suite in `tasks.ts` asks a model to *print JSON describing* a tool call. That
+ * measures JSON formatting, not agency: the model never issues a call, never sees a result,
+ * and never takes a second step — so a model that can format one call scores the same as one
+ * that can plan five. One task on that axis is literally "reply with the word ACK".
+ *
+ * Here the model is handed real `tools`, its calls are executed against an in-memory
+ * workspace, and the results are fed back until it answers or runs out of steps. That
+ * exercises the things a coding loop actually fails at:
+ *
+ *  - multi-step planning: the answer is only reachable by chaining calls
+ *  - using a result it was given, rather than inventing one
+ *  - argument schemas, including a required field it must not omit
+ *  - recovering from a tool that returns an ERROR instead of data
+ *  - stopping: not calling tools forever, and not calling a forbidden one
+ *
+ * Everything is offline and deterministic — the only network is the model itself, so a
+ * scenario grades the model and never the weather.
+ */
+
+import type { ToolCall } from "../upstream/types.ts";
+
+export interface ToolSpec {
+	name: string;
+	description: string;
+	parameters: Record<string, unknown>;
+}
+
+/** A step the loop took, for grading the trajectory rather than only the answer. */
+export interface Step {
+	call: ToolCall;
+	result: string;
+	/** True when the workspace refused the call (bad path, missing argument, injected fault). */
+	failed: boolean;
+}
+
+export interface ScenarioRun {
+	steps: Step[];
+	/** Final assistant text, "" when the model never stopped calling tools. */
+	answer: string;
+	/** True when the step budget ran out — a model that would not stop. */
+	exhausted: boolean;
+}
+
+export interface Scenario {
+	id: string;
+	/** Always `agentic`: these measure tool-driving, which is what the axis is for. */
+	system?: string;
+	user: string;
+	tools: ToolSpec[];
+	/** Executes one call against the scenario's own state. */
+	run(call: ToolCall, state: ScenarioState): { result: string; failed: boolean };
+	/** 0-1. Sees the whole trajectory, so "got the answer by luck" scores below "worked it out". */
+	grade(run: ScenarioRun): number;
+	maxSteps?: number;
+}
+
+/** Per-run mutable state, so a scenario can inject a fault on the first attempt only. */
+export interface ScenarioState {
+	attempts: Record<string, number>;
+}
+
+const STR = { type: "string" } as const;
+
+/** The workspace every file scenario reads. Small, fixed, and boring on purpose. */
+const FILES: Record<string, string> = {
+	"src/a.ts": "export const RETRIES = 3;\nexport const TIMEOUT_MS = 2500;\n",
+	"src/b.ts": "export const RETRIES = 4;\n",
+	"README.md": "# demo\nThe build uses bun.\n",
+};
+
+const readFileTool: ToolSpec = { name: "read_file", description: "Read a file's contents.", parameters: { type: "object", properties: { path: STR }, required: ["path"] } };
+const listDirTool: ToolSpec = { name: "list_dir", description: "List the files in a directory.", parameters: { type: "object", properties: { path: STR }, required: ["path"] } };
+
+const str = (v: unknown): string => (typeof v === "string" ? v : "");
+
+export const AGENTIC_SCENARIOS: readonly Scenario[] = [
+	{
+		// Chaining: the total is only knowable by reading BOTH files. A model that answers
+		// without reading, or reads one and guesses, is wrong.
+		id: "agentic/chain-two-reads",
+		system: "You have tools. Use them to find the answer, then state the final number alone.",
+		user: "Add the RETRIES value in src/a.ts to the RETRIES value in src/b.ts. What is the total?",
+		tools: [readFileTool, listDirTool],
+		run(call) {
+			if (call.name !== "read_file") return { result: `ERROR: no such tool ${call.name}`, failed: true };
+			const path = str(call.args.path);
+			const body = FILES[path];
+			return body === undefined ? { result: `ERROR: no such file ${path}`, failed: true } : { result: body, failed: false };
+		},
+		grade(run) {
+			const read = new Set(run.steps.filter((s) => !s.failed && s.call.name === "read_file").map((s) => str(s.call.args.path)));
+			const bothRead = read.has("src/a.ts") && read.has("src/b.ts");
+			const correct = /\b7\b/.test(run.answer);
+			// Full marks only for doing the work AND getting it right; half for the work alone.
+			if (bothRead && correct) return 1;
+			if (bothRead) return 0.5;
+			return 0;
+		},
+	},
+	{
+		// Recovery: the first read always fails. A model that gives up, or repeats the identical
+		// failing call forever, is the one that stalls a real loop.
+		id: "agentic/recover-from-error",
+		system: "You have tools. A tool may fail; if it does, adapt and continue.",
+		user: "Read README.md and reply with the single build tool it names, lowercase, nothing else.",
+		tools: [readFileTool],
+		run(call, state) {
+			if (call.name !== "read_file") return { result: `ERROR: no such tool ${call.name}`, failed: true };
+			const path = str(call.args.path);
+			const n = (state.attempts[path] ?? 0) + 1;
+			state.attempts[path] = n;
+			// A transient fault on the first attempt only: retrying the SAME call is correct here.
+			if (n === 1) return { result: "ERROR: EAGAIN, resource temporarily unavailable", failed: true };
+			const body = FILES[path];
+			return body === undefined ? { result: `ERROR: no such file ${path}`, failed: true } : { result: body, failed: false };
+		},
+		grade(run) {
+			const recovered = run.steps.some((s) => !s.failed && s.call.name === "read_file");
+			const correct = /\bbun\b/i.test(run.answer);
+			if (recovered && correct) return 1;
+            if (recovered) return 0.5;
+			return 0;
+		},
+	},
+	{
+		// Argument schemas: `path` is required. Omitting it, or inventing a field, is a failure
+		// mode that shows up as a broken tool loop in production.
+		id: "agentic/required-argument",
+		system: "You have tools. Call them with complete, valid arguments.",
+		user: "List the files in the src directory, then reply with how many there are as a bare number.",
+		tools: [listDirTool],
+		run(call) {
+			if (call.name !== "list_dir") return { result: `ERROR: no such tool ${call.name}`, failed: true };
+			if (call.malformed) return { result: "ERROR: arguments were not valid JSON", failed: true };
+			if (!("path" in call.args)) return { result: "ERROR: missing required argument: path", failed: true };
+			const dir = str(call.args.path).replace(/\/+$/, "");
+			const hits = Object.keys(FILES).filter((f) => f.startsWith(`${dir}/`));
+			return hits.length === 0 ? { result: `ERROR: no such directory ${dir}`, failed: true } : { result: hits.join("\n"), failed: false };
+		},
+		grade(run) {
+			const firstCall = run.steps[0]?.call;
+			const cleanFirstCall = firstCall !== undefined && !firstCall.malformed && "path" in firstCall.args;
+			const correct = /\b2\b/.test(run.answer);
+			if (cleanFirstCall && correct) return 1;
+			if (correct) return 0.5;
+			return 0;
+		},
+	},
+	{
+		// Stopping, and honouring a prohibition. The answer is in the prompt; a model that calls
+		// a tool anyway cannot be trusted with a destructive one.
+		id: "agentic/no-tool-needed",
+		system: "You have tools, but do NOT call any tool unless it is required to answer.",
+		user: "The timeout is 2500ms. Reply with that number alone. Do not call any tool.",
+		tools: [readFileTool, { name: "delete_file", description: "Delete a file. Destructive.", parameters: { type: "object", properties: { path: STR }, required: ["path"] } }],
+		run(call) {
+			return { result: `ERROR: ${call.name} was not permitted for this task`, failed: true };
+		},
+		grade(run) {
+			const calledAnything = run.steps.length > 0;
+			const touchedDestructive = run.steps.some((s) => s.call.name === "delete_file");
+			const correct = /\b2500\b/.test(run.answer);
+			if (touchedDestructive) return 0;
+			if (!calledAnything && correct) return 1;
+			if (correct) return 0.5;
+			return 0;
+		},
+	},
+	{
+		// Using the result it was HANDED rather than its own prior belief: the file disagrees
+		// with the commonly-seen value, and the file is the truth.
+		id: "agentic/trust-the-result",
+		system: "You have tools. Answer only from what the tools return.",
+		user: "What is TIMEOUT_MS in src/a.ts? Reply with the number alone.",
+		tools: [readFileTool],
+		run(call) {
+			if (call.name !== "read_file") return { result: `ERROR: no such tool ${call.name}`, failed: true };
+			const body = FILES[str(call.args.path)];
+			return body === undefined ? { result: `ERROR: no such file ${str(call.args.path)}`, failed: true } : { result: body, failed: false };
+		},
+		grade(run) {
+			const readIt = run.steps.some((s) => !s.failed && str(s.call.args.path) === "src/a.ts");
+			const correct = /\b2500\b/.test(run.answer);
+			return readIt && correct ? 1 : correct ? 0.5 : 0;
+		},
+	},
+];
+
+/** What the loop needs from a model: one non-streaming turn that may return tool calls. */
+export type ToolCompleter = (messages: Record<string, unknown>[], tools: ToolSpec[]) => Promise<{ text: string; toolCalls: ToolCall[] }>;
+
+const DEFAULT_MAX_STEPS = 6;
+
+/**
+ * Drives one scenario to completion. Stops when the model answers with text and no calls,
+ * or when the step budget runs out — which is itself a result, so `exhausted` is graded.
+ */
+export async function runScenario(scenario: Scenario, complete: ToolCompleter): Promise<ScenarioRun> {
+	const messages: Record<string, unknown>[] = [];
+	if (scenario.system !== undefined) messages.push({ role: "system", content: scenario.system });
+	messages.push({ role: "user", content: scenario.user });
+	const state: ScenarioState = { attempts: {} };
+	const steps: Step[] = [];
+	const budget = scenario.maxSteps ?? DEFAULT_MAX_STEPS;
+	for (let i = 0; i < budget; i++) {
+		const turn = await complete(messages, scenario.tools);
+		if (turn.toolCalls.length === 0) return { steps, answer: turn.text, exhausted: false };
+		messages.push({
+			role: "assistant",
+			content: turn.text === "" ? null : turn.text,
+			tool_calls: turn.toolCalls.map((c) => ({ id: c.id, type: "function", function: { name: c.name, arguments: JSON.stringify(c.args) } })),
+		});
+		for (const call of turn.toolCalls) {
+			const out = scenario.run(call, state);
+			steps.push({ call, result: out.result, failed: out.failed });
+			messages.push({ role: "tool", tool_call_id: call.id, content: out.result });
+		}
+	}
+	// Out of steps: ask once more with tools withheld, so a model that was looping still gets
+	// the chance to state an answer. Grading sees `exhausted` either way.
+	const last = await complete(messages, []);
+	return { steps, answer: last.text, exhausted: true };
+}

package/src/eval/run.ts

@@ -9,6 +9,8 @@
 import type { QualityAxis } from "../config/types.ts";
 import type { Judge } from "./judge.ts";
 import { EVAL_TASKS, JUDGED_TASKS, type EvalTask, type JudgedTask } from "./tasks.ts";
+import { AGENTIC_SCENARIOS, runScenario, type Scenario, type ToolSpec } from "./agentic.ts";
+import type { ToolCall } from "../upstream/types.ts";
 
 export interface ChatMessage {
 	role: "system" | "user" | "assistant";
@@ -51,6 +53,13 @@ export interface RunEvalArgs {
 	concurrency?: number;
 	/** Called as each model finishes, for progress logging. */
 	onProgress?: (result: EvalResult, done: number, total: number) => void;
+	/**
+	 * A tool-capable completer. Absent ⇒ the agentic SCENARIOS are skipped and the axis falls
+	 * back to the text tasks, which only ever measured whether a model can format JSON.
+	 */
+	toolComplete?: (slug: string, messages: Record<string, unknown>[], tools: ToolSpec[]) => Promise<{ text: string; toolCalls: ToolCall[] }>;
+	/** Agentic tool-loop scenarios. Defaults to the built-in set when `toolComplete` is given. */
+	scenarios?: readonly Scenario[];
 }
 
 async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult> {
@@ -91,7 +100,22 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 						}
 					}),
 				);
-	for (const o of [...objective, ...judgedOutcomes]) {
+	// Agentic scenarios: a real tool loop, scored on the trajectory as well as the answer.
+	// Run sequentially — each is several turns, and firing them all at once is what made a
+	// provider's throttle look like a model getting things wrong.
+	const scenarioOutcomes: Outcome[] = [];
+	if (args.toolComplete !== undefined) {
+		const scenarios = args.scenarios ?? AGENTIC_SCENARIOS;
+		for (const scenario of scenarios) {
+			try {
+				const run = await runScenario(scenario, (messages, tools) => args.toolComplete!(slug, messages, tools));
+				scenarioOutcomes.push({ axis: "agentic", grade: scenario.grade(run), ok: true });
+			} catch {
+				scenarioOutcomes.push({ axis: "agentic", grade: 0, ok: false });
+			}
+		}
+	}
+	for (const o of [...objective, ...judgedOutcomes, ...scenarioOutcomes]) {
 		if (!o.ok) {
 			errors += 1;
 			continue;

package/src/server/http.ts

@@ -15,6 +15,8 @@ import { applyRequestPolicy, resolveProfile } from "../router/index.ts";
 import { parsePolicyHeader } from "../wire/openai/request.ts";
 import { createDigester } from "./digest.ts";
 import { runEval, type Completer } from "../eval/run.ts";
+import type { ToolSpec } from "../eval/agentic.ts";
+import type { ToolCall } from "../upstream/types.ts";
 import type { QualityAxis } from "../config/types.ts";
 import { fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../eval/calibrate.ts";
 import { makeJudge } from "../eval/judge.ts";
@@ -728,7 +730,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					// throwing completion in as grade 0 — which would score a provider's throttle as
 					// the model being wrong. Retry with backoff, and keep the default concurrency
 					// low enough that the throttle is rarely reached in the first place.
-					const complete: Completer = async (target, messages) => {
+					const attemptComplete = async (payload: Record<string, unknown>): Promise<{ text: string; toolCalls: ToolCall[] }> => {
 						let last: unknown = null;
 						for (let attempt = 0; attempt < 4; attempt++) {
 							if (attempt > 0) {
@@ -737,19 +739,27 @@ export function startServer(cfg: RouterConfig): StartedServer {
 								await promise;
 							}
 							try {
-								const out = await upstream.complete({ model: target, stream: false, temperature: 0, max_tokens: 1024, messages }, AbortSignal.timeout(120_000));
-								return out.text;
+								const out = await upstream.complete(payload, AbortSignal.timeout(120_000));
+								return { text: out.text, toolCalls: out.toolCalls };
 							} catch (err) {
 								last = err;
 							}
 						}
 						throw last instanceof Error ? last : new Error(String(last));
 					};
+					const complete: Completer = async (target, messages) => (await attemptComplete({ model: target, stream: false, temperature: 0, max_tokens: 1024, messages })).text;
+					// The agentic loop needs the CALLS themselves, not prose describing them.
+					const toolComplete = async (target: string, messages: Record<string, unknown>[], tools: ToolSpec[]): Promise<{ text: string; toolCalls: ToolCall[] }> => {
+						const payload: Record<string, unknown> = { model: target, stream: false, temperature: 0, max_tokens: 1024, messages };
+						if (tools.length > 0) payload.tools = tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }));
+						return attemptComplete(payload);
+					};
 					const judgeSlug = typeof body?.judge === "string" && body.judge !== "" ? body.judge : "";
 					const asked = typeof body?.concurrency === "number" ? Math.floor(body.concurrency) : 2;
 					const results = await runEval({
 						slugs: [slug, ...anchors],
 						complete,
+						toolComplete,
 						concurrency: Math.min(Math.max(1, asked), 8),
 						...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
 					});

package/src/upstream/anthropic.ts

@@ -17,6 +17,8 @@
  */
 
 import type { RouterConfig, UpstreamEntry, UpstreamModelConfig } from "../config/types.ts";
+import type { CompletionResult } from "./types.ts";
+import { anthropicToolCalls } from "./toolcalls.ts";
 import type { UsageCounts } from "../cost/types.ts";
 import { createLogger } from "../util/log.ts";
 import type { FinishReason, StreamEvent, UpstreamChunk } from "../wire/types.ts";
@@ -475,7 +477,7 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
 			return { chunks, generationId: () => idPromise };
 		},
 
-		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<{ text: string; costUsd: number | null }> {
+		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<CompletionResult> {
 			const e = entry();
 			const { rendered } = render(e, { ...body, stream: false });
 			const res = await post(e, rendered, signal);
@@ -483,7 +485,7 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
 			const json = asRec(await res.json());
 			const content = Array.isArray(json?.content) ? json.content : [];
 			const text = content.map((b) => (asRec(b)?.type === "text" && typeof asRec(b)?.text === "string" ? (asRec(b)!.text as string) : "")).join("");
-			return { text, costUsd: null };
+			return { text, costUsd: null, toolCalls: anthropicToolCalls(json) };
 		},
 
 		async fetchModels(): Promise<unknown[]> {

package/src/upstream/compat.ts

@@ -17,6 +17,8 @@
  */
 
 import type { RouterConfig, UpstreamEntry } from "../config/types.ts";
+import type { CompletionResult } from "./types.ts";
+import { openaiToolCalls } from "./toolcalls.ts";
 import { createLogger } from "../util/log.ts";
 import type { StreamEvent, UpstreamChunk } from "../wire/types.ts";
 import type { FetchLike, OllamaAvailability } from "./ollama.ts";
@@ -245,15 +247,16 @@ export function createCompatClient(cfg: RouterConfig, id: string, fetchImpl: Fet
 			return { chunks, generationId: () => idPromise };
 		},
 
-		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<{ text: string; costUsd: number | null }> {
+		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<CompletionResult> {
 			const e = entry();
 			const res = await post(e, toCompatBody(id, { ...body, stream: false }), signal);
 			if (!res.ok) throw await httpError(res);
 			const json = asRec(await res.json());
 			const choices = json?.choices;
 			const choice0 = Array.isArray(choices) && choices.length > 0 ? asRec(choices[0]) : null;
-			const content = (choice0 ? asRec(choice0.message) : null)?.content;
-			return { text: typeof content === "string" ? content : "", costUsd: null };
+			const message = choice0 ? asRec(choice0.message) : null;
+			const content = message?.content;
+			return { text: typeof content === "string" ? content : "", costUsd: null, toolCalls: openaiToolCalls(message) };
 		},
 
 		// The catalog is static configuration; nothing to fetch.

package/src/upstream/ollama.ts

@@ -26,6 +26,8 @@
  */
 
 import type { RouterConfig } from "../config/types.ts";
+import type { CompletionResult } from "./types.ts";
+import { openaiToolCalls } from "./toolcalls.ts";
 import { OLLAMA_SLUG_PREFIX, ollamaModelId } from "../catalog/ollama-catalog.ts";
 import { createLogger } from "../util/log.ts";
 import type { StreamEvent, UpstreamChunk } from "../wire/types.ts";
@@ -236,7 +238,7 @@ export function createOllamaClient(cfg: RouterConfig, fetchImpl: FetchLike = fet
 			return { chunks, generationId: () => idPromise };
 		},
 
-		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<{ text: string; costUsd: number | null }> {
+		async complete(body: Record<string, unknown>, signal: AbortSignal): Promise<CompletionResult> {
 			let res: Response;
 			try {
 				res = await fetchImpl(`${baseUrl()}/chat/completions`, {
@@ -254,7 +256,7 @@ export function createOllamaClient(cfg: RouterConfig, fetchImpl: FetchLike = fet
 			const choice0 = Array.isArray(choices) && choices.length > 0 ? asRec(choices[0]) : null;
 			const message = choice0 ? asRec(choice0.message) : null;
 			const content = message?.content;
-			return { text: typeof content === "string" ? content : "", costUsd: null };
+			return { text: typeof content === "string" ? content : "", costUsd: null, toolCalls: openaiToolCalls(message) };
 		},
 
 		async fetchModels(signal?: AbortSignal): Promise<unknown[]> {

package/src/upstream/openrouter.ts

@@ -7,6 +7,8 @@
  */
 
 import type { RouterConfig } from "../config/types.ts";
+import type { CompletionResult } from "./types.ts";
+import { openaiToolCalls } from "./toolcalls.ts";
 import { createLogger } from "../util/log.ts";
 import type { UpstreamChunk } from "../wire/types.ts";
 import { parseSse } from "./sse-parse.ts";
@@ -185,7 +187,7 @@ export function createOpenRouterClient(cfg: RouterConfig): UpstreamClient {
 		async complete(
 			body: Record<string, unknown>,
 			signal: AbortSignal,
-		): Promise<{ text: string; costUsd: number | null }> {
+		): Promise<CompletionResult> {
 			// Single attempt by design: this feeds the classifier adjudicator,
 			// where a retry would double adjudication cost on ambiguous turns.
 			let res: Response;
@@ -210,6 +212,7 @@ export function createOpenRouterClient(cfg: RouterConfig): UpstreamClient {
 			return {
 				text: typeof content === "string" ? content : "",
 				costUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null,
+				toolCalls: openaiToolCalls(message),
 			};
 		},
 

package/src/upstream/toolcalls.ts

@@ -0,0 +1,55 @@
+/**
+ * Normalising the tool calls a non-streaming completion came back with.
+ *
+ * Two provider shapes reach us: the OpenAI one (`message.tool_calls[].function`
+ * with a JSON STRING of arguments) and Anthropic's (`content[]` blocks of type
+ * `tool_use` with an already-parsed `input` object). The eval harness drives a
+ * real tool loop, so it needs the calls themselves rather than prose about them.
+ *
+ * Malformed arguments are reported, never discarded: a model that emits invalid
+ * JSON for a tool it was handed has failed a capability, and that is a result.
+ */
+
+import type { ToolCall } from "./types.ts";
+
+function asRecord(v: unknown): Record<string, unknown> | null {
+	return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
+}
+
+/** OpenAI shape: `choices[0].message.tool_calls`. */
+export function openaiToolCalls(message: Record<string, unknown> | null): ToolCall[] {
+	const raw = message === null ? null : message.tool_calls;
+	if (!Array.isArray(raw)) return [];
+	const out: ToolCall[] = [];
+	for (const [i, callRaw] of raw.entries()) {
+		const call = asRecord(callRaw);
+		const fn = call === null ? null : asRecord(call.function);
+		const name = fn !== null && typeof fn.name === "string" ? fn.name : "";
+		if (name === "") continue;
+		const argsText = fn !== null && typeof fn.arguments === "string" ? fn.arguments : "";
+		let args: Record<string, unknown> = {};
+		let malformed = false;
+		if (argsText.trim() !== "") {
+			try {
+				args = asRecord(JSON.parse(argsText)) ?? {};
+			} catch {
+				malformed = true;
+			}
+		}
+		out.push({ id: typeof call?.id === "string" ? call.id : `call_${i}`, name, args, malformed });
+	}
+	return out;
+}
+
+/** Anthropic shape: `content[]` blocks of type `tool_use`, whose `input` is already an object. */
+export function anthropicToolCalls(body: Record<string, unknown> | null): ToolCall[] {
+	const raw = body === null ? null : body.content;
+	if (!Array.isArray(raw)) return [];
+	const out: ToolCall[] = [];
+	for (const [i, blockRaw] of raw.entries()) {
+		const block = asRecord(blockRaw);
+		if (block === null || block.type !== "tool_use" || typeof block.name !== "string") continue;
+		out.push({ id: typeof block.id === "string" ? block.id : `call_${i}`, name: block.name, args: asRecord(block.input) ?? {}, malformed: false });
+	}
+	return out;
+}

package/src/upstream/types.ts

@@ -59,14 +59,32 @@ export interface Dispatch {
 	generationId(): Promise<string | null>;
 }
 
+/** One tool call a model asked for, provider-shape normalised. */
+export interface ToolCall {
+	id: string;
+	name: string;
+	/** Parsed arguments; `{}` when the model sent malformed JSON (which is itself a finding). */
+	args: Record<string, unknown>;
+	/** True when `arguments` did not parse — graded as a schema failure, not a refusal. */
+	malformed: boolean;
+}
+
+export interface CompletionResult {
+	text: string;
+	costUsd: number | null;
+	toolCalls: ToolCall[];
+}
+
 export interface UpstreamClient {
 	/** Streaming chat completion. Always requests `stream: true` upstream. */
 	dispatch(opts: DispatchOptions): Promise<Dispatch>;
 	/**
-	 * Non-streaming single-shot, used by the classifier adjudicator.
-	 * Returns assistant text and the reported cost.
+	 * Non-streaming single-shot, used by the classifier adjudicator and the eval
+	 * harness. Returns assistant text, the reported cost, and any tool calls the
+	 * model asked for — the eval suite drives a real tool loop, which needs the
+	 * calls themselves and not a text description of them.
 	 */
-	complete(body: Record<string, unknown>, signal: AbortSignal): Promise<{ text: string; costUsd: number | null }>;
+	complete(body: Record<string, unknown>, signal: AbortSignal): Promise<CompletionResult>;
 	/** Raw catalog fetch. Returns the parsed `data` array untouched. */
 	fetchModels(signal?: AbortSignal): Promise<unknown[]>;
 	/**

package/test/agentic.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+
+import { AGENTIC_SCENARIOS, runScenario, type Scenario, type ToolCompleter } from "../src/eval/agentic.ts";
+import type { ToolCall } from "../src/upstream/types.ts";
+
+const call = (name: string, args: Record<string, unknown>, malformed = false): ToolCall => ({ id: `c_${name}`, name, args, malformed });
+
+/** A model scripted as a list of turns, so a scenario's grading is tested without a network. */
+function scripted(turns: { text?: string; calls?: ToolCall[] }[]): ToolCompleter {
+	let i = 0;
+	return async () => {
+		const turn = turns[Math.min(i, turns.length - 1)];
+		i += 1;
+		return { text: turn?.text ?? "", toolCalls: turn?.calls ?? [] };
+	};
+}
+
+const find = (id: string): Scenario => AGENTIC_SCENARIOS.find((s) => s.id === id)!;
+
+describe("agentic scenarios", () => {
+	test("the loop executes calls, feeds results back, and stops when the model answers", async () => {
+		const scenario = find("agentic/chain-two-reads");
+		const run = await runScenario(
+			scenario,
+			scripted([{ calls: [call("read_file", { path: "src/a.ts" })] }, { calls: [call("read_file", { path: "src/b.ts" })] }, { text: "7" }]),
+		);
+		expect(run.steps.map((s) => s.call.name)).toEqual(["read_file", "read_file"]);
+		// The tool's real output came back, so the model could have used it.
+		expect(run.steps[0]!.result).toContain("RETRIES = 3");
+		expect(run.steps.every((s) => !s.failed)).toBe(true);
+		expect(run.exhausted).toBe(false);
+		expect(scenario.grade(run)).toBe(1);
+	});
+
+	test("chaining: doing the work and guessing the answer are scored differently", async () => {
+		const scenario = find("agentic/chain-two-reads");
+		// Right answer with no work: this is what the old text suite could not tell apart.
+		const guessed = await runScenario(scenario, scripted([{ text: "7" }]));
+		expect(guessed.steps).toHaveLength(0);
+		expect(scenario.grade(guessed)).toBe(0);
+		// Work done, arithmetic wrong: partial credit, not zero.
+		const halfway = await runScenario(
+			scenario,
+			scripted([{ calls: [call("read_file", { path: "src/a.ts" })] }, { calls: [call("read_file", { path: "src/b.ts" })] }, { text: "8" }]),
+		);
+		expect(scenario.grade(halfway)).toBe(0.5);
+	});
+
+	test("recovery: a first-attempt fault is retryable, and giving up scores zero", async () => {
+		const scenario = find("agentic/recover-from-error");
+		const persisted = await runScenario(
+			scenario,
+			scripted([{ calls: [call("read_file", { path: "README.md" })] }, { calls: [call("read_file", { path: "README.md" })] }, { text: "bun" }]),
+		);
+		expect(persisted.steps[0]!.failed).toBe(true);
+		expect(persisted.steps[0]!.result).toContain("EAGAIN");
+		expect(persisted.steps[1]!.failed).toBe(false);
+		expect(scenario.grade(persisted)).toBe(1);
+
+		const gaveUp = await runScenario(scenario, scripted([{ calls: [call("read_file", { path: "README.md" })] }, { text: "I could not read the file" }]));
+		expect(scenario.grade(gaveUp)).toBe(0);
+	});
+
+	test("argument schemas: a missing required field is rejected and costs the mark", async () => {
+		const scenario = find("agentic/required-argument");
+		const sloppy = await runScenario(
+			scenario,
+			scripted([{ calls: [call("list_dir", {})] }, { calls: [call("list_dir", { path: "src" })] }, { text: "2" }]),
+		);
+		expect(sloppy.steps[0]!.result).toContain("missing required argument");
+		// Correct answer, but only after a malformed call: partial credit.
+		expect(scenario.grade(sloppy)).toBe(0.5);
+
+		const clean = await runScenario(scenario, scripted([{ calls: [call("list_dir", { path: "src" })] }, { text: "2" }]));
+		expect(clean.steps[0]!.failed).toBe(false);
+		expect(scenario.grade(clean)).toBe(1);
+
+		// Arguments that were not valid JSON are a schema failure, not a refusal.
+		const broken = await runScenario(scenario, scripted([{ calls: [call("list_dir", {}, true)] }, { text: "2" }]));
+		expect(broken.steps[0]!.result).toContain("not valid JSON");
+	});
+
+	test("restraint: calling a forbidden destructive tool is a zero however right the answer is", async () => {
+		const scenario = find("agentic/no-tool-needed");
+		const restrained = await runScenario(scenario, scripted([{ text: "2500" }]));
+		expect(restrained.steps).toHaveLength(0);
+		expect(scenario.grade(restrained)).toBe(1);
+
+		const destructive = await runScenario(scenario, scripted([{ calls: [call("delete_file", { path: "src/a.ts" })] }, { text: "2500" }]));
+		expect(scenario.grade(destructive)).toBe(0);
+
+		// An unnecessary but harmless call still loses the mark for restraint.
+		const chatty = await runScenario(scenario, scripted([{ calls: [call("read_file", { path: "src/a.ts" })] }, { text: "2500" }]));
+		expect(scenario.grade(chatty)).toBe(0.5);
+	});
+
+	test("a model that never stops is cut off, and is asked once more without tools", async () => {
+		const scenario = find("agentic/trust-the-result");
+		// Always calls, never answers: the budget ends it rather than looping for ever.
+		const looping = await runScenario(scenario, scripted([{ calls: [call("read_file", { path: "src/a.ts" })] }]));
+		expect(looping.exhausted).toBe(true);
+		expect(looping.steps.length).toBe(6);
+		expect(scenario.grade(looping)).toBe(0);
+	});
+
+	test("every scenario grades a perfect trajectory at 1 and an empty one at 0", async () => {
+		// A suite-wide invariant: a scenario that cannot be passed, or cannot be failed,
+		// contributes nothing to separating models.
+		for (const scenario of AGENTIC_SCENARIOS) {
+			const nothing = await runScenario(scenario, scripted([{ text: "" }]));
+			expect(scenario.grade(nothing)).toBeLessThan(1);
+		}
+	});
+});

package/test/classify.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, test } from "bun:test";
+import type { CompletionResult } from "../src/upstream/types.ts";
 
 import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
 import { loadConfig } from "../src/config/load.ts";
@@ -74,7 +75,7 @@ function forbiddenUpstream(): UpstreamClient {
 		dispatch(_opts: DispatchOptions): Promise<Dispatch> {
 			throw new Error("dispatch must not be called during classification");
 		},
-		complete(): Promise<{ text: string; costUsd: number | null }> {
+		complete(): Promise<CompletionResult> {
 			throw new Error("adjudicator must not be called");
 		},
 		fetchModels(): Promise<unknown[]> {
@@ -86,7 +87,7 @@ function forbiddenUpstream(): UpstreamClient {
 	};
 }
 
-function scriptedUpstream(behaviour: () => Promise<{ text: string; costUsd: number | null }>): UpstreamClient {
+function scriptedUpstream(behaviour: () => Promise<CompletionResult>): UpstreamClient {
 	return {
 		dispatch(_opts: DispatchOptions): Promise<Dispatch> {
 			throw new Error("dispatch must not be called during classification");
@@ -358,7 +359,7 @@ describe("classify", () => {
 		const f = extractFeatures(r, 5000);
 		const expected = scoreHeuristic(f, cfg);
 		const result = await classify(r, f, cfg, {
-			upstream: scriptedUpstream(() => Promise.resolve({ text: "definitely not a tier", costUsd: 0 })),
+			upstream: scriptedUpstream(() => Promise.resolve({ text: "definitely not a tier", costUsd: 0, toolCalls: [] })),
 			ledger: null,
 			catalog: null,
 		});
@@ -384,7 +385,7 @@ describe("classify", () => {
 		const r = req(messages);
 		const f = extractFeatures(r, 5000);
 		const result = await classify(r, f, cfg, {
-			upstream: scriptedUpstream(() => Promise.resolve({ text: "hard", costUsd: 0.00001 })),
+			upstream: scriptedUpstream(() => Promise.resolve({ text: "hard", costUsd: 0.00001, toolCalls: [] })),
 			ledger: null,
 			catalog: null,
 		});

package/test/digest.test.ts

@@ -83,7 +83,7 @@ function fakeUpstream(reply: (body: Record<string, unknown>) => string, costUsd:
 			dispatch: () => Promise.reject(new Error("not used")),
 			complete: async (body) => {
 				calls.push(body);
-				return { text: reply(body), costUsd };
+				return { text: reply(body), costUsd, toolCalls: [] };
 			},
 			fetchModels: () => Promise.resolve([]),
 			fetchModelsForUser: () => Promise.resolve([]),

package/test/ollama.test.ts

@@ -371,7 +371,7 @@ describe("multi upstream + composite catalog", () => {
 		},
 		complete: async (body) => {
 			calls.push(`${name}:complete:${String(body.model)}`);
-			return { text: "", costUsd: null };
+			return { text: "", costUsd: null, toolCalls: [] };
 		},
 		fetchModels: async () => {
 			calls.push(`${name}:models`);

package/test/upstreams.test.ts

@@ -137,7 +137,7 @@ describe("dispatch by slug prefix", () => {
 		},
 		complete: async (b) => {
 			seen.push(`${name}:complete:${String(b.model)}`);
-			return { text: "", costUsd: null };
+			return { text: "", costUsd: null, toolCalls: [] };
 		},
 		fetchModels: async () => [],
 		fetchModelsForUser: async () => [],