auto-model-router 0.1.0

package/tools/mock-openrouter.ts

@@ -0,0 +1,269 @@
+/**
+ * Scriptable stand-in for the OpenRouter API, for end-to-end verification
+ * without an API key or real spend.
+ *
+ * Serves the real 147-model catalog fixture from `GET /models`, and synthesizes
+ * SSE chat completions whose shape and `usage` accounting mirror OpenRouter's
+ * wire format (including `prompt_tokens_details` and `cost`).
+ *
+ * Behaviour is driven at runtime via `POST /__control` so a driver can force
+ * the exact failure a guarded probe is supposed to catch.
+ */
+
+import { existsSync } from "node:fs";
+
+export interface MockControl {
+	/** Slugs that emit a truncated tool call, tripping `malformed_tool_args`. */
+	truncateToolArgs: string[];
+	/**
+	 * Truncate the tool call on the next N tool-offering generations, whatever
+	 * model serves them. Lets a test exercise escalation without predicting
+	 * which slug the router will pick.
+	 */
+	truncateFirstN: number;
+	/** Slugs that finish `stop` with no content, tripping `empty_completion`. */
+	emptyCompletion: string[];
+	/** Slugs that open with a refusal, tripping `refusal`. */
+	refuse: string[];
+	/** Slugs that answer with the tool call named here, to trip `repeat_tool_call`. */
+	echoToolCall: Record<string, { name: string; arguments: string }>;
+	/** Slugs that fail with this HTTP status before streaming. */
+	failWith: Record<string, number>;
+	/**
+	 * Guardrail allowlist: when non-empty, `/models/user` returns ONLY models
+	 * matching these slugs.
+	 */
+	allowedModels: string[];
+	/** Fraction of prompt tokens reported as cache reads. */
+	cacheHitRate: number;
+	/** Artificial delay before the first content chunk, ms. */
+	ttftMs: number;
+}
+const DEFAULT_CONTROL: MockControl = {
+	truncateToolArgs: [],
+	truncateFirstN: 0,
+	emptyCompletion: [],
+	refuse: [],
+	echoToolCall: {},
+	failWith: {},
+	allowedModels: [],
+	cacheHitRate: 0,
+	ttftMs: 0,
+};
+
+export interface MockServer {
+	url: string;
+	port: number;
+	control: MockControl;
+	/** Every completion request observed, in order. Assertions read this. */
+	requests: { model: string; models: string[] | undefined; sessionId: string | null; body: Record<string, unknown> }[];
+	stop(): Promise<void>;
+}
+
+function sse(payload: unknown): string {
+	return `data: ${JSON.stringify(payload)}\n\n`;
+}
+
+export async function startMockOpenRouter(fixturePath: string, port = 0): Promise<MockServer> {
+	if (!existsSync(fixturePath)) throw new Error(`mock catalog fixture missing: ${fixturePath}`);
+	const catalog = await Bun.file(fixturePath).text();
+	const control: MockControl = { ...DEFAULT_CONTROL };
+	const requests: MockServer["requests"] = [];
+
+	const server = Bun.serve({
+		port,
+		hostname: "127.0.0.1",
+		idleTimeout: 60,
+		async fetch(req) {
+			const url = new URL(req.url);
+
+			if (url.pathname === "/__control" && req.method === "POST") {
+				Object.assign(control, (await req.json()) as Partial<MockControl>);
+				return Response.json({ ok: true, control });
+			}
+			if (url.pathname === "/__requests") return Response.json(requests);
+			if (url.pathname.endsWith("/models/user")) {
+				if (control.allowedModels.length === 0) {
+					return new Response(catalog, { headers: { "content-type": "application/json" } });
+				}
+				try {
+					const parsed = JSON.parse(catalog) as { data: Array<{ id?: string }> };
+					const filtered = parsed.data.filter((m) => typeof m.id === "string" && control.allowedModels.includes(m.id));
+					return Response.json({ ...parsed, data: filtered });
+				} catch {
+					return new Response(catalog, { headers: { "content-type": "application/json" } });
+				}
+			}
+			if (url.pathname.endsWith("/models")) {
+				return new Response(catalog, { headers: { "content-type": "application/json" } });
+			}
+			if (!url.pathname.endsWith("/chat/completions")) {
+				return Response.json({ error: { message: "not found" } }, { status: 404 });
+			}
+
+			const body = (await req.json()) as Record<string, unknown>;
+			const model = String(body.model ?? "");
+			requests.push({
+				model,
+				models: Array.isArray(body.models) ? (body.models as string[]) : undefined,
+				sessionId: req.headers.get("x-session-id"),
+				body,
+			});
+
+			const status = control.failWith[model];
+			if (status !== undefined) {
+				return Response.json(
+					{ error: { message: `mock forced status ${status} for ${model}`, code: status } },
+					{ status },
+				);
+			}
+
+			// Plausible magnitudes suffice here; the mock is not a tokenizer.
+			const promptTokens = Math.max(1, Math.ceil(JSON.stringify(body).length / 3.6));
+			const cached = Math.floor(promptTokens * control.cacheHitRate);
+			const offersTools = Array.isArray(body.tools) && body.tools.length > 0;
+			const generationId = `gen-mock-${Math.random().toString(36).slice(2, 10)}`;
+
+			const stream = new ReadableStream<Uint8Array>({
+				async start(controller) {
+					const enc = new TextEncoder();
+					const push = (s: string) => controller.enqueue(enc.encode(s));
+					const base = { id: generationId, object: "chat.completion.chunk", created: Math.floor(Date.now() / 1e3), model };
+
+					// OpenRouter emits keep-alive comments on slow upstreams; exercise that path.
+					push(": OPENROUTER PROCESSING\n\n");
+					push(sse({ ...base, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] }));
+					if (control.ttftMs > 0) await Bun.sleep(control.ttftMs);
+
+					let finishReason = "stop";
+					let completionTokens = 8;
+
+					const echo = control.echoToolCall[model];
+					if (control.emptyCompletion.includes(model)) {
+						completionTokens = 0;
+					} else if (control.refuse.includes(model)) {
+						for (const piece of ["I'm sorry, ", "but I can't ", "help with that."]) {
+							push(sse({ ...base, choices: [{ index: 0, delta: { content: piece }, finish_reason: null }] }));
+						}
+					} else if (echo) {
+						push(
+							sse({
+								...base,
+								choices: [
+									{
+										index: 0,
+										delta: {
+											tool_calls: [
+												{ index: 0, id: "call_echo", type: "function", function: { name: echo.name, arguments: "" } },
+											],
+										},
+										finish_reason: null,
+									},
+								],
+							}),
+						);
+						push(
+							sse({
+								...base,
+								choices: [
+									{
+										index: 0,
+										delta: { tool_calls: [{ index: 0, function: { arguments: echo.arguments } }] },
+										finish_reason: null,
+									},
+								],
+							}),
+						);
+						finishReason = "tool_calls";
+					} else if (offersTools) {
+						const full = JSON.stringify({ path: "src/index.ts" });
+						// Split the argument JSON across frames; truncate it when scripted to,
+						// which is exactly the malformed-tool-args case the probe must catch.
+						let truncate = control.truncateToolArgs.includes(model);
+						if (!truncate && control.truncateFirstN > 0) {
+							truncate = true;
+							control.truncateFirstN--;
+						}
+						const args = truncate ? full.slice(0, full.length - 4) : full;
+						push(
+							sse({
+								...base,
+								choices: [
+									{
+										index: 0,
+										delta: {
+											tool_calls: [
+												{ index: 0, id: "call_mock", type: "function", function: { name: "read", arguments: "" } },
+											],
+										},
+										finish_reason: null,
+									},
+								],
+							}),
+						);
+						for (let i = 0; i < args.length; i += 7) {
+							push(
+								sse({
+									...base,
+									choices: [
+										{
+											index: 0,
+											delta: { tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 7) } }] },
+											finish_reason: null,
+										},
+									],
+								}),
+							);
+						}
+						finishReason = "tool_calls";
+						completionTokens = 24;
+					} else {
+						push(sse({ ...base, choices: [{ index: 0, delta: { reasoning: "weighing options" }, finish_reason: null }] }));
+						for (const piece of ["Mock ", "answer ", "from ", model, "."]) {
+							push(sse({ ...base, choices: [{ index: 0, delta: { content: piece }, finish_reason: null }] }));
+						}
+						completionTokens = 12;
+					}
+
+					push(sse({ ...base, choices: [{ index: 0, delta: {}, finish_reason: finishReason }] }));
+					push(
+						sse({
+							...base,
+							choices: [],
+							usage: {
+								prompt_tokens: promptTokens,
+								completion_tokens: completionTokens,
+								total_tokens: promptTokens + completionTokens,
+								prompt_tokens_details: { cached_tokens: cached, cache_write_tokens: cached > 0 ? 0 : promptTokens },
+								completion_tokens_details: { reasoning_tokens: 0 },
+								cost: (promptTokens * 3e-6 + completionTokens * 1.5e-5) as number,
+								cost_details: { upstream_inference_cost: 0 },
+							},
+						}),
+					);
+					push("data: [DONE]\n\n");
+					controller.close();
+				},
+			});
+
+			return new Response(stream, {
+				headers: { "content-type": "text/event-stream", "cache-control": "no-cache" },
+			});
+		},
+	});
+
+	return {
+		url: `http://127.0.0.1:${server.port}`,
+		port: server.port,
+		control,
+		requests,
+		stop: async () => {
+			await server.stop(true);
+		},
+	};
+}
+
+if (import.meta.main) {
+	const mock = await startMockOpenRouter("test/fixtures/openrouter-models.json", Number(process.env.MOCK_PORT ?? 8799));
+	console.log(`mock openrouter listening on ${mock.url}`);
+}

package/tools/smoke.ts

@@ -0,0 +1,326 @@
+/**
+ * End-to-end smoke test: real router process, real HTTP, real SQLite ledger,
+ * mock OpenRouter upstream. Proves the pipeline actually routes, escalates,
+ * and accounts for spend without needing an API key or spending money.
+ *
+ * Responses are parsed with schemas rather than asserted with casts, so the
+ * harness also verifies the client-facing wire contract.
+ *
+ * Run: bun run tools/smoke.ts
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+
+import { loadConfig } from "../src/config/load.ts";
+import { startServer } from "../src/server/http.ts";
+import { startMockOpenRouter } from "./mock-openrouter.ts";
+
+const failures: string[] = [];
+let checks = 0;
+
+function check(label: string, ok: boolean, detail?: unknown) {
+	checks++;
+	if (ok) {
+		console.log(`  PASS  ${label}`);
+		return;
+	}
+	failures.push(label);
+	console.log(`  FAIL  ${label}${detail === undefined ? "" : `\n        ${JSON.stringify(detail)}`}`);
+}
+
+const HealthSchema = z
+	.object({
+		status: z.string(),
+		apiKeyConfigured: z.boolean(),
+		catalog: z.object({ models: z.number(), fetchedAtMs: z.number(), ageMs: z.number() }).nullable(),
+	})
+	.loose();
+const ModelListSchema = z.object({ data: z.array(z.object({ id: z.string() }).loose()) });
+const StatsSchema = z
+	.object({
+		spendAllTimeUsd: z.number(),
+		requests: z.number(),
+		escalations: z.number(),
+		perModel: z.array(z.object({ slug: z.string(), spendUsd: z.number() }).loose()),
+	})
+	.loose();
+const DecisionsSchema = z.object({
+	entries: z.array(
+		z.object({ wasted: z.boolean(), slug: z.string(), tier: z.string(), attempt: z.number() }).loose(),
+	),
+});
+const BufferedSchema = z
+	.object({
+		choices: z.array(z.object({ message: z.object({ content: z.string().nullable() }).loose() }).loose()),
+		usage: z.object({ prompt_tokens: z.number(), completion_tokens: z.number() }).loose(),
+	})
+	.loose();
+
+/** Streaming frame, modelled loosely: unknown fields must survive passthrough. */
+const FrameSchema = z.object({
+	model: z.string().optional(),
+	x_auto_model_router: z.object({ model: z.string().optional(), tier: z.string().optional() }).loose().optional(),
+	choices: z
+		.array(
+			z.object({
+				delta: z
+					.object({
+						content: z.string().nullish(),
+						tool_calls: z
+							.array(z.object({ function: z.object({ arguments: z.string().optional() }).loose().optional() }).loose())
+							.optional(),
+					})
+					.loose()
+					.optional(),
+			}).loose(),
+		)
+		.optional(),
+}).loose();
+
+const TOOLS = [
+	{
+		type: "function",
+		function: {
+			name: "read",
+			description: "Read a file from disk",
+			parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] },
+		},
+	},
+	{
+		type: "function",
+		function: {
+			name: "bash",
+			description: "Run a shell command",
+			parameters: { type: "object", properties: { command: { type: "string" } }, required: ["command"] },
+		},
+	},
+];
+
+const SYSTEM = "You are a coding agent operating in a repository. Use tools to inspect files before editing.";
+
+interface Turn {
+	role: string;
+	content: unknown;
+	tool_calls?: unknown[];
+	tool_call_id?: string;
+	name?: string;
+}
+
+function body(messages: Turn[], opts: { tools?: boolean; model?: string; stream?: boolean } = {}) {
+	const out: Record<string, unknown> = {
+		model: opts.model ?? "auto",
+		messages: [{ role: "system", content: SYSTEM }, ...messages],
+		stream: opts.stream ?? true,
+	};
+	if (opts.tools !== false) out.tools = TOOLS;
+	return out;
+}
+
+/** Drains an SSE response into concatenated content, tool arguments, and router metadata. */
+async function drain(res: Response) {
+	const text = await res.text();
+	let content = "";
+	let toolArgs = "";
+	let seenModel = "";
+	let meta: { model?: string; tier?: string } | undefined;
+
+	for (const line of text.split("\n")) {
+		if (!line.startsWith("data: ")) continue;
+		const payload = line.slice(6).trim();
+		if (payload === "[DONE]") break;
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(payload);
+		} catch {
+			continue;
+		}
+		const frame = FrameSchema.safeParse(parsed);
+		if (!frame.success) continue;
+		if (frame.data.model !== undefined) seenModel = frame.data.model;
+		if (frame.data.x_auto_model_router !== undefined) meta = frame.data.x_auto_model_router;
+		const delta = frame.data.choices?.[0]?.delta;
+		if (typeof delta?.content === "string") content += delta.content;
+		for (const call of delta?.tool_calls ?? []) toolArgs += call.function?.arguments ?? "";
+	}
+	return { content, meta, toolArgs, seenModel, raw: text };
+}
+
+const home = mkdtempSync(join(tmpdir(), "auto-model-router-smoke-"));
+const mock = await startMockOpenRouter("test/fixtures/openrouter-models.json");
+console.log(`mock openrouter: ${mock.url}`);
+
+const cfg = loadConfig({});
+cfg.server = { host: "127.0.0.1", port: 0 };
+cfg.openrouter.baseUrl = `${mock.url}/api/v1`;
+cfg.openrouter.apiKey = "sk-mock";
+cfg.ledger.path = join(home, "router.db");
+cfg.logLevel = "warn";
+// The adjudicator would call the mock and obscure which tier the heuristic chose.
+cfg.classifier.ambiguityThreshold = 0;
+
+const app = startServer(cfg);
+const base = `http://127.0.0.1:${app.server.port}`;
+console.log(`auto-model-router: ${base}\n`);
+
+const post = (path: string, payload: unknown) =>
+	fetch(`${base}${path}`, {
+		method: "POST",
+		headers: { "content-type": "application/json" },
+		body: JSON.stringify(payload),
+	});
+
+
+const TIER_RANK = ["trivial", "simple", "moderate", "hard"];
+const tierRank = (tier: string): number => TIER_RANK.indexOf(tier);
+
+/**
+ * Tier the most recent turn's FIRST attempt used. `/decisions` is newest-first,
+ * so the first `attempt === 0` row belongs to the turn that just finished.
+ */
+async function firstAttemptTier(): Promise<string> {
+	const res = await fetch(`${base}/v1/router/decisions?limit=50`);
+	const entries = DecisionsSchema.parse(await res.json()).entries;
+	return entries.find((e) => e.attempt === 0)?.tier ?? "";
+}
+
+try {
+	console.log("[1] discovery surface");
+	// The server warms the catalog without blocking listen, so poll rather than race it.
+	let health = HealthSchema.parse(await (await fetch(`${base}/health`)).json());
+	for (let i = 0; i < 100 && health.catalog === null; i++) {
+		await Bun.sleep(100);
+		health = HealthSchema.parse(await (await fetch(`${base}/health`)).json());
+	}
+	check("health reports a warm catalog", (health.catalog?.models ?? 0) > 50, health);
+	const models = ModelListSchema.parse(await (await fetch(`${base}/v1/models`)).json());
+	const ids = models.data.map((m) => m.id);
+	check("advertises the auto profiles", ids.includes("auto") && ids.includes("auto-cheap"), ids);
+
+	console.log("\n[2] plain chat turn routes cheap");
+	mock.requests.length = 0;
+	const chat = await drain(await post("/v1/chat/completions", body([{ role: "user", content: "hi" }], { tools: false })));
+	const chatSlug = mock.requests[0]?.model ?? "";
+	check("dispatched exactly one upstream generation", mock.requests.length === 1, mock.requests.length);
+	check("client sees the virtual model, not the slug", chat.seenModel === "auto", chat.seenModel);
+	check("served slug is reported in metadata", typeof chat.meta?.model === "string", chat.meta);
+	check("never routes to an openrouter meta-model", !chatSlug.startsWith("openrouter/"), chatSlug);
+	check("never routes to a batch endpoint", !chatSlug.endsWith(":batch"), chatSlug);
+	check("never routes to a floating alias", !chatSlug.startsWith("~"), chatSlug);
+	check("forwards a session id for cache stickiness", Boolean(mock.requests[0]?.sessionId), mock.requests[0]?.sessionId);
+	console.log(`        -> ${chatSlug} (tier ${chat.meta?.tier})`);
+
+	console.log("\n[3] complexity separates tiers within one conversation");
+	mock.requests.length = 0;
+	const hardRes = await drain(
+		await post(
+			"/v1/chat/completions",
+			body([
+				{
+					role: "user",
+					content:
+						"Our worker pool deadlocks under load. Walk through the root cause, explain the race between the queue drain and the shutdown path, and propose an architecture that removes the invariant violation.",
+				},
+			]),
+		),
+	);
+	const hardSlug = mock.requests[0]?.model ?? "";
+	const hardFirstTier = await firstAttemptTier();
+
+	mock.requests.length = 0;
+	const mechRes = await drain(
+		await post(
+			"/v1/chat/completions",
+			body([
+				{ role: "user", content: "read src/index.ts" },
+				{
+					role: "assistant",
+					content: null,
+					tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"src/index.ts"}' } }],
+				},
+				{ role: "tool", tool_call_id: "c1", name: "read", content: "export const version = '1.0.0';\n" },
+			]),
+		),
+	);
+	const mechSlug = mock.requests[0]?.model ?? "";
+	const mechFirstTier = await firstAttemptTier();
+
+	check("hard reasoning and mechanical tool-result turns route differently", hardSlug !== mechSlug, {
+		hard: hardSlug,
+		mechanical: mechSlug,
+	});
+	// Compare what CLASSIFICATION chose, i.e. each turn's first attempt. The
+	// final tier is a poor probe here: the mock replays the same tool call the
+	// request already contains, so the mechanical turn legitimately trips
+	// `repeat_tool_call` and escalates, which would mask the classifier's
+	// separation with an escalation artifact.
+	check("the hard turn classified into a higher tier than the mechanical one", tierRank(hardFirstTier) > tierRank(mechFirstTier), {
+		hard: hardFirstTier,
+		mechanical: mechFirstTier,
+	});
+	console.log(`        hard       -> ${hardSlug} (classified ${hardFirstTier}, served ${hardRes.meta?.tier})`);
+	console.log(`        mechanical -> ${mechSlug} (classified ${mechFirstTier}, served ${mechRes.meta?.tier})`);
+
+	console.log("\n[4] guarded probe escalates on a malformed tool call");
+	mock.requests.length = 0;
+	// Truncate only the first generation, whichever model serves it: the retry
+	// must then succeed on a stronger model.
+	mock.control.truncateFirstN = 1;
+	const escalated = await drain(
+		await post(
+			"/v1/chat/completions",
+			body([
+				{ role: "user", content: "open the config file" },
+				{
+					role: "assistant",
+					content: null,
+					tool_calls: [{ id: "c9", type: "function", function: { name: "bash", arguments: '{"command":"ls"}' } }],
+				},
+				{ role: "tool", tool_call_id: "c9", name: "bash", content: "config.yml\n" },
+			]),
+		),
+	);
+	let toolArgsValid = false;
+	try {
+		JSON.parse(escalated.toolArgs);
+		toolArgsValid = escalated.toolArgs.length > 0;
+	} catch {
+		toolArgsValid = false;
+	}
+	const attemptSlugs = mock.requests.map((r) => r.model);
+	check("retried upstream after the malformed tool call", mock.requests.length >= 2, attemptSlugs);
+	check("escalated to a different model", new Set(attemptSlugs).size >= 2, attemptSlugs);
+	check("client received only well-formed tool arguments", toolArgsValid, escalated.toolArgs);
+	mock.control.truncateFirstN = 0;
+	console.log(`        attempts -> ${attemptSlugs.join(" then ")}`);
+
+	console.log("\n[5] buffered (non-streaming) responses");
+	mock.requests.length = 0;
+	const buffered = BufferedSchema.parse(
+		await (
+			await post("/v1/chat/completions", body([{ role: "user", content: "say hello" }], { tools: false, stream: false }))
+		).json(),
+	);
+	check("buffered response carries assembled content", Boolean(buffered.choices[0]?.message.content), buffered.choices);
+	check("buffered response reports usage", buffered.usage.prompt_tokens > 0, buffered.usage);
+
+	console.log("\n[6] ledger accounting");
+	const stats = StatsSchema.parse(await (await fetch(`${base}/v1/router/stats`)).json());
+	const decisions = DecisionsSchema.parse(await (await fetch(`${base}/v1/router/decisions?limit=50`)).json()).entries;
+	check("ledger recorded every attempt", decisions.length >= 6, decisions.length);
+	check("stats report non-zero spend", stats.spendAllTimeUsd > 0, stats);
+	const wasted = decisions.filter((d) => d.wasted).length;
+	check("the abandoned escalation attempt is booked as wasted spend", wasted >= 1, wasted);
+} finally {
+	await app.stop();
+	await mock.stop();
+	rmSync(home, { recursive: true, force: true });
+}
+
+console.log(`\n${checks - failures.length}/${checks} checks passed`);
+if (failures.length > 0) {
+	console.log(`failed:\n  ${failures.join("\n  ")}`);
+	process.exit(1);
+}

package/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext"],
+    "target": "ESNext",
+    "module": "Preserve",
+    "moduleResolution": "bundler",
+    "moduleDetection": "force",
+    "types": ["bun"],
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    "strict": true,
+    "exactOptionalPropertyTypes": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "skipLibCheck": true
+  },
+  "include": ["src", "test"]
+}