auto-model-router 0.1.0

package/src/upstream/sse-parse.ts

@@ -0,0 +1,208 @@
+/**
+ * Incremental SSE decoder for OpenRouter's chat-completions stream.
+ *
+ * Handles the realities of the wire: frames split across TCP chunks (even
+ * mid-line or mid-codepoint), `: OPENROUTER PROCESSING` keep-alive comments,
+ * and the terminal `data: [DONE]`. A malformed JSON frame is skipped with a
+ * warning rather than thrown — one bad frame must not abort a paid generation.
+ */
+
+import type { UsageCounts } from "../cost/types.ts";
+import type { FinishReason, StreamEvent, UpstreamChunk } from "../wire/types.ts";
+
+export type SseWarning = (message: string, fields?: Record<string, unknown>) => void;
+
+function asRec(v: unknown): Record<string, unknown> | null {
+	return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
+}
+
+function num(v: unknown): number {
+	return typeof v === "number" && Number.isFinite(v) ? v : 0;
+}
+
+function mapFinishReason(v: unknown): FinishReason | null {
+	if (typeof v !== "string" || v === "") return null;
+	switch (v) {
+		case "stop":
+		case "length":
+		case "tool_calls":
+		case "content_filter":
+			return v;
+		default:
+			// Anything else ("error", provider-specific strings) means the
+			// generation failed; the escalation guard treats it as such.
+			return "error";
+	}
+}
+
+function reasoningDelta(delta: Record<string, unknown>): string | null {
+	// Providers disagree on the field: OpenAI-style `reasoning`, DeepSeek-style
+	// `reasoning_content`, OpenRouter-normalized `reasoning_details`.
+	const reasoning = delta.reasoning;
+	if (typeof reasoning === "string" && reasoning.length > 0) return reasoning;
+	const reasoningContent = delta.reasoning_content;
+	if (typeof reasoningContent === "string" && reasoningContent.length > 0) return reasoningContent;
+	const details = delta.reasoning_details;
+	if (Array.isArray(details)) {
+		let out = "";
+		for (const d of details) {
+			const rec = asRec(d);
+			if (rec && typeof rec.text === "string") out += rec.text;
+		}
+		if (out.length > 0) return out;
+	}
+	return null;
+}
+
+function usageEvent(payload: Record<string, unknown>): StreamEvent | null {
+	const usage = asRec(payload.usage);
+	if (!usage) return null;
+	const promptDetails = asRec(usage.prompt_tokens_details);
+	const completionDetails = asRec(usage.completion_tokens_details);
+	const counts: UsageCounts = {
+		// prompt_tokens already INCLUDES cached tokens; cachedTokens is a
+		// sub-count for pricing, never added on top.
+		promptTokens: num(usage.prompt_tokens),
+		cachedTokens: num(promptDetails?.cached_tokens),
+		cacheWriteTokens: num(promptDetails?.cache_write_tokens),
+		completionTokens: num(usage.completion_tokens),
+		reasoningTokens: num(completionDetails?.reasoning_tokens),
+		// Usage reports carry no image count; per-image surcharges are forecast-side.
+		images: 0,
+	};
+	const cost = usage.cost;
+	const reportedCostUsd = typeof cost === "number" && Number.isFinite(cost) ? cost : null;
+	return { type: "usage", usage: counts, reportedCostUsd };
+}
+
+/**
+ * Decodes an SSE byte stream into interpreted chunks. Single-pass; abandoning
+ * the generator releases the underlying reader, but callers should abort the
+ * fetch signal to actually tear down the connection.
+ */
+export async function* parseSse(
+	stream: ReadableStream<Uint8Array>,
+	warn?: SseWarning,
+): AsyncGenerator<UpstreamChunk> {
+	const reader = stream.getReader();
+	// Streaming decoder so a multi-byte codepoint split across chunks survives.
+	const decoder = new TextDecoder();
+	let buf = "";
+	let dataLines: string[] = [];
+	let started = false;
+
+	const buildChunk = (data: string): UpstreamChunk | "done" | null => {
+		if (data === "[DONE]") return "done";
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(data);
+		} catch (err) {
+			warn?.("skipping malformed SSE payload", {
+				error: err instanceof Error ? err.message : String(err),
+				snippet: data.length > 160 ? `${data.slice(0, 160)}…` : data,
+			});
+			return null;
+		}
+		const raw = asRec(parsed);
+		if (!raw) {
+			warn?.("skipping non-object SSE payload", { snippet: data.slice(0, 160) });
+			return null;
+		}
+
+		const events: StreamEvent[] = [];
+		const id = typeof raw.id === "string" && raw.id !== "" ? raw.id : null;
+		const model = typeof raw.model === "string" && raw.model !== "" ? raw.model : null;
+		if (!started && id !== null && model !== null) {
+			started = true;
+			events.push({ type: "start", servedSlug: model, generationId: id });
+		}
+
+		const choices = raw.choices;
+		const choice0 = Array.isArray(choices) && choices.length > 0 ? asRec(choices[0]) : null;
+		const delta = choice0 ? asRec(choice0.delta) : null;
+		if (delta) {
+			const content = delta.content;
+			if (typeof content === "string" && content.length > 0) {
+				events.push({ type: "text", delta: content });
+			}
+			const reasoning = reasoningDelta(delta);
+			if (reasoning !== null) events.push({ type: "reasoning", delta: reasoning });
+			const toolCalls = delta.tool_calls;
+			if (Array.isArray(toolCalls)) {
+				for (const tcRaw of toolCalls) {
+					const tc = asRec(tcRaw);
+					if (!tc) continue;
+					const ev: {
+						type: "tool_call";
+						index: number;
+						id?: string;
+						name?: string;
+						argsDelta?: string;
+					} = { type: "tool_call", index: typeof tc.index === "number" ? tc.index : 0 };
+					// Fragments only carry id/name on their first chunk and arguments
+					// as they stream; attach exactly what this fragment supplies.
+					if (typeof tc.id === "string" && tc.id !== "") ev.id = tc.id;
+					const fn = asRec(tc.function);
+					if (fn) {
+						if (typeof fn.name === "string" && fn.name !== "") ev.name = fn.name;
+						if (typeof fn.arguments === "string" && fn.arguments !== "") ev.argsDelta = fn.arguments;
+					}
+					events.push(ev);
+				}
+			}
+		}
+		const finish = mapFinishReason(choice0?.finish_reason);
+		if (finish !== null) events.push({ type: "finish", reason: finish });
+		const usage = usageEvent(raw);
+		if (usage !== null) events.push(usage);
+
+		return { raw, events };
+	};
+
+	// Returns a chunk when the line completed a frame, "done" on [DONE], else null.
+	const processLine = (line: string): UpstreamChunk | "done" | null => {
+		if (line === "") {
+			// Blank line terminates a frame.
+			if (dataLines.length === 0) return null;
+			const data = dataLines.join("\n");
+			dataLines = [];
+			return buildChunk(data);
+		}
+		// Keep-alive comments (": OPENROUTER PROCESSING") carry no data.
+		if (line.startsWith(":")) return null;
+		if (line.startsWith("data:")) {
+			// Per SSE spec, a single leading space after the colon is stripped.
+			dataLines.push(line.slice(5).replace(/^ /, ""));
+		}
+		// event:/id:/retry: fields carry nothing we route on.
+		return null;
+	};
+
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			buf += decoder.decode(value, { stream: true });
+			let nl: number;
+			while ((nl = buf.indexOf("\n")) !== -1) {
+				const line = buf.slice(0, nl);
+				buf = buf.slice(nl + 1);
+				const frame = processLine(line.endsWith("\r") ? line.slice(0, -1) : line);
+				if (frame === "done") return;
+				if (frame !== null) yield frame;
+			}
+		}
+		// Flush a trailing partial line and a final frame missing its blank-line
+		// terminator; some providers close the stream right after the payload.
+		buf += decoder.decode();
+		if (buf.length > 0) {
+			const frame = processLine(buf.endsWith("\r") ? buf.slice(0, -1) : buf);
+			if (frame === "done") return;
+			if (frame !== null) yield frame;
+		}
+		const frame = processLine("");
+		if (frame !== null && frame !== "done") yield frame;
+	} finally {
+		reader.releaseLock();
+	}
+}

package/src/upstream/types.ts

@@ -0,0 +1,75 @@
+/**
+ * OpenRouter transport contracts.
+ *
+ * The client is deliberately thin: it POSTs a body the wire layer rendered,
+ * parses SSE into `UpstreamChunk`s, and classifies failures. It holds no
+ * routing policy — that lives in `router/`.
+ */
+
+import type { UpstreamChunk, WireError } from "../wire/types.ts";
+
+export type UpstreamErrorKind =
+	| "auth"
+	| "rate_limit"
+	| "context_length"
+	| "model_unavailable"
+	| "invalid_request"
+	| "moderation"
+	| "upstream_error"
+	| "timeout"
+	| "network"
+	| "aborted";
+
+export class UpstreamError extends Error {
+	constructor(
+		readonly kind: UpstreamErrorKind,
+		readonly status: number,
+		message: string,
+		readonly retryable: boolean,
+		readonly body?: unknown,
+	) {
+		super(message);
+		this.name = "UpstreamError";
+	}
+
+	toWireError(): WireError {
+		return { status: this.status, code: this.kind, message: this.message };
+	}
+}
+
+export interface DispatchOptions {
+	body: Record<string, unknown>;
+	/** Forwarded as the `x-session-id` header, mirroring body `session_id`. */
+	sessionId: string;
+	signal: AbortSignal;
+}
+
+/**
+ * A live upstream generation.
+ *
+ * `chunks` is single-pass. The escalation guard may abandon it before
+ * completion; callers MUST abort the signal in that case so the upstream
+ * connection is torn down and no further tokens are billed.
+ */
+export interface Dispatch {
+	chunks: AsyncIterable<UpstreamChunk>;
+	/** Resolves once the generation id is known, i.e. on the first chunk. */
+	generationId(): Promise<string | null>;
+}
+
+export interface UpstreamClient {
+	/** Streaming chat completion. Always requests `stream: true` upstream. */
+	dispatch(opts: DispatchOptions): Promise<Dispatch>;
+	/**
+	 * Non-streaming single-shot, used by the classifier adjudicator.
+	 * Returns assistant text and the reported cost.
+	 */
+	complete(body: Record<string, unknown>, signal: AbortSignal): Promise<{ text: string; costUsd: number | null }>;
+	/** Raw catalog fetch. Returns the parsed `data` array untouched. */
+	fetchModels(signal?: AbortSignal): Promise<unknown[]>;
+	/**
+	 * Key-scoped catalog fetch (`GET /models/user`). Returns the models
+	 * available to the configured key under active guardrails and preferences.
+	 */
+	fetchModelsForUser(signal?: AbortSignal): Promise<unknown[]>;
+}

package/src/util/log.ts

@@ -0,0 +1,53 @@
+import type { RouterConfig } from "../config/types.ts";
+
+export type LogLevel = RouterConfig["logLevel"];
+
+export interface Logger {
+	error(msg: string, fields?: Record<string, unknown>): void;
+	warn(msg: string, fields?: Record<string, unknown>): void;
+	info(msg: string, fields?: Record<string, unknown>): void;
+	debug(msg: string, fields?: Record<string, unknown>): void;
+}
+
+const LEVEL_RANK: Record<LogLevel, number> = {
+	silent: Number.POSITIVE_INFINITY,
+	error: 40,
+	warn: 30,
+	info: 20,
+	debug: 10,
+};
+
+function formatField(v: unknown): string {
+	if (typeof v === "string") return /[\s=]/.test(v) ? JSON.stringify(v) : v;
+	if (typeof v === "number" || typeof v === "boolean" || typeof v === "bigint") return String(v);
+	try {
+		return JSON.stringify(v) ?? "null";
+	} catch {
+		return String(v);
+	}
+}
+
+/**
+ * Single-line stderr logger. Suppressed levels return before any formatting
+ * or allocation, so debug calls on the hot path are effectively free.
+ */
+export function createLogger(level: LogLevel): Logger {
+	const threshold = LEVEL_RANK[level];
+	const emit = (rank: number, tag: string, msg: string, fields?: Record<string, unknown>): void => {
+		if (rank < threshold) return;
+		let line = `${new Date().toISOString()} ${tag} ${msg}`;
+		if (fields !== undefined) {
+			for (const [k, v] of Object.entries(fields)) {
+				if (v === undefined) continue;
+				line += ` ${k}=${formatField(v)}`;
+			}
+		}
+		process.stderr.write(line + "\n");
+	};
+	return {
+		error: (msg, fields) => emit(LEVEL_RANK.error, "ERROR", msg, fields),
+		warn: (msg, fields) => emit(LEVEL_RANK.warn, "WARN", msg, fields),
+		info: (msg, fields) => emit(LEVEL_RANK.info, "INFO", msg, fields),
+		debug: (msg, fields) => emit(LEVEL_RANK.debug, "DEBUG", msg, fields),
+	};
+}

package/src/util/sqlite.ts

@@ -0,0 +1,140 @@
+/**
+ * SQLite bootstrap. This is the ONLY migration path: every table the router
+ * needs is created here, idempotently, so boot order never matters.
+ *
+ * Tables:
+ *  - catalog_cache: single-row (id = 1) store of the last raw OpenRouter
+ *    catalog payload, so restarts route from disk while a refresh is pending.
+ *  - ledger: one row per dispatched upstream generation (see cost/types.ts
+ *    LedgerEntry). `reasons`, `usage`, and `cost_breakdown` are JSON text.
+ *  - token_calibration: running sums of estimated prompt bytes vs. actual
+ *    billed prompt tokens per tokenizer family (see tokens/estimate.ts).
+ *  - conversations: per-conversation routing memory, written by the
+ *    RouterBrain slice's ConversationStore (it codes against these columns).
+ */
+
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+
+/** Bump when a migration is added; guarded below so reopening never regresses it. */
+const USER_VERSION = 4;
+
+const MIGRATIONS = `
+CREATE TABLE IF NOT EXISTS catalog_cache (
+  id INTEGER PRIMARY KEY CHECK (id = 1),
+  payload TEXT NOT NULL,
+  fetched_at_ms INTEGER NOT NULL,
+  etag TEXT,
+  key_scoped INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS ledger (
+  id TEXT PRIMARY KEY,
+  created_at_ms INTEGER NOT NULL,
+  conversation_key TEXT NOT NULL,
+  session_id TEXT NOT NULL,
+  turn INTEGER NOT NULL,
+  requested_model TEXT NOT NULL,
+  harness_id TEXT NOT NULL DEFAULT '',
+  slug TEXT NOT NULL,
+  served_slug TEXT,
+  tier TEXT NOT NULL,
+  classification_source TEXT NOT NULL,
+  reasons TEXT NOT NULL,
+  predicted_usd REAL NOT NULL,
+  reported_usd REAL,
+  usage TEXT NOT NULL,
+  cost_breakdown TEXT,
+  attempt INTEGER NOT NULL,
+  escalation_signal TEXT,
+  latency_ms INTEGER NOT NULL,
+  ttft_ms INTEGER,
+  finish_reason TEXT,
+  wasted INTEGER NOT NULL DEFAULT 0,
+  upstream_generation_id TEXT,
+  error TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_ledger_conversation ON ledger (conversation_key);
+CREATE INDEX IF NOT EXISTS idx_ledger_created ON ledger (created_at_ms);
+CREATE INDEX IF NOT EXISTS idx_ledger_slug ON ledger (slug);
+
+CREATE TABLE IF NOT EXISTS token_calibration (
+  tokenizer TEXT PRIMARY KEY,
+  est_bytes INTEGER NOT NULL,
+  actual_tokens INTEGER NOT NULL,
+  samples INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS conversations (
+  key TEXT PRIMARY KEY,
+  session_id TEXT NOT NULL,
+  turn INTEGER NOT NULL DEFAULT 0,
+  current_slug TEXT,
+  current_tier TEXT,
+  sticky_until_turn INTEGER NOT NULL DEFAULT 0,
+  escalations INTEGER NOT NULL DEFAULT 0,
+  spent_usd REAL NOT NULL DEFAULT 0,
+  last_prompt_tokens INTEGER NOT NULL DEFAULT 0,
+  cache_warm_slug TEXT,
+  cache_warm_at_ms INTEGER NOT NULL DEFAULT 0,
+  updated_at_ms INTEGER NOT NULL DEFAULT 0
+);
+
+-- v2: catalog_cache gains key_scoped provenance. ALTER TABLE ADD COLUMN is
+-- idempotent only via a guard; SQLite has no IF NOT EXISTS for columns, so
+-- probe pragma_table_info and add when absent.
+`;
+
+const MIGRATE_V2 = `
+ALTER TABLE catalog_cache ADD COLUMN key_scoped INTEGER NOT NULL DEFAULT 0;
+`;
+
+const MIGRATE_V3 = `
+ALTER TABLE ledger ADD COLUMN harness_id TEXT NOT NULL DEFAULT '';
+`;
+
+// v4: ledger gains error_kind, so per-model trust can count only failures the
+// MODEL is responsible for. Before this, any non-null `error` counted against a
+// model's reliability — including client aborts, account-level auth/policy
+// refusals, and guardrail `model_unavailable`, none of which say anything about
+// the model's quality. Those spurious demotions shrink the candidate pool and
+// push traffic onto a handful of survivors.
+//
+// Existing rows are backfilled from the stored `error` text, which turn.ts
+// writes as `"<kind>: <message>"` (the abort path writes the bare message
+// "request aborted"). Anything unrecognised stays NULL and is treated as
+// model-attributable, preserving the old, stricter behaviour for rows we
+// cannot classify.
+const MIGRATE_V4 = `
+ALTER TABLE ledger ADD COLUMN error_kind TEXT;
+
+UPDATE ledger SET error_kind = CASE
+  WHEN error IS NULL THEN NULL
+  WHEN error = 'request aborted' THEN 'aborted'
+  WHEN instr(error, ': ') > 0 THEN substr(error, 1, instr(error, ': ') - 1)
+  ELSE NULL
+END
+WHERE error IS NOT NULL;
+`;
+
+export function openDb(path: string): Database {
+	// ":memory:" has no parent directory to create.
+	if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
+	const db = new Database(path);
+	// WAL + NORMAL: single-writer local service; favours read latency on the turn hot path.
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec("PRAGMA synchronous = NORMAL");
+	db.exec(MIGRATIONS);
+	// PRAGMA user_version always returns exactly one row with one integer column.
+	const versionRow = db.query("PRAGMA user_version").get() as { user_version: number };
+	if (versionRow.user_version < USER_VERSION) {
+		const cacheCols = db.query("PRAGMA table_info(catalog_cache)").all() as { name: string }[];
+		if (!cacheCols.some((c) => c.name === "key_scoped")) db.exec(MIGRATE_V2);
+		const ledgerCols = db.query("PRAGMA table_info(ledger)").all() as { name: string }[];
+		if (!ledgerCols.some((c) => c.name === "harness_id")) db.exec(MIGRATE_V3);
+		if (!ledgerCols.some((c) => c.name === "error_kind")) db.exec(MIGRATE_V4);
+		db.exec(`PRAGMA user_version = ${USER_VERSION}`);
+	}
+	return db;
+}

package/src/util/sse.ts

@@ -0,0 +1,23 @@
+/**
+ * SSE encoding helpers for the client-facing wire. The OpenAI front end needs
+ * exactly two frame shapes; both go through one shared TextEncoder because a
+ * frame is emitted per token on the hot path.
+ */
+
+const encoder = new TextEncoder();
+
+/** One `data:` frame carrying a JSON payload, terminated by a blank line. */
+export function sseDataFrame(value: unknown): string {
+	return `data: ${JSON.stringify(value)}\n\n`;
+}
+
+/** Encoded variant of {@link sseDataFrame} for stream hot paths. */
+export function encodeSseData(value: unknown): Uint8Array {
+	return encoder.encode(sseDataFrame(value));
+}
+
+/** Terminal frame of every OpenAI SSE stream. */
+export const SSE_DONE_FRAME = "data: [DONE]\n\n";
+
+/** Pre-encoded {@link SSE_DONE_FRAME}. */
+export const SSE_DONE_BYTES: Uint8Array = encoder.encode(SSE_DONE_FRAME);

package/src/wire/openai/errors.ts

@@ -0,0 +1,48 @@
+import type { WireError } from "../types.ts";
+
+/** A wire-format failure carrying its structured payload through the stack. */
+export class WireErrorException extends Error {
+	readonly wireError: WireError;
+
+	constructor(wireError: WireError) {
+		super(wireError.message);
+		this.name = "WireErrorException";
+		this.wireError = wireError;
+	}
+}
+
+export function invalidRequest(message: string): WireErrorException {
+	return new WireErrorException({ status: 400, code: "invalid_request", message });
+}
+
+export function modelNotFound(model: string): WireErrorException {
+	return new WireErrorException({
+		status: 404,
+		code: "model_not_found",
+		message: `Unknown model: ${model}`,
+	});
+}
+
+export function unauthorized(message = "Missing or invalid API key"): WireErrorException {
+	return new WireErrorException({ status: 401, code: "unauthorized", message });
+}
+
+/** Render a WireError in the OpenAI error envelope shape. */
+export function renderErrorEnvelope(err: WireError): {
+	error: { message: string; type: string; code: string };
+} {
+	return {
+		error: {
+			message: err.message,
+			type:
+				err.status === 401
+					? "authentication_error"
+					: err.status === 429
+						? "rate_limit_error"
+						: err.status >= 500
+							? "server_error"
+							: "invalid_request_error",
+			code: err.code,
+		},
+	};
+}

package/src/wire/openai/models.ts

@@ -0,0 +1,37 @@
+import type { RouterConfig } from "../../config/types.ts";
+import type { BlendedRate } from "../../cost/types.ts";
+
+/**
+ * Virtual profiles have no real creation date; a pinned constant keeps
+ * /v1/models output stable for client caches.
+ */
+const PROFILES_CREATED = 1767225600; // 2026-01-01T00:00:00Z
+
+export function renderModelList(cfg: RouterConfig, blend: BlendedRate | null): unknown {
+	const inputPerMtok = blend ? blend.inputPerMtok : cfg.ledger.fallbackBlend.inputPerMtok;
+	const outputPerMtok = blend ? blend.outputPerMtok : cfg.ledger.fallbackBlend.outputPerMtok;
+	return {
+		object: "list",
+		data: cfg.profiles.map((p) => {
+			// OpenRouter-style per-token price strings, converted from our
+			// per-million-token internal representation.
+			const pricing: Record<string, string> = {
+				prompt: String(inputPerMtok / 1e6),
+				completion: String(outputPerMtok / 1e6),
+			};
+			if (blend) {
+				pricing.input_cache_read = String(blend.cacheReadPerMtok / 1e6);
+				pricing.input_cache_write = String(blend.cacheWritePerMtok / 1e6);
+			}
+			return {
+				id: p.id,
+				object: "model",
+				created: PROFILES_CREATED,
+				owned_by: "auto-model-router",
+				name: p.name,
+				context_length: p.contextWindow,
+				pricing,
+			};
+		}),
+	};
+}