auto-model-router 0.1.0

package/src/wire/openai/request.ts

@@ -0,0 +1,279 @@
+import type {
+	NormMessage,
+	NormRequest,
+	NormTool,
+	NormToolCall,
+	ReasoningLevel,
+	Role,
+	UpstreamMutations,
+} from "../types.ts";
+import { conversationKeyOf } from "../../util/hash.ts";
+import { invalidRequest, modelNotFound } from "./errors.ts";
+
+const ROLES: Record<string, true> = { system: true, developer: true, user: true, assistant: true, tool: true };
+
+const REASONING_LEVELS: Record<string, true> = {
+	off: true,
+	minimal: true,
+	low: true,
+	medium: true,
+	high: true,
+	xhigh: true,
+	max: true,
+};
+
+/**
+ * UTF-8 byte length without allocating an encoded copy. Lone surrogates count
+ * as 3 bytes, matching TextEncoder's U+FFFD replacement.
+ */
+function utf8Bytes(s: string): number {
+	let n = 0;
+	for (let i = 0; i < s.length; i++) {
+		const c = s.charCodeAt(i);
+		if (c < 0x80) n += 1;
+		else if (c < 0x800) n += 2;
+		else if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) {
+			n += 4;
+			i++;
+		} else n += 3;
+	}
+	return n;
+}
+
+function normalizeMessage(raw: unknown, index: number): NormMessage {
+	if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+		throw invalidRequest(`messages[${index}] must be an object`);
+	}
+	const m = raw as {
+		role?: unknown;
+		content?: unknown;
+		tool_calls?: unknown;
+		tool_call_id?: unknown;
+		name?: unknown;
+	};
+	if (typeof m.role !== "string" || !(m.role in ROLES)) {
+		throw invalidRequest(`messages[${index}].role must be one of: system, developer, user, assistant, tool`);
+	}
+
+	// Flatten content into lossy text plus an image count. The flattened view
+	// feeds classification only; dispatch bytes come from renderUpstreamBody.
+	let text = "";
+	let images = 0;
+	if (typeof m.content === "string") {
+		text = m.content;
+	} else if (Array.isArray(m.content)) {
+		const parts: string[] = [];
+		for (const part of m.content) {
+			if (typeof part !== "object" || part === null) continue;
+			const p = part as { type?: unknown; text?: unknown };
+			if (p.type === "text" && typeof p.text === "string") parts.push(p.text);
+			else if (p.type === "image_url") images++;
+		}
+		text = parts.join("\n");
+	} else if (m.content !== null && m.content !== undefined) {
+		throw invalidRequest(`messages[${index}].content must be a string or a content-part array`);
+	}
+
+	const toolCalls: NormToolCall[] = [];
+	if (Array.isArray(m.tool_calls)) {
+		for (const tc of m.tool_calls) {
+			if (typeof tc !== "object" || tc === null) continue;
+			const t = tc as { id?: unknown; function?: unknown };
+			const fn =
+				typeof t.function === "object" && t.function !== null
+					? (t.function as { name?: unknown; arguments?: unknown })
+					: {};
+			toolCalls.push({
+				id: typeof t.id === "string" ? t.id : "",
+				name: typeof fn.name === "string" ? fn.name : "",
+				argsJson: typeof fn.arguments === "string" ? fn.arguments : "",
+			});
+		}
+	}
+
+	const msg: NormMessage = {
+		role: m.role as Role,
+		text,
+		images,
+		textBytes: utf8Bytes(text),
+		toolCalls,
+	};
+	if (m.role === "tool") {
+		if (typeof m.tool_call_id === "string") msg.toolCallId = m.tool_call_id;
+		if (typeof m.name === "string") msg.toolName = m.name;
+	}
+	return msg;
+}
+
+function normalizeTools(raw: unknown): NormTool[] {
+	if (!Array.isArray(raw)) return [];
+	const tools: NormTool[] = [];
+	for (const t of raw) {
+		if (typeof t !== "object" || t === null) continue;
+		const fn = (t as { function?: unknown }).function;
+		if (typeof fn !== "object" || fn === null) continue;
+		const f = fn as { name?: unknown; description?: unknown; parameters?: unknown };
+		tools.push({
+			name: typeof f.name === "string" ? f.name : "",
+			description: typeof f.description === "string" ? f.description : "",
+			schemaBytes: utf8Bytes(JSON.stringify(f.parameters ?? {})),
+		});
+	}
+	return tools;
+}
+
+function parseReasoning(b: { reasoning?: unknown; reasoning_effort?: unknown }): ReasoningLevel | undefined {
+	// The OpenRouter-style object wins over the OpenAI-style flat field: it is
+	// the spelling we emit upstream, so it is the client's clearest intent.
+	if (typeof b.reasoning === "object" && b.reasoning !== null) {
+		const r = b.reasoning as { effort?: unknown; enabled?: unknown };
+		if (typeof r.effort === "string" && r.effort in REASONING_LEVELS) return r.effort as ReasoningLevel;
+		if (r.enabled === false) return "off";
+	}
+	if (typeof b.reasoning_effort === "string" && b.reasoning_effort in REASONING_LEVELS) {
+		return b.reasoning_effort as ReasoningLevel;
+	}
+	return undefined;
+}
+
+function renderUpstreamBody(
+	original: Record<string, unknown>,
+	m: UpstreamMutations,
+): Record<string, unknown> {
+	// Deep clone: an escalation retry renders the same original again for a
+	// different model, so a render must never touch the stored body.
+	const body = structuredClone(original);
+	body.model = m.slug;
+	if (m.fallbacks.length > 0) body.models = [m.slug, ...m.fallbacks];
+	body.session_id = m.sessionId;
+	// The guard always consumes a stream; the sink re-buffers for non-streaming clients.
+	body.stream = true;
+	// OpenRouter returns usage unconditionally and the parameter is deprecated.
+	delete body.stream_options;
+
+	if (m.maxTokens !== undefined) {
+		// Respect whichever max-token spelling the client used.
+		if ("max_completion_tokens" in body) body.max_completion_tokens = m.maxTokens;
+		else body.max_tokens = m.maxTokens;
+	}
+
+	// Normalize to the OpenRouter spelling; the two fields must never coexist.
+	delete body.reasoning_effort;
+	if (m.reasoning === undefined) delete body.reasoning;
+	else if (m.reasoning === "off") body.reasoning = { enabled: false };
+	else body.reasoning = { effort: m.reasoning };
+
+	// messages was validated to be an array of objects at parse time.
+	const messages = body.messages as Record<string, unknown>[];
+	if (m.stripAssistantReasoning) {
+		// omp replays reasoning fields for what it believes is a local backend;
+		// most OpenRouter upstreams reject every spelling.
+		for (const msg of messages) {
+			if (msg.role !== "assistant") continue;
+			delete msg.reasoning;
+			delete msg.reasoning_content;
+			delete msg.reasoning_details;
+		}
+	}
+	for (const idx of m.cacheBreakpointMessageIndices) {
+		const msg = messages[idx];
+		if (!msg) continue;
+		const content = msg.content;
+		if (typeof content === "string") {
+			msg.content = [{ type: "text", text: content, cache_control: { type: "ephemeral" } }];
+		} else if (Array.isArray(content)) {
+			for (let i = content.length - 1; i >= 0; i--) {
+				const part = content[i] as { type?: unknown; cache_control?: unknown } | undefined;
+				if (part && typeof part === "object" && part.type === "text") {
+					part.cache_control = { type: "ephemeral" };
+					break;
+				}
+			}
+		}
+	}
+	return body;
+}
+
+export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
+	if (typeof body !== "object" || body === null || Array.isArray(body)) {
+		throw invalidRequest("Request body must be a JSON object");
+	}
+	const b = body as Record<string, unknown>;
+
+	// Harness identity for per-harness budgets and toast scoping. omp sends
+	// this via the provider block's `headers:` override; absent ⇒ single harness.
+	const harnessId = (headers.get("x-omp-harness") ?? "").trim();
+
+	if (typeof b.model !== "string" || b.model.length === 0) {
+		throw invalidRequest("model must be a non-empty string");
+	}
+	// Strip any provider prefix so `auto` and `auto-model-router/auto` both resolve to
+	// the profile id.
+	const requestedModel = b.model.slice(b.model.lastIndexOf("/") + 1);
+	if (requestedModel.length === 0) throw modelNotFound(b.model);
+
+	if (!Array.isArray(b.messages) || b.messages.length === 0) {
+		throw invalidRequest("messages must be a non-empty array");
+	}
+	const messages = b.messages.map((msg, i) => normalizeMessage(msg, i));
+	const tools = normalizeTools(b.tools);
+
+	const tc = b.tool_choice;
+	const forcedToolChoice =
+		tc !== null &&
+		tc !== undefined &&
+		(typeof tc === "object" || (typeof tc === "string" && tc !== "auto" && tc !== "none"));
+
+	let promptBytes = 0;
+	for (const msg of messages) promptBytes += msg.textBytes;
+	for (const t of tools) promptBytes += t.schemaBytes + utf8Bytes(t.name) + utf8Bytes(t.description);
+
+	let hasImages = false;
+	for (const msg of messages) {
+		if (msg.images > 0) {
+			hasImages = true;
+			break;
+		}
+	}
+
+	// Conversation identity: the leading system/developer run plus the first
+	// message after it, matching how OpenRouter fingerprints conversations.
+	const systemParts: string[] = [];
+	let firstNonSystemText = "";
+	for (const msg of messages) {
+		if (msg.role === "system" || msg.role === "developer") systemParts.push(msg.text);
+		else {
+			firstNonSystemText = msg.text;
+			break;
+		}
+	}
+	const conversationKey = conversationKeyOf(systemParts.join("\n"), firstNonSystemText);
+
+	const maxTokens =
+		typeof b.max_completion_tokens === "number"
+			? b.max_completion_tokens
+			: typeof b.max_tokens === "number"
+				? b.max_tokens
+				: undefined;
+	const temperature = typeof b.temperature === "number" ? b.temperature : undefined;
+	const reasoning = parseReasoning(b);
+
+	return {
+		protocol: "openai-chat",
+		conversationKey,
+		harnessId,
+		requestedModel,
+		messages,
+		tools,
+		forcedToolChoice,
+		stream: b.stream === true,
+		hasImages,
+		promptBytes,
+		...(maxTokens !== undefined ? { maxTokens } : {}),
+		...(temperature !== undefined ? { temperature } : {}),
+		...(reasoning !== undefined ? { reasoning } : {}),
+		renderUpstreamBody(m: UpstreamMutations): Record<string, unknown> {
+			return renderUpstreamBody(b, m);
+		},
+	};
+}

package/src/wire/openai/sink.ts

@@ -0,0 +1,213 @@
+import type { ResponseSink, TurnSummary, UpstreamChunk, WireError } from "../types.ts";
+import { encodeSseData, SSE_DONE_BYTES } from "../../util/sse.ts";
+import { renderErrorEnvelope } from "./errors.ts";
+
+/**
+ * The four observability fields every routed response carries. Streaming
+ * responses cannot use real headers (they flush with the first chunk, long
+ * before the TurnSummary exists), so the streaming sink emits these in a final
+ * `x_auto_model_router` SSE frame instead — see createStreamingSink.finish.
+ */
+function summaryFields(summary: TurnSummary): Record<string, unknown> {
+	return {
+		model: summary.servedSlug,
+		tier: summary.tier,
+		cost_usd: summary.reportedUsd ?? summary.predictedUsd,
+		attempts: summary.attempts,
+	};
+}
+
+export function createStreamingSink(virtualModel: string): { sink: ResponseSink; response: Response } {
+	let controller: ReadableStreamDefaultController<Uint8Array> | null = null;
+	let closed = false;
+	const body = new ReadableStream<Uint8Array>({
+		start(c) {
+			controller = c;
+		},
+	});
+	const send = (bytes: Uint8Array): void => {
+		if (!closed) controller?.enqueue(bytes);
+	};
+	const close = (): void => {
+		if (!closed) {
+			closed = true;
+			controller?.close();
+		}
+	};
+
+	const sink: ResponseSink = {
+		chunk(chunk: UpstreamChunk) {
+			const raw = chunk.raw;
+			// The client asked for the virtual id and must see it, so its own
+			// bookkeeping stays consistent; the served slug stays observable
+			// under x_auto_model_router. The tier is only known at finish time, so
+			// per-chunk frames carry the slug and the final frame carries all
+			// summary fields.
+			send(
+				encodeSseData({
+					...raw,
+					model: virtualModel,
+					x_auto_model_router: { model: typeof raw.model === "string" ? raw.model : null },
+				}),
+			);
+		},
+		error(error: WireError) {
+			// Headers (and likely chunks) are already on the wire; the only
+			// channel left for the failure is one SSE frame in the OpenAI error
+			// envelope, then an early close.
+			send(encodeSseData(renderErrorEnvelope(error)));
+			close();
+		},
+		finish(summary: TurnSummary) {
+			// Response headers flushed with the first chunk, so x-auto-model-router-*
+			// cannot be real headers here; this final frame is their carrier.
+			send(encodeSseData({ x_auto_model_router: summaryFields(summary) }));
+			send(SSE_DONE_BYTES);
+			close();
+		},
+	};
+
+	const response = new Response(body, {
+		status: 200,
+		headers: {
+			"content-type": "text/event-stream",
+			"cache-control": "no-cache",
+			connection: "keep-alive",
+		},
+	});
+	return { sink, response };
+}
+
+/** Aggregation state for one completion choice. */
+interface ChoiceAggregate {
+	role: string;
+	text: string;
+	reasoning: string;
+	toolCalls: Map<number, { id: string; name: string; args: string }>;
+	finishReason: string | null;
+}
+
+export function createBufferedSink(virtualModel: string): {
+	sink: ResponseSink;
+	response: Promise<Response>;
+} {
+	let resolveResponse!: (r: Response) => void;
+	const response = new Promise<Response>((resolve) => {
+		resolveResponse = resolve;
+	});
+	let settled = false;
+
+	let id: string | null = null;
+	let created: number | null = null;
+	const choices = new Map<number, ChoiceAggregate>();
+	let usage: unknown = null;
+
+	const sink: ResponseSink = {
+		chunk(chunk: UpstreamChunk) {
+			const raw = chunk.raw;
+			if (typeof raw.id === "string") id = raw.id;
+			if (typeof raw.created === "number") created = raw.created;
+			if (Array.isArray(raw.choices)) {
+				for (const c of raw.choices) {
+					if (typeof c !== "object" || c === null) continue;
+					const ch = c as { index?: unknown; delta?: unknown; finish_reason?: unknown };
+					const index = typeof ch.index === "number" ? ch.index : 0;
+					let agg = choices.get(index);
+					if (!agg) {
+						agg = { role: "assistant", text: "", reasoning: "", toolCalls: new Map(), finishReason: null };
+						choices.set(index, agg);
+					}
+					if (typeof ch.delta === "object" && ch.delta !== null) {
+						const delta = ch.delta as {
+							role?: unknown;
+							content?: unknown;
+							reasoning?: unknown;
+							reasoning_content?: unknown;
+							tool_calls?: unknown;
+						};
+						if (typeof delta.role === "string") agg.role = delta.role;
+						if (typeof delta.content === "string") agg.text += delta.content;
+						// Upstreams disagree on the spelling; both land in `reasoning`.
+						if (typeof delta.reasoning === "string") agg.reasoning += delta.reasoning;
+						else if (typeof delta.reasoning_content === "string") agg.reasoning += delta.reasoning_content;
+						if (Array.isArray(delta.tool_calls)) {
+							for (const tc of delta.tool_calls) {
+								if (typeof tc !== "object" || tc === null) continue;
+								const t = tc as { index?: unknown; id?: unknown; function?: unknown };
+								const ti = typeof t.index === "number" ? t.index : 0;
+								let ta = agg.toolCalls.get(ti);
+								if (!ta) {
+									ta = { id: "", name: "", args: "" };
+									agg.toolCalls.set(ti, ta);
+								}
+								if (typeof t.id === "string" && t.id.length > 0 && ta.id.length === 0) ta.id = t.id;
+								if (typeof t.function === "object" && t.function !== null) {
+									const fn = t.function as { name?: unknown; arguments?: unknown };
+									if (typeof fn.name === "string" && fn.name.length > 0 && ta.name.length === 0) {
+										ta.name = fn.name;
+									}
+									if (typeof fn.arguments === "string") ta.args += fn.arguments;
+								}
+							}
+						}
+					}
+					if (typeof ch.finish_reason === "string") agg.finishReason = ch.finish_reason;
+				}
+			}
+			if (typeof raw.usage === "object" && raw.usage !== null) usage = raw.usage;
+		},
+		error(error: WireError) {
+			if (settled) return;
+			settled = true;
+			resolveResponse(
+				new Response(JSON.stringify(renderErrorEnvelope(error)), {
+					status: error.status,
+					headers: { "content-type": "application/json" },
+				}),
+			);
+		},
+		finish(summary: TurnSummary) {
+			if (settled) return;
+			settled = true;
+			if (choices.size === 0) {
+				choices.set(0, { role: "assistant", text: "", reasoning: "", toolCalls: new Map(), finishReason: null });
+			}
+			const completion: Record<string, unknown> = {
+				id: id ?? "chatcmpl-auto-model-router",
+				object: "chat.completion",
+				created: created ?? Math.floor(Date.now() / 1000),
+				model: virtualModel,
+				choices: [...choices.entries()]
+					.sort((a, b) => a[0] - b[0])
+					.map(([index, agg]) => {
+						const message: Record<string, unknown> = { role: agg.role, content: agg.text };
+						if (agg.reasoning.length > 0) message.reasoning = agg.reasoning;
+						if (agg.toolCalls.size > 0) {
+							message.tool_calls = [...agg.toolCalls.entries()]
+								.sort((a, b) => a[0] - b[0])
+								.map(([, ta]) => ({
+									id: ta.id,
+									type: "function",
+									function: { name: ta.name, arguments: ta.args },
+								}));
+						}
+						return { index, message, finish_reason: agg.finishReason ?? "stop" };
+					}),
+			};
+			if (usage !== null) completion.usage = usage;
+			// Header names are hyphenated by HTTP convention while the SSE frame
+			// keys are snake_case by JSON convention. Deriving one from the other
+			// silently produced `x-auto-model-router-cost_usd`.
+			const headers: Record<string, string> = {
+				"content-type": "application/json",
+				"x-auto-model-router-model": summary.servedSlug,
+				"x-auto-model-router-tier": summary.tier,
+				"x-auto-model-router-cost-usd": String(summary.reportedUsd ?? summary.predictedUsd),
+				"x-auto-model-router-attempts": String(summary.attempts),
+			};
+			resolveResponse(new Response(JSON.stringify(completion), { status: 200, headers }));
+		},
+	};
+
+	return { sink, response };
+}

package/src/wire/types.ts

@@ -0,0 +1,156 @@
+/**
+ * Protocol-agnostic boundary between a client-facing wire (OpenAI chat
+ * completions today, pi-native later) and the routing core.
+ *
+ * The core never parses a wire format. A front end produces a `NormRequest`
+ * and consumes `UpstreamChunk`s through a `ResponseSink`. Anything the core
+ * does not understand rides along in `renderUpstreamBody()` output and in
+ * `UpstreamChunk.raw`, so unknown fields survive the round trip untouched.
+ */
+
+import type { UsageCounts } from "../cost/types.ts";
+
+export type WireProtocol = "openai-chat" | "pi-native";
+
+export type Role = "system" | "developer" | "user" | "assistant" | "tool";
+
+/** One tool call requested by an assistant turn. */
+export interface NormToolCall {
+	id: string;
+	name: string;
+	/** Raw JSON argument text as the model emitted it (may be invalid JSON). */
+	argsJson: string;
+}
+
+/**
+ * A conversation message flattened for feature extraction.
+ *
+ * `text` is a lossy concatenation used only for classification. Nothing is
+ * ever dispatched from it — the wire's own `renderUpstreamBody()` owns the
+ * bytes that reach OpenRouter.
+ */
+export interface NormMessage {
+	role: Role;
+	text: string;
+	/** Number of image parts on this message. */
+	images: number;
+	/** Bytes of text content, cheaper than recounting. */
+	textBytes: number;
+	toolCalls: NormToolCall[];
+	/** Set when `role === "tool"`; links back to the assistant call. */
+	toolCallId?: string;
+	/** Name of the tool, for `role === "tool"` messages that carry it. */
+	toolName?: string;
+}
+
+/** A tool exposed to the model, sized for prompt-cost accounting. */
+export interface NormTool {
+	name: string;
+	description: string;
+	/** Serialized byte length of the JSON schema. Tool schemas dominate omp prompts. */
+	schemaBytes: number;
+}
+
+export type ReasoningLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
+
+export interface NormRequest {
+	protocol: WireProtocol;
+	/**
+	 * Stable conversation identity: sha256 over the system prompt plus the
+	 * first non-system message. Matches how OpenRouter fingerprints
+	 * conversations, so our `session_id` and their implicit key agree.
+	 */
+	conversationKey: string;
+	/**
+	 * Harness/session identifier from the `X-Omp-Harness` request header, when
+	 * the client sends one. Lets multiple coding harnesses share one router
+	 * while keeping per-harness daily budgets and toast scoping. Empty when the
+	 * client sends no header (single-harness default).
+	 */
+	harnessId: string;
+	/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
+	requestedModel: string;
+	messages: NormMessage[];
+	tools: NormTool[];
+	/** True when the client forced a specific tool. */
+	forcedToolChoice: boolean;
+	stream: boolean;
+	maxTokens?: number;
+	temperature?: number;
+	reasoning?: ReasoningLevel;
+	hasImages: boolean;
+	/** Total prompt bytes across messages, system prompt, and tool schemas. */
+	promptBytes: number;
+	/**
+	 * Renders the body to POST to OpenRouter for a chosen model. The core passes
+	 * mutations it computed; the wire owns serialization so unknown client
+	 * fields pass through verbatim.
+	 */
+	renderUpstreamBody(m: UpstreamMutations): Record<string, unknown>;
+}
+
+/** Core-computed changes the wire must apply when rendering the upstream body. */
+export interface UpstreamMutations {
+	/** Concrete slug to dispatch to. Replaces the virtual model. */
+	slug: string;
+	/** Same-tier fallbacks for OpenRouter's `models[]` array. */
+	fallbacks: string[];
+	/** Forwarded as `session_id` to pin provider stickiness and group logs. */
+	sessionId: string;
+	/** Cache breakpoints to inject, as message indices. Empty ⇒ inject none. */
+	cacheBreakpointMessageIndices: number[];
+	/** Effective reasoning level, after clamping to what the target supports. */
+	reasoning: ReasoningLevel | undefined;
+	/** Clamp for the target's published completion ceiling. */
+	maxTokens: number | undefined;
+	/** Drop assistant reasoning-replay fields the target rejects. */
+	stripAssistantReasoning: boolean;
+}
+
+export type FinishReason = "stop" | "length" | "tool_calls" | "content_filter" | "error";
+
+/** Interpreted view of one upstream SSE chunk. */
+export type StreamEvent =
+	| { type: "start"; servedSlug: string; generationId: string | null }
+	| { type: "text"; delta: string }
+	| { type: "reasoning"; delta: string }
+	| { type: "tool_call"; index: number; id?: string; name?: string; argsDelta?: string }
+	| { type: "finish"; reason: FinishReason }
+	| { type: "usage"; usage: UsageCounts; reportedCostUsd: number | null };
+
+/**
+ * One upstream chunk, carried as both raw bytes and interpreted events.
+ *
+ * The escalation guard reads `events`; the wire forwards `raw` (with `model`
+ * rewritten). Keeping both means interpretation gaps never drop client-visible
+ * fields.
+ */
+export interface UpstreamChunk {
+	raw: Record<string, unknown>;
+	events: StreamEvent[];
+}
+
+export interface WireError {
+	status: number;
+	code: string;
+	message: string;
+}
+
+/** Outcome of a fully-resolved turn, after any escalation retries. */
+export interface TurnSummary {
+	servedSlug: string;
+	tier: string;
+	attempts: number;
+	predictedUsd: number;
+	reportedUsd: number | null;
+	usage: UsageCounts;
+	reasons: string[];
+	escalated: boolean;
+}
+
+/** Client-facing sink. A front end renders these to its own wire format. */
+export interface ResponseSink {
+	chunk(chunk: UpstreamChunk): void | Promise<void>;
+	error(error: WireError): void | Promise<void>;
+	finish(summary: TurnSummary): void | Promise<void>;
+}