auto-model-router 0.1.0

package/src/router/state.ts

@@ -0,0 +1,118 @@
+/**
+ * Per-conversation routing memory.
+ *
+ * Persisted rather than in-memory so hysteresis and cache-warmth tracking
+ * survive a restart: an omp session outlives this process, and forgetting
+ * which model is warm would cold-start a paid prompt cache for no reason.
+ *
+ * The `conversations` table is created by `util/sqlite.ts`, the single
+ * migration path.
+ */
+
+import type { Database, Statement } from "bun:sqlite";
+
+import type { ConversationState, ConversationStore, Tier } from "./types.ts";
+
+/** Row shape as stored; column names are snake_case per the schema. */
+interface Row {
+	key: string;
+	session_id: string;
+	turn: number;
+	current_slug: string | null;
+	current_tier: string | null;
+	sticky_until_turn: number;
+	escalations: number;
+	spent_usd: number;
+	last_prompt_tokens: number;
+	cache_warm_slug: string | null;
+	cache_warm_at_ms: number;
+	updated_at_ms: number;
+}
+
+function toState(row: Row): ConversationState {
+	return {
+		key: row.key,
+		sessionId: row.session_id,
+		turn: row.turn,
+		currentSlug: row.current_slug,
+		// Stored as free text; the column is only ever written from a Tier.
+		currentTier: row.current_tier as Tier | null,
+		stickyUntilTurn: row.sticky_until_turn,
+		escalations: row.escalations,
+		spentUsd: row.spent_usd,
+		lastPromptTokens: row.last_prompt_tokens,
+		cacheWarmSlug: row.cache_warm_slug,
+		cacheWarmAtMs: row.cache_warm_at_ms,
+		updatedAtMs: row.updated_at_ms,
+	};
+}
+
+export function createConversationStore(db: Database): ConversationStore {
+	// Hoisted: this runs on every turn, twice when an escalation retries.
+	const selectOne: Statement<Row, [string]> = db.query("SELECT * FROM conversations WHERE key = ?");
+	const insertOne: Statement<unknown, [string, string, number]> = db.query(
+		"INSERT INTO conversations (key, session_id, updated_at_ms) VALUES (?, ?, ?)",
+	);
+	const upsert = db.query(`
+		INSERT INTO conversations (
+			key, session_id, turn, current_slug, current_tier, sticky_until_turn,
+			escalations, spent_usd, last_prompt_tokens, cache_warm_slug, cache_warm_at_ms, updated_at_ms
+		) VALUES ($key, $sessionId, $turn, $currentSlug, $currentTier, $stickyUntilTurn,
+			$escalations, $spentUsd, $lastPromptTokens, $cacheWarmSlug, $cacheWarmAtMs, $updatedAtMs)
+		ON CONFLICT(key) DO UPDATE SET
+			session_id = excluded.session_id,
+			turn = excluded.turn,
+			current_slug = excluded.current_slug,
+			current_tier = excluded.current_tier,
+			sticky_until_turn = excluded.sticky_until_turn,
+			escalations = excluded.escalations,
+			spent_usd = excluded.spent_usd,
+			last_prompt_tokens = excluded.last_prompt_tokens,
+			cache_warm_slug = excluded.cache_warm_slug,
+			cache_warm_at_ms = excluded.cache_warm_at_ms,
+			updated_at_ms = excluded.updated_at_ms
+	`);
+	const deleteStale: Statement<unknown, [number]> = db.query("DELETE FROM conversations WHERE updated_at_ms < ?");
+
+	return {
+		get(key) {
+			const row = selectOne.get(key);
+			return row === null ? null : toState(row);
+		},
+
+		load(key) {
+			const existing = selectOne.get(key);
+			if (existing !== null) return toState(existing);
+			// Session id is derived, not random, so it stays stable if this row is
+			// ever pruned and the same conversation continues afterwards.
+			const sessionId = `omp-${key}`;
+			insertOne.run(key, sessionId, Date.now());
+			const inserted = selectOne.get(key);
+			if (inserted === null) throw new Error(`conversation row vanished immediately after insert: ${key}`);
+			return toState(inserted);
+		},
+
+		save(state) {
+			// bun:sqlite matches named parameters by their literal `$name` key;
+			// bare keys bind nothing at all and every column silently lands NULL.
+			upsert.run({
+				$key: state.key,
+				$sessionId: state.sessionId,
+				$turn: state.turn,
+				$currentSlug: state.currentSlug,
+				$currentTier: state.currentTier,
+				$stickyUntilTurn: state.stickyUntilTurn,
+				$escalations: state.escalations,
+				$spentUsd: state.spentUsd,
+				$lastPromptTokens: state.lastPromptTokens,
+				$cacheWarmSlug: state.cacheWarmSlug,
+				$cacheWarmAtMs: state.cacheWarmAtMs,
+				$updatedAtMs: Date.now(),
+			});
+		},
+
+		prune(maxAgeMs) {
+			return deleteStale.run(Date.now() - maxAgeMs).changes;
+		},
+	};
+}

package/src/router/tier-plan.ts

@@ -0,0 +1,151 @@
+/**
+ * Adaptive tier floors: derive each tier's quality floor from the models that
+ * are ACTUALLY available, recomputed whenever a catalog refresh installs a new
+ * snapshot.
+ *
+ * Why this exists. The configured floors (trivial 0, simple 40, moderate 60,
+ * hard 72) are absolute points on the Artificial Analysis index, tuned against
+ * the full ~420-model catalog. An OpenRouter key's guardrail can narrow the
+ * available set to a handful of models that all sit below those points, and an
+ * absolute floor then admits NOTHING: every tier above `trivial` goes
+ * permanently empty, selection widens down, and the router is trapped serving
+ * the cheapest model for every turn regardless of how hard the work is.
+ *
+ * The fix is to treat the floors as relative when the absolute ones cannot be
+ * met. Rank the available scored models, split them into four quantile bands,
+ * and take each band's lower bound as that tier's adaptive floor. The effective
+ * floor is then `min(configured, adaptive)`:
+ *
+ *  - A healthy catalog keeps the configured floors verbatim (the adaptive floor
+ *    sits above them, so `min` picks the configured value) — no behaviour change.
+ *  - A narrowed catalog falls back to the adaptive floor, so `hard` still gets
+ *    the best quartile of what is available instead of nothing at all.
+ *
+ * `min` is deliberate: adaptive floors may only RELAX a floor, never tighten
+ * one. Tightening would let a rich catalog silently price us out of a tier the
+ * operator explicitly configured.
+ *
+ * Unscored models are never imputed a score (see `candidates.ts`), so a catalog
+ * with no benchmarks at all yields all-zero floors: every tier admits every
+ * model and the price ceiling plus `qualityExponent` do the differentiating.
+ * That is the honest degradation — there is genuinely no measured quality
+ * spread to rank on.
+ */
+
+import type { CatalogModel, CatalogSnapshot } from "../catalog/types.ts";
+import type { QualityAxis, RouterConfig } from "../config/types.ts";
+import { TIER_ORDER, type Tier } from "./types.ts";
+
+const AXES: readonly QualityAxis[] = ["coding", "agentic", "intelligence"];
+
+/** Adaptive floor per tier, for one quality axis. */
+export type AxisFloors = Record<Tier, number>;
+
+export interface TierPlan {
+	/** Adaptive floor per axis per tier. */
+	floors: Record<QualityAxis, AxisFloors>;
+	/** How many available models carried a score on each axis. */
+	scoredCount: Record<QualityAxis, number>;
+}
+
+/**
+ * Models that could plausibly serve a turn, for ranking purposes: the built-in
+ * denials (floating aliases, batch endpoints, stealth, meta-routers) and free
+ * models would otherwise skew the quantiles with entries selection can never
+ * pick. This mirrors the built-in denials in `buildCandidates`; per-request
+ * filters (tools, context, images) are deliberately NOT applied, because the
+ * plan is computed once per refresh, not once per request.
+ */
+function isRankable(model: CatalogModel, includeFree: boolean): boolean {
+	const slug = model.slug;
+	if (slug.startsWith("~") || slug.endsWith(":batch") || slug.startsWith("stealth/")) return false;
+	if (model.author === "openrouter") return false;
+	if (model.price.prompt < 0 || model.price.completion < 0) return false;
+	if (model.isFree && !includeFree) return false;
+	return true;
+}
+
+/**
+ * Quantile band lower bounds over an ascending score list, one per tier.
+ *
+ * Tier k of n takes the score at index `floor(len * k / n)`. With four tiers
+ * that is the min, p25, p50 and p75, so each tier's floor admits roughly the
+ * top `(n-k)/n` of the available models — every tier non-empty by construction
+ * whenever at least one model is scored.
+ */
+function bandFloors(ascending: readonly number[]): AxisFloors {
+	const floors: Record<string, number> = {};
+	const len = ascending.length;
+	for (let k = 0; k < TIER_ORDER.length; k++) {
+		const tier = TIER_ORDER[k];
+		if (tier === undefined) continue;
+		if (len === 0) {
+			floors[tier] = 0;
+			continue;
+		}
+		const index = Math.min(len - 1, Math.floor((len * k) / TIER_ORDER.length));
+		floors[tier] = ascending[index] ?? 0;
+	}
+	return floors as AxisFloors;
+}
+
+/** Computes the adaptive plan for a set of available models. */
+export function computeTierPlan(models: readonly CatalogModel[], cfg: RouterConfig): TierPlan {
+	const floors: Record<string, AxisFloors> = {};
+	const scoredCount: Record<string, number> = {};
+	const includeFree = cfg.filters.includeFree;
+
+	for (const axis of AXES) {
+		const scores: number[] = [];
+		for (const model of models) {
+			if (!isRankable(model, includeFree)) continue;
+			const score = model.quality[axis];
+			if (score === undefined) continue;
+			scores.push(score);
+		}
+		scores.sort((a, b) => a - b);
+		floors[axis] = bandFloors(scores);
+		scoredCount[axis] = scores.length;
+	}
+
+	return {
+		floors: floors as Record<QualityAxis, AxisFloors>,
+		scoredCount: scoredCount as Record<QualityAxis, number>,
+	};
+}
+
+/**
+ * Memoized per snapshot object AND config object. A catalog refresh installs a
+ * fresh `CatalogSnapshot`, which misses the cache and recomputes the plan — so
+ * the plan tracks availability on exactly the polling interval, with no timer
+ * of its own. Keyed by config too so separate routers (and tests) never share.
+ */
+const planCache = new WeakMap<CatalogSnapshot, WeakMap<RouterConfig, TierPlan>>();
+
+export function tierPlanFor(snapshot: CatalogSnapshot, cfg: RouterConfig): TierPlan {
+	let perConfig = planCache.get(snapshot);
+	if (perConfig === undefined) {
+		perConfig = new WeakMap<RouterConfig, TierPlan>();
+		planCache.set(snapshot, perConfig);
+	}
+	let plan = perConfig.get(cfg);
+	if (plan === undefined) {
+		plan = computeTierPlan(snapshot.models, cfg);
+		perConfig.set(cfg, plan);
+	}
+	return plan;
+}
+
+/**
+ * The floor to actually enforce for a tier on an axis. Never tightens the
+ * configured floor; only relaxes it when the available catalog cannot meet it.
+ */
+export function effectiveQualityFloor(
+	configured: number,
+	tier: Tier,
+	axis: QualityAxis,
+	plan: TierPlan,
+): number {
+	const adaptive = plan.floors[axis][tier];
+	return Math.min(configured, adaptive);
+}

package/src/router/types.ts

@@ -0,0 +1,222 @@
+/**
+ * Routing brain contracts: features -> complexity tier -> concrete model.
+ *
+ * Every stage is a pure function of explicit inputs so decisions are
+ * reproducible and `auto-model-router explain` can replay them offline.
+ */
+
+import type { CatalogModel } from "../catalog/types.ts";
+import type { CostForecast } from "../cost/types.ts";
+import type { NormRequest, ReasoningLevel } from "../wire/types.ts";
+
+export type Tier = "trivial" | "simple" | "moderate" | "hard";
+
+export const TIER_ORDER: readonly Tier[] = ["trivial", "simple", "moderate", "hard"] as const;
+
+/**
+ * Task type: the KIND of work, orthogonal to complexity tier. A vision task
+ * routes to the best vision-capable model even if the tier would otherwise
+ * pick a cheaper one; a documentation task stays cheap. Classified from
+ * Features with no tokenizer or model call.
+ */
+export type TaskType = "coding" | "vision" | "documentation" | "data" | "chat";
+
+export const TASK_ORDER: readonly TaskType[] = ["coding", "vision", "documentation", "data", "chat"] as const;
+
+/**
+ * Signals extracted from a request. Deliberately cheap: no tokenizer, no
+ * network, no model call. Field names are stable because they are logged
+ * verbatim into the ledger for later calibration.
+ */
+export interface Features {
+	/** Estimated prompt tokens (see `tokens/estimate.ts`). */
+	promptTokens: number;
+	/** Estimated tokens in the newest user-authored content only. */
+	newContentTokens: number;
+	/** Assistant + user turns in history (excludes system). */
+	turnDepth: number;
+	/** Tools offered. omp exposes ~15-25; a bare chat request offers none. */
+	toolCount: number;
+	/** Total bytes of tool JSON schemas — usually the largest prompt component. */
+	toolSchemaBytes: number;
+	/**
+	 * The last message is a tool result, i.e. this is a mechanical continuation
+	 * of an agent loop rather than fresh human intent. The single strongest
+	 * cheap-routing signal in agent traffic.
+	 */
+	isToolResultContinuation: boolean;
+	/** Consecutive tool-result messages at the tail — loop depth. */
+	toolLoopDepth: number;
+	/** Distinct tool names used across the conversation. */
+	distinctToolsUsed: number;
+	/** A tool result at the tail reports an error or non-zero exit. */
+	lastToolFailed: boolean;
+	/** The same tool was called with identical arguments twice in a row. */
+	repeatedToolCall: boolean;
+	hasImages: boolean;
+	/** Fenced code blocks in the newest user content. */
+	codeBlocks: number;
+	/** Bytes inside fenced code blocks in the newest user content. */
+	codeBytes: number;
+	/** Diff/patch markers in the newest user content. */
+	looksLikeDiff: boolean;
+	/** Matched complexity keywords (architecture, debug, why, race, optimize, ...). */
+	complexityKeywords: string[];
+	/** Matched triviality keywords (rename, format, typo, bump, ...). */
+	trivialityKeywords: string[];
+	/** Client asked for reasoning, a direct statement of expected difficulty. */
+	requestedReasoning: ReasoningLevel | undefined;
+	/** Question marks in the newest user content. */
+	questionCount: number;
+	/** Newest user content is a single short imperative sentence. */
+	isTerseInstruction: boolean;
+}
+
+export type ClassificationSource = "heuristic" | "llm" | "sticky" | "forced" | "escalation";
+
+export interface Classification {
+	tier: Tier;
+	/** Task type: the kind of work, orthogonal to complexity. */
+	task: TaskType;
+	/** 0-1. Below the config's ambiguity band, the LLM adjudicator is consulted. */
+	confidence: number;
+	source: ClassificationSource;
+	/** Ordered, human-readable justification. Logged and surfaced by `explain`. */
+	reasons: string[];
+	/** Raw heuristic score before tier bucketing, 0-1. */
+	score: number;
+}
+
+/** A model that survived capability filtering, with its economics attached. */
+export interface Candidate {
+	model: CatalogModel;
+	forecast: CostForecast;
+	/** Quality on the axis chosen for this request (coding/agentic/intelligence), 0-100. */
+	qualityScore: number;
+	/** Laplace-smoothed success rate from our ledger, 0-1. Defaults to a neutral prior. */
+	trustScore: number;
+	/** Final ranking score. Higher wins. */
+	score: number;
+	/** Why this candidate ranked where it did. */
+	reasons: string[];
+}
+
+export type RejectionReason =
+	| "no_tool_support"
+	| "context_too_small"
+	| "no_image_support"
+	| "below_quality_floor"
+	| "over_price_ceiling"
+	| "over_budget"
+	| "denylisted"
+	| "not_allowlisted"
+	| "free_tier_excluded"
+	| "reasoning_mandatory"
+	| "untrusted"
+	/** Already failed on this turn; excluded so failover picks a different model. */
+	| "failed_this_turn";
+
+export interface Rejection {
+	slug: string;
+	reason: RejectionReason;
+	detail?: string;
+}
+
+/** Per-conversation routing memory. Persisted so restarts do not reset hysteresis. */
+export interface ConversationState {
+	key: string;
+	/** Forwarded to OpenRouter as `session_id`. */
+	sessionId: string;
+	turn: number;
+	/** Slug that served the previous committed turn. */
+	currentSlug: string | null;
+	currentTier: Tier | null;
+	/**
+	 * Hold the current tier until this turn index, to stop per-turn flapping
+	 * that would repeatedly cold-start prompt caches.
+	 */
+	stickyUntilTurn: number;
+	escalations: number;
+	spentUsd: number;
+	/** Prompt tokens on the previous turn, for cache-warmth arithmetic. */
+	lastPromptTokens: number;
+	/** Model whose prompt cache we believe is still warm. */
+	cacheWarmSlug: string | null;
+	/** When that cache was last touched; OpenRouter sticky sessions expire in 5-10 min. */
+	cacheWarmAtMs: number;
+	updatedAtMs: number;
+}
+
+export interface ConversationStore {
+	get(key: string): ConversationState | null;
+	/** Loads existing state or creates a fresh record. */
+	load(key: string): ConversationState;
+	save(state: ConversationState): void;
+	/** Drops records untouched for longer than `maxAgeMs`. */
+	prune(maxAgeMs: number): number;
+}
+
+/** Guarded-probe configuration for one dispatch. */
+export interface ProbePlan {
+	enabled: boolean;
+	/** Hold output until this many text tokens have arrived. */
+	maxTokens: number;
+	/** Hard ceiling on hold time so a slow model cannot stall the client. */
+	maxHoldMs: number;
+	/** Tier to escalate to when the probe rejects the attempt. */
+	escalateTo: Tier | null;
+}
+
+/** The routing decision for one turn. */
+export interface Decision {
+	slug: string;
+	/** Same-tier fallbacks for OpenRouter's `models[]` array; transient-error only. */
+	fallbacks: string[];
+	tier: Tier;
+	classification: Classification;
+	/** The extracted feature vector, retained verbatim for `explain`. */
+	features: Features;
+	forecast: CostForecast;
+	sessionId: string;
+	/** Reused the previous turn's model because switching was not worth the cache loss. */
+	sticky: boolean;
+	/** Message indices to mark with cache breakpoints. */
+	cacheBreakpointMessageIndices: number[];
+	reasoning: ReasoningLevel | undefined;
+	maxTokens: number | undefined;
+	stripAssistantReasoning: boolean;
+	probe: ProbePlan;
+	/** Candidates considered, ranked. Retained for `explain`. */
+	considered: Candidate[];
+	/** Filtered-out models with cause. Retained for `explain`. */
+	rejected: Rejection[];
+	reasons: string[];
+	/** Budget guard forced a cheaper tier than the classifier asked for. */
+	budgetDowngraded: boolean;
+}
+
+/** Why a guarded probe rejected an attempt. */
+export type EscalationSignal =
+	| "malformed_tool_args"
+	| "refusal"
+	| "empty_completion"
+	| "repeat_tool_call"
+	| "length_stop"
+	| "missing_expected_tool_call"
+	| "upstream_error";
+
+export type ProbeVerdict =
+	| { action: "commit"; reason: string }
+	| { action: "escalate"; signal: EscalationSignal; reason: string };
+
+export interface Router {
+	/**
+	 * Chooses a model for a request. Pure w.r.t. everything except the stores it
+	 * reads. `excludeSlugs` removes models that already failed on this turn, so
+	 * a failover retry cannot re-pick the slug that just errored.
+	 */
+	route(
+		req: NormRequest,
+		opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[] },
+	): Promise<Decision>;
+}