auto-model-router 0.1.3 → 0.2.0

package/src/config/types.ts

@@ -56,6 +56,42 @@ export interface OpenRouterConfig {
 	catalogRefreshMs: number;
 }
 
+/**
+ * External benchmark feeds that BACKFILL quality scores OpenRouter does not
+ * publish. OpenRouter embeds Artificial Analysis scores for the models it has,
+ * but returns many (GLM, MiniMax, smaller vendors) unscored — which strands
+ * them below every tier floor above `trivial`. These feeds fill only the axes
+ * a model is missing; a score OpenRouter already published is never overwritten.
+ *
+ * Refreshed on their OWN slow cadence (`refreshMs`, ~daily), independent of the
+ * catalog's minute-scale availability refresh, and cached in `benchmark_cache`.
+ * Every fetch is best-effort: a feed failure leaves the catalog on published
+ * scores rather than failing a refresh.
+ */
+export interface BenchmarksConfig {
+	/** Master switch. Off ⇒ the catalog carries only OpenRouter-published scores. */
+	enabled: boolean;
+	/**
+	 * Artificial Analysis API key (v2 data API). Resolved from config, then
+	 * `ARTIFICIAL_ANALYSIS_API_KEY`. Empty ⇒ the AA feed is skipped; BenchLM
+	 * (keyless) still runs.
+	 */
+	artificialAnalysisApiKey: string;
+	/** Pull the keyless BenchLM leaderboard, which covers models AA omits. */
+	benchlm: boolean;
+	/** Feed cache freshness, ms: re-fetch the feeds only when older than this. */
+	refreshMs: number;
+	/** Per-feed HTTP timeout, ms. */
+	timeoutMs: number;
+	/**
+	 * Apply calibrated scores from our own eval harness (`src/eval`, the
+	 * `local_scores` table) as a last-resort source. Off by default: local
+	 * scores change routing, so they stay inert until deliberately enabled —
+	 * e.g. after a data-collection window closes.
+	 */
+	useLocalScores: boolean;
+}
+
 /** Quality/price envelope for one complexity tier. */
 export interface TierConfig {
 	/**
@@ -163,6 +199,65 @@ export interface HysteresisConfig {
 	maxDowngradePerTurn: number;
 }
 
+/**
+ * Epsilon-greedy exploration: deliberately route a small fraction of turns
+ * one tier BELOW the classified tier, to learn whether the cheaper model
+ * would have sufficed.
+ *
+ * Without it the ledger only ever witnesses UNDER-routing: a tier that was
+ * too low escalates and is recorded, while over-routing stays invisible
+ * because the cheaper model was never run. Weights fit on that one-sided
+ * evidence can only ever ratchet toward more expensive routing.
+ */
+export interface ExplorationConfig {
+	/** Off by default: this deliberately degrades a slice of real turns. */
+	enabled: boolean;
+	/**
+	 * Per-tier sampling rate, 0-1. A tier that is absent, or set to 0, is
+	 * never explored. `trivial` is the floor and cannot drop, so a rate for
+	 * it has no effect.
+	 *
+	 * Rates are per-tier because the tiers are wildly unequal as evidence.
+	 * In one observed window 635 explorable turns were `simple` and 1 was
+	 * `hard`, while `hard` carried ~30% of all spend. A single uniform rate
+	 * therefore spends nearly the whole exploration budget on the cheapest
+	 * question in the system.
+	 */
+	rates: Partial<Record<Tier, number>>;
+	/**
+	 * Which hysteresis-held turns exploration may touch.
+	 *
+	 * `never`      the held population is untouchable.
+	 * `cold-cache` explore a hold only after its prompt cache has expired.
+	 * `always`     explore holds regardless, forfeiting a live cache read.
+	 *
+	 * This matters more than it sounds. ~95% of hard-tier spend arrives by
+	 * hold rather than by classification, so `never` confines exploration to
+	 * the cheapest boundary in the system. But held turns are consecutive
+	 * turns of an active loop and are therefore warm BY CONSTRUCTION, so
+	 * `cold-cache` barely reaches them either: on one real window it moved
+	 * explorable hard turns from 1 to 11. Reaching that population in any
+	 * useful volume means `always`, and paying the forfeited cache read --
+	 * a real cost, but a bounded and directly measurable one.
+	 */
+	stickyPolicy: "never" | "cold-cache" | "always";
+	/**
+	 * Randomise the POST-ESCALATION hold length per conversation, to learn
+	 * what it should be.
+	 *
+	 * `holdTurnsAfterEscalation` is a hand-picked constant that nothing has
+	 * ever validated, and it governs most expensive spend: a turn escalates
+	 * once, then the hold bills the next several turns at the escalated
+	 * tier. Assignment is per conversation, so each conversation is one
+	 * clean randomised arm rather than a confounded mixture.
+	 */
+	holdTurns: {
+		enabled: boolean;
+		/** Candidate hold lengths. One is drawn per conversation. */
+		values: number[];
+	};
+}
+
 export interface CacheConfig {
 	/**
 	 * Inject Anthropic-style `cache_control` breakpoints. OpenRouter translates
@@ -223,12 +318,14 @@ export interface LedgerConfig {
 export interface RouterConfig {
 	server: ServerConfig;
 	openrouter: OpenRouterConfig;
+	benchmarks: BenchmarksConfig;
 	tiers: Record<Tier, TierConfig>;
 	tasks: Record<TaskType, TaskConfig>;
 	filters: FilterConfig;
 	classifier: ClassifierConfig;
 	escalation: EscalationConfig;
 	hysteresis: HysteresisConfig;
+	exploration: ExplorationConfig;
 	cache: CacheConfig;
 	budget: BudgetConfig;
 	profiles: ProfileConfig[];
@@ -240,5 +337,14 @@ export interface RouterConfig {
 	 * `trivial` permanently empty and the router is stuck on the cheapest model.
 	 */
 	adaptiveTierFloors: boolean;
+	/**
+	 * Derive each tier's input-price ceiling from the price spread of the models
+	 * actually available at every catalog refresh (quantile bands), instead of
+	 * fixed `tiers.*.maxInputPerMtok` dollars. Lets the same config self-tune to
+	 * whatever models a key admits — a hard cap becomes "drop this catalog's
+	 * priciest outliers", not a magic dollar value. An explicit ceiling still
+	 * tightens further. Off by default (fixed ceilings).
+	 */
+	adaptivePriceCeilings: boolean;
 	logLevel: "silent" | "error" | "warn" | "info" | "debug";
 }

package/src/cost/ledger.ts

@@ -32,11 +32,19 @@ interface LedgerRow {
 	turn: number;
 	requested_model: string;
 	harness_id: string;
+	omp_session_id: string;
 	slug: string;
 	served_slug: string | null;
 	tier: string;
 	classification_source: string;
 	reasons: string;
+	features: string | null;
+	score: number | null;
+	confidence: number | null;
+	task: string | null;
+	classifier_reasons: string | null;
+	explored_from: string | null;
+	hold_arm: number | null;
 	predicted_usd: number;
 	reported_usd: number | null;
 	usage: string;
@@ -128,11 +136,19 @@ function toEntry(row: LedgerRow): LedgerEntry {
 		turn: row.turn,
 		requestedModel: row.requested_model,
 		harnessId: row.harness_id,
+		ompSessionId: row.omp_session_id,
 		slug: row.slug,
 		servedSlug: row.served_slug,
 		tier: row.tier,
 		classificationSource: row.classification_source,
 		reasons: JSON.parse(row.reasons) as string[],
+		features: row.features === null ? null : (JSON.parse(row.features) as object),
+		score: row.score,
+		confidence: row.confidence,
+		task: row.task,
+		classifierReasons: row.classifier_reasons === null ? null : (JSON.parse(row.classifier_reasons) as string[]),
+		exploredFrom: row.explored_from,
+		holdArm: row.hold_arm,
 		predictedUsd: row.predicted_usd,
 		reportedUsd: row.reported_usd,
 		usage: JSON.parse(row.usage) as UsageCounts,
@@ -151,11 +167,11 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 	// Prepared once: record() runs on every turn.
 	const insertStmt = db.query(
 		`INSERT INTO ledger (
-			id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, slug, served_slug,
+			id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, omp_session_id, slug, served_slug,
 			tier, classification_source, reasons, predicted_usd, reported_usd, usage, cost_breakdown,
 			attempt, escalation_signal, latency_ms, ttft_ms, finish_reason, wasted, upstream_generation_id, error,
-			error_kind
-		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			error_kind, features, score, confidence, task, classifier_reasons, explored_from, hold_arm
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 	);
 	const calibrationStmt = db.query(
 		`INSERT INTO token_calibration (tokenizer, est_bytes, actual_tokens, samples) VALUES (?, ?, ?, 1)
@@ -215,6 +231,7 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 				entry.turn,
 				entry.requestedModel,
 				entry.harnessId,
+				entry.ompSessionId,
 				entry.slug,
 				entry.servedSlug,
 				entry.tier,
@@ -233,6 +250,13 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
 				entry.upstreamGenerationId,
 				entry.error,
 				errorKindOf(entry.error),
+				entry.features === null ? null : JSON.stringify(entry.features),
+				entry.score,
+				entry.confidence,
+				entry.task,
+				entry.classifierReasons === null ? null : JSON.stringify(entry.classifierReasons),
+				entry.exploredFrom,
+				entry.holdArm,
 			);
 			// Always consume the pending estimate, even when the turn failed, so a
 			// dead turn's bytes can never pair with a later turn's tokens. Only

package/src/cost/types.ts

@@ -74,6 +74,12 @@ export interface LedgerEntry {
 	requestedModel: string;
 	/** Harness id from the request header; empty for the default harness. */
 	harnessId: string;
+	/**
+	 * omp UI session id from the `X-Omp-Session` request header; empty when the
+	 * client sends no header. Scopes toasts to a single interactive session so
+	 * concurrent sessions sharing one ledger don't surface each other's choices.
+	 */
+	ompSessionId: string;
 	/** Concrete slug we dispatched to. */
 	slug: string;
 	/** Slug that actually served it, per the response `model` field. */
@@ -82,6 +88,30 @@ export interface LedgerEntry {
 	classificationSource: string;
 	/** Human-readable decision trail. */
 	reasons: string[];
+	/**
+	 * Classifier inputs, persisted verbatim as JSON so any score is
+	 * recomputable offline. Opaque here on purpose: the ledger sits below the
+	 * router in the layering and must not import its types. NULL before v6.
+	 */
+	features: object | null;
+	/** Raw heuristic score, 0-1, before tier bucketing. NULL before v6. */
+	score: number | null;
+	/** Classifier confidence, 0-1. Drives adjudication. NULL before v6. */
+	confidence: number | null;
+	/** Task kind (coding, vision, ...), orthogonal to tier. NULL before v6. */
+	task: string | null;
+	/** Per-feature score breakdown, which the decision trail drops. NULL before v6. */
+	classifierReasons: string[] | null;
+	/**
+	 * Tier the classifier chose on a turn that exploration deliberately routed
+	 * one step cheaper. NULL when the turn was routed normally.
+	 */
+	exploredFrom: string | null;
+	/**
+	 * Hold-length arm this conversation was assigned by hold exploration,
+	 * or NULL when it was not part of that experiment.
+	 */
+	holdArm: number | null;
 	predictedUsd: number;
 	reportedUsd: number | null;
 	usage: UsageCounts;

package/src/eval/calibrate.ts

@@ -0,0 +1,131 @@
+/**
+ * Calibration: map raw suite scores (mean grade, 0-1) onto the Artificial
+ * Analysis 0-100 index the tier floors are defined against.
+ *
+ * A home-grown "0.72" is meaningless next to AA's "72" unless the two scales are
+ * tied together. We do that empirically: run the SAME suite on models AA has
+ * already scored (the anchors), fit raw -> AA per axis by least squares, then
+ * apply that line to the unscored targets. Without enough anchors on an axis we
+ * emit nothing for it — honest degradation, never an imputed number.
+ */
+
+import type { QualityAxis } from "../config/types.ts";
+import type { FeedScore } from "../catalog/benchmark-feeds.ts";
+import { normalizeModelKey } from "../catalog/benchmark-feeds.ts";
+import type { AxisScore, EvalResult } from "./run.ts";
+
+/** Minimum anchor models with a known AA score on an axis before we trust a fit. */
+export const MIN_ANCHORS = 3;
+
+/** Minimum Pearson correlation between raw suite scores and AA before a fit is trusted. */
+export const MIN_R = 0.5;
+
+export interface LineFit {
+	slope: number;
+	intercept: number;
+	/** Pearson correlation of the anchor fit, 0-1. A quality signal on the calibration itself. */
+	r: number;
+	/** Anchors used. */
+	n: number;
+}
+
+export type Calibration = Partial<Record<QualityAxis, LineFit>>;
+
+export interface AnchorPoint {
+	/** Raw mean grade on the axis, 0-1. */
+	raw: number;
+	/** Known AA index on the axis, 0-100. */
+	aa: number;
+}
+
+/**
+ * OLS fit, or null when the calibration cannot be trusted: too few points, no
+ * spread, a non-positive slope, or weak correlation. A suite that does not track
+ * AA positively and with real correlation would turn a target's score into noise
+ * dressed as signal, so we refuse it and emit nothing for that axis.
+ */
+export function fitAxis(points: readonly AnchorPoint[]): LineFit | null {
+	if (points.length < MIN_ANCHORS) return null;
+	const n = points.length;
+	let sx = 0;
+	let sy = 0;
+	for (const p of points) {
+		sx += p.raw;
+		sy += p.aa;
+	}
+	const mx = sx / n;
+	const my = sy / n;
+	let sxx = 0;
+	let syy = 0;
+	let sxy = 0;
+	for (const p of points) {
+		const dx = p.raw - mx;
+		const dy = p.aa - my;
+		sxx += dx * dx;
+		syy += dy * dy;
+		sxy += dx * dy;
+	}
+	// No spread on either axis ⇒ undefined slope or correlation.
+	if (sxx < 1e-9 || syy < 1e-9) return null;
+	const slope = sxy / sxx;
+	const r = sxy / Math.sqrt(sxx * syy);
+	// The suite must rank models the same way AA does, and meaningfully so.
+	if (slope <= 0 || r < MIN_R) return null;
+	return { slope, intercept: my - slope * mx, r, n };
+}
+
+export function applyFit(fit: LineFit, raw: number): number {
+	const y = fit.slope * raw + fit.intercept;
+	return Math.min(100, Math.max(0, y));
+}
+
+const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
+
+/**
+ * Fit every axis from the anchors' raw suite scores paired with their known AA
+ * scores. `anchorAa` supplies the AA index per slug+axis (absent ⇒ that anchor
+ * is not used on that axis).
+ */
+export function fitCalibration(
+	anchors: readonly EvalResult[],
+	anchorAa: (slug: string, axis: QualityAxis) => number | undefined,
+): Calibration {
+	const cal: Calibration = {};
+	for (const axis of AXES) {
+		const points: AnchorPoint[] = [];
+		for (const r of anchors) {
+			const raw = axisMean(r.axes[axis]);
+			const aa = anchorAa(r.slug, axis);
+			if (raw !== null && aa !== undefined) points.push({ raw, aa });
+		}
+		const fit = fitAxis(points);
+		if (fit !== null) cal[axis] = fit;
+	}
+	return cal;
+}
+
+function axisMean(a: AxisScore | undefined): number | null {
+	return a === undefined || a.n === 0 ? null : a.sum / a.n;
+}
+
+/** Calibrated local FeedScores for the targets, one axis at a time, skipping axes with no fit. */
+export function toLocalFeedScores(
+	targets: readonly EvalResult[],
+	cal: Calibration,
+	authorOf: (slug: string) => string,
+): FeedScore[] {
+	const out: FeedScore[] = [];
+	for (const r of targets) {
+		const entry: FeedScore = { key: normalizeModelKey(r.slug), creator: authorOf(r.slug), source: "local" };
+		let any = false;
+		for (const axis of AXES) {
+			const fit = cal[axis];
+			const raw = axisMean(r.axes[axis]);
+			if (fit === undefined || raw === null) continue;
+			entry[axis] = applyFit(fit, raw);
+			any = true;
+		}
+		if (any) out.push(entry);
+	}
+	return out;
+}

package/src/eval/grade.ts

@@ -0,0 +1,115 @@
+/**
+ * Objective, deterministic grading proxies over an assistant's TEXT reply.
+ *
+ * These measure whether a model can follow an instruction, emit valid
+ * structured output, and hit a checkable answer — a FLOOR on quality, which is
+ * exactly what a tier gate needs ("is it good enough for this tier"). They are
+ * not an absolute capability index, and deliberately use no LLM judge: every
+ * grade is reproducible from the text alone.
+ */
+
+const REFUSAL = /\b(?:i can(?:'|no)?t|i am unable|i'm unable|cannot help|as an ai)\b/i;
+
+export function isRefusalOrEmpty(output: string): boolean {
+	const t = output.trim();
+	return t.length === 0 || REFUSAL.test(t);
+}
+
+/** Lowercased, whitespace-collapsed. Reused by every text comparator here. */
+export function normalizeText(s: string): string {
+	return s.trim().toLowerCase().replace(/\s+/g, " ");
+}
+
+function lastLine(s: string): string {
+	const lines = s.split(/\r?\n/).map((l) => l.trim()).filter((l) => l.length > 0);
+	return lines.length === 0 ? "" : (lines[lines.length - 1] ?? "");
+}
+
+function escapeRegex(text: string): string {
+	return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * 1 when the expected answer is the whole reply, the last non-empty line, or a
+ * standalone token in it; else 0. Standalone so "9.9" does not match "19.99".
+ */
+export function answerScore(output: string, expected: string): number {
+	if (isRefusalOrEmpty(output)) return 0;
+	const exp = normalizeText(expected);
+	if (normalizeText(output) === exp) return 1;
+	if (normalizeText(lastLine(output)) === exp) return 1;
+	const re = new RegExp(`(?:^|[^\\w.])${escapeRegex(exp)}(?:$|[^\\w.])`);
+	return re.test(normalizeText(output)) ? 1 : 0;
+}
+
+/** Fraction of required tokens present, case-insensitive. Smooth partial credit. */
+export function tokenCoverage(output: string, tokens: readonly string[]): number {
+	if (isRefusalOrEmpty(output) || tokens.length === 0) return 0;
+	const hay = normalizeText(output);
+	let hit = 0;
+	for (const t of tokens) if (hay.includes(t.toLowerCase())) hit += 1;
+	return hit / tokens.length;
+}
+
+/**
+ * The first balanced JSON value in the text, tolerating ``` fences and prose
+ * around it. Scans by bracket depth with string/escape awareness so a brace
+ * inside a string literal does not throw off the balance.
+ */
+export function extractJson(output: string): unknown {
+	const fenced = output.match(/```(?:json)?\s*([\s\S]*?)```/i);
+	const body = fenced?.[1] ?? output;
+	const start = body.search(/[[{]/);
+	if (start === -1) return undefined;
+	const open = body[start];
+	const close = open === "{" ? "}" : "]";
+	let depth = 0;
+	let inStr = false;
+	let esc = false;
+	for (let i = start; i < body.length; i++) {
+		const ch = body[i];
+		if (inStr) {
+			if (esc) esc = false;
+			else if (ch === "\\") esc = true;
+			else if (ch === '"') inStr = false;
+			continue;
+		}
+		if (ch === '"') inStr = true;
+		else if (ch === open) depth += 1;
+		else if (ch === close) {
+			depth -= 1;
+			if (depth === 0) {
+				try {
+					return JSON.parse(body.slice(start, i + 1));
+				} catch {
+					return undefined;
+				}
+			}
+		}
+	}
+	return undefined;
+}
+
+/** A named field of a JSON object, or undefined. Guarded boundary cast, read is checked. */
+export function jsonField(value: unknown, key: string): unknown {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
+	// Narrowed to a non-null, non-array object; index as a string map at this boundary.
+	const rec = value as Record<string, unknown>;
+	return rec[key];
+}
+
+/**
+ * Fraction of expected answers present as standalone tokens anywhere in the
+ * reply. Partial credit — the point of these is to SPREAD models by how many
+ * parts they get right, where an all-or-nothing grade would pin everyone at 1.
+ */
+export function multiAnswerCoverage(output: string, expected: readonly string[]): number {
+	if (isRefusalOrEmpty(output) || expected.length === 0) return 0;
+	const hay = normalizeText(output);
+	let hit = 0;
+	for (const e of expected) {
+		const re = new RegExp(`(?:^|[^\\w.])${escapeRegex(normalizeText(e))}(?:$|[^\\w.])`);
+		if (re.test(hay)) hit += 1;
+	}
+	return hit / expected.length;
+}

package/src/eval/judge.ts

@@ -0,0 +1,71 @@
+/**
+ * LLM-judge grading for the open-ended tasks.
+ *
+ * Objective proxies pin frontier models at the ceiling; open-ended tasks graded
+ * by a strong model spread them. The judge is used ONLY to produce raw scores —
+ * whether those scores are trustworthy is decided downstream by the SAME
+ * calibration guard the objective path uses: if the judge's scores do not track
+ * AA on the anchors (Pearson r >= MIN_R, positive slope), no fit is emitted.
+ * So "validate the judge against AA" is not a separate step; it is enforced by
+ * refusing to ship an uncorrelated calibration.
+ *
+ * Absolute-rubric scoring (0-10), temperature 0, one judge model. A judge that
+ * errors or returns no parseable score yields null — treated as no observation,
+ * never a 0.
+ */
+
+import type { QualityAxis } from "../config/types.ts";
+import type { ChatMessage, Completer } from "./run.ts";
+import type { JudgedTask } from "./tasks.ts";
+
+export type Judge = (task: JudgedTask, answer: string) => Promise<number | null>;
+
+const JUDGE_SYSTEM = [
+	"You are a strict, impartial grader of an AI assistant's answer to a developer task.",
+	"Judge ONLY the quality of the answer for the stated dimension: correctness, completeness,",
+	"and whether it followed the instruction. Ignore style and length.",
+	"Reply with a SINGLE integer from 0 to 10 and nothing else:",
+	"0 = wrong, empty, or ignores the instruction; 5 = partially correct or incomplete;",
+	"10 = fully correct, complete, and follows the instruction exactly.",
+].join("\n");
+
+const AXIS_LABEL: Record<QualityAxis, string> = {
+	coding: "coding / implementation quality",
+	intelligence: "reasoning quality",
+	agentic: "tool-use / planning quality",
+};
+
+function buildPrompt(task: JudgedTask, answer: string): string {
+	const ref = task.reference === undefined ? "" : `\n=== A STRONG REFERENCE ANSWER ===\n${task.reference}\n`;
+	return `Dimension: ${AXIS_LABEL[task.axis]}\n\n=== TASK ===\n${task.user}\n\n=== ANSWER TO GRADE ===\n${answer}\n${ref}\nScore (0-10):`;
+}
+
+/**
+ * The 0-1 score in the judge's reply: the LAST standalone integer 0-10, since a
+ * model that reasons before answering tends to end on the score. Null when no
+ * such integer appears.
+ */
+export function parseScore(text: string): number | null {
+	const matches = [...text.matchAll(/\b(10|[0-9])\b/g)];
+	if (matches.length === 0) return null;
+	const last = matches[matches.length - 1];
+	const n = Number(last?.[1]);
+	if (!Number.isFinite(n)) return null;
+	return n / 10;
+}
+
+export function makeJudge(complete: Completer, judgeSlug: string): Judge {
+	return async (task, answer) => {
+		const messages: ChatMessage[] = [
+			{ role: "system", content: JUDGE_SYSTEM },
+			{ role: "user", content: buildPrompt(task, answer) },
+		];
+		let text: string;
+		try {
+			text = await complete(judgeSlug, messages);
+		} catch {
+			return null;
+		}
+		return parseScore(text);
+	};
+}