auto-model-router 0.1.3 → 0.2.0

package/research/project-yield.ts

@@ -0,0 +1,96 @@
+/**
+ * Projects how many exploration samples a given set of rates would actually
+ * yield, from a ledger snapshot of real traffic.
+ *
+ * Choosing exploration rates blind is how you end up spending a month
+ * sampling the cheapest boundary in the system. This answers the only
+ * question that matters up front: at these rates, how many turns of each
+ * tier do I get, and how long until there are enough to fit on?
+ *
+ * Cache-coldness is approximated by the gap to the previous turn in the same
+ * conversation: if more than `cacheWarmTtlMs` elapsed, the prompt cache would
+ * have expired, which is what makes a hysteresis-held turn explorable.
+ *
+ * Usage: bun run research/project-yield.ts [snapshot.db]
+ */
+
+import { Database } from "bun:sqlite";
+
+import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
+import type { Tier } from "../src/router/types.ts";
+
+const path = process.argv[2] ?? "research-data/snapshot.db";
+const db = new Database(path, { readonly: true });
+
+const TTL = DEFAULT_CONFIG.hysteresis.cacheWarmTtlMs;
+const RATES = DEFAULT_CONFIG.exploration.rates;
+
+interface Row {
+	tier: Tier;
+	total: number;
+	old_eligible: number;
+	new_eligible: number;
+	spend: number;
+}
+
+const rows = db
+	.query(
+		`WITH t AS (
+			SELECT tier, classification_source AS src,
+			       COALESCE(reported_usd, predicted_usd) AS usd,
+			       created_at_ms - LAG(created_at_ms) OVER (
+			           PARTITION BY conversation_key ORDER BY created_at_ms
+			       ) AS gap
+			FROM ledger
+		)
+		SELECT tier,
+		       COUNT(*) AS total,
+		       ROUND(SUM(usd), 2) AS spend,
+		       SUM(CASE WHEN src = 'heuristic' THEN 1 ELSE 0 END) AS old_eligible,
+		       SUM(CASE WHEN src = 'heuristic'
+		                  OR (src = 'sticky' AND (gap IS NULL OR gap > ${TTL}))
+		                THEN 1 ELSE 0 END) AS new_eligible
+		FROM t
+		WHERE tier IN ('simple', 'moderate', 'hard')
+		GROUP BY tier`,
+	)
+	.all() as unknown as Row[];
+
+const { days } = db.query("SELECT (MAX(created_at_ms) - MIN(created_at_ms)) / 86400000.0 AS days FROM ledger").get() as {
+	days: number;
+};
+
+const order: Tier[] = ["hard", "moderate", "simple"];
+const byTier = new Map(rows.map((r) => [r.tier, r]));
+
+console.log(`window: ${days.toFixed(2)} days\n`);
+console.log("tier        spend   turns   elig(sticky-excl)  elig(cold-ok)   rate   /day    2wk    4wk");
+
+let oldPerDay = 0;
+let newPerDay = 0;
+for (const tier of order) {
+	const r = byTier.get(tier);
+	if (r === undefined) continue;
+	const rate = RATES[tier] ?? 0;
+	const perDay = (r.new_eligible / days) * rate;
+	oldPerDay += (r.old_eligible / days) * rate;
+	newPerDay += perDay;
+	console.log(
+		"  " +
+			tier.padEnd(10) +
+			("$" + r.spend).padStart(6) +
+			String(r.total).padStart(8) +
+			String(r.old_eligible).padStart(19) +
+			String(r.new_eligible).padStart(15) +
+			String(rate).padStart(7) +
+			perDay.toFixed(1).padStart(7) +
+			(perDay * 14).toFixed(0).padStart(7) +
+			(perDay * 28).toFixed(0).padStart(7),
+	);
+}
+
+console.log("");
+console.log(`total explored/day  sticky-excluded ${oldPerDay.toFixed(1)}  ->  cold-cache allowed ${newPerDay.toFixed(1)}`);
+console.log(`over 4 weeks        sticky-excluded ${(oldPerDay * 28).toFixed(0)}  ->  cold-cache allowed ${(newPerDay * 28).toFixed(0)}`);
+
+db.close();

package/research/run-eval.ts

@@ -0,0 +1,133 @@
+/**
+ * Run our own on-distribution benchmark and calibrate it to the AA scale.
+ *
+ * Reads the live cached catalog, splits it into ANCHORS (models AA already
+ * scored) and TARGETS (unscored, routable), runs the curated suite against every
+ * one via real OpenRouter completions, fits raw->AA per axis from the anchors,
+ * and writes calibrated `local` scores for the targets into `local_scores`.
+ *
+ * Those scores are INERT until `benchmarks.useLocalScores` is turned on — this
+ * script never flips it, so a run cannot perturb live routing on its own.
+ *
+ *   bun run research/run-eval.ts
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+// Force the PRODUCTION home; the fork's .env points it at research-data/home.
+process.env.AUTO_MODEL_ROUTER_HOME = join(homedir(), ".auto-model-router");
+
+import type { QualityAxis } from "../src/config/types.ts";
+import type { CatalogModel } from "../src/catalog/types.ts";
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import { saveLocalScores, normalizeModelKey } from "../src/catalog/benchmark-feeds.ts";
+import { loadConfig } from "../src/config/load.ts";
+import { fitCalibration, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { runEval, type ChatMessage, type EvalResult } from "../src/eval/run.ts";
+import { makeJudge } from "../src/eval/judge.ts";
+import { EVAL_TASKS, JUDGED_TASKS } from "../src/eval/tasks.ts";
+import { createOpenRouterClient } from "../src/upstream/openrouter.ts";
+import { openDb } from "../src/util/sqlite.ts";
+
+const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
+const cfg = loadConfig({});
+if (cfg.openrouter.apiKey.trim() === "") {
+	console.log("no OpenRouter key resolved; the eval needs one to dispatch completions.");
+	process.exit(1);
+}
+
+const db = openDb(cfg.ledger.path);
+const row = db.query("SELECT payload FROM catalog_cache WHERE id = 1").get() as { payload: string } | null;
+if (row === null) {
+	console.log("no cached catalog");
+	process.exit(1);
+}
+const models: CatalogModel[] = (JSON.parse(row.payload) as unknown[])
+	.map(normalizeCatalogModel)
+	.filter((m): m is CatalogModel => m !== null);
+
+const routable = (m: CatalogModel): boolean => !m.slug.startsWith("~") && !m.isFree && m.author !== "openrouter";
+const scored = (m: CatalogModel): boolean => m.quality.coding !== undefined || m.quality.intelligence !== undefined || m.quality.agentic !== undefined;
+
+const anchors = models.filter((m) => routable(m) && scored(m));
+const targets = models.filter((m) => routable(m) && !scored(m));
+console.log(`catalog ${models.length}: ${anchors.length} anchors (AA-scored), ${targets.length} targets (unscored)\n`);
+if (anchors.length < MIN_ANCHORS) {
+	console.log(`only ${anchors.length} anchors (< ${MIN_ANCHORS}); cannot calibrate. Aborting.`);
+	process.exit(1);
+}
+
+const client = createOpenRouterClient(cfg);
+// Open-ended (judged) answers are long and slow reasoning models need room, or
+// they time out / truncate and get spuriously excluded — which would make raw
+// scores incomparable across models. Generous ceiling; objective tasks ignore it.
+const complete = async (slug: string, messages: ChatMessage[]): Promise<string> => {
+	const body = { model: slug, messages, temperature: 0, max_tokens: 2048 };
+	const { text } = await client.complete(body, AbortSignal.timeout(120_000));
+	return text;
+};
+
+// The judge is the highest-intelligence anchor: the strongest grader available.
+// It also gets scored here (minor self-preference bias, called out in the report).
+const judgeModel = anchors.reduce((best, m) => ((m.quality.intelligence ?? 0) > (best.quality.intelligence ?? 0) ? m : best));
+const judge = makeJudge(complete, judgeModel.slug);
+const allSlugs = [...anchors, ...targets].map((m) => m.slug);
+console.log(`judge: ${judgeModel.slug} (intelligence ${judgeModel.quality.intelligence ?? "-"})`);
+console.log(`running ${EVAL_TASKS.length} objective + ${JUDGED_TASKS.length} judged tasks against ${allSlugs.length} models...`);
+const results = await runEval({
+	slugs: allSlugs,
+	complete,
+	judge,
+	concurrency: 4,
+	onProgress: (r, done, total) => console.log(`  [${done}/${total}] ${r.slug}${r.errors > 0 ? ` (errors: ${r.errors})` : ""}`),
+});
+const bySlug = new Map<string, EvalResult>(results.map((r) => [r.slug, r]));
+
+const qualityOf = new Map<string, CatalogModel["quality"]>(models.map((m) => [m.slug, m.quality]));
+const anchorAa = (slug: string, axis: QualityAxis): number | undefined => qualityOf.get(slug)?.[axis];
+
+const mean = (r: EvalResult | undefined, axis: QualityAxis): number | null =>
+	r === undefined || r.axes[axis].n === 0 ? null : r.axes[axis].sum / r.axes[axis].n;
+
+const anchorResults = anchors.map((m) => bySlug.get(m.slug)).filter((r): r is EvalResult => r !== undefined);
+const targetResults = targets.map((m) => bySlug.get(m.slug)).filter((r): r is EvalResult => r !== undefined);
+
+const cal = fitCalibration(anchorResults, anchorAa);
+
+console.log("\n=== anchors: raw suite mean vs known AA ===");
+for (const m of anchors) {
+	const r = bySlug.get(m.slug);
+	const cells = AXES.map((a) => {
+		const raw = mean(r, a);
+		const aa = m.quality[a];
+		return `${a[0]}: raw ${raw === null ? "-" : raw.toFixed(2)} / aa ${aa ?? "-"}`;
+	}).join("   ");
+	console.log(`  ${m.slug.padEnd(34)} ${cells}${r !== undefined && r.errors > 0 ? `   (errors: ${r.errors})` : ""}`);
+}
+
+console.log("\n=== calibration fit (raw 0-1 -> AA 0-100) ===");
+for (const a of AXES) {
+	const f = cal[a];
+	console.log(`  ${a.padEnd(13)} ${f === undefined ? "(no fit — too few anchors, no spread, or weak/negative correlation)" : `aa = ${f.slope.toFixed(1)}*raw + ${f.intercept.toFixed(1)}  (r=${f.r.toFixed(2)}, n=${f.n})`}`);
+}
+
+const localScores = toLocalFeedScores(targetResults, cal, (slug) => {
+	const bare = slug.startsWith("~") ? slug.slice(1) : slug;
+	const slash = bare.indexOf("/");
+	return slash === -1 ? "" : bare.slice(0, slash);
+});
+
+console.log("\n=== targets: calibrated local scores ===");
+for (const m of targets) {
+	const r = bySlug.get(m.slug);
+	const fs = localScores.find((s) => s.key === normalizeModelKey(m.slug));
+	const raws = AXES.map((a) => `${a[0]}:${mean(r, a)?.toFixed(2) ?? "-"}`).join(" ");
+	const cal2 = fs === undefined ? "(none)" : AXES.map((a) => `${a[0]}:${fs[a]?.toFixed(1) ?? "-"}`).join(" ");
+	console.log(`  ${m.slug.padEnd(34)} raw[${raws}]  ->  local[${cal2}]`);
+}
+
+saveLocalScores(db, localScores);
+db.close();
+console.log(`\nwrote ${localScores.length} local score row(s) to local_scores.`);
+console.log(`benchmarks.useLocalScores = ${cfg.benchmarks.useLocalScores} — scores are INERT until this is true.`);

package/research/status.ts

@@ -0,0 +1,55 @@
+/**
+ * Quick "is the experiment actually running?" check.
+ *
+ * Distinct from analyze-ledger.ts, which reads results. This answers the
+ * narrower question you ask right after a restart: is the fork serving turns,
+ * and are both experiments recording?
+ *
+ * Usage: bun run research/status.ts [path-to-router.db]
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { Database } from "bun:sqlite";
+
+const path = process.argv[2] ?? join(homedir(), ".auto-model-router", "router.db");
+const db = new Database(path, { readonly: true });
+
+const one = <T>(sql: string): T => db.query(sql).get() as T;
+
+const version = one<{ user_version: number }>("PRAGMA user_version").user_version;
+const total = one<{ n: number }>("SELECT COUNT(*) AS n FROM ledger").n;
+
+// Instrumented rows are the only ones the fork can have written, so they are
+// the dividing line between "old build" and "this build" traffic.
+const instrumented = one<{ n: number }>("SELECT COUNT(*) AS n FROM ledger WHERE features IS NOT NULL").n;
+const withArm = one<{ n: number }>("SELECT COUNT(*) AS n FROM ledger WHERE hold_arm IS NOT NULL").n;
+const explored = one<{ n: number }>("SELECT COUNT(*) AS n FROM ledger WHERE explored_from IS NOT NULL").n;
+const newest = one<{ m: number | null }>("SELECT MAX(created_at_ms) AS m FROM ledger").m;
+const newestInstrumented = one<{ m: number | null }>(
+	"SELECT MAX(created_at_ms) AS m FROM ledger WHERE features IS NOT NULL",
+).m;
+
+const stamp = (ms: number | null): string => (ms === null ? "never" : new Date(ms).toISOString());
+const ok = (b: boolean): string => (b ? "OK  " : "WAIT");
+
+console.log(`db                 ${path}`);
+console.log(`schema             v${version}${version === 8 ? "" : "   <-- expected v8"}`);
+console.log(`rows               ${total}`);
+console.log("");
+console.log(`${ok(instrumented > 0)} instrumented    ${instrumented}  (feature vectors recorded)`);
+console.log(`${ok(withArm > 0)} hold arms       ${withArm}  (should equal instrumented once the hold experiment is live)`);
+console.log(`${ok(explored > 0)} explored        ${explored}  (sampled, so stays 0 for a while at low rates)`);
+console.log("");
+console.log(`newest row             ${stamp(newest)}`);
+console.log(`newest instrumented    ${stamp(newestInstrumented)}`);
+
+if (instrumented > 0 && withArm < instrumented) {
+	console.log("");
+	console.log(`note: ${instrumented - withArm} instrumented row(s) have no hold arm.`);
+	console.log("      Rows written before the hold experiment shipped are expected to be null.");
+	console.log("      If the count keeps growing, the running router predates it -- restart omp.");
+}
+
+db.close();

package/research/tier-fill.ts

@@ -0,0 +1,109 @@
+/**
+ * Reports how many candidates each tier actually has, from the live cached
+ * catalog.
+ *
+ * The exploration plan assumes `hard` turns exist and have somewhere cheaper
+ * to land. If a narrow OpenRouter key-scoping leaves upper tiers empty, or
+ * only a handful of models carry benchmark scores, exploration cannot yield
+ * what the projection promised. This checks that rather than assuming it.
+ *
+ * Usage: bun run research/tier-fill.ts [path-to-router.db]
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { Database } from "bun:sqlite";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
+import { loadConfig } from "../src/config/load.ts";
+import { scoreHeuristic } from "../src/router/classify.ts";
+import { extractFeatures } from "../src/router/features.ts";
+import { select } from "../src/router/select.ts";
+import type { ConversationState, Tier } from "../src/router/types.ts";
+import { parseChatRequest } from "../src/wire/openai/request.ts";
+
+const dbPath = process.argv[2] ?? join(homedir(), ".auto-model-router", "router.db");
+const db = new Database(dbPath, { readonly: true });
+const row = db.query("SELECT payload FROM catalog_cache WHERE id = 1").get() as { payload: string } | null;
+db.close();
+
+if (row === null) {
+	console.log("no cached catalog in " + dbPath);
+	process.exit(0);
+}
+
+const models: CatalogModel[] = (JSON.parse(row.payload) as unknown[])
+	.map(normalizeCatalogModel)
+	.filter((m): m is CatalogModel => m !== null);
+
+const snapshot: CatalogSnapshot = { models, fetchedAtMs: Date.now() };
+const cfg = loadConfig({});
+
+console.log(`catalog: ${models.length} models`);
+console.log(`adaptiveTierFloors: ${cfg.adaptiveTierFloors}`);
+console.log("");
+
+const req = parseChatRequest(
+	{
+		model: "auto",
+		messages: [
+			{ role: "system", content: "You are a coding agent." },
+			{ role: "user", content: "refactor the retry helper and explain the race condition" },
+		],
+	},
+	new Headers(),
+);
+const features = extractFeatures(req, 4000);
+const heuristic = scoreHeuristic(features, cfg);
+
+const state: ConversationState = {
+	key: "tier-fill-probe",
+	sessionId: "probe",
+	turn: 1,
+	currentSlug: null,
+	currentTier: null,
+	stickyUntilTurn: 0,
+	escalations: 0,
+	spentUsd: 0,
+	lastPromptTokens: 0,
+	cacheWarmSlug: null,
+	cacheWarmAtMs: 0,
+	updatedAtMs: Date.now(),
+};
+
+const profile = cfg.profiles.find((p) => p.id === "auto") ?? cfg.profiles[0];
+if (profile === undefined) {
+	console.log("no profiles configured");
+	process.exit(1);
+}
+
+console.log("tier      candidates  chosen                          note");
+for (const tier of ["trivial", "simple", "moderate", "hard"] as Tier[]) {
+	try {
+		const d = select({
+			req,
+			features,
+			classification: { ...heuristic, tier },
+			profile,
+			state,
+			snapshot,
+			ledger: null,
+			cfg,
+			nowMs: Date.now(),
+		});
+		// A widened decision means the requested tier had nothing of its own.
+		const widened = d.reasons.find((r) => r.startsWith("widened"));
+		console.log(
+			"  " +
+				tier.padEnd(10) +
+				String(d.considered.length).padStart(10) +
+				"  " +
+				d.slug.padEnd(32) +
+				(widened ?? ""),
+		);
+	} catch (err) {
+		console.log("  " + tier.padEnd(10) + "  ERROR: " + (err instanceof Error ? err.message : String(err)));
+	}
+}

package/research/tier-map.ts

@@ -0,0 +1,123 @@
+/**
+ * Full placement matrix: for every catalog model, which tiers it is eligible
+ * for, on each routed axis (coding task -> coding axis, chat task -> intelligence
+ * axis). Faithful to src/router/candidates.ts (adaptive floors, price ceilings,
+ * capability filters). Reads the on-disk catalog cache; no network.
+ *
+ *   bun run research/tier-map.ts [db]
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { Database } from "bun:sqlite";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
+import { loadConfig } from "../src/config/load.ts";
+import { buildCandidates } from "../src/router/candidates.ts";
+import { extractFeatures } from "../src/router/features.ts";
+import { effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
+import { TIER_ORDER, type Tier, type TaskType } from "../src/router/types.ts";
+import { parseChatRequest } from "../src/wire/openai/request.ts";
+
+const dbPath = process.argv[2] ?? join(homedir(), ".auto-model-router", "router.db");
+const db = new Database(dbPath, { readonly: true });
+const row = db.query("SELECT payload FROM catalog_cache WHERE id = 1").get() as { payload: string } | null;
+db.close();
+if (row === null) {
+	console.log("no cached catalog");
+	process.exit(1);
+}
+
+const models: CatalogModel[] = (JSON.parse(row.payload) as unknown[])
+	.map(normalizeCatalogModel)
+	.filter((m): m is CatalogModel => m !== null)
+	.sort((a, b) => a.price.prompt - b.price.prompt);
+
+const snapshot: CatalogSnapshot = { models, fetchedAtMs: Date.now() };
+const cfg = loadConfig({});
+
+const req = parseChatRequest(
+	{ model: "auto", messages: [{ role: "user", content: "Implement and verify the change." }] },
+	new Headers(),
+);
+const features = extractFeatures(req, 4000);
+
+// reason -> single-char code for the matrix cell
+function code(reason: string | undefined): string {
+	if (reason === undefined) return "OK ";
+	if (reason === "below_quality_floor") return "qual";
+	if (reason === "over_price_ceiling") return "pric";
+	if (reason === "free_tier_excluded") return "free";
+	return reason.slice(0, 4);
+}
+
+const axes: Array<{ label: string; task: TaskType }> = [
+	{ label: "coding axis (task=coding)", task: "coding" as TaskType },
+	{ label: "intelligence axis (task=chat)", task: "chat" as TaskType },
+];
+
+for (const { label, task } of axes) {
+	const axis = cfg.tasks[task].axis;
+	console.log(`\n===== ${label} =====`);
+
+	// Effective (adaptive) floor per tier on this axis.
+	const floors: Record<string, number> = {};
+	for (const tier of TIER_ORDER) {
+		const tc = cfg.tiers[tier];
+		const adaptive = cfg.adaptiveTierFloors
+			? effectiveQualityFloor(tc.minQuality, tier, axis, tierPlanFor(snapshot, cfg))
+			: tc.minQuality;
+		floors[tier] = Math.max(adaptive, cfg.tasks[task].minQuality ?? 0);
+	}
+	console.log(
+		`floors  ` +
+			TIER_ORDER.map((t) => `${t}=${floors[t]?.toFixed(1)}(cfg ${cfg.tiers[t].minQuality})`).join("  ") +
+			`\nceil$   ` +
+			TIER_ORDER.map((t) => `${t}=${cfg.tiers[t].maxInputPerMtok ?? "inf"}`).join("  ") +
+			`\n`,
+	);
+
+	// Per tier: eligible set + rejection reasons.
+	const perTier = TIER_ORDER.map((tier) => {
+		const { candidates, rejected } = buildCandidates({
+			req,
+			features,
+			tier,
+			task,
+			snapshot,
+			ledger: null,
+			cfg,
+			expectedCompletionTokens: 4000,
+			warmSlug: null,
+		});
+		const elig = new Set(candidates.map((c) => c.model.slug));
+		const rej = new Map(rejected.map((r) => [r.slug, r.reason]));
+		return { tier, elig, rej };
+	});
+
+	const qOf = (m: CatalogModel): number | undefined =>
+		axis === "coding" ? m.quality.coding : axis === "agentic" ? m.quality.agentic : m.quality.intelligence;
+
+	const nameW = Math.max(...models.map((m) => m.slug.length));
+	const header =
+		"model".padEnd(nameW) + "  in$   " + "q".padEnd(6) + TIER_ORDER.map((t) => t.slice(0, 4).padEnd(6)).join("");
+	console.log(header);
+	console.log("-".repeat(header.length));
+	for (const m of models) {
+		const q = qOf(m);
+		const cells = perTier
+			.map((pt) => (pt.elig.has(m.slug) ? "OK " : code(pt.rej.get(m.slug))).padEnd(6))
+			.join("");
+		console.log(
+			m.slug.padEnd(nameW) +
+				"  " +
+				(m.price.prompt * 1e6).toFixed(2).padStart(5) +
+				"  " +
+				(q === undefined ? "-" : q.toFixed(1)).padEnd(6) +
+				cells,
+		);
+	}
+}
+console.log("\ncells: OK=eligible  qual=below_quality_floor  pric=over_price_ceiling  free=free_tier_excluded");