auto-model-router 0.1.3 → 0.2.0

package/omp-extension/router-embed.ts

@@ -43,15 +43,20 @@ import {
  * Registers the auto-model-router provider (and its virtual models) into omp's model
  * registry at a specific bound port.
  */
-function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig): void {
+function registerRouterProvider(pi: ExtensionAPI, port: number, cfg: RouterConfig, sessionId: string): void {
 	const providerConfig = buildProviderConfig(port, cfg);
+	const headers: Record<string, string> = {};
+	if (providerConfig.harnessId !== undefined && providerConfig.harnessId !== "") {
+		headers["X-Omp-Harness"] = providerConfig.harnessId;
+	}
+	// Per-session scoping: lets the toast surface only this session's decisions
+	// even when several omp sessions share one embedded router's ledger.
+	if (sessionId !== "") headers["X-Omp-Session"] = sessionId;
 	pi.registerProvider(EMBED_PROVIDER_ID, {
 		baseUrl: providerConfig.baseUrl,
 		api: "openai-completions",
 		apiKey: EMBED_DUMMY_API_KEY,
-		...(providerConfig.harnessId !== undefined && providerConfig.harnessId !== ""
-			? { headers: { "X-Omp-Harness": providerConfig.harnessId } }
-			: {}),
+		...(Object.keys(headers).length > 0 ? { headers } : {}),
 		models: providerConfig.models.map((m) => ({
 			id: m.id,
 			name: m.name,
@@ -90,9 +95,12 @@ export default function (pi: ExtensionAPI): void {
 		// Subagents and headless sessions do not bind their own router; they
 		// route to the main's router via the shared port file. The main writes
 		// the file before spawning subagents, so the port is available here.
+		// The omp UI session id tags every request so the toast can scope its
+		// notifications to that exact session (see router-toast.ts).
+		const sessionId = ctx.sessionManager.getSessionId();
 		if (!ctx.hasUI) {
 			const port = readEmbedPort(portFile);
-			if (port !== null) registerRouterProvider(pi, port, cfg);
+			if (port !== null) registerRouterProvider(pi, port, cfg, sessionId);
 			return;
 		}
 
@@ -108,7 +116,7 @@ export default function (pi: ExtensionAPI): void {
 
 		// Publish the shared port; subagents and the toast read it from here.
 		writeEmbedPort(portFile, actualPort);
-		registerRouterProvider(pi, actualPort, cfg);
+		registerRouterProvider(pi, actualPort, cfg, sessionId);
 
 		pi.on("session_shutdown", () => {
 			void app?.stop().catch(() => {});

package/omp-extension/router-toast.ts

@@ -73,6 +73,11 @@ export default function (pi: ExtensionAPI): void {
 		// poll loop entirely rather than waking every 2s to do nothing.
 		if (!ctx.hasUI) return;
 
+		// This session's omp id. The embed extension tags every request with it
+		// (X-Omp-Session), so filtering on it scopes toasts to this session even
+		// when several omp sessions share one embedded router's ledger.
+		const sessionId = ctx.sessionManager.getSessionId();
+
 		// The poll request's own deadline (3s) exceeds the poll period (2s), so
 		// a slow router could let a second tick start while the first is still in
 		// flight — both read the same lastSeenId and raise duplicate toasts. An
@@ -115,7 +120,7 @@ export default function (pi: ExtensionAPI): void {
 				const entries = body.entries;
 				if (!Array.isArray(entries) || entries.length === 0) return;
 
-				for (const t of selectToasts(entries, lastSeenId, HARNESS_ID)) {
+				for (const t of selectToasts(entries, lastSeenId, HARNESS_ID, sessionId)) {
 					ctx.ui.notify(t.text, "info");
 				}
 				lastSeenId = newestId(entries) ?? lastSeenId;

package/omp-extension/toast-logic.ts

@@ -83,6 +83,11 @@ export interface ToastDecision {
 	wasted: boolean;
 	/** Harness id from the request header; empty for the default harness. */
 	harnessId: string;
+	/**
+	 * omp UI session id from the `X-Omp-Session` header; empty for the no-header
+	 * default. Lets the toast scope to a single interactive session.
+	 */
+	ompSessionId?: string;
 }
 
 export interface ToastMessage {
@@ -111,6 +116,7 @@ export function selectToasts(
 	entries: ToastDecision[],
 	lastSeenId: string | null,
 	harnessId = "",
+	ompSessionId = "",
 ): ToastMessage[] {
 	if (lastSeenId === null) return [];
 	// `entries` is newest-first. Entries strictly newer than lastSeenId are the
@@ -124,6 +130,7 @@ export function selectToasts(
 		if (d === undefined) continue;
 		if (d.wasted) continue;
 		if (harnessId !== "" && d.harnessId !== harnessId) continue;
+		if (ompSessionId !== "" && d.ompSessionId !== ompSessionId) continue;
 		out.push({ model: d.servedSlug ?? d.slug, tier: d.tier, costUsd: d.reportedUsd, text: toToastText(d) });
 	}
 	return out;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",
@@ -11,6 +11,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "smoke": "bun run tools/smoke.ts",
+    "version": "bun run tools/sync-marketplace-version.ts",
     "release": "npm version $1 && git push --follow-tags"
   },
   "dependencies": {

package/research/analyze-ledger.ts

@@ -0,0 +1,173 @@
+/**
+ * Read-only analysis of a ledger snapshot.
+ *
+ * Answers the questions that decide whether a learned classifier is worth
+ * building at all: does the adjudicator ever actually run, how are tiers
+ * distributed, and how much label signal (escalations) exists.
+ *
+ * Usage: bun run research/analyze-ledger.ts [path-to-snapshot.db]
+ */
+
+import { Database } from "bun:sqlite";
+
+const path = process.argv[2] ?? "research-data/snapshot.db";
+const db = new Database(path, { readonly: true });
+
+function section(label: string, sql: string): void {
+	console.log(`\n=== ${label} ===`);
+	try {
+		const rows = db.query(sql).all() as Record<string, unknown>[];
+		if (rows.length === 0) {
+			console.log("  (no rows)");
+			return;
+		}
+		for (const r of rows) {
+			console.log("  " + Object.entries(r).map(([k, v]) => `${k}=${v}`).join("  "));
+		}
+	} catch (err) {
+		console.log("  ERROR: " + (err instanceof Error ? err.message : String(err)));
+	}
+}
+
+section(
+	"tier distribution",
+	`SELECT tier, COUNT(*) n, ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM ledger), 1) pct
+	 FROM ledger GROUP BY tier ORDER BY n DESC`,
+);
+
+section(
+	"classification source",
+	`SELECT classification_source src, COUNT(*) n,
+	        ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM ledger), 1) pct
+	 FROM ledger GROUP BY src ORDER BY n DESC`,
+);
+
+section(
+	"adjudicator activity (from reasons text)",
+	`SELECT
+	   SUM(CASE WHEN reasons LIKE '%adjudicator skipped%' THEN 1 ELSE 0 END) AS skipped_cost_guard,
+	   SUM(CASE WHEN reasons LIKE '%adjudicator failed%'  THEN 1 ELSE 0 END) AS failed,
+	   SUM(CASE WHEN reasons LIKE '%not a tier word%'     THEN 1 ELSE 0 END) AS bad_reply,
+	   SUM(CASE WHEN reasons LIKE '%adjudicator:%'        THEN 1 ELSE 0 END) AS verdict_used,
+	   COUNT(*) AS total_turns
+	 FROM ledger`,
+);
+
+section(
+	"escalation signals",
+	`SELECT COALESCE(escalation_signal, '(none)') signal, COUNT(*) n
+	 FROM ledger GROUP BY signal ORDER BY n DESC`,
+);
+
+section(
+	"spend and waste",
+	`SELECT SUM(wasted) wasted_turns,
+	        ROUND(SUM(COALESCE(reported_usd, predicted_usd)), 4) total_usd,
+	        ROUND(SUM(CASE WHEN wasted = 1 THEN COALESCE(reported_usd, predicted_usd) ELSE 0 END), 4) wasted_usd
+	 FROM ledger`,
+);
+
+section(
+	"spend by served model",
+	`SELECT COALESCE(served_slug, slug) model, COUNT(*) n,
+	        ROUND(SUM(COALESCE(reported_usd, predicted_usd)), 4) usd
+	 FROM ledger GROUP BY model ORDER BY usd DESC LIMIT 10`,
+);
+
+section(
+	"error kinds",
+	`SELECT COALESCE(error_kind, '(none)') kind, COUNT(*) n
+	 FROM ledger GROUP BY kind ORDER BY n DESC`,
+);
+
+
+
+// ---------------------------------------------------------------------------
+// Exploration (schema v7): the counterfactual natural traffic cannot supply.
+//
+// A turn we deliberately routed one tier cheaper either escalated (the cheap
+// model genuinely could not do it) or committed (the classifier was over-
+// routing, and the cheaper tier would have served the turn fine).
+// ---------------------------------------------------------------------------
+
+section(
+	"exploration coverage",
+	`SELECT
+	   SUM(CASE WHEN explored_from IS NOT NULL THEN 1 ELSE 0 END) AS explored,
+	   COUNT(*) AS total,
+	   ROUND(100.0 * SUM(CASE WHEN explored_from IS NOT NULL THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct
+	 FROM ledger`,
+);
+
+section(
+	"exploration verdict by dropped-from tier",
+	`SELECT
+	   explored_from AS dropped_from,
+	   COUNT(*) AS n,
+	   SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalated,
+	   ROUND(100.0 * SUM(CASE WHEN escalation_signal IS NULL THEN 1 ELSE 0 END) / COUNT(*), 1) AS pct_cheap_sufficed
+	 FROM ledger
+	 WHERE explored_from IS NOT NULL
+	 GROUP BY explored_from
+	 ORDER BY n DESC`,
+);
+
+section(
+	"what exploration cost vs. what it revealed",
+	`SELECT
+	   ROUND(SUM(CASE WHEN explored_from IS NOT NULL AND wasted = 1
+	                  THEN COALESCE(reported_usd, predicted_usd) ELSE 0 END), 4) AS wasted_on_exploration_usd,
+	   ROUND(SUM(CASE WHEN explored_from IS NOT NULL THEN COALESCE(reported_usd, predicted_usd) ELSE 0 END), 4) AS explored_spend_usd
+	 FROM ledger`,
+);
+
+// Confidence distribution: this is what determines whether the LLM adjudicator
+// is ever reached at all. Turns below the configured ambiguityThreshold should
+// be adjudicated; if that bucket is populated but no row has source 'llm', the
+// adjudicator is failing silently rather than never being needed.
+section(
+	"confidence distribution vs. the adjudication band",
+	`SELECT
+	   CASE
+	     WHEN confidence IS NULL THEN '(uninstrumented)'
+	     WHEN confidence < 0.6 THEN 'below 0.6 (should adjudicate)'
+	     WHEN confidence < 0.8 THEN '0.6 - 0.8'
+	     ELSE '0.8 - 1.0'
+	   END AS band,
+	   COUNT(*) AS n,
+	   SUM(CASE WHEN classification_source = 'llm' THEN 1 ELSE 0 END) AS adjudicated
+	 FROM ledger
+	 GROUP BY band
+	 ORDER BY n DESC`,
+);
+
+// ---------------------------------------------------------------------------
+// Hold-length experiment (schema v8).
+//
+// ~95% of hard-tier spend arrives by hysteresis hold rather than by
+// classification, and the hold length is a hand-picked constant. Each
+// conversation is assigned one arm, so arms compare on cost PER
+// CONVERSATION -- comparing per-turn would reward short holds automatically,
+// since a shorter hold simply produces fewer expensive turns.
+//
+// Escalations are the quality guardrail: a shorter hold that costs less but
+// escalates more has moved the cost, not removed it.
+// ---------------------------------------------------------------------------
+
+section(
+	"hold-length arms",
+	`SELECT hold_arm AS arm,
+	        COUNT(DISTINCT conversation_key) AS conversations,
+	        COUNT(*) AS turns,
+	        ROUND(SUM(COALESCE(reported_usd, predicted_usd)), 4) AS usd,
+	        ROUND(SUM(COALESCE(reported_usd, predicted_usd))
+	              / NULLIF(COUNT(DISTINCT conversation_key), 0), 4) AS usd_per_conversation,
+	        SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
+	        ROUND(100.0 * SUM(CASE WHEN wasted = 1 THEN 1 ELSE 0 END) / COUNT(*), 1) AS pct_wasted
+	 FROM ledger
+	 WHERE hold_arm IS NOT NULL
+	 GROUP BY hold_arm
+	 ORDER BY hold_arm`,
+);
+
+db.close();

package/research/apply-cost-tuning.ts

@@ -0,0 +1,73 @@
+/**
+ * Applies the cost-optimization config changes to the PRODUCTION router config
+ * (`~/.auto-model-router/config.yml`), merged + validated + backed up. Never
+ * prints the AA key.
+ *
+ *   bun run research/apply-cost-tuning.ts          # SAFE NOW: disable the dead
+ *                                                  # adjudicator only (routing-neutral).
+ *   bun run research/apply-cost-tuning.ts --full   # WINDOW-CLOSE: also raise
+ *                                                  # switchMargin and lower contextWindow
+ *                                                  # (these CHANGE live routing).
+ *
+ * See [[omp-router-cost-optimization]]. Restart all omp windows after applying.
+ */
+
+import { chmodSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+// The fork's .env points AUTO_MODEL_ROUTER_HOME at research-data/home; these
+// changes must land in the PRODUCTION config the live routers read.
+process.env.AUTO_MODEL_ROUTER_HOME = join(homedir(), ".auto-model-router");
+
+import { routerConfigPath, writeRouterConfig } from "../src/cli/config-cmd.ts";
+import { loadConfig } from "../src/config/load.ts";
+
+const full = process.argv.includes("--full");
+const cfg = loadConfig({});
+
+// (3) Disable the broken adjudicator: classifier.model is absent from the catalog,
+// so it never returns a verdict — always keeps the heuristic. Setting the
+// threshold to 0 skips the doomed call. Routing-identical, just faster.
+const patch: Record<string, unknown> = { classifier: { ambiguityThreshold: 0 } };
+
+if (full) {
+	// (2) Stop forfeiting warm cache for tiny completion savings.
+	patch.hysteresis = { switchMargin: 4 };
+	// (1) Force earlier compaction by advertising a smaller context window.
+	// Preserve every other profile field; only contextWindow changes.
+	patch.profiles = cfg.profiles.map((p) => ({
+		id: p.id,
+		name: p.name,
+		minTier: p.minTier,
+		maxTier: p.maxTier,
+		contextWindow: 200_000,
+		maxTokens: p.maxTokens,
+	}));
+	// (4) Cap the input-dominated hard cost. Now SAFE: the key admits ~13 hard
+	// candidates, so a $3/Mtok ceiling still leaves gpt-5.6-sol (77.4), grok-4.6,
+	// gpt-5.6-terra, glm-5.3, qwen3.8, kimi-k3 with failover; it drops only the
+	// $5-10 models (opus-5, gpt-5.5, fable). Raise to ~$5 if you want opus-5
+	// available for the very hardest turns. Merges into hard, preserving its floor.
+	patch.tiers = { hard: { maxInputPerMtok: 3 } };
+}
+
+const target = routerConfigPath();
+const backup = writeRouterConfig(target, patch);
+try {
+	chmodSync(target, 0o600); // keep the AA-key-bearing file locked down (advisory on Windows)
+} catch {
+	/* best-effort */
+}
+
+const after = loadConfig({});
+console.log(`wrote ${target}${backup === null ? "" : `\nbackup ${backup}`}`);
+console.log(`mode                     ${full ? "FULL (routing-changing)" : "adjudicator-only (safe)"}`);
+console.log(`classifier.ambiguity     ${after.classifier.ambiguityThreshold}`);
+console.log(`hysteresis.switchMargin  ${after.hysteresis.switchMargin}`);
+console.log(`profiles contextWindow   ${after.profiles.map((p) => `${p.id}:${p.contextWindow}`).join(" ")}`);
+// Confirm nothing unrelated was disturbed.
+const k = after.benchmarks.artificialAnalysisApiKey;
+console.log(`AA key intact            ${k.trim() === "" ? "MISSING" : `yes (…${k.slice(-4)})`}`);
+console.log(`exploration intact       enabled=${after.exploration.enabled} holdArms=[${after.exploration.holdTurns.values.join(",")}] sticky=${after.exploration.stickyPolicy}`);
+console.log("\nrestart all omp windows for this to take effect.");

package/research/cost-analysis.ts

@@ -0,0 +1,150 @@
+/**
+ * Where the money actually goes, from the live ledger. Read-only.
+ *
+ * Reframes optimization around the finding that ~96% of spend is the PROMPT,
+ * not the model's output: decomposes spend by cost component, measures the
+ * realized cache hit rate, and LOCALIZES cache misses to their cause (model
+ * switching, warm-window expiry, first-turns, exploration). Also quantifies
+ * abandoned-attempt waste and whether the LLM adjudicator ever fires.
+ *
+ *   bun run research/cost-analysis.ts
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+// Force PRODUCTION home; the fork's .env points it at research-data/home.
+process.env.AUTO_MODEL_ROUTER_HOME = join(homedir(), ".auto-model-router");
+
+import { Database } from "bun:sqlite";
+import { loadConfig } from "../src/config/load.ts";
+
+const cfg = loadConfig({});
+const db = new Database(cfg.ledger.path, { readonly: true });
+
+function rows(sql: string): Record<string, unknown>[] {
+	return db.query(sql).all() as Record<string, unknown>[];
+}
+function table(title: string, data: Record<string, unknown>[]): void {
+	console.log(`\n=== ${title} ===`);
+	if (data.length === 0) {
+		console.log("  (no rows)");
+		return;
+	}
+	const cols = Object.keys(data[0]!);
+	const w = cols.map((c) => Math.max(c.length, ...data.map((r) => String(r[c] ?? "").length)));
+	console.log("  " + cols.map((c, i) => c.padEnd(w[i]!)).join("  "));
+	for (const r of data) console.log("  " + cols.map((c, i) => String(r[c] ?? "").padEnd(w[i]!)).join("  "));
+}
+
+const usd = "json_extract(cost_breakdown,'$.%s')";
+const comp = (k: string) => usd.replace("%s", k);
+
+// 1. Spend by cost component.
+table("spend by component (USD)", rows(`
+	SELECT
+		round(sum(${comp("freshPrompt")}),2) AS fresh_in,
+		round(sum(${comp("cacheRead")}),2)   AS cache_read,
+		round(sum(${comp("cacheWrite")}),2)  AS cache_write,
+		round(sum(${comp("completion")}),2)  AS completion,
+		round(sum(${comp("reasoning")}),2)   AS reasoning,
+		round(sum(reported_usd),2)           AS total
+	FROM ledger WHERE cost_breakdown IS NOT NULL`));
+
+// 2. Token totals + input:output ratio + overall cache hit rate.
+table("tokens", rows(`
+	SELECT
+		sum(json_extract(usage,'$.promptTokens'))     AS prompt_toks,
+		sum(json_extract(usage,'$.completionTokens'))  AS completion_toks,
+		round(sum(json_extract(usage,'$.promptTokens'))*1.0/max(1,sum(json_extract(usage,'$.completionTokens'))),0) AS in_out_ratio,
+		round(sum(json_extract(usage,'$.cachedTokens'))*1.0/max(1,sum(json_extract(usage,'$.promptTokens'))),3) AS cache_hit_rate
+	FROM ledger WHERE usage IS NOT NULL`));
+
+// 3. Cache hit rate + spend by tier.
+table("by tier", rows(`
+	SELECT tier,
+		count(*) AS n,
+		round(sum(reported_usd),2) AS usd,
+		round(avg(reported_usd),5) AS per_turn,
+		round(sum(json_extract(usage,'$.cachedTokens'))*1.0/max(1,sum(json_extract(usage,'$.promptTokens'))),3) AS cache_hit
+	FROM ledger WHERE usage IS NOT NULL GROUP BY tier ORDER BY usd DESC`));
+
+// Per-turn view: first attempt only, with previous turn's model + timestamp.
+const SEQ = `
+	WITH seq AS (
+		SELECT conversation_key, tier, served_slug, created_at_ms, reported_usd, usage, cost_breakdown, explored_from,
+			LAG(served_slug)  OVER w AS prev_slug,
+			LAG(created_at_ms) OVER w AS prev_ms,
+			ROW_NUMBER()       OVER w AS rn
+		FROM ledger
+		WHERE served_slug IS NOT NULL AND reported_usd IS NOT NULL AND attempt = 0
+		WINDOW w AS (PARTITION BY conversation_key ORDER BY turn, created_at_ms)
+	)`;
+
+// 4. Cache misses localized to model switching (continuation turns only).
+table("switch effect (continuation turns)", rows(`${SEQ}
+	SELECT
+		CASE WHEN served_slug = prev_slug THEN 'stayed' ELSE 'switched' END AS kind,
+		count(*) AS n,
+		round(sum(reported_usd),2) AS usd,
+		round(sum(json_extract(usage,'$.cachedTokens'))*1.0/max(1,sum(json_extract(usage,'$.promptTokens'))),3) AS cache_hit,
+		round(sum(${comp("cacheWrite")}),2) AS cache_write
+	FROM seq WHERE rn > 1 GROUP BY kind`));
+
+// 5. Cache misses localized to warm-window expiry (continuation turns only).
+const ttlMin = Math.round(cfg.hysteresis.cacheWarmTtlMs / 60000);
+table(`gap effect (warm window = ${ttlMin} min)`, rows(`${SEQ}
+	SELECT
+		CASE WHEN (created_at_ms - prev_ms) < ${cfg.hysteresis.cacheWarmTtlMs} THEN 'within_window' ELSE 'expired' END AS gap,
+		count(*) AS n,
+		round(sum(json_extract(usage,'$.cachedTokens'))*1.0/max(1,sum(json_extract(usage,'$.promptTokens'))),3) AS cache_hit,
+		round(sum(${comp("cacheWrite")}),2) AS cache_write
+	FROM seq WHERE rn > 1 GROUP BY gap`));
+
+// 6. First-turn vs continuation cache-write (first turns are unavoidably cold).
+table("cache-write: first vs continuation", rows(`${SEQ}
+	SELECT
+		CASE WHEN rn = 1 THEN 'first_turn' ELSE 'continuation' END AS kind,
+		count(*) AS n,
+		round(sum(${comp("cacheWrite")}),2) AS cache_write,
+		round(sum(${comp("freshPrompt")}),2) AS fresh_in
+	FROM seq GROUP BY kind`));
+
+// 7. Exploration's cache cost (stickyPolicy: always forfeits warm cache).
+table("exploration effect", rows(`${SEQ}
+	SELECT
+		CASE WHEN explored_from IS NULL THEN 'normal' ELSE 'explored' END AS kind,
+		count(*) AS n,
+		round(sum(json_extract(usage,'$.cachedTokens'))*1.0/max(1,sum(json_extract(usage,'$.promptTokens'))),3) AS cache_hit,
+		round(sum(${comp("cacheWrite")}),2) AS cache_write
+	FROM seq WHERE rn > 1 GROUP BY kind`));
+
+// 8. Abandoned-attempt waste.
+table("waste (abandoned escalation attempts)", rows(`
+	SELECT
+		round(sum(CASE WHEN wasted = 1 THEN reported_usd ELSE 0 END),2) AS wasted_usd,
+		sum(CASE WHEN wasted = 1 THEN 1 ELSE 0 END) AS wasted_rows,
+		round(sum(reported_usd),2) AS total_usd
+	FROM ledger WHERE reported_usd IS NOT NULL`));
+
+// 9. Does the LLM adjudicator ever fire? confidence vs the ambiguity band.
+table(`adjudicator (ambiguityThreshold = ${cfg.classifier.ambiguityThreshold})`, rows(`
+	SELECT classification_source AS source, count(*) AS n,
+		round(avg(confidence),3) AS avg_conf,
+		sum(CASE WHEN confidence < ${cfg.classifier.ambiguityThreshold} THEN 1 ELSE 0 END) AS below_thresh
+	FROM ledger WHERE confidence IS NOT NULL GROUP BY classification_source`));
+
+// 10. Spend by prompt-size bucket.
+table("spend by prompt size", rows(`
+	SELECT
+		CASE
+			WHEN json_extract(usage,'$.promptTokens') < 50000  THEN 'a <50k'
+			WHEN json_extract(usage,'$.promptTokens') < 150000 THEN 'b 50-150k'
+			WHEN json_extract(usage,'$.promptTokens') < 300000 THEN 'c 150-300k'
+			ELSE 'd 300k+' END AS bucket,
+		count(*) AS n,
+		round(sum(reported_usd),2) AS usd,
+		round(avg(reported_usd),5) AS per_turn
+	FROM ledger WHERE usage IS NOT NULL GROUP BY bucket ORDER BY bucket`));
+
+db.close();

package/research/feed-check.ts

@@ -0,0 +1,64 @@
+/**
+ * Live proof: fetch the external benchmark feeds for real, apply them to a COPY
+ * of the live catalog, and print which previously-unscored models gained scores.
+ * Read-only w.r.t. the production DB — never writes benchmark_cache.
+ *
+ *   bun run research/feed-check.ts [db]
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { Database } from "bun:sqlite";
+
+import { applyFeedScores, fetchAaScores, fetchBenchlmScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import { loadConfig } from "../src/config/load.ts";
+
+const dbPath = process.argv[2] ?? join(homedir(), ".auto-model-router", "router.db");
+const cfg = loadConfig({});
+
+const db = new Database(dbPath, { readonly: true });
+const row = db.query("SELECT payload FROM catalog_cache WHERE id = 1").get() as { payload: string } | null;
+db.close();
+if (row === null) {
+	console.log("no cached catalog");
+	process.exit(1);
+}
+
+const parsed: unknown = JSON.parse(row.payload);
+const rawModels: unknown[] = Array.isArray(parsed) ? parsed : [];
+
+// Quality per slug BEFORE the fill.
+function qualityBySlug(models: unknown[]): Map<string, object> {
+	const out = new Map<string, object>();
+	for (const raw of models) {
+		const m = normalizeCatalogModel(raw);
+		if (m !== null) out.set(m.slug, m.quality);
+	}
+	return out;
+}
+
+const before = qualityBySlug(structuredClone(rawModels));
+
+const key = cfg.benchmarks.artificialAnalysisApiKey;
+console.log(`AA key: ${key.trim() === "" ? "(none — AA feed skipped)" : "present"}`);
+const [aa, bl] = await Promise.all([fetchAaScores(key, { timeoutMs: 30_000 }), fetchBenchlmScores({ timeoutMs: 30_000 })]);
+const feeds: FeedScore[] = [...aa, ...bl];
+console.log(`feeds fetched: artificial_analysis=${aa.length}  benchlm=${bl.length}\n`);
+
+if (feeds.length === 0) {
+	console.log("no feed data (endpoints unreachable or empty); nothing to apply.");
+	process.exit(0);
+}
+
+const result = applyFeedScores(rawModels, feeds);
+const after = qualityBySlug(rawModels);
+
+console.log(`filled ${result.modelsFilled} model(s): coding=${result.axes.coding} intelligence=${result.axes.intelligence} agentic=${result.axes.agentic} (aa=${result.sources.artificial_analysis} benchlm=${result.sources.benchlm})\n`);
+
+for (const [slug, q] of after) {
+	const prev = JSON.stringify(before.get(slug) ?? {});
+	const now = JSON.stringify(q);
+	if (prev !== now) console.log(`${slug}\n    before ${prev}\n    after  ${now}`);
+}

package/research/model-recommendations.ts

@@ -0,0 +1,86 @@
+/**
+ * Which models to ENABLE on the OpenRouter key. The key-scoped catalog is narrow
+ * (guardrails/preferences), so tiers — `hard` especially — resolve to one
+ * candidate and can't take a price ceiling or same-tier failover. This fetches
+ * the FULL public catalog, runs it through the router's real tier filters, and
+ * reports the models that WOULD be eligible per tier but are not currently
+ * admitted by the key. Read-only. Needs network.
+ *
+ *   bun run research/model-recommendations.ts
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+process.env.AUTO_MODEL_ROUTER_HOME = join(homedir(), ".auto-model-router");
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
+import { loadConfig } from "../src/config/load.ts";
+import { buildCandidates } from "../src/router/candidates.ts";
+import { extractFeatures } from "../src/router/features.ts";
+import { effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
+import type { Tier } from "../src/router/types.ts";
+import { createOpenRouterClient } from "../src/upstream/openrouter.ts";
+import { parseChatRequest } from "../src/wire/openai/request.ts";
+
+const cfg = loadConfig({});
+
+const client = createOpenRouterClient(cfg);
+
+// Current key-scoped availability, fetched LIVE (/models/user) so it reflects
+// account changes immediately — the cached snapshot lags until a router refresh.
+const userRaw = await client.fetchModelsForUser(AbortSignal.timeout(30_000));
+const keyScoped = new Set<string>(
+	userRaw.map(normalizeCatalogModel).filter((m): m is CatalogModel => m !== null).map((m) => m.slug),
+);
+
+// Full public catalog (carries AA benchmarks).
+const raw = await client.fetchModels(AbortSignal.timeout(30_000));
+const models = raw.map(normalizeCatalogModel).filter((m): m is CatalogModel => m !== null);
+const snapshot: CatalogSnapshot = { models, fetchedAtMs: Date.now() };
+console.log(`public catalog: ${models.length} models; key-scoped now (live): ${keyScoped.size}\n`);
+
+// A representative hard coding turn, so task=coding (axis=coding).
+const req = parseChatRequest(
+	{ model: "auto", messages: [{ role: "user", content: "Refactor the scheduler to remove the global lock and prove it stays correct under concurrency." }] },
+	new Headers(),
+);
+const features = extractFeatures(req, 8000);
+
+// Cover both axes a coding agent actually routes on: coding (task=coding) and
+// intelligence (task=chat). Model choice for `hard` should be strong on both.
+const combos: Array<[Tier, "coding" | "chat"]> = [
+	["hard", "coding"],
+	["hard", "chat"],
+	["moderate", "coding"],
+];
+for (const [tier, task] of combos) {
+	const axis = cfg.tasks[task].axis;
+	const tc = cfg.tiers[tier];
+	// Floor if the FULL catalog were available (not the collapsed key-scoped floor).
+	const floor = cfg.adaptiveTierFloors
+		? effectiveQualityFloor(tc.minQuality, tier, axis, tierPlanFor(snapshot, cfg))
+		: tc.minQuality;
+	const { candidates } = buildCandidates({
+		req,
+		features,
+		tier,
+		task,
+		snapshot,
+		ledger: null,
+		cfg,
+		expectedCompletionTokens: 8000,
+		warmSlug: null,
+	});
+	const ranked = [...candidates].sort((a, b) => b.qualityScore - a.qualityScore);
+	const haveNow = ranked.filter((c) => keyScoped.has(c.model.slug)).length;
+	console.log(`\n===== ${tier} (axis ${axis}, floor ${floor.toFixed(1)}) — ${ranked.length} eligible in full catalog, ${haveNow} already on your key =====`);
+	console.log("  add?  slug".padEnd(46) + "q     in$/Mtok  ctx");
+	for (const c of ranked.slice(0, 14)) {
+		const tag = keyScoped.has(c.model.slug) ? "have " : "ADD  ";
+		console.log(
+			`  ${tag} ${c.model.slug.padEnd(38)} ${String(c.qualityScore).padEnd(5)} ${(c.model.price.prompt * 1e6).toFixed(2).padStart(7)}  ${c.model.contextLength}`,
+		);
+	}
+}