auto-model-router 0.21.0 → 0.22.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.21.0",
+    "version": "0.22.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.21.0",
+      "version": "0.22.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/README.md

@@ -1942,6 +1942,19 @@ rather than the refresh failing.
 **Roughly 60% of the catalog is unscored anyway.** Scores are never imputed
 from price, so unscored models are only ever eligible where the floor is zero.
 
+**External feeds backfill what neither list scores** (the `benchmarks` block, on
+by default). Artificial Analysis's own v2 data API
+(`benchmarks.artificialAnalysisApiKey`, or `ARTIFICIAL_ANALYSIS_API_KEY`) and the
+keyless BenchLM leaderboard fill only the axes a model is *missing* — a score
+OpenRouter published is never overwritten, and a name matches exactly or the
+model stays honestly unscored. The feeds are cached in `benchmark_cache` on their
+own slow cadence (`benchmarks.refreshMs`, a day) so the minute-scale catalog
+refresh never hits them. Changing the `benchmarks` block itself is the exception:
+a key pasted into a front door's settings, or cleared out of them, ages that cache
+out and rebuilds the catalog immediately instead of leaving the change inert until
+tomorrow. Every fetch stays best-effort — one that fails or returns nothing leaves
+the scores already serving in place.
+
 ---
 
 ## Adaptive tier floors

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/catalog/benchmark-feeds.ts

@@ -318,7 +318,10 @@ export async function refreshFeedScores(cfg: RouterConfig, db: Database, opts: R
 
 	const row = db.query("SELECT payload, fetched_at_ms FROM benchmark_cache WHERE id = 1").get() as CacheRow | null;
 	const cached: FeedScore[] | null = row === null ? null : parseFeedScores(row.payload);
-	if (row !== null && cached !== null && now - row.fetched_at_ms < bm.refreshMs) return cached;
+	// A zero timestamp is `invalidateFeedCache`'s marker, not a real fetch time:
+	// the payload stays readable as a fallback but never counts as fresh again.
+	const fresh = row !== null && row.fetched_at_ms > 0 && now - row.fetched_at_ms < bm.refreshMs;
+	if (fresh && cached !== null) return cached;
 
 	const feedOpts: FetchOpts = { timeoutMs: bm.timeoutMs };
 	if (opts.fetchImpl !== undefined) feedOpts.fetchImpl = opts.fetchImpl;
@@ -347,6 +350,21 @@ export async function refreshFeedScores(cfg: RouterConfig, db: Database, opts: R
 	return merged;
 }
 
+/**
+ * Mark the cached feeds stale so the next `refreshFeedScores` re-fetches instead
+ * of sitting out `benchmarks.refreshMs` (~a day). The benchmarks config changing
+ * is what calls this: an Artificial Analysis key that only takes effect tomorrow
+ * is a key the operator will believe is broken, and a key taken away has to stop
+ * filling scores just as promptly.
+ *
+ * The row is aged out, never deleted. A forced re-fetch that then fails must
+ * still find the previous scores to fall back on — best-effort is the rule here,
+ * and invalidation must not be the one path that empties the catalog.
+ */
+export function invalidateFeedCache(db: Database): void {
+	db.query("UPDATE benchmark_cache SET fetched_at_ms = 0 WHERE id = 1").run();
+}
+
 /** Validate a persisted `FeedScore[]` blob, skipping any entry that drifted. */
 function parseFeedScores(payload: string): FeedScore[] | null {
 	let parsed: unknown;

package/src/server/http.ts

@@ -10,6 +10,7 @@ import { redactionRulesFor } from "../config/redaction.ts";
 import { createSessionOverrides } from "./overrides.ts";
 import { catalogView } from "./catalog-view.ts";
 import { buildUpstreamModels } from "../catalog/static-catalog.ts";
+import { invalidateFeedCache } from "../catalog/benchmark-feeds.ts";
 import { applyRequestPolicy, resolveProfile } from "../router/index.ts";
 import { parsePolicyHeader } from "../wire/openai/request.ts";
 import { createDigester } from "./digest.ts";
@@ -799,15 +800,25 @@ export function startServer(cfg: RouterConfig): StartedServer {
 				defaultScope: cfg.context.defaultScope === "" ? "(per-request header only)" : cfg.context.defaultScope,
 			});
 		}
-		if (!touched(changed, "openrouter", "ollama")) return false;
+		const benchmarksChanged = touched(changed, "benchmarks");
+		if (!touched(changed, "openrouter", "ollama") && !benchmarksChanged) return false;
+		if (benchmarksChanged) {
+			// The feed cache keeps its own ~daily TTL, so a refresh alone would rebuild
+			// the catalog from yesterday's feeds and the new Artificial Analysis key
+			// would do nothing until it expired. Age the row out so the refresh below
+			// re-fetches; the payload stays put, so a fetch that fails leaves the
+			// scores already serving in place. Only a benchmarks change does this —
+			// every other reconfigure keeps the cadence the cache is there for.
+			invalidateFeedCache(db);
+		}
 		// A key change makes the catalog key-scoped (or not), and enabling Ollama adds
 		// its models: the snapshot is rebuilt before the next turn ranks. Started, not
 		// awaited — the caller is a settings save, not a network client, and the
 		// previous snapshot serves turns until the new one lands.
 		void catalog
 			.refresh()
-			.then((snap) => log.info("catalog refreshed after an upstream change", { models: snap.models.length, ollama: cfg.ollama.enabled }))
-			.catch((err: unknown) => log.warn("catalog refresh after an upstream change failed; the previous snapshot stands", { error: err instanceof Error ? err.message : String(err) }));
+			.then((snap) => log.info("catalog refreshed after a live config change", { models: snap.models.length, ollama: cfg.ollama.enabled, benchmarks: benchmarksChanged }))
+			.catch((err: unknown) => log.warn("catalog refresh after a live config change failed; the previous snapshot stands", { error: err instanceof Error ? err.message : String(err) }));
 		return true;
 	}
 

package/test/benchmark-feeds.test.ts

@@ -4,6 +4,7 @@ import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
 import {
 	applyFeedScores,
 	fetchBenchlmScores,
+	invalidateFeedCache,
 	normalizeModelKey,
 	parseAaModels,
 	parseBenchlmModels,
@@ -198,6 +199,40 @@ describe("refreshFeedScores", () => {
 		db.close();
 	});
 
+	test("invalidateFeedCache re-fetches inside the TTL, and a failed forced fetch keeps the scores", async () => {
+		const db = openDb(":memory:");
+		let calls = 0;
+		let coding = 58;
+		const fakeFetch: FetchLike = async () => {
+			calls += 1;
+			return Response.json({
+				models: coding === 0 ? [] : [{ model: "MiniMax M3", creator: "MiniMax", evidenceStatus: "supported", categoryScores: { coding } }],
+			});
+		};
+
+		const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 1_000_000 });
+		await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 1000 });
+		await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 2000 });
+		expect(calls).toBe(1); // deep inside the TTL
+
+		// An Artificial Analysis key arriving cannot wait out the day still left on
+		// the cache; invalidating is what makes the next refresh actually fetch.
+		invalidateFeedCache(db);
+		coding = 71;
+		const forced = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 3000 });
+		expect(calls).toBe(2);
+		expect(forced[0]).toMatchObject({ key: "minimax-m3", coding: 71 });
+
+		// And the forced fetch is still best-effort: a feed that answers with nothing
+		// leaves the scores already serving in place rather than emptying them.
+		invalidateFeedCache(db);
+		coding = 0;
+		const after = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 4000 });
+		expect(calls).toBe(3);
+		expect(after[0]).toMatchObject({ key: "minimax-m3", coding: 71 });
+		db.close();
+	});
+
 	test("falls back to the stale cache when a refresh returns nothing", async () => {
 		const db = openDb(":memory:");
 		const seed = [{ key: "minimax-m3", creator: "minimax", coding: 58, source: "benchlm" }];

package/test/reconfigure.test.ts

@@ -1,8 +1,12 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, test } from "bun:test";
 import { applyConfigPatch, assignInPlace, touched } from "../src/config/apply.ts";
 import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
 import type { RouterConfig } from "../src/config/types.ts";
 import { startServer } from "../src/server/http.ts";
+import { openDb } from "../src/util/sqlite.ts";
 
 /**
  * Live reconfiguration: a running router follows a config change without a
@@ -101,6 +105,55 @@ describe("a running server reconfigures", () => {
 		}
 	});
 
+	test("a benchmarks change ages the feed cache out; an unrelated change leaves it alone", async () => {
+		const dir = mkdtempSync(join(tmpdir(), "amr-benchfeed-"));
+		const cfg: RouterConfig = {
+			...base(),
+			ledger: { ...DEFAULT_CONFIG.ledger, path: join(dir, "ledger.db") },
+			// Off so the background catalog refresh cannot re-fetch the feeds under the
+			// assertion. Invalidation does not depend on it: the point is that the row
+			// is aged out before the refresh runs, whatever the refresh then does.
+			benchmarks: { ...DEFAULT_CONFIG.benchmarks, enabled: false },
+		};
+		const started = startServer(cfg);
+		const db = openDb(cfg.ledger.path);
+		const fetchedAt = (): number | null => {
+			const row = db.query("SELECT fetched_at_ms FROM benchmark_cache WHERE id = 1").get() as { fetched_at_ms: number } | null;
+			return row === null ? null : row.fetched_at_ms;
+		};
+		try {
+			const seeded = Date.now();
+			db.query("INSERT INTO benchmark_cache (id, payload, fetched_at_ms) VALUES (1, ?, ?)").run("[]", seeded);
+
+			// The ~daily feed cadence is not every settings save's to reset.
+			const unrelated = await started.reconfigure({ filters: { latencyWeight: 0.42 } });
+			expect(unrelated.catalogRefreshing).toBe(false);
+			expect(fetchedAt()).toBe(seeded);
+
+			// A key an operator just pasted has to reach the catalog now, not tomorrow.
+			const r = await started.reconfigure({ benchmarks: { artificialAnalysisApiKey: "aa-key" } });
+			expect(r.rejected).toEqual([]);
+			expect(r.changed).toEqual(["benchmarks.artificialAnalysisApiKey"]);
+			expect(r.catalogRefreshing).toBe(true);
+			expect(fetchedAt()).toBe(0);
+			// The payload survives the invalidation, so a failed re-fetch has something
+			// to fall back on.
+			const row = db.query("SELECT payload FROM benchmark_cache WHERE id = 1").get() as { payload: string } | null;
+			expect(row?.payload).toBe("[]");
+
+			// Clearing it again is a benchmarks change too: scores from a key that is
+			// gone must stop being used just as promptly.
+			db.query("UPDATE benchmark_cache SET fetched_at_ms = ? WHERE id = 1").run(seeded);
+			const cleared = await started.reconfigure({ benchmarks: { artificialAnalysisApiKey: "" } });
+			expect(cleared.catalogRefreshing).toBe(true);
+			expect(fetchedAt()).toBe(0);
+		} finally {
+			db.close();
+			await started.stop();
+			rmSync(dir, { recursive: true, force: true });
+		}
+	});
+
 	test("the socket and the ledger file are refused rather than half-applied", async () => {
 		const cfg = base();
 		const started = startServer(cfg);