auto-model-router 0.20.0 → 0.22.0

package/src/server/http.ts

@@ -5,9 +5,12 @@ import { createProviders } from "./providers.ts";
 import { createBridgeFromConfig } from "../context/index.ts";
 import { createFeedbackStore, type Verdict } from "../cost/feedback.ts";
 import { createLedger } from "../cost/ledger.ts";
+import { createRetentionRunner } from "../cost/retention.ts";
+import { redactionRulesFor } from "../config/redaction.ts";
 import { createSessionOverrides } from "./overrides.ts";
 import { catalogView } from "./catalog-view.ts";
 import { buildUpstreamModels } from "../catalog/static-catalog.ts";
+import { invalidateFeedCache } from "../catalog/benchmark-feeds.ts";
 import { applyRequestPolicy, resolveProfile } from "../router/index.ts";
 import { parsePolicyHeader } from "../wire/openai/request.ts";
 import { createDigester } from "./digest.ts";
@@ -265,6 +268,18 @@ export function startServer(cfg: RouterConfig): StartedServer {
 		},
 	);
 
+	// Compile the redaction rules before the listener exists: a rule that does
+	// not load is a hole in the guard, and an operator who configured redaction
+	// must not get a router that started and forwarded anyway. Programmatic
+	// overrides (an embedder's) never pass through the config schema, so this is
+	// the only place their rules are checked.
+	const redactionRules = redactionRulesFor(cfg.redaction);
+	if (redactionRules.length > 0) {
+		// Names only. The patterns describe the secrets and the matches are the
+		// secrets; neither belongs in a log line.
+		log.info("redaction enabled", { rules: redactionRules.map((r) => r.name).join(","), scanTools: cfg.redaction.scanTools });
+	}
+
 	if (context.enabled) {
 		log.info("agentdox context bridge enabled", {
 			url: cfg.context.baseUrl,
@@ -296,8 +311,24 @@ export function startServer(cfg: RouterConfig): StartedServer {
 		log.warn("initial catalog fetch failed", { error: err instanceof Error ? err.message : String(err) });
 	});
 
-	// One housekeeping timer for both tables. `unref`'d so it never holds the
-	// process open.
+	// Ledger retention. The runner owns the once-an-hour floor, so the minute
+	// timer below, the boot run and `POST /v1/router/prune` cannot between them
+	// run a whole-ledger delete more often than that. It reads the window live,
+	// so a hot reload that lowers it applies on the next tick.
+	const retention = createRetentionRunner({ ledger, retentionDays: () => cfg.ledger.retentionDays });
+	const retain = (): void => {
+		try {
+			const result = retention.maybeRun();
+			if (result !== null && result.deleted > 0) {
+				log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
+			}
+		} catch (err) {
+			log.warn("ledger retention prune failed", { error: err instanceof Error ? err.message : String(err) });
+		}
+	};
+
+	// One housekeeping timer for all three tables. `unref`'d so it never holds
+	// the process open.
 	const pruneTimer = setInterval(() => {
 		try {
 			const dropped = conversations.prune(cfg.ledger.conversationTtlMs);
@@ -315,21 +346,12 @@ export function startServer(cfg: RouterConfig): StartedServer {
 		} catch (err) {
 			log.warn("context block prune failed", { error: err instanceof Error ? err.message : String(err) });
 		}
+		retain();
 	}, 60_000);
 	pruneTimer.unref();
 
-	// Ledger retention: hourly, and once at boot so a lowered setting takes
-	// effect without waiting. Reads the live config, so it hot-reloads.
-	const retain = (): void => {
-		try {
-			const dropped = ledger.prune?.(cfg.ledger.retentionDays) ?? 0;
-			if (dropped > 0) log.info("pruned ledger rows past retention", { dropped, retentionDays: cfg.ledger.retentionDays });
-		} catch (err) {
-			log.warn("ledger retention prune failed", { error: err instanceof Error ? err.message : String(err) });
-		}
-	};
-	const retentionTimer = setInterval(retain, 3_600_000);
-	retentionTimer.unref();
+	// Once shortly after boot, so a lowered window takes effect without waiting
+	// out an hour; the runner's own floor governs everything after that.
 	setTimeout(retain, 5_000).unref();
 
 	// Periodically refetch the (key-scoped) catalog in the background so
@@ -665,6 +687,15 @@ export function startServer(cfg: RouterConfig): StartedServer {
 						}),
 					);
 				}
+				if (req.method === "POST" && url.pathname === "/v1/router/prune") {
+					// A front door of the team edition holds a READ-ONLY handle on the
+					// ledger file by design, so this route is the only way it can act
+					// on its own retention policy — and the once-an-hour floor is the
+					// runner's, not the caller's, so calling it in a loop is harmless.
+					const result = retention.runNow();
+					if (result.deleted > 0) log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
+					return json({ ...result, retentionDays: cfg.ledger.retentionDays });
+				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.
 					const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
@@ -769,15 +800,25 @@ export function startServer(cfg: RouterConfig): StartedServer {
 				defaultScope: cfg.context.defaultScope === "" ? "(per-request header only)" : cfg.context.defaultScope,
 			});
 		}
-		if (!touched(changed, "openrouter", "ollama")) return false;
+		const benchmarksChanged = touched(changed, "benchmarks");
+		if (!touched(changed, "openrouter", "ollama") && !benchmarksChanged) return false;
+		if (benchmarksChanged) {
+			// The feed cache keeps its own ~daily TTL, so a refresh alone would rebuild
+			// the catalog from yesterday's feeds and the new Artificial Analysis key
+			// would do nothing until it expired. Age the row out so the refresh below
+			// re-fetches; the payload stays put, so a fetch that fails leaves the
+			// scores already serving in place. Only a benchmarks change does this —
+			// every other reconfigure keeps the cadence the cache is there for.
+			invalidateFeedCache(db);
+		}
 		// A key change makes the catalog key-scoped (or not), and enabling Ollama adds
 		// its models: the snapshot is rebuilt before the next turn ranks. Started, not
 		// awaited — the caller is a settings save, not a network client, and the
 		// previous snapshot serves turns until the new one lands.
 		void catalog
 			.refresh()
-			.then((snap) => log.info("catalog refreshed after an upstream change", { models: snap.models.length, ollama: cfg.ollama.enabled }))
-			.catch((err: unknown) => log.warn("catalog refresh after an upstream change failed; the previous snapshot stands", { error: err instanceof Error ? err.message : String(err) }));
+			.then((snap) => log.info("catalog refreshed after a live config change", { models: snap.models.length, ollama: cfg.ollama.enabled, benchmarks: benchmarksChanged }))
+			.catch((err: unknown) => log.warn("catalog refresh after a live config change failed; the previous snapshot stands", { error: err instanceof Error ? err.message : String(err) }));
 		return true;
 	}
 

package/src/server/redact.ts

@@ -0,0 +1,104 @@
+/**
+ * Redaction: strings an operator forbids from leaving the process.
+ *
+ * An operator with a compliance obligation needs two things a router can
+ * actually give: certainty that certain shapes of text never reach a provider,
+ * and evidence that the guard ran. This module is the first half; the ledger's
+ * `redactions` count is the second. Neither ever records WHAT was matched — a
+ * redaction log that quotes the secret is just a second copy of the secret.
+ *
+ * `runTurn` applies this to the rendered upstream body, which is the
+ * chat-completions shape every front end normalises to and every provider
+ * client renders from, so the guard sits between the router and ALL of them:
+ * a new upstream cannot bypass it by construction.
+ */
+
+import type { CompiledRedactionRule } from "../config/redaction.ts";
+
+/** Applies every rule to one string. Returns the text and how many matches were replaced. */
+export function redactText(text: string, rules: readonly CompiledRedactionRule[]): { text: string; count: number } {
+	let out = text;
+	let count = 0;
+	for (const rule of rules) {
+		// `replace` with a global regex resets and advances `lastIndex` itself,
+		// including over a zero-width position — which compilation refuses anyway.
+		out = out.replace(rule.regex, () => {
+			count++;
+			return rule.replacement;
+		});
+	}
+	return { text: out, count };
+}
+
+type Rec = Record<string, unknown>;
+const isRec = (v: unknown): v is Rec => v !== null && typeof v === "object" && !Array.isArray(v);
+
+/** Redacts a message's `content`, whether it is a string or an array of parts. Returns matches replaced. */
+function redactContent(message: Rec, rules: readonly CompiledRedactionRule[]): number {
+	const content = message.content;
+	if (typeof content === "string") {
+		const r = redactText(content, rules);
+		if (r.count > 0) message.content = r.text;
+		return r.count;
+	}
+	if (!Array.isArray(content)) return 0;
+	let count = 0;
+	for (const part of content) {
+		// Text parts only: an `image_url` part carries a data URI, which no
+		// redaction rule can meaningfully read and every rule would be slow over.
+		if (!isRec(part) || part.type !== "text" || typeof part.text !== "string") continue;
+		const r = redactText(part.text, rules);
+		if (r.count > 0) part.text = r.text;
+		count += r.count;
+	}
+	return count;
+}
+
+/**
+ * Redacts the rendered upstream body in place and returns how many matches
+ * were replaced.
+ *
+ * The body is the chat-completions shape: every front end (chat completions,
+ * Responses, Anthropic Messages) translates into it before parsing, and every
+ * upstream client renders its own protocol FROM it, so this one pass covers
+ * every wire in and every provider out.
+ *
+ * What is scanned:
+ *  - the text content of every message that is not a tool result — the system
+ *    prompt (including the injected agentdox block), the user's words, the
+ *    assistant's replay;
+ *  - with `scanTools`, tool-call arguments and tool-result content as well.
+ *    Off by default because tool results are where the bytes are: a turn's
+ *    prompt is mostly file content, so scanning them is most of the cost — and
+ *    also, for an operator who cares about a secret in a file the agent read,
+ *    most of the point.
+ *
+ * Tool NAMES, ids and the tool schemas are left alone: they are the harness's
+ * own vocabulary, not conversation content, and rewriting one breaks the
+ * call/result pairing the model needs.
+ */
+export function redactUpstreamBody(
+	body: Rec,
+	rules: readonly CompiledRedactionRule[],
+	opts: { scanTools: boolean },
+): number {
+	if (rules.length === 0) return 0;
+	const messages = body.messages;
+	if (!Array.isArray(messages)) return 0;
+	let count = 0;
+	for (const message of messages) {
+		if (!isRec(message)) continue;
+		const isToolResult = message.role === "tool";
+		if (!isToolResult || opts.scanTools) count += redactContent(message, rules);
+		if (!opts.scanTools || !Array.isArray(message.tool_calls)) continue;
+		for (const call of message.tool_calls) {
+			if (!isRec(call) || !isRec(call.function)) continue;
+			const args = call.function.arguments;
+			if (typeof args !== "string") continue;
+			const r = redactText(args, rules);
+			if (r.count > 0) call.function.arguments = r.text;
+			count += r.count;
+		}
+	}
+	return count;
+}

package/src/server/turn.ts

@@ -29,6 +29,8 @@ import {
 import { UpstreamError, type Dispatch, type UpstreamClient } from "../upstream/types.ts";
 import type { SessionOverrides } from "./overrides.ts";
 import { digestCompactionEdits, type CompactionDigester } from "./compaction-digest.ts";
+import { redactUpstreamBody } from "./redact.ts";
+import { redactionRulesFor } from "../config/redaction.ts";
 import { createLogger } from "../util/log.ts";
 import type { NormRequest, ResponseSink, TurnSummary, UpstreamChunk } from "../wire/types.ts";
 
@@ -165,6 +167,10 @@ export async function runTurn(
 	let pendingDecision: Decision | null = null;
 	// Digests made this turn, by edit; a retry re-plans and must not pay twice.
 	const digestMemo = new Map<string, string>();
+	// Compiled once per turn (and memoised across turns on the rules' own text),
+	// never per attempt: a retry renders the same request again and would
+	// otherwise recompile every pattern.
+	const redactionRules = redactionRulesFor(config.redaction);
 
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		// Client disconnected before anything was dispatched: spend nothing.
@@ -261,6 +267,18 @@ export async function runTurn(
 			...(decision.compactionPlan.length > 0 ? { compactionPlan: decision.compactionPlan } : {}),
 		});
 
+		// Redaction, immediately after the body is rendered and before ANYTHING
+		// can dispatch it. This is the only choke point that covers every path:
+		// all three front ends normalise to this shape, and every upstream client
+		// renders its own protocol from it, so a provider added later is covered
+		// without being told. The count is recorded on the ledger row below; the
+		// matched text is never logged, at any level.
+		let redactions = 0;
+		if (redactionRules.length > 0) {
+			redactions = redactUpstreamBody(body, redactionRules, { scanTools: config.redaction.scanTools });
+			if (redactions > 0) log.debug("redacted outgoing request", { matches: redactions, rules: redactionRules.length });
+		}
+
 		// Calibrate the token estimate against the bytes that actually go out —
 		// after compaction shrank the prompt and the context block was appended
 		// — not the raw request the estimate was taken from.
@@ -333,6 +351,9 @@ export async function runTurn(
 				upstreamGenerationId: generationId,
 				error: fields.error,
 				promptTokensSaved: decision.promptTokensSaved,
+				// Evidence that the guard ran, as a count and nothing more. Absent
+				// while redaction is off, so an untouched row stays NULL.
+				...(redactionRules.length === 0 ? {} : { redactions }),
 				// The ledger prices OpenRouter slugs from its own cached payload; any
 				// other provider's model exists only in the live catalog.
 				...(priceModel === undefined ? {} : { priceModel }),

package/src/util/sqlite.ts

@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 
 /** Bump when a migration is added; guarded below so reopening never regresses it. */
-const USER_VERSION = 18;
+const USER_VERSION = 19;
 
 const MIGRATIONS = `
 CREATE TABLE IF NOT EXISTS catalog_cache (
@@ -296,6 +296,16 @@ const MIGRATE_V18 = `
 ALTER TABLE ledger ADD COLUMN scope TEXT;
 `;
 
+// v19: ledger records how many strings redaction removed from the turn's
+// outgoing request (see `redaction` in the config). A COUNT and nothing else:
+// the whole point of redaction is that the matched text does not exist outside
+// the client, so the evidence that it ran must not reintroduce it. NULL on
+// every row written before this and on every turn with redaction off; 0 means
+// the rules ran and matched nothing.
+const MIGRATE_V19 = `
+ALTER TABLE ledger ADD COLUMN redactions INTEGER;
+`;
+
 // v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
 // BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
 // whole new table, created idempotently by the MIGRATIONS block above, so there
@@ -320,6 +330,17 @@ export function openDb(path: string): Database {
 	// ":memory:" has no parent directory to create.
 	if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
 	const db = new Database(path);
+	// Incremental auto-vacuum so retention can hand freed pages back to the
+	// filesystem (`PRAGMA incremental_vacuum` after a prune). SQLite only honours
+	// a change of vacuum mode on an empty database or across a full VACUUM, so
+	// this takes effect for NEW ledgers; an existing one keeps its mode and
+	// simply reuses freed pages instead of releasing them. Must precede the
+	// journal-mode change, and is best-effort: never fail an open over it.
+	try {
+		db.exec("PRAGMA auto_vacuum = INCREMENTAL");
+	} catch {
+		/* an existing database in another mode; freed pages are reused instead */
+	}
 	// WAL + NORMAL: single-writer local service; favours read latency on the turn hot path.
 	db.exec("PRAGMA journal_mode = WAL");
 	db.exec("PRAGMA synchronous = NORMAL");
@@ -338,6 +359,7 @@ export function openDb(path: string): Database {
 		if (!ledgerCols.some((c) => c.name === "hold_arm")) db.exec(MIGRATE_V8);
 		if (!ledgerCols.some((c) => c.name === "prompt_tokens_saved")) db.exec(MIGRATE_V12);
 		if (!ledgerCols.some((c) => c.name === "scope")) db.exec(MIGRATE_V18);
+		if (!ledgerCols.some((c) => c.name === "redactions")) db.exec(MIGRATE_V19);
 		const convCols = db.query("PRAGMA table_info(conversations)").all() as { name: string }[];
 		if (!convCols.some((c) => c.name === "context_version")) db.exec(MIGRATE_V11);
 		if (!convCols.some((c) => c.name === "compaction_plan")) db.exec(MIGRATE_V13);

package/test/benchmark-feeds.test.ts

@@ -4,6 +4,7 @@ import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
 import {
 	applyFeedScores,
 	fetchBenchlmScores,
+	invalidateFeedCache,
 	normalizeModelKey,
 	parseAaModels,
 	parseBenchlmModels,
@@ -198,6 +199,40 @@ describe("refreshFeedScores", () => {
 		db.close();
 	});
 
+	test("invalidateFeedCache re-fetches inside the TTL, and a failed forced fetch keeps the scores", async () => {
+		const db = openDb(":memory:");
+		let calls = 0;
+		let coding = 58;
+		const fakeFetch: FetchLike = async () => {
+			calls += 1;
+			return Response.json({
+				models: coding === 0 ? [] : [{ model: "MiniMax M3", creator: "MiniMax", evidenceStatus: "supported", categoryScores: { coding } }],
+			});
+		};
+
+		const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 1_000_000 });
+		await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 1000 });
+		await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 2000 });
+		expect(calls).toBe(1); // deep inside the TTL
+
+		// An Artificial Analysis key arriving cannot wait out the day still left on
+		// the cache; invalidating is what makes the next refresh actually fetch.
+		invalidateFeedCache(db);
+		coding = 71;
+		const forced = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 3000 });
+		expect(calls).toBe(2);
+		expect(forced[0]).toMatchObject({ key: "minimax-m3", coding: 71 });
+
+		// And the forced fetch is still best-effort: a feed that answers with nothing
+		// leaves the scores already serving in place rather than emptying them.
+		invalidateFeedCache(db);
+		coding = 0;
+		const after = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 4000 });
+		expect(calls).toBe(3);
+		expect(after[0]).toMatchObject({ key: "minimax-m3", coding: 71 });
+		db.close();
+	});
+
 	test("falls back to the stale cache when a refresh returns nothing", async () => {
 		const db = openDb(":memory:");
 		const seed = [{ key: "minimax-m3", creator: "minimax", coding: 58, source: "benchlm" }];

package/test/config-wizard.test.ts

@@ -142,7 +142,7 @@ describe("validateField", () => {
 
 describe("WIZARD_SECTIONS coverage", () => {
 	/** Leaves that are edited as whole records/arrays rather than fields. */
-	const RECORD_PATHS = new Set(["ollama.prices", "ollama.twins", "digest.toolAliases", "profiles", "upstreams"]);
+	const RECORD_PATHS = new Set(["ollama.prices", "ollama.twins", "digest.toolAliases", "profiles", "upstreams", "redaction.rules"]);
 
 	function leaves(obj: unknown, prefix = ""): string[] {
 		if (typeof obj !== "object" || obj === null || Array.isArray(obj)) return [prefix];

package/test/failover.test.ts

@@ -83,6 +83,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 		digest: { enabled: false, minBytes: 12_000, maxBytes: 400_000, tools: ["read"], fromTier: "moderate", tier: "simple", model: "", maxOutputTokens: 700, maxCostUsd: 0.02, timeoutMs: 25_000, toolAliases: {} },
 		profiles: [],
 		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 , retentionDays: 0,},
+		redaction: { enabled: false, rules: [], scanTools: false },
 		adaptiveTierFloors: true,
 		adaptivePriceCeilings: false,
 		logLevel: "silent",

package/test/migrations.test.ts

@@ -24,7 +24,7 @@ import { openDb } from "../src/util/sqlite.ts";
 
 const FIXTURES = join(import.meta.dir, "fixtures", "migrations");
 const files = readdirSync(FIXTURES).filter((f) => /^router-v\d+\.db$/.test(f)).sort((a, b) => Number(/\d+/.exec(a)![0]) - Number(/\d+/.exec(b)![0]));
-const CURRENT_VERSION = 18;
+const CURRENT_VERSION = 19;
 
 describe("schema migrations from every shipped version", () => {
 	test("fixtures exist for the versions that shipped", () => {
@@ -44,7 +44,7 @@ describe("schema migrations from every shipped version", () => {
 				expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
 				// Every column the current code writes exists after migration.
 				const ledgerCols = new Set((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).map((c) => c.name));
-				for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved", "scope"]) expect(ledgerCols.has(c)).toBe(true);
+				for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved", "scope", "redactions"]) expect(ledgerCols.has(c)).toBe(true);
 				const convCols = new Set((db.query("PRAGMA table_info(conversations)").all() as { name: string }[]).map((c) => c.name));
 				for (const c of ["context_version", "compaction_plan", "compaction_plan_tokens", "upgrade_deferred_tier"]) expect(convCols.has(c)).toBe(true);
 				// The fixture's ledger row survived the ALTERs with its values.
@@ -67,7 +67,10 @@ describe("schema migrations from every shipped version", () => {
 				expect(exportRows(db, 0, null).map((r) => r.scope)).toEqual([""]);
 				expect(spendUsdSince(db, 0, null, "acme.api")).toBe(0);
 				expect(spendUsdSince(db, 0, null)).toBeGreaterThanOrEqual(0);
-				expect(ledger.prune?.(0)).toBe(0);
+				expect(ledger.prune?.(0)?.deleted).toBe(0);
+				// v19: the fixture's row predates `redactions`, so nothing claims a
+				// redaction happened on it and the report totals it as zero.
+				expect(buildUsageReport(db, { windowDays: 3650 }).totals.redactions).toBe(0);
 			} finally {
 				db.close();
 				try {

package/test/reconfigure.test.ts

@@ -1,8 +1,12 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, test } from "bun:test";
 import { applyConfigPatch, assignInPlace, touched } from "../src/config/apply.ts";
 import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
 import type { RouterConfig } from "../src/config/types.ts";
 import { startServer } from "../src/server/http.ts";
+import { openDb } from "../src/util/sqlite.ts";
 
 /**
  * Live reconfiguration: a running router follows a config change without a
@@ -101,6 +105,55 @@ describe("a running server reconfigures", () => {
 		}
 	});
 
+	test("a benchmarks change ages the feed cache out; an unrelated change leaves it alone", async () => {
+		const dir = mkdtempSync(join(tmpdir(), "amr-benchfeed-"));
+		const cfg: RouterConfig = {
+			...base(),
+			ledger: { ...DEFAULT_CONFIG.ledger, path: join(dir, "ledger.db") },
+			// Off so the background catalog refresh cannot re-fetch the feeds under the
+			// assertion. Invalidation does not depend on it: the point is that the row
+			// is aged out before the refresh runs, whatever the refresh then does.
+			benchmarks: { ...DEFAULT_CONFIG.benchmarks, enabled: false },
+		};
+		const started = startServer(cfg);
+		const db = openDb(cfg.ledger.path);
+		const fetchedAt = (): number | null => {
+			const row = db.query("SELECT fetched_at_ms FROM benchmark_cache WHERE id = 1").get() as { fetched_at_ms: number } | null;
+			return row === null ? null : row.fetched_at_ms;
+		};
+		try {
+			const seeded = Date.now();
+			db.query("INSERT INTO benchmark_cache (id, payload, fetched_at_ms) VALUES (1, ?, ?)").run("[]", seeded);
+
+			// The ~daily feed cadence is not every settings save's to reset.
+			const unrelated = await started.reconfigure({ filters: { latencyWeight: 0.42 } });
+			expect(unrelated.catalogRefreshing).toBe(false);
+			expect(fetchedAt()).toBe(seeded);
+
+			// A key an operator just pasted has to reach the catalog now, not tomorrow.
+			const r = await started.reconfigure({ benchmarks: { artificialAnalysisApiKey: "aa-key" } });
+			expect(r.rejected).toEqual([]);
+			expect(r.changed).toEqual(["benchmarks.artificialAnalysisApiKey"]);
+			expect(r.catalogRefreshing).toBe(true);
+			expect(fetchedAt()).toBe(0);
+			// The payload survives the invalidation, so a failed re-fetch has something
+			// to fall back on.
+			const row = db.query("SELECT payload FROM benchmark_cache WHERE id = 1").get() as { payload: string } | null;
+			expect(row?.payload).toBe("[]");
+
+			// Clearing it again is a benchmarks change too: scores from a key that is
+			// gone must stop being used just as promptly.
+			db.query("UPDATE benchmark_cache SET fetched_at_ms = ? WHERE id = 1").run(seeded);
+			const cleared = await started.reconfigure({ benchmarks: { artificialAnalysisApiKey: "" } });
+			expect(cleared.catalogRefreshing).toBe(true);
+			expect(fetchedAt()).toBe(0);
+		} finally {
+			db.close();
+			await started.stop();
+			rmSync(dir, { recursive: true, force: true });
+		}
+	});
+
 	test("the socket and the ledger file are refused rather than half-applied", async () => {
 		const cfg = base();
 		const started = startServer(cfg);