auto-model-router 0.36.0 → 0.38.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.36.0",
+    "version": "0.38.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.36.0",
+      "version": "0.38.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/README.md

@@ -753,8 +753,9 @@ modelRoles:
 
 The router decides the concrete OpenRouter model **per turn**; omp only sees the
 virtual profile it picked. Every routed response carries
-`x-auto-model-router-model`, `x-auto-model-router-tier`, `x-auto-model-router-cost-usd`, and
-`x-auto-model-router-attempts`.
+`x-auto-model-router-model`, `x-auto-model-router-tier`, `x-auto-model-router-cost-usd`,
+`x-auto-model-router-attempts`, and `x-request-id` — the id the turn's ledger rows
+are filed under (see [Request ids](#request-ids)).
 
 ---
 
@@ -764,8 +765,10 @@ The ledger records every dispatch: model decided and served, tier, provider,
 tokens (including cached), reported cost, time to first token, total latency,
 escalation signal, error, the agentdox context scope the turn carried (the
 project it belongs to; NULL for a turn that carried none, and for every row
-written before v0.19.0) and how many strings redaction removed from the request
-(NULL when redaction was off, and for every row written before v0.21.0). Three views aggregate it, all from the same
+written before v0.19.0), how many strings redaction removed from the request
+(NULL when redaction was off, and for every row written before v0.21.0) and the
+`request_id` the turn answered (indexed; NULL for every row written before
+v0.37.0 — see [Request ids](#request-ids)). Three views aggregate it, all from the same
 `buildUsageReport` in `src/cost/report.ts`:
 
 - `/router report` in omp — a fullscreen hub with the `/models` look: views
@@ -786,9 +789,10 @@ written before v0.19.0) and how many strings redaction removed from the request
   lists verdicts by model and the recent ones with the harness that gave them, and
   `GET /v1/router/decisions?harness=&days=|since=&slug=&tier=&limit=` is the decision trail
   itself, newest first, each turn with its reasons, the classifier's view, forecast against
-  bill, escalation signal and verdicts (`?session=` narrows to one omp session, as `/router
-  why` does). These are what a front door such as the team edition reads instead of the
-  ledger file.
+  bill, escalation signal, verdicts and its `requestId` (`?session=` narrows to one omp
+  session, as `/router why` does; `?requestId=` narrows to the one request a customer
+  quoted, every attempt of it included). These are what a front door such as the team
+  edition reads instead of the ledger file.
 - `GET /v1/router/catalog[?policy=<X-Omp-Policy JSON>]` — the model catalog as data: every
   model the router knows, sorted by slug, with provider (`openrouter`, `ollama`, a named
   upstream's id), vendor, context, capabilities, prices in USD per million tokens and quality
@@ -830,6 +834,35 @@ to save money (after a burst, the next 15 minutes ran 84–1,577 successes per
 escalation already cover the retry. Use a spike as the cue to `/router pin`
 or deny a model for the session.
 
+### Request ids
+
+Every turn is recorded under the id of the HTTP request it answered, so an id a
+customer quotes names **that** turn instead of "whichever turn of theirs was
+closest in time".
+
+- **Read.** `X-Request-Id` on `/v1/chat/completions`, `/v1/responses` and
+  `/v1/messages`. It is opaque, and it comes from outside, so it is accepted
+  only when it is at most 128 characters of letters, digits and `. _ - : + =`
+  (every id in circulation: a UUID, a ULID, a `traceparent`, a proxy's own).
+  Anything else — too long, a control character, a newline, whitespace — is
+  refused **whole** rather than truncated or stripped: a repaired id looks
+  valid and matches no row.
+- **Minted when absent.** A caller that sends no usable header gets one minted,
+  prefixed `amr-`, so every turn is addressable and a direct user of the router
+  has what a front door's customers have. The prefix is also refused *inbound*:
+  no caller can pass an id off as one the router minted.
+- **Returned.** The turn's response carries `x-request-id` (the caller's value
+  unchanged, or the minted one), and every ledger row of the turn — each
+  escalation attempt, and a digest taken during it — carries it as `request_id`,
+  reported as `requestId` on `GET /v1/router/decisions`. `?requestId=` there
+  filters to exactly those rows; an id nothing was recorded under answers with
+  no entries, and a mangled one is a 400 rather than a silently ignored filter.
+- **Both directions.** A caller that sends no header behaves exactly as before.
+  A row written before v0.37.0 reads as having no id (`null`), never as an
+  error, and is still found by every other filter. `/health` lists `request-id`
+  in `features`, so a front door can tell whether the router it is talking to
+  records ids before it relies on them.
+
 Spend follows the ledger's rule — the provider's reported cost when it gave
 one, else the usage-priced figure the router computed, else the forecast.
 Speed uses only clean streamed rows (TTFT recorded, no error); tokens/s is
@@ -1038,11 +1071,19 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
 
 The model that produced the rejected output never serves the retry, at this
 tier or the next. Signals that indict the *provider* rather than the tier —
-`empty_completion`, `refusal`, and an error finish — first try a different
-model in the **same** tier (bounded, like a 5xx failover) and only then step
-up; structural signals (`malformed_tool_args`, `repeat_tool_call`, a truncated
-tool call) escalate a tier directly. A client that hangs up after the finish
-event has already arrived is treated as a completed turn, not an error.
+`empty_completion`, `refusal`, `content_filter`, and an error finish — first
+try a different model in the **same** tier (bounded, like a 5xx failover) and
+only then step up; structural signals (`malformed_tool_args`,
+`repeat_tool_call`, a truncated tool call) escalate a tier directly. A client
+that hangs up after the finish event has already arrived is treated as a
+completed turn, not an error.
+
+A `content_filter` finish (the provider's filter stopping the completion,
+including Anthropic's `refusal` stop reason, which maps to it) rides the
+`refusal` trigger: the same failure class — the provider declining the work —
+so a config that already opts into refusal escalation gets it without a
+change. A filter finish that produced NO text is an `empty_completion`
+instead: nothing was filtered, so a plain retry is the fix.
 
 ### `hysteresis` — cache-aware model stickiness
 
@@ -1648,9 +1689,30 @@ left alone with a note. A remote without skills answers 404 and nothing happens.
 
 A remote whose `/setup/info` says `mcp: true` (a team edition serving shared context) also
 gets a `team-context` MCP server written for omp (`~/.omp/agent/mcp.json`) and Claude Code
-(`~/.claude.json`): `type: http`, the remote's `/mcp`, the member key in the Authorization
-header. Every refresh rewrites it with the current key, like models.yml; other servers in
-those files are untouched, and a remote that stops serving MCP has the entry removed.
+(`~/.claude.json`): `type: http`, the remote's `/mcp`, and a bearer token in the
+Authorization header. Other servers in those files are untouched, and a remote that stops
+serving MCP has the entry removed.
+
+**Which bearer** is what `/setup/info`'s `mcpAuth` decides. An MCP client substitutes its
+configuration once, at startup, while a team edition rotates the member access key roughly
+every 72 hours - so an entry carrying the access key 401s mid-session every few days and
+reads to the member as "re-authorise". A team that answers `mcpAuth: "context-token"` mints
+a second credential instead (`amrctx_...`, a year long) that can do nothing but read and
+write that member's shared project context: it is refused on turns, on the admin API, on
+the portal and on SCIM. `connect` takes it from `/setup/exchange` when a setup token was
+used, and mints one at `POST /me/context-tokens` otherwise, so `--key` and `--setup-token`
+both end up with one; the team binds it to the calling device, so revoking the machine
+revokes the token with it.
+
+That token is a long-lived secret, so it goes where the refresh token goes - the OS
+credential store, in its own slot (`<refresh account>#context`, falling back to
+`<router home>/context.token`, owner-readable only). `remote.json` records only which store
+holds it, its expiry and its id. **A refresh does not rewrite the MCP entry**: it re-writes
+models.yml and every other harness config with the new access key and leaves `team-context`
+holding the token it already has, renewing it only when there is none or it is within 30
+days of expiring. A team edition that answers `mcpAuth: "member-key"`, or says nothing at
+all, keeps the old behaviour exactly - the entry carries the access key and every refresh
+rewrites it.
 
 ## Direct upstreams: OpenAI, Azure OpenAI, Anthropic, vLLM
 
@@ -2024,6 +2086,46 @@ out and rebuilds the catalog immediately instead of leaving the change inert unt
 tomorrow. Every fetch stays best-effort — one that fails or returns nothing leaves
 the scores already serving in place.
 
+**A front door can supply the rest** (`benchmarks.extraScores`, empty by default).
+The feeds cover a lot and still leave holes, and a *partial* hole is the expensive
+kind: a model carrying intelligence and neither coding nor agentic clears no tier
+floor above the cheapest on those axes, so it is never picked no matter how good
+or how cheap it is. Whatever curates models in front of the router — a team
+dashboard, a config file you maintain — can hand over the numbers it has:
+
+```yaml
+benchmarks:
+  extraScores:
+    - key: deepseek/deepseek-v4.1-flash  # the OpenRouter slug is fine; it is normalised
+      creator: deepseek                  # optional, and only ever a tie-break
+      coding: 55.2
+      agentic: 31.8
+      source: vendor                     # or `neutral`
+```
+
+`source` is the provenance and only two values are accepted: **`neutral`**, a
+benchmark's own leaderboard, and **`vendor`**, a self-reported model-card number
+that the supplier is expected to have discounted already — the router applies no
+discount of its own, because a discount applied twice is its own distortion.
+Nothing else may be claimed: an entry calling itself `artificial_analysis` would
+outrank BenchLM on the strength of a label, and one calling itself `local` would
+write into the lane `useLocalScores` gates.
+
+These obey exactly the rules the feeds do, and sit exactly where the name says in
+the order: **Artificial Analysis → BenchLM → `neutral` → `vendor` → `local`**, and
+none of them touches an axis that already has a value. So a supplied score can
+only ever fill a hole; it can never move a number a published source measured.
+Which axis came from where is recorded per model under `benchmarks.fill_sources`.
+
+Rows arriving over a live config change (a dashboard save) are sanitised on use:
+a malformed one is dropped with a warning and the rest still apply, because the
+same patch carries unrelated settings and one bad row must not take an operator's
+key save down with it. An unreadable score leaves its axis **absent**, never zero
+— a zero would satisfy `trivial` and bid for every turn. Rows written into
+`config.yml` are validated strictly instead, like every other key in the file. A
+changed table ages the feed cache out and rebuilds the catalog straight away, so
+it applies on the next refresh rather than waiting out the day.
+
 ---
 
 ## Adaptive tier floors

package/omp-extension/remote-logic.ts

@@ -32,6 +32,17 @@ export interface RemoteRouter {
 	refreshTokenStore?: "dpapi" | "keychain" | "secret-service" | "file";
 	/** The account the store files it under (`<userId>@<remote host>`). */
 	refreshAccount?: string;
+	/**
+	 * The team edition's context token — the credential the shared-context MCP
+	 * server holds. It is long-lived (a year) and powerless outside that member's
+	 * project context, so it never sits in remote.json: the store named here holds
+	 * it, under `contextAccount` (`<refresh account>#context`). The expiry and the
+	 * id are recorded so a refresh knows when to renew it and which one it is.
+	 */
+	contextTokenStore?: "dpapi" | "keychain" | "secret-service" | "file";
+	contextAccount?: string;
+	contextTokenExpiresAtMs?: number;
+	contextTokenId?: string;
 	keyExpiresAtMs?: number;
 	refreshExpiresAtMs?: number;
 	/** What the remote calls this machine. */
@@ -62,6 +73,10 @@ export function parseRemoteRouter(text: string): RemoteRouter | null {
 			...(typeof raw.refreshToken === "string" && raw.refreshToken !== "" ? { refreshToken: raw.refreshToken } : {}),
 			...(raw.refreshTokenStore === "dpapi" || raw.refreshTokenStore === "keychain" || raw.refreshTokenStore === "secret-service" || raw.refreshTokenStore === "file" ? { refreshTokenStore: raw.refreshTokenStore } : {}),
 			...(typeof raw.refreshAccount === "string" && raw.refreshAccount !== "" ? { refreshAccount: raw.refreshAccount } : {}),
+			...(raw.contextTokenStore === "dpapi" || raw.contextTokenStore === "keychain" || raw.contextTokenStore === "secret-service" || raw.contextTokenStore === "file" ? { contextTokenStore: raw.contextTokenStore } : {}),
+			...(typeof raw.contextAccount === "string" && raw.contextAccount !== "" ? { contextAccount: raw.contextAccount } : {}),
+			...(typeof raw.contextTokenExpiresAtMs === "number" ? { contextTokenExpiresAtMs: raw.contextTokenExpiresAtMs } : {}),
+			...(typeof raw.contextTokenId === "string" && raw.contextTokenId !== "" ? { contextTokenId: raw.contextTokenId } : {}),
 			...(typeof raw.keyExpiresAtMs === "number" ? { keyExpiresAtMs: raw.keyExpiresAtMs } : {}),
 			...(typeof raw.refreshExpiresAtMs === "number" ? { refreshExpiresAtMs: raw.refreshExpiresAtMs } : {}),
 			...(typeof raw.device === "string" && raw.device !== "" ? { device: raw.device } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.36.0",
+  "version": "0.38.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/catalog/benchmark-feeds.ts

@@ -11,14 +11,23 @@
  *    baseline, but only when a key is configured.
  *  - BenchLM (`/api/data/leaderboard`, keyless): covers the models AA omits.
  *
+ * Both still leave holes, and the expensive kind is a PARTIAL one: a model with
+ * intelligence and neither coding nor agentic clears no tier floor above the
+ * cheapest on those axes and is simply never picked. `benchmarks.extraScores`
+ * is the seam for that — curated rows a front door supplies (`suppliedScores`),
+ * under exactly the rules below.
+ *
  * Three rules, all load-bearing:
  *
  *  - FILL, NEVER OVERWRITE. A score OpenRouter already published wins; the feeds
  *    only supply axes that are absent. Two suites measure the same idea on
  *    different tests, so letting one overwrite the other would make a model's
  *    score jump with whichever feed refreshed last.
- *  - PER AXIS, AA BEFORE BENCHLM. AA is the stronger source and fills first;
- *    BenchLM fills whatever axis AA still left empty.
+ *  - PER AXIS, STRONGEST SOURCE FIRST (`FILL_ORDER`). AA fills first, BenchLM
+ *    fills whatever axis AA left empty, then anything the front door supplied
+ *    through `benchmarks.extraScores` (a neutral leaderboard ahead of a
+ *    self-reported vendor number), then our own eval. Every one of them only
+ *    ever fills a hole the ones above it left.
  *  - MATCH EXACTLY OR NOT AT ALL. Matching is on a normalized model-name key,
  *    with the creator used only to break a tie between two rows that share a
  *    key. A fuzzy match would let a 7B inherit a 72B's score and then be handed
@@ -36,7 +45,40 @@ import type { QualityAxis } from "../config/types.ts";
 export const AA_MODELS_URL = "https://artificialanalysis.ai/api/v2/data/llms/models";
 export const BENCHLM_URL = "https://benchlm.ai/api/data/leaderboard";
 
-export type FeedSource = "artificial_analysis" | "benchlm" | "local";
+/**
+ * Where one score came from, and — through `FILL_ORDER` — what it may outrank.
+ *
+ *  - `artificial_analysis`, `benchlm`: the fetched feeds described above.
+ *  - `neutral`, `vendor`: supplied through `benchmarks.extraScores` by whatever
+ *    sits in front of the router. `neutral` is a benchmark's own leaderboard;
+ *    `vendor` is a self-reported model-card number. They stay two members rather
+ *    than collapsing into one `supplied` because that distinction is the whole
+ *    reason the supplier curated the table, and collapsing would discard it at
+ *    the boundary. The router rescales neither: a `vendor` number is expected to
+ *    arrive already discounted, and a discount applied twice is its own lie.
+ *  - `local`: our own eval harness, gated by `benchmarks.useLocalScores`.
+ */
+export type FeedSource = "artificial_analysis" | "benchlm" | "neutral" | "vendor" | "local";
+
+/**
+ * Fill priority, strongest first — one list rather than a chain of ifs, so the
+ * ordering rule is a thing a test can point at. A published score is not in it
+ * at all: an axis that already has a value is skipped before this is consulted.
+ */
+export const FILL_ORDER: readonly FeedSource[] = ["artificial_analysis", "benchlm", "neutral", "vendor", "local"];
+
+/** The sources the fetched-feed cache may hold — exactly what fetches write it. */
+const FETCHED_SOURCES: readonly FeedSource[] = ["artificial_analysis", "benchlm"];
+
+/** The sources `benchmarks.extraScores` may claim. Anything else is dropped. */
+export const SUPPLIED_SOURCES: readonly FeedSource[] = ["neutral", "vendor"];
+
+/**
+ * Cap on `benchmarks.extraScores`. The curated table is a few hundred rows; this
+ * sits far above it and exists only so one config patch cannot hand the fill
+ * path an unbounded list. The excess is dropped; the rest still applies.
+ */
+export const MAX_EXTRA_SCORES = 2_000;
 
 /** One model's scores from one feed, on the router's three axes (0-100). */
 export interface FeedScore {
@@ -61,6 +103,13 @@ export interface FillResult {
 
 const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
 
+/** A zeroed per-source counter, derived from `FILL_ORDER` so it cannot drift from it. */
+function zeroSources(): Record<FeedSource, number> {
+	const out = {} as Record<FeedSource, number>;
+	for (const source of FILL_ORDER) out[source] = 0;
+	return out;
+}
+
 function asRecord(value: unknown): Record<string, unknown> | null {
 	return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
 }
@@ -231,7 +280,7 @@ export function applyFeedScores(rawModels: unknown[], feeds: FeedScore[]): FillR
 	const result: FillResult = {
 		modelsFilled: 0,
 		axes: { coding: 0, intelligence: 0, agentic: 0 },
-		sources: { artificial_analysis: 0, benchlm: 0, local: 0 },
+		sources: zeroSources(),
 	};
 	if (feeds.length === 0) return result;
 
@@ -258,22 +307,21 @@ export function applyFeedScores(rawModels: unknown[], feeds: FeedScore[]): FillR
 
 		for (const axis of AXES) {
 			if (score100(aa[`${axis}_index`]) !== null) continue; // published; never overwrite
-			const aaHit = pick(candidates, "artificial_analysis", author);
-			let value = aaHit === null ? undefined : axisValue(aaHit, axis);
-			let source: FeedSource = "artificial_analysis";
-			if (value === undefined) {
-				const blHit = pick(candidates, "benchlm", author);
-				value = blHit === null ? undefined : axisValue(blHit, axis);
-				source = "benchlm";
-			}
-			if (value === undefined) {
-				// Our own calibrated eval is the weakest source: only where neither
-				// published nor third-party feeds have anything.
-				const localHit = pick(candidates, "local", author);
-				value = localHit === null ? undefined : axisValue(localHit, axis);
-				source = "local";
+			// Walk the sources strongest-first and take the first that has this axis:
+			// the measured feeds, then whatever the front door supplied (a neutral
+			// leaderboard ahead of a self-reported vendor number), then our own
+			// calibrated eval, which only ever fills what nothing else does.
+			let value: number | undefined;
+			let source: FeedSource | undefined;
+			for (const candidate of FILL_ORDER) {
+				const hit = pick(candidates, candidate, author);
+				const found = hit === null ? undefined : axisValue(hit, axis);
+				if (found === undefined) continue;
+				value = found;
+				source = candidate;
+				break;
 			}
-			if (value === undefined) continue;
+			if (value === undefined || source === undefined) continue;
 			aa[`${axis}_index`] = value;
 			fillSources[axis] = source;
 			result.axes[axis] += 1;
@@ -317,7 +365,7 @@ export async function refreshFeedScores(cfg: RouterConfig, db: Database, opts: R
 	const bm = cfg.benchmarks;
 
 	const row = db.query("SELECT payload, fetched_at_ms FROM benchmark_cache WHERE id = 1").get() as CacheRow | null;
-	const cached: FeedScore[] | null = row === null ? null : parseFeedScores(row.payload);
+	const cached: FeedScore[] | null = row === null ? null : parseFeedScores(row.payload, FETCHED_SOURCES);
 	// A zero timestamp is `invalidateFeedCache`'s marker, not a real fetch time:
 	// the payload stays readable as a fallback but never counts as fresh again.
 	const fresh = row !== null && row.fetched_at_ms > 0 && now - row.fetched_at_ms < bm.refreshMs;
@@ -365,8 +413,76 @@ export function invalidateFeedCache(db: Database): void {
 	db.query("UPDATE benchmark_cache SET fetched_at_ms = 0 WHERE id = 1").run();
 }
 
-/** Validate a persisted `FeedScore[]` blob, skipping any entry that drifted. */
-function parseFeedScores(payload: string): FeedScore[] | null {
+interface SanitizeOpts {
+	/** Which `source` values are acceptable here. */
+	allow: readonly FeedSource[];
+	/** Re-run `normalizeModelKey`/`normalizeCreator` on the way in. */
+	normalize?: boolean;
+}
+
+interface SanitizeResult {
+	scores: FeedScore[];
+	/** Entries thrown away whole: not an object, no key, or a source not allowed. */
+	dropped: number;
+	/** Axes thrown away from an otherwise usable entry: not a finite number in [0, 100]. */
+	droppedAxes: number;
+}
+
+/**
+ * Turn an arbitrary array into `FeedScore[]`, keeping only what is usable.
+ *
+ * The one invariant everything downstream leans on: an axis this cannot read is
+ * OMITTED, never defaulted. `applyFeedScores` writes only defined values, so a
+ * `"61"` or a `-3` or a `NaN` leaves the axis exactly as unscored as it was —
+ * a bad entry can no more zero a score than it can raise one.
+ */
+function sanitizeFeedScores(value: unknown, opts: SanitizeOpts): SanitizeResult {
+	const result: SanitizeResult = { scores: [], dropped: 0, droppedAxes: 0 };
+	if (!Array.isArray(value)) return result;
+	for (const item of value) {
+		const rec = asRecord(item);
+		if (rec === null || typeof rec.key !== "string") {
+			result.dropped += 1;
+			continue;
+		}
+		const source = rec.source;
+		if (typeof source !== "string" || !opts.allow.includes(source as FeedSource)) {
+			result.dropped += 1;
+			continue;
+		}
+		const rawCreator = typeof rec.creator === "string" ? rec.creator : "";
+		const key = opts.normalize === true ? normalizeModelKey(rec.key) : rec.key;
+		if (key === "") {
+			result.dropped += 1;
+			continue;
+		}
+		const entry: FeedScore = {
+			key,
+			creator: opts.normalize === true ? normalizeCreator(rawCreator) : rawCreator,
+			source: source as FeedSource,
+		};
+		for (const axis of AXES) {
+			const present = rec[axis];
+			if (present === undefined || present === null) continue;
+			const parsed = score100(present);
+			if (parsed === null) {
+				result.droppedAxes += 1;
+				continue;
+			}
+			entry[axis] = parsed;
+		}
+		result.scores.push(entry);
+	}
+	return result;
+}
+
+/**
+ * Validate a persisted `FeedScore[]` blob, skipping any entry that drifted.
+ * `allow` is what WROTE this particular blob, so a row can never reach a rank by
+ * sitting in a table that does not produce that source — the keys are already
+ * normalized here, having been normalized when they were parsed out of a feed.
+ */
+function parseFeedScores(payload: string, allow: readonly FeedSource[]): FeedScore[] | null {
 	let parsed: unknown;
 	try {
 		parsed = JSON.parse(payload);
@@ -374,25 +490,45 @@ function parseFeedScores(payload: string): FeedScore[] | null {
 		return null;
 	}
 	if (!Array.isArray(parsed)) return null;
-	const out: FeedScore[] = [];
-	for (const item of parsed) {
-		const rec = asRecord(item);
-		if (rec === null || typeof rec.key !== "string") continue;
-		if (rec.source !== "artificial_analysis" && rec.source !== "benchlm" && rec.source !== "local") continue;
-		const entry: FeedScore = {
-			key: rec.key,
-			creator: typeof rec.creator === "string" ? rec.creator : "",
-			source: rec.source,
-		};
-		const coding = score100(rec.coding);
-		if (coding !== null) entry.coding = coding;
-		const intelligence = score100(rec.intelligence);
-		if (intelligence !== null) entry.intelligence = intelligence;
-		const agentic = score100(rec.agentic);
-		if (agentic !== null) entry.agentic = agentic;
-		out.push(entry);
+	return sanitizeFeedScores(parsed, { allow }).scores;
+}
+
+/**
+ * `benchmarks.extraScores`, sanitised. The front door curates a table of
+ * vendor-published and neutral-leaderboard numbers for axes the feeds leave
+ * empty — the case this exists for is a model carrying intelligence and nothing
+ * else, which no tier floor above the cheapest can admit.
+ *
+ * Three decisions live here, all about the fact that this arrives over a config
+ * patch from ANOTHER PROCESS rather than out of a file the operator wrote:
+ *
+ *  - A malformed entry is DROPPED, LOUDLY — never a refused patch. The patch
+ *    carries unrelated settings (the Artificial Analysis key rides in the same
+ *    `benchmarks` block), and one bad row out of six hundred must not take an
+ *    operator's key save down with it. Silence was the other option and is worse:
+ *    a table that quietly stopped applying looks exactly like one that worked.
+ *  - Only `neutral` and `vendor` are accepted. Config claiming
+ *    `artificial_analysis` would outrank BenchLM on the strength of a label, and
+ *    config claiming `local` would write into the lane `useLocalScores` gates —
+ *    which is precisely the switch this design refuses to make into a lie.
+ *  - Keys are normalised here, so the supplier may send the OpenRouter slug
+ *    (`deepseek/deepseek-v4.1-flash`) and need not reimplement the match key.
+ */
+export function suppliedScores(cfg: RouterConfig, log?: Logger): FeedScore[] {
+	const supplied = cfg.benchmarks.extraScores;
+	if (supplied === undefined || supplied.length === 0) return [];
+	const capped = supplied.length > MAX_EXTRA_SCORES ? supplied.slice(0, MAX_EXTRA_SCORES) : supplied;
+	const { scores, dropped, droppedAxes } = sanitizeFeedScores(capped, { allow: SUPPLIED_SOURCES, normalize: true });
+	const over = supplied.length - capped.length;
+	if (dropped > 0 || droppedAxes > 0 || over > 0) {
+		(log ?? createLogger(cfg.logLevel)).warn("supplied benchmark scores partly unusable; the rest still apply", {
+			kept: scores.length,
+			droppedEntries: dropped + over,
+			droppedAxes,
+			...(over > 0 ? { overCap: MAX_EXTRA_SCORES } : {}),
+		});
 	}
-	return out;
+	return scores;
 }
 
 /**
@@ -403,7 +539,9 @@ function parseFeedScores(payload: string): FeedScore[] | null {
 export function loadLocalScores(db: Database): FeedScore[] {
 	const row = db.query("SELECT payload FROM local_scores WHERE id = 1").get() as { payload: string } | null;
 	if (row === null) return [];
-	return parseFeedScores(row.payload) ?? [];
+	// Only `local` may come out of the local lane: the table `useLocalScores`
+	// gates must not be a way to claim a rank it does not have.
+	return parseFeedScores(row.payload, ["local"]) ?? [];
 }
 
 /** Persist local eval scores (source `local`) for `doRefresh` to pick up when enabled. */

package/src/catalog/openrouter-catalog.ts

@@ -31,7 +31,7 @@ import type {
  */
 const SHRINK_KEEP_RATIO = 0.5;
 const SHRINK_MIN_PREVIOUS = 20;
-import { applyFeedScores, loadLocalScores, refreshFeedScores } from "./benchmark-feeds.ts";
+import { applyFeedScores, loadLocalScores, refreshFeedScores, suppliedScores } from "./benchmark-feeds.ts";
 
 function asRecord(value: unknown): Record<string, unknown> | null {
 	return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
@@ -394,10 +394,16 @@ export function createCatalog(cfg: RouterConfig, upstream: UpstreamClient, db: D
 		if (cfg.benchmarks.enabled) {
 			try {
 				const feeds = await refreshFeedScores(cfg, db, { log });
+				// Scores the front door supplied for axes the feeds leave empty. Read
+				// from the live config on every refresh, never cached — a set that just
+				// arrived over a patch applies to the very next catalog build.
+				const supplied = suppliedScores(cfg, log);
 				// Local eval scores are last-resort and gated: they change routing, so
 				// they apply only when the operator opts in.
 				const local = cfg.benchmarks.useLocalScores ? loadLocalScores(db) : [];
-				const filled = applyFeedScores(raw, [...feeds, ...local]);
+				// Order within this array is irrelevant: `applyFeedScores` ranks by
+				// `source` through FILL_ORDER, not by position.
+				const filled = applyFeedScores(raw, [...feeds, ...supplied, ...local]);
 				if (filled.modelsFilled > 0) {
 					log.debug("backfilled missing benchmarks from external feeds", {
 						models: filled.modelsFilled,
@@ -406,6 +412,8 @@ export function createCatalog(cfg: RouterConfig, upstream: UpstreamClient, db: D
 						agentic: filled.axes.agentic,
 						aa: filled.sources.artificial_analysis,
 						benchlm: filled.sources.benchlm,
+						neutral: filled.sources.neutral,
+						vendor: filled.sources.vendor,
 						local: filled.sources.local,
 					});
 				}