auto-model-router 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +127 -46
  3. package/bun.lock +606 -0
  4. package/omp-extension/router-embed.ts +14 -6
  5. package/omp-extension/router-toast.ts +6 -1
  6. package/omp-extension/toast-logic.ts +7 -0
  7. package/package.json +2 -1
  8. package/research/analyze-ledger.ts +173 -0
  9. package/research/apply-cost-tuning.ts +73 -0
  10. package/research/cost-analysis.ts +150 -0
  11. package/research/feed-check.ts +64 -0
  12. package/research/model-recommendations.ts +86 -0
  13. package/research/project-yield.ts +96 -0
  14. package/research/run-eval.ts +133 -0
  15. package/research/status.ts +55 -0
  16. package/research/tier-fill.ts +109 -0
  17. package/research/tier-map.ts +123 -0
  18. package/src/catalog/benchmark-feeds.ts +397 -0
  19. package/src/catalog/openrouter-catalog.ts +30 -0
  20. package/src/config/defaults.ts +30 -0
  21. package/src/config/load.ts +2 -0
  22. package/src/config/schema.ts +34 -0
  23. package/src/config/types.ts +106 -0
  24. package/src/cost/ledger.ts +27 -3
  25. package/src/cost/types.ts +30 -0
  26. package/src/eval/calibrate.ts +131 -0
  27. package/src/eval/grade.ts +115 -0
  28. package/src/eval/judge.ts +71 -0
  29. package/src/eval/run.ts +126 -0
  30. package/src/eval/tasks.ts +272 -0
  31. package/src/index.ts +0 -1
  32. package/src/router/candidates.ts +13 -6
  33. package/src/router/explore.ts +59 -0
  34. package/src/router/select.ts +54 -4
  35. package/src/router/tier-plan.ts +57 -1
  36. package/src/router/types.ts +13 -0
  37. package/src/server/turn.ts +10 -2
  38. package/src/util/sqlite.ts +79 -1
  39. package/src/wire/openai/request.ts +5 -0
  40. package/src/wire/types.ts +7 -0
  41. package/test/benchmark-feeds.test.ts +222 -0
  42. package/test/escalate.test.ts +1 -0
  43. package/test/eval.test.ts +184 -0
  44. package/test/exploration.test.ts +251 -0
  45. package/test/failover.test.ts +5 -0
  46. package/test/hold-exploration.test.ts +124 -0
  47. package/test/tier-plan.test.ts +55 -1
  48. package/test/toast-logic.test.ts +32 -0
  49. package/test/tokens.test.ts +8 -0
  50. package/test/trust-attribution.test.ts +110 -2
  51. package/test/turn.test.ts +46 -0
  52. package/test/wire-request.test.ts +11 -0
  53. package/tools/smoke.ts +2 -0
  54. package/tools/sync-marketplace-version.ts +60 -0
@@ -42,10 +42,12 @@ const AXES: readonly QualityAxis[] = ["coding", "agentic", "intelligence"];
42
42
  export type AxisFloors = Record<Tier, number>;
43
43
 
44
44
  export interface TierPlan {
45
- /** Adaptive floor per axis per tier. */
45
+ /** Adaptive quality floor per axis per tier. */
46
46
  floors: Record<QualityAxis, AxisFloors>;
47
47
  /** How many available models carried a score on each axis. */
48
48
  scoredCount: Record<QualityAxis, number>;
49
+ /** Adaptive input-price ceiling ($/Mtok) per tier, from the catalog's price spread. */
50
+ priceCeilings: Record<Tier, number>;
49
51
  }
50
52
 
51
53
  /**
@@ -89,6 +91,32 @@ function bandFloors(ascending: readonly number[]): AxisFloors {
89
91
  return floors as AxisFloors;
90
92
  }
91
93
 
94
+ /**
95
+ * Per-tier input-price ceiling ($/Mtok) from an ascending price list: quantile
96
+ * UPPER bounds, so each higher tier admits a larger, pricier slice. p25/p50/p75
97
+ * for trivial/simple/moderate, and p90 for `hard` — high enough to keep the
98
+ * strong models but dropping the priciest outliers. The catalog decides the
99
+ * dollar values, so this self-tunes to whatever models a key actually admits.
100
+ */
101
+ const CEILING_QUANTILES: readonly number[] = [0.25, 0.5, 0.75, 0.9];
102
+
103
+ function bandCeilings(ascending: readonly number[]): Record<Tier, number> {
104
+ const out: Record<string, number> = {};
105
+ const len = ascending.length;
106
+ for (let k = 0; k < TIER_ORDER.length; k++) {
107
+ const tier = TIER_ORDER[k];
108
+ if (tier === undefined) continue;
109
+ if (len === 0) {
110
+ out[tier] = Number.POSITIVE_INFINITY; // no data ⇒ no cap
111
+ continue;
112
+ }
113
+ const q = CEILING_QUANTILES[k] ?? 1;
114
+ const index = Math.min(len - 1, Math.max(0, Math.ceil(len * q) - 1));
115
+ out[tier] = ascending[index] ?? Number.POSITIVE_INFINITY;
116
+ }
117
+ return out as Record<Tier, number>;
118
+ }
119
+
92
120
  /** Computes the adaptive plan for a set of available models. */
93
121
  export function computeTierPlan(models: readonly CatalogModel[], cfg: RouterConfig): TierPlan {
94
122
  const floors: Record<string, AxisFloors> = {};
@@ -107,10 +135,18 @@ export function computeTierPlan(models: readonly CatalogModel[], cfg: RouterConf
107
135
  floors[axis] = bandFloors(scores);
108
136
  scoredCount[axis] = scores.length;
109
137
  }
138
+ // Input prices ($/Mtok) of the rankable models, ascending, for the ceilings.
139
+ const prices: number[] = [];
140
+ for (const model of models) {
141
+ if (!isRankable(model, includeFree)) continue;
142
+ prices.push(model.price.prompt * 1e6);
143
+ }
144
+ prices.sort((a, b) => a - b);
110
145
 
111
146
  return {
112
147
  floors: floors as Record<QualityAxis, AxisFloors>,
113
148
  scoredCount: scoredCount as Record<QualityAxis, number>,
149
+ priceCeilings: bandCeilings(prices),
114
150
  };
115
151
  }
116
152
 
@@ -149,3 +185,23 @@ export function effectiveQualityFloor(
149
185
  const adaptive = plan.floors[axis][tier];
150
186
  return Math.min(configured, adaptive);
151
187
  }
188
+
189
+ /**
190
+ * The input-price ceiling ($/Mtok) to actually enforce for a tier. When adaptive
191
+ * ceilings are off, this is just the configured value (possibly unset). When on,
192
+ * the catalog-derived band applies, and an explicit config can only tighten it
193
+ * further — never loosen it. Returns undefined for "no ceiling".
194
+ */
195
+ export function effectivePriceCeiling(
196
+ configured: number | undefined,
197
+ tier: Tier,
198
+ plan: TierPlan,
199
+ enabled: boolean,
200
+ ): number | undefined {
201
+ if (!enabled) return configured;
202
+ const adaptive = plan.priceCeilings[tier];
203
+ const band = Number.isFinite(adaptive) ? adaptive : undefined;
204
+ if (configured === undefined) return band;
205
+ if (band === undefined) return configured;
206
+ return Math.min(configured, band);
207
+ }
@@ -168,6 +168,17 @@ export interface ProbePlan {
168
168
  }
169
169
 
170
170
  /** The routing decision for one turn. */
171
+ /**
172
+ * A turn that was deliberately routed below its classified tier, so the
173
+ * ledger witnesses whether the cheaper model would have sufficed.
174
+ */
175
+ export interface Exploration {
176
+ /** Tier the classifier actually chose. */
177
+ from: Tier;
178
+ /** Tier routed instead, exactly one step cheaper. */
179
+ to: Tier;
180
+ }
181
+
171
182
  export interface Decision {
172
183
  slug: string;
173
184
  /** Same-tier fallbacks for OpenRouter's `models[]` array; transient-error only. */
@@ -191,6 +202,8 @@ export interface Decision {
191
202
  /** Filtered-out models with cause. Retained for `explain`. */
192
203
  rejected: Rejection[];
193
204
  reasons: string[];
205
+ /** Set when epsilon-greedy exploration deliberately routed below the classified tier. */
206
+ explored: Exploration | null;
194
207
  /** Budget guard forced a cheaper tier than the classifier asked for. */
195
208
  budgetDowngraded: boolean;
196
209
  }
@@ -12,6 +12,7 @@ import type { CatalogSource } from "../catalog/types.ts";
12
12
  import type { RouterConfig } from "../config/types.ts";
13
13
  import { EMPTY_USAGE, type Ledger, type UsageCounts } from "../cost/types.ts";
14
14
  import { createProbe, type Probe } from "../router/escalate.ts";
15
+ import { resolveHoldTurns } from "../router/explore.ts";
15
16
  import {
16
17
  TIER_ORDER,
17
18
  type ConversationStore,
@@ -141,11 +142,19 @@ export async function runTurn(
141
142
  turn: turnNumber,
142
143
  requestedModel: req.requestedModel,
143
144
  harnessId: req.harnessId,
145
+ ompSessionId: req.ompSessionId,
144
146
  slug: decision.slug,
145
147
  servedSlug,
146
148
  tier: decision.tier,
147
149
  classificationSource: decision.classification.source,
148
150
  reasons: decision.reasons,
151
+ features: decision.features,
152
+ score: decision.classification.score,
153
+ confidence: decision.classification.confidence,
154
+ task: decision.classification.task,
155
+ classifierReasons: decision.classification.reasons,
156
+ exploredFrom: decision.explored?.from ?? null,
157
+ holdArm: resolveHoldTurns(config, req.conversationKey, escalations > 0).arm,
149
158
  predictedUsd: decision.forecast.expectedUsd,
150
159
  reportedUsd,
151
160
  usage,
@@ -357,8 +366,7 @@ export async function runTurn(
357
366
  // actually easy.
358
367
  const tierChanged = prevTier !== decision.tier;
359
368
  if (tierChanged || escalations > 0) {
360
- state.stickyUntilTurn =
361
- turnNumber + (escalations > 0 ? config.hysteresis.holdTurnsAfterEscalation : config.hysteresis.holdTurns);
369
+ state.stickyUntilTurn = turnNumber + resolveHoldTurns(config, req.conversationKey, escalations > 0).turns;
362
370
  }
363
371
  // Reported cost is authoritative; fall back to the forecast so the
364
372
  // budget guard still works when the provider omits cost.
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
18
18
  import { dirname } from "node:path";
19
19
 
20
20
  /** Bump when a migration is added; guarded below so reopening never regresses it. */
21
- const USER_VERSION = 4;
21
+ const USER_VERSION = 10;
22
22
 
23
23
  const MIGRATIONS = `
24
24
  CREATE TABLE IF NOT EXISTS catalog_cache (
@@ -29,6 +29,18 @@ CREATE TABLE IF NOT EXISTS catalog_cache (
29
29
  key_scoped INTEGER NOT NULL DEFAULT 0
30
30
  );
31
31
 
32
+ CREATE TABLE IF NOT EXISTS benchmark_cache (
33
+ id INTEGER PRIMARY KEY CHECK (id = 1),
34
+ payload TEXT NOT NULL,
35
+ fetched_at_ms INTEGER NOT NULL
36
+ );
37
+
38
+ CREATE TABLE IF NOT EXISTS local_scores (
39
+ id INTEGER PRIMARY KEY CHECK (id = 1),
40
+ payload TEXT NOT NULL,
41
+ fetched_at_ms INTEGER NOT NULL
42
+ );
43
+
32
44
  CREATE TABLE IF NOT EXISTS ledger (
33
45
  id TEXT PRIMARY KEY,
34
46
  created_at_ms INTEGER NOT NULL,
@@ -37,6 +49,7 @@ CREATE TABLE IF NOT EXISTS ledger (
37
49
  turn INTEGER NOT NULL,
38
50
  requested_model TEXT NOT NULL,
39
51
  harness_id TEXT NOT NULL DEFAULT '',
52
+ omp_session_id TEXT NOT NULL DEFAULT '',
40
53
  slug TEXT NOT NULL,
41
54
  served_slug TEXT,
42
55
  tier TEXT NOT NULL,
@@ -118,6 +131,67 @@ END
118
131
  WHERE error IS NOT NULL;
119
132
  `;
120
133
 
134
+ // v5: ledger gains omp_session_id, so the toast extension can scope decisions to
135
+ // its own omp session. Before this, the only scoping was per-harness, so two
136
+ // interactive omp sessions of the same harness (the default: empty) each
137
+ // surfaced the other's routing toasts from the shared ledger. Existing rows
138
+ // backfill to '' (unknown session), matching the no-header default.
139
+ const MIGRATE_V5 = `
140
+ ALTER TABLE ledger ADD COLUMN omp_session_id TEXT NOT NULL DEFAULT '';
141
+ `;
142
+
143
+ // v6: classifier INPUTS. Before this the ledger recorded only what routing
144
+ // decided (tier, slug, cost, escalation) and never what it decided FROM, so a
145
+ // turn could not be replayed, and no weight could be fit offline. `features`
146
+ // is the verbatim feature vector as JSON; `score` and `confidence` are the
147
+ // classifier's own outputs, kept alongside because they are cheap and make
148
+ // weight drift detectable when the scorer changes under a fixed feature set.
149
+ // `classifier_reasons` is the per-feature breakdown, which `select.ts` drops
150
+ // from the decision trail on every path except a hysteresis hold.
151
+ //
152
+ // All nullable with no backfill: pre-v6 rows genuinely lack these inputs and
153
+ // NULL says so honestly. A DEFAULT would invent data that was never observed.
154
+ const MIGRATE_V6 = `
155
+ ALTER TABLE ledger ADD COLUMN features TEXT;
156
+ ALTER TABLE ledger ADD COLUMN score REAL;
157
+ ALTER TABLE ledger ADD COLUMN confidence REAL;
158
+ ALTER TABLE ledger ADD COLUMN task TEXT;
159
+ ALTER TABLE ledger ADD COLUMN classifier_reasons TEXT;
160
+ `;
161
+
162
+ // v7: records epsilon-greedy exploration. `explored_from` is the tier the
163
+ // classifier actually chose on a turn we deliberately routed one step
164
+ // cheaper; NULL means the turn was routed normally.
165
+ //
166
+ // This is the counterfactual the ledger could never observe before. Natural
167
+ // traffic only reveals UNDER-routing, because a tier that was too low
168
+ // escalates and leaves a trace, while a tier that was too high looks
169
+ // indistinguishable from a tier that was exactly right.
170
+ const MIGRATE_V7 = `
171
+ ALTER TABLE ledger ADD COLUMN explored_from TEXT;
172
+ `;
173
+
174
+ // v8: records the hold-length arm a conversation was assigned by hold
175
+ // exploration. NULL means the conversation was not part of the experiment.
176
+ //
177
+ // Recorded on EVERY turn of the conversation, not only the turns a hold
178
+ // actually affects, so arms can be compared on total conversation cost
179
+ // rather than on the subset the treatment happened to touch.
180
+ const MIGRATE_V8 = `
181
+ ALTER TABLE ledger ADD COLUMN hold_arm INTEGER;
182
+ `;
183
+
184
+ // v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
185
+ // BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
186
+ // whole new table, created idempotently by the MIGRATIONS block above, so there
187
+ // is no ALTER guard here — the version bump alone records that the schema now
188
+ // includes it.
189
+
190
+ // v10: local_scores holds calibrated scores from our OWN eval harness
191
+ // (src/eval), a `local`-source feed applied only when benchmarks.useLocalScores
192
+ // is on. Another new table via the idempotent MIGRATIONS block; version bump
193
+ // only, no ALTER guard.
194
+
121
195
  export function openDb(path: string): Database {
122
196
  // ":memory:" has no parent directory to create.
123
197
  if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
@@ -134,6 +208,10 @@ export function openDb(path: string): Database {
134
208
  const ledgerCols = db.query("PRAGMA table_info(ledger)").all() as { name: string }[];
135
209
  if (!ledgerCols.some((c) => c.name === "harness_id")) db.exec(MIGRATE_V3);
136
210
  if (!ledgerCols.some((c) => c.name === "error_kind")) db.exec(MIGRATE_V4);
211
+ if (!ledgerCols.some((c) => c.name === "omp_session_id")) db.exec(MIGRATE_V5);
212
+ if (!ledgerCols.some((c) => c.name === "features")) db.exec(MIGRATE_V6);
213
+ if (!ledgerCols.some((c) => c.name === "explored_from")) db.exec(MIGRATE_V7);
214
+ if (!ledgerCols.some((c) => c.name === "hold_arm")) db.exec(MIGRATE_V8);
137
215
  db.exec(`PRAGMA user_version = ${USER_VERSION}`);
138
216
  }
139
217
  return db;
@@ -204,6 +204,10 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
204
204
  // this via the provider block's `headers:` override; absent ⇒ single harness.
205
205
  const harnessId = (headers.get("x-omp-harness") ?? "").trim();
206
206
 
207
+ // omp UI session id for per-session toast scoping. The embed extension sets
208
+ // this header to ctx.sessionManager.getSessionId(); absent ⇒ unknown session.
209
+ const ompSessionId = (headers.get("x-omp-session") ?? "").trim();
210
+
207
211
  if (typeof b.model !== "string" || b.model.length === 0) {
208
212
  throw invalidRequest("model must be a non-empty string");
209
213
  }
@@ -262,6 +266,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
262
266
  protocol: "openai-chat",
263
267
  conversationKey,
264
268
  harnessId,
269
+ ompSessionId,
265
270
  requestedModel,
266
271
  messages,
267
272
  tools,
package/src/wire/types.ts CHANGED
@@ -68,6 +68,13 @@ export interface NormRequest {
68
68
  * client sends no header (single-harness default).
69
69
  */
70
70
  harnessId: string;
71
+ /**
72
+ * omp UI session id from the `X-Omp-Session` request header, when the client
73
+ * sends one. Scopes toasts to a single interactive session so concurrent
74
+ * sessions sharing one router don't surface each other's choices. Empty when
75
+ * the client sends no header.
76
+ */
77
+ ompSessionId: string;
71
78
  /** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
72
79
  requestedModel: string;
73
80
  messages: NormMessage[];
@@ -0,0 +1,222 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
4
+ import {
5
+ applyFeedScores,
6
+ fetchBenchlmScores,
7
+ normalizeModelKey,
8
+ parseAaModels,
9
+ parseBenchlmModels,
10
+ refreshFeedScores,
11
+ type FeedScore,
12
+ type FetchLike,
13
+ } from "../src/catalog/benchmark-feeds.ts";
14
+ import { loadConfig } from "../src/config/load.ts";
15
+ import type { RouterConfig } from "../src/config/types.ts";
16
+ import { openDb } from "../src/util/sqlite.ts";
17
+
18
+ /** A bare OpenRouter `/models` record, optionally pre-scored. */
19
+ function raw(id: string, benchmarks?: Record<string, unknown>): Record<string, unknown> {
20
+ const record: Record<string, unknown> = {
21
+ id,
22
+ canonical_slug: id,
23
+ name: id,
24
+ context_length: 131_072,
25
+ pricing: { prompt: "0.0000003", completion: "0.0000011" },
26
+ supported_parameters: ["tools"],
27
+ architecture: { input_modalities: ["text"], tokenizer: "Other" },
28
+ created: 1_700_000_000,
29
+ };
30
+ if (benchmarks !== undefined) record.benchmarks = benchmarks;
31
+ return record;
32
+ }
33
+
34
+ function aaScore(over: Partial<FeedScore> & { key: string }): FeedScore {
35
+ return { creator: "", source: "artificial_analysis", ...over };
36
+ }
37
+ function blScore(over: Partial<FeedScore> & { key: string }): FeedScore {
38
+ return { creator: "", source: "benchlm", ...over };
39
+ }
40
+
41
+ describe("normalizeModelKey", () => {
42
+ test("strips provider, tilde, and release words but keeps the parameter size", () => {
43
+ expect(normalizeModelKey("z-ai/glm-5.3-flash")).toBe("glm-5-3-flash");
44
+ expect(normalizeModelKey("~deepseek/deepseek-v4-flash-latest")).toBe("deepseek-v4-flash");
45
+ expect(normalizeModelKey("meta/muse-glimmer-30b")).toBe("muse-glimmer-30b");
46
+ // The feed's own display spelling collapses onto the same key.
47
+ expect(normalizeModelKey("Muse Glimmer 30B")).toBe("muse-glimmer-30b");
48
+ expect(normalizeModelKey("MiniMax M3")).toBe("minimax-m3");
49
+ });
50
+ });
51
+
52
+ describe("parseAaModels", () => {
53
+ test("reads the three indices, keeps in-range values, and skips empty rows", () => {
54
+ const body = {
55
+ data: [
56
+ {
57
+ slug: "glm-5.3-flash",
58
+ model_creator: { slug: "z-ai" },
59
+ evaluations: {
60
+ artificial_analysis_coding_index: 61.2,
61
+ artificial_analysis_intelligence_index: 58.4,
62
+ artificial_analysis_agentic_index: 150, // out of range → dropped
63
+ },
64
+ },
65
+ { slug: "no-evals", model_creator: { slug: "x" }, evaluations: {} },
66
+ ],
67
+ };
68
+ const parsed = parseAaModels(body);
69
+ expect(parsed).toHaveLength(1);
70
+ expect(parsed[0]).toMatchObject({ key: "glm-5-3-flash", creator: "z-ai", coding: 61.2, intelligence: 58.4 });
71
+ expect(parsed[0]?.agentic).toBeUndefined();
72
+ });
73
+ });
74
+
75
+ describe("parseBenchlmModels", () => {
76
+ test("maps categories to axes, drops estimated rows, and ignores out-of-range", () => {
77
+ const body = {
78
+ models: [
79
+ {
80
+ model: "Muse Glimmer 30B",
81
+ creator: "Meta",
82
+ evidenceStatus: "supported",
83
+ categoryScores: { coding: 55, reasoning: 52, agentic: 48 },
84
+ },
85
+ {
86
+ model: "Guessed Model",
87
+ creator: "x",
88
+ evidenceStatus: "estimated",
89
+ categoryScores: { coding: 90 },
90
+ },
91
+ ],
92
+ };
93
+ const parsed = parseBenchlmModels(body);
94
+ expect(parsed).toHaveLength(1);
95
+ expect(parsed[0]).toMatchObject({ key: "muse-glimmer-30b", coding: 55, intelligence: 52, agentic: 48 });
96
+ });
97
+ });
98
+
99
+ describe("applyFeedScores", () => {
100
+ test("fills the real gap models and reaches normalizeCatalogModel", () => {
101
+ const catalog = [
102
+ raw("meta/muse-glimmer-30b"),
103
+ raw("z-ai/glm-5.3-flash"),
104
+ // Already scored by OpenRouter on coding; a feed must not overwrite it.
105
+ raw("google/gemini-3.7-flash", { artificial_analysis: { coding_index: 76.1 } }),
106
+ ];
107
+ const feeds: FeedScore[] = [
108
+ aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58 }),
109
+ aaScore({ key: "gemini-3-7-flash", creator: "google", coding: 40, intelligence: 63 }),
110
+ blScore({ key: "muse-glimmer-30b", creator: "meta", coding: 55, agentic: 48 }),
111
+ blScore({ key: "glm-5-3-flash", creator: "z-ai", agentic: 44 }),
112
+ ];
113
+
114
+ const result = applyFeedScores(catalog, feeds);
115
+
116
+ // muse-glimmer: was empty, gains coding + agentic from BenchLM.
117
+ const muse = normalizeCatalogModel(catalog[0]);
118
+ expect(muse?.quality).toEqual({ coding: 55, agentic: 48 });
119
+
120
+ // glm: coding + intelligence from AA (stronger), agentic from BenchLM.
121
+ const glm = normalizeCatalogModel(catalog[1]);
122
+ expect(glm?.quality).toEqual({ coding: 61, intelligence: 58, agentic: 44 });
123
+
124
+ // gemini: published coding survives untouched; intelligence filled from AA.
125
+ const gemini = normalizeCatalogModel(catalog[2]);
126
+ expect(gemini?.quality.coding).toBe(76.1);
127
+ expect(gemini?.quality.intelligence).toBe(63);
128
+
129
+ // Provenance recorded, counts add up.
130
+ const museBench = catalog[0]?.benchmarks;
131
+ expect(museBench).toMatchObject({ fill_sources: { coding: "benchlm", agentic: "benchlm" } });
132
+ expect(result.modelsFilled).toBe(3);
133
+ expect(result.sources.artificial_analysis).toBe(3); // glm coding+intel, gemini intel
134
+ expect(result.sources.benchlm).toBe(3); // muse coding+agentic, glm agentic
135
+ });
136
+
137
+ test("AA wins over BenchLM for the same axis", () => {
138
+ const catalog = [raw("z-ai/glm-5.3-flash")];
139
+ const feeds: FeedScore[] = [
140
+ blScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 10 }),
141
+ aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
142
+ ];
143
+ applyFeedScores(catalog, feeds);
144
+ expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
145
+ });
146
+
147
+ test("never fuzzy-matches a different model", () => {
148
+ const catalog = [raw("meta/muse-glimmer-30b")];
149
+ // Same family, different model — must not lend its score.
150
+ const feeds: FeedScore[] = [aaScore({ key: "muse-spark-1-2", creator: "meta", coding: 72 })];
151
+ const result = applyFeedScores(catalog, feeds);
152
+ expect(result.modelsFilled).toBe(0);
153
+ expect(normalizeCatalogModel(catalog[0])?.quality).toEqual({});
154
+ });
155
+
156
+ test("a shared key with conflicting creators fills only the creator that matches", () => {
157
+ const catalog = [raw("z-ai/glm-5.3-flash")];
158
+ const feeds: FeedScore[] = [
159
+ aaScore({ key: "glm-5-3-flash", creator: "someone-else", coding: 5 }),
160
+ aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
161
+ ];
162
+ applyFeedScores(catalog, feeds);
163
+ expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
164
+ });
165
+ });
166
+
167
+ describe("refreshFeedScores", () => {
168
+ function cfgWith(over: Partial<RouterConfig["benchmarks"]>): RouterConfig {
169
+ const base = loadConfig({});
170
+ return { ...base, benchmarks: { ...base.benchmarks, ...over } };
171
+ }
172
+
173
+ test("fetches once, then serves the cache within the TTL", async () => {
174
+ const db = openDb(":memory:");
175
+ let calls = 0;
176
+ const fakeFetch: FetchLike = async (url) => {
177
+ calls += 1;
178
+ const u = String(url);
179
+ if (u.includes("benchlm")) {
180
+ return Response.json({
181
+ models: [
182
+ { model: "MiniMax M3", creator: "MiniMax", evidenceStatus: "supported", categoryScores: { coding: 58 } },
183
+ ],
184
+ });
185
+ }
186
+ return Response.json({ data: [] });
187
+ };
188
+
189
+ const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 1_000_000 });
190
+ const first = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 1000 });
191
+ expect(first).toHaveLength(1);
192
+ expect(first[0]).toMatchObject({ key: "minimax-m3", coding: 58, source: "benchlm" });
193
+ expect(calls).toBe(1); // AA skipped (no key), BenchLM fetched once
194
+
195
+ const second = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 2000 });
196
+ expect(second).toHaveLength(1);
197
+ expect(calls).toBe(1); // within TTL → no new fetch
198
+ db.close();
199
+ });
200
+
201
+ test("falls back to the stale cache when a refresh returns nothing", async () => {
202
+ const db = openDb(":memory:");
203
+ const seed = [{ key: "minimax-m3", creator: "minimax", coding: 58, source: "benchlm" }];
204
+ db.query("INSERT INTO benchmark_cache (id, payload, fetched_at_ms) VALUES (1, ?, ?)").run(JSON.stringify(seed), 0);
205
+ const emptyFetch: FetchLike = async () => Response.json({ models: [] });
206
+ const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 10 });
207
+ const got = await refreshFeedScores(cfg, db, { fetchImpl: emptyFetch, now: 1_000_000 });
208
+ expect(got).toHaveLength(1);
209
+ expect(got[0]).toMatchObject({ key: "minimax-m3", coding: 58 });
210
+ db.close();
211
+ });
212
+ });
213
+
214
+ // fetchBenchlmScores over a fake fetch: the keyless path parses end to end.
215
+ test("fetchBenchlmScores parses a keyless leaderboard response", async () => {
216
+ const fake: FetchLike = async () =>
217
+ Response.json({
218
+ models: [{ model: "GLM 5.3 Flash", creator: "Z-AI", evidenceStatus: "supported", categoryScores: { coding: 61, reasoning: 58 } }],
219
+ });
220
+ const scores = await fetchBenchlmScores({ fetchImpl: fake });
221
+ expect(scores).toEqual([{ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58, source: "benchlm" }]);
222
+ });
@@ -22,6 +22,7 @@ function req(messages: NormMessage[] = [], over: Partial<NormRequest> = {}): Nor
22
22
  protocol: "openai-chat",
23
23
  conversationKey: "k",
24
24
  harnessId: "",
25
+ ompSessionId: "",
25
26
  requestedModel: "auto",
26
27
  messages,
27
28
  tools: [],