auto-model-router 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +127 -46
- package/bun.lock +606 -0
- package/omp-extension/router-embed.ts +14 -6
- package/omp-extension/router-toast.ts +6 -1
- package/omp-extension/toast-logic.ts +7 -0
- package/package.json +2 -1
- package/research/analyze-ledger.ts +173 -0
- package/research/apply-cost-tuning.ts +73 -0
- package/research/cost-analysis.ts +150 -0
- package/research/feed-check.ts +64 -0
- package/research/model-recommendations.ts +86 -0
- package/research/project-yield.ts +96 -0
- package/research/run-eval.ts +133 -0
- package/research/status.ts +55 -0
- package/research/tier-fill.ts +109 -0
- package/research/tier-map.ts +123 -0
- package/src/catalog/benchmark-feeds.ts +397 -0
- package/src/catalog/openrouter-catalog.ts +30 -0
- package/src/config/defaults.ts +30 -0
- package/src/config/load.ts +2 -0
- package/src/config/schema.ts +34 -0
- package/src/config/types.ts +106 -0
- package/src/cost/ledger.ts +27 -3
- package/src/cost/types.ts +30 -0
- package/src/eval/calibrate.ts +131 -0
- package/src/eval/grade.ts +115 -0
- package/src/eval/judge.ts +71 -0
- package/src/eval/run.ts +126 -0
- package/src/eval/tasks.ts +272 -0
- package/src/index.ts +0 -1
- package/src/router/candidates.ts +13 -6
- package/src/router/explore.ts +59 -0
- package/src/router/select.ts +54 -4
- package/src/router/tier-plan.ts +57 -1
- package/src/router/types.ts +13 -0
- package/src/server/turn.ts +10 -2
- package/src/util/sqlite.ts +79 -1
- package/src/wire/openai/request.ts +5 -0
- package/src/wire/types.ts +7 -0
- package/test/benchmark-feeds.test.ts +222 -0
- package/test/escalate.test.ts +1 -0
- package/test/eval.test.ts +184 -0
- package/test/exploration.test.ts +251 -0
- package/test/failover.test.ts +5 -0
- package/test/hold-exploration.test.ts +124 -0
- package/test/tier-plan.test.ts +55 -1
- package/test/toast-logic.test.ts +32 -0
- package/test/tokens.test.ts +8 -0
- package/test/trust-attribution.test.ts +110 -2
- package/test/turn.test.ts +46 -0
- package/test/wire-request.test.ts +11 -0
- package/tools/smoke.ts +2 -0
- package/tools/sync-marketplace-version.ts +60 -0
package/src/router/tier-plan.ts
CHANGED
|
@@ -42,10 +42,12 @@ const AXES: readonly QualityAxis[] = ["coding", "agentic", "intelligence"];
|
|
|
42
42
|
export type AxisFloors = Record<Tier, number>;
|
|
43
43
|
|
|
44
44
|
export interface TierPlan {
|
|
45
|
-
/** Adaptive floor per axis per tier. */
|
|
45
|
+
/** Adaptive quality floor per axis per tier. */
|
|
46
46
|
floors: Record<QualityAxis, AxisFloors>;
|
|
47
47
|
/** How many available models carried a score on each axis. */
|
|
48
48
|
scoredCount: Record<QualityAxis, number>;
|
|
49
|
+
/** Adaptive input-price ceiling ($/Mtok) per tier, from the catalog's price spread. */
|
|
50
|
+
priceCeilings: Record<Tier, number>;
|
|
49
51
|
}
|
|
50
52
|
|
|
51
53
|
/**
|
|
@@ -89,6 +91,32 @@ function bandFloors(ascending: readonly number[]): AxisFloors {
|
|
|
89
91
|
return floors as AxisFloors;
|
|
90
92
|
}
|
|
91
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Per-tier input-price ceiling ($/Mtok) from an ascending price list: quantile
|
|
96
|
+
* UPPER bounds, so each higher tier admits a larger, pricier slice. p25/p50/p75
|
|
97
|
+
* for trivial/simple/moderate, and p90 for `hard` — high enough to keep the
|
|
98
|
+
* strong models but dropping the priciest outliers. The catalog decides the
|
|
99
|
+
* dollar values, so this self-tunes to whatever models a key actually admits.
|
|
100
|
+
*/
|
|
101
|
+
const CEILING_QUANTILES: readonly number[] = [0.25, 0.5, 0.75, 0.9];
|
|
102
|
+
|
|
103
|
+
function bandCeilings(ascending: readonly number[]): Record<Tier, number> {
|
|
104
|
+
const out: Record<string, number> = {};
|
|
105
|
+
const len = ascending.length;
|
|
106
|
+
for (let k = 0; k < TIER_ORDER.length; k++) {
|
|
107
|
+
const tier = TIER_ORDER[k];
|
|
108
|
+
if (tier === undefined) continue;
|
|
109
|
+
if (len === 0) {
|
|
110
|
+
out[tier] = Number.POSITIVE_INFINITY; // no data ⇒ no cap
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
const q = CEILING_QUANTILES[k] ?? 1;
|
|
114
|
+
const index = Math.min(len - 1, Math.max(0, Math.ceil(len * q) - 1));
|
|
115
|
+
out[tier] = ascending[index] ?? Number.POSITIVE_INFINITY;
|
|
116
|
+
}
|
|
117
|
+
return out as Record<Tier, number>;
|
|
118
|
+
}
|
|
119
|
+
|
|
92
120
|
/** Computes the adaptive plan for a set of available models. */
|
|
93
121
|
export function computeTierPlan(models: readonly CatalogModel[], cfg: RouterConfig): TierPlan {
|
|
94
122
|
const floors: Record<string, AxisFloors> = {};
|
|
@@ -107,10 +135,18 @@ export function computeTierPlan(models: readonly CatalogModel[], cfg: RouterConf
|
|
|
107
135
|
floors[axis] = bandFloors(scores);
|
|
108
136
|
scoredCount[axis] = scores.length;
|
|
109
137
|
}
|
|
138
|
+
// Input prices ($/Mtok) of the rankable models, ascending, for the ceilings.
|
|
139
|
+
const prices: number[] = [];
|
|
140
|
+
for (const model of models) {
|
|
141
|
+
if (!isRankable(model, includeFree)) continue;
|
|
142
|
+
prices.push(model.price.prompt * 1e6);
|
|
143
|
+
}
|
|
144
|
+
prices.sort((a, b) => a - b);
|
|
110
145
|
|
|
111
146
|
return {
|
|
112
147
|
floors: floors as Record<QualityAxis, AxisFloors>,
|
|
113
148
|
scoredCount: scoredCount as Record<QualityAxis, number>,
|
|
149
|
+
priceCeilings: bandCeilings(prices),
|
|
114
150
|
};
|
|
115
151
|
}
|
|
116
152
|
|
|
@@ -149,3 +185,23 @@ export function effectiveQualityFloor(
|
|
|
149
185
|
const adaptive = plan.floors[axis][tier];
|
|
150
186
|
return Math.min(configured, adaptive);
|
|
151
187
|
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* The input-price ceiling ($/Mtok) to actually enforce for a tier. When adaptive
|
|
191
|
+
* ceilings are off, this is just the configured value (possibly unset). When on,
|
|
192
|
+
* the catalog-derived band applies, and an explicit config can only tighten it
|
|
193
|
+
* further — never loosen it. Returns undefined for "no ceiling".
|
|
194
|
+
*/
|
|
195
|
+
export function effectivePriceCeiling(
|
|
196
|
+
configured: number | undefined,
|
|
197
|
+
tier: Tier,
|
|
198
|
+
plan: TierPlan,
|
|
199
|
+
enabled: boolean,
|
|
200
|
+
): number | undefined {
|
|
201
|
+
if (!enabled) return configured;
|
|
202
|
+
const adaptive = plan.priceCeilings[tier];
|
|
203
|
+
const band = Number.isFinite(adaptive) ? adaptive : undefined;
|
|
204
|
+
if (configured === undefined) return band;
|
|
205
|
+
if (band === undefined) return configured;
|
|
206
|
+
return Math.min(configured, band);
|
|
207
|
+
}
|
package/src/router/types.ts
CHANGED
|
@@ -168,6 +168,17 @@ export interface ProbePlan {
|
|
|
168
168
|
}
|
|
169
169
|
|
|
170
170
|
/** The routing decision for one turn. */
|
|
171
|
+
/**
|
|
172
|
+
* A turn that was deliberately routed below its classified tier, so the
|
|
173
|
+
* ledger witnesses whether the cheaper model would have sufficed.
|
|
174
|
+
*/
|
|
175
|
+
export interface Exploration {
|
|
176
|
+
/** Tier the classifier actually chose. */
|
|
177
|
+
from: Tier;
|
|
178
|
+
/** Tier routed instead, exactly one step cheaper. */
|
|
179
|
+
to: Tier;
|
|
180
|
+
}
|
|
181
|
+
|
|
171
182
|
export interface Decision {
|
|
172
183
|
slug: string;
|
|
173
184
|
/** Same-tier fallbacks for OpenRouter's `models[]` array; transient-error only. */
|
|
@@ -191,6 +202,8 @@ export interface Decision {
|
|
|
191
202
|
/** Filtered-out models with cause. Retained for `explain`. */
|
|
192
203
|
rejected: Rejection[];
|
|
193
204
|
reasons: string[];
|
|
205
|
+
/** Set when epsilon-greedy exploration deliberately routed below the classified tier. */
|
|
206
|
+
explored: Exploration | null;
|
|
194
207
|
/** Budget guard forced a cheaper tier than the classifier asked for. */
|
|
195
208
|
budgetDowngraded: boolean;
|
|
196
209
|
}
|
package/src/server/turn.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type { CatalogSource } from "../catalog/types.ts";
|
|
|
12
12
|
import type { RouterConfig } from "../config/types.ts";
|
|
13
13
|
import { EMPTY_USAGE, type Ledger, type UsageCounts } from "../cost/types.ts";
|
|
14
14
|
import { createProbe, type Probe } from "../router/escalate.ts";
|
|
15
|
+
import { resolveHoldTurns } from "../router/explore.ts";
|
|
15
16
|
import {
|
|
16
17
|
TIER_ORDER,
|
|
17
18
|
type ConversationStore,
|
|
@@ -141,11 +142,19 @@ export async function runTurn(
|
|
|
141
142
|
turn: turnNumber,
|
|
142
143
|
requestedModel: req.requestedModel,
|
|
143
144
|
harnessId: req.harnessId,
|
|
145
|
+
ompSessionId: req.ompSessionId,
|
|
144
146
|
slug: decision.slug,
|
|
145
147
|
servedSlug,
|
|
146
148
|
tier: decision.tier,
|
|
147
149
|
classificationSource: decision.classification.source,
|
|
148
150
|
reasons: decision.reasons,
|
|
151
|
+
features: decision.features,
|
|
152
|
+
score: decision.classification.score,
|
|
153
|
+
confidence: decision.classification.confidence,
|
|
154
|
+
task: decision.classification.task,
|
|
155
|
+
classifierReasons: decision.classification.reasons,
|
|
156
|
+
exploredFrom: decision.explored?.from ?? null,
|
|
157
|
+
holdArm: resolveHoldTurns(config, req.conversationKey, escalations > 0).arm,
|
|
149
158
|
predictedUsd: decision.forecast.expectedUsd,
|
|
150
159
|
reportedUsd,
|
|
151
160
|
usage,
|
|
@@ -357,8 +366,7 @@ export async function runTurn(
|
|
|
357
366
|
// actually easy.
|
|
358
367
|
const tierChanged = prevTier !== decision.tier;
|
|
359
368
|
if (tierChanged || escalations > 0) {
|
|
360
|
-
state.stickyUntilTurn =
|
|
361
|
-
turnNumber + (escalations > 0 ? config.hysteresis.holdTurnsAfterEscalation : config.hysteresis.holdTurns);
|
|
369
|
+
state.stickyUntilTurn = turnNumber + resolveHoldTurns(config, req.conversationKey, escalations > 0).turns;
|
|
362
370
|
}
|
|
363
371
|
// Reported cost is authoritative; fall back to the forecast so the
|
|
364
372
|
// budget guard still works when the provider omits cost.
|
package/src/util/sqlite.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
|
|
|
18
18
|
import { dirname } from "node:path";
|
|
19
19
|
|
|
20
20
|
/** Bump when a migration is added; guarded below so reopening never regresses it. */
|
|
21
|
-
const USER_VERSION =
|
|
21
|
+
const USER_VERSION = 10;
|
|
22
22
|
|
|
23
23
|
const MIGRATIONS = `
|
|
24
24
|
CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
@@ -29,6 +29,18 @@ CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
|
29
29
|
key_scoped INTEGER NOT NULL DEFAULT 0
|
|
30
30
|
);
|
|
31
31
|
|
|
32
|
+
CREATE TABLE IF NOT EXISTS benchmark_cache (
|
|
33
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
34
|
+
payload TEXT NOT NULL,
|
|
35
|
+
fetched_at_ms INTEGER NOT NULL
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
CREATE TABLE IF NOT EXISTS local_scores (
|
|
39
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
40
|
+
payload TEXT NOT NULL,
|
|
41
|
+
fetched_at_ms INTEGER NOT NULL
|
|
42
|
+
);
|
|
43
|
+
|
|
32
44
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
33
45
|
id TEXT PRIMARY KEY,
|
|
34
46
|
created_at_ms INTEGER NOT NULL,
|
|
@@ -37,6 +49,7 @@ CREATE TABLE IF NOT EXISTS ledger (
|
|
|
37
49
|
turn INTEGER NOT NULL,
|
|
38
50
|
requested_model TEXT NOT NULL,
|
|
39
51
|
harness_id TEXT NOT NULL DEFAULT '',
|
|
52
|
+
omp_session_id TEXT NOT NULL DEFAULT '',
|
|
40
53
|
slug TEXT NOT NULL,
|
|
41
54
|
served_slug TEXT,
|
|
42
55
|
tier TEXT NOT NULL,
|
|
@@ -118,6 +131,67 @@ END
|
|
|
118
131
|
WHERE error IS NOT NULL;
|
|
119
132
|
`;
|
|
120
133
|
|
|
134
|
+
// v5: ledger gains omp_session_id, so the toast extension can scope decisions to
|
|
135
|
+
// its own omp session. Before this, the only scoping was per-harness, so two
|
|
136
|
+
// interactive omp sessions of the same harness (the default: empty) each
|
|
137
|
+
// surfaced the other's routing toasts from the shared ledger. Existing rows
|
|
138
|
+
// backfill to '' (unknown session), matching the no-header default.
|
|
139
|
+
const MIGRATE_V5 = `
|
|
140
|
+
ALTER TABLE ledger ADD COLUMN omp_session_id TEXT NOT NULL DEFAULT '';
|
|
141
|
+
`;
|
|
142
|
+
|
|
143
|
+
// v6: classifier INPUTS. Before this the ledger recorded only what routing
|
|
144
|
+
// decided (tier, slug, cost, escalation) and never what it decided FROM, so a
|
|
145
|
+
// turn could not be replayed, and no weight could be fit offline. `features`
|
|
146
|
+
// is the verbatim feature vector as JSON; `score` and `confidence` are the
|
|
147
|
+
// classifier's own outputs, kept alongside because they are cheap and make
|
|
148
|
+
// weight drift detectable when the scorer changes under a fixed feature set.
|
|
149
|
+
// `classifier_reasons` is the per-feature breakdown, which `select.ts` drops
|
|
150
|
+
// from the decision trail on every path except a hysteresis hold.
|
|
151
|
+
//
|
|
152
|
+
// All nullable with no backfill: pre-v6 rows genuinely lack these inputs and
|
|
153
|
+
// NULL says so honestly. A DEFAULT would invent data that was never observed.
|
|
154
|
+
const MIGRATE_V6 = `
|
|
155
|
+
ALTER TABLE ledger ADD COLUMN features TEXT;
|
|
156
|
+
ALTER TABLE ledger ADD COLUMN score REAL;
|
|
157
|
+
ALTER TABLE ledger ADD COLUMN confidence REAL;
|
|
158
|
+
ALTER TABLE ledger ADD COLUMN task TEXT;
|
|
159
|
+
ALTER TABLE ledger ADD COLUMN classifier_reasons TEXT;
|
|
160
|
+
`;
|
|
161
|
+
|
|
162
|
+
// v7: records epsilon-greedy exploration. `explored_from` is the tier the
|
|
163
|
+
// classifier actually chose on a turn we deliberately routed one step
|
|
164
|
+
// cheaper; NULL means the turn was routed normally.
|
|
165
|
+
//
|
|
166
|
+
// This is the counterfactual the ledger could never observe before. Natural
|
|
167
|
+
// traffic only reveals UNDER-routing, because a tier that was too low
|
|
168
|
+
// escalates and leaves a trace, while a tier that was too high looks
|
|
169
|
+
// indistinguishable from a tier that was exactly right.
|
|
170
|
+
const MIGRATE_V7 = `
|
|
171
|
+
ALTER TABLE ledger ADD COLUMN explored_from TEXT;
|
|
172
|
+
`;
|
|
173
|
+
|
|
174
|
+
// v8: records the hold-length arm a conversation was assigned by hold
|
|
175
|
+
// exploration. NULL means the conversation was not part of the experiment.
|
|
176
|
+
//
|
|
177
|
+
// Recorded on EVERY turn of the conversation, not only the turns a hold
|
|
178
|
+
// actually affects, so arms can be compared on total conversation cost
|
|
179
|
+
// rather than on the subset the treatment happened to touch.
|
|
180
|
+
const MIGRATE_V8 = `
|
|
181
|
+
ALTER TABLE ledger ADD COLUMN hold_arm INTEGER;
|
|
182
|
+
`;
|
|
183
|
+
|
|
184
|
+
// v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
|
|
185
|
+
// BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
|
|
186
|
+
// whole new table, created idempotently by the MIGRATIONS block above, so there
|
|
187
|
+
// is no ALTER guard here — the version bump alone records that the schema now
|
|
188
|
+
// includes it.
|
|
189
|
+
|
|
190
|
+
// v10: local_scores holds calibrated scores from our OWN eval harness
|
|
191
|
+
// (src/eval), a `local`-source feed applied only when benchmarks.useLocalScores
|
|
192
|
+
// is on. Another new table via the idempotent MIGRATIONS block; version bump
|
|
193
|
+
// only, no ALTER guard.
|
|
194
|
+
|
|
121
195
|
export function openDb(path: string): Database {
|
|
122
196
|
// ":memory:" has no parent directory to create.
|
|
123
197
|
if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
|
|
@@ -134,6 +208,10 @@ export function openDb(path: string): Database {
|
|
|
134
208
|
const ledgerCols = db.query("PRAGMA table_info(ledger)").all() as { name: string }[];
|
|
135
209
|
if (!ledgerCols.some((c) => c.name === "harness_id")) db.exec(MIGRATE_V3);
|
|
136
210
|
if (!ledgerCols.some((c) => c.name === "error_kind")) db.exec(MIGRATE_V4);
|
|
211
|
+
if (!ledgerCols.some((c) => c.name === "omp_session_id")) db.exec(MIGRATE_V5);
|
|
212
|
+
if (!ledgerCols.some((c) => c.name === "features")) db.exec(MIGRATE_V6);
|
|
213
|
+
if (!ledgerCols.some((c) => c.name === "explored_from")) db.exec(MIGRATE_V7);
|
|
214
|
+
if (!ledgerCols.some((c) => c.name === "hold_arm")) db.exec(MIGRATE_V8);
|
|
137
215
|
db.exec(`PRAGMA user_version = ${USER_VERSION}`);
|
|
138
216
|
}
|
|
139
217
|
return db;
|
|
@@ -204,6 +204,10 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
204
204
|
// this via the provider block's `headers:` override; absent ⇒ single harness.
|
|
205
205
|
const harnessId = (headers.get("x-omp-harness") ?? "").trim();
|
|
206
206
|
|
|
207
|
+
// omp UI session id for per-session toast scoping. The embed extension sets
|
|
208
|
+
// this header to ctx.sessionManager.getSessionId(); absent ⇒ unknown session.
|
|
209
|
+
const ompSessionId = (headers.get("x-omp-session") ?? "").trim();
|
|
210
|
+
|
|
207
211
|
if (typeof b.model !== "string" || b.model.length === 0) {
|
|
208
212
|
throw invalidRequest("model must be a non-empty string");
|
|
209
213
|
}
|
|
@@ -262,6 +266,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
262
266
|
protocol: "openai-chat",
|
|
263
267
|
conversationKey,
|
|
264
268
|
harnessId,
|
|
269
|
+
ompSessionId,
|
|
265
270
|
requestedModel,
|
|
266
271
|
messages,
|
|
267
272
|
tools,
|
package/src/wire/types.ts
CHANGED
|
@@ -68,6 +68,13 @@ export interface NormRequest {
|
|
|
68
68
|
* client sends no header (single-harness default).
|
|
69
69
|
*/
|
|
70
70
|
harnessId: string;
|
|
71
|
+
/**
|
|
72
|
+
* omp UI session id from the `X-Omp-Session` request header, when the client
|
|
73
|
+
* sends one. Scopes toasts to a single interactive session so concurrent
|
|
74
|
+
* sessions sharing one router don't surface each other's choices. Empty when
|
|
75
|
+
* the client sends no header.
|
|
76
|
+
*/
|
|
77
|
+
ompSessionId: string;
|
|
71
78
|
/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
|
|
72
79
|
requestedModel: string;
|
|
73
80
|
messages: NormMessage[];
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
|
|
4
|
+
import {
|
|
5
|
+
applyFeedScores,
|
|
6
|
+
fetchBenchlmScores,
|
|
7
|
+
normalizeModelKey,
|
|
8
|
+
parseAaModels,
|
|
9
|
+
parseBenchlmModels,
|
|
10
|
+
refreshFeedScores,
|
|
11
|
+
type FeedScore,
|
|
12
|
+
type FetchLike,
|
|
13
|
+
} from "../src/catalog/benchmark-feeds.ts";
|
|
14
|
+
import { loadConfig } from "../src/config/load.ts";
|
|
15
|
+
import type { RouterConfig } from "../src/config/types.ts";
|
|
16
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
17
|
+
|
|
18
|
+
/** A bare OpenRouter `/models` record, optionally pre-scored. */
|
|
19
|
+
function raw(id: string, benchmarks?: Record<string, unknown>): Record<string, unknown> {
|
|
20
|
+
const record: Record<string, unknown> = {
|
|
21
|
+
id,
|
|
22
|
+
canonical_slug: id,
|
|
23
|
+
name: id,
|
|
24
|
+
context_length: 131_072,
|
|
25
|
+
pricing: { prompt: "0.0000003", completion: "0.0000011" },
|
|
26
|
+
supported_parameters: ["tools"],
|
|
27
|
+
architecture: { input_modalities: ["text"], tokenizer: "Other" },
|
|
28
|
+
created: 1_700_000_000,
|
|
29
|
+
};
|
|
30
|
+
if (benchmarks !== undefined) record.benchmarks = benchmarks;
|
|
31
|
+
return record;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function aaScore(over: Partial<FeedScore> & { key: string }): FeedScore {
|
|
35
|
+
return { creator: "", source: "artificial_analysis", ...over };
|
|
36
|
+
}
|
|
37
|
+
function blScore(over: Partial<FeedScore> & { key: string }): FeedScore {
|
|
38
|
+
return { creator: "", source: "benchlm", ...over };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe("normalizeModelKey", () => {
|
|
42
|
+
test("strips provider, tilde, and release words but keeps the parameter size", () => {
|
|
43
|
+
expect(normalizeModelKey("z-ai/glm-5.3-flash")).toBe("glm-5-3-flash");
|
|
44
|
+
expect(normalizeModelKey("~deepseek/deepseek-v4-flash-latest")).toBe("deepseek-v4-flash");
|
|
45
|
+
expect(normalizeModelKey("meta/muse-glimmer-30b")).toBe("muse-glimmer-30b");
|
|
46
|
+
// The feed's own display spelling collapses onto the same key.
|
|
47
|
+
expect(normalizeModelKey("Muse Glimmer 30B")).toBe("muse-glimmer-30b");
|
|
48
|
+
expect(normalizeModelKey("MiniMax M3")).toBe("minimax-m3");
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
describe("parseAaModels", () => {
|
|
53
|
+
test("reads the three indices, keeps in-range values, and skips empty rows", () => {
|
|
54
|
+
const body = {
|
|
55
|
+
data: [
|
|
56
|
+
{
|
|
57
|
+
slug: "glm-5.3-flash",
|
|
58
|
+
model_creator: { slug: "z-ai" },
|
|
59
|
+
evaluations: {
|
|
60
|
+
artificial_analysis_coding_index: 61.2,
|
|
61
|
+
artificial_analysis_intelligence_index: 58.4,
|
|
62
|
+
artificial_analysis_agentic_index: 150, // out of range → dropped
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
{ slug: "no-evals", model_creator: { slug: "x" }, evaluations: {} },
|
|
66
|
+
],
|
|
67
|
+
};
|
|
68
|
+
const parsed = parseAaModels(body);
|
|
69
|
+
expect(parsed).toHaveLength(1);
|
|
70
|
+
expect(parsed[0]).toMatchObject({ key: "glm-5-3-flash", creator: "z-ai", coding: 61.2, intelligence: 58.4 });
|
|
71
|
+
expect(parsed[0]?.agentic).toBeUndefined();
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe("parseBenchlmModels", () => {
|
|
76
|
+
test("maps categories to axes, drops estimated rows, and ignores out-of-range", () => {
|
|
77
|
+
const body = {
|
|
78
|
+
models: [
|
|
79
|
+
{
|
|
80
|
+
model: "Muse Glimmer 30B",
|
|
81
|
+
creator: "Meta",
|
|
82
|
+
evidenceStatus: "supported",
|
|
83
|
+
categoryScores: { coding: 55, reasoning: 52, agentic: 48 },
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
model: "Guessed Model",
|
|
87
|
+
creator: "x",
|
|
88
|
+
evidenceStatus: "estimated",
|
|
89
|
+
categoryScores: { coding: 90 },
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
};
|
|
93
|
+
const parsed = parseBenchlmModels(body);
|
|
94
|
+
expect(parsed).toHaveLength(1);
|
|
95
|
+
expect(parsed[0]).toMatchObject({ key: "muse-glimmer-30b", coding: 55, intelligence: 52, agentic: 48 });
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
describe("applyFeedScores", () => {
|
|
100
|
+
test("fills the real gap models and reaches normalizeCatalogModel", () => {
|
|
101
|
+
const catalog = [
|
|
102
|
+
raw("meta/muse-glimmer-30b"),
|
|
103
|
+
raw("z-ai/glm-5.3-flash"),
|
|
104
|
+
// Already scored by OpenRouter on coding; a feed must not overwrite it.
|
|
105
|
+
raw("google/gemini-3.7-flash", { artificial_analysis: { coding_index: 76.1 } }),
|
|
106
|
+
];
|
|
107
|
+
const feeds: FeedScore[] = [
|
|
108
|
+
aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58 }),
|
|
109
|
+
aaScore({ key: "gemini-3-7-flash", creator: "google", coding: 40, intelligence: 63 }),
|
|
110
|
+
blScore({ key: "muse-glimmer-30b", creator: "meta", coding: 55, agentic: 48 }),
|
|
111
|
+
blScore({ key: "glm-5-3-flash", creator: "z-ai", agentic: 44 }),
|
|
112
|
+
];
|
|
113
|
+
|
|
114
|
+
const result = applyFeedScores(catalog, feeds);
|
|
115
|
+
|
|
116
|
+
// muse-glimmer: was empty, gains coding + agentic from BenchLM.
|
|
117
|
+
const muse = normalizeCatalogModel(catalog[0]);
|
|
118
|
+
expect(muse?.quality).toEqual({ coding: 55, agentic: 48 });
|
|
119
|
+
|
|
120
|
+
// glm: coding + intelligence from AA (stronger), agentic from BenchLM.
|
|
121
|
+
const glm = normalizeCatalogModel(catalog[1]);
|
|
122
|
+
expect(glm?.quality).toEqual({ coding: 61, intelligence: 58, agentic: 44 });
|
|
123
|
+
|
|
124
|
+
// gemini: published coding survives untouched; intelligence filled from AA.
|
|
125
|
+
const gemini = normalizeCatalogModel(catalog[2]);
|
|
126
|
+
expect(gemini?.quality.coding).toBe(76.1);
|
|
127
|
+
expect(gemini?.quality.intelligence).toBe(63);
|
|
128
|
+
|
|
129
|
+
// Provenance recorded, counts add up.
|
|
130
|
+
const museBench = catalog[0]?.benchmarks;
|
|
131
|
+
expect(museBench).toMatchObject({ fill_sources: { coding: "benchlm", agentic: "benchlm" } });
|
|
132
|
+
expect(result.modelsFilled).toBe(3);
|
|
133
|
+
expect(result.sources.artificial_analysis).toBe(3); // glm coding+intel, gemini intel
|
|
134
|
+
expect(result.sources.benchlm).toBe(3); // muse coding+agentic, glm agentic
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test("AA wins over BenchLM for the same axis", () => {
|
|
138
|
+
const catalog = [raw("z-ai/glm-5.3-flash")];
|
|
139
|
+
const feeds: FeedScore[] = [
|
|
140
|
+
blScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 10 }),
|
|
141
|
+
aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
|
|
142
|
+
];
|
|
143
|
+
applyFeedScores(catalog, feeds);
|
|
144
|
+
expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("never fuzzy-matches a different model", () => {
|
|
148
|
+
const catalog = [raw("meta/muse-glimmer-30b")];
|
|
149
|
+
// Same family, different model — must not lend its score.
|
|
150
|
+
const feeds: FeedScore[] = [aaScore({ key: "muse-spark-1-2", creator: "meta", coding: 72 })];
|
|
151
|
+
const result = applyFeedScores(catalog, feeds);
|
|
152
|
+
expect(result.modelsFilled).toBe(0);
|
|
153
|
+
expect(normalizeCatalogModel(catalog[0])?.quality).toEqual({});
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("a shared key with conflicting creators fills only the creator that matches", () => {
|
|
157
|
+
const catalog = [raw("z-ai/glm-5.3-flash")];
|
|
158
|
+
const feeds: FeedScore[] = [
|
|
159
|
+
aaScore({ key: "glm-5-3-flash", creator: "someone-else", coding: 5 }),
|
|
160
|
+
aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
|
|
161
|
+
];
|
|
162
|
+
applyFeedScores(catalog, feeds);
|
|
163
|
+
expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
describe("refreshFeedScores", () => {
|
|
168
|
+
function cfgWith(over: Partial<RouterConfig["benchmarks"]>): RouterConfig {
|
|
169
|
+
const base = loadConfig({});
|
|
170
|
+
return { ...base, benchmarks: { ...base.benchmarks, ...over } };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
test("fetches once, then serves the cache within the TTL", async () => {
|
|
174
|
+
const db = openDb(":memory:");
|
|
175
|
+
let calls = 0;
|
|
176
|
+
const fakeFetch: FetchLike = async (url) => {
|
|
177
|
+
calls += 1;
|
|
178
|
+
const u = String(url);
|
|
179
|
+
if (u.includes("benchlm")) {
|
|
180
|
+
return Response.json({
|
|
181
|
+
models: [
|
|
182
|
+
{ model: "MiniMax M3", creator: "MiniMax", evidenceStatus: "supported", categoryScores: { coding: 58 } },
|
|
183
|
+
],
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
return Response.json({ data: [] });
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 1_000_000 });
|
|
190
|
+
const first = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 1000 });
|
|
191
|
+
expect(first).toHaveLength(1);
|
|
192
|
+
expect(first[0]).toMatchObject({ key: "minimax-m3", coding: 58, source: "benchlm" });
|
|
193
|
+
expect(calls).toBe(1); // AA skipped (no key), BenchLM fetched once
|
|
194
|
+
|
|
195
|
+
const second = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 2000 });
|
|
196
|
+
expect(second).toHaveLength(1);
|
|
197
|
+
expect(calls).toBe(1); // within TTL → no new fetch
|
|
198
|
+
db.close();
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test("falls back to the stale cache when a refresh returns nothing", async () => {
|
|
202
|
+
const db = openDb(":memory:");
|
|
203
|
+
const seed = [{ key: "minimax-m3", creator: "minimax", coding: 58, source: "benchlm" }];
|
|
204
|
+
db.query("INSERT INTO benchmark_cache (id, payload, fetched_at_ms) VALUES (1, ?, ?)").run(JSON.stringify(seed), 0);
|
|
205
|
+
const emptyFetch: FetchLike = async () => Response.json({ models: [] });
|
|
206
|
+
const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 10 });
|
|
207
|
+
const got = await refreshFeedScores(cfg, db, { fetchImpl: emptyFetch, now: 1_000_000 });
|
|
208
|
+
expect(got).toHaveLength(1);
|
|
209
|
+
expect(got[0]).toMatchObject({ key: "minimax-m3", coding: 58 });
|
|
210
|
+
db.close();
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
// fetchBenchlmScores over a fake fetch: the keyless path parses end to end.
|
|
215
|
+
test("fetchBenchlmScores parses a keyless leaderboard response", async () => {
|
|
216
|
+
const fake: FetchLike = async () =>
|
|
217
|
+
Response.json({
|
|
218
|
+
models: [{ model: "GLM 5.3 Flash", creator: "Z-AI", evidenceStatus: "supported", categoryScores: { coding: 61, reasoning: 58 } }],
|
|
219
|
+
});
|
|
220
|
+
const scores = await fetchBenchlmScores({ fetchImpl: fake });
|
|
221
|
+
expect(scores).toEqual([{ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58, source: "benchlm" }]);
|
|
222
|
+
});
|
package/test/escalate.test.ts
CHANGED