llm-relay 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -6
- package/config.example.json +37 -3
- package/dist/benchmarks.d.ts +7 -14
- package/dist/benchmarks.js +6 -83
- package/dist/benchmarks.js.map +1 -1
- package/dist/candidates.d.ts +27 -13
- package/dist/candidates.js +30 -17
- package/dist/candidates.js.map +1 -1
- package/dist/catalog.d.ts +38 -0
- package/dist/catalog.js +78 -3
- package/dist/catalog.js.map +1 -1
- package/dist/cli.js +19 -13
- package/dist/cli.js.map +1 -1
- package/dist/metadata.d.ts +46 -7
- package/dist/metadata.js +43 -36
- package/dist/metadata.js.map +1 -1
- package/dist/ping/ping.d.ts +0 -16
- package/dist/ping/ping.js +0 -96
- package/dist/ping/ping.js.map +1 -1
- package/dist/server.js +16 -7
- package/dist/server.js.map +1 -1
- package/dist/tier-data.d.ts +0 -2
- package/dist/tier-data.js +0 -4
- package/dist/tier-data.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -312,9 +312,9 @@ Precedence for a subagent request: `@relay:` directive → `subagents[<tier>]`
|
|
|
312
312
|
|
|
313
313
|
```
|
|
314
314
|
target pools / tiers str agentic coding BFCL arena $/Mout verdict p95 quota breaker ctx
|
|
315
|
-
nim/z-ai/glm-5.2 coding,@opus 83.3/4 43.1 68.8 - - $2.402
|
|
316
|
-
nim/moonshotai/kimi-k2.6 coding,@sonnet 77.4/5 30.3 61.8 - 1461 $2.72
|
|
317
|
-
nim/meta/llama-3.1-8b-instruct fast,@haiku,@fable 9.7/6 0.5 5.4 25.83 1211 $0.08
|
|
315
|
+
nim/z-ai/glm-5.2 coding,@opus 83.3/4 43.1 68.8 - - $2.402~ Perfect 310ms 84% closed 1049k~
|
|
316
|
+
nim/moonshotai/kimi-k2.6 coding,@sonnet 77.4/5 30.3 61.8 - 1461 $2.72~ Perfect 280ms 84% closed 262k~
|
|
317
|
+
nim/meta/llama-3.1-8b-instruct fast,@haiku,@fable 9.7/6 0.5 5.4 25.83 1211 $0.08~ Perfect 120ms 84% closed 131k~
|
|
318
318
|
```
|
|
319
319
|
|
|
320
320
|
Every offload target with its dimensions side by side: capability from each leaderboard separately,
|
|
@@ -343,9 +343,19 @@ available".
|
|
|
343
343
|
`str` is the single exception, and it exists only because ordering a pool requires an order. It is a
|
|
344
344
|
weighted mean of whatever rank-normalized signals a model actually has (tool-use and agentic ability
|
|
345
345
|
weighted highest — this proxy drives tool loops), and it never appears without its provenance:
|
|
346
|
-
`83.3/4` means four published signals backed it, while `
|
|
347
|
-
|
|
348
|
-
|
|
346
|
+
`83.3/4` means four published signals backed it, while `obs` (ranked on this proxy's own traffic,
|
|
347
|
+
≥5 calls) and `neut` (nothing known) mark the fallbacks.
|
|
348
|
+
|
|
349
|
+
**Limits and prices are per-(provider, model), and labelled.** The same model id on two providers is
|
|
350
|
+
two deployments — different context ceilings, different output caps, and possibly free on one and
|
|
351
|
+
metered on the other. Where a provider publishes its own figures (Groq, Mistral, OpenRouter) those
|
|
352
|
+
are used and shown unmarked; where it publishes none (NIM returns only `id`/`object`/`created`/
|
|
353
|
+
`owned_by`) the table falls back to another provider's figure for the same id and marks it `~`.
|
|
354
|
+
If nobody publishes one, the cell is blank — llm-relay does not guess a limit or a price.
|
|
355
|
+
|
|
356
|
+
That honesty is load-bearing: the **context guardrail only fires against a limit the serving
|
|
357
|
+
provider published**. If the limit is unknown the request goes upstream and the backend answers with
|
|
358
|
+
its own error, rather than llm-relay rejecting it against a number it made up.
|
|
349
359
|
|
|
350
360
|
`GET /candidates` returns the full JSON (the table shows a subset). The CLI prefers a running proxy
|
|
351
361
|
so the live columns come from warm ping history rather than a cold start.
|
package/config.example.json
CHANGED
|
@@ -60,12 +60,46 @@
|
|
|
60
60
|
"groq/llama-3.1-8b-instant",
|
|
61
61
|
"ollama/deepseek-r1:14b"
|
|
62
62
|
]
|
|
63
|
-
}
|
|
63
|
+
},
|
|
64
|
+
"pools": {
|
|
65
|
+
"coding": [
|
|
66
|
+
"nim/z-ai/glm-5.2",
|
|
67
|
+
"groq/llama-3.3-70b-versatile",
|
|
68
|
+
"gemini/gemini-2.5-flash"
|
|
69
|
+
],
|
|
70
|
+
"reasoning": [
|
|
71
|
+
"nim/nvidia/nemotron-3-super-120b-a12b",
|
|
72
|
+
"gemini/gemini-2.5-pro"
|
|
73
|
+
],
|
|
74
|
+
"fast": [
|
|
75
|
+
"groq/llama-3.1-8b-instant",
|
|
76
|
+
"nim/meta/llama-3.1-8b-instruct"
|
|
77
|
+
]
|
|
78
|
+
},
|
|
79
|
+
"subagents": {
|
|
80
|
+
"opus": "pool/reasoning",
|
|
81
|
+
"sonnet": "pool/coding",
|
|
82
|
+
"haiku": "pool/fast",
|
|
83
|
+
"fable": "pool/fast",
|
|
84
|
+
"default": "pool/coding"
|
|
85
|
+
},
|
|
86
|
+
"offload": false
|
|
64
87
|
},
|
|
65
88
|
"mode": "repair",
|
|
66
89
|
"repair": {
|
|
67
90
|
"maxAttempts": 2,
|
|
68
|
-
"destructiveTools": [
|
|
91
|
+
"destructiveTools": [
|
|
92
|
+
"rm",
|
|
93
|
+
"delete",
|
|
94
|
+
"push",
|
|
95
|
+
"force",
|
|
96
|
+
"overwrite",
|
|
97
|
+
"drop",
|
|
98
|
+
"reset"
|
|
99
|
+
]
|
|
69
100
|
},
|
|
70
|
-
"log": {
|
|
101
|
+
"log": {
|
|
102
|
+
"level": "metadata",
|
|
103
|
+
"file": null
|
|
104
|
+
}
|
|
71
105
|
}
|
package/dist/benchmarks.d.ts
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import type { ResolvedTarget } from "./config.js";
|
|
2
|
-
export interface BenchmarkScores {
|
|
3
|
-
sweBench?: number;
|
|
4
|
-
humanEval?: number;
|
|
5
|
-
liveCodeBench?: number;
|
|
6
|
-
arenaElo?: number;
|
|
7
|
-
}
|
|
8
|
-
/** Find benchmark scores for a model ID or provider/model spec. */
|
|
9
|
-
export declare function getBenchmarkScores(modelId: string): BenchmarkScores;
|
|
10
|
-
/** Calculate a single composite quality score (0-100) for ranking targets. */
|
|
11
|
-
export declare function calculateQualityScore(scores: BenchmarkScores): number;
|
|
12
2
|
/** Where a strength score came from. Ranking is only as trustworthy as its basis. */
|
|
13
|
-
export type StrengthBasis = "snapshot" | "
|
|
3
|
+
export type StrengthBasis = "snapshot" | "telemetry" | "neutral";
|
|
14
4
|
export interface Strength {
|
|
15
5
|
/** 0-100, comparable across bases. Higher is better. */
|
|
16
6
|
score: number;
|
|
@@ -27,14 +17,17 @@ export interface Strength {
|
|
|
27
17
|
* Strength of one target, from the best evidence available, in this order:
|
|
28
18
|
*
|
|
29
19
|
* 1. the synced multi-source snapshot — real published capability, refreshed by `sync:tiers`;
|
|
30
|
-
* 2.
|
|
31
|
-
* 3. observed runtime telemetry — not capability, but this deployment's own evidence that the
|
|
20
|
+
* 2. observed runtime telemetry — not capability, but this deployment's own evidence that the
|
|
32
21
|
* model answers successfully and quickly. Needs ≥5 real calls, so a brand-new model does not
|
|
33
22
|
* get ranked off one lucky request;
|
|
34
|
-
*
|
|
23
|
+
* 3. neutral — nothing is known, so claim nothing.
|
|
35
24
|
*
|
|
36
25
|
* The basis travels with the score precisely so a telemetry-derived number is never mistaken for
|
|
37
26
|
* a benchmark one.
|
|
27
|
+
*
|
|
28
|
+
* There is deliberately no hardcoded-table rung. `BENCHMARK_DB` used to sit here; every pattern it
|
|
29
|
+
* carried is present in the snapshot, so it contributed nothing but a stale, provenance-free number
|
|
30
|
+
* that outranked the synced data for any model it happened to substring-match.
|
|
38
31
|
*/
|
|
39
32
|
export declare function getStrength(spec: string, opts?: {
|
|
40
33
|
telemetryPath?: string;
|
package/dist/benchmarks.js
CHANGED
|
@@ -1,94 +1,21 @@
|
|
|
1
1
|
import { loadTierData, findTierModel } from "./tier-data.js";
|
|
2
2
|
import { getRealWorldScore } from "./ping/runtime-telemetry.js";
|
|
3
|
-
/**
|
|
4
|
-
* ⚠ LEGACY FALLBACK ONLY — do not add rows here.
|
|
5
|
-
*
|
|
6
|
-
* Hand-typed scores for a 2025-era roster, matched by substring. It was the ONLY ranking input
|
|
7
|
-
* until 0.5.0, which meant pool order came from a table nobody had updated: models it never heard
|
|
8
|
-
* of all collapsed to the flat 50.0 baseline and therefore tied, so `sort` silently fell back to
|
|
9
|
-
* config order. Capability now comes from `docs/tier-data.json` (`npm run sync:tiers`, multi-source,
|
|
10
|
-
* ~770 models); this survives only to score the handful of models no source has published on.
|
|
11
|
-
*
|
|
12
|
-
* It has no provenance: `glm-5` here matches `glm-5.2`, so its numbers can be a different SKU's.
|
|
13
|
-
*/
|
|
14
|
-
const BENCHMARK_DB = [
|
|
15
|
-
// Anthropic / Claude models
|
|
16
|
-
{ pattern: "claude-3-7-sonnet", scores: { sweBench: 70.3, humanEval: 92.0, liveCodeBench: 64.5, arenaElo: 1320 } },
|
|
17
|
-
{ pattern: "claude-3-5-sonnet", scores: { sweBench: 49.0, humanEval: 93.7, liveCodeBench: 58.2, arenaElo: 1280 } },
|
|
18
|
-
{ pattern: "claude-3-opus", scores: { sweBench: 38.0, humanEval: 84.9, liveCodeBench: 45.0, arenaElo: 1250 } },
|
|
19
|
-
{ pattern: "claude-3-5-haiku", scores: { sweBench: 40.6, humanEval: 88.1, liveCodeBench: 48.0, arenaElo: 1220 } },
|
|
20
|
-
// OpenAI / GPT models
|
|
21
|
-
{ pattern: "o3-mini", scores: { sweBench: 71.0, humanEval: 94.5, liveCodeBench: 66.0, arenaElo: 1330 } },
|
|
22
|
-
{ pattern: "o1", scores: { sweBench: 48.9, humanEval: 92.4, liveCodeBench: 61.5, arenaElo: 1310 } },
|
|
23
|
-
{ pattern: "gpt-4o", scores: { sweBench: 38.8, humanEval: 90.2, liveCodeBench: 52.1, arenaElo: 1286 } },
|
|
24
|
-
{ pattern: "gpt-4o-mini", scores: { sweBench: 29.0, humanEval: 87.2, liveCodeBench: 41.5, arenaElo: 1200 } },
|
|
25
|
-
// DeepSeek models
|
|
26
|
-
{ pattern: "deepseek-r1", scores: { sweBench: 49.2, humanEval: 96.1, liveCodeBench: 65.9, arenaElo: 1350 } },
|
|
27
|
-
{ pattern: "deepseek-v3", scores: { sweBench: 42.0, humanEval: 90.0, liveCodeBench: 58.0, arenaElo: 1310 } },
|
|
28
|
-
{ pattern: "deepseek-coder", scores: { sweBench: 38.5, humanEval: 90.2, liveCodeBench: 53.0, arenaElo: 1240 } },
|
|
29
|
-
// Qwen models
|
|
30
|
-
{ pattern: "qwen-2.5-coder-32b", scores: { sweBench: 41.2, humanEval: 92.7, liveCodeBench: 56.4, arenaElo: 1250 } },
|
|
31
|
-
{ pattern: "qwen-2.5-72b", scores: { sweBench: 37.0, humanEval: 86.6, liveCodeBench: 51.0, arenaElo: 1240 } },
|
|
32
|
-
{ pattern: "qwq-32b", scores: { sweBench: 44.0, humanEval: 93.0, liveCodeBench: 59.0, arenaElo: 1270 } },
|
|
33
|
-
// GLM / Z-AI / Nemotron / Llama / Mistral models
|
|
34
|
-
{ pattern: "glm-4", scores: { sweBench: 35.0, humanEval: 85.0, liveCodeBench: 47.0, arenaElo: 1200 } },
|
|
35
|
-
{ pattern: "glm-5", scores: { sweBench: 42.0, humanEval: 89.0, liveCodeBench: 53.0, arenaElo: 1240 } },
|
|
36
|
-
{ pattern: "nemotron", scores: { sweBench: 36.5, humanEval: 86.0, liveCodeBench: 48.0, arenaElo: 1210 } },
|
|
37
|
-
{ pattern: "llama-3.3-70b", scores: { sweBench: 36.0, humanEval: 88.6, liveCodeBench: 49.2, arenaElo: 1230 } },
|
|
38
|
-
{ pattern: "mistral-large", scores: { sweBench: 34.0, humanEval: 84.0, liveCodeBench: 45.0, arenaElo: 1210 } },
|
|
39
|
-
{ pattern: "codestral", scores: { sweBench: 33.5, humanEval: 81.1, liveCodeBench: 46.0, arenaElo: 1200 } },
|
|
40
|
-
];
|
|
41
|
-
/** Find benchmark scores for a model ID or provider/model spec. */
|
|
42
|
-
export function getBenchmarkScores(modelId) {
|
|
43
|
-
const norm = modelId.toLowerCase();
|
|
44
|
-
for (const entry of BENCHMARK_DB) {
|
|
45
|
-
if (typeof entry.pattern === "string") {
|
|
46
|
-
if (norm.includes(entry.pattern))
|
|
47
|
-
return entry.scores;
|
|
48
|
-
}
|
|
49
|
-
else if (entry.pattern.test(norm)) {
|
|
50
|
-
return entry.scores;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
return {};
|
|
54
|
-
}
|
|
55
|
-
/** Calculate a single composite quality score (0-100) for ranking targets. */
|
|
56
|
-
export function calculateQualityScore(scores) {
|
|
57
|
-
let totalWeight = 0;
|
|
58
|
-
let weightedSum = 0;
|
|
59
|
-
if (scores.sweBench !== undefined) {
|
|
60
|
-
weightedSum += scores.sweBench * 3.0; // Primary coding agent benchmark
|
|
61
|
-
totalWeight += 3.0;
|
|
62
|
-
}
|
|
63
|
-
if (scores.liveCodeBench !== undefined) {
|
|
64
|
-
weightedSum += scores.liveCodeBench * 2.0;
|
|
65
|
-
totalWeight += 2.0;
|
|
66
|
-
}
|
|
67
|
-
if (scores.humanEval !== undefined) {
|
|
68
|
-
weightedSum += scores.humanEval * 1.0;
|
|
69
|
-
totalWeight += 1.0;
|
|
70
|
-
}
|
|
71
|
-
if (totalWeight > 0) {
|
|
72
|
-
return Math.round((weightedSum / totalWeight) * 10) / 10;
|
|
73
|
-
}
|
|
74
|
-
if (scores.arenaElo !== undefined) {
|
|
75
|
-
return Math.max(0, Math.min(100, Math.round((scores.arenaElo - 1000) / 4)));
|
|
76
|
-
}
|
|
77
|
-
return 50.0; // Default baseline score for unlisted models
|
|
78
|
-
}
|
|
79
3
|
const NEUTRAL = 50;
|
|
80
4
|
/**
|
|
81
5
|
* Strength of one target, from the best evidence available, in this order:
|
|
82
6
|
*
|
|
83
7
|
* 1. the synced multi-source snapshot — real published capability, refreshed by `sync:tiers`;
|
|
84
|
-
* 2.
|
|
85
|
-
* 3. observed runtime telemetry — not capability, but this deployment's own evidence that the
|
|
8
|
+
* 2. observed runtime telemetry — not capability, but this deployment's own evidence that the
|
|
86
9
|
* model answers successfully and quickly. Needs ≥5 real calls, so a brand-new model does not
|
|
87
10
|
* get ranked off one lucky request;
|
|
88
|
-
*
|
|
11
|
+
* 3. neutral — nothing is known, so claim nothing.
|
|
89
12
|
*
|
|
90
13
|
* The basis travels with the score precisely so a telemetry-derived number is never mistaken for
|
|
91
14
|
* a benchmark one.
|
|
15
|
+
*
|
|
16
|
+
* There is deliberately no hardcoded-table rung. `BENCHMARK_DB` used to sit here; every pattern it
|
|
17
|
+
* carried is present in the snapshot, so it contributed nothing but a stale, provenance-free number
|
|
18
|
+
* that outranked the synced data for any model it happened to substring-match.
|
|
92
19
|
*/
|
|
93
20
|
export function getStrength(spec, opts = {}) {
|
|
94
21
|
const hit = findTierModel(spec, loadTierData()?.byNorm ?? []);
|
|
@@ -102,10 +29,6 @@ export function getStrength(spec, opts = {}) {
|
|
|
102
29
|
matchedName: hit.rec.norm,
|
|
103
30
|
};
|
|
104
31
|
}
|
|
105
|
-
const scores = getBenchmarkScores(spec);
|
|
106
|
-
if (Object.keys(scores).length > 0) {
|
|
107
|
-
return { score: calculateQualityScore(scores), basis: "static-table" };
|
|
108
|
-
}
|
|
109
32
|
const i = spec.indexOf("/");
|
|
110
33
|
if (i !== -1) {
|
|
111
34
|
const observed = getRealWorldScore(spec.slice(0, i), spec.slice(i + 1), opts.telemetryPath ? { path: opts.telemetryPath } : {});
|
package/dist/benchmarks.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmarks.js","sourceRoot":"","sources":["../src/benchmarks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmarks.js","sourceRoot":"","sources":["../src/benchmarks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAkBhE,MAAM,OAAO,GAAG,EAAE,CAAC;AAEnB;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAAmC,EAAE;IAC7E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;IAC9D,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,cAAc;YAC/D,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE;YAC9B,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC;YACtC,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI;SAC1B,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,iBAAiB,CAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAChB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EACjB,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CACvD,CAAC;QACF,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IACxE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,CAAC,CAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,KAAK,GAAG,CAAC,CAAiB,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
package/dist/candidates.d.ts
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import type { Config } from "./config.js";
|
|
2
2
|
import type { ModelCatalog } from "./catalog.js";
|
|
3
3
|
import type { PingLoop } from "./ping/cadence.js";
|
|
4
|
-
import { type
|
|
4
|
+
import { type StrengthBasis } from "./benchmarks.js";
|
|
5
|
+
import { type MetadataSource } from "./metadata.js";
|
|
5
6
|
import { type CircuitBreaker } from "./circuit-breaker.js";
|
|
6
|
-
import { type CapabilityScore } from "./registry.js";
|
|
7
7
|
/**
|
|
8
8
|
* Everything known about one offload destination, kept as SEPARATE raw dimensions.
|
|
9
9
|
*
|
|
10
|
-
* Deliberately un-blended:
|
|
11
|
-
* different things and trade off against each other differently per task ("cheapest that
|
|
12
|
-
* it" vs "best available"). Averaging them into one number would bury exactly the judgement
|
|
13
|
-
* reader is here to make. The
|
|
14
|
-
*
|
|
10
|
+
* Deliberately un-blended: each leaderboard, live stability, cost, quota and observed traffic
|
|
11
|
+
* measure different things and trade off against each other differently per task ("cheapest that
|
|
12
|
+
* can do it" vs "best available"). Averaging them into one number would bury exactly the judgement
|
|
13
|
+
* the reader is here to make. The one scalar that does exist lives under `sortInputs` with the
|
|
14
|
+
* basis and signals that produced it — because ordering a pool requires an order, not because it
|
|
15
|
+
* is a recommendation.
|
|
15
16
|
*/
|
|
16
17
|
export interface Candidate {
|
|
17
18
|
spec: string;
|
|
@@ -24,9 +25,12 @@ export interface Candidate {
|
|
|
24
25
|
hasKey: boolean;
|
|
25
26
|
/** In the provider's live /models catalog. null = not checkable (anthropic kind, or catalog down). */
|
|
26
27
|
listed: boolean | null;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
/** Which snapshot row `scores` came from, and how confidently. `fuzzy` = a similarly-named but
|
|
29
|
+
* DIFFERENT model's row, so those numbers are indicative. Null = no row matched. */
|
|
30
|
+
capabilityMatch: {
|
|
31
|
+
name: string;
|
|
32
|
+
match: "exact" | "fuzzy";
|
|
33
|
+
} | null;
|
|
30
34
|
health: {
|
|
31
35
|
verdict: string;
|
|
32
36
|
avgMs: number | null;
|
|
@@ -50,12 +54,22 @@ export interface Candidate {
|
|
|
50
54
|
avgLatencyMs: number | null;
|
|
51
55
|
lastCalledAt: string | null;
|
|
52
56
|
} | null;
|
|
57
|
+
/**
|
|
58
|
+
* Limits, each with its own provenance. `provider` = this provider published it about its own
|
|
59
|
+
* deployment; `reference` = borrowed from another provider serving the same model id (different
|
|
60
|
+
* deployment, so indicative only — `metadataReferenceFrom` names it); `static-table` = the
|
|
61
|
+
* hardcoded table, including its blanket 128k/4096 guess.
|
|
62
|
+
*/
|
|
53
63
|
contextLength: number | null;
|
|
54
|
-
|
|
55
|
-
contextLengthSource: "snapshot" | "static-table" | null;
|
|
64
|
+
contextLengthSource: MetadataSource | null;
|
|
56
65
|
maxOutputTokens: number | null;
|
|
66
|
+
maxOutputTokensSource: MetadataSource | null;
|
|
67
|
+
metadataReferenceFrom?: string;
|
|
68
|
+
/** Per-million-token price, with the same provenance rules — a model free on one host and
|
|
69
|
+
* metered on another must not report the other's rate. */
|
|
57
70
|
pricePerMTokIn: number | null;
|
|
58
71
|
pricePerMTokOut: number | null;
|
|
72
|
+
priceSource: MetadataSource | null;
|
|
59
73
|
supportsTools: boolean | null;
|
|
60
74
|
/** Which leaderboards published anything about this model. */
|
|
61
75
|
capabilitySources: string[];
|
|
@@ -83,7 +97,7 @@ export interface Candidate {
|
|
|
83
97
|
sortInputs: {
|
|
84
98
|
/** Drives `routing.benchmarkSort` ordering within a pool. */
|
|
85
99
|
strength: number;
|
|
86
|
-
/** snapshot |
|
|
100
|
+
/** snapshot | telemetry | neutral. A telemetry score is not a capability score. */
|
|
87
101
|
strengthBasis: StrengthBasis;
|
|
88
102
|
/** How many published signals backed it. 1 is a guess; 5 is a consensus. */
|
|
89
103
|
strengthSignals: string[];
|
package/dist/candidates.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { POOL_PREFIX } from "./config.js";
|
|
2
|
-
import {
|
|
2
|
+
import { getStrength } from "./benchmarks.js";
|
|
3
3
|
import { findTierModel } from "./tier-data.js";
|
|
4
|
-
import {
|
|
4
|
+
import { resolveMetadata } from "./metadata.js";
|
|
5
5
|
import { globalCircuitBreaker } from "./circuit-breaker.js";
|
|
6
6
|
import { loadRuntimeTelemetry } from "./ping/runtime-telemetry.js";
|
|
7
|
-
import { loadTierData
|
|
7
|
+
import { loadTierData } from "./registry.js";
|
|
8
8
|
const NOTE = "Raw per-dimension data for choosing an offload target. Nothing here is ranked or averaged — " +
|
|
9
9
|
"order is config order (pools, then subagent targets), and every source's score is kept " +
|
|
10
10
|
"separately under `scores`. `sortInputs` reports the ONE scalar the proxy needs for pool " +
|
|
@@ -71,14 +71,26 @@ export async function buildCandidates(cfg, opts = {}) {
|
|
|
71
71
|
const summary = opts.pingLoop && model ? opts.pingLoop.getModelSummary(provider, model) : null;
|
|
72
72
|
const state = breaker.getState(spec);
|
|
73
73
|
const obs = telemetry.models[`${provider}/${model}`];
|
|
74
|
-
const meta = getModelMetadata(model ?? provider);
|
|
75
|
-
const benchmarks = getBenchmarkScores(spec);
|
|
76
74
|
const strength = getStrength(spec);
|
|
77
|
-
const
|
|
75
|
+
const matched = findTierModel(model ?? spec, byNorm);
|
|
76
|
+
const tier = matched?.rec;
|
|
78
77
|
const num = (k) => (typeof tier?.[k] === "number" ? tier[k] : null);
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
|
|
78
|
+
// Limits THIS provider publishes about its own deployment, if any. NIM publishes none;
|
|
79
|
+
// Groq and Mistral publish real ones. The snapshot's numbers come from OpenRouter, so for a
|
|
80
|
+
// NIM target they are a different deployment's figures and are labelled `reference`, never
|
|
81
|
+
// presented as this provider's own.
|
|
82
|
+
const providerLimits = p && p.kind === "openai" && model && opts.catalog
|
|
83
|
+
? await opts.catalog.limits(provider, p, model).catch(() => null)
|
|
84
|
+
: null;
|
|
85
|
+
const meta = resolveMetadata(model ?? provider, {
|
|
86
|
+
providerLimits,
|
|
87
|
+
reference: {
|
|
88
|
+
contextLength: num("context_length"),
|
|
89
|
+
pricePromptPerToken: num("price_prompt"),
|
|
90
|
+
priceCompletionPerToken: num("price_completion"),
|
|
91
|
+
from: "openrouter",
|
|
92
|
+
},
|
|
93
|
+
});
|
|
82
94
|
candidates.push({
|
|
83
95
|
spec,
|
|
84
96
|
provider,
|
|
@@ -87,8 +99,7 @@ export async function buildCandidates(cfg, opts = {}) {
|
|
|
87
99
|
subagentTiers: membership.subagentTiers,
|
|
88
100
|
hasKey: p?.authEnv ? !!process.env[p.authEnv]?.trim() : true,
|
|
89
101
|
listed,
|
|
90
|
-
|
|
91
|
-
capability: model ? joinCapability(model, byNorm) : null,
|
|
102
|
+
capabilityMatch: matched ? { name: matched.rec.norm, match: matched.match } : null,
|
|
92
103
|
health: summary
|
|
93
104
|
? {
|
|
94
105
|
verdict: summary.verdict,
|
|
@@ -115,12 +126,14 @@ export async function buildCandidates(cfg, opts = {}) {
|
|
|
115
126
|
lastCalledAt: obs.lastCalledAt ? new Date(obs.lastCalledAt).toISOString() : null,
|
|
116
127
|
}
|
|
117
128
|
: null,
|
|
118
|
-
contextLength:
|
|
119
|
-
contextLengthSource:
|
|
120
|
-
maxOutputTokens: meta.maxOutputTokens
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
129
|
+
contextLength: meta.contextLength,
|
|
130
|
+
contextLengthSource: meta.contextLengthSource,
|
|
131
|
+
maxOutputTokens: meta.maxOutputTokens,
|
|
132
|
+
maxOutputTokensSource: meta.maxOutputTokensSource,
|
|
133
|
+
...(meta.referenceFrom ? { metadataReferenceFrom: meta.referenceFrom } : {}),
|
|
134
|
+
pricePerMTokIn: meta.pricePerMTokIn,
|
|
135
|
+
pricePerMTokOut: meta.pricePerMTokOut,
|
|
136
|
+
priceSource: meta.priceSource,
|
|
124
137
|
supportsTools: typeof tier?.supports_tools === "boolean" ? tier.supports_tools : null,
|
|
125
138
|
capabilitySources: Array.isArray(tier?.sources) ? tier.sources : [],
|
|
126
139
|
scores: {
|
package/dist/candidates.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"candidates.js","sourceRoot":"","sources":["../src/candidates.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"candidates.js","sourceRoot":"","sources":["../src/candidates.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,EAAE,WAAW,EAAsB,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AA4G7C,MAAM,IAAI,GACR,8FAA8F;IAC9F,yFAAyF;IACzF,0FAA0F;IAC1F,wFAAwF,CAAC;AAE3F,2FAA2F;AAC3F,SAAS,UAAU,CAAC,IAAY,EAAE,GAAW;IAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChD,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,GAAG,GAAG,IAAI,GAAG,EAAwD,CAAC;IAC5E,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE;QAC7B,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;YACrC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;AAClG,CAAC;AAED,+DAA+D;AAC/D,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAW,EACX,OAOI,EAAE;IAEN,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,oBAAoB,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,YAAY,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,oBAAoB,EAAE,CAAC;IAEzC,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ;YAAE,SAAS;QAC1D,MAAM,CAAC,GAA+B,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAE9D,IAAI,MAAM,GAAmB,IAAI,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/F,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,IAAI,IAAI,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,OAAO,EAAE,GAAG,CAAC;QAC1B,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAExF,uFAAuF;QACvF,4FAA4F;QAC5F,2FAA2F;QAC3F,oCAAoC;QACpC,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;YACtE,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;YACjE,CAAC,CAAC,IAAI,CAAC;QACT,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,IAAI,QAAQ,EAAE;YAC9C,cAAc;YACd,SAAS,EAAE;gBACT,aAAa,EAAE,GAAG,CAAC,gBAAgB,CAAC;gBACpC,mBAAmB,EAAE,GAAG,CAAC,cAAc,CAAC;gBACxC,uBAAuB,EAAE,GAAG,CAAC,kBAAkB,CAAC;gBAChD,IAAI,EAAE,YAAY;aACnB;SACF,CAAC,CAAC;QAEH,UAAU,CAAC,IAAI,CAAC;YACd,IAAI;YACJ,QAAQ;YACR,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,aAAa,EAAE,UAAU,CAAC,aAAa;YACvC,MAAM,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;YAC5D,MAAM;YACN,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YAClF,MAAM,EAAE,OAAO;gBACb,CAAC,CAAC;oBACE,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;oBAChD,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;oBAChD,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;oBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;iBAC/B;gBACH,CAAC,CAAC,IAAI;YACR,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;YAC7E,OAAO,EAAE;gBACP,IAAI,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;gBACrC,mBAAmB,EAAE,KAAK,EAAE,mBAAmB,IAAI,CAAC;gBACpD,UAAU,EAAE,KAAK,EAAE,UAAU,IAAI,IAAI;gBACrC,mBAAmB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;aACtE;YACD,QAAQ,EAAE,GAAG;gBACX,CAAC,CAAC;oBACE,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,YAAY,EAAE,GAAG,CAAC,YAAY;oBAC9B,YAAY,EAAE,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;oBACzF,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;iBACjF;gBACH,CAAC,CAAC,IAAI;YACR,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,aAAa,EAAE,OAAO,IAAI,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YACrF,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,OAAoB,CAAC,CAAC,CAAC,EAAE;YACjF,MAAM,EAAE;gBACN,WAAW,EAAE,GAAG,CAAC,cAAc,CAAC;gBAChC,aAAa,EAAE,GAAG,CAAC,iBAAiB,CAAC;gBACrC,eAAe,EAAE,GAAG,CAAC,kBAAkB,CAAC;gBACxC,cAAc,EAAE,GAAG,CAAC,iBAAiB,CAAC;gBACtC,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC;gBAC1B,SAAS,EAAE,GAAG,CAAC,YAAY,CAAC;gBAC5B,aAAa,EAAE,GAAG,CAAC,iBAAiB,CAAC;gBACrC,eAAe,EAAE,GAAG,CAAC,mBAAmB,CAAC;gBACzC,wBAAwB,EAAE,GAAG,CAAC,8BAA8B,CAAC;gBAC7D,wBAAwB,EAAE,GAAG,CAAC,8BAA8B,CAAC;gBAC7D,WAAW,EAAE,GAAG,CAAC,cAAc,CAAC;gBAChC,SAAS,EAAE,GAAG,CAAC,YAAY,CAAC;aAC7B;YACD,UAAU,EAAE;gBACV,QAAQ,EAAE,QAAQ,CAAC,KAAK;gBACxB,aAAa,EAAE,QAAQ,CAAC,KAAK;gBAC7B,eAAe,EAAE,QAAQ,CAAC,OAAO,IAAI,EAAE;gBACvC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;aAClD;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;QACvD,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,IAAI;QAC7C,IAAI,EAAE,IAAI;QACV,UAAU;KACX,CAAC;AACJ,CAAC"}
|
package/dist/catalog.d.ts
CHANGED
|
@@ -1,4 +1,21 @@
|
|
|
1
1
|
import type { ProviderConfig } from "./config.js";
|
|
2
|
+
/**
|
|
3
|
+
* Limits a provider publishes about its OWN deployment of a model.
|
|
4
|
+
*
|
|
5
|
+
* Deliberately per-(provider, model): the same model id served by two providers is two different
|
|
6
|
+
* deployments with different ceilings, so one provider's numbers must never be presented as
|
|
7
|
+
* another's. Null means "this provider does not publish it" — NIM's /models returns only
|
|
8
|
+
* id/object/created/owned_by, while Groq and Mistral publish real limits.
|
|
9
|
+
*/
|
|
10
|
+
export interface ModelLimits {
|
|
11
|
+
contextLength: number | null;
|
|
12
|
+
maxOutputTokens: number | null;
|
|
13
|
+
/** Per-TOKEN price, as published. Per-provider for the same reason limits are. */
|
|
14
|
+
pricePromptPerToken: number | null;
|
|
15
|
+
priceCompletionPerToken: number | null;
|
|
16
|
+
}
|
|
17
|
+
/** Read limits + pricing out of one `/models` record, including a nested `top_provider` (OpenRouter). */
|
|
18
|
+
export declare function limitsFromRecord(rec: Record<string, unknown>): ModelLimits;
|
|
2
19
|
/**
|
|
3
20
|
* Live per-provider model catalog — model ids are DISCOVERED from each provider's
|
|
4
21
|
* OpenAI-compatible `/models` endpoint, never hand-maintained. In-memory TTL cache
|
|
@@ -14,6 +31,8 @@ export declare class ModelCatalog {
|
|
|
14
31
|
private refreshing;
|
|
15
32
|
/** In-flight blocking fetches (cold start / forced) — dedups concurrent requests. */
|
|
16
33
|
private pending;
|
|
34
|
+
/** Limits harvested by the most recent `fetch()`, handed to the Entry by its caller. */
|
|
35
|
+
private lastLimits;
|
|
17
36
|
constructor(opts?: {
|
|
18
37
|
ttlMs?: number;
|
|
19
38
|
cachePath?: string | null;
|
|
@@ -53,5 +72,24 @@ export declare class ModelCatalog {
|
|
|
53
72
|
now?: number;
|
|
54
73
|
fetchFn?: typeof fetch;
|
|
55
74
|
}): Promise<boolean | null>;
|
|
75
|
+
/**
|
|
76
|
+
* Already-cached limits for a model — synchronous, never fetches.
|
|
77
|
+
*
|
|
78
|
+
* For the request hot path, where a blocking upstream fetch to learn a context window would be a
|
|
79
|
+
* worse outcome than simply not enforcing a guardrail on the first request. Returns null until
|
|
80
|
+
* the catalog has been warmed (startup does that), which callers must treat as "unknown".
|
|
81
|
+
*/
|
|
82
|
+
cachedLimits(name: string, model: string): ModelLimits | null;
|
|
83
|
+
/**
|
|
84
|
+
* Limits this provider publishes for one of its own models, or null when it publishes none.
|
|
85
|
+
*
|
|
86
|
+
* Null is meaningful and must not be papered over with another provider's number — see
|
|
87
|
+
* `resolveMetadata()` in metadata.ts, which decides what to fall back to and labels it.
|
|
88
|
+
*/
|
|
89
|
+
limits(name: string, cfg: ProviderConfig, model: string, opts?: {
|
|
90
|
+
force?: boolean;
|
|
91
|
+
now?: number;
|
|
92
|
+
fetchFn?: typeof fetch;
|
|
93
|
+
}): Promise<ModelLimits | null>;
|
|
56
94
|
private fetch;
|
|
57
95
|
}
|
package/dist/catalog.js
CHANGED
|
@@ -3,6 +3,45 @@ import { join, dirname } from "node:path";
|
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
const DEFAULT_TTL_MS = 10 * 60 * 1000; // 10 min
|
|
5
5
|
const DEFAULT_CACHE = join(homedir(), ".llm-relay", "models-cache.json");
|
|
6
|
+
/**
|
|
7
|
+
* Field aliases across OpenAI-compatible `/models` implementations. Kept as a generic alias list
|
|
8
|
+
* rather than a per-provider switch — a new provider that happens to publish `context_window` is
|
|
9
|
+
* picked up with no code change, and no provider name is hardcoded.
|
|
10
|
+
*/
|
|
11
|
+
const CONTEXT_FIELDS = ["context_length", "context_window", "max_context_length", "max_model_len"];
|
|
12
|
+
const MAX_OUTPUT_FIELDS = ["max_completion_tokens", "max_output_length", "max_output_tokens", "max_tokens"];
|
|
13
|
+
const PRICE_IN_FIELDS = ["prompt", "input", "input_tokens"];
|
|
14
|
+
const PRICE_OUT_FIELDS = ["completion", "output", "output_tokens"];
|
|
15
|
+
/** Providers publish prices as numeric STRINGS ("0.00000015") as often as numbers. */
|
|
16
|
+
function pickNumber(rec, fields, allowZero = false) {
|
|
17
|
+
for (const f of fields) {
|
|
18
|
+
const raw = rec[f];
|
|
19
|
+
const v = typeof raw === "string" ? Number(raw) : raw;
|
|
20
|
+
if (typeof v === "number" && Number.isFinite(v) && (allowZero ? v >= 0 : v > 0))
|
|
21
|
+
return v;
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
/** Read limits + pricing out of one `/models` record, including a nested `top_provider` (OpenRouter). */
|
|
26
|
+
export function limitsFromRecord(rec) {
|
|
27
|
+
const obj = (v) => (typeof v === "object" && v !== null ? v : {});
|
|
28
|
+
const top = obj(rec.top_provider);
|
|
29
|
+
const pricing = obj(rec.pricing);
|
|
30
|
+
return {
|
|
31
|
+
contextLength: pickNumber(rec, CONTEXT_FIELDS) ?? pickNumber(top, CONTEXT_FIELDS),
|
|
32
|
+
maxOutputTokens: pickNumber(rec, MAX_OUTPUT_FIELDS) ?? pickNumber(top, MAX_OUTPUT_FIELDS),
|
|
33
|
+
// Zero is a real, meaningful price (free tiers) — not "unpublished".
|
|
34
|
+
pricePromptPerToken: pickNumber(pricing, PRICE_IN_FIELDS, true),
|
|
35
|
+
priceCompletionPerToken: pickNumber(pricing, PRICE_OUT_FIELDS, true),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** True when a provider published nothing at all about a model. */
|
|
39
|
+
function isEmpty(l) {
|
|
40
|
+
return (l.contextLength === null &&
|
|
41
|
+
l.maxOutputTokens === null &&
|
|
42
|
+
l.pricePromptPerToken === null &&
|
|
43
|
+
l.priceCompletionPerToken === null);
|
|
44
|
+
}
|
|
6
45
|
/**
|
|
7
46
|
* Live per-provider model catalog — model ids are DISCOVERED from each provider's
|
|
8
47
|
* OpenAI-compatible `/models` endpoint, never hand-maintained. In-memory TTL cache
|
|
@@ -18,6 +57,8 @@ export class ModelCatalog {
|
|
|
18
57
|
refreshing = new Set();
|
|
19
58
|
/** In-flight blocking fetches (cold start / forced) — dedups concurrent requests. */
|
|
20
59
|
pending = new Map();
|
|
60
|
+
/** Limits harvested by the most recent `fetch()`, handed to the Entry by its caller. */
|
|
61
|
+
lastLimits = {};
|
|
21
62
|
constructor(opts = {}) {
|
|
22
63
|
this.ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;
|
|
23
64
|
this.cachePath = opts.cachePath === undefined ? DEFAULT_CACHE : opts.cachePath;
|
|
@@ -87,7 +128,7 @@ export class ModelCatalog {
|
|
|
87
128
|
const p = (async () => {
|
|
88
129
|
try {
|
|
89
130
|
const models = await this.fetch(cfg, opts.fetchFn ?? fetch);
|
|
90
|
-
this.mem.set(name, { fetchedAt: now, models });
|
|
131
|
+
this.mem.set(name, { fetchedAt: now, models, limits: this.lastLimits });
|
|
91
132
|
this.saveDisk();
|
|
92
133
|
return models;
|
|
93
134
|
}
|
|
@@ -113,7 +154,7 @@ export class ModelCatalog {
|
|
|
113
154
|
void (async () => {
|
|
114
155
|
try {
|
|
115
156
|
const models = await this.fetch(cfg, fetchFn ?? fetch);
|
|
116
|
-
this.mem.set(name, { fetchedAt: Date.now(), models });
|
|
157
|
+
this.mem.set(name, { fetchedAt: Date.now(), models, limits: this.lastLimits });
|
|
117
158
|
this.saveDisk();
|
|
118
159
|
}
|
|
119
160
|
catch {
|
|
@@ -135,6 +176,31 @@ export class ModelCatalog {
|
|
|
135
176
|
return null;
|
|
136
177
|
return models.includes(model);
|
|
137
178
|
}
|
|
179
|
+
/**
|
|
180
|
+
* Already-cached limits for a model — synchronous, never fetches.
|
|
181
|
+
*
|
|
182
|
+
* For the request hot path, where a blocking upstream fetch to learn a context window would be a
|
|
183
|
+
* worse outcome than simply not enforcing a guardrail on the first request. Returns null until
|
|
184
|
+
* the catalog has been warmed (startup does that), which callers must treat as "unknown".
|
|
185
|
+
*/
|
|
186
|
+
cachedLimits(name, model) {
|
|
187
|
+
this.loadDisk();
|
|
188
|
+
const l = this.mem.get(name)?.limits?.[model];
|
|
189
|
+
return l && !isEmpty(l) ? l : null;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Limits this provider publishes for one of its own models, or null when it publishes none.
|
|
193
|
+
*
|
|
194
|
+
* Null is meaningful and must not be papered over with another provider's number — see
|
|
195
|
+
* `resolveMetadata()` in metadata.ts, which decides what to fall back to and labels it.
|
|
196
|
+
*/
|
|
197
|
+
async limits(name, cfg, model, opts = {}) {
|
|
198
|
+
await this.list(name, cfg, opts);
|
|
199
|
+
const l = this.mem.get(name)?.limits?.[model];
|
|
200
|
+
if (!l)
|
|
201
|
+
return null;
|
|
202
|
+
return isEmpty(l) ? null : l;
|
|
203
|
+
}
|
|
138
204
|
async fetch(cfg, fetchFn) {
|
|
139
205
|
// Anthropic-kind backends have no OpenAI-style /models list we consume.
|
|
140
206
|
if (cfg.kind !== "openai")
|
|
@@ -152,7 +218,16 @@ export class ModelCatalog {
|
|
|
152
218
|
if (!res.ok)
|
|
153
219
|
throw new Error(`models fetch HTTP ${res.status}`);
|
|
154
220
|
const j = (await res.json());
|
|
155
|
-
|
|
221
|
+
const records = j.data ?? [];
|
|
222
|
+
this.lastLimits = {};
|
|
223
|
+
for (const rec of records) {
|
|
224
|
+
if (typeof rec?.id !== "string")
|
|
225
|
+
continue;
|
|
226
|
+
const l = limitsFromRecord(rec);
|
|
227
|
+
if (!isEmpty(l))
|
|
228
|
+
this.lastLimits[rec.id] = l;
|
|
229
|
+
}
|
|
230
|
+
return records
|
|
156
231
|
.map((m) => m.id)
|
|
157
232
|
.filter((s) => typeof s === "string")
|
|
158
233
|
.sort();
|