llm-relay 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -11
- package/dist/benchmarks.d.ts +34 -1
- package/dist/benchmarks.js +67 -9
- package/dist/benchmarks.js.map +1 -1
- package/dist/candidates.d.ts +34 -3
- package/dist/candidates.js +36 -9
- package/dist/candidates.js.map +1 -1
- package/dist/cli.js +56 -22
- package/dist/cli.js.map +1 -1
- package/dist/registry.d.ts +8 -5
- package/dist/registry.js +8 -18
- package/dist/registry.js.map +1 -1
- package/dist/tier-data.d.ts +43 -0
- package/dist/tier-data.js +61 -0
- package/dist/tier-data.js.map +1 -0
- package/docs/tier-data.json +15114 -2268
- package/package.json +1 -1
package/dist/registry.js
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
try {
|
|
5
|
-
const path = fileURLToPath(new URL("../docs/tier-data.json", import.meta.url));
|
|
6
|
-
const j = JSON.parse(readFileSync(path, "utf8"));
|
|
7
|
-
return { ...(j.synced_at ? { synced_at: j.synced_at } : {}), models: Array.isArray(j.models) ? j.models : [] };
|
|
8
|
-
}
|
|
9
|
-
catch {
|
|
10
|
-
return null;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
function toScore(r) {
|
|
1
|
+
import { loadTierData } from "./tier-data.js";
|
|
2
|
+
export { loadTierData };
|
|
3
|
+
function toScore(r, match) {
|
|
14
4
|
const n = (k) => (typeof r[k] === "number" ? r[k] : null);
|
|
15
5
|
return {
|
|
16
6
|
bfcl_overall: n("bfcl_overall"),
|
|
@@ -19,6 +9,8 @@ function toScore(r) {
|
|
|
19
9
|
arena_rating: n("arena_rating"),
|
|
20
10
|
arena_rank: n("arena_rank"),
|
|
21
11
|
composite_rank: n("composite_rank"),
|
|
12
|
+
matched_name: typeof r.norm === "string" ? r.norm : null,
|
|
13
|
+
match,
|
|
22
14
|
};
|
|
23
15
|
}
|
|
24
16
|
/**
|
|
@@ -34,16 +26,14 @@ export function joinCapability(modelId, byNorm) {
|
|
|
34
26
|
return null;
|
|
35
27
|
const exact = byNorm.find((e) => e.norm === seg);
|
|
36
28
|
if (exact)
|
|
37
|
-
return toScore(exact.rec);
|
|
29
|
+
return toScore(exact.rec, "exact");
|
|
38
30
|
const contained = byNorm.find((e) => e.norm.includes(seg));
|
|
39
|
-
return contained ? toScore(contained.rec) : null;
|
|
31
|
+
return contained ? toScore(contained.rec, "fuzzy") : null;
|
|
40
32
|
}
|
|
41
33
|
/** Build the discovery view: providers × live models (best-effort capability) + routing + raw scores. */
|
|
42
34
|
export async function buildRegistry(cfg, catalog, opts = {}) {
|
|
43
35
|
const tierData = loadTierData();
|
|
44
|
-
const byNorm =
|
|
45
|
-
.filter((r) => typeof r.norm === "string")
|
|
46
|
-
.map((r) => ({ norm: r.norm.toLowerCase(), rec: r }));
|
|
36
|
+
const byNorm = tierData?.byNorm ?? [];
|
|
47
37
|
const providers = {};
|
|
48
38
|
for (const [name, p] of Object.entries(cfg.providers)) {
|
|
49
39
|
const has_key = p.authEnv ? !!process.env[p.authEnv]?.trim() : true;
|
package/dist/registry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,CAAC;AA2DxB,SAAS,OAAO,CAAC,CAA0B,EAAE,KAAwB;IACnE,MAAM,CAAC,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9E,OAAO;QACL,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC;QAC/B,eAAe,EAAE,CAAC,CAAC,iBAAiB,CAAC;QACrC,gBAAgB,EAAE,CAAC,CAAC,kBAAkB,CAAC;QACvC,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC;QAC/B,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC;QAC3B,cAAc,EAAE,CAAC,CAAC,gBAAgB,CAAC;QACnC,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;QACxD,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,MAA6D;IAE7D,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACvE,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IACjD,IAAI,KAAK;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,OAAO,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5D,CAAC;AAED,yGAAyG;AACzG,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,OAAqB,EACrB,OAA8C,EAAE;IAEhD,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC;IAEtC,MAAM,SAAS,GAAqC,EAAE,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAoC,EAAE,CAAC;QACzF,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACpE,IAAI,MAAM,GAAoB,EAAE,CAAC;QACjC,IAAI,SAAS,GAAmB,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACxC,SAAS,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;gBACtB,MAAM,UAAU,GAAG,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACnF,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAEvF,SAAS,CAAC,IAAI,CAAC,GAAG;YAChB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,OAAO;YACP,SAAS;YACT,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM;SACP,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAClD,OAAO,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE;QACnE,SAAS;QACT,iBAAiB,EAAE;YACjB,OAAO,EAAE,CAAC,CAAC,QAAQ;YACnB,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,EAAE,QAAQ;gBACZ,CAAC,CAAC,0IAA0I;gBAC5I,CAAC,CAAC,qFAAqF;YACzF,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Access to the synced capability snapshot (`docs/tier-data.json`, written by
|
|
3
|
+
* `npm run sync:tiers` from OpenRouter + BFCL + LMArena + Aider).
|
|
4
|
+
*
|
|
5
|
+
* Its own module so both `registry.ts` (the /registry view) and `benchmarks.ts` (pool ranking)
|
|
6
|
+
* can read it without an import cycle — `config.ts` imports `benchmarks.ts`, so anything
|
|
7
|
+
* `benchmarks.ts` pulls in must not reach back into `config.ts`.
|
|
8
|
+
*/
|
|
9
|
+
export interface TierModel extends Record<string, unknown> {
|
|
10
|
+
name?: string;
|
|
11
|
+
norm: string;
|
|
12
|
+
sources?: string[];
|
|
13
|
+
strength?: number | null;
|
|
14
|
+
strength_rank?: number | null;
|
|
15
|
+
signals?: string[];
|
|
16
|
+
signal_count?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface TierData {
|
|
19
|
+
synced_at?: string;
|
|
20
|
+
models: TierModel[];
|
|
21
|
+
/** Lower-cased `norm` index, built once per load — lookups are a scan over this. */
|
|
22
|
+
byNorm: Array<{
|
|
23
|
+
norm: string;
|
|
24
|
+
rec: TierModel;
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
export declare function loadTierData(): TierData | null;
|
|
28
|
+
/** Test seam: drop the memo so a fixture written mid-test is re-read. */
|
|
29
|
+
export declare function _resetTierDataCache(): void;
|
|
30
|
+
export interface TierMatch {
|
|
31
|
+
rec: TierModel;
|
|
32
|
+
/** `fuzzy` means a DIFFERENT model's row whose name contains this one's — indicative, not measured. */
|
|
33
|
+
match: "exact" | "fuzzy";
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Look a routing spec up in the snapshot. Matches on the id's last segment, which is exactly the
|
|
37
|
+
* key `sync-tiers.mjs` stores for OpenRouter models — so a spec like `nim/z-ai/glm-5.2` hits the
|
|
38
|
+
* `glm-5.2` row exactly rather than fuzzily landing on `glm-5.2-max`.
|
|
39
|
+
*/
|
|
40
|
+
export declare function findTierModel(modelId: string, byNorm: Array<{
|
|
41
|
+
norm: string;
|
|
42
|
+
rec: TierModel;
|
|
43
|
+
}>): TierMatch | null;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
/**
|
|
4
|
+
* Memoized on the file's mtime so `npm run sync:tiers` is picked up without a restart. The file is
|
|
5
|
+
* ~770 rows and three endpoints need it per request; re-reading and re-indexing each time is pure
|
|
6
|
+
* waste. Negative results are cached too, so a missing file is not a stat+throw per request.
|
|
7
|
+
*/
|
|
8
|
+
let _cache = null;
|
|
9
|
+
export function loadTierData() {
|
|
10
|
+
let mtimeMs = null;
|
|
11
|
+
let path;
|
|
12
|
+
try {
|
|
13
|
+
path = fileURLToPath(new URL("../docs/tier-data.json", import.meta.url));
|
|
14
|
+
mtimeMs = statSync(path).mtimeMs;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
if (_cache && _cache.mtimeMs === null)
|
|
18
|
+
return _cache.data;
|
|
19
|
+
_cache = { mtimeMs: null, data: null };
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
if (_cache && _cache.mtimeMs === mtimeMs)
|
|
23
|
+
return _cache.data;
|
|
24
|
+
try {
|
|
25
|
+
const j = JSON.parse(readFileSync(path, "utf8"));
|
|
26
|
+
const models = Array.isArray(j.models) ? j.models : [];
|
|
27
|
+
const data = {
|
|
28
|
+
...(j.synced_at ? { synced_at: j.synced_at } : {}),
|
|
29
|
+
models,
|
|
30
|
+
byNorm: models
|
|
31
|
+
.filter((r) => typeof r.norm === "string")
|
|
32
|
+
.map((r) => ({ norm: r.norm.toLowerCase(), rec: r })),
|
|
33
|
+
};
|
|
34
|
+
_cache = { mtimeMs, data };
|
|
35
|
+
return data;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
_cache = { mtimeMs, data: null };
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Test seam: drop the memo so a fixture written mid-test is re-read. */
|
|
43
|
+
export function _resetTierDataCache() {
|
|
44
|
+
_cache = null;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Look a routing spec up in the snapshot. Matches on the id's last segment, which is exactly the
|
|
48
|
+
* key `sync-tiers.mjs` stores for OpenRouter models — so a spec like `nim/z-ai/glm-5.2` hits the
|
|
49
|
+
* `glm-5.2` row exactly rather than fuzzily landing on `glm-5.2-max`.
|
|
50
|
+
*/
|
|
51
|
+
export function findTierModel(modelId, byNorm) {
|
|
52
|
+
const seg = (modelId.split("/").pop() ?? modelId).toLowerCase().trim();
|
|
53
|
+
if (seg.length < 5)
|
|
54
|
+
return null;
|
|
55
|
+
const exact = byNorm.find((e) => e.norm === seg);
|
|
56
|
+
if (exact)
|
|
57
|
+
return { rec: exact.rec, match: "exact" };
|
|
58
|
+
const contained = byNorm.find((e) => e.norm.includes(seg));
|
|
59
|
+
return contained ? { rec: contained.rec, match: "fuzzy" } : null;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=tier-data.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tier-data.js","sourceRoot":"","sources":["../src/tier-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA2BzC;;;;GAIG;AACH,IAAI,MAAM,GAA6D,IAAI,CAAC;AAE5E,MAAM,UAAU,YAAY;IAC1B,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC;QAC1D,MAAM,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IAE7D,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAiD,CAAC;QACjG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAa;YACrB,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM;YACN,MAAM,EAAE,MAAM;iBACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;iBACzC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;SACxD,CAAC;QACF,MAAM,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,mBAAmB;IACjC,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAQD;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,MAA+C;IAE/C,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACvE,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IACjD,IAAI,KAAK;QAAE,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IACrD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,OAAO,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACnE,CAAC"}
|