llm-relay 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -77
- package/config.example.json +27 -41
- package/dist/backend.js +320 -19
- package/dist/backend.js.map +1 -1
- package/dist/benchmarks.d.ts +96 -19
- package/dist/benchmarks.js +168 -27
- package/dist/benchmarks.js.map +1 -1
- package/dist/candidates.d.ts +23 -10
- package/dist/candidates.js +80 -21
- package/dist/candidates.js.map +1 -1
- package/dist/catalog.d.ts +12 -0
- package/dist/catalog.js +47 -2
- package/dist/catalog.js.map +1 -1
- package/dist/circuit-breaker.d.ts +59 -33
- package/dist/circuit-breaker.js +177 -43
- package/dist/circuit-breaker.js.map +1 -1
- package/dist/cli.d.ts +11 -2
- package/dist/cli.js +265 -192
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +12 -1
- package/dist/config.js +34 -6
- package/dist/config.js.map +1 -1
- package/dist/control-authorization.d.ts +45 -0
- package/dist/control-authorization.js +208 -0
- package/dist/control-authorization.js.map +1 -0
- package/dist/dispatch.d.ts +12 -2
- package/dist/dispatch.js +19 -4
- package/dist/dispatch.js.map +1 -1
- package/dist/dynamic-pools.d.ts +20 -3
- package/dist/dynamic-pools.js +103 -38
- package/dist/dynamic-pools.js.map +1 -1
- package/dist/kernel/contracts.d.ts +253 -0
- package/dist/kernel/contracts.js +9 -0
- package/dist/kernel/contracts.js.map +1 -0
- package/dist/kernel/index.d.ts +3 -0
- package/dist/kernel/index.js +4 -0
- package/dist/kernel/index.js.map +1 -0
- package/dist/kernel/request-lifecycle.d.ts +54 -0
- package/dist/kernel/request-lifecycle.js +178 -0
- package/dist/kernel/request-lifecycle.js.map +1 -0
- package/dist/kernel/tier-snapshot.d.ts +48 -0
- package/dist/kernel/tier-snapshot.js +126 -0
- package/dist/kernel/tier-snapshot.js.map +1 -0
- package/dist/offload.js +108 -36
- package/dist/offload.js.map +1 -1
- package/dist/ping/cadence.d.ts +6 -1
- package/dist/ping/cadence.js +87 -9
- package/dist/ping/cadence.js.map +1 -1
- package/dist/ping/probe-cache.d.ts +5 -0
- package/dist/ping/probe-cache.js +56 -3
- package/dist/ping/probe-cache.js.map +1 -1
- package/dist/ping/runtime-telemetry.d.ts +10 -0
- package/dist/ping/runtime-telemetry.js +55 -5
- package/dist/ping/runtime-telemetry.js.map +1 -1
- package/dist/pool-health.d.ts +7 -1
- package/dist/pool-health.js +21 -9
- package/dist/pool-health.js.map +1 -1
- package/dist/registry.d.ts +1 -1
- package/dist/registry.js +3 -3
- package/dist/registry.js.map +1 -1
- package/dist/repair.js +8 -1
- package/dist/repair.js.map +1 -1
- package/dist/request-log.d.ts +6 -0
- package/dist/request-log.js +29 -0
- package/dist/request-log.js.map +1 -0
- package/dist/routes/admin.d.ts +2 -0
- package/dist/routes/admin.js +3 -3
- package/dist/routes/admin.js.map +1 -1
- package/dist/server.d.ts +11 -28
- package/dist/server.js +435 -123
- package/dist/server.js.map +1 -1
- package/dist/tier-data.d.ts +29 -3
- package/dist/tier-data.js +26 -12
- package/dist/tier-data.js.map +1 -1
- package/dist/validator.d.ts +4 -4
- package/dist/validator.js +15 -3
- package/dist/validator.js.map +1 -1
- package/docs/tier-data.json +24842 -8365
- package/package.json +4 -1
- package/scripts/install-skill.mjs +2 -2
- package/scripts/sync-tiers.mjs +326 -0
- package/scripts/tier-scoring.d.mts +20 -0
- package/scripts/tier-scoring.mjs +327 -0
- package/skills/llm-relay/SKILL.md +71 -34
package/README.md
CHANGED
|
@@ -73,7 +73,8 @@ llm-relay
|
|
|
73
73
|
|
|
74
74
|
### 4. Background Adaptive Health Monitoring & Persistent Caching
|
|
75
75
|
- **Adaptive Cadence Loop**: Background `PingLoop` dynamically adjusts probe frequency across 4 operational modes: `speed` (2s interval at startup/activity), `normal` (10s), `slow` (30s after 5m idle), and `forced` (4s).
|
|
76
|
-
- **
|
|
76
|
+
- **Selective probes**: The background loop probes only deployments present in materialized routing, with pool leaders first. A recent successful real request satisfies freshness; broken targets retry with exponential backoff instead of being hammered every tick. Explicit `llm-relay ping` remains a full-catalog diagnostic.
|
|
77
|
+
- **Persistent State**: Background probes, real-world proxy calls, dynamic catalogs, and local keys persist under `~/.llm-relay/` (`models-cache.json`, `probe-cache.json`, `runtime-telemetry.json`, `.env`). JSON caches use bounded write-behind and flush during graceful shutdown, keeping whole-file rewrites out of request/probe hot paths.
|
|
77
78
|
|
|
78
79
|
### 5. Document (PDF/Office) Attachments on Non-Anthropic Backends
|
|
79
80
|
- Anthropic `document` content blocks are converted to markdown **before** the request reaches an
|
|
@@ -88,8 +89,8 @@ llm-relay
|
|
|
88
89
|
the prompt.
|
|
89
90
|
|
|
90
91
|
### 6. Programmatic Telemetry & Quota Access for Claude
|
|
91
|
-
- **
|
|
92
|
-
- **
|
|
92
|
+
- **Tokenless status endpoints**: `GET /models`, `GET /telemetry`, `GET /offload`, and `GET /dispatch` are side-effect-free status reads.
|
|
93
|
+
- **Capability-protected control endpoints**: `POST /offload`, `POST /dispatch`, `GET /ping`, `GET /registry`, `GET /health`, and `GET /candidates`. The CLI automatically carries the per-install 256-bit capability stored in `~/.llm-relay/control-token`; it is never forwarded to providers. ⚠ **Loopback is not authorization** — every request also requires a `Host` exactly matching the bound listener authority, a present `Origin` must match its exact scheme/host/effective port, and `Origin: null` is rejected. Mutating requests additionally require `content-type: application/json`.
|
|
93
94
|
- **CLI Commands**: `llm-relay telemetry` outputs live telemetry metrics; `llm-relay models` lists live model catalogs with SWE-bench & quality scores; `llm-relay ping` performs live health & latency probes.
|
|
94
95
|
- **Response Headers**: Proxy responses include `x-llm-relay-quota-percent`, `x-llm-relay-stability-score`, and `x-llm-relay-target`.
|
|
95
96
|
|
|
@@ -99,21 +100,21 @@ llm-relay
|
|
|
99
100
|
|
|
100
101
|
| Command | Description |
|
|
101
102
|
| :--- | :--- |
|
|
102
|
-
| `llm-relay` | Start
|
|
103
|
-
| `llm-relay onboard` |
|
|
104
|
-
| `llm-relay setup
|
|
105
|
-
| `llm-relay
|
|
106
|
-
| `llm-relay
|
|
107
|
-
| `llm-relay pools [
|
|
108
|
-
| `llm-relay
|
|
109
|
-
| `llm-relay
|
|
110
|
-
| `llm-relay
|
|
111
|
-
| `llm-relay
|
|
112
|
-
| `llm-relay
|
|
113
|
-
| `llm-relay
|
|
114
|
-
| `llm-relay offload
|
|
115
|
-
| `llm-relay candidates [-p <name>]` |
|
|
116
|
-
| `llm-relay dispatch [lane] [
|
|
103
|
+
| `llm-relay` | Start proxy |
|
|
104
|
+
| `llm-relay onboard` | Set up provider keys |
|
|
105
|
+
| `llm-relay setup [target]` | `target`: `claude-cli` | `claude-desktop` |
|
|
106
|
+
| `llm-relay keys | check-keys` | Check provider keys |
|
|
107
|
+
| `llm-relay pools [--probe]` | List pool members; `--probe` tests each |
|
|
108
|
+
| `llm-relay pools <action> <name> [<spec>...]` | `action`: `set` | `add` | `remove` | `delete` |
|
|
109
|
+
| `llm-relay routing <action> ...` | `action`: `show` | `get` | `default` | `tier` | `subagent` | `sort` | `benchmark` | `set` | `unset` |
|
|
110
|
+
| `llm-relay config <action> [<path>] [<value>]` | `action`: `show` | `get` | `set` | `unset` |
|
|
111
|
+
| `llm-relay telemetry` | Print telemetry/quota JSON |
|
|
112
|
+
| `llm-relay models [-p <name>] [-r]` | List provider models |
|
|
113
|
+
| `llm-relay ping [-p <name>]` | Probe providers |
|
|
114
|
+
| `llm-relay offload [status]` | Show aggregate offload state |
|
|
115
|
+
| `llm-relay offload <harness> <on\|off> [--scope <scope>]` | Set one harness's rule |
|
|
116
|
+
| `llm-relay candidates [-p <name>]` | Show offload target data |
|
|
117
|
+
| `llm-relay dispatch [lane] [options]` | Choose next dispatch lane |
|
|
117
118
|
|
|
118
119
|
---
|
|
119
120
|
|
|
@@ -124,25 +125,25 @@ and validate the complete result before writing it. Restart a running proxy afte
|
|
|
124
125
|
|
|
125
126
|
```bash
|
|
126
127
|
# Static pool: members are tried/ranked according to the normal pool rules.
|
|
127
|
-
llm-relay pools set
|
|
128
|
-
llm-relay pools add
|
|
129
|
-
llm-relay pools remove
|
|
130
|
-
llm-relay pools delete
|
|
128
|
+
llm-relay pools set medium nim/z-ai/glm-5.2 openrouter/openai/gpt-5.2-codex
|
|
129
|
+
llm-relay pools add medium gemini/gemini-2.5-flash
|
|
130
|
+
llm-relay pools remove medium gemini/gemini-2.5-flash
|
|
131
|
+
llm-relay pools delete medium
|
|
131
132
|
|
|
132
|
-
# Dynamic pool:
|
|
133
|
-
llm-relay pools set
|
|
133
|
+
# Dynamic effort pool: an empty configured prefix, then evidence-ranked free models.
|
|
134
|
+
llm-relay pools set medium --free --effort medium
|
|
134
135
|
|
|
135
136
|
# Main fallback, Claude tier maps, subagent destinations, and ranking.
|
|
136
137
|
llm-relay routing default nim/z-ai/glm-5.2 openrouter/openai/gpt-5.2-codex
|
|
137
|
-
llm-relay routing tier sonnet pool/
|
|
138
|
-
llm-relay routing subagent default pool/
|
|
138
|
+
llm-relay routing tier sonnet pool/high
|
|
139
|
+
llm-relay routing subagent default pool/medium
|
|
139
140
|
llm-relay routing sort off
|
|
140
141
|
llm-relay routing tier opus --clear
|
|
141
142
|
|
|
142
143
|
# Inspect or change any less-common routing field using a JSON value.
|
|
143
144
|
llm-relay routing show
|
|
144
145
|
llm-relay config get routing.pools
|
|
145
|
-
llm-relay config set routing.ladder '[{"id":"
|
|
146
|
+
llm-relay config set routing.ladder '[{"id":"medium","kind":"relay","spec":"pool/medium"}]'
|
|
146
147
|
llm-relay config unset routing.ladder
|
|
147
148
|
```
|
|
148
149
|
|
|
@@ -315,17 +316,18 @@ a `routing` block that maps each request's `model` to one provider + backend mod
|
|
|
315
316
|
"gemini": { "base": "https://generativelanguage.googleapis.com/v1beta/openai", "kind": "openai", "authEnv": "GEMINI_API_KEY" }
|
|
316
317
|
},
|
|
317
318
|
"routing": {
|
|
318
|
-
|
|
319
|
-
// (it only sorts when there is more than one) AND failover. A lone pinned model disables both.
|
|
320
|
-
"default": ["nim/z-ai/glm-5.2", "nim/deepseek-ai/deepseek-v4-pro"],
|
|
319
|
+
"default": "pool/medium",
|
|
321
320
|
"tiers": { // Claude tier (substring match) → provider/model
|
|
322
|
-
"opus":
|
|
323
|
-
"
|
|
324
|
-
"
|
|
325
|
-
"
|
|
321
|
+
"opus": "pool/xhigh",
|
|
322
|
+
"fable": "pool/xhigh",
|
|
323
|
+
"sonnet": "pool/high",
|
|
324
|
+
"haiku": "pool/medium"
|
|
326
325
|
},
|
|
327
326
|
"pools": { // addressable as model "pool/<name>"
|
|
328
|
-
"
|
|
327
|
+
"low": { "preferred": [], "include": "free", "effort": "low" },
|
|
328
|
+
"medium": { "preferred": [], "include": "free", "effort": "medium" },
|
|
329
|
+
"high": { "preferred": [], "include": "free", "effort": "high" },
|
|
330
|
+
"xhigh": { "preferred": [], "include": "free", "effort": "xhigh" }
|
|
329
331
|
}
|
|
330
332
|
},
|
|
331
333
|
"mode": "repair", // detect | repair (strict accepted, aliases detect)
|
|
@@ -340,7 +342,7 @@ a `routing` block that maps each request's `model` to one provider + backend mod
|
|
|
340
342
|
|
|
341
343
|
**Routing (lifted from free-claude-code's proven scheme — split on the first `/` only):**
|
|
342
344
|
1. **Pool** — a request `model` of `pool/<name>` expands to that pool's whole candidate list,
|
|
343
|
-
which is then
|
|
345
|
+
which is then fitness-ranked and failed over. Use this to ask for *the best available*
|
|
344
346
|
model instead of naming one. An unknown pool is a **400, never a silent fallback** to the
|
|
345
347
|
default — a typo must not quietly succeed against a different model.
|
|
346
348
|
2. **Namespaced** — a request `model` of `provider/rest` where `provider` is a configured
|
|
@@ -360,18 +362,28 @@ reserved provider name; configuring a provider called `pool` fails at load.
|
|
|
360
362
|
Pools can be static arrays, or automatic free-model pools:
|
|
361
363
|
|
|
362
364
|
```jsonc
|
|
363
|
-
"
|
|
364
|
-
"preferred": [
|
|
365
|
-
"include": "free"
|
|
365
|
+
"medium": {
|
|
366
|
+
"preferred": [],
|
|
367
|
+
"include": "free",
|
|
368
|
+
"effort": "medium"
|
|
366
369
|
}
|
|
367
370
|
```
|
|
368
371
|
|
|
369
|
-
The
|
|
372
|
+
The configured prefix remains first in exactly the written order. The relay then appends every
|
|
370
373
|
model discovered from a `tierType: "free"` provider (excluding a model when its catalog publishes
|
|
371
374
|
a positive price), plus zero-priced or explicitly free-labelled models from `tierType: "mixed"`
|
|
372
|
-
providers
|
|
375
|
+
providers. `effort` may be `low`, `medium`, `high`, or `xhigh`. These are cumulative raw-capability
|
|
376
|
+
floors (50/60/70/80), not ceilings. Admission compares a whole-point capability score; an existing
|
|
377
|
+
member remains until it falls two points below its floor, preventing refresh noise from flapping the
|
|
378
|
+
pool. Automatic membership also requires an exact SKU match and at least three published capability
|
|
379
|
+
or task-fit signals; confidence, stability, and metadata affect ordering, not eligibility.
|
|
380
|
+
A strong free model remains eligible for `low`, while higher effort narrows upward
|
|
381
|
+
(`xhigh ⊆ high ⊆ medium ⊆ low`). Exact SKUs known not to support tools are excluded.
|
|
373
382
|
Catalog refreshes re-materialize the pool automatically; adding new free models never requires a
|
|
374
|
-
config edit.
|
|
383
|
+
config edit. Materialization builds and ranks one common discovered roster, then filters that
|
|
384
|
+
snapshot into all effort pools; the result is reused for a 30-second ranking epoch and invalidated
|
|
385
|
+
immediately by a catalog revision. Legacy array pools keep their existing whole-array
|
|
386
|
+
`benchmarkSort` behaviour, with their ranking likewise reused within a short epoch.
|
|
375
387
|
|
|
376
388
|
**What failover actually does** (both `/v1/messages` and `/v1/chat/completions`):
|
|
377
389
|
|
|
@@ -453,7 +465,10 @@ to marked subagents only (the current behavior) or to the whole conversation:
|
|
|
453
465
|
```jsonc
|
|
454
466
|
"routing": {
|
|
455
467
|
"tiers": { "opus": "anthropic", "sonnet": "anthropic", "haiku": "anthropic", "fable": "anthropic" },
|
|
456
|
-
"subagents": {
|
|
468
|
+
"subagents": {
|
|
469
|
+
"opus": "pool/xhigh", "fable": "pool/xhigh",
|
|
470
|
+
"sonnet": "pool/high", "haiku": "pool/medium", "default": "pool/medium"
|
|
471
|
+
},
|
|
457
472
|
"offload": {
|
|
458
473
|
"claude": { "enabled": true, "scope": "subagents" },
|
|
459
474
|
"codex": { "enabled": false, "scope": "all" }
|
|
@@ -466,17 +481,18 @@ to marked subagents only (the current behavior) or to the whole conversation:
|
|
|
466
481
|
Claude or Codex quota is exhausted. Rules may use any future client name; an explicit `default`
|
|
467
482
|
rule is the opt-in catch-all for otherwise unnamed front doors. All rules are off by default.
|
|
468
483
|
|
|
469
|
-
The CLI changes one
|
|
484
|
+
The CLI changes one harness without restarting the proxy:
|
|
470
485
|
|
|
471
486
|
```bash
|
|
472
487
|
llm-relay offload status
|
|
473
|
-
llm-relay offload
|
|
474
|
-
llm-relay offload codex on --scope all
|
|
475
|
-
llm-relay offload claude off
|
|
488
|
+
llm-relay offload <harness> <on|off> [--scope <scope>]
|
|
476
489
|
```
|
|
477
490
|
|
|
478
|
-
|
|
479
|
-
|
|
491
|
+
`<harness>` is `claude`, `codex`, or another configured client. `<scope>` is `subagents` or
|
|
492
|
+
`all` (default: `subagents`).
|
|
493
|
+
|
|
494
|
+
The legacy boolean form remains supported in config files as a global subagents-only rule
|
|
495
|
+
(`"offload": false`). The CLI requires a harness name for changes. `GET /offload?client=claude` reads one rule;
|
|
480
496
|
`POST /offload` accepts `{"client":"claude","enabled":true,"scope":"all"}`. Changes are
|
|
481
497
|
persisted and take effect on the next request.
|
|
482
498
|
|
|
@@ -533,7 +549,7 @@ description = "Read-only coding child routed through llm-relay."
|
|
|
533
549
|
developer_instructions = "Work read-only. Return a concise result to the parent and do not modify files."
|
|
534
550
|
|
|
535
551
|
model_provider = "llm-relay"
|
|
536
|
-
model = "pool/
|
|
552
|
+
model = "pool/medium"
|
|
537
553
|
model_reasoning_effort = "medium"
|
|
538
554
|
```
|
|
539
555
|
|
|
@@ -546,12 +562,12 @@ description = "General-purpose read-only child routed through llm-relay."
|
|
|
546
562
|
developer_instructions = "Work read-only. Return a concise result to the parent and do not modify files."
|
|
547
563
|
|
|
548
564
|
model_provider = "llm-relay"
|
|
549
|
-
model = "pool/
|
|
565
|
+
model = "pool/medium"
|
|
550
566
|
model_reasoning_effort = "medium"
|
|
551
567
|
```
|
|
552
568
|
|
|
553
569
|
With that override, a normal “use a subagent” request keeps the parent native while the generic child
|
|
554
|
-
goes through `pool/
|
|
570
|
+
goes through `pool/medium`; named agents can still select a different pool explicitly.
|
|
555
571
|
|
|
556
572
|
Run Codex normally, without the `llm-relay` profile. Ask the parent to use exactly one subagent of
|
|
557
573
|
type `relay_coding`; Codex keeps the parent on its normal provider and starts the child through the
|
|
@@ -560,11 +576,11 @@ relay. The relay pool then chooses the configured provider and can fail over nor
|
|
|
560
576
|
Enable only Codex child offload in `~/.llm-relay/config.json`:
|
|
561
577
|
|
|
562
578
|
```bash
|
|
563
|
-
llm-relay offload
|
|
579
|
+
llm-relay offload <harness> on --scope <scope>
|
|
564
580
|
```
|
|
565
581
|
|
|
566
|
-
|
|
567
|
-
|
|
582
|
+
For Codex, use `harness=codex` with `scope=subagents`; use `scope=all` to include the parent
|
|
583
|
+
conversation. Claude's rule is unaffected.
|
|
568
584
|
|
|
569
585
|
The `llm-relay` profile remains available as an explicit all-relay mode, but it routes the parent
|
|
570
586
|
through the relay too and is not the split setup described above. The automatic
|
|
@@ -576,18 +592,18 @@ This applies to local Codex clients that can reach `127.0.0.1`. Hosted ChatGPT/C
|
|
|
576
592
|
reach a loopback relay, and the relay cannot spend a ChatGPT subscription on behalf of an upstream
|
|
577
593
|
request; those remain separate CLI/client-bound dispatch lanes.
|
|
578
594
|
|
|
579
|
-
Whole-task CLI dispatch can likewise vary by tier with `routing.ladders.{
|
|
580
|
-
Use `llm-relay dispatch --tier
|
|
581
|
-
`subagents.default` is selected (normally `
|
|
595
|
+
Whole-task CLI dispatch can likewise vary by tier with `routing.ladders.{low,medium,high,xhigh}`.
|
|
596
|
+
Use `llm-relay dispatch --tier high -t "..."`; without `--tier`, the ladder matching
|
|
597
|
+
`subagents.default` is selected (normally `medium`). The legacy single `routing.ladder` remains
|
|
582
598
|
supported for configurations that do not need tier-specific CLI models.
|
|
583
599
|
|
|
584
600
|
### Choosing where to offload (`llm-relay candidates`)
|
|
585
601
|
|
|
586
602
|
```
|
|
587
|
-
target pools / tiers
|
|
588
|
-
|
|
589
|
-
nim/
|
|
590
|
-
nim/
|
|
603
|
+
target pools / tiers fit raw cap agentic coding BFCL arena $/Mout verdict p95 quota breaker ctx
|
|
604
|
+
ollama-cloud/kimi-k3 low,medium,high... 79.9 96.6 87.3/4 50.1 76.2 - - - Pending - - closed 1049k~
|
|
605
|
+
nim/z-ai/glm-5.2 low,medium,high... 71.8 83.3 76.6/4 43.1 68.8 - - $2.402~ Pending - - closed 1049k~
|
|
606
|
+
nim/deepseek-ai/deepseek-v4-pro low,medium,@haiku 64.9 67.4 67.4/5 36.4 59.4 - 1457 $0.87~ Pending - - closed 1049k~
|
|
591
607
|
```
|
|
592
608
|
|
|
593
609
|
Every offload target with its dimensions side by side: capability from each leaderboard separately,
|
|
@@ -609,15 +625,34 @@ They **disagree** — the agentic index puts deepseek above kimi while the codin
|
|
|
609
625
|
above deepseek — which is exactly why each keeps its own column, and why a blank cell means *not
|
|
610
626
|
measured*, never *bad*.
|
|
611
627
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
628
|
+
The raw dimensions remain separate — capability, latency and remaining quota answer different
|
|
629
|
+
questions. Pool ordering uses three explicit derived scores:
|
|
630
|
+
|
|
631
|
+
- `raw` is fixed at 40% agentic/tool use, 35% coding, and 25% general reasoning. Each source is
|
|
632
|
+
mapped through persisted raw-value calibration anchors, so an unrelated leaderboard addition
|
|
633
|
+
cannot silently move every model. If an entire dimension is missing, it is estimated by ridge
|
|
634
|
+
regression from models with overlapping dimensions rather than disappearing from the denominator.
|
|
635
|
+
Artificial Analysis Agentic and BFCL Overall feed agentic capability; AA Coding and Aider pass
|
|
636
|
+
rate feed coding; AA Intelligence and LMArena feed general reasoning.
|
|
637
|
+
- Design Arena's differently covered specialist categories, BFCL irrelevance, and Aider formatting
|
|
638
|
+
compliance are task-fit signals, not raw capability. This prevents a model measured on a favorable
|
|
639
|
+
specialized subset from gaining an effort tier.
|
|
640
|
+
- `cap` is `raw` shrunk toward neutral by capability evidence confidence. Direct dimension coverage,
|
|
641
|
+
published capability signals, and imputation quality determine confidence; fuzzy model-name
|
|
642
|
+
matches get half confidence. It affects ordering, never effort eligibility. `/4c5p` means four
|
|
643
|
+
direct capability signals and five total publications; `neut` means no capability evidence.
|
|
644
|
+
Operational telemetry never substitutes for capability.
|
|
645
|
+
- `fit` is 75% `cap`, 20% deployment operations, and 5% task-fit metadata. Operations combine
|
|
646
|
+
synthetic probe stability with success/speed/recency from at least five real calls. Metadata
|
|
647
|
+
uses the separate specialist/behavior score, exact-SKU tool support, and provider/reference
|
|
648
|
+
context and output limits. Missing inputs are neutral (50), not zero. A known tool-incompatible
|
|
649
|
+
SKU is excluded from automatic effort pools; breaker-open and credential-faulted deployments are
|
|
650
|
+
demoted after scoring.
|
|
651
|
+
|
|
652
|
+
Only coarse `raw` capability plus the exact-match/three-publication gate decides whether a model
|
|
653
|
+
clears an effort floor. The generated snapshot persists the two-point exit band. `fit` decides the
|
|
654
|
+
order among eligible deployments. The JSON view exposes dimensions, direct/imputed coverage, task
|
|
655
|
+
fit, and confidence.
|
|
621
656
|
|
|
622
657
|
**Limits and prices are per-(provider, model), and labelled.** The same model id on two providers is
|
|
623
658
|
two deployments — different context ceilings, different output caps, and possibly free on one and
|
|
@@ -644,7 +679,7 @@ an anthropic provider has no fixed model id to reshape on, so it needs an explic
|
|
|
644
679
|
**Prefer the pool form — do not pin one reshaper model:**
|
|
645
680
|
|
|
646
681
|
```jsonc
|
|
647
|
-
"reshaper": { "pool": "
|
|
682
|
+
"reshaper": { "pool": "medium" } // ranked candidates, tried in order
|
|
648
683
|
```
|
|
649
684
|
|
|
650
685
|
A pinned `{ "base": …, "model": … }` still works, but if the provider stops serving that exact id
|
|
@@ -684,9 +719,10 @@ llm-relay models --provider nim # one provider
|
|
|
684
719
|
llm-relay models --provider nim --refresh # force a re-fetch
|
|
685
720
|
```
|
|
686
721
|
|
|
687
|
-
On startup the proxy warms
|
|
688
|
-
|
|
689
|
-
|
|
722
|
+
On startup the proxy warms providers referenced by routing plus free/mixed providers that can
|
|
723
|
+
contribute to dynamic pools, then **warns about any routing target its provider doesn't serve**.
|
|
724
|
+
Unrelated subscription catalogs stay lazy until first use, while a stale/typo'd routed model is
|
|
725
|
+
still caught at boot rather than silently failing on its first request.
|
|
690
726
|
|
|
691
727
|
> Provider notes: **Groq** returns `403 "check your network settings"` from some
|
|
692
728
|
> IPs/regions (a network-side block, not a key issue) — it works once your network
|
|
@@ -720,7 +756,7 @@ no mode switch. So the tier map stays the default, and dispatcher-style usage is
|
|
|
720
756
|
"send namespaced ids + read `/registry`".
|
|
721
757
|
|
|
722
758
|
OpenAI-native clients can point their base URL at `http://127.0.0.1:8791/v1` and use a
|
|
723
|
-
namespaced model such as `anthropic/claude-sonnet-4-20250514` or `pool/
|
|
759
|
+
namespaced model such as `anthropic/claude-sonnet-4-20250514` or `pool/medium`. Codex uses
|
|
724
760
|
`/v1/responses`; other IDEs commonly use `/v1/chat/completions`. Configure the Anthropic
|
|
725
761
|
provider with `kind: "anthropic"` and `authEnv: "ANTHROPIC_API_KEY"` when the relay should
|
|
726
762
|
use its own key, or omit `authEnv` for an intentional caller-credential passthrough.
|
|
@@ -790,7 +826,7 @@ provider and point every tier at it:
|
|
|
790
826
|
"routing": {
|
|
791
827
|
"default": "anthropic",
|
|
792
828
|
"tiers": { "opus": "anthropic", "sonnet": "anthropic", "haiku": "anthropic", "fable": "anthropic" },
|
|
793
|
-
"pools": { "
|
|
829
|
+
"pools": { "medium": ["nim/z-ai/glm-5.2", "nim/deepseek-ai/deepseek-v4-pro"] }
|
|
794
830
|
}
|
|
795
831
|
```
|
|
796
832
|
|
package/config.example.json
CHANGED
|
@@ -38,55 +38,41 @@
|
|
|
38
38
|
}
|
|
39
39
|
},
|
|
40
40
|
"routing": {
|
|
41
|
-
"default":
|
|
42
|
-
"groq/llama-3.3-70b-versatile",
|
|
43
|
-
"openrouter/openrouter/free",
|
|
44
|
-
"nim/z-ai/glm-5.2",
|
|
45
|
-
"gemini/gemini-2.5-flash",
|
|
46
|
-
"ollama/qwen2.5-coder:32b"
|
|
47
|
-
],
|
|
41
|
+
"default": "pool/medium",
|
|
48
42
|
"tiers": {
|
|
49
|
-
"opus":
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
"sonnet": [
|
|
54
|
-
"groq/llama-3.3-70b-versatile",
|
|
55
|
-
"nim/z-ai/glm-5.2",
|
|
56
|
-
"gemini/gemini-2.5-flash",
|
|
57
|
-
"ollama/qwen2.5-coder:32b"
|
|
58
|
-
],
|
|
59
|
-
"haiku": [
|
|
60
|
-
"groq/llama-3.1-8b-instant",
|
|
61
|
-
"nim/meta/llama-3.1-8b-instruct",
|
|
62
|
-
"openrouter/openrouter/free",
|
|
63
|
-
"ollama/deepseek-r1:14b"
|
|
64
|
-
],
|
|
65
|
-
"fable": [
|
|
66
|
-
"groq/llama-3.1-8b-instant",
|
|
67
|
-
"ollama/deepseek-r1:14b"
|
|
68
|
-
]
|
|
43
|
+
"opus": "pool/xhigh",
|
|
44
|
+
"fable": "pool/xhigh",
|
|
45
|
+
"sonnet": "pool/high",
|
|
46
|
+
"haiku": "pool/medium"
|
|
69
47
|
},
|
|
70
48
|
"pools": {
|
|
71
|
-
"
|
|
72
|
-
"preferred": [
|
|
73
|
-
"include": "free"
|
|
49
|
+
"low": {
|
|
50
|
+
"preferred": [],
|
|
51
|
+
"include": "free",
|
|
52
|
+
"effort": "low"
|
|
74
53
|
},
|
|
75
|
-
"
|
|
76
|
-
"preferred": [
|
|
77
|
-
"include": "free"
|
|
54
|
+
"medium": {
|
|
55
|
+
"preferred": [],
|
|
56
|
+
"include": "free",
|
|
57
|
+
"effort": "medium"
|
|
78
58
|
},
|
|
79
|
-
"
|
|
80
|
-
"preferred": [
|
|
81
|
-
"include": "free"
|
|
59
|
+
"high": {
|
|
60
|
+
"preferred": [],
|
|
61
|
+
"include": "free",
|
|
62
|
+
"effort": "high"
|
|
63
|
+
},
|
|
64
|
+
"xhigh": {
|
|
65
|
+
"preferred": [],
|
|
66
|
+
"include": "free",
|
|
67
|
+
"effort": "xhigh"
|
|
82
68
|
}
|
|
83
69
|
},
|
|
84
70
|
"subagents": {
|
|
85
|
-
"opus": "pool/
|
|
86
|
-
"sonnet": "pool/
|
|
87
|
-
"haiku": "pool/
|
|
88
|
-
"fable": "pool/
|
|
89
|
-
"default": "pool/
|
|
71
|
+
"opus": "pool/xhigh",
|
|
72
|
+
"sonnet": "pool/high",
|
|
73
|
+
"haiku": "pool/medium",
|
|
74
|
+
"fable": "pool/xhigh",
|
|
75
|
+
"default": "pool/medium"
|
|
90
76
|
},
|
|
91
77
|
"offload": {
|
|
92
78
|
"claude": { "enabled": false, "scope": "subagents" },
|