llm-relay 0.70.0 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accounting-store-schema.d.ts +8 -1
- package/dist/accounting-store-schema.js +3 -1
- package/dist/accounting-store-schema.js.map +1 -1
- package/dist/accounting-store.d.ts +16 -12
- package/dist/accounting-store.js +1 -1
- package/dist/accounting-store.js.map +1 -1
- package/dist/accounting.d.ts +2 -3
- package/dist/accounting.js +2 -1
- package/dist/accounting.js.map +1 -1
- package/dist/candidate-runner.d.ts +36 -2
- package/dist/candidate-runner.js +56 -7
- package/dist/candidate-runner.js.map +1 -1
- package/dist/cli.d.ts +22 -0
- package/dist/cli.js +92 -19
- package/dist/cli.js.map +1 -1
- package/dist/config-types.d.ts +270 -26
- package/dist/config-types.js.map +1 -1
- package/dist/config.d.ts +2 -554
- package/dist/config.js +2 -2
- package/dist/config.js.map +1 -1
- package/dist/dashboard-routes.d.ts +1 -10
- package/dist/dashboard-routes.js +2 -1
- package/dist/dashboard-routes.js.map +1 -1
- package/dist/delegate-gate/test-assertions.js +27 -6
- package/dist/delegate-gate/test-assertions.js.map +1 -1
- package/dist/dispatch-lane-stats.d.ts +133 -0
- package/dist/dispatch-lane-stats.js +274 -0
- package/dist/dispatch-lane-stats.js.map +1 -0
- package/dist/dispatch.d.ts +49 -1
- package/dist/dispatch.js +29 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/hedge-trigger.d.ts +14 -26
- package/dist/hedge-trigger.js.map +1 -1
- package/dist/mcp/lane-runner.d.ts +20 -1
- package/dist/mcp/lane-runner.js +11 -1
- package/dist/mcp/lane-runner.js.map +1 -1
- package/dist/mcp/server.d.ts +19 -0
- package/dist/mcp/server.js +47 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/openai-request.d.ts +1 -4
- package/dist/openai-request.js.map +1 -1
- package/dist/quota-observation.d.ts +2 -3
- package/dist/quota-observation.js.map +1 -1
- package/dist/routes/admin.d.ts +8 -0
- package/dist/routes/admin.js +97 -18
- package/dist/routes/admin.js.map +1 -1
- package/dist/routes/messages.js +8 -4
- package/dist/routes/messages.js.map +1 -1
- package/dist/routes/openai-front.js +8 -4
- package/dist/routes/openai-front.js.map +1 -1
- package/dist/server.js +6 -2
- package/dist/server.js.map +1 -1
- package/dist/storage/json-store.d.ts +0 -34
- package/dist/storage/json-store.js +0 -44
- package/dist/storage/json-store.js.map +1 -1
- package/dist/stream-pipeline.d.ts +11 -0
- package/dist/stream-pipeline.js +7 -1
- package/dist/stream-pipeline.js.map +1 -1
- package/package.json +1 -1
package/dist/config-types.d.ts
CHANGED
|
@@ -6,6 +6,17 @@ export type ProviderTierType = "free" | "mixed" | "subscription";
|
|
|
6
6
|
/**
|
|
7
7
|
* The effort bands, WEAKEST FIRST — one declaration, and the type is derived from it rather than
|
|
8
8
|
* the other way round.
|
|
9
|
+
*
|
|
10
|
+
* ⚠ Order is load-bearing: `dynamic-pools.ts` derives the degrade tail from this array's index, so
|
|
11
|
+
* a band inserted out of order silently reorders which weaker members an exhausted pool falls back
|
|
12
|
+
* to.
|
|
13
|
+
*
|
|
14
|
+
* ⚠ Why an ordered tuple and not four hand-lists: the member list stood restated in four places
|
|
15
|
+
* across three modules, and only ONE of them was exhaustiveness-checked. `new Set<EffortLevel>([…])`
|
|
16
|
+
* and `EffortLevel[]` both accept a SUBSET, and `cli.ts` validated against a bare `string[]` with
|
|
17
|
+
* no link to the type at all — so a fifth band would have compiled everywhere and silently failed
|
|
18
|
+
* in three of the four. `Record<EffortLevel, …>` (`EFFORT_FLOORS` in `benchmarks.ts`) is the shape
|
|
19
|
+
* that already caught it, and is why that one is left as it is.
|
|
9
20
|
*/
|
|
10
21
|
export declare const EFFORT_LEVELS: readonly ["low", "medium", "high", "xhigh"];
|
|
11
22
|
/** Requested reasoning/capability band for an automatically discovered pool. */
|
|
@@ -21,7 +32,10 @@ export interface OffloadRule {
|
|
|
21
32
|
scope: OffloadScope;
|
|
22
33
|
/**
|
|
23
34
|
* Refuse — loudly — rather than let this client's rerouted traffic reach a deployment that
|
|
24
|
-
* is not assessed `free
|
|
35
|
+
* is not assessed `free` (see `assessCost`; `unknown` counts as paid on purpose). Applies to
|
|
36
|
+
* everything `subagentSpec` reroutes for this client, INCLUDING a per-call `@relay:` directive:
|
|
37
|
+
* the flag is the owner's standing "this lane never spends money", and a subagent prompt must
|
|
38
|
+
* not be able to out-rank it. The Anthropic passthrough is primary quota and is never free.
|
|
25
39
|
*/
|
|
26
40
|
freeOnly?: boolean;
|
|
27
41
|
}
|
|
@@ -49,15 +63,37 @@ export interface ReshaperConfig {
|
|
|
49
63
|
export type CredentialMode = "passthrough" | "contained";
|
|
50
64
|
/**
|
|
51
65
|
* What shape a provider's own validator demands of the tool-call ids the relay puts on the wire.
|
|
66
|
+
*
|
|
67
|
+
* - `"preserve"` — forward the caller's ids verbatim. The default everywhere, and what the relay
|
|
68
|
+
* did for its whole life before 2026-08-23: an id is linkage, so not touching it is the safest
|
|
69
|
+
* thing a translation can do.
|
|
70
|
+
* - `"strict9"` — rewrite every outbound `tool_calls[].id` / `tool_call_id` to mistral's stated
|
|
71
|
+
* `^[a-zA-Z0-9]{9}$` shape. Mistral's `mistral-common` validator enforces it on BOTH halves of
|
|
72
|
+
* the pair (and, from v13, linkage and uniqueness on top), so an Anthropic `toolu_01…` id is a
|
|
73
|
+
* hard 400 there — see `src/openai-request.ts`.
|
|
52
74
|
*/
|
|
53
75
|
export type ToolCallIdMode = "preserve" | "strict9";
|
|
54
76
|
/**
|
|
55
77
|
* Whether a replayed assistant tool call must carry gemini 3.x's `thought_signature` field.
|
|
78
|
+
*
|
|
79
|
+
* - `"none"` — emit nothing. The default everywhere, and byte for byte what this relay put on the
|
|
80
|
+
* wire before 2026-08-23.
|
|
81
|
+
* - `"sentinel"` — stamp Google's own documented opt-out token,
|
|
82
|
+
* `skip_thought_signature_validator`, at `tool_calls[].extra_content.google.thought_signature`
|
|
83
|
+
* on every replayed tool call. See `src/openai-request.ts` for the 400 that states the rule and
|
|
84
|
+
* for why echoing a REAL signature is not an option here.
|
|
56
85
|
*/
|
|
57
86
|
export type ThoughtSignatureMode = "none" | "sentinel";
|
|
58
87
|
/**
|
|
59
88
|
* Per-provider WIRE-SHAPE quirks — things a specific host's request validator demands that the
|
|
60
|
-
* protocol itself does not.
|
|
89
|
+
* protocol itself does not. Deliberately not routing configuration and deliberately not a
|
|
90
|
+
* per-provider switch in `src/`: a labelled provider fact may live in code only while config can
|
|
91
|
+
* override it (see the "Provider knowledge is data" invariant), which is exactly the shape here —
|
|
92
|
+
* a base-host default that any explicit value beats in both directions.
|
|
93
|
+
*
|
|
94
|
+
* Two keys today, one per vendor rule the relay has first-party evidence for. An unknown key or
|
|
95
|
+
* value is a HARD load error naming it: the `configured-limits` precedent — an ignored typo
|
|
96
|
+
* silently no-ops while reading like a declaration that took effect.
|
|
61
97
|
*/
|
|
62
98
|
export interface ProviderCompatConfig {
|
|
63
99
|
/** Absent ⇒ resolved from the base host by `resolveToolCallIdMode`. */
|
|
@@ -126,7 +162,17 @@ export interface ProviderConfig {
|
|
|
126
162
|
/** Explicit multi-key declarations. Presence of an empty array means an empty fleet. */
|
|
127
163
|
credentials?: ProviderCredentialConfig[];
|
|
128
164
|
/**
|
|
129
|
-
* What to do with the caller's own credential when this provider declares no `authEnv
|
|
165
|
+
* What to do with the caller's own credential when this provider declares no `authEnv`:
|
|
166
|
+
* - `"passthrough"` — forward it. This is the declaration that makes an Anthropic
|
|
167
|
+
* passthrough deliberate rather than a side effect of leaving `authEnv` out.
|
|
168
|
+
* - `"contained"` — strip it and send none. The right answer for a keyless backend that
|
|
169
|
+
* is not the caller's own vendor: a local daemon, a second relay, someone else's
|
|
170
|
+
* Anthropic-format endpoint.
|
|
171
|
+
*
|
|
172
|
+
* Omitting it still forwards, so existing configs keep working, but config load warns for an
|
|
173
|
+
* `anthropic`-kind provider. "This backend needs no key of its own" and "send this host the
|
|
174
|
+
* user's subscription credential" are different intentions, and only the first one should be
|
|
175
|
+
* inferable from an omission. Illegal together with `authEnv` — that pair states both at once.
|
|
130
176
|
*/
|
|
131
177
|
credentialMode?: CredentialMode;
|
|
132
178
|
/** Maximum concurrent requests for this provider credential domain; null/omitted is unlimited. */
|
|
@@ -136,17 +182,27 @@ export interface ProviderConfig {
|
|
|
136
182
|
/** Backend request deadline in ms. Default 120000. */
|
|
137
183
|
timeoutMs: number;
|
|
138
184
|
/**
|
|
139
|
-
* Inter-byte stall watchdog for STREAMED responses, in ms.
|
|
185
|
+
* Inter-byte stall watchdog for STREAMED responses, in ms. Once a stream is being served, the
|
|
186
|
+
* total `timeoutMs` deadline disarms — one flat deadline kills a healthy long generation at
|
|
187
|
+
* minute two while letting a dead stream hang until the same minute two — and this watchdog
|
|
188
|
+
* aborts only when NO byte arrives for this long. 0 keeps the old single-deadline behavior.
|
|
189
|
+
* Default 90000 (fork-validated in freellmapi). Adoption review §1.2.
|
|
140
190
|
*/
|
|
141
191
|
stallTimeoutMs?: number;
|
|
142
192
|
/** "free": wholly free/free-tier catalog. "mixed": catalog contains free and paid models. */
|
|
143
193
|
tierType?: ProviderTierType;
|
|
144
194
|
/**
|
|
145
|
-
* Operator-asserted rate limits (spec §4 rung 3, basis "configured").
|
|
195
|
+
* Operator-asserted rate limits (spec §4 rung 3, basis "configured"). The provider-level block
|
|
196
|
+
* is the default for every credential of this provider; a slot's own `limits` overrides it per
|
|
197
|
+
* credential; a `models` entry overrides per deployment — each axis independently. See
|
|
198
|
+
* `src/configured-limits.ts`. These never refuse a request by themselves; they feed the
|
|
199
|
+
* availability/headroom surfaces.
|
|
146
200
|
*/
|
|
147
201
|
limits?: ProviderLimitsConfig;
|
|
148
202
|
/**
|
|
149
|
-
* Wire-shape quirks this host's own request validator enforces.
|
|
203
|
+
* Wire-shape quirks this host's own request validator enforces. Absent keys fall back to a
|
|
204
|
+
* labelled base-host default (`resolveToolCallIdMode`, `resolveThoughtSignatureMode`); an
|
|
205
|
+
* explicit value always wins.
|
|
150
206
|
*/
|
|
151
207
|
compat?: ProviderCompatConfig;
|
|
152
208
|
/** Web URL where users can sign up or obtain API keys. */
|
|
@@ -154,6 +210,15 @@ export interface ProviderConfig {
|
|
|
154
210
|
}
|
|
155
211
|
/**
|
|
156
212
|
* How an inbound request's `model` maps to a provider + backend model.
|
|
213
|
+
* - `default`: fallback "provider/model" spec (string or array of target specs) when nothing else matches.
|
|
214
|
+
* - `tiers`: Claude tier name ("opus"/"sonnet"/"haiku"/"fable") → "provider/model" spec or array.
|
|
215
|
+
* - `pools`: named candidate list addressed as "pool/<name>" — the ranked, discovery-friendly
|
|
216
|
+
* alternative to pinning one model. A pool resolves to ALL its specs, so `benchmarkSort` ranks
|
|
217
|
+
* them and the existing failover walks them in order. This is the only routing form that lets a
|
|
218
|
+
* caller say "the best available coding model" instead of naming one; a bare namespaced spec is
|
|
219
|
+
* deliberately verbatim and never ranked.
|
|
220
|
+
* A request may also address a provider directly with a namespaced model id
|
|
221
|
+
* ("nim/z-ai/glm-5.2") — the prefix picks the provider, the rest is the model.
|
|
157
222
|
*/
|
|
158
223
|
export interface Routing {
|
|
159
224
|
default: string | string[];
|
|
@@ -162,48 +227,106 @@ export interface Routing {
|
|
|
162
227
|
/** Dynamic pool policies are normalized separately from their materialized target arrays. */
|
|
163
228
|
poolPolicies?: Record<string, PoolPolicy>;
|
|
164
229
|
/**
|
|
165
|
-
* Pool → the specs in its DEGRADE TAIL: live members below the pool's effort band
|
|
230
|
+
* Pool → the specs in its DEGRADE TAIL: live members below the pool's effort band, appended so
|
|
231
|
+
* an exhausted band still has somewhere to go.
|
|
232
|
+
*
|
|
233
|
+
* Materialized alongside `pools`, never written to config.json — it is derived state, and the
|
|
234
|
+
* same refresh that rebuilds a pool rebuilds this. It exists so a served response can say the
|
|
235
|
+
* answer came from below the requested band: degrading automatically is only acceptable if it is
|
|
236
|
+
* never silent, since a capability downgrade that reads as an ordinary success is indistinguishable
|
|
237
|
+
* from having got what you asked for.
|
|
166
238
|
*/
|
|
167
239
|
poolDegraded?: Record<string, string[]>;
|
|
168
240
|
/**
|
|
169
|
-
* Tier → spec for SUBAGENT requests only (`cc_is_subagent=true`).
|
|
241
|
+
* Tier → spec for SUBAGENT requests only (`cc_is_subagent=true`). Lets the dispatcher pick a
|
|
242
|
+
* destination with the one per-call knob it actually has — the Agent tool's `model` enum
|
|
243
|
+
* (sonnet|opus|haiku|fable) — without writing an agent file. `default` catches anything that
|
|
244
|
+
* matches no tier. Main-conversation requests never consult this, which is what keeps
|
|
245
|
+
* `routing.tiers` free to stay on an Anthropic passthrough.
|
|
170
246
|
*/
|
|
171
247
|
subagents?: Record<string, string>;
|
|
172
248
|
/**
|
|
173
|
-
* Offload admission.
|
|
249
|
+
* Offload admission. The legacy boolean is a global subagent-only switch. The object form is
|
|
250
|
+
* independently keyed by originating client (`claude`, `codex`, or a future name), and each
|
|
251
|
+
* rule chooses whether it applies to marked subagents only or to every conversation from that
|
|
252
|
+
* client. **Default false** in either form.
|
|
174
253
|
*/
|
|
175
254
|
offload?: OffloadConfig;
|
|
176
255
|
benchmarkSort?: boolean;
|
|
177
256
|
/** Ephemeral session affinity. Boolean shorthand uses the 30m/1,000-entry defaults. */
|
|
178
257
|
sticky?: StickyConfig;
|
|
179
258
|
/**
|
|
180
|
-
* Quota-as-demotion enforcement (spec §5.4 / Gap 12).
|
|
259
|
+
* Quota-as-demotion enforcement (spec §5.4 / Gap 12). `enforce` (default true) lets a SPENT
|
|
260
|
+
* quota demote a candidate to the cooling band — provider-stated observations and
|
|
261
|
+
* operator-declared limits only. `enforceLearned` (default false) additionally admits limits
|
|
262
|
+
* LEARNED from vendor prose (decision M2): display-only until this is set to true.
|
|
263
|
+
*
|
|
264
|
+
* A demotion never drops and never refuses — it only reorders, expiring at the resetsAt the
|
|
265
|
+
* evidence stated. Unknown quota has no effect whatsoever.
|
|
181
266
|
*/
|
|
182
267
|
quota?: QuotaEnforcementConfig;
|
|
183
268
|
/**
|
|
184
|
-
* Sustained MEASURED latency as a demotion term (owner decision 2026-08-30).
|
|
269
|
+
* Sustained MEASURED latency as a demotion term (owner decision 2026-08-30). **Default ON** —
|
|
270
|
+
* absent means enabled with the tunable defaults in `src/latency-demotion.ts`. `false` is the
|
|
271
|
+
* shorthand for `{ enabled: false }` and restores the pre-2026-08-30 behaviour exactly.
|
|
272
|
+
*
|
|
273
|
+
* Like quota, it only ever REORDERS: never drops, never refuses, and unmeasured latency has no
|
|
274
|
+
* effect whatsoever.
|
|
185
275
|
*/
|
|
186
276
|
latency?: LatencyDemotionConfig;
|
|
187
277
|
/**
|
|
188
|
-
* Hedged attempts
|
|
278
|
+
* Hedged attempts (owner proposal + decisions 2026-08-30,
|
|
279
|
+
* docs/hedged-attempts-design-2026-08-30.md §7). **Default ON**, and confined to deployments
|
|
280
|
+
* `assessCost()` calls FREE.
|
|
281
|
+
*
|
|
282
|
+
* ⚠ This is the FIRST behaviour here that does not merely reorder — it DUPLICATES a request onto
|
|
283
|
+
* a second candidate. The `CLAUDE.md` invariant reads "Acting on counts is optional, always
|
|
284
|
+
* announced, and may only reorder"; the owner amended it for this feature on 2026-08-30 and
|
|
285
|
+
* bounded the duplication three ways: free deployments only (D1), the loser aborted the moment a
|
|
286
|
+
* winner commits, and the response announcing it (`x-llm-relay-hedged`).
|
|
287
|
+
*
|
|
288
|
+
* `false` is the shorthand for `{ enabled: false }` and restores the pre-hedge behaviour exactly.
|
|
189
289
|
*/
|
|
190
290
|
hedge?: HedgeConfig;
|
|
191
291
|
/**
|
|
192
|
-
* Background lane re-probing
|
|
292
|
+
* Background lane re-probing (owner decision 2026-08-29,
|
|
293
|
+
* docs/quota-reprobe-design-2026-08-29.md): keeping lane metadata fresh is the relay's own
|
|
294
|
+
* job, the way the ping loop already does for HTTP. **Default ON** — catalog probes are
|
|
295
|
+
* metadata commands that spend no quota, and quota probes fire only for buckets carrying an
|
|
296
|
+
* ACTIVE recorded death (an alive lane is re-tested by real use for free). Boolean shorthand
|
|
297
|
+
* toggles `enabled` with the default intervals. Absent on a hand-built `Config` means the
|
|
298
|
+
* defaults too — the cadence resolves absence itself.
|
|
193
299
|
*/
|
|
194
300
|
laneProbe?: LaneProbeSettings;
|
|
195
301
|
/**
|
|
196
|
-
* Settings for `llm-relay mcp
|
|
302
|
+
* Settings for `llm-relay mcp`, the stdio MCP server that exposes the dispatch verb to any MCP
|
|
303
|
+
* host. Read ONLY by that server — the daemon never consults this block, because the daemon
|
|
304
|
+
* never serves MCP and never spawns a lane for an HTTP turn.
|
|
305
|
+
*
|
|
306
|
+
* Absent means every default: the lane runs in the MCP server's own working directory unless the
|
|
307
|
+
* caller names another, and any existing directory is accepted.
|
|
197
308
|
*/
|
|
198
309
|
mcp?: McpSettings;
|
|
199
310
|
/**
|
|
200
|
-
* Ordered dispatch ladder consulted by `/dispatch
|
|
311
|
+
* Ordered dispatch ladder consulted by `/dispatch` — which LANE a host agent should hand a
|
|
312
|
+
* whole delegated task to, and in what order to fall back. Distinct from `subagents`, which
|
|
313
|
+
* routes one HTTP turn: a ladder rung may be an agent CLI that never traverses this proxy,
|
|
314
|
+
* because its quota is client-bound and only the vendor's own binary can spend it.
|
|
315
|
+
*
|
|
316
|
+
* Absent means the relay expresses no opinion and dispatch order stays the host's to choose.
|
|
201
317
|
*/
|
|
202
318
|
ladder?: LadderRung[];
|
|
203
319
|
/** Tier-specific dispatch ladders. `dispatch --tier <name>` selects one. */
|
|
204
320
|
ladders?: Record<string, LadderRung[]>;
|
|
205
321
|
/**
|
|
206
|
-
* How to reach a `relay` rung's spec by SHELLING OUT
|
|
322
|
+
* How to reach a `relay` rung's spec by SHELLING OUT, for a host whose own traffic does not
|
|
323
|
+
* reach this relay (see `src/host-routing.ts`). `/dispatch` substitutes the rung's spec into
|
|
324
|
+
* `{spec}` and the task into `{task}`, and hands back an ordinary `cli` invoke.
|
|
325
|
+
*
|
|
326
|
+
* Declared, never invented: the relay must not learn what a `claude` binary is or how to
|
|
327
|
+
* address one — that is the provider/model-agnostic invariant applied to lane rendering. Absent
|
|
328
|
+
* means no transposition is possible, and such rungs are reported unreachable rather than
|
|
329
|
+
* quietly replaced with something the operator never authorised.
|
|
207
330
|
*/
|
|
208
331
|
cliLane?: CliLaneTemplate;
|
|
209
332
|
}
|
|
@@ -213,6 +336,17 @@ export interface StickyRoutingConfig {
|
|
|
213
336
|
maxSessions?: number;
|
|
214
337
|
}
|
|
215
338
|
export type StickyConfig = boolean | StickyRoutingConfig;
|
|
339
|
+
/**
|
|
340
|
+
* `routing.quota`. Both keys are optional booleans with deliberate defaults: enforcement of
|
|
341
|
+
* provider-stated/derived-from-configured figures is ON unless switched off (decision M1), and
|
|
342
|
+
* learned prose parses stay display-only unless explicitly admitted (decision M2). An object with
|
|
343
|
+
* neither key is legal and means exactly the defaults — writing it down is documentation, not a
|
|
344
|
+
* behaviour change.
|
|
345
|
+
*
|
|
346
|
+
* G2 adds `hardCaps` (default true): an operator who wrote a `hard` block inside a `limits`
|
|
347
|
+
* declaration meant it, so the refusal ceilings are live unless this switch turns every one of
|
|
348
|
+
* them into a soft limit. false demotes nothing per request and logs nothing.
|
|
349
|
+
*/
|
|
216
350
|
export interface QuotaEnforcementConfig {
|
|
217
351
|
/** Default true. false disables quota demotion entirely. */
|
|
218
352
|
enforce?: boolean;
|
|
@@ -221,6 +355,19 @@ export interface QuotaEnforcementConfig {
|
|
|
221
355
|
/** Default true. false treats every `hard` cap as an ordinary (soft) configured limit. */
|
|
222
356
|
hardCaps?: boolean;
|
|
223
357
|
}
|
|
358
|
+
/**
|
|
359
|
+
* `routing.latency` — sustained MEASURED latency as a demotion term (owner decision 2026-08-30).
|
|
360
|
+
*
|
|
361
|
+
* **Default ON.** A candidate whose measured p95 exceeds `p95Ms`, over at least `minSamples`
|
|
362
|
+
* measurable samples, joins the cooling band instead of leading the walk. It is a demotion, so the
|
|
363
|
+
* worst case is a reorder: nothing is dropped and nothing is refused.
|
|
364
|
+
*
|
|
365
|
+
* `false` is the documented shorthand for `{ enabled: false }` and restores the pre-2026-08-30
|
|
366
|
+
* behaviour exactly. An object with no keys is legal and means the defaults — writing it down is
|
|
367
|
+
* documentation, not a behaviour change.
|
|
368
|
+
*
|
|
369
|
+
* The thresholds live in `src/latency-demotion.ts` beside the measurement that calibrated them.
|
|
370
|
+
*/
|
|
224
371
|
export interface LatencyDemotionConfig {
|
|
225
372
|
/** Default true. false disables latency demotion entirely. */
|
|
226
373
|
enabled?: boolean;
|
|
@@ -228,28 +375,76 @@ export interface LatencyDemotionConfig {
|
|
|
228
375
|
p95Ms?: number;
|
|
229
376
|
/**
|
|
230
377
|
* PRIMARY ceiling: measured p95 latency per OUTPUT TOKEN, in ms.
|
|
378
|
+
*
|
|
379
|
+
* Absolute latency cannot compare a probe with a generation - a probe asks for one token, a real
|
|
380
|
+
* request may produce hundreds and amortise the same fixed overhead. The per-token rate is the
|
|
381
|
+
* fair figure, so it is tested first, over real request samples only.
|
|
231
382
|
*/
|
|
232
383
|
msPerToken?: number;
|
|
233
384
|
/** Minimum measurable samples before latency may demote anything at all. */
|
|
234
385
|
minSamples?: number;
|
|
235
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* `routing.hedge` — start the NEXT candidate beside a slow in-flight attempt, instead of after it
|
|
389
|
+
* (owner proposal 2026-08-30; the four decisions are in
|
|
390
|
+
* `docs/hedged-attempts-design-2026-08-30.md` §7).
|
|
391
|
+
*
|
|
392
|
+
* **Default ON, free deployments only.** That is owner decision D1, taken against the
|
|
393
|
+
* recommendation of off-by-default. `assessCost()` treats an UNKNOWN price as paid, so the rule is
|
|
394
|
+
* fail-safe in the only direction that matters: a duplicate can never land on a deployment whose
|
|
395
|
+
* price this relay cannot establish. The stated cost is that hedging silently does not fire on many
|
|
396
|
+
* members whose prices are simply unpublished.
|
|
397
|
+
*
|
|
398
|
+
* ⚠ **Hedging DUPLICATES; every other term here only reorders.** `false` is the documented
|
|
399
|
+
* shorthand for `{ enabled: false }` and restores the pre-hedge behaviour exactly, byte for byte.
|
|
400
|
+
* An object with no keys is legal and means the defaults.
|
|
401
|
+
*
|
|
402
|
+
* The thresholds live in `src/hedge-trigger.ts`. ⚠ `margin` and `minSamples` are still PLACEHOLDERS
|
|
403
|
+
* awaiting calibration and say so at their definition — do not quote them as measurements.
|
|
404
|
+
* `msPerInputToken` IS calibrated (`scripts/calibrate-hedge-floor.mjs`, 2026-09-04) — see its own
|
|
405
|
+
* doc comment in `hedge-trigger.ts` for why the fit came back out of range and what shipped instead.
|
|
406
|
+
*/
|
|
236
407
|
export interface HedgeConfig {
|
|
237
408
|
/** Default true. false disables hedging entirely. */
|
|
238
409
|
enabled?: boolean;
|
|
239
410
|
/**
|
|
240
|
-
* The floor
|
|
411
|
+
* The floor's flat component, in ms. Owner direction 2026-09-04: the floor is no longer flat on
|
|
412
|
+
* its own — see `msPerInputToken` — but this still bounds the SMALL-prompt case, where the
|
|
413
|
+
* size-scaled component is negligible. Without it a deployment with a tiny p90 is hedged on
|
|
414
|
+
* ordinary noise, and a fast pool duplicates almost every request.
|
|
415
|
+
*/
|
|
416
|
+
minFloorMs?: number;
|
|
417
|
+
/**
|
|
418
|
+
* LEGACY alias of `minFloorMs`, kept for backward compatibility — an operator config written
|
|
419
|
+
* before 2026-09-04 (`{"floorMs": 8000}`) keeps loading and keeps meaning exactly what it always
|
|
420
|
+
* meant: the floor never drops below 8000 ms. Honoured only when `minFloorMs` itself is absent;
|
|
421
|
+
* `resolveHedgeSettings` in `hedge-trigger.ts` is the ONE place that resolves the alias, so a new
|
|
422
|
+
* caller of that function can never re-decide the precedence.
|
|
241
423
|
*/
|
|
242
424
|
floorMs?: number;
|
|
425
|
+
/**
|
|
426
|
+
* The floor's size-scaled component, in ms per estimated INPUT token
|
|
427
|
+
* (`estimateRequestTokens` in `metadata.ts`). Without it a large prompt is hedged against the
|
|
428
|
+
* time it simply takes a healthy deployment to read the prompt, not against real slowness.
|
|
429
|
+
*/
|
|
430
|
+
msPerInputToken?: number;
|
|
243
431
|
/** How far past the expected time an attempt must run before a hedge starts. */
|
|
244
432
|
margin?: number;
|
|
245
433
|
/** Minimum samples before a measured statistic may set the bar instead of the floor. */
|
|
246
434
|
minSamples?: number;
|
|
247
435
|
}
|
|
436
|
+
/**
|
|
437
|
+
* Template for rendering a `relay` rung as a shelled-out CLI command.
|
|
438
|
+
*
|
|
439
|
+
* Same executable shape as a `cli` rung — the host runs `command` with `args`, applying `env`
|
|
440
|
+
* (string sets, `null` unsets) — with one extra placeholder: `{spec}`, replaced by the rung's
|
|
441
|
+
* routing spec so ONE template serves every pool and pinned model in the ladder.
|
|
442
|
+
*/
|
|
248
443
|
export interface CliLaneTemplate {
|
|
249
444
|
command: string;
|
|
250
|
-
/** Must contain `{spec}`; `{task}` too. */
|
|
445
|
+
/** Must contain `{spec}`; `{task}` too, on the same reasoning as a cli rung's args. */
|
|
251
446
|
args: string[];
|
|
252
|
-
/** Applied by the host when spawning. */
|
|
447
|
+
/** Applied by the host when spawning. Placeholders are never substituted here. */
|
|
253
448
|
env?: Record<string, string | null>;
|
|
254
449
|
}
|
|
255
450
|
/** A fixed configured prefix followed by automatically discovered free models. */
|
|
@@ -261,6 +456,15 @@ export interface PoolPolicy {
|
|
|
261
456
|
/** Optional evidence-aware effort band for the discovered tail. */
|
|
262
457
|
effort?: EffortLevel;
|
|
263
458
|
}
|
|
459
|
+
/**
|
|
460
|
+
* One rung of `routing.ladder`.
|
|
461
|
+
*
|
|
462
|
+
* `cli` rungs are executed by the HOST (the relay never spawns a process — it is a proxy, not a
|
|
463
|
+
* process supervisor); `relay` rungs are addressed through this proxy in the ordinary way.
|
|
464
|
+
* `quota` names the balance a rung draws on so that rungs sharing one are cooled down together
|
|
465
|
+
* and rungs that merely share a binary are not: one CLI can meter two model families against
|
|
466
|
+
* two independent balances, and treating those as one bucket would skip a live lane.
|
|
467
|
+
*/
|
|
264
468
|
export interface LadderRung {
|
|
265
469
|
id: string;
|
|
266
470
|
kind: "cli" | "relay";
|
|
@@ -271,7 +475,16 @@ export interface LadderRung {
|
|
|
271
475
|
/** cli rungs: the binary to run, and its args — one of which must contain the task placeholder. */
|
|
272
476
|
command?: string;
|
|
273
477
|
args?: string[];
|
|
274
|
-
/**
|
|
478
|
+
/**
|
|
479
|
+
* cli rungs: environment the HOST applies when spawning the command. A string value sets the
|
|
480
|
+
* variable; `null` unsets an inherited one. Both directions are load-bearing for the lane this
|
|
481
|
+
* exists for — a `claude -p` child routed through this proxy: `ANTHROPIC_BASE_URL` must be SET
|
|
482
|
+
* (a terminal-spawned `claude` honours it even though Claude Desktop pins its own sessions to
|
|
483
|
+
* api.anthropic.com), and `CLAUDECODE`/`CLAUDE_CODE_SSE_PORT`/`CLAUDE_CODE_ENTRYPOINT`/
|
|
484
|
+
* `ANTHROPIC_API_KEY` must be UNSET or a child spawned from inside a Claude session inherits
|
|
485
|
+
* the parent's harness wiring and refuses to start cleanly. The task placeholder is never
|
|
486
|
+
* substituted here — env values are operator-authored routing, not task content.
|
|
487
|
+
*/
|
|
275
488
|
env?: Record<string, string | null>;
|
|
276
489
|
/** relay rungs: the spec to address (`pool/<name>`, `<provider>/<model>`, a provider name). */
|
|
277
490
|
spec?: string;
|
|
@@ -295,11 +508,17 @@ export interface ResolvedTarget {
|
|
|
295
508
|
/** Carried from the provider: inter-byte stall watchdog for streamed responses. */
|
|
296
509
|
stallTimeoutMs?: number;
|
|
297
510
|
/**
|
|
298
|
-
* RESOLVED outbound tool-call-id shape (`resolveToolCallIdMode`)
|
|
511
|
+
* RESOLVED outbound tool-call-id shape (`resolveToolCallIdMode`) — an explicit
|
|
512
|
+
* `compat.toolCallIds` or the labelled base-host default. Resolved here so the request mapper
|
|
513
|
+
* is handed a mode and never a provider identity to re-derive one from. Absent (a hand-built
|
|
514
|
+
* target) reads as `"preserve"`, which is the pre-2026-08-23 behaviour byte for byte.
|
|
299
515
|
*/
|
|
300
516
|
toolCallIds?: ToolCallIdMode;
|
|
301
517
|
/**
|
|
302
|
-
* RESOLVED thought-signature mode (`resolveThoughtSignatureMode`)
|
|
518
|
+
* RESOLVED thought-signature mode (`resolveThoughtSignatureMode`) — an explicit
|
|
519
|
+
* `compat.thoughtSignature` or the labelled base-host default. Resolved here for the same reason
|
|
520
|
+
* as `toolCallIds`: the request mapper is handed a mode, never a provider identity to sniff one
|
|
521
|
+
* from. Absent (a hand-built target) reads as `"none"` — the pre-2026-08-23 bytes exactly.
|
|
303
522
|
*/
|
|
304
523
|
thoughtSignature?: ThoughtSignatureMode;
|
|
305
524
|
}
|
|
@@ -312,7 +531,9 @@ export interface Config {
|
|
|
312
531
|
/** Explicit global reshaper override; otherwise repair reshapes on the resolved target itself. */
|
|
313
532
|
reshaper?: ReshaperConfig;
|
|
314
533
|
/**
|
|
315
|
-
* Ranked reshaper candidates, from `reshaper: { pool: "<name>" }`.
|
|
534
|
+
* Ranked reshaper candidates, from `reshaper: { pool: "<name>" }`. Tried in order, so a single
|
|
535
|
+
* de-listed model cannot take repair down with it — the whole point of not pinning one model.
|
|
536
|
+
* `reshaper` is candidates[0] so every existing single-reshaper path keeps working unchanged.
|
|
316
537
|
*/
|
|
317
538
|
reshaperCandidates?: ReshaperConfig[];
|
|
318
539
|
/** Dynamic reshaper pool resolved lazily after its catalog-backed tail is materialized. */
|
|
@@ -325,7 +546,10 @@ export interface Config {
|
|
|
325
546
|
destructiveTools: string[];
|
|
326
547
|
};
|
|
327
548
|
/**
|
|
328
|
-
* Wall-clock ceiling (ms) on STARTING further failover attempts within one request's pool
|
|
549
|
+
* Wall-clock ceiling (ms) on STARTING further failover attempts within one request's pool
|
|
550
|
+
* walk. The first two attempts are always allowed and an attempt already in flight is never
|
|
551
|
+
* aborted — the budget bounds the walk, not the answer. 0 disables. Absent ⇒ the server's
|
|
552
|
+
* DEFAULT_WALK_BUDGET_MS (45s).
|
|
329
553
|
*/
|
|
330
554
|
walkBudgetMs?: number;
|
|
331
555
|
/** Maximum inbound request-body size in bytes. Absent ⇒ 36 MiB. */
|
|
@@ -337,15 +561,26 @@ export interface Config {
|
|
|
337
561
|
};
|
|
338
562
|
/**
|
|
339
563
|
* Providers the onboarding nudge must stop asking about (`leave_me_alone` in config.json).
|
|
564
|
+
*
|
|
565
|
+
* Scope is deliberately narrow: it silences the "❌ Missing Key / 👉 get one here" prompt in
|
|
566
|
+
* `llm-relay onboard`, and nothing else. Suppressed providers still appear in `llm-relay keys`,
|
|
567
|
+
* in `/registry` and in every status surface — silencing a nudge is not hiding state, and a
|
|
568
|
+
* provider that vanished from the status commands would be undebuggable later.
|
|
569
|
+
*
|
|
570
|
+
* Names that match no known provider are legal (see `parseLeaveMeAlone`).
|
|
340
571
|
*/
|
|
341
572
|
leaveMeAlone?: string[];
|
|
342
|
-
/** Path this config was loaded from. Set by `loadConfig`; absent for hand-built test configs.
|
|
573
|
+
/** Path this config was loaded from. Set by `loadConfig`; absent for hand-built test configs.
|
|
574
|
+
* Only consumer is the runtime offload toggle, which persists back to the same file. */
|
|
343
575
|
sourcePath?: string;
|
|
344
576
|
/**
|
|
345
|
-
* Non-fatal load-time problems
|
|
577
|
+
* Non-fatal load-time problems (a provider disabled for an unset `${ENV}`, a pool member
|
|
578
|
+
* dropped with it). Present so startup can print them — a degraded config that boots
|
|
579
|
+
* silently is how you end up running on one provider without noticing.
|
|
346
580
|
*/
|
|
347
581
|
warnings?: string[];
|
|
348
582
|
}
|
|
583
|
+
/** Background lane re-probing settings — see the `laneProbe` field doc on `Routing`. */
|
|
349
584
|
export interface LaneProbeSettings {
|
|
350
585
|
enabled: boolean;
|
|
351
586
|
/** Gate between quota probes of ONE dead bucket. A probe spends that lane's quota. */
|
|
@@ -353,6 +588,15 @@ export interface LaneProbeSettings {
|
|
|
353
588
|
/** Gate between catalog re-probes of ONE lane. Metadata commands, no quota spent. */
|
|
354
589
|
catalogIntervalMs: number;
|
|
355
590
|
}
|
|
591
|
+
/** `llm-relay mcp` settings — see the `mcp` field doc on `Routing`. */
|
|
356
592
|
export interface McpSettings {
|
|
593
|
+
/**
|
|
594
|
+
* Directories a caller-supplied `cwd` must sit under. Absent or empty ⇒ no bound beyond the
|
|
595
|
+
* directory existing.
|
|
596
|
+
*
|
|
597
|
+
* ⚠ Offered, never imposed. The MCP caller is already a trusted agent on the operator's own
|
|
598
|
+
* machine, and defaulting to a bound would make the tool useless for its stated purpose. Whether
|
|
599
|
+
* to narrow it is the operator's decision to record here, not this file's to assume.
|
|
600
|
+
*/
|
|
357
601
|
allowedRoots?: string[];
|
|
358
602
|
}
|
package/dist/config-types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-types.js","sourceRoot":"","sources":["../src/config-types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"config-types.js","sourceRoot":"","sources":["../src/config-types.ts"],"names":[],"mappings":"AAyBA,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAU,CAAC;AAMzE,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;AAyF/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC"}
|