llm-relay 0.70.0 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accounting-store-schema.d.ts +8 -1
- package/dist/accounting-store-schema.js +3 -1
- package/dist/accounting-store-schema.js.map +1 -1
- package/dist/accounting-store.d.ts +16 -12
- package/dist/accounting-store.js +1 -1
- package/dist/accounting-store.js.map +1 -1
- package/dist/accounting.d.ts +2 -3
- package/dist/accounting.js +2 -1
- package/dist/accounting.js.map +1 -1
- package/dist/candidate-runner.d.ts +36 -2
- package/dist/candidate-runner.js +56 -7
- package/dist/candidate-runner.js.map +1 -1
- package/dist/cli.d.ts +22 -0
- package/dist/cli.js +92 -19
- package/dist/cli.js.map +1 -1
- package/dist/config-types.d.ts +270 -26
- package/dist/config-types.js.map +1 -1
- package/dist/config.d.ts +2 -554
- package/dist/config.js +2 -2
- package/dist/config.js.map +1 -1
- package/dist/dashboard-routes.d.ts +1 -10
- package/dist/dashboard-routes.js +2 -1
- package/dist/dashboard-routes.js.map +1 -1
- package/dist/delegate-gate/test-assertions.js +27 -6
- package/dist/delegate-gate/test-assertions.js.map +1 -1
- package/dist/dispatch-lane-stats.d.ts +133 -0
- package/dist/dispatch-lane-stats.js +274 -0
- package/dist/dispatch-lane-stats.js.map +1 -0
- package/dist/dispatch.d.ts +49 -1
- package/dist/dispatch.js +29 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/hedge-trigger.d.ts +14 -26
- package/dist/hedge-trigger.js.map +1 -1
- package/dist/mcp/lane-runner.d.ts +20 -1
- package/dist/mcp/lane-runner.js +11 -1
- package/dist/mcp/lane-runner.js.map +1 -1
- package/dist/mcp/server.d.ts +19 -0
- package/dist/mcp/server.js +47 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/openai-request.d.ts +1 -4
- package/dist/openai-request.js.map +1 -1
- package/dist/quota-observation.d.ts +2 -3
- package/dist/quota-observation.js.map +1 -1
- package/dist/routes/admin.d.ts +8 -0
- package/dist/routes/admin.js +97 -18
- package/dist/routes/admin.js.map +1 -1
- package/dist/routes/messages.js +8 -4
- package/dist/routes/messages.js.map +1 -1
- package/dist/routes/openai-front.js +8 -4
- package/dist/routes/openai-front.js.map +1 -1
- package/dist/server.js +6 -2
- package/dist/server.js.map +1 -1
- package/dist/storage/json-store.d.ts +0 -34
- package/dist/storage/json-store.js +0 -44
- package/dist/storage/json-store.js.map +1 -1
- package/dist/stream-pipeline.d.ts +11 -0
- package/dist/stream-pipeline.js +7 -1
- package/dist/stream-pipeline.js.map +1 -1
- package/package.json +1 -1
package/dist/config.d.ts
CHANGED
|
@@ -1,107 +1,6 @@
|
|
|
1
|
-
import { type ProviderLimitsConfig } from "./configured-limits.js";
|
|
2
|
-
import { type CredentialSlot, type ProviderCredentialConfig } from "./credential-fleet.js";
|
|
3
1
|
import { type KeystoreOptions } from "./keystore.js";
|
|
4
|
-
|
|
5
|
-
export type AuthHeader
|
|
6
|
-
export type Kind = "anthropic" | "openai";
|
|
7
|
-
export type ProviderTierType = "free" | "mixed" | "subscription";
|
|
8
|
-
/**
|
|
9
|
-
* The effort bands, WEAKEST FIRST — one declaration, and the type is derived from it rather than
|
|
10
|
-
* the other way round.
|
|
11
|
-
*
|
|
12
|
-
* ⚠ Order is load-bearing: `dynamic-pools.ts` derives the degrade tail from this array's index, so
|
|
13
|
-
* a band inserted out of order silently reorders which weaker members an exhausted pool falls back
|
|
14
|
-
* to.
|
|
15
|
-
*
|
|
16
|
-
* ⚠ Why an ordered tuple and not four hand-lists: the member list stood restated in four places
|
|
17
|
-
* across three modules, and only ONE of them was exhaustiveness-checked. `new Set<EffortLevel>([…])`
|
|
18
|
-
* and `EffortLevel[]` both accept a SUBSET, and `cli.ts` validated against a bare `string[]` with
|
|
19
|
-
* no link to the type at all — so a fifth band would have compiled everywhere and silently failed
|
|
20
|
-
* in three of the four. `Record<EffortLevel, …>` (`EFFORT_FLOORS` in `benchmarks.ts`) is the shape
|
|
21
|
-
* that already caught it, and is why that one is left as it is.
|
|
22
|
-
*/
|
|
23
|
-
export declare const EFFORT_LEVELS: readonly ["low", "medium", "high", "xhigh"];
|
|
24
|
-
/** Requested reasoning/capability band for an automatically discovered pool. */
|
|
25
|
-
export type EffortLevel = (typeof EFFORT_LEVELS)[number];
|
|
26
|
-
export declare const CLAUDE_TIER_NAMES: readonly ["opus", "sonnet", "haiku", "fable"];
|
|
27
|
-
export type ClaudeTierName = (typeof CLAUDE_TIER_NAMES)[number];
|
|
28
|
-
/** Which requests a client-specific offload rule may reroute. */
|
|
29
|
-
export type OffloadScope = "subagents" | "all";
|
|
30
|
-
/** One independently controlled offload rule. */
|
|
31
|
-
export interface OffloadRule {
|
|
32
|
-
enabled: boolean;
|
|
33
|
-
scope: OffloadScope;
|
|
34
|
-
/**
|
|
35
|
-
* Refuse — loudly — rather than let this client's rerouted traffic reach a deployment that
|
|
36
|
-
* is not assessed `free` (see `assessCost`; `unknown` counts as paid on purpose). Applies to
|
|
37
|
-
* everything `subagentSpec` reroutes for this client, INCLUDING a per-call `@relay:` directive:
|
|
38
|
-
* the flag is the owner's standing "this lane never spends money", and a subagent prompt must
|
|
39
|
-
* not be able to out-rank it. The Anthropic passthrough is primary quota and is never free.
|
|
40
|
-
*/
|
|
41
|
-
freeOnly?: boolean;
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* `false`/`true` is the backwards-compatible global switch. The object form is keyed by
|
|
45
|
-
* originating harness (`claude`, `codex`, or a future client name), with `default` available as
|
|
46
|
-
* an explicit catch-all for a client that has not got a dedicated rule yet.
|
|
47
|
-
*/
|
|
48
|
-
export type OffloadConfig = boolean | Record<string, OffloadRule>;
|
|
49
|
-
export interface ReshaperConfig {
|
|
50
|
-
base: string;
|
|
51
|
-
model: string;
|
|
52
|
-
/** "anthropic": call /v1/messages. "openai": call /chat/completions (NIM/vLLM). */
|
|
53
|
-
kind: Kind;
|
|
54
|
-
/** Provider name for provider-aware credential resolution (auth alias + passthrough rules). */
|
|
55
|
-
provider?: string;
|
|
56
|
-
authEnv?: string;
|
|
57
|
-
authHeader: AuthHeader;
|
|
58
|
-
timeoutMs: number;
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* What happens to the CALLER's own `Authorization`/`x-api-key` at a provider that declares no
|
|
62
|
-
* `authEnv` of its own. See `buildForwardHeaders`.
|
|
63
|
-
*/
|
|
64
|
-
export type CredentialMode = "passthrough" | "contained";
|
|
65
|
-
/**
|
|
66
|
-
* What shape a provider's own validator demands of the tool-call ids the relay puts on the wire.
|
|
67
|
-
*
|
|
68
|
-
* - `"preserve"` — forward the caller's ids verbatim. The default everywhere, and what the relay
|
|
69
|
-
* did for its whole life before 2026-08-23: an id is linkage, so not touching it is the safest
|
|
70
|
-
* thing a translation can do.
|
|
71
|
-
* - `"strict9"` — rewrite every outbound `tool_calls[].id` / `tool_call_id` to mistral's stated
|
|
72
|
-
* `^[a-zA-Z0-9]{9}$` shape. Mistral's `mistral-common` validator enforces it on BOTH halves of
|
|
73
|
-
* the pair (and, from v13, linkage and uniqueness on top), so an Anthropic `toolu_01…` id is a
|
|
74
|
-
* hard 400 there — see `src/openai-request.ts`.
|
|
75
|
-
*/
|
|
76
|
-
export type ToolCallIdMode = "preserve" | "strict9";
|
|
77
|
-
/**
|
|
78
|
-
* Whether a replayed assistant tool call must carry gemini 3.x's `thought_signature` field.
|
|
79
|
-
*
|
|
80
|
-
* - `"none"` — emit nothing. The default everywhere, and byte for byte what this relay put on the
|
|
81
|
-
* wire before 2026-08-23.
|
|
82
|
-
* - `"sentinel"` — stamp Google's own documented opt-out token,
|
|
83
|
-
* `skip_thought_signature_validator`, at `tool_calls[].extra_content.google.thought_signature`
|
|
84
|
-
* on every replayed tool call. See `src/openai-request.ts` for the 400 that states the rule and
|
|
85
|
-
* for why echoing a REAL signature is not an option here.
|
|
86
|
-
*/
|
|
87
|
-
export type ThoughtSignatureMode = "none" | "sentinel";
|
|
88
|
-
/**
|
|
89
|
-
* Per-provider WIRE-SHAPE quirks — things a specific host's request validator demands that the
|
|
90
|
-
* protocol itself does not. Deliberately not routing configuration and deliberately not a
|
|
91
|
-
* per-provider switch in `src/`: a labelled provider fact may live in code only while config can
|
|
92
|
-
* override it (see the "Provider knowledge is data" invariant), which is exactly the shape here —
|
|
93
|
-
* a base-host default that any explicit value beats in both directions.
|
|
94
|
-
*
|
|
95
|
-
* Two keys today, one per vendor rule the relay has first-party evidence for. An unknown key or
|
|
96
|
-
* value is a HARD load error naming it: the `configured-limits` precedent — an ignored typo
|
|
97
|
-
* silently no-ops while reading like a declaration that took effect.
|
|
98
|
-
*/
|
|
99
|
-
export interface ProviderCompatConfig {
|
|
100
|
-
/** Absent ⇒ resolved from the base host by `resolveToolCallIdMode`. */
|
|
101
|
-
toolCallIds?: ToolCallIdMode;
|
|
102
|
-
/** Absent ⇒ resolved from the base host by `resolveThoughtSignatureMode`. */
|
|
103
|
-
thoughtSignature?: ThoughtSignatureMode;
|
|
104
|
-
}
|
|
2
|
+
import { type OffloadRule, type OffloadConfig, type ReshaperConfig, type ToolCallIdMode, type ThoughtSignatureMode, type ProviderCompatConfig, type RequestHeaders, type ResolvedTarget, type Config, type LaneProbeSettings } from "./config-types.js";
|
|
3
|
+
export { EFFORT_LEVELS, CLAUDE_TIER_NAMES, type Mode, type AuthHeader, type Kind, type ProviderTierType, type EffortLevel, type ClaudeTierName, type OffloadScope, type OffloadRule, type OffloadConfig, type ReshaperConfig, type CredentialMode, type ToolCallIdMode, type ThoughtSignatureMode, type ProviderCompatConfig, type ProviderConfig, type Routing, type StickyRoutingConfig, type StickyConfig, type QuotaEnforcementConfig, type LatencyDemotionConfig, type HedgeConfig, type CliLaneTemplate, type PoolPolicy, type LadderRung, type RequestHeaders, type ResolvedTarget, type Config, type LaneProbeSettings, type McpSettings, } from "./config-types.js";
|
|
105
4
|
/**
|
|
106
5
|
* The resolved outbound tool-call-id shape for one provider.
|
|
107
6
|
*
|
|
@@ -129,352 +28,10 @@ export declare function resolveThoughtSignatureMode(p: {
|
|
|
129
28
|
base: string;
|
|
130
29
|
compat?: ProviderCompatConfig;
|
|
131
30
|
}): ThoughtSignatureMode;
|
|
132
|
-
/** One HTTP backend provider in the registry (NIM, OpenRouter, Gemini API, …). */
|
|
133
|
-
export interface ProviderConfig {
|
|
134
|
-
base: string;
|
|
135
|
-
/**
|
|
136
|
-
* "anthropic": provider speaks Anthropic Messages; forward as-is.
|
|
137
|
-
* "openai": provider is OpenAI-compatible (NIM/vLLM/OpenRouter/Gemini-openai);
|
|
138
|
-
* the proxy translates request+response via llm-bridge.
|
|
139
|
-
*/
|
|
140
|
-
kind: Kind;
|
|
141
|
-
authEnv?: string;
|
|
142
|
-
/** Explicit multi-key declarations. Presence of an empty array means an empty fleet. */
|
|
143
|
-
credentials?: ProviderCredentialConfig[];
|
|
144
|
-
/**
|
|
145
|
-
* What to do with the caller's own credential when this provider declares no `authEnv`:
|
|
146
|
-
* - `"passthrough"` — forward it. This is the declaration that makes an Anthropic
|
|
147
|
-
* passthrough deliberate rather than a side effect of leaving `authEnv` out.
|
|
148
|
-
* - `"contained"` — strip it and send none. The right answer for a keyless backend that
|
|
149
|
-
* is not the caller's own vendor: a local daemon, a second relay, someone else's
|
|
150
|
-
* Anthropic-format endpoint.
|
|
151
|
-
*
|
|
152
|
-
* Omitting it still forwards, so existing configs keep working, but config load warns for an
|
|
153
|
-
* `anthropic`-kind provider. "This backend needs no key of its own" and "send this host the
|
|
154
|
-
* user's subscription credential" are different intentions, and only the first one should be
|
|
155
|
-
* inferable from an omission. Illegal together with `authEnv` — that pair states both at once.
|
|
156
|
-
*/
|
|
157
|
-
credentialMode?: CredentialMode;
|
|
158
|
-
/** Maximum concurrent requests for this provider credential domain; null/omitted is unlimited. */
|
|
159
|
-
maxConcurrent?: number | null;
|
|
160
|
-
/** Which header to inject the provider key into. Default: authorization (openai) / x-api-key (anthropic). */
|
|
161
|
-
authHeader: AuthHeader;
|
|
162
|
-
/** Backend request deadline in ms. Default 120000. */
|
|
163
|
-
timeoutMs: number;
|
|
164
|
-
/**
|
|
165
|
-
* Inter-byte stall watchdog for STREAMED responses, in ms. Once a stream is being served, the
|
|
166
|
-
* total `timeoutMs` deadline disarms — one flat deadline kills a healthy long generation at
|
|
167
|
-
* minute two while letting a dead stream hang until the same minute two — and this watchdog
|
|
168
|
-
* aborts only when NO byte arrives for this long. 0 keeps the old single-deadline behavior.
|
|
169
|
-
* Default 90000 (fork-validated in freellmapi). Adoption review §1.2.
|
|
170
|
-
*/
|
|
171
|
-
stallTimeoutMs?: number;
|
|
172
|
-
/** "free": wholly free/free-tier catalog. "mixed": catalog contains free and paid models. */
|
|
173
|
-
tierType?: ProviderTierType;
|
|
174
|
-
/**
|
|
175
|
-
* Operator-asserted rate limits (spec §4 rung 3, basis "configured"). The provider-level block
|
|
176
|
-
* is the default for every credential of this provider; a slot's own `limits` overrides it per
|
|
177
|
-
* credential; a `models` entry overrides per deployment — each axis independently. See
|
|
178
|
-
* `src/configured-limits.ts`. These never refuse a request by themselves; they feed the
|
|
179
|
-
* availability/headroom surfaces.
|
|
180
|
-
*/
|
|
181
|
-
limits?: ProviderLimitsConfig;
|
|
182
|
-
/**
|
|
183
|
-
* Wire-shape quirks this host's own request validator enforces. Absent keys fall back to a
|
|
184
|
-
* labelled base-host default (`resolveToolCallIdMode`, `resolveThoughtSignatureMode`); an
|
|
185
|
-
* explicit value always wins.
|
|
186
|
-
*/
|
|
187
|
-
compat?: ProviderCompatConfig;
|
|
188
|
-
/** Web URL where users can sign up or obtain API keys. */
|
|
189
|
-
signupUrl?: string;
|
|
190
|
-
}
|
|
191
|
-
/**
|
|
192
|
-
* How an inbound request's `model` maps to a provider + backend model.
|
|
193
|
-
* - `default`: fallback "provider/model" spec (string or array of target specs) when nothing else matches.
|
|
194
|
-
* - `tiers`: Claude tier name ("opus"/"sonnet"/"haiku"/"fable") → "provider/model" spec or array.
|
|
195
|
-
* - `pools`: named candidate list addressed as "pool/<name>" — the ranked, discovery-friendly
|
|
196
|
-
* alternative to pinning one model. A pool resolves to ALL its specs, so `benchmarkSort` ranks
|
|
197
|
-
* them and the existing failover walks them in order. This is the only routing form that lets a
|
|
198
|
-
* caller say "the best available coding model" instead of naming one; a bare namespaced spec is
|
|
199
|
-
* deliberately verbatim and never ranked.
|
|
200
|
-
* A request may also address a provider directly with a namespaced model id
|
|
201
|
-
* ("nim/z-ai/glm-5.2") — the prefix picks the provider, the rest is the model.
|
|
202
|
-
*/
|
|
203
|
-
export interface Routing {
|
|
204
|
-
default: string | string[];
|
|
205
|
-
tiers: Record<string, string | string[]>;
|
|
206
|
-
pools?: Record<string, string[]>;
|
|
207
|
-
/** Dynamic pool policies are normalized separately from their materialized target arrays. */
|
|
208
|
-
poolPolicies?: Record<string, PoolPolicy>;
|
|
209
|
-
/**
|
|
210
|
-
* Pool → the specs in its DEGRADE TAIL: live members below the pool's effort band, appended so
|
|
211
|
-
* an exhausted band still has somewhere to go.
|
|
212
|
-
*
|
|
213
|
-
* Materialized alongside `pools`, never written to config.json — it is derived state, and the
|
|
214
|
-
* same refresh that rebuilds a pool rebuilds this. It exists so a served response can say the
|
|
215
|
-
* answer came from below the requested band: degrading automatically is only acceptable if it is
|
|
216
|
-
* never silent, since a capability downgrade that reads as an ordinary success is indistinguishable
|
|
217
|
-
* from having got what you asked for.
|
|
218
|
-
*/
|
|
219
|
-
poolDegraded?: Record<string, string[]>;
|
|
220
|
-
/**
|
|
221
|
-
* Tier → spec for SUBAGENT requests only (`cc_is_subagent=true`). Lets the dispatcher pick a
|
|
222
|
-
* destination with the one per-call knob it actually has — the Agent tool's `model` enum
|
|
223
|
-
* (sonnet|opus|haiku|fable) — without writing an agent file. `default` catches anything that
|
|
224
|
-
* matches no tier. Main-conversation requests never consult this, which is what keeps
|
|
225
|
-
* `routing.tiers` free to stay on an Anthropic passthrough.
|
|
226
|
-
*/
|
|
227
|
-
subagents?: Record<string, string>;
|
|
228
|
-
/**
|
|
229
|
-
* Offload admission. The legacy boolean is a global subagent-only switch. The object form is
|
|
230
|
-
* independently keyed by originating client (`claude`, `codex`, or a future name), and each
|
|
231
|
-
* rule chooses whether it applies to marked subagents only or to every conversation from that
|
|
232
|
-
* client. **Default false** in either form.
|
|
233
|
-
*/
|
|
234
|
-
offload?: OffloadConfig;
|
|
235
|
-
benchmarkSort?: boolean;
|
|
236
|
-
/** Ephemeral session affinity. Boolean shorthand uses the 30m/1,000-entry defaults. */
|
|
237
|
-
sticky?: StickyConfig;
|
|
238
|
-
/**
|
|
239
|
-
* Quota-as-demotion enforcement (spec §5.4 / Gap 12). `enforce` (default true) lets a SPENT
|
|
240
|
-
* quota demote a candidate to the cooling band — provider-stated observations and
|
|
241
|
-
* operator-declared limits only. `enforceLearned` (default false) additionally admits limits
|
|
242
|
-
* LEARNED from vendor prose (decision M2): display-only until this is set to true.
|
|
243
|
-
*
|
|
244
|
-
* A demotion never drops and never refuses — it only reorders, expiring at the resetsAt the
|
|
245
|
-
* evidence stated. Unknown quota has no effect whatsoever.
|
|
246
|
-
*/
|
|
247
|
-
quota?: QuotaEnforcementConfig;
|
|
248
|
-
/**
|
|
249
|
-
* Sustained MEASURED latency as a demotion term (owner decision 2026-08-30). **Default ON** —
|
|
250
|
-
* absent means enabled with the tunable defaults in `src/latency-demotion.ts`. `false` is the
|
|
251
|
-
* shorthand for `{ enabled: false }` and restores the pre-2026-08-30 behaviour exactly.
|
|
252
|
-
*
|
|
253
|
-
* Like quota, it only ever REORDERS: never drops, never refuses, and unmeasured latency has no
|
|
254
|
-
* effect whatsoever.
|
|
255
|
-
*/
|
|
256
|
-
latency?: LatencyDemotionConfig;
|
|
257
|
-
/**
|
|
258
|
-
* Hedged attempts (owner proposal + decisions 2026-08-30,
|
|
259
|
-
* docs/hedged-attempts-design-2026-08-30.md §7). **Default ON**, and confined to deployments
|
|
260
|
-
* `assessCost()` calls FREE.
|
|
261
|
-
*
|
|
262
|
-
* ⚠ This is the FIRST behaviour here that does not merely reorder — it DUPLICATES a request onto
|
|
263
|
-
* a second candidate. The `CLAUDE.md` invariant reads "Acting on counts is optional, always
|
|
264
|
-
* announced, and may only reorder"; the owner amended it for this feature on 2026-08-30 and
|
|
265
|
-
* bounded the duplication three ways: free deployments only (D1), the loser aborted the moment a
|
|
266
|
-
* winner commits, and the response announcing it (`x-llm-relay-hedged`).
|
|
267
|
-
*
|
|
268
|
-
* `false` is the shorthand for `{ enabled: false }` and restores the pre-hedge behaviour exactly.
|
|
269
|
-
*/
|
|
270
|
-
hedge?: HedgeConfig;
|
|
271
|
-
/**
|
|
272
|
-
* Background lane re-probing (owner decision 2026-08-29,
|
|
273
|
-
* docs/quota-reprobe-design-2026-08-29.md): keeping lane metadata fresh is the relay's own
|
|
274
|
-
* job, the way the ping loop already does for HTTP. **Default ON** — catalog probes are
|
|
275
|
-
* metadata commands that spend no quota, and quota probes fire only for buckets carrying an
|
|
276
|
-
* ACTIVE recorded death (an alive lane is re-tested by real use for free). Boolean shorthand
|
|
277
|
-
* toggles `enabled` with the default intervals. Absent on a hand-built `Config` means the
|
|
278
|
-
* defaults too — the cadence resolves absence itself.
|
|
279
|
-
*/
|
|
280
|
-
laneProbe?: LaneProbeSettings;
|
|
281
|
-
/**
|
|
282
|
-
* Settings for `llm-relay mcp`, the stdio MCP server that exposes the dispatch verb to any MCP
|
|
283
|
-
* host. Read ONLY by that server — the daemon never consults this block, because the daemon
|
|
284
|
-
* never serves MCP and never spawns a lane for an HTTP turn.
|
|
285
|
-
*
|
|
286
|
-
* Absent means every default: the lane runs in the MCP server's own working directory unless the
|
|
287
|
-
* caller names another, and any existing directory is accepted.
|
|
288
|
-
*/
|
|
289
|
-
mcp?: McpSettings;
|
|
290
|
-
/**
|
|
291
|
-
* Ordered dispatch ladder consulted by `/dispatch` — which LANE a host agent should hand a
|
|
292
|
-
* whole delegated task to, and in what order to fall back. Distinct from `subagents`, which
|
|
293
|
-
* routes one HTTP turn: a ladder rung may be an agent CLI that never traverses this proxy,
|
|
294
|
-
* because its quota is client-bound and only the vendor's own binary can spend it.
|
|
295
|
-
*
|
|
296
|
-
* Absent means the relay expresses no opinion and dispatch order stays the host's to choose.
|
|
297
|
-
*/
|
|
298
|
-
ladder?: LadderRung[];
|
|
299
|
-
/** Tier-specific dispatch ladders. `dispatch --tier <name>` selects one. */
|
|
300
|
-
ladders?: Record<string, LadderRung[]>;
|
|
301
|
-
/**
|
|
302
|
-
* How to reach a `relay` rung's spec by SHELLING OUT, for a host whose own traffic does not
|
|
303
|
-
* reach this relay (see `src/host-routing.ts`). `/dispatch` substitutes the rung's spec into
|
|
304
|
-
* `{spec}` and the task into `{task}`, and hands back an ordinary `cli` invoke.
|
|
305
|
-
*
|
|
306
|
-
* Declared, never invented: the relay must not learn what a `claude` binary is or how to
|
|
307
|
-
* address one — that is the provider/model-agnostic invariant applied to lane rendering. Absent
|
|
308
|
-
* means no transposition is possible, and such rungs are reported unreachable rather than
|
|
309
|
-
* quietly replaced with something the operator never authorised.
|
|
310
|
-
*/
|
|
311
|
-
cliLane?: CliLaneTemplate;
|
|
312
|
-
}
|
|
313
|
-
export interface StickyRoutingConfig {
|
|
314
|
-
enabled: boolean;
|
|
315
|
-
ttlMs?: number;
|
|
316
|
-
maxSessions?: number;
|
|
317
|
-
}
|
|
318
|
-
export type StickyConfig = boolean | StickyRoutingConfig;
|
|
319
|
-
/**
|
|
320
|
-
* `routing.quota`. Both keys are optional booleans with deliberate defaults: enforcement of
|
|
321
|
-
* provider-stated/derived-from-configured figures is ON unless switched off (decision M1), and
|
|
322
|
-
* learned prose parses stay display-only unless explicitly admitted (decision M2). An object with
|
|
323
|
-
* neither key is legal and means exactly the defaults — writing it down is documentation, not a
|
|
324
|
-
* behaviour change.
|
|
325
|
-
*
|
|
326
|
-
* G2 adds `hardCaps` (default true): an operator who wrote a `hard` block inside a `limits`
|
|
327
|
-
* declaration meant it, so the refusal ceilings are live unless this switch turns every one of
|
|
328
|
-
* them into a soft limit. false demotes nothing per request and logs nothing.
|
|
329
|
-
*/
|
|
330
|
-
export interface QuotaEnforcementConfig {
|
|
331
|
-
/** Default true. false disables quota demotion entirely. */
|
|
332
|
-
enforce?: boolean;
|
|
333
|
-
/** Default false. true additionally lets `derived:learned` figures gate routing. */
|
|
334
|
-
enforceLearned?: boolean;
|
|
335
|
-
/** Default true. false treats every `hard` cap as an ordinary (soft) configured limit. */
|
|
336
|
-
hardCaps?: boolean;
|
|
337
|
-
}
|
|
338
|
-
/**
|
|
339
|
-
* `routing.latency` — sustained MEASURED latency as a demotion term (owner decision 2026-08-30).
|
|
340
|
-
*
|
|
341
|
-
* **Default ON.** A candidate whose measured p95 exceeds `p95Ms`, over at least `minSamples`
|
|
342
|
-
* measurable samples, joins the cooling band instead of leading the walk. It is a demotion, so the
|
|
343
|
-
* worst case is a reorder: nothing is dropped and nothing is refused.
|
|
344
|
-
*
|
|
345
|
-
* `false` is the documented shorthand for `{ enabled: false }` and restores the pre-2026-08-30
|
|
346
|
-
* behaviour exactly. An object with no keys is legal and means the defaults — writing it down is
|
|
347
|
-
* documentation, not a behaviour change.
|
|
348
|
-
*
|
|
349
|
-
* The thresholds live in `src/latency-demotion.ts` beside the measurement that calibrated them.
|
|
350
|
-
*/
|
|
351
|
-
export interface LatencyDemotionConfig {
|
|
352
|
-
/** Default true. false disables latency demotion entirely. */
|
|
353
|
-
enabled?: boolean;
|
|
354
|
-
/** Fallback ceiling: measured absolute p95 in ms. Above it, the candidate is demoted. */
|
|
355
|
-
p95Ms?: number;
|
|
356
|
-
/**
|
|
357
|
-
* PRIMARY ceiling: measured p95 latency per OUTPUT TOKEN, in ms.
|
|
358
|
-
*
|
|
359
|
-
* Absolute latency cannot compare a probe with a generation - a probe asks for one token, a real
|
|
360
|
-
* request may produce hundreds and amortise the same fixed overhead. The per-token rate is the
|
|
361
|
-
* fair figure, so it is tested first, over real request samples only.
|
|
362
|
-
*/
|
|
363
|
-
msPerToken?: number;
|
|
364
|
-
/** Minimum measurable samples before latency may demote anything at all. */
|
|
365
|
-
minSamples?: number;
|
|
366
|
-
}
|
|
367
|
-
/**
|
|
368
|
-
* `routing.hedge` — start the NEXT candidate beside a slow in-flight attempt, instead of after it
|
|
369
|
-
* (owner proposal 2026-08-30; the four decisions are in
|
|
370
|
-
* `docs/hedged-attempts-design-2026-08-30.md` §7).
|
|
371
|
-
*
|
|
372
|
-
* **Default ON, free deployments only.** That is owner decision D1, taken against the
|
|
373
|
-
* recommendation of off-by-default. `assessCost()` treats an UNKNOWN price as paid, so the rule is
|
|
374
|
-
* fail-safe in the only direction that matters: a duplicate can never land on a deployment whose
|
|
375
|
-
* price this relay cannot establish. The stated cost is that hedging silently does not fire on many
|
|
376
|
-
* members whose prices are simply unpublished.
|
|
377
|
-
*
|
|
378
|
-
* ⚠ **Hedging DUPLICATES; every other term here only reorders.** `false` is the documented
|
|
379
|
-
* shorthand for `{ enabled: false }` and restores the pre-hedge behaviour exactly, byte for byte.
|
|
380
|
-
* An object with no keys is legal and means the defaults.
|
|
381
|
-
*
|
|
382
|
-
* The thresholds live in `src/hedge-trigger.ts`. ⚠ `margin` and `minSamples` are still PLACEHOLDERS
|
|
383
|
-
* awaiting calibration and say so at their definition — do not quote them as measurements.
|
|
384
|
-
* `msPerInputToken` IS calibrated (`scripts/calibrate-hedge-floor.mjs`, 2026-09-04) — see its own
|
|
385
|
-
* doc comment in `hedge-trigger.ts` for why the fit came back out of range and what shipped instead.
|
|
386
|
-
*/
|
|
387
|
-
export interface HedgeConfig {
|
|
388
|
-
/** Default true. false disables hedging entirely. */
|
|
389
|
-
enabled?: boolean;
|
|
390
|
-
/**
|
|
391
|
-
* The floor's flat component, in ms. Owner direction 2026-09-04: the floor is no longer flat on
|
|
392
|
-
* its own — see `msPerInputToken` — but this still bounds the SMALL-prompt case, where the
|
|
393
|
-
* size-scaled component is negligible. Without it a deployment with a tiny p90 is hedged on
|
|
394
|
-
* ordinary noise, and a fast pool duplicates almost every request.
|
|
395
|
-
*/
|
|
396
|
-
minFloorMs?: number;
|
|
397
|
-
/**
|
|
398
|
-
* LEGACY alias of `minFloorMs`, kept for backward compatibility — an operator config written
|
|
399
|
-
* before 2026-09-04 (`{"floorMs": 8000}`) keeps loading and keeps meaning exactly what it always
|
|
400
|
-
* meant: the floor never drops below 8000 ms. Honoured only when `minFloorMs` itself is absent;
|
|
401
|
-
* `resolveHedgeSettings` in `hedge-trigger.ts` is the ONE place that resolves the alias, so a new
|
|
402
|
-
* caller of that function can never re-decide the precedence.
|
|
403
|
-
*/
|
|
404
|
-
floorMs?: number;
|
|
405
|
-
/**
|
|
406
|
-
* The floor's size-scaled component, in ms per estimated INPUT token
|
|
407
|
-
* (`estimateRequestTokens` in `metadata.ts`). Without it a large prompt is hedged against the
|
|
408
|
-
* time it simply takes a healthy deployment to read the prompt, not against real slowness.
|
|
409
|
-
*/
|
|
410
|
-
msPerInputToken?: number;
|
|
411
|
-
/** How far past the expected time an attempt must run before a hedge starts. */
|
|
412
|
-
margin?: number;
|
|
413
|
-
/** Minimum samples before a measured statistic may set the bar instead of the floor. */
|
|
414
|
-
minSamples?: number;
|
|
415
|
-
}
|
|
416
|
-
/**
|
|
417
|
-
* Template for rendering a `relay` rung as a shelled-out CLI command.
|
|
418
|
-
*
|
|
419
|
-
* Same executable shape as a `cli` rung — the host runs `command` with `args`, applying `env`
|
|
420
|
-
* (string sets, `null` unsets) — with one extra placeholder: `{spec}`, replaced by the rung's
|
|
421
|
-
* routing spec so ONE template serves every pool and pinned model in the ladder.
|
|
422
|
-
*/
|
|
423
|
-
export interface CliLaneTemplate {
|
|
424
|
-
command: string;
|
|
425
|
-
/** Must contain `{spec}`; `{task}` too, on the same reasoning as a cli rung's args. */
|
|
426
|
-
args: string[];
|
|
427
|
-
/** Applied by the host when spawning. Placeholders are never substituted here. */
|
|
428
|
-
env?: Record<string, string | null>;
|
|
429
|
-
}
|
|
430
|
-
/** A fixed configured prefix followed by automatically discovered free models. */
|
|
431
|
-
export interface PoolPolicy {
|
|
432
|
-
preferred: string[];
|
|
433
|
-
include: "free";
|
|
434
|
-
/** Permanent user tombstones, applied to both the fixed prefix and discovered tail. */
|
|
435
|
-
exclude?: string[];
|
|
436
|
-
/** Optional evidence-aware effort band for the discovered tail. */
|
|
437
|
-
effort?: EffortLevel;
|
|
438
|
-
}
|
|
439
|
-
/**
|
|
440
|
-
* One rung of `routing.ladder`.
|
|
441
|
-
*
|
|
442
|
-
* `cli` rungs are executed by the HOST (the relay never spawns a process — it is a proxy, not a
|
|
443
|
-
* process supervisor); `relay` rungs are addressed through this proxy in the ordinary way.
|
|
444
|
-
* `quota` names the balance a rung draws on so that rungs sharing one are cooled down together
|
|
445
|
-
* and rungs that merely share a binary are not: one CLI can meter two model families against
|
|
446
|
-
* two independent balances, and treating those as one bucket would skip a live lane.
|
|
447
|
-
*/
|
|
448
|
-
export interface LadderRung {
|
|
449
|
-
id: string;
|
|
450
|
-
kind: "cli" | "relay";
|
|
451
|
-
/** Parked rungs stay visible in the ladder but are never auto-selected. Default true. */
|
|
452
|
-
enabled: boolean;
|
|
453
|
-
quota?: string;
|
|
454
|
-
note?: string;
|
|
455
|
-
/** cli rungs: the binary to run, and its args — one of which must contain the task placeholder. */
|
|
456
|
-
command?: string;
|
|
457
|
-
args?: string[];
|
|
458
|
-
/**
|
|
459
|
-
* cli rungs: environment the HOST applies when spawning the command. A string value sets the
|
|
460
|
-
* variable; `null` unsets an inherited one. Both directions are load-bearing for the lane this
|
|
461
|
-
* exists for — a `claude -p` child routed through this proxy: `ANTHROPIC_BASE_URL` must be SET
|
|
462
|
-
* (a terminal-spawned `claude` honours it even though Claude Desktop pins its own sessions to
|
|
463
|
-
* api.anthropic.com), and `CLAUDECODE`/`CLAUDE_CODE_SSE_PORT`/`CLAUDE_CODE_ENTRYPOINT`/
|
|
464
|
-
* `ANTHROPIC_API_KEY` must be UNSET or a child spawned from inside a Claude session inherits
|
|
465
|
-
* the parent's harness wiring and refuses to start cleanly. The task placeholder is never
|
|
466
|
-
* substituted here — env values are operator-authored routing, not task content.
|
|
467
|
-
*/
|
|
468
|
-
env?: Record<string, string | null>;
|
|
469
|
-
/** relay rungs: the spec to address (`pool/<name>`, `<provider>/<model>`, a provider name). */
|
|
470
|
-
spec?: string;
|
|
471
|
-
}
|
|
472
31
|
/** Reserved provider-namespace prefix for `pool/<name>` routing. */
|
|
473
32
|
export declare const POOL_PREFIX = "pool";
|
|
474
33
|
/** Reserved model name for auto-dispatch to the ladder's first ready relay rung. */
|
|
475
34
|
export declare const AUTO_MODEL = "auto";
|
|
476
|
-
/** The request headers needed for protocol-specific subagent markers. */
|
|
477
|
-
export type RequestHeaders = Readonly<Record<string, string | string[] | undefined>>;
|
|
478
35
|
/** True when this request is a marked Claude Code or local Codex child turn. */
|
|
479
36
|
export declare function isSubagentRequest(reqJson: unknown, headers?: RequestHeaders): boolean;
|
|
480
37
|
/**
|
|
@@ -533,95 +90,6 @@ export declare function anyOffloadEnabled(cfg: Pick<Config, "routing">): boolean
|
|
|
533
90
|
* fail and name what IS configured.
|
|
534
91
|
*/
|
|
535
92
|
export declare function subagentSpec(reqJson: unknown, model: string | null, cfg: Config, headers?: RequestHeaders, client?: string): string | null;
|
|
536
|
-
/** A request routed to a concrete provider + backend model. */
|
|
537
|
-
export interface ResolvedTarget {
|
|
538
|
-
provider: string;
|
|
539
|
-
base: string;
|
|
540
|
-
kind: Kind;
|
|
541
|
-
/** Real backend model id (required for openai; absent = anthropic passthrough). */
|
|
542
|
-
model?: string;
|
|
543
|
-
authEnv?: string;
|
|
544
|
-
/** Normalized non-secret credential slots for this provider. */
|
|
545
|
-
credentialSlots?: readonly CredentialSlot[];
|
|
546
|
-
/** Carried from the provider: whether the caller's own credential may travel to this target. */
|
|
547
|
-
credentialMode?: CredentialMode;
|
|
548
|
-
authHeader: AuthHeader;
|
|
549
|
-
timeoutMs: number;
|
|
550
|
-
/** Carried from the provider: inter-byte stall watchdog for streamed responses. */
|
|
551
|
-
stallTimeoutMs?: number;
|
|
552
|
-
/**
|
|
553
|
-
* RESOLVED outbound tool-call-id shape (`resolveToolCallIdMode`) — an explicit
|
|
554
|
-
* `compat.toolCallIds` or the labelled base-host default. Resolved here so the request mapper
|
|
555
|
-
* is handed a mode and never a provider identity to re-derive one from. Absent (a hand-built
|
|
556
|
-
* target) reads as `"preserve"`, which is the pre-2026-08-23 behaviour byte for byte.
|
|
557
|
-
*/
|
|
558
|
-
toolCallIds?: ToolCallIdMode;
|
|
559
|
-
/**
|
|
560
|
-
* RESOLVED thought-signature mode (`resolveThoughtSignatureMode`) — an explicit
|
|
561
|
-
* `compat.thoughtSignature` or the labelled base-host default. Resolved here for the same reason
|
|
562
|
-
* as `toolCallIds`: the request mapper is handed a mode, never a provider identity to sniff one
|
|
563
|
-
* from. Absent (a hand-built target) reads as `"none"` — the pre-2026-08-23 bytes exactly.
|
|
564
|
-
*/
|
|
565
|
-
thoughtSignature?: ThoughtSignatureMode;
|
|
566
|
-
}
|
|
567
|
-
export interface Config {
|
|
568
|
-
host: string;
|
|
569
|
-
port: number;
|
|
570
|
-
providers: Record<string, ProviderConfig>;
|
|
571
|
-
routing: Routing;
|
|
572
|
-
mode: Mode;
|
|
573
|
-
/** Explicit global reshaper override; otherwise repair reshapes on the resolved target itself. */
|
|
574
|
-
reshaper?: ReshaperConfig;
|
|
575
|
-
/**
|
|
576
|
-
* Ranked reshaper candidates, from `reshaper: { pool: "<name>" }`. Tried in order, so a single
|
|
577
|
-
* de-listed model cannot take repair down with it — the whole point of not pinning one model.
|
|
578
|
-
* `reshaper` is candidates[0] so every existing single-reshaper path keeps working unchanged.
|
|
579
|
-
*/
|
|
580
|
-
reshaperCandidates?: ReshaperConfig[];
|
|
581
|
-
/** Dynamic reshaper pool resolved lazily after its catalog-backed tail is materialized. */
|
|
582
|
-
reshaperPool?: {
|
|
583
|
-
name: string;
|
|
584
|
-
timeoutMs?: number;
|
|
585
|
-
};
|
|
586
|
-
repair: {
|
|
587
|
-
maxAttempts: number;
|
|
588
|
-
destructiveTools: string[];
|
|
589
|
-
};
|
|
590
|
-
/**
|
|
591
|
-
* Wall-clock ceiling (ms) on STARTING further failover attempts within one request's pool
|
|
592
|
-
* walk. The first two attempts are always allowed and an attempt already in flight is never
|
|
593
|
-
* aborted — the budget bounds the walk, not the answer. 0 disables. Absent ⇒ the server's
|
|
594
|
-
* DEFAULT_WALK_BUDGET_MS (45s).
|
|
595
|
-
*/
|
|
596
|
-
walkBudgetMs?: number;
|
|
597
|
-
/** Maximum inbound request-body size in bytes. Absent ⇒ 36 MiB. */
|
|
598
|
-
maxBodyBytes?: number;
|
|
599
|
-
log: {
|
|
600
|
-
level: "metadata" | "silent";
|
|
601
|
-
file: string | null;
|
|
602
|
-
maxBytes?: number;
|
|
603
|
-
};
|
|
604
|
-
/**
|
|
605
|
-
* Providers the onboarding nudge must stop asking about (`leave_me_alone` in config.json).
|
|
606
|
-
*
|
|
607
|
-
* Scope is deliberately narrow: it silences the "❌ Missing Key / 👉 get one here" prompt in
|
|
608
|
-
* `llm-relay onboard`, and nothing else. Suppressed providers still appear in `llm-relay keys`,
|
|
609
|
-
* in `/registry` and in every status surface — silencing a nudge is not hiding state, and a
|
|
610
|
-
* provider that vanished from the status commands would be undebuggable later.
|
|
611
|
-
*
|
|
612
|
-
* Names that match no known provider are legal (see `parseLeaveMeAlone`).
|
|
613
|
-
*/
|
|
614
|
-
leaveMeAlone?: string[];
|
|
615
|
-
/** Path this config was loaded from. Set by `loadConfig`; absent for hand-built test configs.
|
|
616
|
-
* Only consumer is the runtime offload toggle, which persists back to the same file. */
|
|
617
|
-
sourcePath?: string;
|
|
618
|
-
/**
|
|
619
|
-
* Non-fatal load-time problems (a provider disabled for an unset `${ENV}`, a pool member
|
|
620
|
-
* dropped with it). Present so startup can print them — a degraded config that boots
|
|
621
|
-
* silently is how you end up running on one provider without noticing.
|
|
622
|
-
*/
|
|
623
|
-
warnings?: string[];
|
|
624
|
-
}
|
|
625
93
|
/**
|
|
626
94
|
* Tools a repaired call is never allowed to name. THE single definition — the
|
|
627
95
|
* shipped config template, config.example.json and the docs all derive from or
|
|
@@ -674,26 +142,6 @@ export interface ConfigOverrides {
|
|
|
674
142
|
}
|
|
675
143
|
/** Load + validate a config file, failing loudly on anything unusable. */
|
|
676
144
|
export declare function loadConfig(path: string, overrides?: ConfigOverrides, keystoreOptions?: KeystoreOptions): Config;
|
|
677
|
-
/** `llm-relay mcp` settings — see the `mcp` field doc on `Routing`. */
|
|
678
|
-
export interface McpSettings {
|
|
679
|
-
/**
|
|
680
|
-
* Directories a caller-supplied `cwd` must sit under. Absent or empty ⇒ no bound beyond the
|
|
681
|
-
* directory existing.
|
|
682
|
-
*
|
|
683
|
-
* ⚠ Offered, never imposed. The MCP caller is already a trusted agent on the operator's own
|
|
684
|
-
* machine, and defaulting to a bound would make the tool useless for its stated purpose. Whether
|
|
685
|
-
* to narrow it is the operator's decision to record here, not this file's to assume.
|
|
686
|
-
*/
|
|
687
|
-
allowedRoots?: string[];
|
|
688
|
-
}
|
|
689
|
-
/** Background lane re-probing settings — see the `laneProbe` field doc on `Routing`. */
|
|
690
|
-
export interface LaneProbeSettings {
|
|
691
|
-
enabled: boolean;
|
|
692
|
-
/** Gate between quota probes of ONE dead bucket. A probe spends that lane's quota. */
|
|
693
|
-
quotaIntervalMs: number;
|
|
694
|
-
/** Gate between catalog re-probes of ONE lane. Metadata commands, no quota spent. */
|
|
695
|
-
catalogIntervalMs: number;
|
|
696
|
-
}
|
|
697
145
|
export declare const DEFAULT_LANE_PROBE: LaneProbeSettings;
|
|
698
146
|
/** Parse the legacy global switch or the independently keyed client-rule form. */
|
|
699
147
|
export declare function parseOffload(raw: unknown): OffloadConfig;
|
package/dist/config.js
CHANGED
|
@@ -5,8 +5,8 @@ import { CREDENTIAL_LABEL_PATTERN, makeCredentialId } from "./credential-id.js";
|
|
|
5
5
|
import { parseConfiguredLimits } from "./configured-limits.js";
|
|
6
6
|
import { providerCredentialSlots, resolveCredentialSlot, } from "./credential-fleet.js";
|
|
7
7
|
import { createKeystoreResolutionWalk, keystoreStatus, listEntries, } from "./keystore.js";
|
|
8
|
-
|
|
9
|
-
export
|
|
8
|
+
import { EFFORT_LEVELS, CLAUDE_TIER_NAMES, } from "./config-types.js";
|
|
9
|
+
export { EFFORT_LEVELS, CLAUDE_TIER_NAMES, } from "./config-types.js";
|
|
10
10
|
const EFFORT_LEVEL_SET = new Set(EFFORT_LEVELS);
|
|
11
11
|
const COMPAT_KEYS = ["toolCallIds", "thoughtSignature"];
|
|
12
12
|
const TOOL_CALL_ID_MODES = ["preserve", "strict9"];
|