llm-relay 0.62.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accounting-store-io.js +0 -33
- package/dist/accounting-store-io.js.map +1 -1
- package/dist/accounting-store-schema.js +0 -65
- package/dist/accounting-store-schema.js.map +1 -1
- package/dist/accounting-store.js +0 -195
- package/dist/accounting-store.js.map +1 -1
- package/dist/accounting.js +0 -72
- package/dist/accounting.js.map +1 -1
- package/dist/anthropic.js +0 -18
- package/dist/anthropic.js.map +1 -1
- package/dist/authEnv.js +0 -78
- package/dist/authEnv.js.map +1 -1
- package/dist/availability-snapshot.js +0 -58
- package/dist/availability-snapshot.js.map +1 -1
- package/dist/availability.js +0 -162
- package/dist/availability.js.map +1 -1
- package/dist/backend.js +1 -416
- package/dist/backend.js.map +1 -1
- package/dist/benchmarks.js +1 -63
- package/dist/benchmarks.js.map +1 -1
- package/dist/breaker-persistence.js +0 -59
- package/dist/breaker-persistence.js.map +1 -1
- package/dist/candidates.js +0 -64
- package/dist/candidates.js.map +1 -1
- package/dist/catalog.js +1 -140
- package/dist/catalog.js.map +1 -1
- package/dist/circuit-breaker.js +1 -90
- package/dist/circuit-breaker.js.map +1 -1
- package/dist/claude-hook.js +0 -81
- package/dist/claude-hook.js.map +1 -1
- package/dist/cli.d.ts +36 -0
- package/dist/cli.js +73 -730
- package/dist/cli.js.map +1 -1
- package/dist/config-edit.js +0 -11
- package/dist/config-edit.js.map +1 -1
- package/dist/config.d.ts +21 -0
- package/dist/config.js +22 -372
- package/dist/config.js.map +1 -1
- package/dist/configured-limits.js +0 -58
- package/dist/configured-limits.js.map +1 -1
- package/dist/context-limits.js +0 -97
- package/dist/context-limits.js.map +1 -1
- package/dist/control-authorization.js +0 -36
- package/dist/control-authorization.js.map +1 -1
- package/dist/cooldown-clear.js +0 -6
- package/dist/cooldown-clear.js.map +1 -1
- package/dist/credential-fleet.js +0 -5
- package/dist/credential-fleet.js.map +1 -1
- package/dist/credential-select.js +4 -30
- package/dist/credential-select.js.map +1 -1
- package/dist/dashboard-auth.js +0 -29
- package/dist/dashboard-auth.js.map +1 -1
- package/dist/dashboard-contract.js +0 -46
- package/dist/dashboard-contract.js.map +1 -1
- package/dist/dashboard-routes.js +0 -32
- package/dist/dashboard-routes.js.map +1 -1
- package/dist/dashboard-snapshot.js +0 -154
- package/dist/dashboard-snapshot.js.map +1 -1
- package/dist/dashboard-static.js +0 -22
- package/dist/dashboard-static.js.map +1 -1
- package/dist/delegate-gate/cast-necessity.js +1 -22
- package/dist/delegate-gate/cast-necessity.js.map +1 -1
- package/dist/delegate-gate/cli.js +0 -6
- package/dist/delegate-gate/cli.js.map +1 -1
- package/dist/delegate-gate/diff-parser.js +0 -32
- package/dist/delegate-gate/diff-parser.js.map +1 -1
- package/dist/delegate-gate/fix.js +0 -10
- package/dist/delegate-gate/fix.js.map +1 -1
- package/dist/delegate-gate/gate.js +0 -12
- package/dist/delegate-gate/gate.js.map +1 -1
- package/dist/delegate-gate/minimality.js +0 -38
- package/dist/delegate-gate/minimality.js.map +1 -1
- package/dist/delegate-gate/post-image.js +1 -13
- package/dist/delegate-gate/post-image.js.map +1 -1
- package/dist/delegate-gate/shared-state.js +0 -23
- package/dist/delegate-gate/shared-state.js.map +1 -1
- package/dist/delegate-gate/test-assertions.js +1 -54
- package/dist/delegate-gate/test-assertions.js.map +1 -1
- package/dist/delegate-gate/types.js +0 -11
- package/dist/delegate-gate/types.js.map +1 -1
- package/dist/dialect-stream.js +9 -37
- package/dist/dialect-stream.js.map +1 -1
- package/dist/dispatch-exhaustion-persistence.js +0 -35
- package/dist/dispatch-exhaustion-persistence.js.map +1 -1
- package/dist/dispatch.js +0 -269
- package/dist/dispatch.js.map +1 -1
- package/dist/documents.js +0 -92
- package/dist/documents.js.map +1 -1
- package/dist/dotenv.js +0 -22
- package/dist/dotenv.js.map +1 -1
- package/dist/dynamic-pools.js +0 -93
- package/dist/dynamic-pools.js.map +1 -1
- package/dist/emitSse.js +0 -64
- package/dist/emitSse.js.map +1 -1
- package/dist/executable-lookup.js +0 -59
- package/dist/executable-lookup.js.map +1 -1
- package/dist/hard-cap.js +1 -32
- package/dist/hard-cap.js.map +1 -1
- package/dist/host-routing.js +0 -40
- package/dist/host-routing.js.map +1 -1
- package/dist/installed-hosts.js +0 -71
- package/dist/installed-hosts.js.map +1 -1
- package/dist/json-shape.js +0 -19
- package/dist/json-shape.js.map +1 -1
- package/dist/kernel/contracts.js +0 -26
- package/dist/kernel/contracts.js.map +1 -1
- package/dist/kernel/request-lifecycle.js +0 -4
- package/dist/kernel/request-lifecycle.js.map +1 -1
- package/dist/key-checker.js +0 -115
- package/dist/key-checker.js.map +1 -1
- package/dist/key-import.js +0 -20
- package/dist/key-import.js.map +1 -1
- package/dist/keys-cli.js +0 -15
- package/dist/keys-cli.js.map +1 -1
- package/dist/keystore.js +2 -51
- package/dist/keystore.js.map +1 -1
- package/dist/lane-cadence.js +1 -51
- package/dist/lane-cadence.js.map +1 -1
- package/dist/lane-manifest.js +0 -54
- package/dist/lane-manifest.js.map +1 -1
- package/dist/lane-probe.js +0 -36
- package/dist/lane-probe.js.map +1 -1
- package/dist/lane-quota-probe.js +0 -61
- package/dist/lane-quota-probe.js.map +1 -1
- package/dist/log.js +0 -17
- package/dist/log.js.map +1 -1
- package/dist/mcp/lane-runner.d.ts +114 -0
- package/dist/mcp/lane-runner.js +186 -0
- package/dist/mcp/lane-runner.js.map +1 -0
- package/dist/mcp/protocol.d.ts +98 -0
- package/dist/mcp/protocol.js +67 -0
- package/dist/mcp/protocol.js.map +1 -0
- package/dist/mcp/server.d.ts +78 -0
- package/dist/mcp/server.js +360 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/metadata.js +0 -73
- package/dist/metadata.js.map +1 -1
- package/dist/network-block.js +0 -45
- package/dist/network-block.js.map +1 -1
- package/dist/offload.js +0 -21
- package/dist/offload.js.map +1 -1
- package/dist/onboarding.js +0 -36
- package/dist/onboarding.js.map +1 -1
- package/dist/openai-dialect.js +0 -32
- package/dist/openai-dialect.js.map +1 -1
- package/dist/openai-request.js +0 -236
- package/dist/openai-request.js.map +1 -1
- package/dist/os-keyring.js +0 -28
- package/dist/os-keyring.js.map +1 -1
- package/dist/ping/cadence.js +3 -90
- package/dist/ping/cadence.js.map +1 -1
- package/dist/ping/metrics.js +0 -91
- package/dist/ping/metrics.js.map +1 -1
- package/dist/ping/ping.js +0 -17
- package/dist/ping/ping.js.map +1 -1
- package/dist/ping/probe-cache.js +2 -53
- package/dist/ping/probe-cache.js.map +1 -1
- package/dist/ping/quota.js +0 -17
- package/dist/ping/quota.js.map +1 -1
- package/dist/ping/runtime-telemetry.js +1 -18
- package/dist/ping/runtime-telemetry.js.map +1 -1
- package/dist/pool-health.js +0 -49
- package/dist/pool-health.js.map +1 -1
- package/dist/presets.js +0 -4
- package/dist/presets.js.map +1 -1
- package/dist/process-safety-net.js +0 -36
- package/dist/process-safety-net.js.map +1 -1
- package/dist/quota-demotion.js +1 -35
- package/dist/quota-demotion.js.map +1 -1
- package/dist/quota-observation.js +0 -21
- package/dist/quota-observation.js.map +1 -1
- package/dist/rate-limits.js +1 -94
- package/dist/rate-limits.js.map +1 -1
- package/dist/refusal-interpretation.js +2 -327
- package/dist/refusal-interpretation.js.map +1 -1
- package/dist/registry.js +0 -11
- package/dist/registry.js.map +1 -1
- package/dist/repair.js +1 -119
- package/dist/repair.js.map +1 -1
- package/dist/request-log.js +0 -2
- package/dist/request-log.js.map +1 -1
- package/dist/reshaper.js +0 -85
- package/dist/reshaper.js.map +1 -1
- package/dist/resolved-attempt.js +0 -3
- package/dist/resolved-attempt.js.map +1 -1
- package/dist/responses-request.js +0 -187
- package/dist/responses-request.js.map +1 -1
- package/dist/routes/admin.js +0 -38
- package/dist/routes/admin.js.map +1 -1
- package/dist/secret-file-acl.js +0 -26
- package/dist/secret-file-acl.js.map +1 -1
- package/dist/self-update.js +0 -118
- package/dist/self-update.js.map +1 -1
- package/dist/server.js +15 -952
- package/dist/server.js.map +1 -1
- package/dist/session-pin.js +0 -9
- package/dist/session-pin.js.map +1 -1
- package/dist/setup-claude.js +0 -9
- package/dist/setup-claude.js.map +1 -1
- package/dist/spend-headroom.js +0 -20
- package/dist/spend-headroom.js.map +1 -1
- package/dist/sse-frames.js +0 -31
- package/dist/sse-frames.js.map +1 -1
- package/dist/sse.js +1 -19
- package/dist/sse.js.map +1 -1
- package/dist/state-paths.js +0 -16
- package/dist/state-paths.js.map +1 -1
- package/dist/stream-commit.js +1 -27
- package/dist/stream-commit.js.map +1 -1
- package/dist/target-facts.js +2 -130
- package/dist/target-facts.js.map +1 -1
- package/dist/telemetry.js +0 -8
- package/dist/telemetry.js.map +1 -1
- package/dist/think-tags.js +1 -38
- package/dist/think-tags.js.map +1 -1
- package/dist/tier-data.js +0 -29
- package/dist/tier-data.js.map +1 -1
- package/dist/tool-dialects.js +1 -103
- package/dist/tool-dialects.js.map +1 -1
- package/dist/tool-use-ids.js +0 -51
- package/dist/tool-use-ids.js.map +1 -1
- package/dist/usage-observer.js +0 -52
- package/dist/usage-observer.js.map +1 -1
- package/dist/validator.js +1 -25
- package/dist/validator.js.map +1 -1
- package/dist/winenv.js +0 -63
- package/dist/winenv.js.map +1 -1
- package/dist/write-behind.js +0 -12
- package/dist/write-behind.js.map +1 -1
- package/package.json +2 -2
- package/skills/llm-relay/SKILL.md +26 -1
package/dist/server.js
CHANGED
|
@@ -57,9 +57,6 @@ const INTERNAL_REQUEST_HEADERS = new Set([
|
|
|
57
57
|
CONTROL_AUTHORIZATION_HEADER,
|
|
58
58
|
]);
|
|
59
59
|
const INBOUND_AUTH = ["authorization", "x-api-key"];
|
|
60
|
-
// `MAX_TASK_LEN` lived here until the admin routes moved to `routes/admin.ts`, which owns the
|
|
61
|
-
// `?task=` bound now. Two copies of a limit drift; the one at the boundary that accepts the
|
|
62
|
-
// request is the one that counts.
|
|
63
60
|
export const TOKENLESS_CONTROL_READ_PATHS = Object.freeze([
|
|
64
61
|
"/v1/models",
|
|
65
62
|
"/models",
|
|
@@ -77,22 +74,6 @@ const CONTROL_ROUTES = new Set([
|
|
|
77
74
|
"/health",
|
|
78
75
|
"/candidates",
|
|
79
76
|
]);
|
|
80
|
-
/**
|
|
81
|
-
* Request admission for mutating and command-rendering routes.
|
|
82
|
-
*
|
|
83
|
-
* Binding to loopback is NOT authorization. Any web page the user visits can
|
|
84
|
-
* issue a cross-origin POST to 127.0.0.1, and because the handler JSON-parses
|
|
85
|
-
* whatever body arrives regardless of declared content type, a `text/plain` POST
|
|
86
|
-
* is a CORS *simple request* — no preflight, and it succeeds. The attacker never
|
|
87
|
-
* reads the response, but every interesting operation here is a WRITE: flipping
|
|
88
|
-
* offload routing, rewriting config.json on disk, marking dispatch lanes spent,
|
|
89
|
-
* or spending provider keys.
|
|
90
|
-
*
|
|
91
|
-
* Returns null when the request may proceed, or a reason to reject with 403.
|
|
92
|
-
* A CLI sends no Origin, so an ABSENT Origin is allowed; a present-but-unknown
|
|
93
|
-
* one is not. Host is checked against the loopback names to close DNS rebinding,
|
|
94
|
-
* where a hostile name resolves to 127.0.0.1 and thus looks local to the socket.
|
|
95
|
-
*/
|
|
96
77
|
function admissionFailure(req, pathname, server, cfg, authorization) {
|
|
97
78
|
const expected = listenerAuthority(server, cfg);
|
|
98
79
|
if (!expected)
|
|
@@ -104,8 +85,6 @@ function admissionFailure(req, pathname, server, cfg, authorization) {
|
|
|
104
85
|
if (rawHostValues.length !== 1 || !host || host.hostname !== expected.hostname || host.port !== expected.port) {
|
|
105
86
|
return "Host authority does not match the bound listener";
|
|
106
87
|
}
|
|
107
|
-
// Header presence is significant: Origin: null and Origin: "" are present-invalid,
|
|
108
|
-
// rather than falling into the trusted non-browser/CLI path.
|
|
109
88
|
if (Object.hasOwn(req.headers, "origin")) {
|
|
110
89
|
const origin = parseOrigin(req.headers.origin);
|
|
111
90
|
if (!origin || origin.scheme !== "http:" || origin.hostname !== expected.hostname || origin.port !== expected.port) {
|
|
@@ -115,8 +94,6 @@ function admissionFailure(req, pathname, server, cfg, authorization) {
|
|
|
115
94
|
const mutating = req.method !== "GET" && req.method !== "HEAD";
|
|
116
95
|
if (mutating) {
|
|
117
96
|
const ct = (req.headers["content-type"] ?? "").toString().split(";")[0]?.trim().toLowerCase();
|
|
118
|
-
// Requiring application/json is what makes a body-bearing cross-origin POST
|
|
119
|
-
// need a preflight, which a hostile page cannot satisfy.
|
|
120
97
|
if (ct !== "application/json") {
|
|
121
98
|
return `mutating requests require content-type: application/json (got ${ct || "none"})`;
|
|
122
99
|
}
|
|
@@ -174,9 +151,7 @@ function listenerAuthority(server, cfg) {
|
|
|
174
151
|
return { hostname: normalizeHostname(configuredHost), port: address.port };
|
|
175
152
|
}
|
|
176
153
|
const MAX_VALIDATE_BYTES = 8 * 1024 * 1024;
|
|
177
|
-
/** 25 MiB decoded document × base64 expansion, plus JSON-envelope headroom. */
|
|
178
154
|
export const DEFAULT_MAX_BODY_BYTES = 36 * 1024 * 1024;
|
|
179
|
-
/** A bare programmatic proxy has no persistence owner, so dashboard reads fail closed as empty. */
|
|
180
155
|
const UNAVAILABLE_ACCOUNTING_READER = Object.freeze({
|
|
181
156
|
readDay: () => ({ status: "missing", value: null }),
|
|
182
157
|
readDays: () => ({
|
|
@@ -196,55 +171,28 @@ export function createProxy(cfg, deps = {}) {
|
|
|
196
171
|
const logger = new MetadataLogger(cfg.log);
|
|
197
172
|
const isDestructive = destructiveMatcher(cfg.repair.destructiveTools);
|
|
198
173
|
const catalog = deps.catalog ?? new ModelCatalog();
|
|
199
|
-
/**
|
|
200
|
-
* The lane cadence rides the ping tick (owner decision 2026-08-29): catalog re-probes keep
|
|
201
|
-
* rosters fresh, and quota probes re-test recorded lane deaths so no lane stays parked on a
|
|
202
|
-
* stale record. Skipped under vitest — the suite exercises `LaneCadence` directly with
|
|
203
|
-
* injected seams, and a test proxy must never spawn a real lane.
|
|
204
|
-
*/
|
|
205
174
|
const laneCadence = process.env.VITEST ? null : new LaneCadence(cfg);
|
|
206
175
|
const pingLoop = deps.pingLoop ??
|
|
207
176
|
new PingLoop(cfg, catalog, laneCadence ? { onTick: (now) => laneCadence.poke(now) } : {});
|
|
208
177
|
const breaker = deps.breaker ?? new CircuitBreaker();
|
|
209
|
-
/**
|
|
210
|
-
* Cooling state survives a restart, like ping health already did.
|
|
211
|
-
*
|
|
212
|
-
* Without this the relay discarded every cooldown and the whole unexplained-429 escalation
|
|
213
|
-
* ladder on restart, then walked back into the same walls — measured here as a 19.9-hour
|
|
214
|
-
* cooldown learned from 7 consecutive 429s on one deployment and 26 on another.
|
|
215
|
-
*
|
|
216
|
-
* ⚠ Skipped under vitest for the reason the probe loop is: a test proxy must not read or write
|
|
217
|
-
* the developer's live cooling state. `getBreakerStatePath()` also redirects under vitest, so
|
|
218
|
-
* this is belt and braces — the suite exercises the mechanism directly with an explicit path.
|
|
219
|
-
*/
|
|
220
178
|
if (!process.env.VITEST)
|
|
221
179
|
installBreakerPersistence(breaker);
|
|
222
|
-
// The ladder's exhaustion state survives a restart for the same reason — the report route
|
|
223
|
-
// accepts vendor-stated cooldowns up to 30 days, exactly the rows worth keeping.
|
|
224
180
|
if (!process.env.VITEST)
|
|
225
181
|
installDispatchExhaustionPersistence(cfg);
|
|
226
|
-
// Process-local and deliberately credential-wide: a deployment switch must not reset fairness.
|
|
227
182
|
const credentialLru = new CredentialLru();
|
|
228
183
|
const modelCallRecorder = deps.modelCallRecorder ?? (process.env.VITEST ? undefined : recordModelCall);
|
|
229
184
|
const accountingRecorder = deps.accountingRecorder ?? NOOP_ACCOUNTING_RECORDER;
|
|
230
|
-
// One port per proxy: both request fronts price through the SAME published figures.
|
|
231
185
|
const accountingPricePort = deps.accountingPricePortOverride ?? buildAccountingPricePort(catalog, cfg);
|
|
232
|
-
// Do not create a second persistence store here. The CLI owns the production store lifecycle
|
|
233
|
-
// and supplies the same object as recorder and reader; a bare in-memory proxy reports no data.
|
|
234
186
|
const dashboardAuth = new DashboardAuthManager();
|
|
235
187
|
const dashboardAssetRoot = deps.dashboardAssetRoot ?? getProductionDashboardAssetRoot();
|
|
236
188
|
const dashboardStatic = new DashboardStaticHandler({
|
|
237
189
|
assetRoot: dashboardAssetRoot,
|
|
238
|
-
// Vite writes this only at build time. Source-driven runs therefore fail closed for assets
|
|
239
|
-
// rather than discovering or probing a development server on each request.
|
|
240
190
|
manifestPath: join(dashboardAssetRoot, ".vite", "manifest.json"),
|
|
241
191
|
});
|
|
242
192
|
const dashboardRead = createDashboardSnapshotReadPort({
|
|
243
193
|
accounting: deps.accountingReader ?? UNAVAILABLE_ACCOUNTING_READER,
|
|
244
194
|
relayVersion: deps.dashboardRelayVersion ?? "unknown",
|
|
245
195
|
attributionPolicy: deps.dashboardAttributionPolicy ?? "unknown",
|
|
246
|
-
// The Quota/Cooldown panels' producer. `usedInWindow` is store-only, so a bare in-memory
|
|
247
|
-
// proxy passes null and its quota rows simply carry localUsed null.
|
|
248
196
|
availability: createAvailabilityProducer({
|
|
249
197
|
breaker,
|
|
250
198
|
config: cfg,
|
|
@@ -257,10 +205,6 @@ export function createProxy(cfg, deps = {}) {
|
|
|
257
205
|
const stickySessions = stickyConfig === true || (typeof stickyConfig === "object" && stickyConfig.enabled)
|
|
258
206
|
? new StickySessionManager(typeof stickyConfig === "object" ? stickyConfig : undefined)
|
|
259
207
|
: undefined;
|
|
260
|
-
// Gap 12: the deployment-level quota demotion term. Built once per proxy; each call re-reads
|
|
261
|
-
// live breaker state and the store's in-memory window. `deps.accountingReader` is narrowed the
|
|
262
|
-
// same way the dashboard producer above narrows it, so a bare programmatic proxy simply has no
|
|
263
|
-
// localUsed figure and the term stays inert.
|
|
264
208
|
const quotaDemotion = createQuotaDemotionFn({
|
|
265
209
|
cfg,
|
|
266
210
|
breaker,
|
|
@@ -268,18 +212,6 @@ export function createProxy(cfg, deps = {}) {
|
|
|
268
212
|
? deps.accountingReader
|
|
269
213
|
: null,
|
|
270
214
|
});
|
|
271
|
-
// The cost class of a deployment, resolved LIVE from the catalog, for facts that apply to only
|
|
272
|
-
// one class. Built once per proxy like `quotaDemotion` above, and read fresh on every call so it
|
|
273
|
-
// tracks the catalog's own 10-minute refresh.
|
|
274
|
-
//
|
|
275
|
-
// ⚠ Why this exists: OpenRouter's `403 Key limit exceeded (weekly limit)` names the KEY, but it
|
|
276
|
-
// is a SPEND limit — its surface is the PAID subset. A credential-scoped demotion took out 18
|
|
277
|
-
// free models that were answering 200 (measured 2026-08-28). A `group` scope with a member list
|
|
278
|
-
// cannot fix it either, because a provider moves models between free, discounted and paid on its
|
|
279
|
-
// own schedule. Referencing the CLASSIFIER rather than a list is what cannot go stale.
|
|
280
|
-
//
|
|
281
|
-
// ⚠ Anthropic passthrough is definitionally not free — it spends primary quota — so it reports
|
|
282
|
-
// `paid` rather than a guess, exactly as the `targetCandidates` assessment above does.
|
|
283
215
|
const costClassOf = (attempt) => {
|
|
284
216
|
const t = attempt.target;
|
|
285
217
|
if (t.kind !== "openai" || !t.model)
|
|
@@ -288,22 +220,9 @@ export function createProxy(cfg, deps = {}) {
|
|
|
288
220
|
return assessCost(t.model, catalog.cachedLimits(t.provider, t.model), cfg.providers[t.provider]?.tierType).costClass;
|
|
289
221
|
}
|
|
290
222
|
catch {
|
|
291
|
-
// Unknown class ⇒ a cost-filtered fact simply does not apply. Never demote on a
|
|
292
|
-
// classification we could not make.
|
|
293
223
|
return undefined;
|
|
294
224
|
}
|
|
295
225
|
};
|
|
296
|
-
// G2: the operator-set refusal ceilings. Narrowed to the SAME in-memory `usedInWindow` seam the
|
|
297
|
-
// demotion term reads — a bare programmatic proxy has no ledger and its caps simply never fire
|
|
298
|
-
// (unknown usage ⇒ no refusal). Built once per proxy; each call re-reads the live window.
|
|
299
|
-
//
|
|
300
|
-
// ⚠ The read is NARROWED BY THE CAP'S OWN SCOPE, which `evaluateHardCap` states per call: a
|
|
301
|
-
// `limits.models.<id>.hard` ceiling is compared against that deployment's usage, a flat
|
|
302
|
-
// credential/provider ceiling against the credential's usage across every model. Passing one
|
|
303
|
-
// fixed scope is how enforcement and `/candidates` drifted apart — they shared the evaluator
|
|
304
|
-
// but asked the ledger different questions, so a per-deployment cap refused `m/x` for requests
|
|
305
|
-
// spent entirely on `m/y`. All three consumers now narrow on exactly this rule (see
|
|
306
|
-
// `candidates.ts` and `availability-snapshot.ts`).
|
|
307
226
|
const hardCapEvaluator = (attempt, now) => {
|
|
308
227
|
try {
|
|
309
228
|
const parsed = parseCredentialId(attempt.credentialId);
|
|
@@ -318,7 +237,6 @@ export function createProxy(cfg, deps = {}) {
|
|
|
318
237
|
});
|
|
319
238
|
}
|
|
320
239
|
catch {
|
|
321
|
-
// A routing guard must not be able to fail a request; degrade to no opinion.
|
|
322
240
|
return null;
|
|
323
241
|
}
|
|
324
242
|
};
|
|
@@ -331,14 +249,9 @@ export function createProxy(cfg, deps = {}) {
|
|
|
331
249
|
controlAuthorization = createControlAuthorization(resolveControlAuthorizationConfigDir(cfg.sourcePath));
|
|
332
250
|
}
|
|
333
251
|
catch {
|
|
334
|
-
// Data-plane and tokenless status reads remain available. Protected control
|
|
335
|
-
// work fails closed through the absent port instead of preventing startup.
|
|
336
252
|
controlAuthorization = undefined;
|
|
337
253
|
}
|
|
338
254
|
}
|
|
339
|
-
// Secret-bearing reshapers are request-local. Provider-backed forms are rehydrated from the
|
|
340
|
-
// provider config at the repair decision so current fleet slots and current env snapshots bind
|
|
341
|
-
// the egress. The implicit form instead keeps the exact credential that served the bad answer.
|
|
342
255
|
const providerBackedReshaper = (targets, now) => new CredentialWalkReshaper(expandCredentialAttempts(targets), {
|
|
343
256
|
lru: credentialLru,
|
|
344
257
|
walkBudgetMs: cfg.walkBudgetMs ?? DEFAULT_WALK_BUDGET_MS,
|
|
@@ -380,8 +293,6 @@ export function createProxy(cfg, deps = {}) {
|
|
|
380
293
|
return providerBackedReshaper(targets, now);
|
|
381
294
|
}
|
|
382
295
|
if (cfg.reshaper) {
|
|
383
|
-
// A legacy standalone endpoint owns only its declared authEnv. It has no provider identity,
|
|
384
|
-
// so never infer one from a coincidentally matching base/model and never attach a fleet.
|
|
385
296
|
return new HttpReshaper(cfg.reshaper, resolveCredential(cfg.reshaper.authEnv));
|
|
386
297
|
}
|
|
387
298
|
const spec = reshaperForTarget(servedAttempt.target);
|
|
@@ -416,30 +327,11 @@ export function createProxy(cfg, deps = {}) {
|
|
|
416
327
|
hardCap: hardCapEvaluator,
|
|
417
328
|
}).catch((e) => {
|
|
418
329
|
failClosed(res, 502, `llm-relay internal error: ${e.message}`);
|
|
419
|
-
// Last-resort net. `handle` logs every turn it terminates itself, so reaching
|
|
420
|
-
// here means a turn ended with NO log record — the operator would see a client
|
|
421
|
-
// error with nothing at all in the log to match it against. A duplicate line
|
|
422
|
-
// in some future edge case is much cheaper than an invisible request.
|
|
423
330
|
logger.write(baseLog(started, req.url ?? "/", false, false, 502, "skipped", null));
|
|
424
331
|
});
|
|
425
332
|
});
|
|
426
|
-
/**
|
|
427
|
-
* Actually run the background health loop.
|
|
428
|
-
*
|
|
429
|
-
* ⚠ `PingLoop.start()` was called from NOWHERE in `src/`. The class has a complete adaptive
|
|
430
|
-
* cadence — speed/normal/slow modes, idle detection, `noteUserActivity()` — and every bit of it
|
|
431
|
-
* was dead code, so the only probes ever recorded were the ones a human triggered by hitting
|
|
432
|
-
* `/ping` by hand. That is the real reason a long-running relay reported `verdict: Pending` and
|
|
433
|
-
* an empty `p95` for every model it had supposedly been monitoring for days: nothing was
|
|
434
|
-
* monitoring anything. Persisting samples (probe-cache) and rehydrating them (cadence) are both
|
|
435
|
-
* pointless without this line.
|
|
436
|
-
*
|
|
437
|
-
* Skipped under vitest: a test proxy must not start probing real providers in the background.
|
|
438
|
-
*/
|
|
439
333
|
if (!process.env.VITEST) {
|
|
440
334
|
server.on("listening", () => {
|
|
441
|
-
// Give the probe loop a concrete routing roster before its first tick. On a warm restart
|
|
442
|
-
// this is immediate from the catalog cache; a cold network refresh expands it later.
|
|
443
335
|
materializeDynamicPools(cfg, catalog);
|
|
444
336
|
pingLoop.start();
|
|
445
337
|
});
|
|
@@ -456,12 +348,8 @@ function isCallerVisibleAccountingPath(method, pathname) {
|
|
|
456
348
|
|| pathname === "/v1/responses"
|
|
457
349
|
|| pathname === "/responses";
|
|
458
350
|
}
|
|
459
|
-
/** Records caller-visible terminals that occur before parsed-request accounting exists. */
|
|
460
351
|
function recordEarlyTerminalAccounting(recorder, startedAt, client) {
|
|
461
352
|
try {
|
|
462
|
-
// failureKind "unknown", not "protocol": the enum's only protocol kind means a
|
|
463
|
-
// PROVIDER answered with a malformed envelope, and this request never left the
|
|
464
|
-
// relay — nothing reached any provider to be protocol about.
|
|
465
353
|
createAccountingRequest({ recorder, startedAt, client }).complete({
|
|
466
354
|
outcome: "error",
|
|
467
355
|
failureKind: "unknown",
|
|
@@ -470,7 +358,6 @@ function recordEarlyTerminalAccounting(recorder, startedAt, client) {
|
|
|
470
358
|
});
|
|
471
359
|
}
|
|
472
360
|
catch {
|
|
473
|
-
// Accounting is observational and must never change caller traffic.
|
|
474
361
|
}
|
|
475
362
|
}
|
|
476
363
|
function serveAccountingAttribution(attempt) {
|
|
@@ -512,16 +399,6 @@ function accountingTokens(usage, estimatedInputTokens, includeEstimatedOutput) {
|
|
|
512
399
|
: {}),
|
|
513
400
|
};
|
|
514
401
|
}
|
|
515
|
-
/**
|
|
516
|
-
* Build the published-price lookup the accounting lifecycle prices spend through.
|
|
517
|
-
*
|
|
518
|
-
* Request-path safe by construction: `cachedLimits()` reads the in-memory catalog
|
|
519
|
-
* cache and NEVER fetches (same contract as the context guardrail's lookup), and the
|
|
520
|
-
* synced capability snapshot is memoized on its file mtime. A deployment that
|
|
521
|
-
* publishes no price resolves to null and stays UNPRICED — never priced at a
|
|
522
|
-
* fallback or a default. The reference rung uses the snapshot for the same model id,
|
|
523
|
-
* labelled `reference`, exactly as `/candidates` already labels it.
|
|
524
|
-
*/
|
|
525
402
|
function buildAccountingPricePort(catalog, cfg) {
|
|
526
403
|
return (provider, model) => {
|
|
527
404
|
const p = cfg.providers[provider];
|
|
@@ -532,9 +409,6 @@ function buildAccountingPricePort(catalog, cfg) {
|
|
|
532
409
|
const matched = data
|
|
533
410
|
? findTierModel(model, data.byNorm, data.exactByNorm)
|
|
534
411
|
: null;
|
|
535
|
-
// A FUZZY match borrows a different SKU's row (`glm-5.2` → `glm-5.2-max`). A
|
|
536
|
-
// borrowed score mis-ranks a pool; a borrowed PRICE would mis-state spend — so
|
|
537
|
-
// this path takes the stricter exact-only rule, like `contextWindowResolver`.
|
|
538
412
|
const tier = matched?.match === "exact" ? matched.rec : undefined;
|
|
539
413
|
const meta = resolveMetadata(model, {
|
|
540
414
|
providerLimits,
|
|
@@ -555,11 +429,6 @@ function buildAccountingPricePort(catalog, cfg) {
|
|
|
555
429
|
};
|
|
556
430
|
};
|
|
557
431
|
}
|
|
558
|
-
/**
|
|
559
|
-
* Request accounting is intentionally separate from health state: health owns
|
|
560
|
-
* routing decisions, while this helper only observes actual egress and the
|
|
561
|
-
* downstream response lifetime. Every method is fail-open for proxy traffic.
|
|
562
|
-
*/
|
|
563
432
|
class RequestAccountingState {
|
|
564
433
|
estimatedInputTokens;
|
|
565
434
|
pricePort;
|
|
@@ -602,20 +471,6 @@ class RequestAccountingState {
|
|
|
602
471
|
credentialId: attempt.credentialId,
|
|
603
472
|
});
|
|
604
473
|
}
|
|
605
|
-
/**
|
|
606
|
-
* CONTRACT: the returned handle MUST be completed exactly once, on every exit
|
|
607
|
-
* path (success, refusal, transport error, cancellation). Only `complete()`
|
|
608
|
-
* removes an entry from `active`, and `finalizeIfReady()` refuses to finish the
|
|
609
|
-
* request while `active` is non-empty — so a custom `Reshaper` that starts an
|
|
610
|
-
* attempt and drops the handle stalls this request's `request-completed` event
|
|
611
|
-
* until the store LRU-evicts it, and the day aggregate silently misses the turn.
|
|
612
|
-
*
|
|
613
|
-
* Deliberately NO eager sweep here: once the caller response is terminal, a
|
|
614
|
-
* still-active attempt is usually a REAL repair in flight being aborted by the
|
|
615
|
-
* disconnect, and its own `complete(cancelled/aborted)` lands a few ticks later.
|
|
616
|
-
* Sweeping at finalize time races that and would relabel honest cancellations as
|
|
617
|
-
* `error/unknown`; the in-tree reshapers all complete in `finally`.
|
|
618
|
-
*/
|
|
619
474
|
startRepair(options) {
|
|
620
475
|
return this.start("repair", {
|
|
621
476
|
startedAt: options.startedAt,
|
|
@@ -636,16 +491,12 @@ class RequestAccountingState {
|
|
|
636
491
|
}
|
|
637
492
|
}
|
|
638
493
|
catch {
|
|
639
|
-
// Accounting must not perturb a successful response write.
|
|
640
494
|
}
|
|
641
495
|
}
|
|
642
496
|
complete(attempt, outcome, failureKind, usage, endedAt = Date.now()) {
|
|
643
497
|
if (attempt === null)
|
|
644
498
|
return;
|
|
645
499
|
try {
|
|
646
|
-
// The backend observer also sees a provisional prefix that stream-commit may
|
|
647
|
-
// withhold for failover. Only a committed serve attempt actually forwarded it;
|
|
648
|
-
// repair output is consumed internally and remains independently metered.
|
|
649
500
|
attempt.complete({
|
|
650
501
|
outcome,
|
|
651
502
|
failureKind,
|
|
@@ -654,7 +505,6 @@ class RequestAccountingState {
|
|
|
654
505
|
});
|
|
655
506
|
}
|
|
656
507
|
catch {
|
|
657
|
-
// The recorder and its packets are strictly observational.
|
|
658
508
|
}
|
|
659
509
|
this.active.delete(attempt);
|
|
660
510
|
this.committed.delete(attempt);
|
|
@@ -685,20 +535,6 @@ class RequestAccountingState {
|
|
|
685
535
|
if (this.finalized || this.request === null || this.responseTerminal === null)
|
|
686
536
|
return;
|
|
687
537
|
if (this.active.size > 0) {
|
|
688
|
-
// Two very different reasons for a still-active attempt:
|
|
689
|
-
//
|
|
690
|
-
// - On "close" (client disconnect mid-turn) a REAL repair is usually still in
|
|
691
|
-
// flight, being aborted by that same disconnect; its own
|
|
692
|
-
// complete("cancelled"/"aborted") lands within a few ticks. Returning here
|
|
693
|
-
// lets that honest completion happen instead of relabelling it.
|
|
694
|
-
// - On "finish" the handler has fully written, so every in-tree reshaper has
|
|
695
|
-
// long since completed its handle in `finally`. Anything still active is a
|
|
696
|
-
// DROPPED handle from a custom Reshaper that ignored the startRepair
|
|
697
|
-
// contract; sweeping it keeps the request from stalling unfinalized until
|
|
698
|
-
// store eviction. Recorded as error/unknown — the one honest statement,
|
|
699
|
-
// since nothing observed how the abandoned egress ended. A late real
|
|
700
|
-
// completion is harmlessly ignored: complete() is idempotent and this
|
|
701
|
-
// object is finalized immediately after.
|
|
702
538
|
if (this.responseTerminal !== "finished")
|
|
703
539
|
return;
|
|
704
540
|
for (const attempt of [...this.active]) {
|
|
@@ -726,7 +562,6 @@ class RequestAccountingState {
|
|
|
726
562
|
}
|
|
727
563
|
}
|
|
728
564
|
catch {
|
|
729
|
-
// A malformed accounting packet must never affect the caller response.
|
|
730
565
|
}
|
|
731
566
|
}
|
|
732
567
|
}
|
|
@@ -766,7 +601,6 @@ function dashboardHeaders(req) {
|
|
|
766
601
|
}
|
|
767
602
|
return headers;
|
|
768
603
|
}
|
|
769
|
-
/** Canonical dashboard origin synthesis, including the brackets required by IPv6 URLs. */
|
|
770
604
|
export function dashboardExpectedOriginForAuthority(hostname, port) {
|
|
771
605
|
const host = hostname.includes(":") ? `[${hostname}]` : hostname;
|
|
772
606
|
return `http://${host}:${port}`;
|
|
@@ -777,14 +611,8 @@ function dashboardExpectedOrigin(server, cfg) {
|
|
|
777
611
|
return null;
|
|
778
612
|
return dashboardExpectedOriginForAuthority(authority.hostname, authority.port);
|
|
779
613
|
}
|
|
780
|
-
/**
|
|
781
|
-
* Treat percent spellings of the dashboard namespace as dashboard requests even when malformed.
|
|
782
|
-
* They are never canonicalized or served; this is only a fail-closed egress boundary.
|
|
783
|
-
*/
|
|
784
614
|
function dashboardNamespaceTarget(path) {
|
|
785
615
|
const rawPath = path.split("?", 1)[0] ?? "";
|
|
786
|
-
// Literal routes retain their deliberately narrow mount. Percent-bearing lookalikes take the
|
|
787
|
-
// stricter path below so an encoded separator/query/dot/backslash can never fall through.
|
|
788
616
|
if (!rawPath.includes("%"))
|
|
789
617
|
return rawPath === "/dashboard" || rawPath.startsWith("/dashboard/");
|
|
790
618
|
const percentTolerant = rawPath.replace(/%([0-9a-fA-F]{2})/g, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16)));
|
|
@@ -800,8 +628,6 @@ function failDashboardClosed(res, status, message) {
|
|
|
800
628
|
async function handleDashboardAdapter(req, res, path, started, cfg, h) {
|
|
801
629
|
if (!dashboardNamespaceTarget(path))
|
|
802
630
|
return false;
|
|
803
|
-
// Static handling is synchronous and intentionally receives the raw target: query strings and
|
|
804
|
-
// alternate spellings are not cache variants on this tokenless surface.
|
|
805
631
|
const staticResponse = h.dashboardStatic.handle({ method: req.method ?? "GET", path });
|
|
806
632
|
if (staticResponse.handled) {
|
|
807
633
|
const staticHeaders = staticResponse.status === 404 || staticResponse.status === 405
|
|
@@ -825,22 +651,15 @@ async function handleDashboardAdapter(req, res, path, started, cfg, h) {
|
|
|
825
651
|
admission: {
|
|
826
652
|
hostAuthorized: true,
|
|
827
653
|
expectedOrigin,
|
|
828
|
-
// Bootstrap is a dashboard-specific control operation. The generic admission above
|
|
829
|
-
// remains unchanged for every existing route; this only supplies the route contract.
|
|
830
654
|
controlAuthorized: validateControlAuthorization(h.controlAuthorization, req.headers),
|
|
831
655
|
},
|
|
832
656
|
readBody: (maxBytes) => readBody(req, maxBytes),
|
|
833
657
|
}, { auth: h.dashboardAuth, read: h.dashboardRead });
|
|
834
658
|
if (response.handled) {
|
|
835
659
|
writeDashboardResponse(res, response);
|
|
836
|
-
// One metadata record per dashboard answer, mirroring the admin-route convention:
|
|
837
|
-
// status + logSafePath(path) only. The session/bootstrap tokens travel as headers and
|
|
838
|
-
// body bytes, neither of which this record can carry (`served` is null, no attempts).
|
|
839
660
|
h.logger.write(baseLog(started, path, false, false, response.status, "skipped", null));
|
|
840
661
|
return true;
|
|
841
662
|
}
|
|
842
|
-
// Do not let a dashboard lookalike reach proxy routing. This covers unknown API paths and
|
|
843
|
-
// every other /dashboard/* spelling while preserving all existing non-dashboard routes.
|
|
844
663
|
failDashboardClosed(res, 404, "dashboard route not found");
|
|
845
664
|
h.logger.write(baseLog(started, path, false, false, 404, "skipped", null));
|
|
846
665
|
return true;
|
|
@@ -859,9 +678,6 @@ async function handle(req, res, cfg, h) {
|
|
|
859
678
|
h.logger.write(baseLog(started, path, false, false, 403, "skipped", null));
|
|
860
679
|
return;
|
|
861
680
|
}
|
|
862
|
-
// The dashboard owns a deliberately tiny route namespace. It must run before the generic
|
|
863
|
-
// proxy body reader: static GET/HEAD never buffer, and dashboard writes retain their 16 KiB
|
|
864
|
-
// cap rather than inheriting the data-plane's 36 MiB allowance.
|
|
865
681
|
if (await handleDashboardAdapter(req, res, path, started, cfg, h))
|
|
866
682
|
return;
|
|
867
683
|
let reqBuf;
|
|
@@ -892,53 +708,22 @@ async function handle(req, res, cfg, h) {
|
|
|
892
708
|
const handled = await handleAdminRoutes(req, res, pathname, path, started, reqJson, cfg, h);
|
|
893
709
|
if (handled)
|
|
894
710
|
return;
|
|
895
|
-
// ONE full-body token walk per request, shared with the context guardrail below.
|
|
896
|
-
// estimateRequestTokens walks the entire parsed body — potentially tens of MB — so
|
|
897
|
-
// computing it once per consumer paid that twice on every caller-visible request.
|
|
898
|
-
// ⚠ Reassigned below: a stripped `@relay:` directive shrinks the body, and the
|
|
899
|
-
// guardrail must judge the body that will actually reach the backend.
|
|
900
711
|
let estimatedRequestTokens = estimateRequestTokens(reqJson);
|
|
901
|
-
// Only exact caller-visible inference routes create a request lifecycle.
|
|
902
|
-
// Admin/control aliases and prefix lookalikes remain outside accounting.
|
|
903
|
-
// (Deliberately constructed even under the NOOP recorder: the lifecycle also owns
|
|
904
|
-
// `isRequestClosed()`, which is how a running repair learns the client went away.)
|
|
905
712
|
const accounting = isCallerVisibleAccountingPath(req.method, pathname)
|
|
906
713
|
? new RequestAccountingState(h.accountingRecorder, res, started, estimatedRequestTokens, requestClient, h.accountingPricePort)
|
|
907
714
|
: null;
|
|
908
715
|
const isCountTokens = req.method === "POST" && pathname === "/v1/messages/count_tokens";
|
|
909
716
|
const isMessages = req.method === "POST" && pathname.startsWith("/v1/messages") && !isCountTokens;
|
|
910
|
-
// Real traffic is the signal the adaptive cadence was built around: probe briskly while the
|
|
911
|
-
// proxy is in use, drop to `slow` once it has been idle. `noteUserActivity()` had no callers
|
|
912
|
-
// either, so the loop could never have left its startup mode even if it had been running.
|
|
913
717
|
h.pingLoop?.noteUserActivity();
|
|
914
|
-
// Route the request's model to a concrete provider + backend model candidates.
|
|
915
|
-
//
|
|
916
|
-
// ⚠ `subagentSpec` must run INSIDE this try. It throws `RoutingError` for an
|
|
917
|
-
// unresolvable `@relay:` directive, and while it sat outside, that throw escaped
|
|
918
|
-
// to `createProxy`'s top-level catch and surfaced as `502 llm-relay internal
|
|
919
|
-
// error: …` with NO log line — fail-closed and loud, but mislabelled as a proxy
|
|
920
|
-
// bug and invisible to the operator. A bad directive is a client routing error
|
|
921
|
-
// (400) like any other unresolvable spec.
|
|
922
718
|
let targetCandidates;
|
|
923
|
-
// Hoisted out of the try: the served response has to say whether the answer came from below the
|
|
924
|
-
// requested effort band, and that is only knowable from the pool that was actually addressed.
|
|
925
719
|
let degradedSpecs = null;
|
|
926
720
|
let addressedPool = null;
|
|
927
721
|
try {
|
|
928
|
-
// A SUBAGENT request may route somewhere other than its nominal model: either an explicit
|
|
929
|
-
// `@relay: <spec>` in the dispatcher's prompt (stripped here, so the model never sees it),
|
|
930
|
-
// Claude's cc_is_subagent marker, or Codex's request metadata header. Main-conversation
|
|
931
|
-
// requests remain untouched unless that front door's rule explicitly uses scope "all".
|
|
932
722
|
const subSpec = subagentSpec(reqJson, model, cfg, req.headers, requestClient);
|
|
933
723
|
const routedModel = subSpec ?? model;
|
|
934
724
|
materializeDynamicPools(cfg, h.catalog);
|
|
935
|
-
// Re-serialize whenever a subagent spec applied — the @relay: line was stripped from reqJson
|
|
936
|
-
// in place, and it must not reach the backend even when the spec matches the nominal model.
|
|
937
725
|
if (subSpec !== null) {
|
|
938
726
|
reqBuf = Buffer.from(JSON.stringify(reqJson), "utf8");
|
|
939
|
-
// The body just shrank, so the shared token estimate is stale for anything downstream
|
|
940
|
-
// of this point — notably the context guardrail. Accounting already captured the
|
|
941
|
-
// pre-strip figure, which is correct there: that walk measured what the CLIENT sent.
|
|
942
727
|
estimatedRequestTokens = estimateRequestTokens(reqJson);
|
|
943
728
|
}
|
|
944
729
|
targetCandidates = resolveTargets(routedModel, cfg);
|
|
@@ -947,18 +732,6 @@ async function handle(req, res, cfg, h) {
|
|
|
947
732
|
const tail = cfg.routing.poolDegraded?.[addressedPool];
|
|
948
733
|
degradedSpecs = tail && tail.length > 0 ? new Set(tail) : null;
|
|
949
734
|
}
|
|
950
|
-
// The freeOnly guard: rerouted-by-offload traffic must not spend money. Enforced on the
|
|
951
|
-
// RESOLVED candidates, not the spec — a pool lists free and paid members side by side, and
|
|
952
|
-
// "the pool is mostly free" is exactly the assumption this exists to not rely on. Refusal is
|
|
953
|
-
// loud (a clean 503 naming the rule), never a fall-through: falling through to
|
|
954
|
-
// `routing.default` is the Anthropic passthrough, i.e. the very spend being guarded against.
|
|
955
|
-
// ⚠ The guard covers a DIRECTLY ADDRESSED pool too, not only offload-rerouted traffic.
|
|
956
|
-
// Gating it on `subSpec !== null` meant it never ran for the case it most needed to: a
|
|
957
|
-
// dispatch `cliLane` runs `claude -p --model pool/<name>`, whose requests are a MAIN
|
|
958
|
-
// conversation — no subagent marker, no `@relay:` directive — so the free-lane traffic this
|
|
959
|
-
// flag exists to bound walked straight past it. `pool/<name>` is by construction relay-routed
|
|
960
|
-
// free-lane traffic and never the vendor passthrough, and the guard can only ever refuse to
|
|
961
|
-
// spend, so extending it there cannot cost anyone an answer they were entitled to.
|
|
962
735
|
const addressesPool = typeof routedModel === "string" && routedModel.startsWith("pool/");
|
|
963
736
|
if ((subSpec !== null || addressesPool) && freeOnlyApplies(offloadRule(cfg, requestClient), subSpec !== null)) {
|
|
964
737
|
const kept = [];
|
|
@@ -966,11 +739,7 @@ async function handle(req, res, cfg, h) {
|
|
|
966
739
|
for (const t of targetCandidates) {
|
|
967
740
|
const assessment = t.kind === "openai" && t.model
|
|
968
741
|
? assessCost(t.model, h.catalog.cachedLimits(t.provider, t.model), cfg.providers[t.provider]?.tierType)
|
|
969
|
-
: null;
|
|
970
|
-
// A deployment that STATED it is not free outranks a price table that says it is. The
|
|
971
|
-
// `provider-tier` basis is an assumption about a roster; a 403 naming a subscription is
|
|
972
|
-
// that deployment correcting us. (`isCostBlocked` excludes `allowance-exhausted` — a
|
|
973
|
-
// spent allowance is not a price, and must not be laundered into one here either.)
|
|
742
|
+
: null;
|
|
974
743
|
if (assessment?.costClass === "free" && !isCostBlocked(t.provider, null, t.model, { costClass: assessment.costClass }))
|
|
975
744
|
kept.push(t);
|
|
976
745
|
else if (!blocked) {
|
|
@@ -1001,28 +770,9 @@ async function handle(req, res, cfg, h) {
|
|
|
1001
770
|
}
|
|
1002
771
|
throw e;
|
|
1003
772
|
}
|
|
1004
|
-
// Demote unusable candidates — and do NOTHING else to the order.
|
|
1005
|
-
//
|
|
1006
|
-
// ⚠ Deliberately NOT `getHealthyTargets()`: that filters AND re-sorts by measured
|
|
1007
|
-
// stability, which is a second ranking pass competing with the deployment-fitness ranking
|
|
1008
|
-
// `resolveTargets` already applied. Two ranking passes
|
|
1009
|
-
// means neither decides the order, and live health then PROMOTES on evidence that
|
|
1010
|
-
// is often a single request's latency. Health is used here only to demote, never
|
|
1011
|
-
// to promote: a target the breaker is cooling steps aside, everything else keeps
|
|
1012
|
-
// its fitness order. (The re-sort was invisible for as long as an untracked target
|
|
1013
|
-
// scored a flat 100 and `Array.prototype.sort` is stable — INV-TS-7.)
|
|
1014
|
-
// The OpenAI front (Chat Completions / Responses) is detected BEFORE the context guardrail so
|
|
1015
|
-
// the guardrail covers it: both fronts resolve concrete target deployments, and the estimator
|
|
1016
|
-
// walks all three wire shapes. The front went without this pruning until 0.17.0 — the same
|
|
1017
|
-
// "two paths, two policies, one of them empty" failure mode as the pool-failover incident.
|
|
1018
773
|
const openAiFrontProtocol = detectOpenAiFrontProtocol(req.method, pathname);
|
|
1019
774
|
const routingNow = Date.now();
|
|
1020
|
-
// Resolve the credential once per configured candidate after all route-level pruning. The
|
|
1021
|
-
// resulting attempt is immutable for this request: retries and both public fronts must not
|
|
1022
|
-
// observe an environment change halfway through a provider attempt.
|
|
1023
775
|
const attempts = expandCredentialAttempts(targetCandidates);
|
|
1024
|
-
// An explicit empty/disabled/model-scoped/missing fleet cannot make egress. Keeping a
|
|
1025
|
-
// synthetic first attempt here would accidentally turn a configuration error into a request.
|
|
1026
776
|
if (attempts.length === 0) {
|
|
1027
777
|
failClosed(res, 502, "llm-relay configuration: no enabled credential slot is available for the resolved target");
|
|
1028
778
|
h.logger.write(baseLog(started, path, hadTools, false, 502, "skipped", null));
|
|
@@ -1050,15 +800,7 @@ async function handle(req, res, cfg, h) {
|
|
|
1050
800
|
}
|
|
1051
801
|
}
|
|
1052
802
|
}
|
|
1053
|
-
// Context guardrail — enforced ONLY against a limit the serving provider published about its own
|
|
1054
|
-
// deployment. An unknown limit means no guardrail: the request goes upstream and the provider
|
|
1055
|
-
// answers with its own (authoritative) error.
|
|
1056
|
-
//
|
|
1057
|
-
// Candidates whose published context limits are exceeded by the estimated prompt tokens are pruned.
|
|
1058
|
-
// If all candidates are pruned, fail closed with 400 naming the context limit.
|
|
1059
803
|
if ((isMessages || openAiFrontProtocol) && reqJson) {
|
|
1060
|
-
// The single walk computed above (re-run here ONLY if an `@relay:` directive was
|
|
1061
|
-
// stripped, which shrinks the body this guardrail is judging).
|
|
1062
804
|
const estimatedTokens = estimatedRequestTokens;
|
|
1063
805
|
if (estimatedTokens > 0) {
|
|
1064
806
|
const remainingAttempts = [];
|
|
@@ -1096,9 +838,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1096
838
|
evidenceFor: (attempt) => credentialEvidence(attempt, cfg, h.breaker, routingNow),
|
|
1097
839
|
});
|
|
1098
840
|
const credentialTrace = new CredentialAttemptTrace(cfg);
|
|
1099
|
-
// OpenAI-compatible FRONT: route both Chat Completions and Responses requests through the
|
|
1100
|
-
// resolved target. The adapter supports OpenAI-compatible and Anthropic backends, so Codex and
|
|
1101
|
-
// OpenAI-native IDEs can use the same relay that Claude clients use in the other direction.
|
|
1102
841
|
if (openAiFrontProtocol) {
|
|
1103
842
|
await openAiFrontPath(res, credentialWalk, credentialTrace, {
|
|
1104
843
|
reqJson,
|
|
@@ -1118,11 +857,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1118
857
|
}, h);
|
|
1119
858
|
return;
|
|
1120
859
|
}
|
|
1121
|
-
// OpenAI-compatible backends expose ONLY /chat/completions — they have no
|
|
1122
|
-
// count_tokens route and no other Anthropic paths. Rather than mistranslate
|
|
1123
|
-
// those into a chat completion (yielding a spurious 400/garbage), answer
|
|
1124
|
-
// count_tokens locally with a cheap estimate and reject other paths cleanly.
|
|
1125
|
-
// For an Anthropic backend everything forwards as before (it speaks these).
|
|
1126
860
|
if (target.kind === "openai") {
|
|
1127
861
|
if (isCountTokens) {
|
|
1128
862
|
const input_tokens = Math.max(1, estimateRequestTokens(reqJson));
|
|
@@ -1137,14 +871,10 @@ async function handle(req, res, cfg, h) {
|
|
|
1137
871
|
return;
|
|
1138
872
|
}
|
|
1139
873
|
}
|
|
1140
|
-
// Candidate execution loop: CredentialWalk owns breadth-first slot expansion,
|
|
1141
|
-
// request-local suppression, and the wall-clock start budget.
|
|
1142
874
|
const attemptTrace = new RequestAttemptTrace();
|
|
1143
875
|
const pool429 = new Pool429Tracker();
|
|
1144
876
|
const tried = [];
|
|
1145
877
|
while (!res.destroyed) {
|
|
1146
|
-
// G2's attempt boundary: an offer whose operator-set hard cap is reached is refused here,
|
|
1147
|
-
// before any egress, and the loop comes straight back for the next candidate.
|
|
1148
878
|
const resolvedAttempt = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
|
|
1149
879
|
if (!resolvedAttempt)
|
|
1150
880
|
break;
|
|
@@ -1192,8 +922,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1192
922
|
reqJson,
|
|
1193
923
|
anthropicHeaders: forwardHeaders,
|
|
1194
924
|
wantsStream,
|
|
1195
|
-
// Reaches the dialect-rescue commit points: a tool call the relay reconstructs out of
|
|
1196
|
-
// model TEXT is refused when it names a destructive tool.
|
|
1197
925
|
isDestructive: h.isDestructive,
|
|
1198
926
|
usage,
|
|
1199
927
|
signal: controller.signal,
|
|
@@ -1245,8 +973,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1245
973
|
const walkEnd = endWalk(h, res, "anthropic", credentialWalk, attemptTrace, pool429, status, sticky, credentialTrace, () => baseLog(started, path, hadTools, false, status, "skipped", target, attemptTrace.snapshot()), {
|
|
1246
974
|
kind: "transport",
|
|
1247
975
|
message: aborted ? "backend timed out" : `backend unreachable: ${e.message}`,
|
|
1248
|
-
// Preserved cross-front residue: this handle exit omits served-by while its OpenAI
|
|
1249
|
-
// twin includes it. Unifying it is now a one-line `servedBy` follow-up.
|
|
1250
976
|
});
|
|
1251
977
|
if (walkEnd)
|
|
1252
978
|
continue;
|
|
@@ -1301,9 +1027,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1301
1027
|
? await probeStreamForCommit(backendRes.body, "anthropic-messages", {
|
|
1302
1028
|
isCancelled: () => res.destroyed,
|
|
1303
1029
|
malformedProvenance: target.kind === "openai" ? "local" : "upstream",
|
|
1304
|
-
// Declared, never inferred from the wire: only a stream the dialect wrapper actually
|
|
1305
|
-
// wrapped has a signal, so an upstream echoing the refusal code cannot mint `local`
|
|
1306
|
-
// provenance for itself and thereby dodge both failover and breaker accounting.
|
|
1307
1030
|
...(dialectRefusalSignalOf(backendRes) ? { relayRefusal: dialectRefusalSignalOf(backendRes) } : {}),
|
|
1308
1031
|
})
|
|
1309
1032
|
: { kind: "dead", reason: "stream has no body", provenance: "upstream" };
|
|
@@ -1332,14 +1055,9 @@ async function handle(req, res, cfg, h) {
|
|
|
1332
1055
|
kind: "dead-stream",
|
|
1333
1056
|
message: `llm-relay: ${probe.reason}`,
|
|
1334
1057
|
errorType: probe.errorType,
|
|
1335
|
-
// Who produced this status. A dead PRE-COMMIT stream the relay judged `local` is the
|
|
1336
|
-
// proxy's own verdict, and saying so is the header's documented job — it was simply
|
|
1337
|
-
// never written on this path.
|
|
1338
1058
|
errorOrigin: probe.provenance,
|
|
1339
1059
|
servedBy: tried.join(", "),
|
|
1340
1060
|
shouldTryNext: probe.provenance === "upstream",
|
|
1341
|
-
// Preserved residue: handle gates dead streams on `!res.destroyed`; the OpenAI
|
|
1342
|
-
// front additionally requires `!res.writableEnded`.
|
|
1343
1061
|
});
|
|
1344
1062
|
if (walkEnd)
|
|
1345
1063
|
continue;
|
|
@@ -1405,10 +1123,6 @@ async function handle(req, res, cfg, h) {
|
|
|
1405
1123
|
if (repairResult !== null) {
|
|
1406
1124
|
recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, { kind: "protocol" });
|
|
1407
1125
|
credentialRecorded = true;
|
|
1408
|
-
// Deliberate endWalk holdout: this records a dead turn, never records final, and its
|
|
1409
|
-
// continue branch never records failover because recordDeadTurn already counted the cell
|
|
1410
|
-
// and cleared only429. Converting it would need skipFinal/skipFailover flags — exactly the
|
|
1411
|
-
// silent-skip hazard endWalk exists to prevent.
|
|
1412
1126
|
pool429.recordDeadTurn();
|
|
1413
1127
|
const next = !res.destroyed ? nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429) : undefined;
|
|
1414
1128
|
if (next)
|
|
@@ -1470,47 +1184,11 @@ async function handle(req, res, cfg, h) {
|
|
|
1470
1184
|
res.off("close", onResClose);
|
|
1471
1185
|
}
|
|
1472
1186
|
}
|
|
1473
|
-
// THE all-capped refusal site — the only one on this front, deliberately.
|
|
1474
|
-
//
|
|
1475
|
-
// `allCapped()` requires `!egressed`, and every failover arm above runs only after a candidate
|
|
1476
|
-
// egressed, so a check inside one could never be true: the loop exit is the sole place the
|
|
1477
|
-
// condition can hold. (Eight such inner checks existed and were dead code on the request path;
|
|
1478
|
-
// the rule they LOOK like they enforce — a walk that mixes caps with real failures keeps its
|
|
1479
|
-
// last upstream error — is enforced by `egressed`, and pinned by the mixed-walk tests.) A walk
|
|
1480
|
-
// that exhausted for any other reason reaches here with nothing capped and must stay silent:
|
|
1481
|
-
// its caller already replied.
|
|
1482
1187
|
if (!res.headersSent && !res.destroyed && pool429.allCapped()) {
|
|
1483
1188
|
respondAllCapped(res, h, { started, path, hadTools, streamed: wantsStream }, "anthropic", pool429, attemptTrace.snapshot());
|
|
1484
1189
|
}
|
|
1485
1190
|
}
|
|
1486
|
-
/**
|
|
1487
|
-
* Order candidates worst-last WITHOUT dropping any: live, then credential-faulted, then cooling.
|
|
1488
|
-
*
|
|
1489
|
-
* Three states, and the distinction between them is the whole point:
|
|
1490
|
-
* - live — breaker closed, no standing 401/403. Keeps its fitness order.
|
|
1491
|
-
* - credential-faulted — answered 401/403 recently. It is not sick, it is unusable, and it
|
|
1492
|
-
* must not cost a round-trip per request ahead of a working member.
|
|
1493
|
-
* - cooling — breaker open (rate-limited or repeatedly failing).
|
|
1494
|
-
*
|
|
1495
|
-
* Demotion, not filtering. The previous `filter(isHealthy)` with a keep-everything fallback
|
|
1496
|
-
* removed cooling candidates outright whenever at least one was healthy, so a pool could be
|
|
1497
|
-
* narrowed to a single member and then have nothing to fall back to when that one failed too.
|
|
1498
|
-
* Ordering is strictly better: a demoted target is only ever reached after every better one has
|
|
1499
|
-
* actually failed on this request, and a pool with 14 members always has 14 chances.
|
|
1500
|
-
*
|
|
1501
|
-
* Stable within each band, so the already-computed deployment fitness still decides among equals.
|
|
1502
|
-
*/
|
|
1503
|
-
/**
|
|
1504
|
-
* `"<spec> (below <band>)"` when this answer came from the pool's degrade tail, else null.
|
|
1505
|
-
*
|
|
1506
|
-
* The "loudly" half of automatic degradation. Falling back to a weaker live model is the right
|
|
1507
|
-
* behaviour — a band with nothing behind it turns "the strongest models are busy" into "no answer
|
|
1508
|
-
* at all" — but only because the caller is told. An unannounced downgrade is indistinguishable
|
|
1509
|
-
* from getting what you asked for.
|
|
1510
|
-
*/
|
|
1511
1191
|
function paidLabel(cfg, h, target) {
|
|
1512
|
-
// Only openai-kind targets carry a price we can assess; the anthropic passthrough is the
|
|
1513
|
-
// caller's own subscription and is reported by the absence of any pool at all.
|
|
1514
1192
|
if (target.kind !== "openai" || !target.model)
|
|
1515
1193
|
return null;
|
|
1516
1194
|
const assessment = assessCost(target.model, h.catalog.cachedLimits(target.provider, target.model), cfg.providers[target.provider]?.tierType);
|
|
@@ -1524,12 +1202,9 @@ function degradedLabel(pool, degraded, target) {
|
|
|
1524
1202
|
const spec = specOfTarget(target);
|
|
1525
1203
|
return degraded.has(spec) ? `${spec} (below ${pool})` : null;
|
|
1526
1204
|
}
|
|
1527
|
-
/** Is a learned allowance exhaustion still cooling this target? Never throws — no store, no cooling. */
|
|
1528
1205
|
function cooledByAllowance(attempt, now, costClass) {
|
|
1529
1206
|
try {
|
|
1530
1207
|
const { target } = attempt;
|
|
1531
|
-
// `exactOptionalPropertyTypes` is on: an explicit `undefined` is not the same as absent, and
|
|
1532
|
-
// absent is what "the caller could not classify this" must look like to `factsFor`.
|
|
1533
1208
|
const until = cooldownUntil(target.provider, attempt.credentialId, target.model ?? null, {
|
|
1534
1209
|
now,
|
|
1535
1210
|
...(costClass === undefined ? {} : { costClass }),
|
|
@@ -1540,17 +1215,6 @@ function cooledByAllowance(attempt, now, costClass) {
|
|
|
1540
1215
|
return false;
|
|
1541
1216
|
}
|
|
1542
1217
|
}
|
|
1543
|
-
/**
|
|
1544
|
-
* Gap 12: a SPENT quota demotes to the same cooling band as an outage (spec §5.4), because both
|
|
1545
|
-
* mean "this cell cannot serve right now, and it comes back on its own". A spent allowance is not
|
|
1546
|
-
* a sick backend, so the term never touches failure counters; registering the cooldown with
|
|
1547
|
-
* source "quota" exists purely so `/candidates`, the dashboard Cooldowns panel and
|
|
1548
|
-
* `llm-relay candidates` can SEE why the member stepped aside — and so a sticky pin or a degrade
|
|
1549
|
-
* tail check later in this same request reads the demotion too, without a second resolver pass.
|
|
1550
|
-
*
|
|
1551
|
-
* Learned limits gate only under `routing.quota.enforceLearned`; unknown quota yields null here
|
|
1552
|
-
* and has no effect whatsoever. Never throws, never drops, never refuses.
|
|
1553
|
-
*/
|
|
1554
1218
|
function cooledByQuota(attempt, breaker, quotaDemotion, now) {
|
|
1555
1219
|
if (!quotaDemotion)
|
|
1556
1220
|
return false;
|
|
@@ -1566,16 +1230,12 @@ function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf) {
|
|
|
1566
1230
|
return "cooling";
|
|
1567
1231
|
if (cooledByAllowance(attempt, now, costClassOf?.(attempt)))
|
|
1568
1232
|
return "cooling";
|
|
1569
|
-
// A stated/declared quota at zero belongs in the SAME band as allowance-exhaustion above: both
|
|
1570
|
-
// are temporary conditions on a deployment that is otherwise healthy, and ordering them ahead
|
|
1571
|
-
// of live members is the whole point — the walk should spend its round-trips where capacity is.
|
|
1572
1233
|
if (cooledByQuota(attempt, breaker, quotaDemotion, now))
|
|
1573
1234
|
return "cooling";
|
|
1574
1235
|
if (breaker.hasCredentialFault(identity, now))
|
|
1575
1236
|
return "credential-fault";
|
|
1576
1237
|
return "live";
|
|
1577
1238
|
}
|
|
1578
|
-
/** Resolve every configured slot once, then retain deployment order while taking credential rounds. */
|
|
1579
1239
|
function expandCredentialAttempts(targets) {
|
|
1580
1240
|
const expanded = [];
|
|
1581
1241
|
for (const target of targets) {
|
|
@@ -1600,7 +1260,7 @@ function credentialEvidence(attempt, cfg, breaker, now) {
|
|
|
1600
1260
|
try {
|
|
1601
1261
|
facts = factsFor(attempt.target.provider, attempt.credentialId, attempt.target.model ?? null, { now });
|
|
1602
1262
|
}
|
|
1603
|
-
catch {
|
|
1263
|
+
catch { }
|
|
1604
1264
|
const cost = attempt.target.kind === "openai" && attempt.target.model
|
|
1605
1265
|
? assessCost(attempt.target.model, null, provider?.tierType).costClass
|
|
1606
1266
|
: "paid";
|
|
@@ -1608,29 +1268,18 @@ function credentialEvidence(attempt, cfg, breaker, now) {
|
|
|
1608
1268
|
facts,
|
|
1609
1269
|
health: breaker.isHealthy(identity, now) ? "unknown" : "unhealthy",
|
|
1610
1270
|
credentialFault: breaker.hasCredentialFault(identity, now),
|
|
1611
|
-
// `cost` here is deliberately assessed with NO catalog limits (see above), so it can say
|
|
1612
|
-
// `free` from a provider tier but never `paid` from a price. Passing it would under-report a
|
|
1613
|
-
// paid deployment as unclassified, so this evidence view reports cooling for cost-filtered
|
|
1614
|
-
// facts the same fail-safe way any unclassified caller does: they simply do not apply.
|
|
1615
1271
|
cooling: cooledByAllowance(attempt, now, undefined),
|
|
1616
1272
|
saturated: provider?.maxConcurrent != null && breaker.inFlightCredential(attempt.credentialId) >= provider.maxConcurrent,
|
|
1617
1273
|
quota: state?.quotaObservations ?? [],
|
|
1618
1274
|
cost,
|
|
1619
1275
|
};
|
|
1620
1276
|
}
|
|
1621
|
-
/**
|
|
1622
|
-
* Get the soonest cooldown lift time for a cooling attempt.
|
|
1623
|
-
* Checks breaker state first (covers breaker cooling + quota demotion), then target-facts for allowance facts.
|
|
1624
|
-
* Returns null if no known lift time (unknown).
|
|
1625
|
-
*/
|
|
1626
1277
|
function coolingLiftTime(attempt, breaker, now, costClassOf) {
|
|
1627
1278
|
const identity = targetIdentity(attempt);
|
|
1628
|
-
// 1. Breaker cooldown (covers failure/429/402 cooling AND quota demotion cooling)
|
|
1629
1279
|
const breakerState = breaker.getState(identity);
|
|
1630
1280
|
if (breakerState && breakerState.cooldownUntil > now) {
|
|
1631
1281
|
return breakerState.cooldownUntil;
|
|
1632
1282
|
}
|
|
1633
|
-
// 2. Allowance exhaustion fact (target-facts) - only if not already covered by breaker
|
|
1634
1283
|
const costClass = costClassOf?.(attempt);
|
|
1635
1284
|
const factCooldown = cooldownUntil(attempt.target.provider, attempt.credentialId, attempt.target.model ?? null, {
|
|
1636
1285
|
now,
|
|
@@ -1641,14 +1290,6 @@ function coolingLiftTime(attempt, breaker, now, costClassOf) {
|
|
|
1641
1290
|
}
|
|
1642
1291
|
return null;
|
|
1643
1292
|
}
|
|
1644
|
-
/**
|
|
1645
|
-
* Keep every credential row of a deployment together while ordering deployments by their
|
|
1646
|
-
* best-ranked row. CredentialWalk performs the breadth-first interleaving when it offers rows.
|
|
1647
|
-
* Tracked like `orderByUsabilityTracked` so the request path can announce a displaced first choice.
|
|
1648
|
-
*
|
|
1649
|
-
* Within the COOLING band, order by soonest known lift time (ascending). Unknown lifts sort last.
|
|
1650
|
-
* Stable within ties and unknowns.
|
|
1651
|
-
*/
|
|
1652
1293
|
function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion, costClassOf) {
|
|
1653
1294
|
const preferred = groupCredentialAttempts(attempts)[0]?.attempts[0];
|
|
1654
1295
|
const live = [];
|
|
@@ -1663,18 +1304,16 @@ function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion,
|
|
|
1663
1304
|
else
|
|
1664
1305
|
live.push(group);
|
|
1665
1306
|
}
|
|
1666
|
-
// Sort cooling band by soonest lift time (ascending). Unknown (null) sorts last.
|
|
1667
|
-
// Stable sort preserves original order for ties and unknowns.
|
|
1668
1307
|
cooling.sort((a, b) => {
|
|
1669
1308
|
const liftA = coolingLiftTime(a.attempts[0], breaker, now, costClassOf);
|
|
1670
1309
|
const liftB = coolingLiftTime(b.attempts[0], breaker, now, costClassOf);
|
|
1671
1310
|
if (liftA === null && liftB === null)
|
|
1672
|
-
return 0;
|
|
1311
|
+
return 0;
|
|
1673
1312
|
if (liftA === null)
|
|
1674
|
-
return 1;
|
|
1313
|
+
return 1;
|
|
1675
1314
|
if (liftB === null)
|
|
1676
|
-
return -1;
|
|
1677
|
-
return liftA - liftB;
|
|
1315
|
+
return -1;
|
|
1316
|
+
return liftA - liftB;
|
|
1678
1317
|
});
|
|
1679
1318
|
const ordered = [...live, ...faulted, ...cooling].flatMap((group) => group.attempts);
|
|
1680
1319
|
let quotaDemotedFirst = null;
|
|
@@ -1688,7 +1327,6 @@ function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion,
|
|
|
1688
1327
|
}
|
|
1689
1328
|
return { ordered, quotaDemotedFirst };
|
|
1690
1329
|
}
|
|
1691
|
-
/** Metadata-only trace: credential values and storage locations never enter it. */
|
|
1692
1330
|
class CredentialAttemptTrace {
|
|
1693
1331
|
cfg;
|
|
1694
1332
|
entries = [];
|
|
@@ -1769,18 +1407,6 @@ function recordCredentialOutcome(walk, trace, attempt, outcome) {
|
|
|
1769
1407
|
walk.record(attempt, outcome);
|
|
1770
1408
|
trace.record(attempt, outcome);
|
|
1771
1409
|
}
|
|
1772
|
-
/**
|
|
1773
|
-
* G2's attempt boundary — EVERY `CredentialWalk.next()` on both public fronts goes through here.
|
|
1774
|
-
*
|
|
1775
|
-
* A candidate whose operator-set hard cap is reached is refused IN THIS PROCESS, before any
|
|
1776
|
-
* egress and before `recordStarted()`: no provider request, no LRU touch, no breaker mutation,
|
|
1777
|
-
* no accounting attempt — the capped key spends literally nothing. It is still announced three
|
|
1778
|
-
* ways: a status-only `capped` entry in the metadata log's attempt list, an `Nxcapped` tally in
|
|
1779
|
-
* `x-llm-relay-pool-attempts`, and the soonest cap boundary kept on the tracker so an ALL-CAPPED
|
|
1780
|
-
* walk can state its Retry-After instead of inventing one. "Health demotes, never drops" still
|
|
1781
|
-
* governs the WALK — the capped cell stays listed everywhere and lifts at the period boundary;
|
|
1782
|
-
* only this attempt is refused.
|
|
1783
|
-
*/
|
|
1784
1410
|
function nextUncappedAttempt(h, walk, attemptTrace, tracker) {
|
|
1785
1411
|
const now = Date.now();
|
|
1786
1412
|
for (;;) {
|
|
@@ -1789,8 +1415,6 @@ function nextUncappedAttempt(h, walk, attemptTrace, tracker) {
|
|
|
1789
1415
|
return undefined;
|
|
1790
1416
|
const verdict = h.hardCap(candidate, now);
|
|
1791
1417
|
if (verdict === null) {
|
|
1792
|
-
// Announce the offer BEFORE returning: the tracker needs to know a candidate is in flight so
|
|
1793
|
-
// a cap it sees while the loop asks for the next one is ordered after this candidate.
|
|
1794
1418
|
tracker.noteOffered();
|
|
1795
1419
|
return candidate;
|
|
1796
1420
|
}
|
|
@@ -1823,9 +1447,6 @@ function endWalk(h, res, front, walk, attemptTrace, tracker, status, sticky, cre
|
|
|
1823
1447
|
const shouldTryNext = exit.shouldTryNext ?? true;
|
|
1824
1448
|
let next;
|
|
1825
1449
|
if (exit.kind === "dead-stream") {
|
|
1826
|
-
// Dead streams gate liveness BEFORE advancing the walk. That preserves the old lazy policy:
|
|
1827
|
-
// capped/rejected cells are not recorded after the client side is already dead. The two
|
|
1828
|
-
// fronts' liveness spellings intentionally remain different pending a separate unification.
|
|
1829
1450
|
const canContinue = front === "anthropic"
|
|
1830
1451
|
? !res.destroyed
|
|
1831
1452
|
: !res.writableEnded && !res.destroyed;
|
|
@@ -1834,13 +1455,7 @@ function endWalk(h, res, front, walk, attemptTrace, tracker, status, sticky, cre
|
|
|
1834
1455
|
: undefined;
|
|
1835
1456
|
}
|
|
1836
1457
|
else {
|
|
1837
|
-
// Transport and post-header failures advance BEFORE checking response liveness. Preserve that
|
|
1838
|
-
// eager ordering: recordCapped/walk.recordRejected must still fire for skipped capped cells.
|
|
1839
1458
|
next = shouldTryNext ? nextUncappedAttempt(h, walk, attemptTrace, tracker) : undefined;
|
|
1840
|
-
// A response that died mid-walk forces the FINALIZE path even with a next candidate in hand
|
|
1841
|
-
// (the pre-refactor shape): walking on for a gone client would spend another upstream request
|
|
1842
|
-
// with no recordFinal and no log record at this exit. Unreachable today only because every
|
|
1843
|
-
// eager caller pre-guards on res liveness — this keeps the helper safe if one stops.
|
|
1844
1459
|
if (res.writableEnded || res.destroyed)
|
|
1845
1460
|
next = undefined;
|
|
1846
1461
|
}
|
|
@@ -1862,28 +1477,13 @@ function endWalk(h, res, front, walk, attemptTrace, tracker, status, sticky, cre
|
|
|
1862
1477
|
h.logger.write(log());
|
|
1863
1478
|
return false;
|
|
1864
1479
|
}
|
|
1865
|
-
/**
|
|
1866
|
-
* The ALL-CAPPED refusal — the loud half of G2.
|
|
1867
|
-
*
|
|
1868
|
-
* Every walked candidate was refused by an operator-set cap, so the relay answers 429 itself:
|
|
1869
|
-
* there is no upstream error to pass through, and staying silent would hang the client. The body
|
|
1870
|
-
* names the caps (axis/period, inclusive used/cap, basis `operator-declared`) in each front's
|
|
1871
|
-
* NATIVE error shape, so an ordinary client's rate-limit handling just works. `Retry-After` is
|
|
1872
|
-
* derived ONLY from the soonest UTC period boundary any walked cap lifts at; when no boundary
|
|
1873
|
-
* resolves the header is omitted and the body says so — a duration is never invented. Metadata
|
|
1874
|
-
* only: labels, numbers, no message text, no key material.
|
|
1875
|
-
*/
|
|
1876
1480
|
function respondAllCapped(res, h, ctx, front, tracker, attempts) {
|
|
1877
1481
|
if (res.headersSent)
|
|
1878
1482
|
return;
|
|
1879
1483
|
const summary = tracker.summary();
|
|
1880
|
-
// One rendering, shared by the body and the header: two calls could disagree about the bound.
|
|
1881
1484
|
const capped = tracker.cappedSummary();
|
|
1882
1485
|
const labels = capped ?? "every candidate";
|
|
1883
1486
|
const resetAt = tracker.cappedResetAt();
|
|
1884
|
-
// The reset is a DERIVED UTC boundary (minute/day), so it always resolves for any cap this
|
|
1885
|
-
// module can declare — but the fail-safe stays: when it somehow does not, the header is omitted
|
|
1886
|
-
// and the body says the lift time is unknown rather than inventing a duration.
|
|
1887
1487
|
const message = `llm-relay: request refused by operator-declared hard cap(s): ${labels}. ` +
|
|
1888
1488
|
`No provider was contacted.` +
|
|
1889
1489
|
(resetAt !== null
|
|
@@ -1904,10 +1504,6 @@ function respondAllCapped(res, h, ctx, front, tracker, attempts) {
|
|
|
1904
1504
|
res.end(JSON.stringify(body));
|
|
1905
1505
|
h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, ctx.streamed, 429, "skipped", null, attempts));
|
|
1906
1506
|
}
|
|
1907
|
-
/**
|
|
1908
|
-
* Promote a pin only inside its live capability segment. In particular, a live degrade-tail pin
|
|
1909
|
-
* never jumps a live in-band member; the tail remains a fallback after the requested band.
|
|
1910
|
-
*/
|
|
1911
1507
|
function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaDemotion, costClassOf) {
|
|
1912
1508
|
const groups = groupCredentialAttempts(ordered);
|
|
1913
1509
|
const pinnedIndex = groups.findIndex((group) => specOfTarget(group.attempts[0].target) === pinnedSpec);
|
|
@@ -1915,7 +1511,6 @@ function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaD
|
|
|
1915
1511
|
const pinned = pinnedGroup?.attempts[0];
|
|
1916
1512
|
if (!pinned || !pinnedGroup)
|
|
1917
1513
|
return { targets: ordered, status: "bypassed: not-in-pool" };
|
|
1918
|
-
// The first row is the credential selector's best-ranked usable slot for this deployment.
|
|
1919
1514
|
const usability = targetUsability(pinned, breaker, now, quotaDemotion, costClassOf);
|
|
1920
1515
|
if (usability !== "live")
|
|
1921
1516
|
return { targets: ordered, status: `bypassed: ${usability}` };
|
|
@@ -1930,8 +1525,6 @@ function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaD
|
|
|
1930
1525
|
}
|
|
1931
1526
|
if (pinnedIndex === 0)
|
|
1932
1527
|
return { targets: ordered, status: "pinned, natural" };
|
|
1933
|
-
// Move the deployment group, not an individual row. CredentialWalk will still offer one slot
|
|
1934
|
-
// per deployment round, so the pin cannot cluster every credential ahead of other deployments.
|
|
1935
1528
|
const reordered = [pinnedGroup, ...groups.filter((_, index) => index !== pinnedIndex)]
|
|
1936
1529
|
.flatMap((group) => group.attempts);
|
|
1937
1530
|
return { targets: reordered, status: "pinned, reordered" };
|
|
@@ -1957,30 +1550,11 @@ export function orderByUsability(attempts, breaker = globalCircuitBreaker, now =
|
|
|
1957
1550
|
const { ordered } = orderByUsabilityTracked(attempts, breaker, now);
|
|
1958
1551
|
return ordered;
|
|
1959
1552
|
}
|
|
1960
|
-
/**
|
|
1961
|
-
* Gap 12's walk-order orderer. Kept pure: its exported shape must not change, and the
|
|
1962
|
-
* quota-demotion ANNOUNCEMENT lives in exactly one place —
|
|
1963
|
-
* `orderDeploymentGroupsByUsability` above — because two copies of the displacement check
|
|
1964
|
-
* would drift silently (the first version of this block was unreachable here).
|
|
1965
|
-
*
|
|
1966
|
-
* Within the COOLING band, order by soonest known lift time (ascending). Unknown lifts sort last.
|
|
1967
|
-
* Stable within ties and unknowns.
|
|
1968
|
-
*/
|
|
1969
1553
|
function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf) {
|
|
1970
1554
|
const live = [];
|
|
1971
1555
|
const faulted = [];
|
|
1972
1556
|
const cooling = [];
|
|
1973
1557
|
for (const attempt of attempts) {
|
|
1974
|
-
// A learned allowance exhaustion cools a target the breaker may know nothing about. That is
|
|
1975
|
-
// the whole point of the ACCOUNT scope: one member's stated "you have depleted your monthly
|
|
1976
|
-
// included credits" is a fact about the credential, so its siblings are spent too and should
|
|
1977
|
-
// step aside without each first spending a round-trip to be told so individually. Measured
|
|
1978
|
-
// here: `pool/xhigh`'s 15 members share only four independent quota domains, so the walk was
|
|
1979
|
-
// rediscovering four facts fifteen times.
|
|
1980
|
-
//
|
|
1981
|
-
// Demotion, never exclusion — same contract as the rest of this function, and doubly so here:
|
|
1982
|
-
// an exhausted allowance is a temporary condition on a deployment that is still free, and a
|
|
1983
|
-
// pool with nothing else left must still be able to try it.
|
|
1984
1558
|
const usability = targetUsability(attempt, breaker, now, quotaDemotion, costClassOf);
|
|
1985
1559
|
if (usability === "cooling")
|
|
1986
1560
|
cooling.push(attempt);
|
|
@@ -1989,8 +1563,6 @@ function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClas
|
|
|
1989
1563
|
else
|
|
1990
1564
|
live.push(attempt);
|
|
1991
1565
|
}
|
|
1992
|
-
// Sort cooling band by soonest lift time (ascending). Unknown (null) sorts last.
|
|
1993
|
-
// Stable sort preserves original order for ties and unknowns.
|
|
1994
1566
|
cooling.sort((a, b) => {
|
|
1995
1567
|
const liftA = coolingLiftTime(a, breaker, now, costClassOf);
|
|
1996
1568
|
const liftB = coolingLiftTime(b, breaker, now, costClassOf);
|
|
@@ -2013,30 +1585,10 @@ export function classifyStatus(status) {
|
|
|
2013
1585
|
return "retriable";
|
|
2014
1586
|
return "client";
|
|
2015
1587
|
}
|
|
2016
|
-
/** Whether another pool candidate is useful; health mutation occurs only at terminal completion. */
|
|
2017
1588
|
function shouldTryNext(cls) {
|
|
2018
1589
|
return cls === "retriable" || cls === "credential";
|
|
2019
1590
|
}
|
|
2020
|
-
/**
|
|
2021
|
-
* Wall-clock ceiling on STARTING further failover attempts — one policy, both fronts, same maxim
|
|
2022
|
-
* as `classifyStatus`. A deep pool could legitimately spend members × timeoutMs on one request;
|
|
2023
|
-
* the budget bounds the walk, never the answer: an attempt already in flight is not aborted, and
|
|
2024
|
-
* the first TWO attempts are always allowed, so a slow-failing first candidate cannot starve the
|
|
2025
|
-
* request of its one retry. 0 disables. Default fork-validated in freellmapi (45s) — adoption
|
|
2026
|
-
* review §1.5.
|
|
2027
|
-
*/
|
|
2028
1591
|
export const DEFAULT_WALK_BUDGET_MS = 45_000;
|
|
2029
|
-
/**
|
|
2030
|
-
* Streamed-response deadline split (adoption review §1.2) — one policy, both fronts.
|
|
2031
|
-
*
|
|
2032
|
-
* One flat `timeoutMs` spanning the whole attempt mis-serves streams in both directions: a
|
|
2033
|
-
* healthy long generation still emitting bytes at the deadline is killed mid-answer, while a
|
|
2034
|
-
* genuinely dead stream is not detected until the same deadline. So once a stream is chosen for
|
|
2035
|
-
* serving, the total deadline DISARMS and an inter-byte watchdog takes over: every arriving
|
|
2036
|
-
* chunk re-arms it, and only silence for the full window aborts — through the same controller,
|
|
2037
|
-
* so the existing mid-stream error path reports it honestly. Default fork-validated in
|
|
2038
|
-
* freellmapi (90s); `stallTimeoutMs: 0` keeps the old single-deadline behavior.
|
|
2039
|
-
*/
|
|
2040
1592
|
export const DEFAULT_STALL_TIMEOUT_MS = 90_000;
|
|
2041
1593
|
function withStallWatchdog(upstream, controller, stallMs) {
|
|
2042
1594
|
if (!upstream.body)
|
|
@@ -2065,57 +1617,21 @@ function withStallWatchdog(upstream, controller, stallMs) {
|
|
|
2065
1617
|
headers: upstream.headers,
|
|
2066
1618
|
});
|
|
2067
1619
|
}
|
|
2068
|
-
/**
|
|
2069
|
-
* How many capped cells `x-llm-relay-capped` may NAME before the rest are counted (`+K more`).
|
|
2070
|
-
*
|
|
2071
|
-
* The header is diagnostic, not a manifest: five lines is enough to see which credentials and
|
|
2072
|
-
* which axis stopped the request, and it keeps a fully capped 30-member dynamic pool from
|
|
2073
|
-
* emitting a ~1.2 KB header — a capped attempt costs no walk budget, so every member reaches it.
|
|
2074
|
-
*/
|
|
2075
1620
|
const MAX_CAPPED_HEADER_CELLS = 5;
|
|
2076
|
-
/**
|
|
2077
|
-
* All-429 exhaustion policy — one policy, both fronts (same maxim as `classifyStatus`).
|
|
2078
|
-
*
|
|
2079
|
-
* When the whole pool is rate-limited, the client is served the LAST candidate's real 429, but
|
|
2080
|
-
* that candidate's Retry-After may be the pool's worst: the earliest reset among the walked 429s
|
|
2081
|
-
* is when the POOL next has capacity. Only an all-429 walk qualifies — a mixed walk says nothing
|
|
2082
|
-
* about when the pool frees up, so any non-429 failure leaves the final header untouched.
|
|
2083
|
-
*/
|
|
2084
1621
|
class Pool429Tracker {
|
|
2085
1622
|
minRetryAfterMs = null;
|
|
2086
1623
|
only429 = true;
|
|
2087
|
-
/** status → how many candidates answered it. Ordering lives in `firstSeenAt`. */
|
|
2088
1624
|
counts = new Map();
|
|
2089
|
-
/**
|
|
2090
|
-
* Outcome kind → the walk position it was FIRST seen at, so the breakdown reads in WALK order.
|
|
2091
|
-
*
|
|
2092
|
-
* ⚠ Positions are stamped, not appended, because the two kinds of outcome arrive out of order:
|
|
2093
|
-
* a cap is recorded while the walk is being asked for the next candidate — i.e. BEFORE the
|
|
2094
|
-
* candidate already in flight has been counted. Appending made a walk that went `A:429 →
|
|
2095
|
-
* B:capped` render `1xcapped, 1x429`, reversing what actually happened. See `stampCapped`.
|
|
2096
|
-
*/
|
|
2097
1625
|
firstSeenAt = new Map();
|
|
2098
|
-
/** Monotonic walk position handed out by `stamp()`; never a candidate count. */
|
|
2099
1626
|
order = 0;
|
|
2100
|
-
/** True while a candidate has been offered and its provider outcome has not been counted yet. */
|
|
2101
1627
|
inFlight = false;
|
|
2102
|
-
/** Caps observed while a candidate was in flight: they follow that candidate's own outcome. */
|
|
2103
1628
|
deferredCapped = false;
|
|
2104
|
-
/** G2: candidates skipped before egress because an operator-set hard cap was reached. */
|
|
2105
1629
|
cappedLabels = [];
|
|
2106
|
-
/** Soonest UTC boundary any walked cap lifts at — the pool's own next capacity instant. */
|
|
2107
1630
|
soonestCapResetAt = null;
|
|
2108
|
-
/** Set at the ONE true egress boundary (`onEgress`), so "no provider was contacted" stays provable. */
|
|
2109
1631
|
egressed = false;
|
|
2110
|
-
/** Called exactly where a backend request is about to start — see `onEgress` on both fronts. */
|
|
2111
1632
|
noteEgress() {
|
|
2112
1633
|
this.egressed = true;
|
|
2113
1634
|
}
|
|
2114
|
-
/**
|
|
2115
|
-
* A candidate was OFFERED to the loop (it passed the cap gate and will be attempted). Nothing
|
|
2116
|
-
* is tallied here — this only marks that an outcome for it is still to come, which is what
|
|
2117
|
-
* lets a cap seen in the meantime be stamped AFTER it.
|
|
2118
|
-
*/
|
|
2119
1635
|
noteOffered() {
|
|
2120
1636
|
this.inFlight = true;
|
|
2121
1637
|
}
|
|
@@ -2123,14 +1639,12 @@ class Pool429Tracker {
|
|
|
2123
1639
|
if (!this.firstSeenAt.has(key))
|
|
2124
1640
|
this.firstSeenAt.set(key, ++this.order);
|
|
2125
1641
|
}
|
|
2126
|
-
/** Place the deferred `capped` term once the in-flight candidate's own outcome is counted. */
|
|
2127
1642
|
stampCapped() {
|
|
2128
1643
|
if (!this.deferredCapped)
|
|
2129
1644
|
return;
|
|
2130
1645
|
this.deferredCapped = false;
|
|
2131
1646
|
this.stamp("capped");
|
|
2132
1647
|
}
|
|
2133
|
-
/** Record a response being failed over past. */
|
|
2134
1648
|
recordFailover(status, retryAfterMs) {
|
|
2135
1649
|
this.count(status);
|
|
2136
1650
|
if (status === 429) {
|
|
@@ -2142,21 +1656,10 @@ class Pool429Tracker {
|
|
|
2142
1656
|
this.only429 = false;
|
|
2143
1657
|
}
|
|
2144
1658
|
}
|
|
2145
|
-
/** Record the response actually served — the walk's last candidate, success or failure. */
|
|
2146
1659
|
recordFinal(status) {
|
|
2147
1660
|
this.count(status);
|
|
2148
1661
|
}
|
|
2149
|
-
/**
|
|
2150
|
-
* Record a candidate skipped BEFORE egress by an operator-set hard cap (G2).
|
|
2151
|
-
*
|
|
2152
|
-
* Deliberately NOT folded into `recordFailover`: a cap is not a provider answer, so it must
|
|
2153
|
-
* neither trip the breaker nor set `only429 = false` — when every candidate is capped, that
|
|
2154
|
-
* flag's meaning ("nothing else went wrong") is exactly what licenses the synthesized refusal.
|
|
2155
|
-
*/
|
|
2156
1662
|
recordCapped(label, resetsAt) {
|
|
2157
|
-
// One tally entry per OUTCOME KIND, matching how statuses count: two distinct capped cells
|
|
2158
|
-
// are still one `Nxcapped` term in the breakdown. Its POSITION waits for the candidate
|
|
2159
|
-
// already in flight, whose outcome the loop has not counted yet.
|
|
2160
1663
|
if (this.inFlight)
|
|
2161
1664
|
this.deferredCapped = true;
|
|
2162
1665
|
else
|
|
@@ -2165,29 +1668,12 @@ class Pool429Tracker {
|
|
|
2165
1668
|
this.soonestCapResetAt =
|
|
2166
1669
|
this.soonestCapResetAt === null ? resetsAt : Math.min(this.soonestCapResetAt, resetsAt);
|
|
2167
1670
|
}
|
|
2168
|
-
/** The soonest cap boundary across the walk, for the all-capped Retry-After; null otherwise. */
|
|
2169
1671
|
cappedResetAt() {
|
|
2170
1672
|
return this.soonestCapResetAt;
|
|
2171
1673
|
}
|
|
2172
|
-
/**
|
|
2173
|
-
* True when this walk offered at least one candidate and EVERY one of them was refused by an
|
|
2174
|
-
* operator-set hard cap before egress — no provider answered anything at all. That is the exact
|
|
2175
|
-
* license for the relay-synthesized 429: any real provider outcome among the failures means the
|
|
2176
|
-
* last real upstream error is the more honest body, and the caps ride beside it in the
|
|
2177
|
-
* pool-attempts tally instead.
|
|
2178
|
-
*/
|
|
2179
1674
|
allCapped() {
|
|
2180
1675
|
return this.cappedLabels.length > 0 && this.counts.size === 0 && !this.egressed;
|
|
2181
1676
|
}
|
|
2182
|
-
/**
|
|
2183
|
-
* `"a/nim/m requests/day 450/450"` lines, in walk order — metadata only, and BOUNDED.
|
|
2184
|
-
*
|
|
2185
|
-
* A capped attempt consumes no walk start budget (`credential-select.ts` gates that on
|
|
2186
|
-
* `#started`), so every member of a fully capped 30-member dynamic pool reaches this list and
|
|
2187
|
-
* an unbounded join is a ~1.2 KB header. At most `MAX_CAPPED_HEADER_CELLS` cells are named and
|
|
2188
|
-
* the remainder is counted — the client learns the shape of the refusal without the response
|
|
2189
|
-
* carrying the whole roster.
|
|
2190
|
-
*/
|
|
2191
1677
|
cappedSummary() {
|
|
2192
1678
|
if (this.cappedLabels.length === 0)
|
|
2193
1679
|
return null;
|
|
@@ -2196,47 +1682,23 @@ class Pool429Tracker {
|
|
|
2196
1682
|
const shown = this.cappedLabels.slice(0, MAX_CAPPED_HEADER_CELLS).join("; ");
|
|
2197
1683
|
return `${shown}; +${this.cappedLabels.length - MAX_CAPPED_HEADER_CELLS} more`;
|
|
2198
1684
|
}
|
|
2199
|
-
/** A backend answered 200, but its malformed tool call remained unusable after repair. */
|
|
2200
1685
|
recordDeadTurn() {
|
|
2201
1686
|
this.only429 = false;
|
|
2202
1687
|
this.count("dead-turn");
|
|
2203
1688
|
}
|
|
2204
|
-
/** A refusal in this walk whose meaning the relay could not look up. */
|
|
2205
1689
|
unknownRefusals = 0;
|
|
2206
1690
|
noteUnknownRefusal() {
|
|
2207
1691
|
this.unknownRefusals += 1;
|
|
2208
1692
|
}
|
|
2209
|
-
/** How many, or null when every refusal was understood. */
|
|
2210
1693
|
unknownCount() {
|
|
2211
1694
|
return this.unknownRefusals > 0 ? this.unknownRefusals : null;
|
|
2212
1695
|
}
|
|
2213
1696
|
count(status) {
|
|
2214
1697
|
this.counts.set(status, (this.counts.get(status) ?? 0) + 1);
|
|
2215
1698
|
this.stamp(status);
|
|
2216
|
-
// This candidate's outcome is in; anything capped while it was in flight comes next.
|
|
2217
1699
|
this.inFlight = false;
|
|
2218
1700
|
this.stampCapped();
|
|
2219
1701
|
}
|
|
2220
|
-
/**
|
|
2221
|
-
* The walk, as one line: `"13 tried, 0 served: 4x402, 5x429, 3x403, 1x400"`.
|
|
2222
|
-
*
|
|
2223
|
-
* ⚠ ASCII only. This is an HTTP header value, and Node latin1-encodes those — a `×` reaches a
|
|
2224
|
-
* UTF-8 client as mojibake. Caught on a live pool, where the header read `6�402`.
|
|
2225
|
-
*
|
|
2226
|
-
* A pool's error is one member's error, and that is genuinely misleading when the other twelve
|
|
2227
|
-
* failed for three other reasons: the client is handed HuggingFace's 402 and told to go buy
|
|
2228
|
-
* credits, when the correct action is "use another pool". The BODY still carries that member's
|
|
2229
|
-
* real upstream error — a true upstream error beats a synthesized one, the same maxim the
|
|
2230
|
-
* context guardrail and the all-429 policy follow — so the aggregate rides alongside it in a
|
|
2231
|
-
* header and in the log, where it costs the client nothing and answers "what actually happened"
|
|
2232
|
-
* without a round of manual probing.
|
|
2233
|
-
*
|
|
2234
|
-
* G2 caps appear in the breakdown as `Nxcapped`, in walk order beside the provider outcomes:
|
|
2235
|
-
* they ARE candidates this request consumed, even though no provider saw them.
|
|
2236
|
-
*
|
|
2237
|
-
* Null for a single-candidate walk: there is no aggregate to report, and emitting one would
|
|
2238
|
-
* dress up an ordinary passthrough error as a pool exhaustion.
|
|
2239
|
-
*/
|
|
2240
1702
|
summary() {
|
|
2241
1703
|
let tried = 0;
|
|
2242
1704
|
let served = 0;
|
|
@@ -2248,34 +1710,20 @@ class Pool429Tracker {
|
|
|
2248
1710
|
tried += this.cappedLabels.length;
|
|
2249
1711
|
if (tried < 2)
|
|
2250
1712
|
return null;
|
|
2251
|
-
// A walk that ended while a candidate was still in flight never counted its outcome, so a cap
|
|
2252
|
-
// deferred behind it would otherwise be dropped from the breakdown entirely.
|
|
2253
1713
|
this.stampCapped();
|
|
2254
1714
|
const parts = [...this.firstSeenAt.entries()]
|
|
2255
1715
|
.sort((a, b) => a[1] - b[1])
|
|
2256
1716
|
.map(([key]) => (key === "capped" ? `${this.cappedLabels.length}xcapped` : `${this.counts.get(key) ?? 0}x${key}`));
|
|
2257
1717
|
return `${tried} tried, ${served} served: ${parts.join(", ")}`;
|
|
2258
1718
|
}
|
|
2259
|
-
/** Retry-After (ms) to serve on the FINAL response, or undefined to leave its real header alone. */
|
|
2260
1719
|
overrideMs(finalStatus, finalRetryAfterMs) {
|
|
2261
1720
|
if (finalStatus !== 429 || !this.only429 || this.minRetryAfterMs === null)
|
|
2262
1721
|
return undefined;
|
|
2263
1722
|
return Math.min(this.minRetryAfterMs, finalRetryAfterMs ?? Infinity);
|
|
2264
1723
|
}
|
|
2265
1724
|
}
|
|
2266
|
-
/**
|
|
2267
|
-
* Feed the proxy's own request outcome into runtime telemetry — the "observed traffic"
|
|
2268
|
-
* evidence consumed by `deploymentFitness()` on the operational axis (after capability
|
|
2269
|
-
* eligibility is decided by `getStrength()`) and reported under `observed` in `/candidates`.
|
|
2270
|
-
* `getStrength()` ranks capability using ONLY the synced capability snapshot (basis
|
|
2271
|
-
* "snapshot" → neutral 50); it does not consume runtime telemetry. Only targets with a
|
|
2272
|
-
* concrete model id are recorded; the Anthropic passthrough has none. Skipped under vitest
|
|
2273
|
-
* so tests never write the user's real telemetry file.
|
|
2274
|
-
*/
|
|
2275
1725
|
function recordCall(h, attempt, ok, completedAt) {
|
|
2276
1726
|
const { target, usage } = attempt;
|
|
2277
|
-
// Keep the historical default writer out of the test filesystem, but never suppress an
|
|
2278
|
-
// explicit recorder: that seam is how ledger callers observe every terminal attempt.
|
|
2279
1727
|
if (!target.model || !h.modelCallRecorder)
|
|
2280
1728
|
return;
|
|
2281
1729
|
try {
|
|
@@ -2286,10 +1734,8 @@ function recordCall(h, attempt, ok, completedAt) {
|
|
|
2286
1734
|
});
|
|
2287
1735
|
}
|
|
2288
1736
|
catch {
|
|
2289
|
-
/* telemetry is best-effort, never in the request's way */
|
|
2290
1737
|
}
|
|
2291
1738
|
}
|
|
2292
|
-
/** Request-scoped, bounded attempt metadata shared by both public fronts. */
|
|
2293
1739
|
class RequestAttemptTrace {
|
|
2294
1740
|
entries = [];
|
|
2295
1741
|
record(target, status, started, completedAt) {
|
|
@@ -2302,7 +1748,6 @@ class RequestAttemptTrace {
|
|
|
2302
1748
|
ms: Math.max(0, completedAt - started),
|
|
2303
1749
|
});
|
|
2304
1750
|
}
|
|
2305
|
-
/** G2: an operator-set hard cap refused this candidate BEFORE any provider egress. */
|
|
2306
1751
|
recordCapped(target, at) {
|
|
2307
1752
|
this.record(target, "capped", at, at);
|
|
2308
1753
|
}
|
|
@@ -2338,11 +1783,6 @@ function beginHealthAttempt(h, resolvedAttempt, started, trace, usage, accountin
|
|
|
2338
1783
|
completed: false,
|
|
2339
1784
|
};
|
|
2340
1785
|
}
|
|
2341
|
-
/**
|
|
2342
|
-
* The candidate loops' client-gone reaction, shared by both fronts: when the client connection
|
|
2343
|
-
* closes before the response is finished, abort the caller-facing work and the in-flight backend
|
|
2344
|
-
* fetch together. One definition, so the two loops cannot drift.
|
|
2345
|
-
*/
|
|
2346
1786
|
function abortOnClientClose(res, callerController, controller) {
|
|
2347
1787
|
return () => {
|
|
2348
1788
|
if (!res.writableEnded) {
|
|
@@ -2351,21 +1791,7 @@ function abortOnClientClose(res, callerController, controller) {
|
|
|
2351
1791
|
}
|
|
2352
1792
|
};
|
|
2353
1793
|
}
|
|
2354
|
-
/**
|
|
2355
|
-
* Learn a deployment's real context ceiling from an error it just returned.
|
|
2356
|
-
*
|
|
2357
|
-
* ⚠ Called from BOTH request paths, right beside `observeAttemptHeaders`, and for the same reason
|
|
2358
|
-
* that helper is shared: this repo has already shipped one defect where the OpenAI front had no
|
|
2359
|
-
* copy of a policy the Anthropic path enforced ("two paths, two policies, one of them empty" — see
|
|
2360
|
-
* docs/pool-failover.md). A learning loop that only ran on one front would silently know less
|
|
2361
|
-
* about half the traffic.
|
|
2362
|
-
*
|
|
2363
|
-
* Called only with bytes already buffered by inspectCandidateResponse, so learning never tees or
|
|
2364
|
-
* consumes the Response that may still need to become the client's terminal real error.
|
|
2365
|
-
*/
|
|
2366
1794
|
function observeContextLimit(status, target, body) {
|
|
2367
|
-
// Context-length rejections are 400 (OpenAI-compatible) or 413. Anything else is a different
|
|
2368
|
-
// fault, and scanning every error body would be work for nothing.
|
|
2369
1795
|
if (status !== 400 && status !== 413)
|
|
2370
1796
|
return;
|
|
2371
1797
|
if (target.model === undefined)
|
|
@@ -2379,24 +1805,9 @@ function observeContextLimit(status, target, body) {
|
|
|
2379
1805
|
recordObservedContextLimit(target.provider, target.model, stated);
|
|
2380
1806
|
}
|
|
2381
1807
|
catch {
|
|
2382
|
-
// Learning is best-effort and never in the request's way.
|
|
2383
1808
|
}
|
|
2384
1809
|
}
|
|
2385
|
-
/**
|
|
2386
|
-
* Learn a deployment's stated OUTPUT-token ceiling (`max_tokens`) from an error it just returned.
|
|
2387
|
-
*
|
|
2388
|
-
* ⚠ Called from BOTH request paths beside `observeContextLimit`, for the same reason that helper
|
|
2389
|
-
* documents: a learning loop wired into only one front would silently know nothing about half the
|
|
2390
|
-
* traffic (docs/pool-failover.md). Called with bytes already buffered by
|
|
2391
|
-
* `inspectCandidateResponse`, so it never tees or consumes the Response that may still become the
|
|
2392
|
-
* client's terminal real error.
|
|
2393
|
-
*
|
|
2394
|
-
* Display-only by owner decision (docs/max-output-caps-design-2026-08-29.md): the fact renders in
|
|
2395
|
-
* `llm-relay candidates`; nothing clamps, refuses, or routes on it.
|
|
2396
|
-
*/
|
|
2397
1810
|
function observeMaxOutput(status, target, body) {
|
|
2398
|
-
// A max_tokens rejection is a request-validation 400 (or a 413 variant); same gate as the
|
|
2399
|
-
// context observer beside it.
|
|
2400
1811
|
if (status !== 400 && status !== 413)
|
|
2401
1812
|
return;
|
|
2402
1813
|
if (target.model === undefined)
|
|
@@ -2410,21 +1821,8 @@ function observeMaxOutput(status, target, body) {
|
|
|
2410
1821
|
recordObservedMaxOutput(target.provider, target.model, stated);
|
|
2411
1822
|
}
|
|
2412
1823
|
catch {
|
|
2413
|
-
// Learning is best-effort and never in the request's way.
|
|
2414
1824
|
}
|
|
2415
1825
|
}
|
|
2416
|
-
/**
|
|
2417
|
-
* Record the durable half of provider-stated quota observations as learned rate-limit facts.
|
|
2418
|
-
*
|
|
2419
|
-
* A header pair like `x-ratelimit-limit-requests-day: 1000` is the deployment stating its own
|
|
2420
|
-
* ceiling — first-party evidence, same standing as an error body that says it. The observation
|
|
2421
|
-
* already had to be explicitly attributed (axis AND period named in the header, limit AND
|
|
2422
|
-
* remaining in one bucket), so nothing here guesses.
|
|
2423
|
-
*
|
|
2424
|
-
* ⚠ minute/day only. `quota-observation.ts` also recognizes month and unknown periods; there is no
|
|
2425
|
-
* measurement kind for either (a monthly ceiling is allowance territory, not rate), so they are
|
|
2426
|
-
* skipped rather than forced into a kind that would misstate them.
|
|
2427
|
-
*/
|
|
2428
1826
|
function observeStatedRateLimits(attempt, observations) {
|
|
2429
1827
|
if (observations.length === 0)
|
|
2430
1828
|
return;
|
|
@@ -2435,7 +1833,6 @@ function observeStatedRateLimits(attempt, observations) {
|
|
|
2435
1833
|
for (const o of observations) {
|
|
2436
1834
|
if (!Number.isFinite(o.limit) || o.limit <= 0)
|
|
2437
1835
|
continue;
|
|
2438
|
-
// Month/unknown have no fact kind on purpose (see above); only minute/day are measurements.
|
|
2439
1836
|
if (o.period !== "minute" && o.period !== "day")
|
|
2440
1837
|
continue;
|
|
2441
1838
|
recordObservedRateLimit(target.provider, attempt.resolvedAttempt.credentialId, target.model, [
|
|
@@ -2444,25 +1841,8 @@ function observeStatedRateLimits(attempt, observations) {
|
|
|
2444
1841
|
}
|
|
2445
1842
|
}
|
|
2446
1843
|
catch {
|
|
2447
|
-
// Learning is best-effort and never in the request's way — observeAttemptHeaders itself may
|
|
2448
|
-
// throw when the breaker rejects an observation, but this must not.
|
|
2449
1844
|
}
|
|
2450
1845
|
}
|
|
2451
|
-
/**
|
|
2452
|
-
* Learn a deployment's stated rate limits from an error body it just returned.
|
|
2453
|
-
*
|
|
2454
|
-
* ⚠ Called from BOTH request paths beside `observeContextLimit`, for the same reason that helper
|
|
2455
|
-
* documents: this repo has already shipped one defect where the OpenAI front had no copy of a
|
|
2456
|
-
* policy the Anthropic path enforced ("two paths, two policies, one of them empty" — see
|
|
2457
|
-
* docs/pool-failover.md). A learning loop that only ran on one front would silently know less
|
|
2458
|
-
* about half the traffic. Called with bytes already buffered by `inspectCandidateResponse`, so it
|
|
2459
|
-
* never tees or consumes the Response that may still become the client's terminal real error.
|
|
2460
|
-
*
|
|
2461
|
-
* ⚠ 429 only. A 402 (credits) or 403 (gating) is a different fact about a DIFFERENT axis, already
|
|
2462
|
-
* handled by the eligibility/refusal machinery; scanning those bodies for rate wording risks
|
|
2463
|
-
* recording a ceiling from prose that merely mentions one. Conservative default; revisit only with
|
|
2464
|
-
* evidence of a real provider stating its limit on a non-429.
|
|
2465
|
-
*/
|
|
2466
1846
|
function observeRateLimit(attempt, status, body) {
|
|
2467
1847
|
if (status !== 429)
|
|
2468
1848
|
return;
|
|
@@ -2477,15 +1857,8 @@ function observeRateLimit(attempt, status, body) {
|
|
|
2477
1857
|
recordObservedRateLimit(attempt.target.provider, attempt.credentialId, attempt.target.model, stated);
|
|
2478
1858
|
}
|
|
2479
1859
|
catch {
|
|
2480
|
-
// Learning is best-effort and never in the request's way.
|
|
2481
1860
|
}
|
|
2482
1861
|
}
|
|
2483
|
-
/**
|
|
2484
|
-
* Relay protocol adapters sometimes wrap a provider's JSON refusal inside their own error
|
|
2485
|
-
* message. Interpret the original nested payload as well as the outer wire body so a confirmed
|
|
2486
|
-
* signature means the same thing on every front. This is deterministic unwrapping only: no text
|
|
2487
|
-
* is classified or inferred here.
|
|
2488
|
-
*/
|
|
2489
1862
|
function refusalBodyCandidates(body) {
|
|
2490
1863
|
const candidates = [];
|
|
2491
1864
|
const queued = [body];
|
|
@@ -2511,7 +1884,6 @@ function refusalBodyCandidates(body) {
|
|
|
2511
1884
|
break;
|
|
2512
1885
|
}
|
|
2513
1886
|
catch {
|
|
2514
|
-
// Keep looking for the start of a nested JSON payload.
|
|
2515
1887
|
}
|
|
2516
1888
|
}
|
|
2517
1889
|
continue;
|
|
@@ -2554,72 +1926,25 @@ function observeEligibility(attempt, status, retryAfterMs, body) {
|
|
|
2554
1926
|
}
|
|
2555
1927
|
}
|
|
2556
1928
|
if (verdict === null) {
|
|
2557
|
-
// The fail-safe: an unrecognized refusal changes nothing about routing. It is held so a
|
|
2558
|
-
// researcher can say what it means, and only then will it ever bind.
|
|
2559
1929
|
recordUnknownRefusal(target.provider, target.model, status, body);
|
|
2560
1930
|
return { unknown: true };
|
|
2561
1931
|
}
|
|
2562
|
-
// The verdict's scope TEMPLATE becomes a concrete scope here, using this request's provider
|
|
2563
|
-
// and model. A provider-scoped verdict therefore covers every deployment behind that
|
|
2564
|
-
// credential from one observation — which is the whole point: a stated credit balance or a
|
|
2565
|
-
// rejected key is one fact, and rediscovering it once per model is pure waste.
|
|
2566
1932
|
const scope = materializeScope(verdict.scope, target.provider, attempt.credentialId, target.model);
|
|
2567
1933
|
const reset = resolveReset(verdict, retryAfterMs, matchedBody);
|
|
2568
1934
|
recordFact(verdict.class, scope, {
|
|
2569
1935
|
retryAfterMs: reset?.ms ?? null,
|
|
2570
1936
|
...(reset === null ? {} : { untilBasis: reset.basis }),
|
|
2571
|
-
// The verdict's cost filter travels with it. Without this the reviewer's "paid deployments
|
|
2572
|
-
// only" would be accepted, displayed, and then silently dropped on the way to the store —
|
|
2573
|
-
// the verdict would demote everything the scope covers, which is the defect it was written
|
|
2574
|
-
// to prevent.
|
|
2575
1937
|
...(verdict.costClasses === undefined ? {} : { costClasses: verdict.costClasses }),
|
|
2576
1938
|
});
|
|
2577
1939
|
return { unknown: false, scope };
|
|
2578
1940
|
}
|
|
2579
1941
|
catch {
|
|
2580
|
-
/* learning is best-effort and never in the request's way */
|
|
2581
1942
|
}
|
|
2582
1943
|
return { unknown: false };
|
|
2583
1944
|
}
|
|
2584
|
-
/**
|
|
2585
|
-
* Does the free-only guard bind this request?
|
|
2586
|
-
*
|
|
2587
|
-
* explicit `true` → always. The owner's standing "this lane never spends money", which a
|
|
2588
|
-
* per-call `@relay:` directive and a dispatch cliLane must not outrank.
|
|
2589
|
-
* explicit `false` → never. Spending was opted into deliberately.
|
|
2590
|
-
* UNSET → defaults ON for offload-rerouted traffic, OFF for a directly addressed pool.
|
|
2591
|
-
*
|
|
2592
|
-
* ⚠ That asymmetry is the whole point, and it is why `loadConfig` keeps "unset" distinguishable
|
|
2593
|
-
* from "false". Offload exists to spend somebody else's free capacity instead of your
|
|
2594
|
-
* subscription, so an install that has never thought about cost must not discover the feature by
|
|
2595
|
-
* being billed for it — the default belongs there. But a request that *names* `pool/<name>` is an
|
|
2596
|
-
* explicit routing choice by someone who knows what a pool is, and silently gating all of it on a
|
|
2597
|
-
* flag they never set would turn every unpriced deployment into a 503 for a decision they did not
|
|
2598
|
-
* make. Defaulting both the same way was tried: it failed 29 tests, all of them pool traffic that
|
|
2599
|
-
* had nothing to do with offload, which is exactly the surprise a user would have hit.
|
|
2600
|
-
*/
|
|
2601
1945
|
function freeOnlyApplies(rule, rerouted) {
|
|
2602
1946
|
return rule.freeOnly ?? rerouted;
|
|
2603
1947
|
}
|
|
2604
|
-
/**
|
|
2605
|
-
* When the condition this refusal describes clears, best evidence first.
|
|
2606
|
-
*
|
|
2607
|
-
* 1. `Retry-After` — the header, stated by THIS response.
|
|
2608
|
-
* 2. A reviewed `field` rule — also read from THIS response, just from a place only a reviewer
|
|
2609
|
-
* knew to look (Google's `retryDelay`, which no header carries).
|
|
2610
|
-
* 3. The generic body parse — the same class of evidence, found without being told where.
|
|
2611
|
-
* 4. A reviewed `fixed` window — a reviewer's knowledge of the provider, not a measurement of
|
|
2612
|
-
* this response, so it ranks below everything the response actually said.
|
|
2613
|
-
* 5. null ⇒ the fact kind's default TTL.
|
|
2614
|
-
*
|
|
2615
|
-
* ⚠ The ordering is the point. A reviewer may know MORE than the response (that a daily quota
|
|
2616
|
-
* resets in hours when the body says nothing), but never more than the response about itself — so
|
|
2617
|
-
* an assertion can fill a gap and can never overrule a statement.
|
|
2618
|
-
*
|
|
2619
|
-
* Returns the value AND which rung produced it, so `recordFact` can persist the provenance
|
|
2620
|
-
* (`untilBasis`) beside the expiry rather than the availability producer having to re-derive it
|
|
2621
|
-
* from vendor prose in a read path.
|
|
2622
|
-
*/
|
|
2623
1948
|
function resolveReset(interpretation, headerMs, body) {
|
|
2624
1949
|
if (headerMs !== null)
|
|
2625
1950
|
return { ms: headerMs, basis: "retry-after" };
|
|
@@ -2638,14 +1963,6 @@ function resolveReset(interpretation, headerMs, body) {
|
|
|
2638
1963
|
}
|
|
2639
1964
|
return null;
|
|
2640
1965
|
}
|
|
2641
|
-
/**
|
|
2642
|
-
* Statuses whose body can carry a durable fact.
|
|
2643
|
-
*
|
|
2644
|
-
* 429 is included, but narrowly: an ordinary rate limit stays the breaker's business and produces
|
|
2645
|
-
* no fact, because only the small set of messages that NAME an account-level limit matches a seed.
|
|
2646
|
-
* Reading the body costs nothing here — it is already being discarded — and the alternative was
|
|
2647
|
-
* never learning the one 429 that is worth learning.
|
|
2648
|
-
*/
|
|
2649
1966
|
function carriesEligibilityFact(status) {
|
|
2650
1967
|
return (status === 400 ||
|
|
2651
1968
|
status === 401 ||
|
|
@@ -2655,12 +1972,6 @@ function carriesEligibilityFact(status) {
|
|
|
2655
1972
|
status === 410 ||
|
|
2656
1973
|
status === 429);
|
|
2657
1974
|
}
|
|
2658
|
-
/**
|
|
2659
|
-
* Inspect an error response once and rebuild it from the same bytes. The caller may either discard
|
|
2660
|
-
* the rebuilt response when CredentialWalk offers another candidate or serve it as the terminal
|
|
2661
|
-
* real upstream error. A body failure after headers is returned as a distinct protocol outcome;
|
|
2662
|
-
* it must never be rebuilt as an empty response under the provider's original credential status.
|
|
2663
|
-
*/
|
|
2664
1975
|
async function inspectCandidateResponse(res, attempt, retryAfterMs) {
|
|
2665
1976
|
const propagatedFailure = postHeaderBodyFailure(res);
|
|
2666
1977
|
if (propagatedFailure)
|
|
@@ -2706,15 +2017,9 @@ function walkOutcomeForResponse(status, localFailure, scope) {
|
|
|
2706
2017
|
}
|
|
2707
2018
|
function observeAttemptHeaders(h, attempt, status, retryAfterMs, headers) {
|
|
2708
2019
|
const observedAt = Date.now();
|
|
2709
|
-
// Provider-stated quota from the response that just served real traffic. Keep every attributed
|
|
2710
|
-
// axis/period pair: reducing it to one scalar can conflate requests/day with tokens/minute.
|
|
2711
2020
|
const quotaObservations = headers
|
|
2712
2021
|
? extractQuotaObservations(headers, { observedAt })
|
|
2713
2022
|
: [];
|
|
2714
|
-
// The `limit` half of a provider-stated quota observation is DURABLE — it is what the deployment
|
|
2715
|
-
// says it allows — so it is also recorded as a learned rate-limit fact (spec §4 Rung 1: header
|
|
2716
|
-
// observations are not their own rung; the limit IS the rung-1 measurement). `remaining` and
|
|
2717
|
-
// `resetsAt` stay volatile here with the breaker.
|
|
2718
2023
|
observeStatedRateLimits(attempt, quotaObservations);
|
|
2719
2024
|
const result = h.breaker.observeHeaders(attempt.handle, {
|
|
2720
2025
|
target: attempt.identity,
|
|
@@ -2760,22 +2065,13 @@ function completeAttemptSuccess(h, attempt, status) {
|
|
|
2760
2065
|
attempt.trace.record(attempt.target, status, attempt.started, completedAt);
|
|
2761
2066
|
recordCall(h, attempt, true, completedAt);
|
|
2762
2067
|
attempt.accounting?.complete(attempt.accountingAttempt, "success", null, attempt.usage, completedAt);
|
|
2763
|
-
// A served request is first-party proof that this deployment exists and that the credential has
|
|
2764
|
-
// allowance RIGHT NOW — strictly better evidence than any stored refusal, so it clears the
|
|
2765
|
-
// record, including the account-scoped one. That is how a topped-up balance or a rolled-over
|
|
2766
|
-
// month recovers well before the TTL would have expired, with no restart. Same contract as the
|
|
2767
|
-
// breaker clearing a credential fault on success.
|
|
2768
2068
|
try {
|
|
2769
2069
|
const cleared = clearFacts(attempt.target.provider, attempt.resolvedAttempt.credentialId, attempt.target.model ?? null);
|
|
2770
|
-
// A stated bad credential has just been disproved, so the per-deployment 401s it caused are
|
|
2771
|
-
// stale evidence about a problem that no longer exists. Clearing them together is what makes a
|
|
2772
|
-
// key rotation recover the WHOLE provider at once instead of one model per expiry.
|
|
2773
2070
|
if (cleared.includes("credential-invalid")) {
|
|
2774
2071
|
h.breaker.clearCredentialFaults(attempt.resolvedAttempt.credentialId);
|
|
2775
2072
|
}
|
|
2776
2073
|
}
|
|
2777
2074
|
catch {
|
|
2778
|
-
/* best-effort */
|
|
2779
2075
|
}
|
|
2780
2076
|
}
|
|
2781
2077
|
function completeAttemptFailure(h, attempt, options) {
|
|
@@ -2819,11 +2115,6 @@ function completeAttemptCancelled(h, attempt, reason) {
|
|
|
2819
2115
|
attempt.trace.record(attempt.target, "cancelled", attempt.started, completedAt);
|
|
2820
2116
|
attempt.accounting?.complete(attempt.accountingAttempt, "cancelled", "aborted", attempt.usage, completedAt);
|
|
2821
2117
|
}
|
|
2822
|
-
/**
|
|
2823
|
-
* A Response proves fetch reached the provider and received headers. A later body rejection is
|
|
2824
|
-
* therefore not a provider-wide transport failure and a credential-looking status is not usable
|
|
2825
|
-
* evidence about the credential. Close only this deployment unless the client or deadline won.
|
|
2826
|
-
*/
|
|
2827
2118
|
function completePostHeaderBodyFailure(h, downstream, signal, attempt, credentialWalk, credentialTrace, resolvedAttempt) {
|
|
2828
2119
|
if (downstream.destroyed) {
|
|
2829
2120
|
completeAttemptCancelled(h, attempt, "client disconnected while reading provider response body");
|
|
@@ -2856,30 +2147,12 @@ function detectOpenAiFrontProtocol(method, pathname) {
|
|
|
2856
2147
|
return "responses";
|
|
2857
2148
|
return null;
|
|
2858
2149
|
}
|
|
2859
|
-
/**
|
|
2860
|
-
* OpenAI front: serve a resolved candidate's Chat Completions or Responses request, preserving
|
|
2861
|
-
* the OpenAI wire contract. Direct OpenAI Chat Completions remain verbatim; translated paths use
|
|
2862
|
-
* the backend adapter, which keeps the internal Anthropic-shaped seam but does not run the
|
|
2863
|
-
* Anthropic tool-repair layer for OpenAI callers.
|
|
2864
|
-
*
|
|
2865
|
-
* ⚠ This path failed over across candidates for exactly as long as this comment has existed,
|
|
2866
|
-
* which is to say it never did. It was handed `healthyTargets[0]` and returned before the
|
|
2867
|
-
* Anthropic path's failover loop, and it reported outcomes to runtime telemetry but never to the
|
|
2868
|
-
* circuit breaker — so a rate-limited candidate was neither stepped over within a request nor
|
|
2869
|
-
* demoted for the next one, and a 14-member pool served every single request from the same dead
|
|
2870
|
-
* member. Measured: 8 sequential requests to a 14-candidate pool, 6 consecutive 429s, 0 other
|
|
2871
|
-
* candidates tried, breaker `lastStatus: null` throughout. Both halves are fixed here; the
|
|
2872
|
-
* classification and breaker accounting come from the SAME helpers the Anthropic path uses, so
|
|
2873
|
-
* the two cannot drift apart again.
|
|
2874
|
-
*/
|
|
2875
2150
|
async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
2876
2151
|
const tried = [];
|
|
2877
2152
|
const attemptTrace = new RequestAttemptTrace();
|
|
2878
2153
|
const directTools = toolSchemaMap(ctx.reqJson);
|
|
2879
2154
|
const pool429 = new Pool429Tracker();
|
|
2880
2155
|
while (!res.destroyed) {
|
|
2881
|
-
// G2's attempt boundary, shared with the Anthropic front: a capped offer is refused here
|
|
2882
|
-
// before any egress and the loop comes straight back for the next candidate.
|
|
2883
2156
|
const resolvedAttempt = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
|
|
2884
2157
|
if (!resolvedAttempt)
|
|
2885
2158
|
break;
|
|
@@ -2918,7 +2191,6 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
2918
2191
|
}
|
|
2919
2192
|
throw e;
|
|
2920
2193
|
}
|
|
2921
|
-
/* attempt begins at the real egress callback */
|
|
2922
2194
|
const recoveryAudit = { value: null };
|
|
2923
2195
|
const processRecoveredChat = async (recovered) => {
|
|
2924
2196
|
const assistant = {
|
|
@@ -3128,8 +2400,6 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
3128
2400
|
errorOrigin: probe.provenance,
|
|
3129
2401
|
servedBy: tried.join(", "),
|
|
3130
2402
|
shouldTryNext: probe.provenance === "upstream",
|
|
3131
|
-
// Preserved residue: OpenAI dead streams require both writable-open and not
|
|
3132
|
-
// destroyed; handle's Anthropic path checks only `!res.destroyed`.
|
|
3133
2403
|
});
|
|
3134
2404
|
if (walkEnd)
|
|
3135
2405
|
continue;
|
|
@@ -3152,9 +2422,6 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
3152
2422
|
pool429.recordFinal(upstream.status);
|
|
3153
2423
|
let responseBytesWritten = false;
|
|
3154
2424
|
try {
|
|
3155
|
-
// One announcement set for both fronts — see `ServedAnnouncementContext`. The credential
|
|
3156
|
-
// headers are read here, after `recordCredentialOutcome` above settled the winning slot,
|
|
3157
|
-
// which is the per-attempt attribution a bare shared call would otherwise lose.
|
|
3158
2425
|
const headers = responseHeadersForTarget(upstream, {
|
|
3159
2426
|
target,
|
|
3160
2427
|
tried,
|
|
@@ -3163,7 +2430,6 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
3163
2430
|
poolUnknownRefusals: pool429.unknownCount(),
|
|
3164
2431
|
credentialHeaders: credentialTrace.headers(),
|
|
3165
2432
|
degraded: degradedLabel(ctx.addressedPool ?? null, ctx.degradedSpecs ?? null, target),
|
|
3166
|
-
// The announcement computed at walk-order time, beside the same demotion that produced it.
|
|
3167
2433
|
quotaDemoted: ctx.quotaDemotedFirst,
|
|
3168
2434
|
paid: ctx.cfg ? paidLabel(ctx.cfg, h, target) : null,
|
|
3169
2435
|
sticky: ctx.sticky,
|
|
@@ -3212,10 +2478,6 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
3212
2478
|
const audit = recoveryAudit.value;
|
|
3213
2479
|
const log = {
|
|
3214
2480
|
...baseLog(ctx.started, ctx.path, ctx.hadTools, streamed, upstream.status, audit?.validated ?? "skipped", target, attemptTrace.snapshot(), upstreamReportedModel(reportedModelSource), resolvedAttempt.credentialId),
|
|
3215
|
-
// This front's TRANSLATED lane (anything but openai-kind + Chat) runs through
|
|
3216
|
-
// `fetchBackend`, so a Codex `/v1/responses` turn on an openai-kind target gets the same
|
|
3217
|
-
// id mint the Messages front does — and it must be reported on the SERVED turn, not only
|
|
3218
|
-
// on the mid-stream error path that already carried it.
|
|
3219
2481
|
...toolUseIdRewriteField(reportedModelSource),
|
|
3220
2482
|
};
|
|
3221
2483
|
h.logger.write(audit ? {
|
|
@@ -3250,29 +2512,13 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
|
|
|
3250
2512
|
res.off("close", onResClose);
|
|
3251
2513
|
}
|
|
3252
2514
|
}
|
|
3253
|
-
// Same single refusal site as the Anthropic front (see the note there): a walk whose every
|
|
3254
|
-
// candidate was refused before the first egress reaches here with nothing sent, and owes the
|
|
3255
|
-
// client its refusal. Anything that egressed keeps its own upstream error instead.
|
|
3256
2515
|
if (!res.headersSent && !res.destroyed && pool429.allCapped()) {
|
|
3257
2516
|
respondAllCapped(res, h, { started: ctx.started, path: ctx.path, hadTools: ctx.hadTools, streamed: ctx.wantsStream }, "openai", pool429, attemptTrace.snapshot());
|
|
3258
2517
|
}
|
|
3259
2518
|
}
|
|
3260
|
-
/**
|
|
3261
|
-
* The metadata-only `tool_use` id-minting counter for this response, or nothing when the host's
|
|
3262
|
-
* own ids were already unique.
|
|
3263
|
-
*
|
|
3264
|
-
* Read from the response the backend adapter produced, and only at LOG time: on a stream the pass
|
|
3265
|
-
* runs while the body drains, so the figure is final once the request is over and would be a guess
|
|
3266
|
-
* any earlier. A count, never an id.
|
|
3267
|
-
*/
|
|
3268
2519
|
function toolUseIdRewriteField(source) {
|
|
3269
2520
|
const rewrites = toolUseIdRewrites(source);
|
|
3270
|
-
// The REQUEST-direction sibling: ids the mapper rewrote to the provider's stated shape
|
|
3271
|
-
// (`compat.toolCallIds: "strict9"`). Final before egress, but read here so both counters travel
|
|
3272
|
-
// together and a stream reports both in the one place it can.
|
|
3273
2521
|
const outbound = toolCallIdRewrites(source);
|
|
3274
|
-
// The other request-direction pass (`compat.thoughtSignature: "sentinel"`). It has NO response
|
|
3275
|
-
// header of its own, so this is the only surface the operator can see it on.
|
|
3276
2522
|
const sentinels = thoughtSignatureSentinels(source);
|
|
3277
2523
|
return {
|
|
3278
2524
|
...(rewrites === undefined ? {} : { toolUseIdRewrites: rewrites }),
|
|
@@ -3280,16 +2526,8 @@ function toolUseIdRewriteField(source) {
|
|
|
3280
2526
|
...(sentinels === undefined ? {} : { thoughtSignatureSentinels: sentinels }),
|
|
3281
2527
|
};
|
|
3282
2528
|
}
|
|
3283
|
-
/** Winning-candidate response metadata, shared by transparent and repair streaming paths. */
|
|
3284
2529
|
function responseHeadersForTarget(backendRes, ctx) {
|
|
3285
2530
|
const responseHeaders = filterResponseHeaders(backendRes.headers);
|
|
3286
|
-
// The winner below 400, every deployment tried at or above it. The header's own declaration is
|
|
3287
|
-
// the contract — "when every candidate fails it carries the list that was tried instead, so an
|
|
3288
|
-
// exhausted pool is self-describing" — and `docs/reference.md` states it to users. This front
|
|
3289
|
-
// used to omit it entirely on a terminal error while the OpenAI front supplied it, so the same
|
|
3290
|
-
// documented behaviour was delivered by one path and not the other.
|
|
3291
|
-
// ⚠ Not the transport-exit omission at the `handle` catch, which is a RECORDED deliberate
|
|
3292
|
-
// residue: a transport exhaustion has no HTTP response to describe.
|
|
3293
2531
|
responseHeaders[SERVED_BY_HEADER] = backendRes.status < 400 || !ctx.tried?.length
|
|
3294
2532
|
? specOfTarget(ctx.target)
|
|
3295
2533
|
: ctx.tried.join(", ");
|
|
@@ -3298,12 +2536,6 @@ function responseHeadersForTarget(backendRes, ctx) {
|
|
|
3298
2536
|
}
|
|
3299
2537
|
if (ctx.poolSummary)
|
|
3300
2538
|
responseHeaders[POOL_ATTEMPTS_HEADER] = ctx.poolSummary;
|
|
3301
|
-
// Counts only candidates stepped over: terminal refusal bodies are not buffered on this path.
|
|
3302
|
-
// ⚠ Emitted on SUCCESS too, and that is the point: the count is the push half of the eligibility
|
|
3303
|
-
// queue — "a NEW kind of refusal just appeared, run `llm-relay eligibility` while the context is
|
|
3304
|
-
// still in hand". A later candidate answering does not un-see the refusal. The OpenAI front used
|
|
3305
|
-
// to compute it only inside its own `status >= 400` branch and so swallowed exactly that case.
|
|
3306
|
-
// The count is positive-or-null and never 0, so this truthy test is exact.
|
|
3307
2539
|
if (ctx.poolUnknownRefusals)
|
|
3308
2540
|
responseHeaders[UNKNOWN_REFUSAL_HEADER] = String(ctx.poolUnknownRefusals);
|
|
3309
2541
|
if (ctx.degraded)
|
|
@@ -3319,7 +2551,6 @@ function responseHeadersForTarget(backendRes, ctx) {
|
|
|
3319
2551
|
responseHeaders[STICKY_PROVENANCE_HEADER] = sticky;
|
|
3320
2552
|
return responseHeaders;
|
|
3321
2553
|
}
|
|
3322
|
-
/** detect/default: forward bytes unchanged, observe + log if applicable. */
|
|
3323
2554
|
async function transparentPath(res, backendRes, timer, ctx, h) {
|
|
3324
2555
|
if (!res.headersSent)
|
|
3325
2556
|
res.writeHead(backendRes.status, responseHeadersForTarget(backendRes, ctx));
|
|
@@ -3358,29 +2589,17 @@ async function transparentPath(res, backendRes, timer, ctx, h) {
|
|
|
3358
2589
|
markAttemptCommitted(ctx.attempt);
|
|
3359
2590
|
res.end(bytes);
|
|
3360
2591
|
}
|
|
3361
|
-
// Same as the OpenAI front's terminal branch: the served response is the last candidate's,
|
|
3362
|
-
// and for a single-member pool the only refusal we will ever see. Errors are never streamed,
|
|
3363
|
-
// so this branch is where they land.
|
|
3364
|
-
// NOTE: observeEligibility was already called in inspectCandidateResponse for this response.
|
|
3365
|
-
// Calling it again here would double-count the refusal signature in the eligibility queue.
|
|
3366
2592
|
if (ctx.willValidate && bytes.length <= MAX_VALIDATE_BYTES)
|
|
3367
2593
|
assistant = parseAssistant(bytes.toString("utf8"));
|
|
3368
2594
|
}
|
|
3369
2595
|
}
|
|
3370
2596
|
catch (e) {
|
|
3371
|
-
// The head was committed at the top of this function, so the client is already
|
|
3372
|
-
// reading a 200. Say the stream broke rather than closing on a truncated answer,
|
|
3373
|
-
// and LOG the turn — this throw used to escape to the top-level catch, which
|
|
3374
|
-
// could only `res.end()` and never logged.
|
|
3375
2597
|
handleMidStreamError(res, e, ctx.started, ctx.path, ctx.hadTools, ctx.streamed, backendRes.status, ctx.target, ctx.attempt, h, (msg) => ctx.streamed ? sseError(msg) : null, ctx.reportedModelSource, ctx.signal.aborted, responseBytesWritten);
|
|
3376
2598
|
return;
|
|
3377
2599
|
}
|
|
3378
2600
|
finally {
|
|
3379
2601
|
clearTimeout(timer);
|
|
3380
2602
|
}
|
|
3381
|
-
// A client disconnect can make writeChunk stop without throwing. Leave the handle pending so
|
|
3382
|
-
// the routing-level finally records cancellation instead of validating a partial response and
|
|
3383
|
-
// charging it to provider health.
|
|
3384
2603
|
if (res.destroyed)
|
|
3385
2604
|
return;
|
|
3386
2605
|
let validated = "skipped";
|
|
@@ -3420,7 +2639,6 @@ async function transparentPath(res, backendRes, timer, ctx, h) {
|
|
|
3420
2639
|
...toolUseIdRewriteField(ctx.reportedModelSource),
|
|
3421
2640
|
});
|
|
3422
2641
|
}
|
|
3423
|
-
/** repair: route to the streaming or buffered variant. */
|
|
3424
2642
|
async function repairPath(res, backendRes, timer, ctx, h) {
|
|
3425
2643
|
if (ctx.streamed) {
|
|
3426
2644
|
await repairStreamingPath(res, backendRes, timer, ctx, h);
|
|
@@ -3430,21 +2648,12 @@ async function repairPath(res, backendRes, timer, ctx, h) {
|
|
|
3430
2648
|
return repairBufferedPath(res, backendRes, timer, ctx, h);
|
|
3431
2649
|
}
|
|
3432
2650
|
}
|
|
3433
|
-
/**
|
|
3434
|
-
* repair, streaming: forward text-block SSE frames to the client as they arrive;
|
|
3435
|
-
* withhold everything from the first tool_use `content_block_start` onward. At
|
|
3436
|
-
* end-of-stream, validate the reconstructed message — if the tool calls are valid,
|
|
3437
|
-
* flush the withheld frames byte-for-byte (fully transparent); if invalid, repair
|
|
3438
|
-
* and re-emit only the corrected trailing blocks. `message_start` and any leading
|
|
3439
|
-
* text have already reached the client, so a pure-text response streams through
|
|
3440
|
-
* with zero added latency.
|
|
3441
|
-
*/
|
|
3442
2651
|
async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
3443
2652
|
const filtered = responseHeadersForTarget(backendRes, ctx);
|
|
3444
|
-
let acc = "";
|
|
3445
|
-
let overflow = false;
|
|
3446
|
-
let work = Buffer.alloc(0);
|
|
3447
|
-
const held = [];
|
|
2653
|
+
let acc = "";
|
|
2654
|
+
let overflow = false;
|
|
2655
|
+
let work = Buffer.alloc(0);
|
|
2656
|
+
const held = [];
|
|
3448
2657
|
let buffering = false;
|
|
3449
2658
|
let firstToolUseIndex = -1;
|
|
3450
2659
|
let headWritten = res.headersSent;
|
|
@@ -3472,7 +2681,6 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3472
2681
|
if (!overflow) {
|
|
3473
2682
|
acc += frame.toString("utf8");
|
|
3474
2683
|
if (acc.length > MAX_VALIDATE_BYTES) {
|
|
3475
|
-
// Too large to validate/repair safely: stop holding, stream the rest.
|
|
3476
2684
|
overflow = true;
|
|
3477
2685
|
if (buffering) {
|
|
3478
2686
|
await flushHeld();
|
|
@@ -3502,21 +2710,15 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3502
2710
|
let end;
|
|
3503
2711
|
while ((end = frameEnd(work)) !== -1) {
|
|
3504
2712
|
const frame = work.subarray(0, end);
|
|
3505
|
-
work = Buffer.from(work.subarray(end));
|
|
2713
|
+
work = Buffer.from(work.subarray(end));
|
|
3506
2714
|
await processFrame(frame);
|
|
3507
2715
|
}
|
|
3508
2716
|
}
|
|
3509
2717
|
}
|
|
3510
2718
|
if (work.length)
|
|
3511
|
-
await processFrame(work);
|
|
2719
|
+
await processFrame(work);
|
|
3512
2720
|
}
|
|
3513
2721
|
catch (e) {
|
|
3514
|
-
// The upstream stream broke part-way. Anything withheld in `held` is a partial
|
|
3515
|
-
// tool call and is DROPPED rather than flushed — half a tool_use is exactly the
|
|
3516
|
-
// malformed call this proxy exists to keep out of the harness — and the client
|
|
3517
|
-
// gets an explicit error event instead of a stream that simply stops. If the
|
|
3518
|
-
// head has not been written yet (a failure before any text frame) this is still
|
|
3519
|
-
// a clean 502. Either way the turn is logged.
|
|
3520
2722
|
held.length = 0;
|
|
3521
2723
|
handleMidStreamError(res, e, ctx.started, ctx.path, ctx.hadTools, true, backendRes.status, ctx.target, ctx.attempt, h, sseError, ctx.reportedModelSource, ctx.signal.aborted, responseBytesWritten);
|
|
3522
2724
|
return;
|
|
@@ -3532,7 +2734,6 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3532
2734
|
let errorKinds = [];
|
|
3533
2735
|
let repairOutcome = "none";
|
|
3534
2736
|
if (overflow || !buffering) {
|
|
3535
|
-
// Nothing was withheld (pure text, or gave up): stream already complete.
|
|
3536
2737
|
if (!buffering) {
|
|
3537
2738
|
const assistant = overflow ? null : reconstructFromSse(acc);
|
|
3538
2739
|
if (assistant) {
|
|
@@ -3571,7 +2772,7 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3571
2772
|
signal: ctx.callerSignal,
|
|
3572
2773
|
});
|
|
3573
2774
|
repairOutcome = decision.outcome;
|
|
3574
|
-
ensureHead();
|
|
2775
|
+
ensureHead();
|
|
3575
2776
|
if (decision.outcome === "fixed" && decision.message) {
|
|
3576
2777
|
if (!res.writableEnded) {
|
|
3577
2778
|
const tail = emitSseTail(decision.message, firstToolUseIndex);
|
|
@@ -3581,7 +2782,6 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3581
2782
|
}
|
|
3582
2783
|
}
|
|
3583
2784
|
else {
|
|
3584
|
-
// Head already committed — surface a mid-stream SSE error, never a fabricated call.
|
|
3585
2785
|
if (!res.writableEnded)
|
|
3586
2786
|
res.end(sseError(`llm-relay: tool call could not be repaired (${decision.outcome})`));
|
|
3587
2787
|
}
|
|
@@ -3614,16 +2814,12 @@ async function repairStreamingPath(res, backendRes, timer, ctx, h) {
|
|
|
3614
2814
|
...toolUseIdRewriteField(ctx.reportedModelSource),
|
|
3615
2815
|
});
|
|
3616
2816
|
}
|
|
3617
|
-
/** repair, buffered (non-streamed JSON): buffer, validate; if invalid, reshape and re-emit. */
|
|
3618
2817
|
async function repairBufferedPath(res, backendRes, timer, ctx, h) {
|
|
3619
2818
|
let bytes;
|
|
3620
2819
|
try {
|
|
3621
2820
|
bytes = Buffer.from(await backendRes.arrayBuffer());
|
|
3622
2821
|
}
|
|
3623
2822
|
catch (e) {
|
|
3624
|
-
// Nothing has been written yet on this path, so this is a clean 502 rather than
|
|
3625
|
-
// a truncated body — but it still has to be LOGGED, which the bare finally did
|
|
3626
|
-
// not do: the throw went straight to the top-level catch.
|
|
3627
2823
|
handleMidStreamError(res, e, ctx.started, ctx.path, ctx.hadTools, ctx.streamed, backendRes.status, ctx.target, ctx.attempt, h, () => null, ctx.reportedModelSource, ctx.signal.aborted);
|
|
3628
2824
|
return null;
|
|
3629
2825
|
}
|
|
@@ -3635,8 +2831,6 @@ async function repairBufferedPath(res, backendRes, timer, ctx, h) {
|
|
|
3635
2831
|
const assistant = ctx.streamed
|
|
3636
2832
|
? reconstructFromSse(bytes.toString("utf8"))
|
|
3637
2833
|
: parseAssistant(bytes.toString("utf8"));
|
|
3638
|
-
// W8's winning-candidate metadata assembly is shared with transparent and streaming repair;
|
|
3639
|
-
// sticky provenance must describe the candidate that actually commits here too.
|
|
3640
2834
|
const filtered = responseHeadersForTarget(backendRes, ctx);
|
|
3641
2835
|
let repairOutcome = "none";
|
|
3642
2836
|
let validated = "skipped";
|
|
@@ -3654,8 +2848,6 @@ async function repairBufferedPath(res, backendRes, timer, ctx, h) {
|
|
|
3654
2848
|
};
|
|
3655
2849
|
if (!assistant) {
|
|
3656
2850
|
if (backendRes.status < 400) {
|
|
3657
|
-
// A successful status with no Anthropic message is not a successful provider attempt.
|
|
3658
|
-
// Nothing has been committed yet on this buffered path, so return a bounded clean 502.
|
|
3659
2851
|
validated = "fail";
|
|
3660
2852
|
errorKinds = ["invalid_upstream_envelope"];
|
|
3661
2853
|
failClosed(res, 502, "llm-relay: invalid Anthropic upstream envelope", recordFinalWalk(502, stickyProvenanceHeaders(ctx.sticky)));
|
|
@@ -3677,7 +2869,7 @@ async function repairBufferedPath(res, backendRes, timer, ctx, h) {
|
|
|
3677
2869
|
if (r.valid) {
|
|
3678
2870
|
validated = r.uncheckableCount > 0 ? "uncheckable" : "pass";
|
|
3679
2871
|
recordFinalWalk(backendRes.status, filtered);
|
|
3680
|
-
res.writeHead(backendRes.status, filtered);
|
|
2872
|
+
res.writeHead(backendRes.status, filtered);
|
|
3681
2873
|
if (!res.writableEnded) {
|
|
3682
2874
|
if (backendRes.status < 400 && bytes.length > 0)
|
|
3683
2875
|
markAttemptCommitted(ctx.attempt);
|
|
@@ -3701,11 +2893,8 @@ async function repairBufferedPath(res, backendRes, timer, ctx, h) {
|
|
|
3701
2893
|
emitFixed(res, backendRes.status, filtered, decision.message, ctx.wantsStream, () => markAttemptCommitted(ctx.attempt));
|
|
3702
2894
|
}
|
|
3703
2895
|
else if (decision.outcome === "failed") {
|
|
3704
|
-
// Nothing has been committed on the buffered path. The outer candidate loop decides
|
|
3705
|
-
// whether this request can resume or whether this remains the terminal fail-clean 502.
|
|
3706
2896
|
}
|
|
3707
2897
|
else {
|
|
3708
|
-
// fail-clean: loud, well-formed error rather than a silently broken call.
|
|
3709
2898
|
failClosed(res, 502, `llm-relay: tool call could not be repaired (${decision.outcome})`, recordFinalWalk(502, stickyProvenanceHeaders(ctx.sticky)));
|
|
3710
2899
|
}
|
|
3711
2900
|
}
|
|
@@ -3761,18 +2950,6 @@ function emitFixed(res, status, filtered, message, wantsStream, beforeBody) {
|
|
|
3761
2950
|
}
|
|
3762
2951
|
}
|
|
3763
2952
|
}
|
|
3764
|
-
/**
|
|
3765
|
-
* Re-serialize a repaired message as a buffered Anthropic Message — the JSON counterpart
|
|
3766
|
-
* of `emitSse`, and it now follows the same rules.
|
|
3767
|
-
*
|
|
3768
|
-
* It used to hardcode `id: "msg_repair"`, take `model` from what the CLIENT asked for (which
|
|
3769
|
-
* is routinely not the deployment that answered), and zero-fill `usage`. All three rewrote
|
|
3770
|
-
* the response's identity on the way through: every repaired turn looked like the same
|
|
3771
|
-
* message, attributed to the wrong model, reporting a token count nobody measured. The
|
|
3772
|
-
* backend's own values are carried whenever the response had them; an unknown id falls back
|
|
3773
|
-
* to the same relay-marked synthetic id the streaming path uses, and an unreported `usage`
|
|
3774
|
-
* is OMITTED rather than stated as zero.
|
|
3775
|
-
*/
|
|
3776
2953
|
function toAnthropicMessage(msg) {
|
|
3777
2954
|
return {
|
|
3778
2955
|
id: msg.id ?? syntheticMessageId(),
|
|
@@ -3785,46 +2962,14 @@ function toAnthropicMessage(msg) {
|
|
|
3785
2962
|
...(msg.usage ? { usage: msg.usage } : {}),
|
|
3786
2963
|
};
|
|
3787
2964
|
}
|
|
3788
|
-
/** A provider declared an authEnv whose variable is unset — a configuration error, not a passthrough. */
|
|
3789
2965
|
export class CredentialConfigError extends Error {
|
|
3790
2966
|
constructor(provider, authEnv) {
|
|
3791
2967
|
super(`provider "${provider}" declares authEnv ${authEnv} but it is unset or blank`);
|
|
3792
2968
|
this.name = "CredentialConfigError";
|
|
3793
2969
|
}
|
|
3794
2970
|
}
|
|
3795
|
-
/**
|
|
3796
|
-
* The headers forwarded upstream for one request (INV-HS-8).
|
|
3797
|
-
*
|
|
3798
|
-
* Exported so the `declared-missing` branch is directly assertable: it is meant to
|
|
3799
|
-
* be unreachable through the request path (`resolveTargets` drops keyless targets),
|
|
3800
|
-
* so an end-to-end test cannot reach it, and an invariant nothing can check is an
|
|
3801
|
-
* invariant that quietly stops holding.
|
|
3802
|
-
*
|
|
3803
|
-
* The three `CredentialState`s are three different obligations:
|
|
3804
|
-
* - `not-declared` — a real passthrough. The caller's own credential is FORWARDED;
|
|
3805
|
-
* that is the whole point of the anthropic passthrough. A provider may
|
|
3806
|
-
* DECLARE this with `credentialMode: "passthrough"`, and declare the
|
|
3807
|
-
* opposite with `credentialMode: "contained"` — a keyless backend that is
|
|
3808
|
-
* not the caller's own vendor (a local daemon, a second relay, someone
|
|
3809
|
-
* else's Anthropic-format endpoint) must not receive their credential just
|
|
3810
|
-
* because it needs none of its own. Omitting the field still forwards, so
|
|
3811
|
-
* existing configs keep working, and config load warns instead.
|
|
3812
|
-
* - `declared-present` — the provider's own key is attached and the caller's inbound
|
|
3813
|
-
* `Authorization`/`x-api-key` are REMOVED, never merged.
|
|
3814
|
-
* - `declared-missing` — the caller's inbound `Authorization`/`x-api-key` are REMOVED
|
|
3815
|
-
* *and* the request fails. "Removed" is strictly stronger than
|
|
3816
|
-
* "we did not add one of our own": the defect this replaces
|
|
3817
|
-
* satisfied the weaker reading while forwarding the caller's
|
|
3818
|
-
* Anthropic token verbatim to a third-party base URL.
|
|
3819
|
-
*/
|
|
3820
2971
|
export function buildForwardHeaders(inbound, attempt) {
|
|
3821
2972
|
const { target, credential } = attempt;
|
|
3822
|
-
// Containment is DECLARED, not inferred from key presence. The old
|
|
3823
|
-
// `stripAuth = !!apiKey` was identically falsy for two opposite configurations —
|
|
3824
|
-
// "no authEnv declared" (an intentional passthrough: forward the caller's own
|
|
3825
|
-
// credential) and "authEnv declared but unset" (a misconfiguration) — so in the
|
|
3826
|
-
// second case the caller's own Anthropic token was forwarded verbatim to a
|
|
3827
|
-
// third-party base URL. Only a real passthrough forwards inbound auth now.
|
|
3828
2973
|
const stripAuth = credential.state !== "not-declared" || target.credentialMode === "contained";
|
|
3829
2974
|
const out = {};
|
|
3830
2975
|
for (const [k, v] of Object.entries(inbound)) {
|
|
@@ -3833,10 +2978,6 @@ export function buildForwardHeaders(inbound, attempt) {
|
|
|
3833
2978
|
continue;
|
|
3834
2979
|
if (INTERNAL_REQUEST_HEADERS.has(key))
|
|
3835
2980
|
continue;
|
|
3836
|
-
// The REMOVAL, for both declared states. It happens before the throw below so
|
|
3837
|
-
// that no code path can observe a header map still carrying the caller's
|
|
3838
|
-
// credential — not the throw's own error, not a future caller that decides to
|
|
3839
|
-
// handle the error and reuse what was built.
|
|
3840
2981
|
if (stripAuth && INBOUND_AUTH.includes(key))
|
|
3841
2982
|
continue;
|
|
3842
2983
|
if (v === undefined)
|
|
@@ -3846,9 +2987,6 @@ export function buildForwardHeaders(inbound, attempt) {
|
|
|
3846
2987
|
if (!out["anthropic-version"])
|
|
3847
2988
|
out["anthropic-version"] = DEFAULT_ANTHROPIC_VERSION;
|
|
3848
2989
|
if (credential.state === "declared-missing") {
|
|
3849
|
-
// Should be unreachable — resolveTargets drops keyless targets — but thrown
|
|
3850
|
-
// rather than silently proceeding so a routing change that lets one through
|
|
3851
|
-
// fails loudly instead of egressing whatever the caller happened to send.
|
|
3852
2990
|
throw new CredentialConfigError(target.provider, target.authEnv);
|
|
3853
2991
|
}
|
|
3854
2992
|
Object.assign(out, buildAuthHeaders(credential.value, target.authHeader));
|
|
@@ -3885,15 +3023,6 @@ async function forwardLocalResponse(res, response) {
|
|
|
3885
3023
|
res.writeHead(response.status, headers);
|
|
3886
3024
|
res.end(bytes);
|
|
3887
3025
|
}
|
|
3888
|
-
/**
|
|
3889
|
-
* Parse a buffered (non-streamed) backend response into the shape the validator inspects.
|
|
3890
|
-
*
|
|
3891
|
-
* `id` / `model` / `stop_sequence` are read here for the same reason `reconstructFromSse`
|
|
3892
|
-
* reads them off `message_start`: a repaired buffered response is re-serialized from this
|
|
3893
|
-
* shape, so anything not captured here is gone by the time it is re-emitted — which is how
|
|
3894
|
-
* every repaired non-streamed turn used to go out under the constant `msg_repair` with the
|
|
3895
|
-
* backend's real id discarded. Absent fields stay absent; nothing is invented.
|
|
3896
|
-
*/
|
|
3897
3026
|
function parseAssistant(text) {
|
|
3898
3027
|
try {
|
|
3899
3028
|
const j = JSON.parse(text);
|
|
@@ -3917,12 +3046,6 @@ function parseAssistant(text) {
|
|
|
3917
3046
|
return null;
|
|
3918
3047
|
}
|
|
3919
3048
|
}
|
|
3920
|
-
/**
|
|
3921
|
-
* End offset (exclusive) of the first complete SSE frame in `buf`, or -1 if no
|
|
3922
|
-
* frame boundary is present yet. Frames are delimited by a blank line — `\n\n`
|
|
3923
|
-
* (LF) or `\r\n\r\n` (CRLF); whichever boundary comes first wins. Operates on
|
|
3924
|
-
* raw bytes so multibyte UTF-8 in event payloads is never split.
|
|
3925
|
-
*/
|
|
3926
3049
|
function frameEnd(buf) {
|
|
3927
3050
|
const lf = buf.indexOf("\n\n", 0, "latin1");
|
|
3928
3051
|
const crlf = buf.indexOf("\r\n\r\n", 0, "latin1");
|
|
@@ -3932,10 +3055,6 @@ function frameEnd(buf) {
|
|
|
3932
3055
|
return crlf + 4;
|
|
3933
3056
|
return lf + 2;
|
|
3934
3057
|
}
|
|
3935
|
-
/**
|
|
3936
|
-
* If `frame` is a `content_block_start` event opening a `tool_use` block, return
|
|
3937
|
-
* its block index; otherwise null. This is the trigger to start withholding.
|
|
3938
|
-
*/
|
|
3939
3058
|
function frameOpensToolUse(frame) {
|
|
3940
3059
|
const dataLines = [];
|
|
3941
3060
|
for (const line of frame.toString("utf8").split(/\r?\n/)) {
|
|
@@ -3960,39 +3079,14 @@ function frameOpensToolUse(frame) {
|
|
|
3960
3079
|
return null;
|
|
3961
3080
|
return typeof e.index === "number" ? e.index : 0;
|
|
3962
3081
|
}
|
|
3963
|
-
/** A single Anthropic-style SSE `error` event, for failing an already-open stream. */
|
|
3964
3082
|
function sseError(message) {
|
|
3965
3083
|
const data = JSON.stringify({ type: "error", error: { type: "api_error", message } });
|
|
3966
3084
|
return `event: error\ndata: ${data}\n\n`;
|
|
3967
3085
|
}
|
|
3968
|
-
/** The OpenAI-front equivalent: an error object in a plain `data:` frame. */
|
|
3969
3086
|
function openAiSseError(message) {
|
|
3970
3087
|
return `data: ${JSON.stringify({ error: { message, type: "api_error" } })}\n\n`;
|
|
3971
3088
|
}
|
|
3972
|
-
/**
|
|
3973
|
-
* The log `errorKinds` entry for an upstream body that failed part-way through.
|
|
3974
|
-
*
|
|
3975
|
-
* A distinct kind, not a validator error kind: the response was never validated,
|
|
3976
|
-
* it was cut off. It is what makes a truncated turn greppable in the log at all.
|
|
3977
|
-
*/
|
|
3978
3089
|
const MID_STREAM_ERROR_KIND = "backend_stream_failed";
|
|
3979
|
-
/**
|
|
3980
|
-
* Terminate a response whose upstream body threw PART-WAY THROUGH.
|
|
3981
|
-
*
|
|
3982
|
-
* By the time a body is consumed the head is usually already committed, so
|
|
3983
|
-
* `failClosed` degrades to a bare `res.end()` and the client cannot tell a
|
|
3984
|
-
* truncated answer from a complete one. Where the committed response is a stream
|
|
3985
|
-
* we can still say so — `errorFrame` carries the protocol-appropriate error event
|
|
3986
|
-
* (Anthropic `event: error`, or an OpenAI `data:` frame on the OpenAI front) —
|
|
3987
|
-
* and where it is not, closing is all that remains. Pass `null` for a committed
|
|
3988
|
-
* non-stream response; inventing a frame in the wrong protocol is worse than
|
|
3989
|
-
* closing.
|
|
3990
|
-
*
|
|
3991
|
-
* ⚠ The caller MUST also write a log record. Previously the throw propagated out
|
|
3992
|
-
* of these paths past `h.logger.write` to `createProxy`'s top-level catch, so a
|
|
3993
|
-
* mid-stream backend failure produced a truncated 200 AND no log line at all —
|
|
3994
|
-
* invisible to the operator, and indistinguishable from success to the client.
|
|
3995
|
-
*/
|
|
3996
3090
|
function endMidStreamFailure(res, errorFrame, message) {
|
|
3997
3091
|
if (res.writableEnded || res.destroyed)
|
|
3998
3092
|
return;
|
|
@@ -4002,13 +3096,6 @@ function endMidStreamFailure(res, errorFrame, message) {
|
|
|
4002
3096
|
}
|
|
4003
3097
|
res.end(errorFrame ?? undefined);
|
|
4004
3098
|
}
|
|
4005
|
-
/**
|
|
4006
|
-
* Handle a mid-stream failure (socket reset / network drop mid-response).
|
|
4007
|
-
*
|
|
4008
|
-
* Emits protocol-appropriate error frames (SSE event/data or none for non-stream),
|
|
4009
|
-
* completes the attempt's still-provisional health handle as a failure and writes a
|
|
4010
|
-
* metadata log record with `MID_STREAM_ERROR_KIND`.
|
|
4011
|
-
*/
|
|
4012
3099
|
function handleMidStreamError(res, e, started, path, hadTools, streamed, backendStatus, target, attempt, h, errorFrameBuilder, reportedModelSource, deadlineAborted = false, committed = false) {
|
|
4013
3100
|
const message = midStreamMessage(e);
|
|
4014
3101
|
const errorFrame = errorFrameBuilder(message);
|
|
@@ -4030,7 +3117,6 @@ function handleMidStreamError(res, e, started, path, hadTools, streamed, backend
|
|
|
4030
3117
|
...toolUseIdRewriteField(reportedModelSource),
|
|
4031
3118
|
});
|
|
4032
3119
|
}
|
|
4033
|
-
/** The client-facing description of a mid-transfer upstream failure. */
|
|
4034
3120
|
function midStreamMessage(e) {
|
|
4035
3121
|
const errStr = e && typeof e === "object" && "message" in e && typeof e.message === "string"
|
|
4036
3122
|
? e.message
|
|
@@ -4054,11 +3140,7 @@ function readBody(req, maxBytes = DEFAULT_MAX_BODY_BYTES) {
|
|
|
4054
3140
|
if (total > maxBytes) {
|
|
4055
3141
|
done = true;
|
|
4056
3142
|
cleanup();
|
|
4057
|
-
// Drain without retaining the rest so the client can receive the explicit 413 response.
|
|
4058
3143
|
req.resume();
|
|
4059
|
-
// Tagged, not described: the dashboard route classifies 413-vs-500 from this CODE. It used
|
|
4060
|
-
// to regex-match this very message string, i.e. the relay inferring its own intent from
|
|
4061
|
-
// prose — see `BODY_TOO_LARGE_CODE`. Harmless for the other caller, which ignores it.
|
|
4062
3144
|
reject(Object.assign(new Error("request body too large"), { code: BODY_TOO_LARGE_CODE }));
|
|
4063
3145
|
return;
|
|
4064
3146
|
}
|
|
@@ -4112,12 +3194,7 @@ async function writeChunk(res, chunk, beforeWrite) {
|
|
|
4112
3194
|
return false;
|
|
4113
3195
|
}
|
|
4114
3196
|
}
|
|
4115
|
-
function failClosed(res, status, message, headers,
|
|
4116
|
-
/**
|
|
4117
|
-
* Error type for the body. Defaults to `api_error`, which is what every caller wanted before a
|
|
4118
|
-
* relay-authored refusal could reach here — so an omitted argument is byte-identical to before.
|
|
4119
|
-
*/
|
|
4120
|
-
errorType = "api_error") {
|
|
3197
|
+
function failClosed(res, status, message, headers, errorType = "api_error") {
|
|
4121
3198
|
if (res.headersSent) {
|
|
4122
3199
|
if (!res.writableEnded)
|
|
4123
3200
|
res.end();
|
|
@@ -4126,20 +3203,6 @@ errorType = "api_error") {
|
|
|
4126
3203
|
res.writeHead(status, { ...headers, "content-type": "application/json" });
|
|
4127
3204
|
res.end(JSON.stringify({ type: "error", error: { type: errorType, message } }));
|
|
4128
3205
|
}
|
|
4129
|
-
/**
|
|
4130
|
-
* Build the metadata record for one turn.
|
|
4131
|
-
*
|
|
4132
|
-
* `served` is the target a backend request was actually DISPATCHED to, or `null`
|
|
4133
|
-
* when none was — a guardrail rejection, a routing error, an admin endpoint, or
|
|
4134
|
-
* anything answered locally. It is a required parameter, without a default, so
|
|
4135
|
-
* `tsc` names every call site that has not decided which of the two it is; a
|
|
4136
|
-
* default would quietly turn "nobody decided" into a confident claim.
|
|
4137
|
-
*
|
|
4138
|
-
* The model the CLIENT asked for is deliberately NOT recorded (OBS-b5ade458 is
|
|
4139
|
-
* finished): routing resolves a tier/pool spec to a `ResolvedTarget`, so it is
|
|
4140
|
-
* routinely not the model that answered, and it was the id every "which model
|
|
4141
|
-
* trips the validator" reading of this log was attributed to.
|
|
4142
|
-
*/
|
|
4143
3206
|
function pickString(obj, key) {
|
|
4144
3207
|
if (typeof obj === "object" && obj !== null) {
|
|
4145
3208
|
const v = obj[key];
|