free-coding-models 0.5.88 → 0.5.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/bin/free-coding-models.js +33 -2
- package/changelog/v0.5.89.md +20 -0
- package/changelog/v0.5.90.md +12 -0
- package/package.json +1 -1
- package/sources.js +5 -2
- package/src/core/benchmark.js +9 -0
- package/src/core/cloudflare-account.js +311 -0
- package/src/core/endpoint-installer.js +8 -4
- package/src/core/opencode.js +9 -6
- package/src/core/ping.js +52 -16
- package/src/core/provider-key-tester.js +10 -3
- package/src/core/provider-metadata.js +1 -1
- package/src/core/router-daemon.js +1138 -501
- package/src/core/router-v2/anthropic-compat.js +473 -0
- package/src/core/router-v2/bench.js +171 -0
- package/src/core/router-v2/breaker-store.js +265 -0
- package/src/core/router-v2/constants.js +108 -0
- package/src/core/router-v2/decision-trace.js +134 -0
- package/src/core/router-v2/failure-classifier.js +231 -0
- package/src/core/router-v2/request-history.js +137 -0
- package/src/core/router-v2/response-gate.js +175 -0
- package/src/core/router-v2/tui-dashboard.js +632 -0
- package/src/core/schema-normalizer.js +23 -6
- package/src/core/utils.js +12 -0
- package/src/tui/app.js +7 -2
- package/src/tui/cli-help.js +4 -0
- package/src/tui/key-handler.js +117 -2
- package/src/tui/overlays.js +19 -3
- package/src/tui/tui-state.js +22 -0
- package/web/dist/assets/index-CCaxIOti.css +1 -0
- package/web/dist/assets/index-CCkuXrqE.js +48 -0
- package/web/dist/index.html +2 -2
- package/web/server.js +105 -1
- package/web/dist/assets/index-CAzFIt8P.css +0 -1
- package/web/dist/assets/index-DFg1h0Nd.js +0 -44
|
@@ -35,7 +35,7 @@ import { dirname, join, resolve as resolvePath, sep as pathSep } from 'node:path
|
|
|
35
35
|
import { fork, execFileSync } from 'node:child_process'
|
|
36
36
|
import { randomUUID, timingSafeEqual } from 'node:crypto'
|
|
37
37
|
import { appendFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
38
|
-
import { homedir } from 'node:os'
|
|
38
|
+
import { homedir, tmpdir } from 'node:os'
|
|
39
39
|
import { fileURLToPath } from 'node:url'
|
|
40
40
|
import { MODELS, sources } from '../../sources.js'
|
|
41
41
|
import {
|
|
@@ -47,7 +47,7 @@ import {
|
|
|
47
47
|
normalizeRouterConfig,
|
|
48
48
|
saveConfig,
|
|
49
49
|
} from './config.js'
|
|
50
|
-
import { buildChatCompletionPingBody, ping, resolveCloudflareUrl, shouldUseDisabledThinkingForProvider } from './ping.js'
|
|
50
|
+
import { buildChatCompletionPingBody, ping, resolveCloudflareUrl, shouldUseDisabledThinkingForProvider, getProviderSessionHeaders } from './ping.js'
|
|
51
51
|
import { benchmarkModel, BENCHMARK_TIMEOUT_MS } from './benchmark.js'
|
|
52
52
|
import { loadChangelog } from './changelog-loader.js'
|
|
53
53
|
import { sendUsageTelemetry } from './telemetry.js'
|
|
@@ -80,6 +80,23 @@ import {
|
|
|
80
80
|
pruneStaleEntries as pruneRuntimeTelemetry,
|
|
81
81
|
DEFAULT_MIN_CALLS_FOR_SCORE as RUNTIME_MIN_CALLS,
|
|
82
82
|
} from './runtime-telemetry.js'
|
|
83
|
+
// 📖 Router v2 engine (merged into this daemon): typed failure classification,
|
|
84
|
+
// content-level response gating, decision traces, persisted breakers, request
|
|
85
|
+
// history and the Anthropic /v1/messages protocol. The router is now v2
|
|
86
|
+
// internally while keeping every v1 endpoint, flag and port.
|
|
87
|
+
import { classifyFailure, classifyStatus, clientStatusForKind, FAILURE_KINDS } from './router-v2/failure-classifier.js'
|
|
88
|
+
import { validateChatCompletionPayload, createStreamReadinessTracker, estimateTokens } from './router-v2/response-gate.js'
|
|
89
|
+
import { createDecisionTrace, traceSkip, traceAttempt, finishTrace, decisionHeaderValue, traceSummary } from './router-v2/decision-trace.js'
|
|
90
|
+
import { BreakerStore } from './router-v2/breaker-store.js'
|
|
91
|
+
import { RequestHistory } from './router-v2/request-history.js'
|
|
92
|
+
import {
|
|
93
|
+
anthropicErrorPayload,
|
|
94
|
+
anthropicErrorTypeForStatus,
|
|
95
|
+
createAnthropicStreamTransformer,
|
|
96
|
+
translateAnthropicToOpenAI,
|
|
97
|
+
translateOpenAIToAnthropicResponse,
|
|
98
|
+
} from './router-v2/anthropic-compat.js'
|
|
99
|
+
import { parseFcmModel } from './router-v2/constants.js'
|
|
83
100
|
|
|
84
101
|
export const ROUTER_DEFAULT_PORT = 19280
|
|
85
102
|
export const ROUTER_MAX_PORT = 19289
|
|
@@ -118,7 +135,7 @@ export function getRouterPortRange() {
|
|
|
118
135
|
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
119
136
|
const CLI_ENTRY_PATH = join(__dirname, '..', '..', 'bin', 'free-coding-models.js')
|
|
120
137
|
const LOCAL_VERSION = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8')).version
|
|
121
|
-
const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
138
|
+
export const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
122
139
|
/**
|
|
123
140
|
* 📖 normalizeToolCallsResponse — fix malformed tool_calls from upstream.
|
|
124
141
|
* Some providers return finish_reason: "tool_calls" but message has no tool_calls array,
|
|
@@ -128,7 +145,7 @@ const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
|
128
145
|
* @param {object} data - parsed JSON response object
|
|
129
146
|
* @returns {boolean} true if mutated
|
|
130
147
|
*/
|
|
131
|
-
function normalizeToolCallsResponse(data) {
|
|
148
|
+
export function normalizeToolCallsResponse(data) {
|
|
132
149
|
if (!data || typeof data !== 'object' || !Array.isArray(data.choices)) return false
|
|
133
150
|
let mutated = false
|
|
134
151
|
for (const choice of data.choices) {
|
|
@@ -145,16 +162,16 @@ function normalizeToolCallsResponse(data) {
|
|
|
145
162
|
return mutated
|
|
146
163
|
}
|
|
147
164
|
|
|
148
|
-
const MAX_REQUEST_LOG = 200
|
|
149
|
-
const MAX_SSE_CLIENTS = 10
|
|
150
|
-
const MAX_SSE_CLIENTS_PER_ORIGIN = 5
|
|
151
|
-
const MAX_CONCURRENT_REQUESTS = 50
|
|
152
|
-
const MAX_PROBE_WINDOW = 20
|
|
165
|
+
export const MAX_REQUEST_LOG = 200
|
|
166
|
+
export const MAX_SSE_CLIENTS = 10
|
|
167
|
+
export const MAX_SSE_CLIENTS_PER_ORIGIN = 5
|
|
168
|
+
export const MAX_CONCURRENT_REQUESTS = 50
|
|
169
|
+
export const MAX_PROBE_WINDOW = 20
|
|
153
170
|
const TOKEN_FLUSH_INTERVAL_MS = 60000
|
|
154
171
|
const CONFIG_RELOAD_INTERVAL_MS = 10000
|
|
155
|
-
const STATS_RETENTION_DAYS = 90
|
|
156
|
-
const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504, 529])
|
|
157
|
-
const AUTH_STATUS_CODES = new Set([401, 403])
|
|
172
|
+
export const STATS_RETENTION_DAYS = 90
|
|
173
|
+
export const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504, 529])
|
|
174
|
+
export const AUTH_STATUS_CODES = new Set([401, 403])
|
|
158
175
|
const RATE_LIMIT_HEADER_NAMES = [
|
|
159
176
|
'retry-after',
|
|
160
177
|
'x-ratelimit-limit',
|
|
@@ -171,16 +188,16 @@ const RATE_LIMIT_HEADER_NAMES = [
|
|
|
171
188
|
'x-ratelimit-reset-tokens',
|
|
172
189
|
]
|
|
173
190
|
|
|
174
|
-
function nowIso() {
|
|
191
|
+
export function nowIso() {
|
|
175
192
|
return new Date().toISOString()
|
|
176
193
|
}
|
|
177
194
|
|
|
178
|
-
function modelKey(provider, model) {
|
|
195
|
+
export function modelKey(provider, model) {
|
|
179
196
|
return `${provider}/${model}`
|
|
180
197
|
}
|
|
181
198
|
|
|
182
199
|
// 📖 parseJsonResult is still local - it returns {ok, value/error} which is different from safeJsonParse
|
|
183
|
-
function parseJsonResult(raw) {
|
|
200
|
+
export function parseJsonResult(raw) {
|
|
184
201
|
try {
|
|
185
202
|
return { ok: true, value: JSON.parse(raw) }
|
|
186
203
|
} catch (error) {
|
|
@@ -188,7 +205,7 @@ function parseJsonResult(raw) {
|
|
|
188
205
|
}
|
|
189
206
|
}
|
|
190
207
|
|
|
191
|
-
function isProcessAlive(pid) {
|
|
208
|
+
export function isProcessAlive(pid) {
|
|
192
209
|
if (!Number.isInteger(pid) || pid <= 0) return false
|
|
193
210
|
try {
|
|
194
211
|
process.kill(pid, 0)
|
|
@@ -198,7 +215,7 @@ function isProcessAlive(pid) {
|
|
|
198
215
|
}
|
|
199
216
|
}
|
|
200
217
|
|
|
201
|
-
function readNumberFile(path) {
|
|
218
|
+
export function readNumberFile(path) {
|
|
202
219
|
try {
|
|
203
220
|
const value = Number.parseInt(readFileSync(path, 'utf8').trim(), 10)
|
|
204
221
|
return Number.isFinite(value) ? value : null
|
|
@@ -207,7 +224,7 @@ function readNumberFile(path) {
|
|
|
207
224
|
}
|
|
208
225
|
}
|
|
209
226
|
|
|
210
|
-
function headerEntries(headers) {
|
|
227
|
+
export function headerEntries(headers) {
|
|
211
228
|
const entries = {}
|
|
212
229
|
if (!headers || typeof headers.forEach !== 'function') return entries
|
|
213
230
|
headers.forEach((value, key) => {
|
|
@@ -222,12 +239,12 @@ function headerEntries(headers) {
|
|
|
222
239
|
return entries
|
|
223
240
|
}
|
|
224
241
|
|
|
225
|
-
function getHeaderValue(headers, name) {
|
|
242
|
+
export function getHeaderValue(headers, name) {
|
|
226
243
|
if (!headers || typeof headers.get !== 'function') return ''
|
|
227
244
|
return headers.get(name) || ''
|
|
228
245
|
}
|
|
229
246
|
|
|
230
|
-
function extractRateLimitHeaders(headers) {
|
|
247
|
+
export function extractRateLimitHeaders(headers) {
|
|
231
248
|
const values = {}
|
|
232
249
|
for (const name of RATE_LIMIT_HEADER_NAMES) {
|
|
233
250
|
const value = getHeaderValue(headers, name)
|
|
@@ -236,7 +253,7 @@ function extractRateLimitHeaders(headers) {
|
|
|
236
253
|
return values
|
|
237
254
|
}
|
|
238
255
|
|
|
239
|
-
function parseRetryAfterMs(value) {
|
|
256
|
+
export function parseRetryAfterMs(value) {
|
|
240
257
|
if (!value) return null
|
|
241
258
|
const seconds = Number(value)
|
|
242
259
|
if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000))
|
|
@@ -245,7 +262,7 @@ function parseRetryAfterMs(value) {
|
|
|
245
262
|
return null
|
|
246
263
|
}
|
|
247
264
|
|
|
248
|
-
function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
265
|
+
export function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
249
266
|
return Object.entries(rateLimitHeaders).some(([name, value]) => {
|
|
250
267
|
if (!name.includes('remaining')) return false
|
|
251
268
|
const numeric = Number(value)
|
|
@@ -253,11 +270,11 @@ function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
|
253
270
|
})
|
|
254
271
|
}
|
|
255
272
|
|
|
256
|
-
function isLikelyHtmlText(text) {
|
|
273
|
+
export function isLikelyHtmlText(text) {
|
|
257
274
|
return /^\s*(<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>])/i.test(text || '')
|
|
258
275
|
}
|
|
259
276
|
|
|
260
|
-
function isLikelyHtmlResponse(headers, text = '') {
|
|
277
|
+
export function isLikelyHtmlResponse(headers, text = '') {
|
|
261
278
|
const contentType = getHeaderValue(headers, 'content-type').toLowerCase()
|
|
262
279
|
return contentType.includes('text/html') || isLikelyHtmlText(text)
|
|
263
280
|
}
|
|
@@ -268,7 +285,7 @@ function isLikelyHtmlResponse(headers, text = '') {
|
|
|
268
285
|
// 📖 endpoints. Blocks CSRF from malicious tabs and key exfiltration from
|
|
269
286
|
// 📖 cross-origin scripts. Plain CLI calls (curl/fetch without Origin) are
|
|
270
287
|
// 📖 allowed because they cannot be triggered by a browser context.
|
|
271
|
-
function isLoopbackHostname(hostname) {
|
|
288
|
+
export function isLoopbackHostname(hostname) {
|
|
272
289
|
if (!hostname) return false
|
|
273
290
|
const h = hostname.toLowerCase()
|
|
274
291
|
return h === 'localhost' || h === '127.0.0.1' || h === '[::1]' || h === '::1' || h.endsWith('.localhost')
|
|
@@ -279,7 +296,7 @@ function isLoopbackHostname(hostname) {
|
|
|
279
296
|
// 📖 `.local` or `.internal`. Used when the daemon is explicitly bound to a
|
|
280
297
|
// 📖 wildcard address (FCM_HOST=0.0.0.0) so LAN clients keep working; it
|
|
281
298
|
// 📖 never grants origin trust on its own.
|
|
282
|
-
function isPrivateNetworkHostname(hostname) {
|
|
299
|
+
export function isPrivateNetworkHostname(hostname) {
|
|
283
300
|
if (!hostname) return false
|
|
284
301
|
const h = hostname.toLowerCase()
|
|
285
302
|
// 📖 mDNS / zero-conf hostnames
|
|
@@ -304,7 +321,28 @@ function getAllowedOrigins() {
|
|
|
304
321
|
return _allowedOriginsCache
|
|
305
322
|
}
|
|
306
323
|
|
|
307
|
-
|
|
324
|
+
// 📖 v2: CORS for loopback (and explicitly allowed) origins so a browser
|
|
325
|
+
// dashboard served from another local port (web dashboard 3333) can call
|
|
326
|
+
// this daemon directly without a proxy.
|
|
327
|
+
function applyCors(req, res) {
|
|
328
|
+
const origin = typeof req.headers.origin === 'string' ? req.headers.origin : ''
|
|
329
|
+
if (!origin) return
|
|
330
|
+
let hostname = ''
|
|
331
|
+
try {
|
|
332
|
+
hostname = new URL(origin).hostname
|
|
333
|
+
} catch {
|
|
334
|
+
return
|
|
335
|
+
}
|
|
336
|
+
const allowed = isLoopbackHostname(hostname) || getAllowedOrigins().includes(origin)
|
|
337
|
+
if (!allowed) return
|
|
338
|
+
res.setHeader('Access-Control-Allow-Origin', origin)
|
|
339
|
+
res.setHeader('Vary', 'Origin')
|
|
340
|
+
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS')
|
|
341
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, x-request-id, anthropic-version')
|
|
342
|
+
res.setHeader('Access-Control-Max-Age', '600')
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export function isSameOriginOrLocal(req) {
|
|
308
346
|
const origin = req.headers.origin
|
|
309
347
|
const referer = req.headers.referer || req.headers.referrer
|
|
310
348
|
const hasOrigin = typeof origin === 'string' && origin.length > 0
|
|
@@ -346,7 +384,7 @@ function isSameOriginOrLocal(req) {
|
|
|
346
384
|
// 📖 Loopback Host values are always accepted; when the daemon is bound to a
|
|
347
385
|
// 📖 non-loopback FCM_HOST (LAN / Docker), that hostname plus private-network
|
|
348
386
|
// 📖 hosts are accepted too. Public domains stay rejected.
|
|
349
|
-
function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
387
|
+
export function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
350
388
|
if (typeof hostHeader !== 'string' || !hostHeader) return false
|
|
351
389
|
const value = hostHeader.toLowerCase()
|
|
352
390
|
const loopback = [`127.0.0.1:${port}`, `localhost:${port}`, `[::1]:${port}`, '127.0.0.1', 'localhost', '[::1]']
|
|
@@ -366,18 +404,18 @@ function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
|
366
404
|
// 📖 Set FCM_ROUTER_TOKEN to require `Authorization: Bearer <token>` (or
|
|
367
405
|
// 📖 `x-api-key: <token>`) on every /v1/* request; leave it unset for the
|
|
368
406
|
// 📖 default no-auth local behavior.
|
|
369
|
-
function getRouterToken() {
|
|
407
|
+
export function getRouterToken() {
|
|
370
408
|
return (process.env.FCM_ROUTER_TOKEN || '').trim()
|
|
371
409
|
}
|
|
372
410
|
|
|
373
|
-
function safeTokenCompare(candidate, token) {
|
|
411
|
+
export function safeTokenCompare(candidate, token) {
|
|
374
412
|
const bufA = Buffer.from(String(candidate || ''))
|
|
375
413
|
const bufB = Buffer.from(token)
|
|
376
414
|
if (bufA.length !== bufB.length) return false
|
|
377
415
|
return timingSafeEqual(bufA, bufB)
|
|
378
416
|
}
|
|
379
417
|
|
|
380
|
-
function isAuthorizedForV1(req) {
|
|
418
|
+
export function isAuthorizedForV1(req) {
|
|
381
419
|
const token = getRouterToken()
|
|
382
420
|
if (!token) return true
|
|
383
421
|
const auth = req.headers.authorization || ''
|
|
@@ -386,7 +424,7 @@ function isAuthorizedForV1(req) {
|
|
|
386
424
|
return typeof apiKeyHeader === 'string' && safeTokenCompare(apiKeyHeader.trim(), token)
|
|
387
425
|
}
|
|
388
426
|
|
|
389
|
-
const MIME_TYPES = {
|
|
427
|
+
export const MIME_TYPES = {
|
|
390
428
|
'.html': 'text/html; charset=utf-8',
|
|
391
429
|
'.css': 'text/css; charset=utf-8',
|
|
392
430
|
'.js': 'application/javascript; charset=utf-8',
|
|
@@ -396,7 +434,7 @@ const MIME_TYPES = {
|
|
|
396
434
|
'.ico': 'image/x-icon',
|
|
397
435
|
}
|
|
398
436
|
|
|
399
|
-
function getWebModelsPayload(runtime) {
|
|
437
|
+
export function getWebModelsPayload(runtime) {
|
|
400
438
|
// 📖 Hoist router + active set lookups out of the per-model loop so we
|
|
401
439
|
// 📖 don't re-resolve them ~200 times per request.
|
|
402
440
|
const router = runtime.routerConfig()
|
|
@@ -473,7 +511,7 @@ function getWebUpdateStatusPayload() {
|
|
|
473
511
|
}
|
|
474
512
|
}
|
|
475
513
|
|
|
476
|
-
function getWebStatePayload(runtime) {
|
|
514
|
+
export function getWebStatePayload(runtime) {
|
|
477
515
|
const router = runtime.routerConfig()
|
|
478
516
|
const probeInterval = router.probeIntervals?.[router.probeMode] || DEFAULT_ROUTER_SETTINGS.probeIntervals.balanced
|
|
479
517
|
return {
|
|
@@ -491,7 +529,7 @@ function getWebStatePayload(runtime) {
|
|
|
491
529
|
}
|
|
492
530
|
}
|
|
493
531
|
|
|
494
|
-
function getWebConfigPayload(runtime) {
|
|
532
|
+
export function getWebConfigPayload(runtime) {
|
|
495
533
|
const providers = {}
|
|
496
534
|
for (const [key, src] of Object.entries(sources)) {
|
|
497
535
|
const rawKey = runtime.getApiKeyForProvider(key)
|
|
@@ -517,9 +555,9 @@ function getWebConfigPayload(runtime) {
|
|
|
517
555
|
}
|
|
518
556
|
}
|
|
519
557
|
|
|
520
|
-
const WEB_DIST_DIR = resolvePath(__dirname, '..', '..', 'web', 'dist')
|
|
558
|
+
export const WEB_DIST_DIR = resolvePath(__dirname, '..', '..', 'web', 'dist')
|
|
521
559
|
|
|
522
|
-
function serveStaticFromDist(res, absPath) {
|
|
560
|
+
export function serveStaticFromDist(res, absPath) {
|
|
523
561
|
const ext = absPath.slice(absPath.lastIndexOf('.'))
|
|
524
562
|
const ct = MIME_TYPES[ext] || 'application/octet-stream'
|
|
525
563
|
res.writeHead(200, {
|
|
@@ -530,7 +568,7 @@ function serveStaticFromDist(res, absPath) {
|
|
|
530
568
|
res.end(readFileSync(absPath))
|
|
531
569
|
}
|
|
532
570
|
|
|
533
|
-
function serveSpaIndex(res) {
|
|
571
|
+
export function serveSpaIndex(res) {
|
|
534
572
|
const indexPath = resolvePath(WEB_DIST_DIR, 'index.html')
|
|
535
573
|
if (!existsSync(indexPath)) {
|
|
536
574
|
res.writeHead(503, { 'Content-Type': 'text/plain' })
|
|
@@ -545,7 +583,7 @@ function serveSpaIndex(res) {
|
|
|
545
583
|
res.end(readFileSync(indexPath))
|
|
546
584
|
}
|
|
547
585
|
|
|
548
|
-
function serveWebStaticFile(res, pathname, requestId) {
|
|
586
|
+
export function serveWebStaticFile(res, pathname, requestId) {
|
|
549
587
|
// 📖 Resolve to an absolute path and verify it stays inside WEB_DIST_DIR.
|
|
550
588
|
// 📖 Without this, `pathname` like `/../../etc/passwd` escapes the dist root.
|
|
551
589
|
const requested = pathname === '/' ? 'index.html' : pathname.replace(/^\/+/, '')
|
|
@@ -582,7 +620,7 @@ function serveWebStaticFile(res, pathname, requestId) {
|
|
|
582
620
|
serveStaticFromDist(res, candidate)
|
|
583
621
|
}
|
|
584
622
|
|
|
585
|
-
function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
623
|
+
export function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
586
624
|
// 📖 Keep quota diagnostics structural only: headers and retry timing are safe,
|
|
587
625
|
// 📖 while upstream response bodies stay out of logs and telemetry.
|
|
588
626
|
const rateLimitHeaders = extractRateLimitHeaders(response.headers)
|
|
@@ -602,7 +640,7 @@ function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
|
602
640
|
}
|
|
603
641
|
}
|
|
604
642
|
|
|
605
|
-
function attachClientAbort(req, res, controller) {
|
|
643
|
+
export function attachClientAbort(req, res, controller) {
|
|
606
644
|
let clientAborted = false
|
|
607
645
|
const abort = () => {
|
|
608
646
|
if (res.writableEnded) return
|
|
@@ -634,7 +672,10 @@ export function cloneHeadersForUpstream(reqHeaders, apiKey, providerKey) {
|
|
|
634
672
|
const lower = key.toLowerCase()
|
|
635
673
|
// 📖 Drop the client's cookies: they belong to the local browser session
|
|
636
674
|
// 📖 and must never be forwarded to the upstream provider.
|
|
637
|
-
|
|
675
|
+
// 📖 Also strip `x-api-key`: the local router token can arrive under that
|
|
676
|
+
// name and must never leak to an upstream provider (v2 security fix).
|
|
677
|
+
if (['host', 'connection', 'content-length', 'authorization', 'cookie', 'x-api-key'].includes(lower)) continue
|
|
678
|
+
if (lower.startsWith('x-fcm-') || lower === 'x-request-id') continue
|
|
638
679
|
if (typeof value !== 'string') continue
|
|
639
680
|
if (lower === 'content-type') {
|
|
640
681
|
headers['Content-Type'] = value
|
|
@@ -648,26 +689,86 @@ export function cloneHeadersForUpstream(reqHeaders, apiKey, providerKey) {
|
|
|
648
689
|
headers['HTTP-Referer'] = 'https://github.com/vava-nessa/free-coding-models'
|
|
649
690
|
headers['X-Title'] = 'free-coding-models'
|
|
650
691
|
}
|
|
692
|
+
// 📖 Mandatory per-provider headers (issue #181): OpenCode Zen rejects every
|
|
693
|
+
// 📖 request without `x-opencode-session` with HTTP 400 MissingSessionID.
|
|
694
|
+
// 📖 This function builds headers for BOTH health probes and real forwarded
|
|
695
|
+
// 📖 traffic, so applying it here covers every upstream call in one place.
|
|
696
|
+
Object.assign(headers, getProviderSessionHeaders(providerKey))
|
|
651
697
|
return headers
|
|
652
698
|
}
|
|
653
699
|
|
|
654
|
-
|
|
700
|
+
// 📖 v2 defaults for the merged engine. These extend (never replace) the
|
|
701
|
+
// shared failover settings; users override them in ~/.free-coding-models.json
|
|
702
|
+
// under `router.failover` and the raw values are read because the shared
|
|
703
|
+
// normalizer only knows the v1 field names.
|
|
704
|
+
const DEFAULT_BODY_READ_TIMEOUT_MS = 30000
|
|
705
|
+
const DEFAULT_TOTAL_BUDGET_MS = 120000
|
|
706
|
+
const DEFAULT_CONTENT_VALIDATION = 'strict'
|
|
707
|
+
const DEFAULT_QUOTA_PAUSE_MS = 60000
|
|
708
|
+
const MAX_CONCURRENT_QUEUE_RETRY_AFTER_S = 3
|
|
709
|
+
|
|
710
|
+
function clampIntV2(value, fallback, { min, max }) {
|
|
711
|
+
const n = Number(value)
|
|
712
|
+
if (!Number.isFinite(n)) return fallback
|
|
713
|
+
return Math.min(max, Math.max(min, Math.round(n)))
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// 📖 Read the upstream body under a hard deadline (v2 fix): v1 cleared the
|
|
717
|
+
// request timeout as soon as headers arrived, so a provider that trickled the
|
|
718
|
+
// body could hang an agent forever.
|
|
719
|
+
async function readBodyWithTimeout(response, controller, timeoutMs) {
|
|
720
|
+
let timer = null
|
|
721
|
+
try {
|
|
722
|
+
return await Promise.race([
|
|
723
|
+
response.text(),
|
|
724
|
+
new Promise((_, reject) => {
|
|
725
|
+
timer = setTimeout(() => {
|
|
726
|
+
try { controller.abort() } catch {}
|
|
727
|
+
reject(Object.assign(new Error('upstream_body_read_timeout'), { name: 'BodyReadTimeoutError' }))
|
|
728
|
+
}, timeoutMs)
|
|
729
|
+
}),
|
|
730
|
+
])
|
|
731
|
+
} finally {
|
|
732
|
+
if (timer) clearTimeout(timer)
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// 📖 Map a content-gate rejection reason to its failure kind name.
|
|
737
|
+
function gateReasonToKind(reason) {
|
|
738
|
+
switch (reason) {
|
|
739
|
+
case 'error_payload': return 'ERROR_PAYLOAD'
|
|
740
|
+
case 'empty_choices': return 'EMPTY_CHOICES'
|
|
741
|
+
case 'empty_content': return 'EMPTY_CONTENT'
|
|
742
|
+
case 'invalid_json': return 'INVALID_JSON'
|
|
743
|
+
default: return 'INVALID_JSON'
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
function parseLastResortModel(value) {
|
|
748
|
+
if (typeof value !== 'string') return null
|
|
749
|
+
const trimmed = value.trim()
|
|
750
|
+
const slashIdx = trimmed.indexOf('/')
|
|
751
|
+
if (slashIdx <= 0 || slashIdx === trimmed.length - 1) return null
|
|
752
|
+
return { provider: trimmed.slice(0, slashIdx), model: trimmed.slice(slashIdx + 1), key: trimmed }
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
export function getApiModelId(providerKey, modelId) {
|
|
655
756
|
return providerKey === 'zai' ? modelId.replace(/^zai\//, '') : modelId
|
|
656
757
|
}
|
|
657
758
|
|
|
658
|
-
function resolveProviderUrl(providerKey) {
|
|
759
|
+
export function resolveProviderUrl(providerKey) {
|
|
659
760
|
const url = sources[providerKey]?.url
|
|
660
761
|
if (!url) return null
|
|
661
762
|
return providerKey === 'cloudflare' ? resolveCloudflareUrl(url) : url
|
|
662
763
|
}
|
|
663
764
|
|
|
664
|
-
function buildProviderModelsUrl(providerKey) {
|
|
765
|
+
export function buildProviderModelsUrl(providerKey) {
|
|
665
766
|
const url = resolveProviderUrl(providerKey)
|
|
666
767
|
if (typeof url !== 'string' || !url.includes('/chat/completions')) return null
|
|
667
768
|
return url.replace(/\/chat\/completions$/, '/models')
|
|
668
769
|
}
|
|
669
770
|
|
|
670
|
-
function extractUsage(payload) {
|
|
771
|
+
export function extractUsage(payload) {
|
|
671
772
|
const usage = payload?.usage
|
|
672
773
|
if (!usage || typeof usage !== 'object') return null
|
|
673
774
|
const promptTokens = Number(usage.prompt_tokens ?? 0)
|
|
@@ -694,7 +795,7 @@ export function formatOpenAiError(message, type, code, requestId, extra = {}) {
|
|
|
694
795
|
}
|
|
695
796
|
}
|
|
696
797
|
|
|
697
|
-
function sendJson(res, statusCode, payload, headers = {}) {
|
|
798
|
+
export function sendJson(res, statusCode, payload, headers = {}) {
|
|
698
799
|
if (res.writableEnded) return
|
|
699
800
|
const body = JSON.stringify(payload)
|
|
700
801
|
res.writeHead(statusCode, {
|
|
@@ -705,11 +806,11 @@ function sendJson(res, statusCode, payload, headers = {}) {
|
|
|
705
806
|
res.end(body)
|
|
706
807
|
}
|
|
707
808
|
|
|
708
|
-
function sendError(res, statusCode, message, type, code, requestId, extra = {}) {
|
|
809
|
+
export function sendError(res, statusCode, message, type, code, requestId, extra = {}) {
|
|
709
810
|
sendJson(res, statusCode, formatOpenAiError(message, type, code, requestId, extra))
|
|
710
811
|
}
|
|
711
812
|
|
|
712
|
-
function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
813
|
+
export function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
713
814
|
return new Promise((resolve, reject) => {
|
|
714
815
|
let size = 0
|
|
715
816
|
const chunks = []
|
|
@@ -727,7 +828,7 @@ function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
|
727
828
|
})
|
|
728
829
|
}
|
|
729
830
|
|
|
730
|
-
function readJsonBody(req) {
|
|
831
|
+
export function readJsonBody(req) {
|
|
731
832
|
return readRequestBody(req).then((raw) => {
|
|
732
833
|
// 📖 Refuse explicit non-JSON bodies (e.g. text/plain form posts) so a
|
|
733
834
|
// 📖 cross-site form cannot smuggle data into JSON endpoints. Missing or
|
|
@@ -794,7 +895,7 @@ export function applyPrePromptToBody(body, prePrompt) {
|
|
|
794
895
|
return { ...safeBody, messages }
|
|
795
896
|
}
|
|
796
897
|
|
|
797
|
-
class RouterLogger {
|
|
898
|
+
export class RouterLogger {
|
|
798
899
|
constructor(logPath, level = 'info') {
|
|
799
900
|
this.logPath = logPath
|
|
800
901
|
this.level = level
|
|
@@ -844,7 +945,7 @@ class RouterLogger {
|
|
|
844
945
|
debug(message, meta = null) { this.write('debug', message, meta) }
|
|
845
946
|
}
|
|
846
947
|
|
|
847
|
-
class TokenTracker {
|
|
948
|
+
export class TokenTracker {
|
|
848
949
|
constructor(path, logger) {
|
|
849
950
|
this.path = path
|
|
850
951
|
this.logger = logger
|
|
@@ -969,8 +1070,8 @@ class TokenTracker {
|
|
|
969
1070
|
}
|
|
970
1071
|
}
|
|
971
1072
|
|
|
972
|
-
class RouterRuntime {
|
|
973
|
-
constructor({ config, port, logger, tokenPath = ROUTER_TOKENS_PATH, persistConfig = true }) {
|
|
1073
|
+
export class RouterRuntime {
|
|
1074
|
+
constructor({ config, port, logger, tokenPath = ROUTER_TOKENS_PATH, persistConfig = true, paths = {} }) {
|
|
974
1075
|
this.config = config
|
|
975
1076
|
this.port = port
|
|
976
1077
|
this.logger = logger
|
|
@@ -984,11 +1085,28 @@ class RouterRuntime {
|
|
|
984
1085
|
this.configReloadTimer = null
|
|
985
1086
|
this.tokenFlushTimer = null
|
|
986
1087
|
this.probeTimer = null
|
|
1088
|
+
this.probeWatchdog = null
|
|
987
1089
|
this.probeTimeouts = new Set()
|
|
988
1090
|
this.tokenTracker = new TokenTracker(tokenPath, logger)
|
|
989
1091
|
this.modelCatalog = this.buildModelCatalog()
|
|
990
1092
|
this.probeWindows = new Map()
|
|
991
|
-
|
|
1093
|
+
// 📖 v2 engine: persisted circuit breakers (survive restarts, DEGRADED
|
|
1094
|
+
// warning state, escalating backoff). `this.circuit` stays the shared Map
|
|
1095
|
+
// so every v1 read-path keeps working; it IS the breaker store's map.
|
|
1096
|
+
this.breakers = new BreakerStore({
|
|
1097
|
+
path: paths.breakers || join(homedir(), '.free-coding-models-router-v2-breakers.json'),
|
|
1098
|
+
logger,
|
|
1099
|
+
})
|
|
1100
|
+
this.circuit = this.breakers.breakers
|
|
1101
|
+
// 📖 v2 engine: persisted request history (fallback chains per request)
|
|
1102
|
+
// and the in-flight decision traces surfaced on /api/router-v2/*.
|
|
1103
|
+
this.history = new RequestHistory({
|
|
1104
|
+
path: paths.history || join(homedir(), '.free-coding-models-router-v2-history.json'),
|
|
1105
|
+
logger,
|
|
1106
|
+
maxEntries: 500,
|
|
1107
|
+
})
|
|
1108
|
+
this.recentTraces = []
|
|
1109
|
+
this.quotaPauses = new Map()
|
|
992
1110
|
this.requestLog = []
|
|
993
1111
|
this.activeRequests = new Map()
|
|
994
1112
|
this.sseClients = new Set()
|
|
@@ -1078,20 +1196,11 @@ class RouterRuntime {
|
|
|
1078
1196
|
for (const model of set.models || []) {
|
|
1079
1197
|
const key = modelKey(model.provider, model.model)
|
|
1080
1198
|
if (!this.probeWindows.has(key)) this.probeWindows.set(key, [])
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
consecutiveFailures: 0,
|
|
1085
|
-
cooldownMs: router.circuitBreaker.initialCooldownMs,
|
|
1086
|
-
openedAt: null,
|
|
1087
|
-
lastError: null,
|
|
1088
|
-
authError: false,
|
|
1089
|
-
stale: false,
|
|
1090
|
-
})
|
|
1091
|
-
}
|
|
1092
|
-
const entry = this.circuit.get(key)
|
|
1093
|
-
entry.stale = !this.modelCatalog.has(key)
|
|
1199
|
+
// 📖 v2: entries come from the persisted breaker store (restored state
|
|
1200
|
+
// included); catalog-derived flags are refreshed on every boot.
|
|
1201
|
+
const entry = this.breakers.ensure(key, router.circuitBreaker.initialCooldownMs)
|
|
1094
1202
|
const catalogEntry = this.modelCatalog.get(key)
|
|
1203
|
+
entry.stale = !this.modelCatalog.has(key)
|
|
1095
1204
|
entry.unsupported = Boolean(catalogEntry && !catalogEntry.routeable)
|
|
1096
1205
|
if (entry.stale && !this.staleNotifications.has(key)) {
|
|
1097
1206
|
this.staleNotifications.add(key)
|
|
@@ -1149,8 +1258,11 @@ class RouterRuntime {
|
|
|
1149
1258
|
reloadConfigFromDisk() {
|
|
1150
1259
|
try {
|
|
1151
1260
|
const nextConfig = loadConfig()
|
|
1152
|
-
// 📖
|
|
1153
|
-
|
|
1261
|
+
// 📖 v2 fix: do NOT run ensureRouterConfigForDaemon here. It rebuilds
|
|
1262
|
+
// the router section from DEFAULT_ROUTER_SETTINGS and silently
|
|
1263
|
+
// discards user failover tuning (requestTimeoutMs, streamStall, and
|
|
1264
|
+
// the v2-only fields) on every 10s tick. The raw file is adopted
|
|
1265
|
+
// as-is; routerConfig() normalizes on read.
|
|
1154
1266
|
this.config = nextConfig
|
|
1155
1267
|
this.refreshRouteState()
|
|
1156
1268
|
this.scheduleProbeLoop()
|
|
@@ -1178,16 +1290,54 @@ class RouterRuntime {
|
|
|
1178
1290
|
return [...(set?.models || [])].sort((a, b) => a.priority - b.priority)
|
|
1179
1291
|
}
|
|
1180
1292
|
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
this.
|
|
1293
|
+
// ─── v2 engine: failover settings + quota pauses + breaker plumbing ───────
|
|
1294
|
+
|
|
1295
|
+
// 📖 v2-specific failover knobs are read from the RAW config: the shared
|
|
1296
|
+
// normalizer only knows the v1 field names and would drop the new ones.
|
|
1297
|
+
failoverSettings() {
|
|
1298
|
+
const normalized = this.routerConfig().failover
|
|
1299
|
+
const raw = (this.config?.router?.failover && typeof this.config.router.failover === 'object')
|
|
1300
|
+
? this.config.router.failover
|
|
1301
|
+
: {}
|
|
1302
|
+
const validation = raw.contentValidation
|
|
1303
|
+
return {
|
|
1304
|
+
...normalized,
|
|
1305
|
+
bodyReadTimeoutMs: clampIntV2(raw.bodyReadTimeoutMs, DEFAULT_BODY_READ_TIMEOUT_MS, { min: 5000, max: 300000 }),
|
|
1306
|
+
totalBudgetMs: clampIntV2(raw.totalBudgetMs, DEFAULT_TOTAL_BUDGET_MS, { min: 10000, max: 600000 }),
|
|
1307
|
+
contentValidation: ['strict', 'basic', 'off'].includes(validation) ? validation : DEFAULT_CONTENT_VALIDATION,
|
|
1308
|
+
lastResortModel: parseLastResortModel(raw.lastResortModel),
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
breakerParams() {
|
|
1313
|
+
const cb = this.routerConfig().circuitBreaker
|
|
1314
|
+
return {
|
|
1315
|
+
failureThreshold: cb.failureThreshold,
|
|
1316
|
+
initialCooldownMs: cb.initialCooldownMs,
|
|
1317
|
+
maxCooldownMs: cb.maxCooldownMs,
|
|
1318
|
+
backoffMultiplier: cb.backoffMultiplier,
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
quotaPauseActive(key) {
|
|
1323
|
+
const pause = this.quotaPauses.get(key)
|
|
1324
|
+
if (!pause) return false
|
|
1325
|
+
if (Date.now() >= pause.until) {
|
|
1326
|
+
this.quotaPauses.delete(key)
|
|
1327
|
+
return false
|
|
1189
1328
|
}
|
|
1190
|
-
return
|
|
1329
|
+
return true
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
quotaPausesForKeys(keys) {
|
|
1333
|
+
return keys
|
|
1334
|
+
.map((key) => this.quotaPauses.get(key))
|
|
1335
|
+
.filter(Boolean)
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
updateCircuitForCooldown(key) {
|
|
1339
|
+
// 📖 v2: lazy OPEN -> HALF_OPEN promotion lives in the breaker store.
|
|
1340
|
+
return this.breakers.evaluate(key)
|
|
1191
1341
|
}
|
|
1192
1342
|
|
|
1193
1343
|
recordProbeResult(key, result) {
|
|
@@ -1244,69 +1394,86 @@ class RouterRuntime {
|
|
|
1244
1394
|
}
|
|
1245
1395
|
|
|
1246
1396
|
markAuthError(key, detail = 'authentication failed') {
|
|
1247
|
-
const state = this.
|
|
1397
|
+
const state = this.breakers.get(key)
|
|
1248
1398
|
if (!state) return
|
|
1249
|
-
|
|
1250
|
-
state.lastError = detail
|
|
1399
|
+
this.breakers.markFailure(key, { ...this.breakerParams(), detail, authError: true })
|
|
1251
1400
|
this.broadcast('circuit', { model: key, old_state: state.state, new_state: 'AUTH_ERROR', cooldown_ms: 0 })
|
|
1252
1401
|
}
|
|
1253
1402
|
|
|
1254
1403
|
markSuccess(key, latencyMs = null) {
|
|
1255
|
-
const
|
|
1256
|
-
|
|
1257
|
-
const oldState = state.state
|
|
1258
|
-
state.state = 'CLOSED'
|
|
1259
|
-
state.consecutiveFailures = 0
|
|
1260
|
-
state.cooldownMs = this.routerConfig().circuitBreaker.initialCooldownMs
|
|
1261
|
-
state.openedAt = null
|
|
1262
|
-
state.lastError = null
|
|
1263
|
-
state.authError = false
|
|
1404
|
+
const oldState = this.breakers.get(key)?.state
|
|
1405
|
+
this.breakers.markSuccess(key, this.routerConfig().circuitBreaker.initialCooldownMs)
|
|
1264
1406
|
this.quotaExhausted.delete(key)
|
|
1265
1407
|
this.quotaDetails.delete(key)
|
|
1266
|
-
|
|
1267
|
-
|
|
1408
|
+
this.quotaPauses.delete(key)
|
|
1409
|
+
if (oldState && oldState !== 'CLOSED') {
|
|
1410
|
+
this.broadcast('circuit', { model: key, old_state: oldState, new_state: 'CLOSED', cooldown_ms: 0 })
|
|
1268
1411
|
}
|
|
1269
1412
|
if (latencyMs !== null) this.recordProbeResult(key, { ok: true, latencyMs, code: 200 })
|
|
1270
1413
|
}
|
|
1271
1414
|
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
this.
|
|
1415
|
+
// 📖 Single funnel from a failure verdict to health state (v2): circuit
|
|
1416
|
+
// damage, quota pause and probe-window recording all derive from the
|
|
1417
|
+
// classifier's policy instead of ad-hoc per-path bookkeeping.
|
|
1418
|
+
applyFailureVerdict(key, verdict, { detail, statusCode = null, latencyMs = null, meta = {} } = {}) {
|
|
1419
|
+
if (verdict.kind === FAILURE_KINDS.AUTH) {
|
|
1420
|
+
this.breakers.markFailure(key, { ...this.breakerParams(), detail, statusCode, authError: true })
|
|
1421
|
+
this.broadcast('circuit', { model: key, state: 'AUTH_ERROR', reason: detail })
|
|
1422
|
+
} else if (verdict.healthDamage) {
|
|
1423
|
+
const result = this.breakers.markFailure(key, { ...this.breakerParams(), detail, statusCode })
|
|
1424
|
+
this.broadcast('circuit', {
|
|
1281
1425
|
model: key,
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1426
|
+
state: result.state,
|
|
1427
|
+
opened: result.opened,
|
|
1428
|
+
degraded: result.degraded,
|
|
1429
|
+
reason: detail,
|
|
1286
1430
|
})
|
|
1431
|
+
if (result.opened) this.logger.warn(`Circuit opened for ${key}`, { reason: detail })
|
|
1432
|
+
else if (result.degraded) this.logger.warn(`Circuit DEGRADED for ${key}`, { reason: detail })
|
|
1433
|
+
} else {
|
|
1434
|
+
// 📖 No health damage (client-caused 4xx): remember the reason for the
|
|
1435
|
+
// dashboards but never push the breaker toward OPEN.
|
|
1436
|
+
const breaker = this.breakers.ensure(key, this.routerConfig().circuitBreaker.initialCooldownMs)
|
|
1437
|
+
breaker.lastError = detail
|
|
1287
1438
|
}
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
Math.max(router.circuitBreaker.initialCooldownMs, state.cooldownMs * router.circuitBreaker.backoffMultiplier),
|
|
1296
|
-
)
|
|
1297
|
-
this.broadcast('circuit', { model: key, old_state: oldState, new_state: state.state, cooldown_ms: state.cooldownMs })
|
|
1298
|
-
this.logger.warn(`Circuit opened for ${key}`, { reason: detail, cooldown_ms: state.cooldownMs })
|
|
1299
|
-
void sendUsageTelemetry(this.config, {}, {
|
|
1300
|
-
event: 'app_router_circuit_open',
|
|
1301
|
-
mode: 'daemon',
|
|
1302
|
-
properties: {
|
|
1303
|
-
model: key,
|
|
1304
|
-
consecutive_failures: state.consecutiveFailures,
|
|
1305
|
-
cooldown_ms: state.cooldownMs,
|
|
1306
|
-
},
|
|
1307
|
-
})
|
|
1439
|
+
// 📖 Quota bookkeeping: the pause map drives routing skips (Retry-After
|
|
1440
|
+
// aware); the legacy set feeds the all-models-failed error payloads.
|
|
1441
|
+
if ((verdict.quotaPauseMs != null && verdict.quotaPauseMs > 0)
|
|
1442
|
+
|| verdict.kind === FAILURE_KINDS.RATE_LIMIT
|
|
1443
|
+
|| verdict.kind === FAILURE_KINDS.QUOTA
|
|
1444
|
+
|| meta.quotaExhausted) {
|
|
1445
|
+
this.recordQuotaPause(key, verdict.quotaPauseMs || DEFAULT_QUOTA_PAUSE_MS, statusCode, meta)
|
|
1308
1446
|
}
|
|
1309
|
-
|
|
1447
|
+
if (verdict.kind !== FAILURE_KINDS.RATE_LIMIT && verdict.kind !== FAILURE_KINDS.QUOTA) {
|
|
1448
|
+
this.quotaExhausted.delete(key)
|
|
1449
|
+
}
|
|
1450
|
+
this.recordProbeResult(key, { ok: false, latencyMs, code: statusCode || 'ERR', error: detail })
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
recordQuotaPause(key, pauseMs, statusCode, meta = {}) {
|
|
1454
|
+
this.quotaPauses.set(key, {
|
|
1455
|
+
model: key,
|
|
1456
|
+
until: Date.now() + pauseMs,
|
|
1457
|
+
retry_after_ms: pauseMs,
|
|
1458
|
+
status: statusCode,
|
|
1459
|
+
rate_limit_headers: meta.rateLimitHeaders || {},
|
|
1460
|
+
last_seen: nowIso(),
|
|
1461
|
+
})
|
|
1462
|
+
this.quotaExhausted.add(key)
|
|
1463
|
+
this.quotaDetails.set(key, {
|
|
1464
|
+
model: key,
|
|
1465
|
+
status: statusCode,
|
|
1466
|
+
retry_after_ms: pauseMs,
|
|
1467
|
+
rate_limit_headers: meta.rateLimitHeaders || {},
|
|
1468
|
+
last_seen: nowIso(),
|
|
1469
|
+
})
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
markFailure(key, detail, statusCode = null, meta = {}) {
|
|
1473
|
+
// 📖 v2: classify first, then apply the verdict policy. Same call shape
|
|
1474
|
+
// as v1 so probe loops and legacy paths keep working.
|
|
1475
|
+
const verdict = classifyFailure({ status: statusCode, retryAfterMs: meta.retryAfterMs ?? null })
|
|
1476
|
+
this.applyFailureVerdict(key, verdict, { detail, statusCode, meta })
|
|
1310
1477
|
}
|
|
1311
1478
|
|
|
1312
1479
|
quotaDetailsForKeys(keys) {
|
|
@@ -1414,27 +1581,56 @@ class RouterRuntime {
|
|
|
1414
1581
|
// 📖 Circuit-breaker safety is preserved: CLOSED (healthy) models always come
|
|
1415
1582
|
// 📖 before HALF_OPEN (probing after cooldown) models, so a recovering model
|
|
1416
1583
|
// 📖 never pre-empts a known-good one.
|
|
1417
|
-
getRoutingCandidates(set) {
|
|
1584
|
+
getRoutingCandidates(set, { trace = null, blockedProviders = null } = {}) {
|
|
1418
1585
|
const scored = this.scoreCandidates(set)
|
|
1419
|
-
const usable =
|
|
1420
|
-
|
|
1421
|
-
if (
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1586
|
+
const usable = []
|
|
1587
|
+
for (const candidate of scored) {
|
|
1588
|
+
if (blockedProviders?.has(candidate.provider)) {
|
|
1589
|
+
traceSkip(trace, candidate.key, 'provider_blocked')
|
|
1590
|
+
continue
|
|
1591
|
+
}
|
|
1592
|
+
if (!candidate.catalog || candidate.circuit?.stale) {
|
|
1593
|
+
traceSkip(trace, candidate.key, 'stale')
|
|
1594
|
+
continue
|
|
1595
|
+
}
|
|
1596
|
+
if (!candidate.catalog.routeable || candidate.circuit?.unsupported) {
|
|
1597
|
+
traceSkip(trace, candidate.key, 'unsupported')
|
|
1598
|
+
continue
|
|
1599
|
+
}
|
|
1600
|
+
if (candidate.circuit?.authError) {
|
|
1601
|
+
traceSkip(trace, candidate.key, 'auth_error')
|
|
1602
|
+
continue
|
|
1603
|
+
}
|
|
1604
|
+
if (!this.getApiKeyForProvider(candidate.provider)) {
|
|
1605
|
+
traceSkip(trace, candidate.key, 'missing_key')
|
|
1606
|
+
continue
|
|
1607
|
+
}
|
|
1608
|
+
// 📖 v2: a rate-limited model is paused for its Retry-After window and
|
|
1609
|
+
// skipped entirely, instead of being retried until its circuit opens.
|
|
1610
|
+
if (this.quotaPauseActive(candidate.key)) {
|
|
1611
|
+
traceSkip(trace, candidate.key, 'quota_paused')
|
|
1612
|
+
continue
|
|
1613
|
+
}
|
|
1614
|
+
const state = candidate.circuit?.state || 'UNKNOWN'
|
|
1615
|
+
if (state !== 'CLOSED' && state !== 'HALF_OPEN' && state !== 'DEGRADED') {
|
|
1616
|
+
traceSkip(trace, candidate.key, state === 'OPEN' ? 'circuit_open' : 'circuit_state')
|
|
1617
|
+
continue
|
|
1618
|
+
}
|
|
1619
|
+
usable.push(candidate)
|
|
1620
|
+
}
|
|
1426
1621
|
// 📖 New ordering: prioritize by explicit priority first, then by circuit state
|
|
1427
|
-
// 📖 (CLOSED before HALF_OPEN), and finally by health score
|
|
1428
|
-
// 📖 This ensures a higher
|
|
1429
|
-
// 📖 in HALF_OPEN while a lower
|
|
1430
|
-
|
|
1622
|
+
// 📖 (CLOSED before DEGRADED before HALF_OPEN), and finally by health score
|
|
1623
|
+
// 📖 (higher is better). This ensures a higher-priority model is never
|
|
1624
|
+
// 📖 skipped just because it is in HALF_OPEN while a lower-priority CLOSED
|
|
1625
|
+
// 📖 model is available. DEGRADED (failing, not yet tripped) still routes.
|
|
1626
|
+
const stateOrder = { CLOSED: 0, DEGRADED: 1, HALF_OPEN: 2 }
|
|
1431
1627
|
const comparator = (a, b) => {
|
|
1432
1628
|
if (a.priority !== b.priority) return a.priority - b.priority
|
|
1433
1629
|
const aState = a.circuit?.state || 'UNKNOWN'
|
|
1434
1630
|
const bState = b.circuit?.state || 'UNKNOWN'
|
|
1435
1631
|
if (aState !== bState) {
|
|
1436
|
-
const aRank = stateOrder[aState] ??
|
|
1437
|
-
const bRank = stateOrder[bState] ??
|
|
1632
|
+
const aRank = stateOrder[aState] ?? 3
|
|
1633
|
+
const bRank = stateOrder[bState] ?? 3
|
|
1438
1634
|
return aRank - bRank
|
|
1439
1635
|
}
|
|
1440
1636
|
// higher score first
|
|
@@ -1471,11 +1667,16 @@ class RouterRuntime {
|
|
|
1471
1667
|
? 'STALE'
|
|
1472
1668
|
: candidate.circuit?.unsupported
|
|
1473
1669
|
? 'UNSUPPORTED'
|
|
1474
|
-
: candidate.
|
|
1670
|
+
: this.quotaPauseActive(candidate.key)
|
|
1671
|
+
? 'QUOTA_PAUSED'
|
|
1672
|
+
: candidate.circuit?.state || 'UNKNOWN',
|
|
1475
1673
|
score: Number(candidate.score.toFixed(4)),
|
|
1476
1674
|
last_latency_ms: candidate.stats.last?.latencyMs ?? null,
|
|
1477
1675
|
uptime: candidate.stats.uptime,
|
|
1478
1676
|
last_error: candidate.circuit?.lastError || null,
|
|
1677
|
+
quota_paused_until: this.quotaPauses.get(candidate.key)
|
|
1678
|
+
? new Date(this.quotaPauses.get(candidate.key).until).toISOString()
|
|
1679
|
+
: null,
|
|
1479
1680
|
// 📖 AI Latency benchmark results for the Router Dashboard's "Probe all"
|
|
1480
1681
|
// 📖 button. Mirrors the per-model fields already exposed on /api/models
|
|
1481
1682
|
// 📖 so the set list can show live AI latency + TPS after a probe.
|
|
@@ -1484,6 +1685,15 @@ class RouterRuntime {
|
|
|
1484
1685
|
}))
|
|
1485
1686
|
}
|
|
1486
1687
|
|
|
1688
|
+
// 📖 v2: breaker census for dashboards (CLOSED / DEGRADED / OPEN / ...).
|
|
1689
|
+
getModelStates(set = this.getSet()) {
|
|
1690
|
+
const counts = { CLOSED: 0, DEGRADED: 0, OPEN: 0, HALF_OPEN: 0, AUTH_ERROR: 0, QUOTA_PAUSED: 0 }
|
|
1691
|
+
for (const model of this.getModelHealth(set || { models: [] })) {
|
|
1692
|
+
if (counts[model.state] !== undefined) counts[model.state] += 1
|
|
1693
|
+
}
|
|
1694
|
+
return counts
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1487
1697
|
findBestModelForProviderInSources(providerKey) {
|
|
1488
1698
|
const source = sources[providerKey]
|
|
1489
1699
|
if (!source || !Array.isArray(source.models)) return null
|
|
@@ -1675,6 +1885,24 @@ class RouterRuntime {
|
|
|
1675
1885
|
configPath: CONFIG_PATH,
|
|
1676
1886
|
tokenStatsPath: ROUTER_TOKENS_PATH,
|
|
1677
1887
|
logPath: ROUTER_LOG_PATH,
|
|
1888
|
+
// 📖 v2 engine: failover knobs, live breaker census, quota pauses and
|
|
1889
|
+
// the persisted request-history aggregates for dashboards.
|
|
1890
|
+
router: 'v2',
|
|
1891
|
+
failover: {
|
|
1892
|
+
maxRetries: router.failover.maxRetries,
|
|
1893
|
+
requestTimeoutMs: router.failover.requestTimeoutMs,
|
|
1894
|
+
bodyReadTimeoutMs: this.failoverSettings().bodyReadTimeoutMs,
|
|
1895
|
+
totalBudgetMs: this.failoverSettings().totalBudgetMs,
|
|
1896
|
+
contentValidation: this.failoverSettings().contentValidation,
|
|
1897
|
+
lastResortModel: this.failoverSettings().lastResortModel?.key || null,
|
|
1898
|
+
},
|
|
1899
|
+
modelStates: this.getModelStates(activeSet),
|
|
1900
|
+
quotaPauses: this.quotaPausesForKeys([...this.quotaPauses.keys()]).map((p) => ({
|
|
1901
|
+
model: p.model,
|
|
1902
|
+
until: new Date(p.until).toISOString(),
|
|
1903
|
+
retry_after_ms: p.retry_after_ms,
|
|
1904
|
+
})),
|
|
1905
|
+
history: this.history.stats(),
|
|
1678
1906
|
// 📖 Probe-cache (t1): live aggregates from the persistent probe-cache.
|
|
1679
1907
|
// 📖 Surfaced so the Web Dashboard + CLI can show cache hit rate + how many
|
|
1680
1908
|
// 📖 broken models are currently hidden. Refreshed every /stats call.
|
|
@@ -1718,6 +1946,9 @@ class RouterRuntime {
|
|
|
1718
1946
|
completed: this.webGlobalBenchmarkCompleted || 0,
|
|
1719
1947
|
},
|
|
1720
1948
|
requestLog: this.requestLog.slice(0, 20),
|
|
1949
|
+
// 📖 v2 engine: persisted breakers + recent decision traces.
|
|
1950
|
+
breakers: this.breakers.snapshot(),
|
|
1951
|
+
traces: this.recentTraces.slice(-20).map((trace) => this.historyEntryFromTrace(trace)),
|
|
1721
1952
|
activeRequests: Array.from(this.activeRequests.values()).map(r => ({
|
|
1722
1953
|
requestId: r.requestId,
|
|
1723
1954
|
at: r.at,
|
|
@@ -2081,119 +2312,245 @@ class RouterRuntime {
|
|
|
2081
2312
|
this.probeWatchdog.unref?.()
|
|
2082
2313
|
}
|
|
2083
2314
|
|
|
2084
|
-
|
|
2085
|
-
|
|
2315
|
+
// 📖 Shared admission helpers for the v2 engine (decision traces, pinned
|
|
2316
|
+
// models, protocol-aware errors). Everything below keeps the v1 call
|
|
2317
|
+
// shapes so the whole v1 surface (sets API, web dashboard, playground)
|
|
2318
|
+
// keeps working on top of the hardened engine.
|
|
2319
|
+
|
|
2320
|
+
resolvePinnedCandidate(pinned) {
|
|
2321
|
+
const key = modelKey(pinned.provider, pinned.model)
|
|
2322
|
+
const catalog = this.modelCatalog.get(key)
|
|
2323
|
+
if (!catalog) return { error: `Unknown model: ${key}` }
|
|
2324
|
+
if (!isRouteableProvider(pinned.provider, sources)) return { error: `Provider is not routeable: ${pinned.provider}` }
|
|
2325
|
+
if (!this.getApiKeyForProvider(pinned.provider)) return { error: `No API key configured for ${pinned.provider}` }
|
|
2326
|
+
const breaker = this.breakers.get(key) || {}
|
|
2327
|
+
return {
|
|
2328
|
+
candidate: {
|
|
2329
|
+
provider: pinned.provider,
|
|
2330
|
+
model: pinned.model,
|
|
2331
|
+
priority: 1,
|
|
2332
|
+
key,
|
|
2333
|
+
score: 0,
|
|
2334
|
+
stats: this.getWindowStats(key),
|
|
2335
|
+
circuit: breaker,
|
|
2336
|
+
catalog,
|
|
2337
|
+
},
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
rememberTrace(trace) {
|
|
2342
|
+
this.recentTraces.push(trace)
|
|
2343
|
+
while (this.recentTraces.length > 50) this.recentTraces.shift()
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2346
|
+
historyEntryFromTrace(trace, { stream = false, set = null } = {}) {
|
|
2347
|
+
return {
|
|
2348
|
+
request_id: trace.request_id,
|
|
2349
|
+
at: trace.at,
|
|
2350
|
+
set: set || trace.set,
|
|
2351
|
+
protocol: trace.protocol,
|
|
2352
|
+
model_requested: trace.model_requested,
|
|
2353
|
+
pinned_model: trace.pinned_model,
|
|
2354
|
+
served_model: trace.served_model,
|
|
2355
|
+
outcome: trace.outcome,
|
|
2356
|
+
attempts: trace.attempts,
|
|
2357
|
+
skipped: trace.skipped,
|
|
2358
|
+
wall_ms: trace.wall_ms,
|
|
2359
|
+
tokens: trace.tokens,
|
|
2360
|
+
stream,
|
|
2361
|
+
last_resort_used: trace.last_resort_used,
|
|
2362
|
+
summary: traceSummary(trace),
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
decisionHeaders(trace) {
|
|
2367
|
+
const lastModel = trace.attempts.length > 0 ? trace.attempts[trace.attempts.length - 1].model : 'none'
|
|
2368
|
+
return {
|
|
2369
|
+
'x-fcm-router-model': trace.served_model || lastModel,
|
|
2370
|
+
'x-fcm-v2-model': trace.served_model || lastModel,
|
|
2371
|
+
'x-fcm-v2-attempts': String(trace.attempts.length),
|
|
2372
|
+
'x-fcm-v2-decision': decisionHeaderValue(trace),
|
|
2373
|
+
'x-request-id': trace.request_id,
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
sendProtocolError(res, protocol, statusCode, message, requestId, extra = {}) {
|
|
2378
|
+
if (protocol === 'anthropic') {
|
|
2379
|
+
sendJson(res, statusCode, anthropicErrorPayload(anthropicErrorTypeForStatus(statusCode), message), {
|
|
2380
|
+
'x-request-id': requestId,
|
|
2381
|
+
...extra.headers,
|
|
2382
|
+
})
|
|
2383
|
+
return
|
|
2384
|
+
}
|
|
2385
|
+
sendError(res, statusCode, message, 'service_unavailable', extra.code || 'router_error', requestId, extra.payload)
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
retryAfterHeaders() {
|
|
2389
|
+
const pauses = this.quotaPausesForKeys([...this.quotaPauses.keys()])
|
|
2390
|
+
if (pauses.length === 0) return {}
|
|
2391
|
+
const maxUntil = Math.max(...pauses.map((p) => p.until))
|
|
2392
|
+
const seconds = Math.max(1, Math.ceil((maxUntil - Date.now()) / 1000))
|
|
2393
|
+
return { 'Retry-After': String(Math.min(seconds, 900)) }
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
buildUpstreamBody(body, candidate, stream) {
|
|
2397
|
+
const bodyWithPrePrompt = applyPrePromptToBody(body, this.routerConfig().prePrompt)
|
|
2398
|
+
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2399
|
+
const upstreamBody = {
|
|
2400
|
+
...bodyNormalized,
|
|
2401
|
+
model: getApiModelId(candidate.provider, candidate.model),
|
|
2402
|
+
stream,
|
|
2403
|
+
}
|
|
2404
|
+
// 📖 Some providers/models fail if we send custom internal params.
|
|
2405
|
+
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2406
|
+
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2407
|
+
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2408
|
+
return upstreamBody
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
async routeRequest({ req, res, body, setName, requestId, protocol = 'openai', anthropicModelName = null }) {
|
|
2412
|
+
const trace = createDecisionTrace({
|
|
2086
2413
|
requestId,
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
attempts: 0,
|
|
2091
|
-
tokens: 0,
|
|
2092
|
-
stalled: false
|
|
2414
|
+
set: setName || this.routerConfig().activeSet,
|
|
2415
|
+
protocol,
|
|
2416
|
+
modelRequested: body?.model || 'fcm',
|
|
2093
2417
|
})
|
|
2418
|
+
const started = Date.now()
|
|
2419
|
+
|
|
2420
|
+
// 📖 v2 lifecycle fix: every rejection guard runs BEFORE the
|
|
2421
|
+
// active-request entry exists, and the entry only lives inside the
|
|
2422
|
+
// try/finally. v1 created it first, so each rejected request leaked a
|
|
2423
|
+
// ghost "active request" into /stats until restart.
|
|
2094
2424
|
if (this.shuttingDown) {
|
|
2095
|
-
|
|
2425
|
+
this.sendProtocolError(res, protocol, 503, 'Daemon is shutting down', requestId)
|
|
2426
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2427
|
+
this.rememberTrace(trace)
|
|
2096
2428
|
return
|
|
2097
2429
|
}
|
|
2098
2430
|
if (this.inFlight >= MAX_CONCURRENT_REQUESTS) {
|
|
2099
2431
|
sendError(res, 503, 'Router overloaded, too many concurrent requests', 'service_unavailable', 'router_overloaded', requestId)
|
|
2432
|
+
finishTrace(trace, { outcome: 'overloaded', wallMs: Date.now() - started })
|
|
2433
|
+
this.rememberTrace(trace)
|
|
2100
2434
|
return
|
|
2101
2435
|
}
|
|
2102
2436
|
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
|
2103
2437
|
sendError(res, 400, 'Request body must be a JSON object', 'invalid_request_error', 'invalid_json_object', requestId)
|
|
2438
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2439
|
+
this.rememberTrace(trace)
|
|
2104
2440
|
return
|
|
2105
2441
|
}
|
|
2106
2442
|
if (typeof body.model !== 'string' || !body.model.trim()) {
|
|
2107
2443
|
sendError(res, 400, 'Missing required field: model', 'invalid_request_error', 'missing_model', requestId)
|
|
2444
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2445
|
+
this.rememberTrace(trace)
|
|
2108
2446
|
return
|
|
2109
2447
|
}
|
|
2110
2448
|
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2449
|
+
// 📖 Model spec: `fcm` (active set), `fcm:<set>`, or `fcm:@provider/model`
|
|
2450
|
+
// (pinned single-model request, failover disabled - used by the
|
|
2451
|
+
// test-via-router actions to exercise ONE model through the full chain).
|
|
2452
|
+
const spec = parseFcmModel(body.model)
|
|
2453
|
+
let set = null
|
|
2454
|
+
let pinned = null
|
|
2455
|
+
if (spec.kind === 'pinned') {
|
|
2456
|
+
pinned = spec.pinned
|
|
2457
|
+
set = this.getSet(null)
|
|
2458
|
+
if (!set) {
|
|
2459
|
+
this.sendProtocolError(res, protocol, 503, 'No active router set', requestId, { code: 'set_not_found' })
|
|
2460
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2461
|
+
this.rememberTrace(trace)
|
|
2462
|
+
return
|
|
2463
|
+
}
|
|
2464
|
+
trace.pinned_model = `${pinned.provider}/${pinned.model}`
|
|
2465
|
+
} else {
|
|
2466
|
+
const requestedSetName = spec.kind === 'set' ? spec.set : setName
|
|
2467
|
+
set = this.getSet(requestedSetName)
|
|
2468
|
+
if (!set) {
|
|
2469
|
+
sendError(res, 404, `Router set not found: ${requestedSetName || this.routerConfig().activeSet}`, 'invalid_request_error', 'set_not_found', requestId)
|
|
2470
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2471
|
+
this.rememberTrace(trace)
|
|
2472
|
+
return
|
|
2473
|
+
}
|
|
2115
2474
|
}
|
|
2116
2475
|
|
|
2117
|
-
const
|
|
2118
|
-
const
|
|
2119
|
-
const
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2476
|
+
const settings = this.failoverSettings()
|
|
2477
|
+
const maxAttempts = pinned ? 1 : Math.min(1 + this.routerConfig().failover.maxRetries, 6)
|
|
2478
|
+
const deadline = Date.now() + settings.totalBudgetMs
|
|
2479
|
+
const stream = body.stream === true
|
|
2480
|
+
|
|
2481
|
+
this.inFlight += 1
|
|
2482
|
+
const activeReq = {
|
|
2483
|
+
requestId,
|
|
2484
|
+
at: Date.now(),
|
|
2485
|
+
model: body.model,
|
|
2486
|
+
current_model: null,
|
|
2487
|
+
attempts: 0,
|
|
2488
|
+
tokens: 0,
|
|
2489
|
+
stalled: false,
|
|
2490
|
+
last_activity_at: Date.now(),
|
|
2491
|
+
}
|
|
2492
|
+
this.activeRequests.set(requestId, activeReq)
|
|
2493
|
+
try {
|
|
2494
|
+
let candidates
|
|
2495
|
+
if (pinned) {
|
|
2496
|
+
// 📖 Pinned tests deliberately bypass availability pre-skips (circuit
|
|
2497
|
+
// OPEN, quota pause): the point is a genuine attempt that feeds real
|
|
2498
|
+
// health data back into the breakers.
|
|
2499
|
+
const resolved = this.resolvePinnedCandidate(pinned)
|
|
2500
|
+
if (resolved.error) {
|
|
2501
|
+
this.sendProtocolError(res, protocol, 400, resolved.error, requestId, { code: 'invalid_model' })
|
|
2502
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2503
|
+
return
|
|
2143
2504
|
}
|
|
2505
|
+
candidates = [resolved.candidate]
|
|
2506
|
+
} else {
|
|
2507
|
+
candidates = this.getRoutingCandidates(set, { trace })
|
|
2144
2508
|
}
|
|
2145
2509
|
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2151
|
-
model_health: health,
|
|
2152
|
-
})
|
|
2153
|
-
void sendUsageTelemetry(this.config, {}, {
|
|
2154
|
-
event: 'app_router_all_down',
|
|
2155
|
-
mode: 'daemon',
|
|
2156
|
-
properties: {
|
|
2157
|
-
set_name: set.name,
|
|
2158
|
-
models_tried: [],
|
|
2159
|
-
quota_exhausted_count: quotaExhausted.length,
|
|
2160
|
-
},
|
|
2161
|
-
})
|
|
2162
|
-
return
|
|
2163
|
-
}
|
|
2510
|
+
if (candidates.length === 0) {
|
|
2511
|
+
this.sendAllModelsUnavailable(res, trace, set, requestId, protocol)
|
|
2512
|
+
return
|
|
2513
|
+
}
|
|
2164
2514
|
|
|
2165
|
-
this.inFlight += 1
|
|
2166
|
-
try {
|
|
2167
2515
|
const tried = []
|
|
2516
|
+
const failedKinds = []
|
|
2168
2517
|
const blockedProviders = new Set()
|
|
2169
2518
|
let attemptIndex = 0
|
|
2170
|
-
// 📖 attemptChain is a private copy of the routing order: on a family
|
|
2171
|
-
// failover (t8) the same-family candidate is swapped in as the NEXT
|
|
2172
|
-
// attempt, so the iteration order itself follows the two-stage policy.
|
|
2173
2519
|
const attemptChain = candidates.slice()
|
|
2520
|
+
|
|
2174
2521
|
for (let index = 0; index < attemptChain.length && attemptIndex < maxAttempts; index += 1) {
|
|
2522
|
+
if (Date.now() > deadline) {
|
|
2523
|
+
this.logger.warn('Request retry budget exhausted; failing over to error', { request_id: requestId })
|
|
2524
|
+
break
|
|
2525
|
+
}
|
|
2175
2526
|
const candidate = attemptChain[index]
|
|
2176
2527
|
if (blockedProviders.has(candidate.provider)) continue
|
|
2177
2528
|
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
activeReq.current_model = candidate.key
|
|
2181
|
-
activeReq.attempts = attemptIndex + 1
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2529
|
+
activeReq.current_model = candidate.key
|
|
2530
|
+
activeReq.attempts = attemptIndex + 1
|
|
2184
2531
|
tried.push(candidate.key)
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2532
|
+
traceAttempt(trace, candidate.key, { status: null })
|
|
2533
|
+
|
|
2534
|
+
const result = stream
|
|
2535
|
+
? await this.proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, anthropicModelName })
|
|
2536
|
+
: await this.proxyJsonRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace })
|
|
2537
|
+
|
|
2538
|
+
trace.attempts[trace.attempts.length - 1] = {
|
|
2539
|
+
...trace.attempts[trace.attempts.length - 1],
|
|
2540
|
+
model: candidate.key,
|
|
2541
|
+
status: result.status ?? null,
|
|
2542
|
+
latency_ms: result.latencyMs ?? null,
|
|
2543
|
+
error: result.reason || null,
|
|
2544
|
+
at: new Date().toISOString(),
|
|
2545
|
+
}
|
|
2546
|
+
if (result.verdict) failedKinds.push(result.verdict.kind)
|
|
2188
2547
|
if (result.done) return
|
|
2189
2548
|
attemptIndex += 1
|
|
2190
|
-
if (result.
|
|
2549
|
+
if (result.verdict?.blockProvider) blockedProviders.add(candidate.provider)
|
|
2550
|
+
|
|
2191
2551
|
if (result.failoverToNext && attemptIndex < maxAttempts) {
|
|
2192
2552
|
// 📖 Two-stage failover (t8): prefer a healthy model of the SAME
|
|
2193
|
-
// family on another provider
|
|
2194
|
-
// Together) so the user's output style doesn't change mid-request.
|
|
2195
|
-
// Falls back to the historical set-order pick, and is disabled
|
|
2196
|
-
// per-set via familyFailover: false.
|
|
2553
|
+
// family on another provider so output style stays consistent.
|
|
2197
2554
|
const pick = pickNextCandidate({
|
|
2198
2555
|
candidates: attemptChain,
|
|
2199
2556
|
failedCandidate: candidate,
|
|
@@ -2202,9 +2559,6 @@ class RouterRuntime {
|
|
|
2202
2559
|
familyFailover: set.familyFailover !== false,
|
|
2203
2560
|
})
|
|
2204
2561
|
const next = pick?.candidate || null
|
|
2205
|
-
// 📖 Reorder the remaining chain so `next` is genuinely the following
|
|
2206
|
-
// attempt. With set-order picks this is already the case (or the
|
|
2207
|
-
// skipped entries are blocked anyway), so behaviour is unchanged.
|
|
2208
2562
|
if (next && attemptChain[index + 1] !== next) {
|
|
2209
2563
|
const nextIndex = attemptChain.indexOf(next)
|
|
2210
2564
|
if (nextIndex > index) {
|
|
@@ -2212,6 +2566,8 @@ class RouterRuntime {
|
|
|
2212
2566
|
attemptChain.splice(index + 1, 0, next)
|
|
2213
2567
|
}
|
|
2214
2568
|
}
|
|
2569
|
+
// 📖 t8: stamp the reason ('family_failover' | 'set_order') on the
|
|
2570
|
+
// active request so every log entry of the next attempt carries it.
|
|
2215
2571
|
const activeReqForReason = this.activeRequests.get(requestId)
|
|
2216
2572
|
if (next && activeReqForReason) activeReqForReason.failoverReason = pick.reason
|
|
2217
2573
|
this.logger.warn(
|
|
@@ -2233,75 +2589,123 @@ class RouterRuntime {
|
|
|
2233
2589
|
}
|
|
2234
2590
|
}
|
|
2235
2591
|
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2592
|
+
// 📖 Last-resort escape hatch (v2): one final configured model outside
|
|
2593
|
+
// the rotation gets a single shot before the client sees an error.
|
|
2594
|
+
const lastResort = settings.lastResortModel
|
|
2595
|
+
if (!pinned && lastResort && !tried.includes(lastResort.key) && !blockedProviders.has(lastResort.provider) && Date.now() <= deadline) {
|
|
2596
|
+
const resolved = this.resolvePinnedCandidate({ provider: lastResort.provider, model: lastResort.model })
|
|
2597
|
+
if (resolved.candidate) {
|
|
2598
|
+
this.logger.warn(`All candidates failed; trying last-resort model ${lastResort.key}`, { request_id: requestId })
|
|
2599
|
+
trace.last_resort_used = true
|
|
2600
|
+
activeReq.current_model = lastResort.key
|
|
2601
|
+
tried.push(lastResort.key)
|
|
2602
|
+
traceAttempt(trace, lastResort.key, { status: null })
|
|
2603
|
+
const result = stream
|
|
2604
|
+
? await this.proxyStreamingRequest({ req, res, body, candidate: resolved.candidate, requestId, attemptIndex, protocol, trace, isLastResort: true, anthropicModelName })
|
|
2605
|
+
: await this.proxyJsonRequest({ req, res, body, candidate: resolved.candidate, requestId, attemptIndex, protocol, trace, isLastResort: true })
|
|
2606
|
+
trace.attempts[trace.attempts.length - 1] = {
|
|
2607
|
+
...trace.attempts[trace.attempts.length - 1],
|
|
2608
|
+
model: lastResort.key,
|
|
2609
|
+
status: result.status ?? null,
|
|
2610
|
+
latency_ms: result.latencyMs ?? null,
|
|
2611
|
+
error: result.reason || null,
|
|
2612
|
+
at: new Date().toISOString(),
|
|
2613
|
+
}
|
|
2614
|
+
if (result.verdict) failedKinds.push(result.verdict.kind)
|
|
2615
|
+
if (result.done) return
|
|
2260
2616
|
}
|
|
2261
2617
|
}
|
|
2262
2618
|
|
|
2263
|
-
|
|
2264
|
-
set: set.name,
|
|
2265
|
-
models_tried: tried,
|
|
2266
|
-
quota_exhausted: quotaExhausted,
|
|
2267
|
-
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2268
|
-
})
|
|
2619
|
+
this.sendAllModelsFailed(res, trace, set, requestId, protocol, { tried, failedKinds, stream })
|
|
2269
2620
|
} finally {
|
|
2270
2621
|
this.inFlight -= 1
|
|
2271
2622
|
this.activeRequests.delete(requestId)
|
|
2623
|
+
const wallMs = Date.now() - started
|
|
2624
|
+
finishTrace(trace, {
|
|
2625
|
+
outcome: trace.outcome || (trace.served_model ? 'served' : 'all_failed'),
|
|
2626
|
+
wallMs,
|
|
2627
|
+
servedModel: trace.served_model,
|
|
2628
|
+
lastResort: trace.last_resort_used,
|
|
2629
|
+
tokens: activeReq.tokens,
|
|
2630
|
+
})
|
|
2631
|
+
this.rememberTrace(trace)
|
|
2632
|
+
this.history.append(this.historyEntryFromTrace(trace, { stream, set: set?.name || null }))
|
|
2272
2633
|
}
|
|
2273
2634
|
}
|
|
2274
2635
|
|
|
2275
|
-
|
|
2636
|
+
sendAllModelsUnavailable(res, trace, set, requestId, protocol) {
|
|
2637
|
+
const health = this.getModelHealth(set)
|
|
2638
|
+
const quotaExhausted = [...this.quotaExhausted].filter((key) => set.models.some((model) => modelKey(model.provider, model.model) === key))
|
|
2639
|
+
const allAuthError = health.length > 0 && health.every((h) => h.state === 'AUTH_ERROR')
|
|
2640
|
+
const allPaused = health.length > 0 && health.every((h) => h.state === 'QUOTA_PAUSED')
|
|
2641
|
+
const allStaleOrUnsupported = health.length > 0 && health.every((h) => h.state === 'STALE' || h.state === 'UNSUPPORTED')
|
|
2642
|
+
let statusCode = 503
|
|
2643
|
+
if (allAuthError) statusCode = 401
|
|
2644
|
+
else if (allPaused || (quotaExhausted.length === health.length && health.length > 0)) statusCode = 429
|
|
2645
|
+
else if (allStaleOrUnsupported) statusCode = 400
|
|
2646
|
+
const headers = statusCode === 429 ? this.retryAfterHeaders() : {}
|
|
2647
|
+
this.sendProtocolError(res, protocol, statusCode,
|
|
2648
|
+
`All models in set are unavailable: ${set.name}`, requestId,
|
|
2649
|
+
{
|
|
2650
|
+
code: statusCode === 401 ? 'invalid_api_key' : statusCode === 429 ? 'insufficient_quota' : 'all_models_unavailable',
|
|
2651
|
+
headers,
|
|
2652
|
+
payload: { set: set.name, models_tried: [], quota_exhausted: quotaExhausted, quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted), model_health: health },
|
|
2653
|
+
})
|
|
2654
|
+
void sendUsageTelemetry(this.config, {}, {
|
|
2655
|
+
event: 'app_router_all_down',
|
|
2656
|
+
mode: 'daemon',
|
|
2657
|
+
properties: { set_name: set.name, models_tried: [], quota_exhausted_count: quotaExhausted.length },
|
|
2658
|
+
})
|
|
2659
|
+
finishTrace(trace, { outcome: 'all_failed', wallMs: Date.now() - new Date(trace.at).getTime() })
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
sendAllModelsFailed(res, trace, set, requestId, protocol, { tried, failedKinds, stream }) {
|
|
2663
|
+
// 📖 Status refinement by dominant failure kind (v2): an all-auth failure
|
|
2664
|
+
// is 401 for the client, all-quota is 429 (+ Retry-After), all-
|
|
2665
|
+
// invalid-request means the PAYLOAD is the problem (400).
|
|
2666
|
+
const kinds = failedKinds.length > 0 ? failedKinds : ['unknown']
|
|
2667
|
+
const allSame = kinds.every((k) => k === kinds[0])
|
|
2668
|
+
const statusCode = allSame ? clientStatusForKind(kinds[0]) : 503
|
|
2669
|
+
const headers = statusCode === 429 ? this.retryAfterHeaders() : {}
|
|
2670
|
+
const quotaExhausted = [...this.quotaExhausted].filter((key) => tried.includes(key))
|
|
2671
|
+
// 📖 Keep the v1 client-facing error codes: quota exhaustion is
|
|
2672
|
+
// 'insufficient_quota' (OpenAI convention), auth is 'invalid_api_key'.
|
|
2673
|
+
const clientCode = allSame
|
|
2674
|
+
? (kinds[0] === FAILURE_KINDS.RATE_LIMIT || kinds[0] === FAILURE_KINDS.QUOTA
|
|
2675
|
+
? 'insufficient_quota'
|
|
2676
|
+
: kinds[0] === FAILURE_KINDS.AUTH ? 'invalid_api_key' : kinds[0])
|
|
2677
|
+
: 'all_models_failed'
|
|
2678
|
+
this.sendProtocolError(res, protocol, statusCode,
|
|
2679
|
+
`All routed models failed for set: ${set.name}`, requestId,
|
|
2680
|
+
{
|
|
2681
|
+
code: clientCode,
|
|
2682
|
+
headers,
|
|
2683
|
+
payload: {
|
|
2684
|
+
set: set.name,
|
|
2685
|
+
models_tried: tried,
|
|
2686
|
+
failure_kinds: kinds,
|
|
2687
|
+
quota_exhausted: quotaExhausted,
|
|
2688
|
+
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2689
|
+
stream,
|
|
2690
|
+
},
|
|
2691
|
+
})
|
|
2692
|
+
}
|
|
2693
|
+
|
|
2694
|
+
async proxyJsonRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, isLastResort = false }) {
|
|
2276
2695
|
const key = candidate.key
|
|
2277
2696
|
const apiKey = this.getApiKeyForProvider(candidate.provider)
|
|
2278
|
-
// 📖 Guard: bail early if provider URL cannot be resolved
|
|
2279
2697
|
const providerUrl = resolveProviderUrl(candidate.provider)
|
|
2280
2698
|
if (!providerUrl) {
|
|
2281
|
-
|
|
2699
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.PROVIDER_URL })
|
|
2700
|
+
this.applyFailureVerdict(key, verdict, { detail: 'provider URL unresolvable' })
|
|
2282
2701
|
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'provider_url_unresolvable' })
|
|
2283
|
-
return { done: false, failoverToNext: true, reason:
|
|
2702
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2284
2703
|
}
|
|
2285
2704
|
const controller = new AbortController()
|
|
2286
2705
|
const timeout = setTimeout(() => controller.abort(), this.routerConfig().failover.requestTimeoutMs)
|
|
2706
|
+
const settings = this.failoverSettings()
|
|
2287
2707
|
const started = performance.now()
|
|
2288
|
-
|
|
2289
|
-
// 📖 curl, custom Playground) gets the FCM persona without any client
|
|
2290
|
-
// 📖 change. Non-streaming path.
|
|
2291
|
-
const bodyWithPrePrompt = applyPrePromptToBody(body, this.routerConfig().prePrompt)
|
|
2292
|
-
// 📖 Apply per-provider schema normalization (GLM, Mistral, Codestral).
|
|
2293
|
-
// 📖 Returns the body unchanged for providers without a registered normalizer.
|
|
2294
|
-
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2295
|
-
const upstreamBody = {
|
|
2296
|
-
...bodyNormalized,
|
|
2297
|
-
model: getApiModelId(candidate.provider, candidate.model),
|
|
2298
|
-
stream: false,
|
|
2299
|
-
}
|
|
2300
|
-
// 📖 Some providers/models fail if we send custom internal params, so strip them
|
|
2301
|
-
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2302
|
-
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2303
|
-
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2304
|
-
|
|
2708
|
+
const upstreamBody = this.buildUpstreamBody(body, candidate, false)
|
|
2305
2709
|
const clientAbort = attachClientAbort(req, res, controller)
|
|
2306
2710
|
try {
|
|
2307
2711
|
const response = await fetch(providerUrl, {
|
|
@@ -2315,46 +2719,52 @@ class RouterRuntime {
|
|
|
2315
2719
|
})
|
|
2316
2720
|
clearTimeout(timeout)
|
|
2317
2721
|
const latencyMs = Math.round(performance.now() - started)
|
|
2318
|
-
const text = await response.
|
|
2722
|
+
const text = await readBodyWithTimeout(response, controller, settings.bodyReadTimeoutMs)
|
|
2319
2723
|
const upstreamMeta = buildUpstreamMeta(response, text, candidate.provider)
|
|
2320
2724
|
|
|
2321
2725
|
if (isLikelyHtmlResponse(response.headers, text)) {
|
|
2322
|
-
|
|
2323
|
-
this.
|
|
2726
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2727
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2728
|
+
this.recordRouterError('upstream_html_maintenance', requestId, { model: key })
|
|
2324
2729
|
this.addRequestLog({ request_id: requestId, model: key, status: 503, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_html_maintenance' })
|
|
2325
|
-
return { done: false, failoverToNext: true, reason:
|
|
2730
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2326
2731
|
}
|
|
2327
2732
|
|
|
2328
2733
|
if (response.ok) {
|
|
2329
2734
|
const parsed = parseJsonResult(text)
|
|
2330
2735
|
if (!parsed.ok || !parsed.value || typeof parsed.value !== 'object') {
|
|
2331
|
-
|
|
2332
|
-
this.
|
|
2736
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.INVALID_JSON })
|
|
2737
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream invalid json', statusCode: 502, meta: upstreamMeta })
|
|
2738
|
+
this.recordRouterError('upstream_invalid_json', requestId, { model: key })
|
|
2333
2739
|
this.addRequestLog({ request_id: requestId, model: key, status: 502, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_invalid_json' })
|
|
2334
|
-
return { done: false, failoverToNext: true, reason:
|
|
2740
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 502, latencyMs }
|
|
2741
|
+
}
|
|
2742
|
+
// 📖 THE v2 content gate: a 200 only counts as success when the
|
|
2743
|
+
// payload holds real content. Empty choices, embedded error objects
|
|
2744
|
+
// and content-less answers fail over (v1 served those as successes).
|
|
2745
|
+
if (settings.contentValidation !== 'off') {
|
|
2746
|
+
const gate = validateChatCompletionPayload(parsed.value, { mode: settings.contentValidation })
|
|
2747
|
+
if (!gate.ok) {
|
|
2748
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS[gateReasonToKind(gate.reason)] || FAILURE_KINDS.INVALID_JSON })
|
|
2749
|
+
this.applyFailureVerdict(key, verdict, { detail: `gate: ${gate.reason}${gate.detail ? ` (${gate.detail})` : ''}`, statusCode: 200, meta: upstreamMeta })
|
|
2750
|
+
this.recordRouterError('gate_reject', requestId, { model: key, reason: gate.reason })
|
|
2751
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 200, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `gate_${gate.reason}` })
|
|
2752
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2753
|
+
}
|
|
2335
2754
|
}
|
|
2755
|
+
|
|
2336
2756
|
this.markSuccess(key, latencyMs)
|
|
2337
2757
|
const usage = extractUsage(parsed.value)
|
|
2338
2758
|
this.tokenTracker.record(candidate.provider, candidate.model, usage)
|
|
2339
|
-
|
|
2340
|
-
// 📖 real-world score can rank models by what *actually* works.
|
|
2341
|
-
this.recordRuntimeCall({
|
|
2342
|
-
providerKey: candidate.provider,
|
|
2343
|
-
modelId: candidate.model,
|
|
2344
|
-
success: true,
|
|
2345
|
-
latencyMs,
|
|
2346
|
-
usage,
|
|
2347
|
-
})
|
|
2759
|
+
this.recordRuntimeCall({ providerKey: candidate.provider, modelId: candidate.model, success: true, latencyMs, usage })
|
|
2348
2760
|
this.totalRequestsRouted += 1
|
|
2349
|
-
|
|
2761
|
+
trace.served_model = key
|
|
2762
|
+
trace.tokens = usage?.total_tokens || 0
|
|
2350
2763
|
if (this.totalRequestsRouted % 10 === 0) {
|
|
2351
2764
|
void sendUsageTelemetry(this.config, {}, {
|
|
2352
2765
|
event: 'app_router_use',
|
|
2353
2766
|
mode: 'daemon',
|
|
2354
|
-
properties: {
|
|
2355
|
-
total_requests: this.totalRequestsRouted,
|
|
2356
|
-
active_set: this.routerConfig().activeSet,
|
|
2357
|
-
},
|
|
2767
|
+
properties: { total_requests: this.totalRequestsRouted, active_set: this.routerConfig().activeSet },
|
|
2358
2768
|
})
|
|
2359
2769
|
}
|
|
2360
2770
|
this.addRequestLog({
|
|
@@ -2366,121 +2776,110 @@ class RouterRuntime {
|
|
|
2366
2776
|
failover: attemptIndex > 0,
|
|
2367
2777
|
})
|
|
2368
2778
|
this.logger.info(`Routed to ${key} - ${latencyMs}ms`, { request_id: requestId, status: response.status })
|
|
2369
|
-
// 📖
|
|
2779
|
+
// 📖 Record the winning attempt BEFORE the response head is written so
|
|
2780
|
+
// the decision header shows the final status of this model.
|
|
2781
|
+
const winningAttempt = trace.attempts[trace.attempts.length - 1]
|
|
2782
|
+
if (winningAttempt) {
|
|
2783
|
+
winningAttempt.status = response.status
|
|
2784
|
+
winningAttempt.latency_ms = latencyMs
|
|
2785
|
+
}
|
|
2786
|
+
// 📖 Fix #124: normalize malformed tool_calls (finish_reason
|
|
2787
|
+
// tool_calls without a tool_calls array).
|
|
2370
2788
|
let responseText = text
|
|
2371
2789
|
try {
|
|
2372
|
-
if (
|
|
2790
|
+
if (protocol === 'anthropic') {
|
|
2791
|
+
const translated = translateOpenAIToAnthropicResponse(parsed.value, { model: key })
|
|
2792
|
+
responseText = translated.ok ? JSON.stringify(translated.body) : text
|
|
2793
|
+
} else if (normalizeToolCallsResponse(parsed.value)) {
|
|
2373
2794
|
responseText = JSON.stringify(parsed.value)
|
|
2374
2795
|
}
|
|
2375
2796
|
} catch {}
|
|
2376
2797
|
if (!res.writableEnded) {
|
|
2377
2798
|
res.writeHead(response.status, {
|
|
2378
2799
|
...headerEntries(response.headers),
|
|
2379
|
-
|
|
2380
|
-
'x-
|
|
2800
|
+
...this.decisionHeaders(trace),
|
|
2801
|
+
...(isLastResort ? { 'x-fcm-v2-last-resort': 'true' } : {}),
|
|
2381
2802
|
})
|
|
2382
2803
|
res.end(responseText)
|
|
2383
2804
|
}
|
|
2384
|
-
return { done: true }
|
|
2385
|
-
}
|
|
2386
|
-
|
|
2387
|
-
if (AUTH_STATUS_CODES.has(response.status)) {
|
|
2388
|
-
this.markAuthError(key, `HTTP ${response.status}`)
|
|
2389
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'auth_error' })
|
|
2390
|
-
this.recordRuntimeCall({
|
|
2391
|
-
providerKey: candidate.provider, modelId: candidate.model,
|
|
2392
|
-
success: false, latencyMs, error: `auth_${response.status}`,
|
|
2393
|
-
})
|
|
2394
|
-
return { done: false, failoverToNext: true, reason: `auth_${response.status}`, authFailure: true }
|
|
2395
|
-
}
|
|
2396
|
-
|
|
2397
|
-
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
2398
|
-
this.markFailure(key, `HTTP ${response.status}`, response.status, upstreamMeta)
|
|
2399
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}` })
|
|
2400
|
-
this.recordRuntimeCall({
|
|
2401
|
-
providerKey: candidate.provider, modelId: candidate.model,
|
|
2402
|
-
success: false, latencyMs, error: `http_${response.status}`,
|
|
2403
|
-
})
|
|
2404
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2805
|
+
return { done: true, status: response.status, latencyMs }
|
|
2405
2806
|
}
|
|
2406
2807
|
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
}
|
|
2421
|
-
|
|
2422
|
-
if (!res.writableEnded) {
|
|
2423
|
-
res.writeHead(response.status, {
|
|
2424
|
-
...headerEntries(response.headers),
|
|
2425
|
-
'x-fcm-router-model': key,
|
|
2426
|
-
'x-request-id': requestId,
|
|
2427
|
-
})
|
|
2428
|
-
res.end(text)
|
|
2429
|
-
}
|
|
2430
|
-
return { done: true }
|
|
2808
|
+
const verdict = classifyFailure({ status: response.status, retryAfterMs: upstreamMeta.retryAfterMs })
|
|
2809
|
+
this.applyFailureVerdict(key, verdict, {
|
|
2810
|
+
detail: `HTTP ${response.status}`,
|
|
2811
|
+
statusCode: response.status,
|
|
2812
|
+
meta: upstreamMeta,
|
|
2813
|
+
})
|
|
2814
|
+
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: verdict.kind })
|
|
2815
|
+
this.recordRuntimeCall({
|
|
2816
|
+
providerKey: candidate.provider, modelId: candidate.model,
|
|
2817
|
+
success: false, latencyMs, error: verdict.kind,
|
|
2818
|
+
})
|
|
2819
|
+
this.recordRouterError(verdict.kind, requestId, { model: key, status: response.status })
|
|
2820
|
+
return { done: false, failoverToNext: verdict.failover, reason: verdict.kind, verdict, status: response.status, latencyMs }
|
|
2431
2821
|
} catch (error) {
|
|
2822
|
+
// 📖 Blame attribution (v2): a client disconnect is never an upstream
|
|
2823
|
+
// failure and must not damage the model's health.
|
|
2432
2824
|
if (clientAbort.aborted) {
|
|
2433
2825
|
this.logger.info(`Client disconnected before upstream response from ${key}`, { request_id: requestId })
|
|
2434
|
-
|
|
2826
|
+
trace.outcome = 'client_aborted'
|
|
2827
|
+
return { done: true, reason: 'client_aborted' }
|
|
2435
2828
|
}
|
|
2436
|
-
const
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2829
|
+
const isBodyReadTimeout = error?.name === 'BodyReadTimeoutError'
|
|
2830
|
+
const verdict = isBodyReadTimeout || error.name === 'AbortError'
|
|
2831
|
+
? classifyFailure({ kind: FAILURE_KINDS.TIMEOUT })
|
|
2832
|
+
: classifyFailure({ kind: FAILURE_KINDS.NETWORK })
|
|
2833
|
+
const detail = isBodyReadTimeout ? 'body read timeout' : (error.name === 'AbortError' ? 'timeout' : (error.message || String(error)))
|
|
2834
|
+
this.applyFailureVerdict(key, verdict, { detail })
|
|
2835
|
+
this.recordRouterError('upstream_transport_error', requestId, { model: key, reason: detail })
|
|
2836
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: detail })
|
|
2837
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2441
2838
|
} finally {
|
|
2442
2839
|
clearTimeout(timeout)
|
|
2443
2840
|
clientAbort.dispose()
|
|
2444
2841
|
}
|
|
2445
2842
|
}
|
|
2446
2843
|
|
|
2447
|
-
async proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex }) {
|
|
2844
|
+
async proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, isLastResort = false, anthropicModelName = null }) {
|
|
2448
2845
|
const key = candidate.key
|
|
2449
2846
|
const activeReq = this.activeRequests.get(requestId)
|
|
2450
2847
|
if (activeReq) {
|
|
2451
2848
|
activeReq.current_model = key
|
|
2452
|
-
|
|
2849
|
+
activeReq.last_activity_at = Date.now()
|
|
2453
2850
|
}
|
|
2454
2851
|
const apiKey = this.getApiKeyForProvider(candidate.provider)
|
|
2455
|
-
// 📖 Guard: bail early if provider URL cannot be resolved
|
|
2456
2852
|
const providerUrl = resolveProviderUrl(candidate.provider)
|
|
2457
2853
|
if (!providerUrl) {
|
|
2458
|
-
|
|
2854
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.PROVIDER_URL })
|
|
2855
|
+
this.applyFailureVerdict(key, verdict, { detail: 'provider URL unresolvable' })
|
|
2459
2856
|
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'provider_url_unresolvable', stream: true })
|
|
2460
|
-
return { done: false, failoverToNext: true, reason:
|
|
2857
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2461
2858
|
}
|
|
2462
2859
|
const controller = new AbortController()
|
|
2463
2860
|
const started = performance.now()
|
|
2464
|
-
|
|
2465
|
-
// 📖
|
|
2466
|
-
//
|
|
2467
|
-
const
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2471
|
-
const upstreamBody = {
|
|
2472
|
-
...bodyNormalized,
|
|
2473
|
-
model: getApiModelId(candidate.provider, candidate.model),
|
|
2474
|
-
stream: true,
|
|
2475
|
-
}
|
|
2476
|
-
// 📖 Some providers/models fail if we send custom internal params, so strip them
|
|
2477
|
-
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2478
|
-
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2479
|
-
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2480
|
-
|
|
2861
|
+
const upstreamBody = this.buildUpstreamBody(body, candidate, true)
|
|
2862
|
+
// 📖 Anthropic clients receive Anthropic SSE events: every byte written
|
|
2863
|
+
// to the client goes through the transformer sink instead of raw.
|
|
2864
|
+
const sink = protocol === 'anthropic'
|
|
2865
|
+
? createAnthropicStreamTransformer({ model: anthropicModelName || key })
|
|
2866
|
+
: null
|
|
2481
2867
|
const timeout = setTimeout(() => controller.abort(), this.routerConfig().failover.requestTimeoutMs)
|
|
2482
2868
|
let sentToClient = false
|
|
2483
2869
|
const clientAbort = attachClientAbort(req, res, controller)
|
|
2870
|
+
|
|
2871
|
+
const writeToClient = (text) => {
|
|
2872
|
+
if (res.writableEnded) return
|
|
2873
|
+
if (sink) res.write(sink.write(text))
|
|
2874
|
+
else res.write(Buffer.isBuffer(text) ? text : Buffer.from(text))
|
|
2875
|
+
}
|
|
2876
|
+
const endClientStream = () => {
|
|
2877
|
+
try {
|
|
2878
|
+
if (sink && !res.writableEnded) res.write(sink.end())
|
|
2879
|
+
} catch {}
|
|
2880
|
+
try { if (!res.writableEnded) res.end() } catch {}
|
|
2881
|
+
}
|
|
2882
|
+
|
|
2484
2883
|
try {
|
|
2485
2884
|
const response = await fetch(providerUrl, {
|
|
2486
2885
|
method: 'POST',
|
|
@@ -2495,164 +2894,205 @@ class RouterRuntime {
|
|
|
2495
2894
|
const latencyMs = Math.round(performance.now() - started)
|
|
2496
2895
|
const upstreamMeta = buildUpstreamMeta(response, '', candidate.provider)
|
|
2497
2896
|
if (isLikelyHtmlResponse(response.headers)) {
|
|
2498
|
-
|
|
2499
|
-
this.
|
|
2897
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2898
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2899
|
+
this.recordRouterError('upstream_html_maintenance', requestId, { model: key, stream: true })
|
|
2500
2900
|
this.addRequestLog({ request_id: requestId, model: key, status: 503, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_html_maintenance', stream: true })
|
|
2501
|
-
return { done: false, failoverToNext: true, reason:
|
|
2901
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2502
2902
|
}
|
|
2503
2903
|
if (!response.ok) {
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
}
|
|
2509
|
-
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
2510
|
-
this.markFailure(key, `HTTP ${response.status}`, response.status, upstreamMeta)
|
|
2511
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}`, stream: true })
|
|
2512
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2513
|
-
}
|
|
2514
|
-
|
|
2515
|
-
// 📖 Provide failover fallback for non-retryable errors from the provider (like 400 Bad Request)
|
|
2516
|
-
// when they are caused by format idiosyncrasies (e.g. empty tools array that another model might accept)
|
|
2517
|
-
if (response.status >= 400 && response.status < 500) {
|
|
2518
|
-
// 📖 Telemetry keeps structural fields only: upstream response bodies
|
|
2519
|
-
// 📖 must never leave the machine (they can embed user code/prompts).
|
|
2520
|
-
this.recordRouterError(`http_${response.status}`, requestId, { provider: candidate.provider, model: key, status: response.status, stream: true })
|
|
2521
|
-
this.markFailure(key, `HTTP ${response.status}`)
|
|
2522
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}`, stream: true })
|
|
2523
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2524
|
-
}
|
|
2525
|
-
|
|
2526
|
-
if (!res.writableEnded) {
|
|
2527
|
-
res.writeHead(response.status, {
|
|
2528
|
-
...headerEntries(response.headers),
|
|
2529
|
-
'x-fcm-router-model': key,
|
|
2530
|
-
'x-request-id': requestId,
|
|
2531
|
-
})
|
|
2532
|
-
try { res.end(await response.text()) } catch {}
|
|
2533
|
-
}
|
|
2534
|
-
return { done: true }
|
|
2904
|
+
const verdict = classifyFailure({ status: response.status, retryAfterMs: upstreamMeta.retryAfterMs })
|
|
2905
|
+
this.applyFailureVerdict(key, verdict, { detail: `HTTP ${response.status}`, statusCode: response.status, meta: upstreamMeta })
|
|
2906
|
+
this.recordRouterError(verdict.kind, requestId, { model: key, status: response.status, stream: true })
|
|
2907
|
+
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: verdict.kind, stream: true })
|
|
2908
|
+
return { done: false, failoverToNext: verdict.failover, reason: verdict.kind, verdict, status: response.status, latencyMs }
|
|
2535
2909
|
}
|
|
2536
2910
|
|
|
2537
2911
|
const reader = response.body?.getReader()
|
|
2538
2912
|
if (!reader) {
|
|
2539
|
-
|
|
2540
|
-
|
|
2913
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.EMPTY_STREAM })
|
|
2914
|
+
this.applyFailureVerdict(key, verdict, { detail: 'empty stream' })
|
|
2915
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'empty_stream', stream: true })
|
|
2916
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2541
2917
|
}
|
|
2542
2918
|
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2919
|
+
// 📖 v2 readiness gate: hold early chunks until the tracker sees useful
|
|
2920
|
+
// content, an upstream error frame (fail over BEFORE the client gets
|
|
2921
|
+
// bytes), or the hold cap overflows (weird provider: pass through).
|
|
2922
|
+
const tracker = createStreamReadinessTracker()
|
|
2923
|
+
const holdBuffer = []
|
|
2924
|
+
let forwarded = false
|
|
2925
|
+
let forwardedChars = 0
|
|
2926
|
+
let upstreamErrorAfterForward = false
|
|
2927
|
+
|
|
2928
|
+
const flushHold = () => {
|
|
2929
|
+
// 📖 Record this attempt as the serving one before the head is
|
|
2930
|
+
// written so the decision header reflects the streaming model.
|
|
2931
|
+
const winningAttempt = trace.attempts[trace.attempts.length - 1]
|
|
2932
|
+
if (winningAttempt) {
|
|
2933
|
+
winningAttempt.status = 200
|
|
2934
|
+
winningAttempt.latency_ms = latencyMs
|
|
2935
|
+
}
|
|
2936
|
+
// 📖 Issue #137: when a previous model already sent partial data the
|
|
2937
|
+
// headers are on the wire; append with an SSE comment marker instead.
|
|
2938
|
+
if (!res.headersSent) {
|
|
2939
|
+
res.writeHead(200, {
|
|
2940
|
+
...headerEntries(response.headers),
|
|
2941
|
+
// 📖 A forwarded stream is SSE by definition: override whatever
|
|
2942
|
+
// content-type the upstream declared (json/html/...).
|
|
2943
|
+
'Content-Type': 'text/event-stream',
|
|
2944
|
+
'Cache-Control': 'no-cache',
|
|
2945
|
+
Connection: 'keep-alive',
|
|
2946
|
+
...this.decisionHeaders(trace),
|
|
2947
|
+
...(isLastResort ? { 'x-fcm-v2-last-resort': 'true' } : {}),
|
|
2948
|
+
})
|
|
2949
|
+
} else {
|
|
2950
|
+
try { res.write(`: fcm-router-failover-from=${key}\n\n`) } catch {}
|
|
2951
|
+
}
|
|
2952
|
+
for (const text of holdBuffer) {
|
|
2953
|
+
writeToClient(text)
|
|
2954
|
+
forwardedChars += text.length
|
|
2955
|
+
}
|
|
2956
|
+
holdBuffer.length = 0
|
|
2957
|
+
sentToClient = true
|
|
2958
|
+
forwarded = true
|
|
2573
2959
|
}
|
|
2574
|
-
sentToClient = true
|
|
2575
|
-
res.write(firstChunkBuffer)
|
|
2576
2960
|
|
|
2577
|
-
while (
|
|
2961
|
+
while (true) {
|
|
2578
2962
|
const chunk = await this.readStreamChunkWithTimeout(reader)
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2963
|
+
const text = chunk.done || !chunk.value
|
|
2964
|
+
? null
|
|
2965
|
+
: (Buffer.isBuffer(chunk.value) ? chunk.value.toString('utf8') : Buffer.from(chunk.value).toString('utf8'))
|
|
2966
|
+
if (text === null) break
|
|
2967
|
+
tracker.observe(text)
|
|
2583
2968
|
if (activeReq) {
|
|
2584
|
-
|
|
2585
|
-
activeReq.tokens += 1
|
|
2969
|
+
activeReq.last_activity_at = Date.now()
|
|
2970
|
+
activeReq.tokens += 1
|
|
2586
2971
|
}
|
|
2972
|
+
if (!forwarded) {
|
|
2973
|
+
if (tracker.errorPayload) {
|
|
2974
|
+
try { controller.abort() } catch {}
|
|
2975
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.ERROR_PAYLOAD })
|
|
2976
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream error payload before content', statusCode: 200, meta: upstreamMeta })
|
|
2977
|
+
this.recordRouterError('gate_reject', requestId, { model: key, stream: true, reason: 'error_payload' })
|
|
2978
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2979
|
+
}
|
|
2980
|
+
if (tracker.useful) {
|
|
2981
|
+
holdBuffer.push(text)
|
|
2982
|
+
flushHold()
|
|
2983
|
+
} else if (tracker.bytesSeen > tracker.maxHoldBytes) {
|
|
2984
|
+
// 📖 Huge non-JSON preamble: pass it through rather than stall.
|
|
2985
|
+
holdBuffer.push(text)
|
|
2986
|
+
flushHold()
|
|
2987
|
+
} else if (isLikelyHtmlText(text)) {
|
|
2988
|
+
try { controller.abort() } catch {}
|
|
2989
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2990
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2991
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2992
|
+
} else {
|
|
2993
|
+
holdBuffer.push(text)
|
|
2994
|
+
}
|
|
2995
|
+
} else {
|
|
2996
|
+
writeToClient(text)
|
|
2997
|
+
forwardedChars += text.length
|
|
2998
|
+
if (tracker.errorPayload) {
|
|
2999
|
+
// 📖 Upstream errored AFTER real content: keep the partial output,
|
|
3000
|
+
// close cleanly, and record a real failure (v1 marked success).
|
|
3001
|
+
upstreamErrorAfterForward = true
|
|
3002
|
+
break
|
|
3003
|
+
}
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
|
|
3007
|
+
if (!forwarded) {
|
|
3008
|
+
// 📖 The stream closed without ever producing useful content. v1 only
|
|
3009
|
+
// caught the zero-chunk case; the gate also fails over a stream that
|
|
3010
|
+
// sent only framing garbage. Nothing reached the client, so failover
|
|
3011
|
+
// is safe.
|
|
3012
|
+
try { controller.abort() } catch {}
|
|
3013
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.EMPTY_STREAM })
|
|
3014
|
+
this.applyFailureVerdict(key, verdict, { detail: `stream closed without content (${tracker.describe()})`, statusCode: 200, meta: upstreamMeta })
|
|
3015
|
+
this.recordRouterError('gate_reject', requestId, { model: key, stream: true, reason: 'empty_stream' })
|
|
3016
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
3017
|
+
}
|
|
3018
|
+
|
|
3019
|
+
if (upstreamErrorAfterForward) {
|
|
3020
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.ERROR_PAYLOAD })
|
|
3021
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream error payload after content', statusCode: 200, meta: upstreamMeta })
|
|
3022
|
+
endClientStream()
|
|
3023
|
+
return { done: true, status: 200, latencyMs }
|
|
2587
3024
|
}
|
|
2588
3025
|
|
|
2589
3026
|
this.markSuccess(key, latencyMs)
|
|
3027
|
+
const completionTokens = estimateTokens(forwardedChars)
|
|
3028
|
+
this.tokenTracker.record(candidate.provider, candidate.model, {
|
|
3029
|
+
prompt_tokens: 0,
|
|
3030
|
+
completion_tokens: completionTokens,
|
|
3031
|
+
total_tokens: completionTokens,
|
|
3032
|
+
})
|
|
3033
|
+
this.recordRuntimeCall({
|
|
3034
|
+
providerKey: candidate.provider, modelId: candidate.model,
|
|
3035
|
+
success: true, latencyMs,
|
|
3036
|
+
usage: { prompt_tokens: 0, completion_tokens: completionTokens, total_tokens: completionTokens },
|
|
3037
|
+
})
|
|
2590
3038
|
this.totalRequestsRouted += 1
|
|
3039
|
+
trace.served_model = key
|
|
3040
|
+
trace.tokens = completionTokens
|
|
2591
3041
|
this.addRequestLog({
|
|
2592
3042
|
request_id: requestId,
|
|
2593
3043
|
model: key,
|
|
2594
|
-
status:
|
|
3044
|
+
status: 200,
|
|
2595
3045
|
latency_ms: latencyMs,
|
|
2596
|
-
tokens:
|
|
3046
|
+
tokens: completionTokens,
|
|
2597
3047
|
failover: attemptIndex > 0,
|
|
2598
3048
|
stream: true,
|
|
2599
3049
|
})
|
|
2600
|
-
|
|
2601
|
-
return { done: true }
|
|
3050
|
+
endClientStream()
|
|
3051
|
+
return { done: true, status: 200, latencyMs }
|
|
2602
3052
|
} catch (error) {
|
|
2603
3053
|
try { controller.abort() } catch {}
|
|
2604
3054
|
if (clientAbort.aborted) {
|
|
2605
3055
|
this.logger.info(`Client disconnected during streaming response from ${key}`, { request_id: requestId })
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
// 📖 Issue #137: stream-stall timeouts get a special tag so we can
|
|
2610
|
-
// 📖 distinguish them from generic upstream errors below. Only stalls
|
|
2611
|
-
// 📖 should trigger failover after a partial response — generic errors
|
|
2612
|
-
// 📖 (malformed JSON, network reset, etc.) usually mean the partial
|
|
2613
|
-
// 📖 data is invalid anyway, so closing cleanly is safer.
|
|
2614
|
-
const isStall = reason === 'stream_stall_timeout' || reason === 'timeout'
|
|
2615
|
-
this.markFailure(key, reason)
|
|
2616
|
-
if (reason !== 'timeout') {
|
|
2617
|
-
this.recordRouterError('upstream_stream_error', requestId, { model: key, reason, partial: sentToClient })
|
|
2618
|
-
} else {
|
|
2619
|
-
this.recordRouterError('timeout', requestId, { model: key, reason, partial: sentToClient })
|
|
3056
|
+
trace.outcome = 'client_aborted'
|
|
3057
|
+
endClientStream()
|
|
3058
|
+
return { done: true, reason: 'client_aborted' }
|
|
2620
3059
|
}
|
|
2621
|
-
|
|
3060
|
+
const isStall = error?.message === 'stream_stall_timeout' || error.name === 'AbortError'
|
|
3061
|
+
const kind = error?.message === 'stream_stall_timeout'
|
|
3062
|
+
? FAILURE_KINDS.STREAM_STALL
|
|
3063
|
+
: (error.name === 'AbortError' ? FAILURE_KINDS.TIMEOUT : FAILURE_KINDS.NETWORK)
|
|
3064
|
+
const detail = error?.message === 'stream_stall_timeout'
|
|
3065
|
+
? 'stream stall timeout'
|
|
3066
|
+
: (error.name === 'AbortError' ? 'timeout' : (error.message || String(error)))
|
|
3067
|
+
const verdict = classifyFailure({ kind })
|
|
3068
|
+
this.applyFailureVerdict(key, verdict, { detail })
|
|
3069
|
+
this.recordRouterError(isStall ? 'timeout' : 'upstream_stream_error', requestId, { model: key, reason: detail, partial: sentToClient, stream: true })
|
|
3070
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: detail, stream: true })
|
|
2622
3071
|
if (sentToClient) {
|
|
2623
3072
|
if (isStall) {
|
|
2624
|
-
// 📖 Issue #137:
|
|
2625
|
-
//
|
|
2626
|
-
//
|
|
2627
|
-
|
|
2628
|
-
// 📖 upstream connection; its chunks are appended to the same
|
|
2629
|
-
// 📖 response object so the client sees one continuous stream.
|
|
2630
|
-
this.logger.warn(`Stream stall after partial response from ${key}, attempting failover`, { request_id: requestId, reason })
|
|
3073
|
+
// 📖 Issue #137: fail over even after partial output. OpenAI
|
|
3074
|
+
// clients get a synthetic caution delta; Anthropic clients get an
|
|
3075
|
+
// SSE error event before the stream closes.
|
|
3076
|
+
this.logger.warn(`Stream stall after partial response from ${key}, attempting failover`, { request_id: requestId, reason: detail })
|
|
2631
3077
|
if (!res.writableEnded) {
|
|
2632
3078
|
try {
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
index: 0,
|
|
2640
|
-
delta: { content: failoverMsg },
|
|
2641
|
-
finish_reason: null,
|
|
2642
|
-
}],
|
|
2643
|
-
})
|
|
2644
|
-
res.write(`data: ${deltaPayload}\n\n`)
|
|
3079
|
+
if (sink) {
|
|
3080
|
+
res.write(sink.write(`data: ${JSON.stringify({ error: { message: `stream truncated by router (${detail}); failing over to next model`, type: 'api_error' } })}\n\n`))
|
|
3081
|
+
} else {
|
|
3082
|
+
const failoverMsg = `\n\n> [!CAUTION]\n> Stream truncated by router due to upstream ${detail}; failing over to next model.\n\n`
|
|
3083
|
+
res.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: failoverMsg }, finish_reason: null }] })}\n\n`)
|
|
3084
|
+
}
|
|
2645
3085
|
} catch { /* best-effort */ }
|
|
2646
3086
|
}
|
|
2647
|
-
return { done: false, failoverToNext: true, reason: `
|
|
3087
|
+
return { done: false, failoverToNext: true, reason: `stream_stall`, verdict }
|
|
2648
3088
|
}
|
|
2649
|
-
// 📖 Non-stall errors after partial output:
|
|
2650
|
-
//
|
|
2651
|
-
this.logger.warn(`Streaming failure after partial response from ${key}`, { request_id: requestId, reason })
|
|
2652
|
-
|
|
3089
|
+
// 📖 Non-stall errors after partial output: close cleanly, no
|
|
3090
|
+
// failover, to avoid sending malformed data (v1 behavior kept).
|
|
3091
|
+
this.logger.warn(`Streaming failure after partial response from ${key}`, { request_id: requestId, reason: detail })
|
|
3092
|
+
endClientStream()
|
|
2653
3093
|
return { done: true }
|
|
2654
3094
|
}
|
|
2655
|
-
return { done: false, failoverToNext: true, reason }
|
|
3095
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2656
3096
|
} finally {
|
|
2657
3097
|
clearTimeout(timeout)
|
|
2658
3098
|
clientAbort.dispose()
|
|
@@ -2677,6 +3117,41 @@ class RouterRuntime {
|
|
|
2677
3117
|
])
|
|
2678
3118
|
}
|
|
2679
3119
|
|
|
3120
|
+
// ─── Anthropic /v1/messages (v2 protocol support) ─────────────────────────
|
|
3121
|
+
|
|
3122
|
+
async handleAnthropicMessages(req, res, requestId) {
|
|
3123
|
+
if (!isAuthorizedForV1(req)) {
|
|
3124
|
+
sendJson(res, 401, anthropicErrorPayload('authentication_error', 'Missing or invalid router token'), { 'x-request-id': requestId })
|
|
3125
|
+
return
|
|
3126
|
+
}
|
|
3127
|
+
let body
|
|
3128
|
+
try {
|
|
3129
|
+
body = await readJsonBody(req)
|
|
3130
|
+
} catch (error) {
|
|
3131
|
+
if (error.code === 'BODY_TOO_LARGE') {
|
|
3132
|
+
sendJson(res, 413, anthropicErrorPayload('request_too_large', 'Request body too large'), { 'x-request-id': requestId })
|
|
3133
|
+
return
|
|
3134
|
+
}
|
|
3135
|
+
sendJson(res, 400, anthropicErrorPayload('invalid_request_error', 'Invalid JSON body'), { 'x-request-id': requestId })
|
|
3136
|
+
return
|
|
3137
|
+
}
|
|
3138
|
+
const translated = translateAnthropicToOpenAI(body)
|
|
3139
|
+
if (!translated.ok) {
|
|
3140
|
+
sendJson(res, 400, anthropicErrorPayload('invalid_request_error', translated.error), { 'x-request-id': requestId })
|
|
3141
|
+
return
|
|
3142
|
+
}
|
|
3143
|
+
const openaiBody = { ...translated.body, stream: body.stream === true }
|
|
3144
|
+
await this.routeRequest({
|
|
3145
|
+
req,
|
|
3146
|
+
res,
|
|
3147
|
+
body: openaiBody,
|
|
3148
|
+
setName: null,
|
|
3149
|
+
requestId,
|
|
3150
|
+
protocol: 'anthropic',
|
|
3151
|
+
anthropicModelName: typeof body.model === 'string' ? body.model : null,
|
|
3152
|
+
})
|
|
3153
|
+
}
|
|
3154
|
+
|
|
2680
3155
|
async handleSetsRequest(req, res, url, requestId) {
|
|
2681
3156
|
// 📖 Hoisted same-origin guard: covers both the canonical /sets routes and
|
|
2682
3157
|
// 📖 the /api/router/sets alias so no set-mutating path skips the check.
|
|
@@ -3022,6 +3497,12 @@ class RouterRuntime {
|
|
|
3022
3497
|
const requestId = typeof rawRequestId === 'string' && rawRequestId.trim()
|
|
3023
3498
|
? rawRequestId.trim().slice(0, 64)
|
|
3024
3499
|
: `req-${randomUUID()}`
|
|
3500
|
+
applyCors(req, res)
|
|
3501
|
+
if (req.method === 'OPTIONS') {
|
|
3502
|
+
res.writeHead(204)
|
|
3503
|
+
res.end()
|
|
3504
|
+
return
|
|
3505
|
+
}
|
|
3025
3506
|
// 📖 DNS-rebinding guard: reject requests whose Host header is not the
|
|
3026
3507
|
// 📖 loopback (or the configured FCM_HOST) before any routing happens.
|
|
3027
3508
|
if (!isAllowedHostHeader(req.headers.host, this.port, this.boundHost)) {
|
|
@@ -3478,6 +3959,80 @@ class RouterRuntime {
|
|
|
3478
3959
|
serveWebStaticFile(res, url.pathname, requestId)
|
|
3479
3960
|
return
|
|
3480
3961
|
}
|
|
3962
|
+
// ─── Anthropic-compatible routing surface (v2) ──────────────────────
|
|
3963
|
+
if (url.pathname === '/v1/messages') {
|
|
3964
|
+
if (req.method !== 'POST') {
|
|
3965
|
+
sendJson(res, 405, anthropicErrorPayload('invalid_request_error', 'Method not allowed, use POST'), { 'x-request-id': requestId })
|
|
3966
|
+
return
|
|
3967
|
+
}
|
|
3968
|
+
await this.handleAnthropicMessages(req, res, requestId)
|
|
3969
|
+
return
|
|
3970
|
+
}
|
|
3971
|
+
|
|
3972
|
+
// ─── Router v2 dashboard API (beta overlays + web page) ──────────────
|
|
3973
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/status') {
|
|
3974
|
+
sendJson(res, 200, { ...this.statusPayload(), router: 'v2', beta: true }, { 'x-request-id': requestId })
|
|
3975
|
+
return
|
|
3976
|
+
}
|
|
3977
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/stats') {
|
|
3978
|
+
sendJson(res, 200, { ...this.statsPayload(), router: 'v2', beta: true }, { 'x-request-id': requestId })
|
|
3979
|
+
return
|
|
3980
|
+
}
|
|
3981
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/history') {
|
|
3982
|
+
const limitRaw = Number.parseInt(url.searchParams.get('limit') || '50', 10)
|
|
3983
|
+
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(1, limitRaw), 500) : 50
|
|
3984
|
+
sendJson(res, 200, { entries: this.history.recent(limit), stats: this.history.stats() }, { 'x-request-id': requestId })
|
|
3985
|
+
return
|
|
3986
|
+
}
|
|
3987
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/traces') {
|
|
3988
|
+
const limitRaw = Number.parseInt(url.searchParams.get('limit') || '20', 10)
|
|
3989
|
+
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(1, limitRaw), 50) : 20
|
|
3990
|
+
sendJson(res, 200, { traces: this.recentTraces.slice(-limit).reverse() }, { 'x-request-id': requestId })
|
|
3991
|
+
return
|
|
3992
|
+
}
|
|
3993
|
+
if (url.pathname === '/api/router-v2/history' && req.method === 'DELETE') {
|
|
3994
|
+
if (!isSameOriginOrLocal(req)) {
|
|
3995
|
+
sendError(res, 403, 'Forbidden cross-origin request', 'invalid_request_error', 'forbidden_origin', requestId)
|
|
3996
|
+
return
|
|
3997
|
+
}
|
|
3998
|
+
this.history.clear()
|
|
3999
|
+
this.recentTraces = []
|
|
4000
|
+
sendJson(res, 200, { ok: true }, { 'x-request-id': requestId })
|
|
4001
|
+
return
|
|
4002
|
+
}
|
|
4003
|
+
if (url.pathname === '/api/router-v2/test' && req.method === 'POST') {
|
|
4004
|
+
if (!isSameOriginOrLocal(req)) {
|
|
4005
|
+
sendError(res, 403, 'Forbidden cross-origin request', 'invalid_request_error', 'forbidden_origin', requestId)
|
|
4006
|
+
return
|
|
4007
|
+
}
|
|
4008
|
+
const body = await readJsonBody(req)
|
|
4009
|
+
const provider = typeof body.provider === 'string' ? body.provider.trim() : ''
|
|
4010
|
+
const model = typeof body.model === 'string' ? body.model.trim() : ''
|
|
4011
|
+
if (!provider || !model) {
|
|
4012
|
+
sendError(res, 400, 'Both `provider` and `model` are required', 'invalid_request_error', 'missing_model_fields', requestId)
|
|
4013
|
+
return
|
|
4014
|
+
}
|
|
4015
|
+
const { testModelViaRouter } = await import('./router-v2/bench.js')
|
|
4016
|
+
const result = await testModelViaRouter({ port: this.port, provider, model })
|
|
4017
|
+
sendJson(res, 200, result, { 'x-request-id': requestId })
|
|
4018
|
+
return
|
|
4019
|
+
}
|
|
4020
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/events') {
|
|
4021
|
+
if (!this.tryOpenSseConnection(req, res, requestId)) return
|
|
4022
|
+
res.writeHead(200, {
|
|
4023
|
+
'Content-Type': 'text/event-stream',
|
|
4024
|
+
'Cache-Control': 'no-cache',
|
|
4025
|
+
Connection: 'keep-alive',
|
|
4026
|
+
'x-request-id': requestId,
|
|
4027
|
+
})
|
|
4028
|
+
res.flushHeaders?.()
|
|
4029
|
+
res.write(': connected\n\n')
|
|
4030
|
+
res.write(`event: hello\ndata: ${JSON.stringify(this.statusPayload())}\n\n`)
|
|
4031
|
+
this.sseClients.add(res)
|
|
4032
|
+
req.on('close', () => this.sseClients.delete(res))
|
|
4033
|
+
return
|
|
4034
|
+
}
|
|
4035
|
+
|
|
3481
4036
|
if (url.pathname === '/v1/chat/completions' || url.pathname.match(/^\/v1\/sets\/[^/]+\/chat\/completions$/)) {
|
|
3482
4037
|
if (req.method !== 'POST') {
|
|
3483
4038
|
sendError(res, 405, 'Method not allowed', 'invalid_request_error', 'method_not_allowed', requestId, { allowed: ['POST'] })
|
|
@@ -3556,6 +4111,7 @@ class RouterRuntime {
|
|
|
3556
4111
|
this.shuttingDown = true
|
|
3557
4112
|
this.logger.info('Router daemon stopping')
|
|
3558
4113
|
if (this.probeTimer) clearInterval(this.probeTimer)
|
|
4114
|
+
if (this.probeWatchdog) clearInterval(this.probeWatchdog)
|
|
3559
4115
|
if (this.configReloadTimer) clearInterval(this.configReloadTimer)
|
|
3560
4116
|
if (this.tokenFlushTimer) clearInterval(this.tokenFlushTimer)
|
|
3561
4117
|
if (this.probeCacheFlushTimer) clearInterval(this.probeCacheFlushTimer)
|
|
@@ -3566,6 +4122,8 @@ class RouterRuntime {
|
|
|
3566
4122
|
await sleep(100)
|
|
3567
4123
|
}
|
|
3568
4124
|
this.tokenTracker.flush({ force: true })
|
|
4125
|
+
this.breakers.flush()
|
|
4126
|
+
this.history.flush()
|
|
3569
4127
|
flushProbeCache() // 📖 t1: persist any pending probe-cache deltas before exit
|
|
3570
4128
|
if (this.runtimeTelemetryDirty) flushRuntimeTelemetryStore() // 📖 t3
|
|
3571
4129
|
try { this.server?.close() } catch {}
|
|
@@ -3587,7 +4145,7 @@ class RouterRuntime {
|
|
|
3587
4145
|
// 📖 Pinned picks: only used as a *tie-breaker* when multiple models have
|
|
3588
4146
|
// 📖 identical (tier, sweScore, latency) - never a hard requirement, so
|
|
3589
4147
|
// 📖 a user whose NVIDIA key is dead still gets a working set.
|
|
3590
|
-
const PREFERRED_DEFAULT_MODELS = [
|
|
4148
|
+
export const PREFERRED_DEFAULT_MODELS = [
|
|
3591
4149
|
{ provider: 'groq', model: 'openai/gpt-oss-120b' },
|
|
3592
4150
|
{ provider: 'groq', model: 'qwen/qwen3.6-27b' },
|
|
3593
4151
|
{ provider: 'cerebras', model: 'llama3.1-70b' },
|
|
@@ -3755,7 +4313,7 @@ export async function buildDefaultRouterSet(config = {}, maxModels, options = {}
|
|
|
3755
4313
|
}
|
|
3756
4314
|
}
|
|
3757
4315
|
|
|
3758
|
-
export function createRouterRuntimeForTest({ config, port = 0, logger = null, tokenPath = ROUTER_TOKENS_PATH } = {}) {
|
|
4316
|
+
export function createRouterRuntimeForTest({ config, port = 0, logger = null, tokenPath = ROUTER_TOKENS_PATH, breakersPath = null, historyPath = null } = {}) {
|
|
3759
4317
|
const testLogger = logger || {
|
|
3760
4318
|
level: 'error',
|
|
3761
4319
|
error() {},
|
|
@@ -3767,12 +4325,19 @@ export function createRouterRuntimeForTest({ config, port = 0, logger = null, to
|
|
|
3767
4325
|
// 📖 fake providers without spawning a daemon or touching user token files.
|
|
3768
4326
|
// 📖 Router config persistence is disabled here so set/probe-mode endpoint
|
|
3769
4327
|
// 📖 tests cannot write fixture router sets into ~/.free-coding-models.json.
|
|
4328
|
+
// 📖 v2: breaker/history state also lands in unique tmp files, otherwise
|
|
4329
|
+
// 📖 tests would inherit the machine's real persisted breaker state.
|
|
4330
|
+
const testId = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`
|
|
3770
4331
|
return new RouterRuntime({
|
|
3771
4332
|
config: config || {},
|
|
3772
4333
|
port,
|
|
3773
4334
|
logger: testLogger,
|
|
3774
4335
|
tokenPath,
|
|
3775
4336
|
persistConfig: false,
|
|
4337
|
+
paths: {
|
|
4338
|
+
breakers: breakersPath || join(tmpdir(), `fcm-router-test-breakers-${testId}.json`),
|
|
4339
|
+
history: historyPath || join(tmpdir(), `fcm-router-test-history-${testId}.json`),
|
|
4340
|
+
},
|
|
3776
4341
|
})
|
|
3777
4342
|
}
|
|
3778
4343
|
|
|
@@ -3792,7 +4357,7 @@ export function createRouterRuntimeForTest({ config, port = 0, logger = null, to
|
|
|
3792
4357
|
*
|
|
3793
4358
|
* @returns {(entry: { provider: string, model: string }) => Promise<{ ok: boolean, code: string|number, latencyMs: number }>}
|
|
3794
4359
|
*/
|
|
3795
|
-
function createDefaultProbeFn(apiKeys) {
|
|
4360
|
+
export function createDefaultProbeFn(apiKeys) {
|
|
3796
4361
|
return async (entry) => {
|
|
3797
4362
|
const { provider, model } = entry
|
|
3798
4363
|
if (!isRouteableProvider(provider, sources)) return { ok: false, code: 'NOT_ROUTEABLE', latencyMs: 0 }
|
|
@@ -3818,6 +4383,9 @@ function createDefaultProbeFn(apiKeys) {
|
|
|
3818
4383
|
headers['HTTP-Referer'] = 'https://github.com/vava-nessa/free-coding-models'
|
|
3819
4384
|
headers['X-Title'] = 'free-coding-models'
|
|
3820
4385
|
}
|
|
4386
|
+
// 📖 Mandatory per-provider headers (issue #181): OpenCode Zen 400s without
|
|
4387
|
+
// 📖 `x-opencode-session`, so default-set probes must carry it too.
|
|
4388
|
+
Object.assign(headers, getProviderSessionHeaders(provider))
|
|
3821
4389
|
}
|
|
3822
4390
|
const started = Date.now()
|
|
3823
4391
|
try {
|
|
@@ -3839,7 +4407,7 @@ function createDefaultProbeFn(apiKeys) {
|
|
|
3839
4407
|
}
|
|
3840
4408
|
}
|
|
3841
4409
|
|
|
3842
|
-
function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
4410
|
+
export function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
3843
4411
|
// 📖 Synchronous fallback used when async probing isn't available (e.g.
|
|
3844
4412
|
// 📖 routerConfig() getter, which is on the hot path). Falls back to the
|
|
3845
4413
|
// 📖 static tier-based ordering. The async probed version is the one
|
|
@@ -3882,7 +4450,7 @@ function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
|
3882
4450
|
}
|
|
3883
4451
|
}
|
|
3884
4452
|
|
|
3885
|
-
async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
4453
|
+
export async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
3886
4454
|
// 📖 Preserve existing named sets (e.g., created by sync-set) to avoid overwriting
|
|
3887
4455
|
// 📖 user-created configurations. Only rebuild from favorites/defaults when no
|
|
3888
4456
|
// 📖 sets exist at all (fresh install).
|
|
@@ -3924,7 +4492,7 @@ async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
|
3924
4492
|
* 📖 Each favorite "providerKey/modelId" is resolved to its source model entry.
|
|
3925
4493
|
* 📖 Falls back to buildDefaultRouterSet if no favorites exist.
|
|
3926
4494
|
*/
|
|
3927
|
-
function buildRouterSetFromFavorites(config) {
|
|
4495
|
+
export function buildRouterSetFromFavorites(config) {
|
|
3928
4496
|
const favorites = config.favorites
|
|
3929
4497
|
if (!Array.isArray(favorites) || favorites.length === 0) return null
|
|
3930
4498
|
const models = []
|
|
@@ -3952,7 +4520,7 @@ function buildRouterSetFromFavorites(config) {
|
|
|
3952
4520
|
}
|
|
3953
4521
|
}
|
|
3954
4522
|
|
|
3955
|
-
function listenOnPort(server, port, host = '127.0.0.1') {
|
|
4523
|
+
export function listenOnPort(server, port, host = '127.0.0.1') {
|
|
3956
4524
|
return new Promise((resolve, reject) => {
|
|
3957
4525
|
const onError = (error) => {
|
|
3958
4526
|
server.off('error', onError)
|
|
@@ -3968,7 +4536,7 @@ function listenOnPort(server, port, host = '127.0.0.1') {
|
|
|
3968
4536
|
})
|
|
3969
4537
|
}
|
|
3970
4538
|
|
|
3971
|
-
async function listenWithFallback(server, preferredPort, logger, host = '127.0.0.1') {
|
|
4539
|
+
export async function listenWithFallback(server, preferredPort, logger, host = '127.0.0.1') {
|
|
3972
4540
|
const { defaultPort, maxPort } = getRouterPortRange()
|
|
3973
4541
|
const start = Math.max(1, preferredPort || defaultPort)
|
|
3974
4542
|
const candidates = []
|
|
@@ -3989,9 +4557,47 @@ async function listenWithFallback(server, preferredPort, logger, host = '127.0.0
|
|
|
3989
4557
|
throw lastError || new Error('No router ports available')
|
|
3990
4558
|
}
|
|
3991
4559
|
|
|
4560
|
+
// 📖 v2: a set counts as usable when the active set holds at least one model.
|
|
4561
|
+
function hasUsableActiveSet(config) {
|
|
4562
|
+
const router = config?.router
|
|
4563
|
+
if (!router || typeof router !== 'object') return false
|
|
4564
|
+
const activeSet = router.activeSet || DEFAULT_ROUTER_SETTINGS.activeSet
|
|
4565
|
+
const set = router.sets?.[activeSet]
|
|
4566
|
+
return Boolean(set && Array.isArray(set.models) && set.models.length > 0)
|
|
4567
|
+
}
|
|
4568
|
+
|
|
3992
4569
|
export async function runRouterDaemon() {
|
|
3993
4570
|
const config = loadConfig()
|
|
3994
|
-
|
|
4571
|
+
// 📖 v2: listen FIRST. v1 awaited a 24-candidate probe sweep before the
|
|
4572
|
+
// server socket opened, leaving first boots with a ~36s black hole. Build
|
|
4573
|
+
// a fast static set when none exists, serve immediately, and upgrade to
|
|
4574
|
+
// the probe-driven set in the background.
|
|
4575
|
+
let needsProbedSetUpgrade = false
|
|
4576
|
+
if (!hasUsableActiveSet(config)) {
|
|
4577
|
+
const favSet = buildRouterSetFromFavorites(config)
|
|
4578
|
+
if (favSet) {
|
|
4579
|
+
config.router = normalizeRouterConfig({
|
|
4580
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4581
|
+
enabled: true,
|
|
4582
|
+
onboardingSeen: true,
|
|
4583
|
+
activeSet: favSet.name,
|
|
4584
|
+
sets: { [favSet.name]: favSet },
|
|
4585
|
+
})
|
|
4586
|
+
saveConfig(config)
|
|
4587
|
+
} else {
|
|
4588
|
+
const syncSet = buildDefaultRouterSetSync(config, 5)
|
|
4589
|
+
config.router = normalizeRouterConfig({
|
|
4590
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4591
|
+
enabled: true,
|
|
4592
|
+
onboardingSeen: true,
|
|
4593
|
+
activeSet: syncSet.name,
|
|
4594
|
+
sets: { [syncSet.name]: syncSet },
|
|
4595
|
+
})
|
|
4596
|
+
saveConfig(config)
|
|
4597
|
+
needsProbedSetUpgrade = true
|
|
4598
|
+
}
|
|
4599
|
+
}
|
|
4600
|
+
const router = config.router
|
|
3995
4601
|
// 📖 In dev mode, override the saved port with the dev default so a local
|
|
3996
4602
|
// 📖 checkout doesn't clash with a production install on the same machine.
|
|
3997
4603
|
// 📖 The saved config has port: 19280 (production); dev should use 29280.
|
|
@@ -4034,8 +4640,39 @@ export async function runRouterDaemon() {
|
|
|
4034
4640
|
})
|
|
4035
4641
|
runtime.configReloadTimer = setInterval(() => runtime.reloadConfigFromDisk(), CONFIG_RELOAD_INTERVAL_MS)
|
|
4036
4642
|
runtime.tokenFlushTimer = setInterval(() => runtime.tokenTracker.flush(), TOKEN_FLUSH_INTERVAL_MS)
|
|
4037
|
-
|
|
4038
|
-
|
|
4643
|
+
// 📖 v2: probe-driven default set upgrade (when the static one above was
|
|
4644
|
+
// just created) happens in the background, after listen().
|
|
4645
|
+
void (async () => {
|
|
4646
|
+
try {
|
|
4647
|
+
if (needsProbedSetUpgrade) {
|
|
4648
|
+
// 📖 The static tier-ordered pick can contain models the user's key
|
|
4649
|
+
// cannot actually call. Replace it once with the probe-driven set so
|
|
4650
|
+
// the router starts on models that really answer.
|
|
4651
|
+
const fresh = loadConfig()
|
|
4652
|
+
const probed = await buildDefaultRouterSet(fresh, 5, {
|
|
4653
|
+
probeFn: createDefaultProbeFn(fresh.apiKeys || {}),
|
|
4654
|
+
probeTimeoutMs: 1500,
|
|
4655
|
+
probeBudget: 24,
|
|
4656
|
+
})
|
|
4657
|
+
if (probed && Array.isArray(probed.models) && probed.models.length > 0) {
|
|
4658
|
+
fresh.router = normalizeRouterConfig({
|
|
4659
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4660
|
+
enabled: true,
|
|
4661
|
+
onboardingSeen: true,
|
|
4662
|
+
activeSet: probed.name,
|
|
4663
|
+
sets: { [probed.name]: probed },
|
|
4664
|
+
})
|
|
4665
|
+
saveConfig(fresh)
|
|
4666
|
+
runtime.config = fresh
|
|
4667
|
+
runtime.refreshRouteState()
|
|
4668
|
+
}
|
|
4669
|
+
}
|
|
4670
|
+
} catch (error) {
|
|
4671
|
+
logger.debug('Background router set upgrade skipped', { error: error?.message })
|
|
4672
|
+
}
|
|
4673
|
+
void runtime.runProbeBurst()
|
|
4674
|
+
runtime.scheduleProbeLoop()
|
|
4675
|
+
})()
|
|
4039
4676
|
// 📖 Auto-heal: wait for the first probe burst to populate health data,
|
|
4040
4677
|
// 📖 then swap any broken models (AUTH_ERROR / STALE) for working
|
|
4041
4678
|
// 📖 alternatives. This is the M6 promise: the Playground and Router
|
|
@@ -4125,7 +4762,7 @@ export async function startRouterDaemonBackground() {
|
|
|
4125
4762
|
// 📖 Best-effort process command lookup used to verify a PID file before
|
|
4126
4763
|
// 📖 signalling. Returns null when `ps` is unavailable (e.g. Windows) so the
|
|
4127
4764
|
// 📖 caller can keep the previous behavior instead of failing hard.
|
|
4128
|
-
function getProcessCommand(pid) {
|
|
4765
|
+
export function getProcessCommand(pid) {
|
|
4129
4766
|
try {
|
|
4130
4767
|
return execFileSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8' }).trim()
|
|
4131
4768
|
} catch {
|