free-coding-models 0.5.88 → 0.5.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/bin/free-coding-models.js +33 -2
- package/changelog/v0.5.89.md +20 -0
- package/package.json +2 -2
- package/src/core/router-daemon.js +1129 -500
- package/src/core/router-v2/anthropic-compat.js +473 -0
- package/src/core/router-v2/bench.js +171 -0
- package/src/core/router-v2/breaker-store.js +265 -0
- package/src/core/router-v2/constants.js +108 -0
- package/src/core/router-v2/decision-trace.js +134 -0
- package/src/core/router-v2/failure-classifier.js +231 -0
- package/src/core/router-v2/request-history.js +137 -0
- package/src/core/router-v2/response-gate.js +175 -0
- package/src/core/router-v2/tui-dashboard.js +632 -0
- package/src/core/schema-normalizer.js +23 -6
- package/src/core/utils.js +12 -0
- package/src/tui/app.js +7 -2
- package/src/tui/cli-help.js +4 -0
- package/src/tui/key-handler.js +117 -2
- package/src/tui/overlays.js +10 -0
- package/src/tui/tui-state.js +22 -0
- package/web/dist/assets/index-C5hgQLYN.js +48 -0
- package/web/dist/assets/index-CCaxIOti.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/server.js +82 -0
- package/web/dist/assets/index-CAzFIt8P.css +0 -1
- package/web/dist/assets/index-DFg1h0Nd.js +0 -44
|
@@ -35,7 +35,7 @@ import { dirname, join, resolve as resolvePath, sep as pathSep } from 'node:path
|
|
|
35
35
|
import { fork, execFileSync } from 'node:child_process'
|
|
36
36
|
import { randomUUID, timingSafeEqual } from 'node:crypto'
|
|
37
37
|
import { appendFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
38
|
-
import { homedir } from 'node:os'
|
|
38
|
+
import { homedir, tmpdir } from 'node:os'
|
|
39
39
|
import { fileURLToPath } from 'node:url'
|
|
40
40
|
import { MODELS, sources } from '../../sources.js'
|
|
41
41
|
import {
|
|
@@ -80,6 +80,23 @@ import {
|
|
|
80
80
|
pruneStaleEntries as pruneRuntimeTelemetry,
|
|
81
81
|
DEFAULT_MIN_CALLS_FOR_SCORE as RUNTIME_MIN_CALLS,
|
|
82
82
|
} from './runtime-telemetry.js'
|
|
83
|
+
// 📖 Router v2 engine (merged into this daemon): typed failure classification,
|
|
84
|
+
// content-level response gating, decision traces, persisted breakers, request
|
|
85
|
+
// history and the Anthropic /v1/messages protocol. The router is now v2
|
|
86
|
+
// internally while keeping every v1 endpoint, flag and port.
|
|
87
|
+
import { classifyFailure, classifyStatus, clientStatusForKind, FAILURE_KINDS } from './router-v2/failure-classifier.js'
|
|
88
|
+
import { validateChatCompletionPayload, createStreamReadinessTracker, estimateTokens } from './router-v2/response-gate.js'
|
|
89
|
+
import { createDecisionTrace, traceSkip, traceAttempt, finishTrace, decisionHeaderValue, traceSummary } from './router-v2/decision-trace.js'
|
|
90
|
+
import { BreakerStore } from './router-v2/breaker-store.js'
|
|
91
|
+
import { RequestHistory } from './router-v2/request-history.js'
|
|
92
|
+
import {
|
|
93
|
+
anthropicErrorPayload,
|
|
94
|
+
anthropicErrorTypeForStatus,
|
|
95
|
+
createAnthropicStreamTransformer,
|
|
96
|
+
translateAnthropicToOpenAI,
|
|
97
|
+
translateOpenAIToAnthropicResponse,
|
|
98
|
+
} from './router-v2/anthropic-compat.js'
|
|
99
|
+
import { parseFcmModel } from './router-v2/constants.js'
|
|
83
100
|
|
|
84
101
|
export const ROUTER_DEFAULT_PORT = 19280
|
|
85
102
|
export const ROUTER_MAX_PORT = 19289
|
|
@@ -118,7 +135,7 @@ export function getRouterPortRange() {
|
|
|
118
135
|
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
119
136
|
const CLI_ENTRY_PATH = join(__dirname, '..', '..', 'bin', 'free-coding-models.js')
|
|
120
137
|
const LOCAL_VERSION = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8')).version
|
|
121
|
-
const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
138
|
+
export const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
122
139
|
/**
|
|
123
140
|
* 📖 normalizeToolCallsResponse — fix malformed tool_calls from upstream.
|
|
124
141
|
* Some providers return finish_reason: "tool_calls" but message has no tool_calls array,
|
|
@@ -128,7 +145,7 @@ const MAX_BODY_BYTES = 10 * 1024 * 1024
|
|
|
128
145
|
* @param {object} data - parsed JSON response object
|
|
129
146
|
* @returns {boolean} true if mutated
|
|
130
147
|
*/
|
|
131
|
-
function normalizeToolCallsResponse(data) {
|
|
148
|
+
export function normalizeToolCallsResponse(data) {
|
|
132
149
|
if (!data || typeof data !== 'object' || !Array.isArray(data.choices)) return false
|
|
133
150
|
let mutated = false
|
|
134
151
|
for (const choice of data.choices) {
|
|
@@ -145,16 +162,16 @@ function normalizeToolCallsResponse(data) {
|
|
|
145
162
|
return mutated
|
|
146
163
|
}
|
|
147
164
|
|
|
148
|
-
const MAX_REQUEST_LOG = 200
|
|
149
|
-
const MAX_SSE_CLIENTS = 10
|
|
150
|
-
const MAX_SSE_CLIENTS_PER_ORIGIN = 5
|
|
151
|
-
const MAX_CONCURRENT_REQUESTS = 50
|
|
152
|
-
const MAX_PROBE_WINDOW = 20
|
|
165
|
+
export const MAX_REQUEST_LOG = 200
|
|
166
|
+
export const MAX_SSE_CLIENTS = 10
|
|
167
|
+
export const MAX_SSE_CLIENTS_PER_ORIGIN = 5
|
|
168
|
+
export const MAX_CONCURRENT_REQUESTS = 50
|
|
169
|
+
export const MAX_PROBE_WINDOW = 20
|
|
153
170
|
const TOKEN_FLUSH_INTERVAL_MS = 60000
|
|
154
171
|
const CONFIG_RELOAD_INTERVAL_MS = 10000
|
|
155
|
-
const STATS_RETENTION_DAYS = 90
|
|
156
|
-
const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504, 529])
|
|
157
|
-
const AUTH_STATUS_CODES = new Set([401, 403])
|
|
172
|
+
export const STATS_RETENTION_DAYS = 90
|
|
173
|
+
export const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504, 529])
|
|
174
|
+
export const AUTH_STATUS_CODES = new Set([401, 403])
|
|
158
175
|
const RATE_LIMIT_HEADER_NAMES = [
|
|
159
176
|
'retry-after',
|
|
160
177
|
'x-ratelimit-limit',
|
|
@@ -171,16 +188,16 @@ const RATE_LIMIT_HEADER_NAMES = [
|
|
|
171
188
|
'x-ratelimit-reset-tokens',
|
|
172
189
|
]
|
|
173
190
|
|
|
174
|
-
function nowIso() {
|
|
191
|
+
export function nowIso() {
|
|
175
192
|
return new Date().toISOString()
|
|
176
193
|
}
|
|
177
194
|
|
|
178
|
-
function modelKey(provider, model) {
|
|
195
|
+
export function modelKey(provider, model) {
|
|
179
196
|
return `${provider}/${model}`
|
|
180
197
|
}
|
|
181
198
|
|
|
182
199
|
// 📖 parseJsonResult is still local - it returns {ok, value/error} which is different from safeJsonParse
|
|
183
|
-
function parseJsonResult(raw) {
|
|
200
|
+
export function parseJsonResult(raw) {
|
|
184
201
|
try {
|
|
185
202
|
return { ok: true, value: JSON.parse(raw) }
|
|
186
203
|
} catch (error) {
|
|
@@ -188,7 +205,7 @@ function parseJsonResult(raw) {
|
|
|
188
205
|
}
|
|
189
206
|
}
|
|
190
207
|
|
|
191
|
-
function isProcessAlive(pid) {
|
|
208
|
+
export function isProcessAlive(pid) {
|
|
192
209
|
if (!Number.isInteger(pid) || pid <= 0) return false
|
|
193
210
|
try {
|
|
194
211
|
process.kill(pid, 0)
|
|
@@ -198,7 +215,7 @@ function isProcessAlive(pid) {
|
|
|
198
215
|
}
|
|
199
216
|
}
|
|
200
217
|
|
|
201
|
-
function readNumberFile(path) {
|
|
218
|
+
export function readNumberFile(path) {
|
|
202
219
|
try {
|
|
203
220
|
const value = Number.parseInt(readFileSync(path, 'utf8').trim(), 10)
|
|
204
221
|
return Number.isFinite(value) ? value : null
|
|
@@ -207,7 +224,7 @@ function readNumberFile(path) {
|
|
|
207
224
|
}
|
|
208
225
|
}
|
|
209
226
|
|
|
210
|
-
function headerEntries(headers) {
|
|
227
|
+
export function headerEntries(headers) {
|
|
211
228
|
const entries = {}
|
|
212
229
|
if (!headers || typeof headers.forEach !== 'function') return entries
|
|
213
230
|
headers.forEach((value, key) => {
|
|
@@ -222,12 +239,12 @@ function headerEntries(headers) {
|
|
|
222
239
|
return entries
|
|
223
240
|
}
|
|
224
241
|
|
|
225
|
-
function getHeaderValue(headers, name) {
|
|
242
|
+
export function getHeaderValue(headers, name) {
|
|
226
243
|
if (!headers || typeof headers.get !== 'function') return ''
|
|
227
244
|
return headers.get(name) || ''
|
|
228
245
|
}
|
|
229
246
|
|
|
230
|
-
function extractRateLimitHeaders(headers) {
|
|
247
|
+
export function extractRateLimitHeaders(headers) {
|
|
231
248
|
const values = {}
|
|
232
249
|
for (const name of RATE_LIMIT_HEADER_NAMES) {
|
|
233
250
|
const value = getHeaderValue(headers, name)
|
|
@@ -236,7 +253,7 @@ function extractRateLimitHeaders(headers) {
|
|
|
236
253
|
return values
|
|
237
254
|
}
|
|
238
255
|
|
|
239
|
-
function parseRetryAfterMs(value) {
|
|
256
|
+
export function parseRetryAfterMs(value) {
|
|
240
257
|
if (!value) return null
|
|
241
258
|
const seconds = Number(value)
|
|
242
259
|
if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000))
|
|
@@ -245,7 +262,7 @@ function parseRetryAfterMs(value) {
|
|
|
245
262
|
return null
|
|
246
263
|
}
|
|
247
264
|
|
|
248
|
-
function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
265
|
+
export function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
249
266
|
return Object.entries(rateLimitHeaders).some(([name, value]) => {
|
|
250
267
|
if (!name.includes('remaining')) return false
|
|
251
268
|
const numeric = Number(value)
|
|
@@ -253,11 +270,11 @@ function hasZeroRemainingQuota(rateLimitHeaders) {
|
|
|
253
270
|
})
|
|
254
271
|
}
|
|
255
272
|
|
|
256
|
-
function isLikelyHtmlText(text) {
|
|
273
|
+
export function isLikelyHtmlText(text) {
|
|
257
274
|
return /^\s*(<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>])/i.test(text || '')
|
|
258
275
|
}
|
|
259
276
|
|
|
260
|
-
function isLikelyHtmlResponse(headers, text = '') {
|
|
277
|
+
export function isLikelyHtmlResponse(headers, text = '') {
|
|
261
278
|
const contentType = getHeaderValue(headers, 'content-type').toLowerCase()
|
|
262
279
|
return contentType.includes('text/html') || isLikelyHtmlText(text)
|
|
263
280
|
}
|
|
@@ -268,7 +285,7 @@ function isLikelyHtmlResponse(headers, text = '') {
|
|
|
268
285
|
// 📖 endpoints. Blocks CSRF from malicious tabs and key exfiltration from
|
|
269
286
|
// 📖 cross-origin scripts. Plain CLI calls (curl/fetch without Origin) are
|
|
270
287
|
// 📖 allowed because they cannot be triggered by a browser context.
|
|
271
|
-
function isLoopbackHostname(hostname) {
|
|
288
|
+
export function isLoopbackHostname(hostname) {
|
|
272
289
|
if (!hostname) return false
|
|
273
290
|
const h = hostname.toLowerCase()
|
|
274
291
|
return h === 'localhost' || h === '127.0.0.1' || h === '[::1]' || h === '::1' || h.endsWith('.localhost')
|
|
@@ -279,7 +296,7 @@ function isLoopbackHostname(hostname) {
|
|
|
279
296
|
// 📖 `.local` or `.internal`. Used when the daemon is explicitly bound to a
|
|
280
297
|
// 📖 wildcard address (FCM_HOST=0.0.0.0) so LAN clients keep working; it
|
|
281
298
|
// 📖 never grants origin trust on its own.
|
|
282
|
-
function isPrivateNetworkHostname(hostname) {
|
|
299
|
+
export function isPrivateNetworkHostname(hostname) {
|
|
283
300
|
if (!hostname) return false
|
|
284
301
|
const h = hostname.toLowerCase()
|
|
285
302
|
// 📖 mDNS / zero-conf hostnames
|
|
@@ -304,7 +321,28 @@ function getAllowedOrigins() {
|
|
|
304
321
|
return _allowedOriginsCache
|
|
305
322
|
}
|
|
306
323
|
|
|
307
|
-
|
|
324
|
+
// 📖 v2: CORS for loopback (and explicitly allowed) origins so a browser
|
|
325
|
+
// dashboard served from another local port (web dashboard 3333) can call
|
|
326
|
+
// this daemon directly without a proxy.
|
|
327
|
+
function applyCors(req, res) {
|
|
328
|
+
const origin = typeof req.headers.origin === 'string' ? req.headers.origin : ''
|
|
329
|
+
if (!origin) return
|
|
330
|
+
let hostname = ''
|
|
331
|
+
try {
|
|
332
|
+
hostname = new URL(origin).hostname
|
|
333
|
+
} catch {
|
|
334
|
+
return
|
|
335
|
+
}
|
|
336
|
+
const allowed = isLoopbackHostname(hostname) || getAllowedOrigins().includes(origin)
|
|
337
|
+
if (!allowed) return
|
|
338
|
+
res.setHeader('Access-Control-Allow-Origin', origin)
|
|
339
|
+
res.setHeader('Vary', 'Origin')
|
|
340
|
+
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS')
|
|
341
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, x-request-id, anthropic-version')
|
|
342
|
+
res.setHeader('Access-Control-Max-Age', '600')
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export function isSameOriginOrLocal(req) {
|
|
308
346
|
const origin = req.headers.origin
|
|
309
347
|
const referer = req.headers.referer || req.headers.referrer
|
|
310
348
|
const hasOrigin = typeof origin === 'string' && origin.length > 0
|
|
@@ -346,7 +384,7 @@ function isSameOriginOrLocal(req) {
|
|
|
346
384
|
// 📖 Loopback Host values are always accepted; when the daemon is bound to a
|
|
347
385
|
// 📖 non-loopback FCM_HOST (LAN / Docker), that hostname plus private-network
|
|
348
386
|
// 📖 hosts are accepted too. Public domains stay rejected.
|
|
349
|
-
function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
387
|
+
export function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
350
388
|
if (typeof hostHeader !== 'string' || !hostHeader) return false
|
|
351
389
|
const value = hostHeader.toLowerCase()
|
|
352
390
|
const loopback = [`127.0.0.1:${port}`, `localhost:${port}`, `[::1]:${port}`, '127.0.0.1', 'localhost', '[::1]']
|
|
@@ -366,18 +404,18 @@ function isAllowedHostHeader(hostHeader, port, boundHost) {
|
|
|
366
404
|
// 📖 Set FCM_ROUTER_TOKEN to require `Authorization: Bearer <token>` (or
|
|
367
405
|
// 📖 `x-api-key: <token>`) on every /v1/* request; leave it unset for the
|
|
368
406
|
// 📖 default no-auth local behavior.
|
|
369
|
-
function getRouterToken() {
|
|
407
|
+
export function getRouterToken() {
|
|
370
408
|
return (process.env.FCM_ROUTER_TOKEN || '').trim()
|
|
371
409
|
}
|
|
372
410
|
|
|
373
|
-
function safeTokenCompare(candidate, token) {
|
|
411
|
+
export function safeTokenCompare(candidate, token) {
|
|
374
412
|
const bufA = Buffer.from(String(candidate || ''))
|
|
375
413
|
const bufB = Buffer.from(token)
|
|
376
414
|
if (bufA.length !== bufB.length) return false
|
|
377
415
|
return timingSafeEqual(bufA, bufB)
|
|
378
416
|
}
|
|
379
417
|
|
|
380
|
-
function isAuthorizedForV1(req) {
|
|
418
|
+
export function isAuthorizedForV1(req) {
|
|
381
419
|
const token = getRouterToken()
|
|
382
420
|
if (!token) return true
|
|
383
421
|
const auth = req.headers.authorization || ''
|
|
@@ -386,7 +424,7 @@ function isAuthorizedForV1(req) {
|
|
|
386
424
|
return typeof apiKeyHeader === 'string' && safeTokenCompare(apiKeyHeader.trim(), token)
|
|
387
425
|
}
|
|
388
426
|
|
|
389
|
-
const MIME_TYPES = {
|
|
427
|
+
export const MIME_TYPES = {
|
|
390
428
|
'.html': 'text/html; charset=utf-8',
|
|
391
429
|
'.css': 'text/css; charset=utf-8',
|
|
392
430
|
'.js': 'application/javascript; charset=utf-8',
|
|
@@ -396,7 +434,7 @@ const MIME_TYPES = {
|
|
|
396
434
|
'.ico': 'image/x-icon',
|
|
397
435
|
}
|
|
398
436
|
|
|
399
|
-
function getWebModelsPayload(runtime) {
|
|
437
|
+
export function getWebModelsPayload(runtime) {
|
|
400
438
|
// 📖 Hoist router + active set lookups out of the per-model loop so we
|
|
401
439
|
// 📖 don't re-resolve them ~200 times per request.
|
|
402
440
|
const router = runtime.routerConfig()
|
|
@@ -473,7 +511,7 @@ function getWebUpdateStatusPayload() {
|
|
|
473
511
|
}
|
|
474
512
|
}
|
|
475
513
|
|
|
476
|
-
function getWebStatePayload(runtime) {
|
|
514
|
+
export function getWebStatePayload(runtime) {
|
|
477
515
|
const router = runtime.routerConfig()
|
|
478
516
|
const probeInterval = router.probeIntervals?.[router.probeMode] || DEFAULT_ROUTER_SETTINGS.probeIntervals.balanced
|
|
479
517
|
return {
|
|
@@ -491,7 +529,7 @@ function getWebStatePayload(runtime) {
|
|
|
491
529
|
}
|
|
492
530
|
}
|
|
493
531
|
|
|
494
|
-
function getWebConfigPayload(runtime) {
|
|
532
|
+
export function getWebConfigPayload(runtime) {
|
|
495
533
|
const providers = {}
|
|
496
534
|
for (const [key, src] of Object.entries(sources)) {
|
|
497
535
|
const rawKey = runtime.getApiKeyForProvider(key)
|
|
@@ -517,9 +555,9 @@ function getWebConfigPayload(runtime) {
|
|
|
517
555
|
}
|
|
518
556
|
}
|
|
519
557
|
|
|
520
|
-
const WEB_DIST_DIR = resolvePath(__dirname, '..', '..', 'web', 'dist')
|
|
558
|
+
export const WEB_DIST_DIR = resolvePath(__dirname, '..', '..', 'web', 'dist')
|
|
521
559
|
|
|
522
|
-
function serveStaticFromDist(res, absPath) {
|
|
560
|
+
export function serveStaticFromDist(res, absPath) {
|
|
523
561
|
const ext = absPath.slice(absPath.lastIndexOf('.'))
|
|
524
562
|
const ct = MIME_TYPES[ext] || 'application/octet-stream'
|
|
525
563
|
res.writeHead(200, {
|
|
@@ -530,7 +568,7 @@ function serveStaticFromDist(res, absPath) {
|
|
|
530
568
|
res.end(readFileSync(absPath))
|
|
531
569
|
}
|
|
532
570
|
|
|
533
|
-
function serveSpaIndex(res) {
|
|
571
|
+
export function serveSpaIndex(res) {
|
|
534
572
|
const indexPath = resolvePath(WEB_DIST_DIR, 'index.html')
|
|
535
573
|
if (!existsSync(indexPath)) {
|
|
536
574
|
res.writeHead(503, { 'Content-Type': 'text/plain' })
|
|
@@ -545,7 +583,7 @@ function serveSpaIndex(res) {
|
|
|
545
583
|
res.end(readFileSync(indexPath))
|
|
546
584
|
}
|
|
547
585
|
|
|
548
|
-
function serveWebStaticFile(res, pathname, requestId) {
|
|
586
|
+
export function serveWebStaticFile(res, pathname, requestId) {
|
|
549
587
|
// 📖 Resolve to an absolute path and verify it stays inside WEB_DIST_DIR.
|
|
550
588
|
// 📖 Without this, `pathname` like `/../../etc/passwd` escapes the dist root.
|
|
551
589
|
const requested = pathname === '/' ? 'index.html' : pathname.replace(/^\/+/, '')
|
|
@@ -582,7 +620,7 @@ function serveWebStaticFile(res, pathname, requestId) {
|
|
|
582
620
|
serveStaticFromDist(res, candidate)
|
|
583
621
|
}
|
|
584
622
|
|
|
585
|
-
function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
623
|
+
export function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
586
624
|
// 📖 Keep quota diagnostics structural only: headers and retry timing are safe,
|
|
587
625
|
// 📖 while upstream response bodies stay out of logs and telemetry.
|
|
588
626
|
const rateLimitHeaders = extractRateLimitHeaders(response.headers)
|
|
@@ -602,7 +640,7 @@ function buildUpstreamMeta(response, text = '', providerKey = '') {
|
|
|
602
640
|
}
|
|
603
641
|
}
|
|
604
642
|
|
|
605
|
-
function attachClientAbort(req, res, controller) {
|
|
643
|
+
export function attachClientAbort(req, res, controller) {
|
|
606
644
|
let clientAborted = false
|
|
607
645
|
const abort = () => {
|
|
608
646
|
if (res.writableEnded) return
|
|
@@ -634,7 +672,10 @@ export function cloneHeadersForUpstream(reqHeaders, apiKey, providerKey) {
|
|
|
634
672
|
const lower = key.toLowerCase()
|
|
635
673
|
// 📖 Drop the client's cookies: they belong to the local browser session
|
|
636
674
|
// 📖 and must never be forwarded to the upstream provider.
|
|
637
|
-
|
|
675
|
+
// 📖 Also strip `x-api-key`: the local router token can arrive under that
|
|
676
|
+
// name and must never leak to an upstream provider (v2 security fix).
|
|
677
|
+
if (['host', 'connection', 'content-length', 'authorization', 'cookie', 'x-api-key'].includes(lower)) continue
|
|
678
|
+
if (lower.startsWith('x-fcm-') || lower === 'x-request-id') continue
|
|
638
679
|
if (typeof value !== 'string') continue
|
|
639
680
|
if (lower === 'content-type') {
|
|
640
681
|
headers['Content-Type'] = value
|
|
@@ -651,23 +692,78 @@ export function cloneHeadersForUpstream(reqHeaders, apiKey, providerKey) {
|
|
|
651
692
|
return headers
|
|
652
693
|
}
|
|
653
694
|
|
|
654
|
-
|
|
695
|
+
// 📖 v2 defaults for the merged engine. These extend (never replace) the
|
|
696
|
+
// shared failover settings; users override them in ~/.free-coding-models.json
|
|
697
|
+
// under `router.failover` and the raw values are read because the shared
|
|
698
|
+
// normalizer only knows the v1 field names.
|
|
699
|
+
const DEFAULT_BODY_READ_TIMEOUT_MS = 30000
|
|
700
|
+
const DEFAULT_TOTAL_BUDGET_MS = 120000
|
|
701
|
+
const DEFAULT_CONTENT_VALIDATION = 'strict'
|
|
702
|
+
const DEFAULT_QUOTA_PAUSE_MS = 60000
|
|
703
|
+
const MAX_CONCURRENT_QUEUE_RETRY_AFTER_S = 3
|
|
704
|
+
|
|
705
|
+
function clampIntV2(value, fallback, { min, max }) {
|
|
706
|
+
const n = Number(value)
|
|
707
|
+
if (!Number.isFinite(n)) return fallback
|
|
708
|
+
return Math.min(max, Math.max(min, Math.round(n)))
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
// 📖 Read the upstream body under a hard deadline (v2 fix): v1 cleared the
|
|
712
|
+
// request timeout as soon as headers arrived, so a provider that trickled the
|
|
713
|
+
// body could hang an agent forever.
|
|
714
|
+
async function readBodyWithTimeout(response, controller, timeoutMs) {
|
|
715
|
+
let timer = null
|
|
716
|
+
try {
|
|
717
|
+
return await Promise.race([
|
|
718
|
+
response.text(),
|
|
719
|
+
new Promise((_, reject) => {
|
|
720
|
+
timer = setTimeout(() => {
|
|
721
|
+
try { controller.abort() } catch {}
|
|
722
|
+
reject(Object.assign(new Error('upstream_body_read_timeout'), { name: 'BodyReadTimeoutError' }))
|
|
723
|
+
}, timeoutMs)
|
|
724
|
+
}),
|
|
725
|
+
])
|
|
726
|
+
} finally {
|
|
727
|
+
if (timer) clearTimeout(timer)
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
// 📖 Map a content-gate rejection reason to its failure kind name.
|
|
732
|
+
function gateReasonToKind(reason) {
|
|
733
|
+
switch (reason) {
|
|
734
|
+
case 'error_payload': return 'ERROR_PAYLOAD'
|
|
735
|
+
case 'empty_choices': return 'EMPTY_CHOICES'
|
|
736
|
+
case 'empty_content': return 'EMPTY_CONTENT'
|
|
737
|
+
case 'invalid_json': return 'INVALID_JSON'
|
|
738
|
+
default: return 'INVALID_JSON'
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
function parseLastResortModel(value) {
|
|
743
|
+
if (typeof value !== 'string') return null
|
|
744
|
+
const trimmed = value.trim()
|
|
745
|
+
const slashIdx = trimmed.indexOf('/')
|
|
746
|
+
if (slashIdx <= 0 || slashIdx === trimmed.length - 1) return null
|
|
747
|
+
return { provider: trimmed.slice(0, slashIdx), model: trimmed.slice(slashIdx + 1), key: trimmed }
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
export function getApiModelId(providerKey, modelId) {
|
|
655
751
|
return providerKey === 'zai' ? modelId.replace(/^zai\//, '') : modelId
|
|
656
752
|
}
|
|
657
753
|
|
|
658
|
-
function resolveProviderUrl(providerKey) {
|
|
754
|
+
export function resolveProviderUrl(providerKey) {
|
|
659
755
|
const url = sources[providerKey]?.url
|
|
660
756
|
if (!url) return null
|
|
661
757
|
return providerKey === 'cloudflare' ? resolveCloudflareUrl(url) : url
|
|
662
758
|
}
|
|
663
759
|
|
|
664
|
-
function buildProviderModelsUrl(providerKey) {
|
|
760
|
+
export function buildProviderModelsUrl(providerKey) {
|
|
665
761
|
const url = resolveProviderUrl(providerKey)
|
|
666
762
|
if (typeof url !== 'string' || !url.includes('/chat/completions')) return null
|
|
667
763
|
return url.replace(/\/chat\/completions$/, '/models')
|
|
668
764
|
}
|
|
669
765
|
|
|
670
|
-
function extractUsage(payload) {
|
|
766
|
+
export function extractUsage(payload) {
|
|
671
767
|
const usage = payload?.usage
|
|
672
768
|
if (!usage || typeof usage !== 'object') return null
|
|
673
769
|
const promptTokens = Number(usage.prompt_tokens ?? 0)
|
|
@@ -694,7 +790,7 @@ export function formatOpenAiError(message, type, code, requestId, extra = {}) {
|
|
|
694
790
|
}
|
|
695
791
|
}
|
|
696
792
|
|
|
697
|
-
function sendJson(res, statusCode, payload, headers = {}) {
|
|
793
|
+
export function sendJson(res, statusCode, payload, headers = {}) {
|
|
698
794
|
if (res.writableEnded) return
|
|
699
795
|
const body = JSON.stringify(payload)
|
|
700
796
|
res.writeHead(statusCode, {
|
|
@@ -705,11 +801,11 @@ function sendJson(res, statusCode, payload, headers = {}) {
|
|
|
705
801
|
res.end(body)
|
|
706
802
|
}
|
|
707
803
|
|
|
708
|
-
function sendError(res, statusCode, message, type, code, requestId, extra = {}) {
|
|
804
|
+
export function sendError(res, statusCode, message, type, code, requestId, extra = {}) {
|
|
709
805
|
sendJson(res, statusCode, formatOpenAiError(message, type, code, requestId, extra))
|
|
710
806
|
}
|
|
711
807
|
|
|
712
|
-
function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
808
|
+
export function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
713
809
|
return new Promise((resolve, reject) => {
|
|
714
810
|
let size = 0
|
|
715
811
|
const chunks = []
|
|
@@ -727,7 +823,7 @@ function readRequestBody(req, limit = MAX_BODY_BYTES) {
|
|
|
727
823
|
})
|
|
728
824
|
}
|
|
729
825
|
|
|
730
|
-
function readJsonBody(req) {
|
|
826
|
+
export function readJsonBody(req) {
|
|
731
827
|
return readRequestBody(req).then((raw) => {
|
|
732
828
|
// 📖 Refuse explicit non-JSON bodies (e.g. text/plain form posts) so a
|
|
733
829
|
// 📖 cross-site form cannot smuggle data into JSON endpoints. Missing or
|
|
@@ -794,7 +890,7 @@ export function applyPrePromptToBody(body, prePrompt) {
|
|
|
794
890
|
return { ...safeBody, messages }
|
|
795
891
|
}
|
|
796
892
|
|
|
797
|
-
class RouterLogger {
|
|
893
|
+
export class RouterLogger {
|
|
798
894
|
constructor(logPath, level = 'info') {
|
|
799
895
|
this.logPath = logPath
|
|
800
896
|
this.level = level
|
|
@@ -844,7 +940,7 @@ class RouterLogger {
|
|
|
844
940
|
debug(message, meta = null) { this.write('debug', message, meta) }
|
|
845
941
|
}
|
|
846
942
|
|
|
847
|
-
class TokenTracker {
|
|
943
|
+
export class TokenTracker {
|
|
848
944
|
constructor(path, logger) {
|
|
849
945
|
this.path = path
|
|
850
946
|
this.logger = logger
|
|
@@ -969,8 +1065,8 @@ class TokenTracker {
|
|
|
969
1065
|
}
|
|
970
1066
|
}
|
|
971
1067
|
|
|
972
|
-
class RouterRuntime {
|
|
973
|
-
constructor({ config, port, logger, tokenPath = ROUTER_TOKENS_PATH, persistConfig = true }) {
|
|
1068
|
+
export class RouterRuntime {
|
|
1069
|
+
constructor({ config, port, logger, tokenPath = ROUTER_TOKENS_PATH, persistConfig = true, paths = {} }) {
|
|
974
1070
|
this.config = config
|
|
975
1071
|
this.port = port
|
|
976
1072
|
this.logger = logger
|
|
@@ -984,11 +1080,28 @@ class RouterRuntime {
|
|
|
984
1080
|
this.configReloadTimer = null
|
|
985
1081
|
this.tokenFlushTimer = null
|
|
986
1082
|
this.probeTimer = null
|
|
1083
|
+
this.probeWatchdog = null
|
|
987
1084
|
this.probeTimeouts = new Set()
|
|
988
1085
|
this.tokenTracker = new TokenTracker(tokenPath, logger)
|
|
989
1086
|
this.modelCatalog = this.buildModelCatalog()
|
|
990
1087
|
this.probeWindows = new Map()
|
|
991
|
-
|
|
1088
|
+
// 📖 v2 engine: persisted circuit breakers (survive restarts, DEGRADED
|
|
1089
|
+
// warning state, escalating backoff). `this.circuit` stays the shared Map
|
|
1090
|
+
// so every v1 read-path keeps working; it IS the breaker store's map.
|
|
1091
|
+
this.breakers = new BreakerStore({
|
|
1092
|
+
path: paths.breakers || join(homedir(), '.free-coding-models-router-v2-breakers.json'),
|
|
1093
|
+
logger,
|
|
1094
|
+
})
|
|
1095
|
+
this.circuit = this.breakers.breakers
|
|
1096
|
+
// 📖 v2 engine: persisted request history (fallback chains per request)
|
|
1097
|
+
// and the in-flight decision traces surfaced on /api/router-v2/*.
|
|
1098
|
+
this.history = new RequestHistory({
|
|
1099
|
+
path: paths.history || join(homedir(), '.free-coding-models-router-v2-history.json'),
|
|
1100
|
+
logger,
|
|
1101
|
+
maxEntries: 500,
|
|
1102
|
+
})
|
|
1103
|
+
this.recentTraces = []
|
|
1104
|
+
this.quotaPauses = new Map()
|
|
992
1105
|
this.requestLog = []
|
|
993
1106
|
this.activeRequests = new Map()
|
|
994
1107
|
this.sseClients = new Set()
|
|
@@ -1078,20 +1191,11 @@ class RouterRuntime {
|
|
|
1078
1191
|
for (const model of set.models || []) {
|
|
1079
1192
|
const key = modelKey(model.provider, model.model)
|
|
1080
1193
|
if (!this.probeWindows.has(key)) this.probeWindows.set(key, [])
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
consecutiveFailures: 0,
|
|
1085
|
-
cooldownMs: router.circuitBreaker.initialCooldownMs,
|
|
1086
|
-
openedAt: null,
|
|
1087
|
-
lastError: null,
|
|
1088
|
-
authError: false,
|
|
1089
|
-
stale: false,
|
|
1090
|
-
})
|
|
1091
|
-
}
|
|
1092
|
-
const entry = this.circuit.get(key)
|
|
1093
|
-
entry.stale = !this.modelCatalog.has(key)
|
|
1194
|
+
// 📖 v2: entries come from the persisted breaker store (restored state
|
|
1195
|
+
// included); catalog-derived flags are refreshed on every boot.
|
|
1196
|
+
const entry = this.breakers.ensure(key, router.circuitBreaker.initialCooldownMs)
|
|
1094
1197
|
const catalogEntry = this.modelCatalog.get(key)
|
|
1198
|
+
entry.stale = !this.modelCatalog.has(key)
|
|
1095
1199
|
entry.unsupported = Boolean(catalogEntry && !catalogEntry.routeable)
|
|
1096
1200
|
if (entry.stale && !this.staleNotifications.has(key)) {
|
|
1097
1201
|
this.staleNotifications.add(key)
|
|
@@ -1149,8 +1253,11 @@ class RouterRuntime {
|
|
|
1149
1253
|
reloadConfigFromDisk() {
|
|
1150
1254
|
try {
|
|
1151
1255
|
const nextConfig = loadConfig()
|
|
1152
|
-
// 📖
|
|
1153
|
-
|
|
1256
|
+
// 📖 v2 fix: do NOT run ensureRouterConfigForDaemon here. It rebuilds
|
|
1257
|
+
// the router section from DEFAULT_ROUTER_SETTINGS and silently
|
|
1258
|
+
// discards user failover tuning (requestTimeoutMs, streamStall, and
|
|
1259
|
+
// the v2-only fields) on every 10s tick. The raw file is adopted
|
|
1260
|
+
// as-is; routerConfig() normalizes on read.
|
|
1154
1261
|
this.config = nextConfig
|
|
1155
1262
|
this.refreshRouteState()
|
|
1156
1263
|
this.scheduleProbeLoop()
|
|
@@ -1178,16 +1285,54 @@ class RouterRuntime {
|
|
|
1178
1285
|
return [...(set?.models || [])].sort((a, b) => a.priority - b.priority)
|
|
1179
1286
|
}
|
|
1180
1287
|
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
this.
|
|
1288
|
+
// ─── v2 engine: failover settings + quota pauses + breaker plumbing ───────
|
|
1289
|
+
|
|
1290
|
+
// 📖 v2-specific failover knobs are read from the RAW config: the shared
|
|
1291
|
+
// normalizer only knows the v1 field names and would drop the new ones.
|
|
1292
|
+
failoverSettings() {
|
|
1293
|
+
const normalized = this.routerConfig().failover
|
|
1294
|
+
const raw = (this.config?.router?.failover && typeof this.config.router.failover === 'object')
|
|
1295
|
+
? this.config.router.failover
|
|
1296
|
+
: {}
|
|
1297
|
+
const validation = raw.contentValidation
|
|
1298
|
+
return {
|
|
1299
|
+
...normalized,
|
|
1300
|
+
bodyReadTimeoutMs: clampIntV2(raw.bodyReadTimeoutMs, DEFAULT_BODY_READ_TIMEOUT_MS, { min: 5000, max: 300000 }),
|
|
1301
|
+
totalBudgetMs: clampIntV2(raw.totalBudgetMs, DEFAULT_TOTAL_BUDGET_MS, { min: 10000, max: 600000 }),
|
|
1302
|
+
contentValidation: ['strict', 'basic', 'off'].includes(validation) ? validation : DEFAULT_CONTENT_VALIDATION,
|
|
1303
|
+
lastResortModel: parseLastResortModel(raw.lastResortModel),
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
breakerParams() {
|
|
1308
|
+
const cb = this.routerConfig().circuitBreaker
|
|
1309
|
+
return {
|
|
1310
|
+
failureThreshold: cb.failureThreshold,
|
|
1311
|
+
initialCooldownMs: cb.initialCooldownMs,
|
|
1312
|
+
maxCooldownMs: cb.maxCooldownMs,
|
|
1313
|
+
backoffMultiplier: cb.backoffMultiplier,
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
quotaPauseActive(key) {
|
|
1318
|
+
const pause = this.quotaPauses.get(key)
|
|
1319
|
+
if (!pause) return false
|
|
1320
|
+
if (Date.now() >= pause.until) {
|
|
1321
|
+
this.quotaPauses.delete(key)
|
|
1322
|
+
return false
|
|
1189
1323
|
}
|
|
1190
|
-
return
|
|
1324
|
+
return true
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
quotaPausesForKeys(keys) {
|
|
1328
|
+
return keys
|
|
1329
|
+
.map((key) => this.quotaPauses.get(key))
|
|
1330
|
+
.filter(Boolean)
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
updateCircuitForCooldown(key) {
|
|
1334
|
+
// 📖 v2: lazy OPEN -> HALF_OPEN promotion lives in the breaker store.
|
|
1335
|
+
return this.breakers.evaluate(key)
|
|
1191
1336
|
}
|
|
1192
1337
|
|
|
1193
1338
|
recordProbeResult(key, result) {
|
|
@@ -1244,69 +1389,86 @@ class RouterRuntime {
|
|
|
1244
1389
|
}
|
|
1245
1390
|
|
|
1246
1391
|
markAuthError(key, detail = 'authentication failed') {
|
|
1247
|
-
const state = this.
|
|
1392
|
+
const state = this.breakers.get(key)
|
|
1248
1393
|
if (!state) return
|
|
1249
|
-
|
|
1250
|
-
state.lastError = detail
|
|
1394
|
+
this.breakers.markFailure(key, { ...this.breakerParams(), detail, authError: true })
|
|
1251
1395
|
this.broadcast('circuit', { model: key, old_state: state.state, new_state: 'AUTH_ERROR', cooldown_ms: 0 })
|
|
1252
1396
|
}
|
|
1253
1397
|
|
|
1254
1398
|
markSuccess(key, latencyMs = null) {
|
|
1255
|
-
const
|
|
1256
|
-
|
|
1257
|
-
const oldState = state.state
|
|
1258
|
-
state.state = 'CLOSED'
|
|
1259
|
-
state.consecutiveFailures = 0
|
|
1260
|
-
state.cooldownMs = this.routerConfig().circuitBreaker.initialCooldownMs
|
|
1261
|
-
state.openedAt = null
|
|
1262
|
-
state.lastError = null
|
|
1263
|
-
state.authError = false
|
|
1399
|
+
const oldState = this.breakers.get(key)?.state
|
|
1400
|
+
this.breakers.markSuccess(key, this.routerConfig().circuitBreaker.initialCooldownMs)
|
|
1264
1401
|
this.quotaExhausted.delete(key)
|
|
1265
1402
|
this.quotaDetails.delete(key)
|
|
1266
|
-
|
|
1267
|
-
|
|
1403
|
+
this.quotaPauses.delete(key)
|
|
1404
|
+
if (oldState && oldState !== 'CLOSED') {
|
|
1405
|
+
this.broadcast('circuit', { model: key, old_state: oldState, new_state: 'CLOSED', cooldown_ms: 0 })
|
|
1268
1406
|
}
|
|
1269
1407
|
if (latencyMs !== null) this.recordProbeResult(key, { ok: true, latencyMs, code: 200 })
|
|
1270
1408
|
}
|
|
1271
1409
|
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
this.
|
|
1410
|
+
// 📖 Single funnel from a failure verdict to health state (v2): circuit
|
|
1411
|
+
// damage, quota pause and probe-window recording all derive from the
|
|
1412
|
+
// classifier's policy instead of ad-hoc per-path bookkeeping.
|
|
1413
|
+
applyFailureVerdict(key, verdict, { detail, statusCode = null, latencyMs = null, meta = {} } = {}) {
|
|
1414
|
+
if (verdict.kind === FAILURE_KINDS.AUTH) {
|
|
1415
|
+
this.breakers.markFailure(key, { ...this.breakerParams(), detail, statusCode, authError: true })
|
|
1416
|
+
this.broadcast('circuit', { model: key, state: 'AUTH_ERROR', reason: detail })
|
|
1417
|
+
} else if (verdict.healthDamage) {
|
|
1418
|
+
const result = this.breakers.markFailure(key, { ...this.breakerParams(), detail, statusCode })
|
|
1419
|
+
this.broadcast('circuit', {
|
|
1281
1420
|
model: key,
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1421
|
+
state: result.state,
|
|
1422
|
+
opened: result.opened,
|
|
1423
|
+
degraded: result.degraded,
|
|
1424
|
+
reason: detail,
|
|
1286
1425
|
})
|
|
1426
|
+
if (result.opened) this.logger.warn(`Circuit opened for ${key}`, { reason: detail })
|
|
1427
|
+
else if (result.degraded) this.logger.warn(`Circuit DEGRADED for ${key}`, { reason: detail })
|
|
1428
|
+
} else {
|
|
1429
|
+
// 📖 No health damage (client-caused 4xx): remember the reason for the
|
|
1430
|
+
// dashboards but never push the breaker toward OPEN.
|
|
1431
|
+
const breaker = this.breakers.ensure(key, this.routerConfig().circuitBreaker.initialCooldownMs)
|
|
1432
|
+
breaker.lastError = detail
|
|
1287
1433
|
}
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
Math.max(router.circuitBreaker.initialCooldownMs, state.cooldownMs * router.circuitBreaker.backoffMultiplier),
|
|
1296
|
-
)
|
|
1297
|
-
this.broadcast('circuit', { model: key, old_state: oldState, new_state: state.state, cooldown_ms: state.cooldownMs })
|
|
1298
|
-
this.logger.warn(`Circuit opened for ${key}`, { reason: detail, cooldown_ms: state.cooldownMs })
|
|
1299
|
-
void sendUsageTelemetry(this.config, {}, {
|
|
1300
|
-
event: 'app_router_circuit_open',
|
|
1301
|
-
mode: 'daemon',
|
|
1302
|
-
properties: {
|
|
1303
|
-
model: key,
|
|
1304
|
-
consecutive_failures: state.consecutiveFailures,
|
|
1305
|
-
cooldown_ms: state.cooldownMs,
|
|
1306
|
-
},
|
|
1307
|
-
})
|
|
1434
|
+
// 📖 Quota bookkeeping: the pause map drives routing skips (Retry-After
|
|
1435
|
+
// aware); the legacy set feeds the all-models-failed error payloads.
|
|
1436
|
+
if ((verdict.quotaPauseMs != null && verdict.quotaPauseMs > 0)
|
|
1437
|
+
|| verdict.kind === FAILURE_KINDS.RATE_LIMIT
|
|
1438
|
+
|| verdict.kind === FAILURE_KINDS.QUOTA
|
|
1439
|
+
|| meta.quotaExhausted) {
|
|
1440
|
+
this.recordQuotaPause(key, verdict.quotaPauseMs || DEFAULT_QUOTA_PAUSE_MS, statusCode, meta)
|
|
1308
1441
|
}
|
|
1309
|
-
|
|
1442
|
+
if (verdict.kind !== FAILURE_KINDS.RATE_LIMIT && verdict.kind !== FAILURE_KINDS.QUOTA) {
|
|
1443
|
+
this.quotaExhausted.delete(key)
|
|
1444
|
+
}
|
|
1445
|
+
this.recordProbeResult(key, { ok: false, latencyMs, code: statusCode || 'ERR', error: detail })
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
recordQuotaPause(key, pauseMs, statusCode, meta = {}) {
|
|
1449
|
+
this.quotaPauses.set(key, {
|
|
1450
|
+
model: key,
|
|
1451
|
+
until: Date.now() + pauseMs,
|
|
1452
|
+
retry_after_ms: pauseMs,
|
|
1453
|
+
status: statusCode,
|
|
1454
|
+
rate_limit_headers: meta.rateLimitHeaders || {},
|
|
1455
|
+
last_seen: nowIso(),
|
|
1456
|
+
})
|
|
1457
|
+
this.quotaExhausted.add(key)
|
|
1458
|
+
this.quotaDetails.set(key, {
|
|
1459
|
+
model: key,
|
|
1460
|
+
status: statusCode,
|
|
1461
|
+
retry_after_ms: pauseMs,
|
|
1462
|
+
rate_limit_headers: meta.rateLimitHeaders || {},
|
|
1463
|
+
last_seen: nowIso(),
|
|
1464
|
+
})
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
markFailure(key, detail, statusCode = null, meta = {}) {
|
|
1468
|
+
// 📖 v2: classify first, then apply the verdict policy. Same call shape
|
|
1469
|
+
// as v1 so probe loops and legacy paths keep working.
|
|
1470
|
+
const verdict = classifyFailure({ status: statusCode, retryAfterMs: meta.retryAfterMs ?? null })
|
|
1471
|
+
this.applyFailureVerdict(key, verdict, { detail, statusCode, meta })
|
|
1310
1472
|
}
|
|
1311
1473
|
|
|
1312
1474
|
quotaDetailsForKeys(keys) {
|
|
@@ -1414,27 +1576,56 @@ class RouterRuntime {
|
|
|
1414
1576
|
// 📖 Circuit-breaker safety is preserved: CLOSED (healthy) models always come
|
|
1415
1577
|
// 📖 before HALF_OPEN (probing after cooldown) models, so a recovering model
|
|
1416
1578
|
// 📖 never pre-empts a known-good one.
|
|
1417
|
-
getRoutingCandidates(set) {
|
|
1579
|
+
getRoutingCandidates(set, { trace = null, blockedProviders = null } = {}) {
|
|
1418
1580
|
const scored = this.scoreCandidates(set)
|
|
1419
|
-
const usable =
|
|
1420
|
-
|
|
1421
|
-
if (
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1581
|
+
const usable = []
|
|
1582
|
+
for (const candidate of scored) {
|
|
1583
|
+
if (blockedProviders?.has(candidate.provider)) {
|
|
1584
|
+
traceSkip(trace, candidate.key, 'provider_blocked')
|
|
1585
|
+
continue
|
|
1586
|
+
}
|
|
1587
|
+
if (!candidate.catalog || candidate.circuit?.stale) {
|
|
1588
|
+
traceSkip(trace, candidate.key, 'stale')
|
|
1589
|
+
continue
|
|
1590
|
+
}
|
|
1591
|
+
if (!candidate.catalog.routeable || candidate.circuit?.unsupported) {
|
|
1592
|
+
traceSkip(trace, candidate.key, 'unsupported')
|
|
1593
|
+
continue
|
|
1594
|
+
}
|
|
1595
|
+
if (candidate.circuit?.authError) {
|
|
1596
|
+
traceSkip(trace, candidate.key, 'auth_error')
|
|
1597
|
+
continue
|
|
1598
|
+
}
|
|
1599
|
+
if (!this.getApiKeyForProvider(candidate.provider)) {
|
|
1600
|
+
traceSkip(trace, candidate.key, 'missing_key')
|
|
1601
|
+
continue
|
|
1602
|
+
}
|
|
1603
|
+
// 📖 v2: a rate-limited model is paused for its Retry-After window and
|
|
1604
|
+
// skipped entirely, instead of being retried until its circuit opens.
|
|
1605
|
+
if (this.quotaPauseActive(candidate.key)) {
|
|
1606
|
+
traceSkip(trace, candidate.key, 'quota_paused')
|
|
1607
|
+
continue
|
|
1608
|
+
}
|
|
1609
|
+
const state = candidate.circuit?.state || 'UNKNOWN'
|
|
1610
|
+
if (state !== 'CLOSED' && state !== 'HALF_OPEN' && state !== 'DEGRADED') {
|
|
1611
|
+
traceSkip(trace, candidate.key, state === 'OPEN' ? 'circuit_open' : 'circuit_state')
|
|
1612
|
+
continue
|
|
1613
|
+
}
|
|
1614
|
+
usable.push(candidate)
|
|
1615
|
+
}
|
|
1426
1616
|
// 📖 New ordering: prioritize by explicit priority first, then by circuit state
|
|
1427
|
-
// 📖 (CLOSED before HALF_OPEN), and finally by health score
|
|
1428
|
-
// 📖 This ensures a higher
|
|
1429
|
-
// 📖 in HALF_OPEN while a lower
|
|
1430
|
-
|
|
1617
|
+
// 📖 (CLOSED before DEGRADED before HALF_OPEN), and finally by health score
|
|
1618
|
+
// 📖 (higher is better). This ensures a higher-priority model is never
|
|
1619
|
+
// 📖 skipped just because it is in HALF_OPEN while a lower-priority CLOSED
|
|
1620
|
+
// 📖 model is available. DEGRADED (failing, not yet tripped) still routes.
|
|
1621
|
+
const stateOrder = { CLOSED: 0, DEGRADED: 1, HALF_OPEN: 2 }
|
|
1431
1622
|
const comparator = (a, b) => {
|
|
1432
1623
|
if (a.priority !== b.priority) return a.priority - b.priority
|
|
1433
1624
|
const aState = a.circuit?.state || 'UNKNOWN'
|
|
1434
1625
|
const bState = b.circuit?.state || 'UNKNOWN'
|
|
1435
1626
|
if (aState !== bState) {
|
|
1436
|
-
const aRank = stateOrder[aState] ??
|
|
1437
|
-
const bRank = stateOrder[bState] ??
|
|
1627
|
+
const aRank = stateOrder[aState] ?? 3
|
|
1628
|
+
const bRank = stateOrder[bState] ?? 3
|
|
1438
1629
|
return aRank - bRank
|
|
1439
1630
|
}
|
|
1440
1631
|
// higher score first
|
|
@@ -1471,11 +1662,16 @@ class RouterRuntime {
|
|
|
1471
1662
|
? 'STALE'
|
|
1472
1663
|
: candidate.circuit?.unsupported
|
|
1473
1664
|
? 'UNSUPPORTED'
|
|
1474
|
-
: candidate.
|
|
1665
|
+
: this.quotaPauseActive(candidate.key)
|
|
1666
|
+
? 'QUOTA_PAUSED'
|
|
1667
|
+
: candidate.circuit?.state || 'UNKNOWN',
|
|
1475
1668
|
score: Number(candidate.score.toFixed(4)),
|
|
1476
1669
|
last_latency_ms: candidate.stats.last?.latencyMs ?? null,
|
|
1477
1670
|
uptime: candidate.stats.uptime,
|
|
1478
1671
|
last_error: candidate.circuit?.lastError || null,
|
|
1672
|
+
quota_paused_until: this.quotaPauses.get(candidate.key)
|
|
1673
|
+
? new Date(this.quotaPauses.get(candidate.key).until).toISOString()
|
|
1674
|
+
: null,
|
|
1479
1675
|
// 📖 AI Latency benchmark results for the Router Dashboard's "Probe all"
|
|
1480
1676
|
// 📖 button. Mirrors the per-model fields already exposed on /api/models
|
|
1481
1677
|
// 📖 so the set list can show live AI latency + TPS after a probe.
|
|
@@ -1484,6 +1680,15 @@ class RouterRuntime {
|
|
|
1484
1680
|
}))
|
|
1485
1681
|
}
|
|
1486
1682
|
|
|
1683
|
+
// 📖 v2: breaker census for dashboards (CLOSED / DEGRADED / OPEN / ...).
|
|
1684
|
+
getModelStates(set = this.getSet()) {
|
|
1685
|
+
const counts = { CLOSED: 0, DEGRADED: 0, OPEN: 0, HALF_OPEN: 0, AUTH_ERROR: 0, QUOTA_PAUSED: 0 }
|
|
1686
|
+
for (const model of this.getModelHealth(set || { models: [] })) {
|
|
1687
|
+
if (counts[model.state] !== undefined) counts[model.state] += 1
|
|
1688
|
+
}
|
|
1689
|
+
return counts
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1487
1692
|
findBestModelForProviderInSources(providerKey) {
|
|
1488
1693
|
const source = sources[providerKey]
|
|
1489
1694
|
if (!source || !Array.isArray(source.models)) return null
|
|
@@ -1675,6 +1880,24 @@ class RouterRuntime {
|
|
|
1675
1880
|
configPath: CONFIG_PATH,
|
|
1676
1881
|
tokenStatsPath: ROUTER_TOKENS_PATH,
|
|
1677
1882
|
logPath: ROUTER_LOG_PATH,
|
|
1883
|
+
// 📖 v2 engine: failover knobs, live breaker census, quota pauses and
|
|
1884
|
+
// the persisted request-history aggregates for dashboards.
|
|
1885
|
+
router: 'v2',
|
|
1886
|
+
failover: {
|
|
1887
|
+
maxRetries: router.failover.maxRetries,
|
|
1888
|
+
requestTimeoutMs: router.failover.requestTimeoutMs,
|
|
1889
|
+
bodyReadTimeoutMs: this.failoverSettings().bodyReadTimeoutMs,
|
|
1890
|
+
totalBudgetMs: this.failoverSettings().totalBudgetMs,
|
|
1891
|
+
contentValidation: this.failoverSettings().contentValidation,
|
|
1892
|
+
lastResortModel: this.failoverSettings().lastResortModel?.key || null,
|
|
1893
|
+
},
|
|
1894
|
+
modelStates: this.getModelStates(activeSet),
|
|
1895
|
+
quotaPauses: this.quotaPausesForKeys([...this.quotaPauses.keys()]).map((p) => ({
|
|
1896
|
+
model: p.model,
|
|
1897
|
+
until: new Date(p.until).toISOString(),
|
|
1898
|
+
retry_after_ms: p.retry_after_ms,
|
|
1899
|
+
})),
|
|
1900
|
+
history: this.history.stats(),
|
|
1678
1901
|
// 📖 Probe-cache (t1): live aggregates from the persistent probe-cache.
|
|
1679
1902
|
// 📖 Surfaced so the Web Dashboard + CLI can show cache hit rate + how many
|
|
1680
1903
|
// 📖 broken models are currently hidden. Refreshed every /stats call.
|
|
@@ -1718,6 +1941,9 @@ class RouterRuntime {
|
|
|
1718
1941
|
completed: this.webGlobalBenchmarkCompleted || 0,
|
|
1719
1942
|
},
|
|
1720
1943
|
requestLog: this.requestLog.slice(0, 20),
|
|
1944
|
+
// 📖 v2 engine: persisted breakers + recent decision traces.
|
|
1945
|
+
breakers: this.breakers.snapshot(),
|
|
1946
|
+
traces: this.recentTraces.slice(-20).map((trace) => this.historyEntryFromTrace(trace)),
|
|
1721
1947
|
activeRequests: Array.from(this.activeRequests.values()).map(r => ({
|
|
1722
1948
|
requestId: r.requestId,
|
|
1723
1949
|
at: r.at,
|
|
@@ -2081,119 +2307,245 @@ class RouterRuntime {
|
|
|
2081
2307
|
this.probeWatchdog.unref?.()
|
|
2082
2308
|
}
|
|
2083
2309
|
|
|
2084
|
-
|
|
2085
|
-
|
|
2310
|
+
// 📖 Shared admission helpers for the v2 engine (decision traces, pinned
|
|
2311
|
+
// models, protocol-aware errors). Everything below keeps the v1 call
|
|
2312
|
+
// shapes so the whole v1 surface (sets API, web dashboard, playground)
|
|
2313
|
+
// keeps working on top of the hardened engine.
|
|
2314
|
+
|
|
2315
|
+
resolvePinnedCandidate(pinned) {
|
|
2316
|
+
const key = modelKey(pinned.provider, pinned.model)
|
|
2317
|
+
const catalog = this.modelCatalog.get(key)
|
|
2318
|
+
if (!catalog) return { error: `Unknown model: ${key}` }
|
|
2319
|
+
if (!isRouteableProvider(pinned.provider, sources)) return { error: `Provider is not routeable: ${pinned.provider}` }
|
|
2320
|
+
if (!this.getApiKeyForProvider(pinned.provider)) return { error: `No API key configured for ${pinned.provider}` }
|
|
2321
|
+
const breaker = this.breakers.get(key) || {}
|
|
2322
|
+
return {
|
|
2323
|
+
candidate: {
|
|
2324
|
+
provider: pinned.provider,
|
|
2325
|
+
model: pinned.model,
|
|
2326
|
+
priority: 1,
|
|
2327
|
+
key,
|
|
2328
|
+
score: 0,
|
|
2329
|
+
stats: this.getWindowStats(key),
|
|
2330
|
+
circuit: breaker,
|
|
2331
|
+
catalog,
|
|
2332
|
+
},
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
rememberTrace(trace) {
|
|
2337
|
+
this.recentTraces.push(trace)
|
|
2338
|
+
while (this.recentTraces.length > 50) this.recentTraces.shift()
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
historyEntryFromTrace(trace, { stream = false, set = null } = {}) {
|
|
2342
|
+
return {
|
|
2343
|
+
request_id: trace.request_id,
|
|
2344
|
+
at: trace.at,
|
|
2345
|
+
set: set || trace.set,
|
|
2346
|
+
protocol: trace.protocol,
|
|
2347
|
+
model_requested: trace.model_requested,
|
|
2348
|
+
pinned_model: trace.pinned_model,
|
|
2349
|
+
served_model: trace.served_model,
|
|
2350
|
+
outcome: trace.outcome,
|
|
2351
|
+
attempts: trace.attempts,
|
|
2352
|
+
skipped: trace.skipped,
|
|
2353
|
+
wall_ms: trace.wall_ms,
|
|
2354
|
+
tokens: trace.tokens,
|
|
2355
|
+
stream,
|
|
2356
|
+
last_resort_used: trace.last_resort_used,
|
|
2357
|
+
summary: traceSummary(trace),
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
decisionHeaders(trace) {
|
|
2362
|
+
const lastModel = trace.attempts.length > 0 ? trace.attempts[trace.attempts.length - 1].model : 'none'
|
|
2363
|
+
return {
|
|
2364
|
+
'x-fcm-router-model': trace.served_model || lastModel,
|
|
2365
|
+
'x-fcm-v2-model': trace.served_model || lastModel,
|
|
2366
|
+
'x-fcm-v2-attempts': String(trace.attempts.length),
|
|
2367
|
+
'x-fcm-v2-decision': decisionHeaderValue(trace),
|
|
2368
|
+
'x-request-id': trace.request_id,
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
sendProtocolError(res, protocol, statusCode, message, requestId, extra = {}) {
|
|
2373
|
+
if (protocol === 'anthropic') {
|
|
2374
|
+
sendJson(res, statusCode, anthropicErrorPayload(anthropicErrorTypeForStatus(statusCode), message), {
|
|
2375
|
+
'x-request-id': requestId,
|
|
2376
|
+
...extra.headers,
|
|
2377
|
+
})
|
|
2378
|
+
return
|
|
2379
|
+
}
|
|
2380
|
+
sendError(res, statusCode, message, 'service_unavailable', extra.code || 'router_error', requestId, extra.payload)
|
|
2381
|
+
}
|
|
2382
|
+
|
|
2383
|
+
retryAfterHeaders() {
|
|
2384
|
+
const pauses = this.quotaPausesForKeys([...this.quotaPauses.keys()])
|
|
2385
|
+
if (pauses.length === 0) return {}
|
|
2386
|
+
const maxUntil = Math.max(...pauses.map((p) => p.until))
|
|
2387
|
+
const seconds = Math.max(1, Math.ceil((maxUntil - Date.now()) / 1000))
|
|
2388
|
+
return { 'Retry-After': String(Math.min(seconds, 900)) }
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
buildUpstreamBody(body, candidate, stream) {
|
|
2392
|
+
const bodyWithPrePrompt = applyPrePromptToBody(body, this.routerConfig().prePrompt)
|
|
2393
|
+
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2394
|
+
const upstreamBody = {
|
|
2395
|
+
...bodyNormalized,
|
|
2396
|
+
model: getApiModelId(candidate.provider, candidate.model),
|
|
2397
|
+
stream,
|
|
2398
|
+
}
|
|
2399
|
+
// 📖 Some providers/models fail if we send custom internal params.
|
|
2400
|
+
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2401
|
+
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2402
|
+
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2403
|
+
return upstreamBody
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
async routeRequest({ req, res, body, setName, requestId, protocol = 'openai', anthropicModelName = null }) {
|
|
2407
|
+
const trace = createDecisionTrace({
|
|
2086
2408
|
requestId,
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
attempts: 0,
|
|
2091
|
-
tokens: 0,
|
|
2092
|
-
stalled: false
|
|
2409
|
+
set: setName || this.routerConfig().activeSet,
|
|
2410
|
+
protocol,
|
|
2411
|
+
modelRequested: body?.model || 'fcm',
|
|
2093
2412
|
})
|
|
2413
|
+
const started = Date.now()
|
|
2414
|
+
|
|
2415
|
+
// 📖 v2 lifecycle fix: every rejection guard runs BEFORE the
|
|
2416
|
+
// active-request entry exists, and the entry only lives inside the
|
|
2417
|
+
// try/finally. v1 created it first, so each rejected request leaked a
|
|
2418
|
+
// ghost "active request" into /stats until restart.
|
|
2094
2419
|
if (this.shuttingDown) {
|
|
2095
|
-
|
|
2420
|
+
this.sendProtocolError(res, protocol, 503, 'Daemon is shutting down', requestId)
|
|
2421
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2422
|
+
this.rememberTrace(trace)
|
|
2096
2423
|
return
|
|
2097
2424
|
}
|
|
2098
2425
|
if (this.inFlight >= MAX_CONCURRENT_REQUESTS) {
|
|
2099
2426
|
sendError(res, 503, 'Router overloaded, too many concurrent requests', 'service_unavailable', 'router_overloaded', requestId)
|
|
2427
|
+
finishTrace(trace, { outcome: 'overloaded', wallMs: Date.now() - started })
|
|
2428
|
+
this.rememberTrace(trace)
|
|
2100
2429
|
return
|
|
2101
2430
|
}
|
|
2102
2431
|
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
|
2103
2432
|
sendError(res, 400, 'Request body must be a JSON object', 'invalid_request_error', 'invalid_json_object', requestId)
|
|
2433
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2434
|
+
this.rememberTrace(trace)
|
|
2104
2435
|
return
|
|
2105
2436
|
}
|
|
2106
2437
|
if (typeof body.model !== 'string' || !body.model.trim()) {
|
|
2107
2438
|
sendError(res, 400, 'Missing required field: model', 'invalid_request_error', 'missing_model', requestId)
|
|
2439
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2440
|
+
this.rememberTrace(trace)
|
|
2108
2441
|
return
|
|
2109
2442
|
}
|
|
2110
2443
|
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2444
|
+
// 📖 Model spec: `fcm` (active set), `fcm:<set>`, or `fcm:@provider/model`
|
|
2445
|
+
// (pinned single-model request, failover disabled - used by the
|
|
2446
|
+
// test-via-router actions to exercise ONE model through the full chain).
|
|
2447
|
+
const spec = parseFcmModel(body.model)
|
|
2448
|
+
let set = null
|
|
2449
|
+
let pinned = null
|
|
2450
|
+
if (spec.kind === 'pinned') {
|
|
2451
|
+
pinned = spec.pinned
|
|
2452
|
+
set = this.getSet(null)
|
|
2453
|
+
if (!set) {
|
|
2454
|
+
this.sendProtocolError(res, protocol, 503, 'No active router set', requestId, { code: 'set_not_found' })
|
|
2455
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2456
|
+
this.rememberTrace(trace)
|
|
2457
|
+
return
|
|
2458
|
+
}
|
|
2459
|
+
trace.pinned_model = `${pinned.provider}/${pinned.model}`
|
|
2460
|
+
} else {
|
|
2461
|
+
const requestedSetName = spec.kind === 'set' ? spec.set : setName
|
|
2462
|
+
set = this.getSet(requestedSetName)
|
|
2463
|
+
if (!set) {
|
|
2464
|
+
sendError(res, 404, `Router set not found: ${requestedSetName || this.routerConfig().activeSet}`, 'invalid_request_error', 'set_not_found', requestId)
|
|
2465
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2466
|
+
this.rememberTrace(trace)
|
|
2467
|
+
return
|
|
2468
|
+
}
|
|
2115
2469
|
}
|
|
2116
2470
|
|
|
2117
|
-
const
|
|
2118
|
-
const
|
|
2119
|
-
const
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2471
|
+
const settings = this.failoverSettings()
|
|
2472
|
+
const maxAttempts = pinned ? 1 : Math.min(1 + this.routerConfig().failover.maxRetries, 6)
|
|
2473
|
+
const deadline = Date.now() + settings.totalBudgetMs
|
|
2474
|
+
const stream = body.stream === true
|
|
2475
|
+
|
|
2476
|
+
this.inFlight += 1
|
|
2477
|
+
const activeReq = {
|
|
2478
|
+
requestId,
|
|
2479
|
+
at: Date.now(),
|
|
2480
|
+
model: body.model,
|
|
2481
|
+
current_model: null,
|
|
2482
|
+
attempts: 0,
|
|
2483
|
+
tokens: 0,
|
|
2484
|
+
stalled: false,
|
|
2485
|
+
last_activity_at: Date.now(),
|
|
2486
|
+
}
|
|
2487
|
+
this.activeRequests.set(requestId, activeReq)
|
|
2488
|
+
try {
|
|
2489
|
+
let candidates
|
|
2490
|
+
if (pinned) {
|
|
2491
|
+
// 📖 Pinned tests deliberately bypass availability pre-skips (circuit
|
|
2492
|
+
// OPEN, quota pause): the point is a genuine attempt that feeds real
|
|
2493
|
+
// health data back into the breakers.
|
|
2494
|
+
const resolved = this.resolvePinnedCandidate(pinned)
|
|
2495
|
+
if (resolved.error) {
|
|
2496
|
+
this.sendProtocolError(res, protocol, 400, resolved.error, requestId, { code: 'invalid_model' })
|
|
2497
|
+
finishTrace(trace, { outcome: 'rejected', wallMs: Date.now() - started })
|
|
2498
|
+
return
|
|
2143
2499
|
}
|
|
2500
|
+
candidates = [resolved.candidate]
|
|
2501
|
+
} else {
|
|
2502
|
+
candidates = this.getRoutingCandidates(set, { trace })
|
|
2144
2503
|
}
|
|
2145
2504
|
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2151
|
-
model_health: health,
|
|
2152
|
-
})
|
|
2153
|
-
void sendUsageTelemetry(this.config, {}, {
|
|
2154
|
-
event: 'app_router_all_down',
|
|
2155
|
-
mode: 'daemon',
|
|
2156
|
-
properties: {
|
|
2157
|
-
set_name: set.name,
|
|
2158
|
-
models_tried: [],
|
|
2159
|
-
quota_exhausted_count: quotaExhausted.length,
|
|
2160
|
-
},
|
|
2161
|
-
})
|
|
2162
|
-
return
|
|
2163
|
-
}
|
|
2505
|
+
if (candidates.length === 0) {
|
|
2506
|
+
this.sendAllModelsUnavailable(res, trace, set, requestId, protocol)
|
|
2507
|
+
return
|
|
2508
|
+
}
|
|
2164
2509
|
|
|
2165
|
-
this.inFlight += 1
|
|
2166
|
-
try {
|
|
2167
2510
|
const tried = []
|
|
2511
|
+
const failedKinds = []
|
|
2168
2512
|
const blockedProviders = new Set()
|
|
2169
2513
|
let attemptIndex = 0
|
|
2170
|
-
// 📖 attemptChain is a private copy of the routing order: on a family
|
|
2171
|
-
// failover (t8) the same-family candidate is swapped in as the NEXT
|
|
2172
|
-
// attempt, so the iteration order itself follows the two-stage policy.
|
|
2173
2514
|
const attemptChain = candidates.slice()
|
|
2515
|
+
|
|
2174
2516
|
for (let index = 0; index < attemptChain.length && attemptIndex < maxAttempts; index += 1) {
|
|
2517
|
+
if (Date.now() > deadline) {
|
|
2518
|
+
this.logger.warn('Request retry budget exhausted; failing over to error', { request_id: requestId })
|
|
2519
|
+
break
|
|
2520
|
+
}
|
|
2175
2521
|
const candidate = attemptChain[index]
|
|
2176
2522
|
if (blockedProviders.has(candidate.provider)) continue
|
|
2177
2523
|
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
activeReq.current_model = candidate.key
|
|
2181
|
-
activeReq.attempts = attemptIndex + 1
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2524
|
+
activeReq.current_model = candidate.key
|
|
2525
|
+
activeReq.attempts = attemptIndex + 1
|
|
2184
2526
|
tried.push(candidate.key)
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2527
|
+
traceAttempt(trace, candidate.key, { status: null })
|
|
2528
|
+
|
|
2529
|
+
const result = stream
|
|
2530
|
+
? await this.proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, anthropicModelName })
|
|
2531
|
+
: await this.proxyJsonRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace })
|
|
2532
|
+
|
|
2533
|
+
trace.attempts[trace.attempts.length - 1] = {
|
|
2534
|
+
...trace.attempts[trace.attempts.length - 1],
|
|
2535
|
+
model: candidate.key,
|
|
2536
|
+
status: result.status ?? null,
|
|
2537
|
+
latency_ms: result.latencyMs ?? null,
|
|
2538
|
+
error: result.reason || null,
|
|
2539
|
+
at: new Date().toISOString(),
|
|
2540
|
+
}
|
|
2541
|
+
if (result.verdict) failedKinds.push(result.verdict.kind)
|
|
2188
2542
|
if (result.done) return
|
|
2189
2543
|
attemptIndex += 1
|
|
2190
|
-
if (result.
|
|
2544
|
+
if (result.verdict?.blockProvider) blockedProviders.add(candidate.provider)
|
|
2545
|
+
|
|
2191
2546
|
if (result.failoverToNext && attemptIndex < maxAttempts) {
|
|
2192
2547
|
// 📖 Two-stage failover (t8): prefer a healthy model of the SAME
|
|
2193
|
-
// family on another provider
|
|
2194
|
-
// Together) so the user's output style doesn't change mid-request.
|
|
2195
|
-
// Falls back to the historical set-order pick, and is disabled
|
|
2196
|
-
// per-set via familyFailover: false.
|
|
2548
|
+
// family on another provider so output style stays consistent.
|
|
2197
2549
|
const pick = pickNextCandidate({
|
|
2198
2550
|
candidates: attemptChain,
|
|
2199
2551
|
failedCandidate: candidate,
|
|
@@ -2202,9 +2554,6 @@ class RouterRuntime {
|
|
|
2202
2554
|
familyFailover: set.familyFailover !== false,
|
|
2203
2555
|
})
|
|
2204
2556
|
const next = pick?.candidate || null
|
|
2205
|
-
// 📖 Reorder the remaining chain so `next` is genuinely the following
|
|
2206
|
-
// attempt. With set-order picks this is already the case (or the
|
|
2207
|
-
// skipped entries are blocked anyway), so behaviour is unchanged.
|
|
2208
2557
|
if (next && attemptChain[index + 1] !== next) {
|
|
2209
2558
|
const nextIndex = attemptChain.indexOf(next)
|
|
2210
2559
|
if (nextIndex > index) {
|
|
@@ -2212,6 +2561,8 @@ class RouterRuntime {
|
|
|
2212
2561
|
attemptChain.splice(index + 1, 0, next)
|
|
2213
2562
|
}
|
|
2214
2563
|
}
|
|
2564
|
+
// 📖 t8: stamp the reason ('family_failover' | 'set_order') on the
|
|
2565
|
+
// active request so every log entry of the next attempt carries it.
|
|
2215
2566
|
const activeReqForReason = this.activeRequests.get(requestId)
|
|
2216
2567
|
if (next && activeReqForReason) activeReqForReason.failoverReason = pick.reason
|
|
2217
2568
|
this.logger.warn(
|
|
@@ -2233,75 +2584,123 @@ class RouterRuntime {
|
|
|
2233
2584
|
}
|
|
2234
2585
|
}
|
|
2235
2586
|
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2587
|
+
// 📖 Last-resort escape hatch (v2): one final configured model outside
|
|
2588
|
+
// the rotation gets a single shot before the client sees an error.
|
|
2589
|
+
const lastResort = settings.lastResortModel
|
|
2590
|
+
if (!pinned && lastResort && !tried.includes(lastResort.key) && !blockedProviders.has(lastResort.provider) && Date.now() <= deadline) {
|
|
2591
|
+
const resolved = this.resolvePinnedCandidate({ provider: lastResort.provider, model: lastResort.model })
|
|
2592
|
+
if (resolved.candidate) {
|
|
2593
|
+
this.logger.warn(`All candidates failed; trying last-resort model ${lastResort.key}`, { request_id: requestId })
|
|
2594
|
+
trace.last_resort_used = true
|
|
2595
|
+
activeReq.current_model = lastResort.key
|
|
2596
|
+
tried.push(lastResort.key)
|
|
2597
|
+
traceAttempt(trace, lastResort.key, { status: null })
|
|
2598
|
+
const result = stream
|
|
2599
|
+
? await this.proxyStreamingRequest({ req, res, body, candidate: resolved.candidate, requestId, attemptIndex, protocol, trace, isLastResort: true, anthropicModelName })
|
|
2600
|
+
: await this.proxyJsonRequest({ req, res, body, candidate: resolved.candidate, requestId, attemptIndex, protocol, trace, isLastResort: true })
|
|
2601
|
+
trace.attempts[trace.attempts.length - 1] = {
|
|
2602
|
+
...trace.attempts[trace.attempts.length - 1],
|
|
2603
|
+
model: lastResort.key,
|
|
2604
|
+
status: result.status ?? null,
|
|
2605
|
+
latency_ms: result.latencyMs ?? null,
|
|
2606
|
+
error: result.reason || null,
|
|
2607
|
+
at: new Date().toISOString(),
|
|
2608
|
+
}
|
|
2609
|
+
if (result.verdict) failedKinds.push(result.verdict.kind)
|
|
2610
|
+
if (result.done) return
|
|
2260
2611
|
}
|
|
2261
2612
|
}
|
|
2262
2613
|
|
|
2263
|
-
|
|
2264
|
-
set: set.name,
|
|
2265
|
-
models_tried: tried,
|
|
2266
|
-
quota_exhausted: quotaExhausted,
|
|
2267
|
-
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2268
|
-
})
|
|
2614
|
+
this.sendAllModelsFailed(res, trace, set, requestId, protocol, { tried, failedKinds, stream })
|
|
2269
2615
|
} finally {
|
|
2270
2616
|
this.inFlight -= 1
|
|
2271
2617
|
this.activeRequests.delete(requestId)
|
|
2618
|
+
const wallMs = Date.now() - started
|
|
2619
|
+
finishTrace(trace, {
|
|
2620
|
+
outcome: trace.outcome || (trace.served_model ? 'served' : 'all_failed'),
|
|
2621
|
+
wallMs,
|
|
2622
|
+
servedModel: trace.served_model,
|
|
2623
|
+
lastResort: trace.last_resort_used,
|
|
2624
|
+
tokens: activeReq.tokens,
|
|
2625
|
+
})
|
|
2626
|
+
this.rememberTrace(trace)
|
|
2627
|
+
this.history.append(this.historyEntryFromTrace(trace, { stream, set: set?.name || null }))
|
|
2272
2628
|
}
|
|
2273
2629
|
}
|
|
2274
2630
|
|
|
2275
|
-
|
|
2631
|
+
sendAllModelsUnavailable(res, trace, set, requestId, protocol) {
|
|
2632
|
+
const health = this.getModelHealth(set)
|
|
2633
|
+
const quotaExhausted = [...this.quotaExhausted].filter((key) => set.models.some((model) => modelKey(model.provider, model.model) === key))
|
|
2634
|
+
const allAuthError = health.length > 0 && health.every((h) => h.state === 'AUTH_ERROR')
|
|
2635
|
+
const allPaused = health.length > 0 && health.every((h) => h.state === 'QUOTA_PAUSED')
|
|
2636
|
+
const allStaleOrUnsupported = health.length > 0 && health.every((h) => h.state === 'STALE' || h.state === 'UNSUPPORTED')
|
|
2637
|
+
let statusCode = 503
|
|
2638
|
+
if (allAuthError) statusCode = 401
|
|
2639
|
+
else if (allPaused || (quotaExhausted.length === health.length && health.length > 0)) statusCode = 429
|
|
2640
|
+
else if (allStaleOrUnsupported) statusCode = 400
|
|
2641
|
+
const headers = statusCode === 429 ? this.retryAfterHeaders() : {}
|
|
2642
|
+
this.sendProtocolError(res, protocol, statusCode,
|
|
2643
|
+
`All models in set are unavailable: ${set.name}`, requestId,
|
|
2644
|
+
{
|
|
2645
|
+
code: statusCode === 401 ? 'invalid_api_key' : statusCode === 429 ? 'insufficient_quota' : 'all_models_unavailable',
|
|
2646
|
+
headers,
|
|
2647
|
+
payload: { set: set.name, models_tried: [], quota_exhausted: quotaExhausted, quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted), model_health: health },
|
|
2648
|
+
})
|
|
2649
|
+
void sendUsageTelemetry(this.config, {}, {
|
|
2650
|
+
event: 'app_router_all_down',
|
|
2651
|
+
mode: 'daemon',
|
|
2652
|
+
properties: { set_name: set.name, models_tried: [], quota_exhausted_count: quotaExhausted.length },
|
|
2653
|
+
})
|
|
2654
|
+
finishTrace(trace, { outcome: 'all_failed', wallMs: Date.now() - new Date(trace.at).getTime() })
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
sendAllModelsFailed(res, trace, set, requestId, protocol, { tried, failedKinds, stream }) {
|
|
2658
|
+
// 📖 Status refinement by dominant failure kind (v2): an all-auth failure
|
|
2659
|
+
// is 401 for the client, all-quota is 429 (+ Retry-After), all-
|
|
2660
|
+
// invalid-request means the PAYLOAD is the problem (400).
|
|
2661
|
+
const kinds = failedKinds.length > 0 ? failedKinds : ['unknown']
|
|
2662
|
+
const allSame = kinds.every((k) => k === kinds[0])
|
|
2663
|
+
const statusCode = allSame ? clientStatusForKind(kinds[0]) : 503
|
|
2664
|
+
const headers = statusCode === 429 ? this.retryAfterHeaders() : {}
|
|
2665
|
+
const quotaExhausted = [...this.quotaExhausted].filter((key) => tried.includes(key))
|
|
2666
|
+
// 📖 Keep the v1 client-facing error codes: quota exhaustion is
|
|
2667
|
+
// 'insufficient_quota' (OpenAI convention), auth is 'invalid_api_key'.
|
|
2668
|
+
const clientCode = allSame
|
|
2669
|
+
? (kinds[0] === FAILURE_KINDS.RATE_LIMIT || kinds[0] === FAILURE_KINDS.QUOTA
|
|
2670
|
+
? 'insufficient_quota'
|
|
2671
|
+
: kinds[0] === FAILURE_KINDS.AUTH ? 'invalid_api_key' : kinds[0])
|
|
2672
|
+
: 'all_models_failed'
|
|
2673
|
+
this.sendProtocolError(res, protocol, statusCode,
|
|
2674
|
+
`All routed models failed for set: ${set.name}`, requestId,
|
|
2675
|
+
{
|
|
2676
|
+
code: clientCode,
|
|
2677
|
+
headers,
|
|
2678
|
+
payload: {
|
|
2679
|
+
set: set.name,
|
|
2680
|
+
models_tried: tried,
|
|
2681
|
+
failure_kinds: kinds,
|
|
2682
|
+
quota_exhausted: quotaExhausted,
|
|
2683
|
+
quota_exhausted_details: this.quotaDetailsForKeys(quotaExhausted),
|
|
2684
|
+
stream,
|
|
2685
|
+
},
|
|
2686
|
+
})
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2689
|
+
async proxyJsonRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, isLastResort = false }) {
|
|
2276
2690
|
const key = candidate.key
|
|
2277
2691
|
const apiKey = this.getApiKeyForProvider(candidate.provider)
|
|
2278
|
-
// 📖 Guard: bail early if provider URL cannot be resolved
|
|
2279
2692
|
const providerUrl = resolveProviderUrl(candidate.provider)
|
|
2280
2693
|
if (!providerUrl) {
|
|
2281
|
-
|
|
2694
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.PROVIDER_URL })
|
|
2695
|
+
this.applyFailureVerdict(key, verdict, { detail: 'provider URL unresolvable' })
|
|
2282
2696
|
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'provider_url_unresolvable' })
|
|
2283
|
-
return { done: false, failoverToNext: true, reason:
|
|
2697
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2284
2698
|
}
|
|
2285
2699
|
const controller = new AbortController()
|
|
2286
2700
|
const timeout = setTimeout(() => controller.abort(), this.routerConfig().failover.requestTimeoutMs)
|
|
2701
|
+
const settings = this.failoverSettings()
|
|
2287
2702
|
const started = performance.now()
|
|
2288
|
-
|
|
2289
|
-
// 📖 curl, custom Playground) gets the FCM persona without any client
|
|
2290
|
-
// 📖 change. Non-streaming path.
|
|
2291
|
-
const bodyWithPrePrompt = applyPrePromptToBody(body, this.routerConfig().prePrompt)
|
|
2292
|
-
// 📖 Apply per-provider schema normalization (GLM, Mistral, Codestral).
|
|
2293
|
-
// 📖 Returns the body unchanged for providers without a registered normalizer.
|
|
2294
|
-
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2295
|
-
const upstreamBody = {
|
|
2296
|
-
...bodyNormalized,
|
|
2297
|
-
model: getApiModelId(candidate.provider, candidate.model),
|
|
2298
|
-
stream: false,
|
|
2299
|
-
}
|
|
2300
|
-
// 📖 Some providers/models fail if we send custom internal params, so strip them
|
|
2301
|
-
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2302
|
-
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2303
|
-
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2304
|
-
|
|
2703
|
+
const upstreamBody = this.buildUpstreamBody(body, candidate, false)
|
|
2305
2704
|
const clientAbort = attachClientAbort(req, res, controller)
|
|
2306
2705
|
try {
|
|
2307
2706
|
const response = await fetch(providerUrl, {
|
|
@@ -2315,46 +2714,52 @@ class RouterRuntime {
|
|
|
2315
2714
|
})
|
|
2316
2715
|
clearTimeout(timeout)
|
|
2317
2716
|
const latencyMs = Math.round(performance.now() - started)
|
|
2318
|
-
const text = await response.
|
|
2717
|
+
const text = await readBodyWithTimeout(response, controller, settings.bodyReadTimeoutMs)
|
|
2319
2718
|
const upstreamMeta = buildUpstreamMeta(response, text, candidate.provider)
|
|
2320
2719
|
|
|
2321
2720
|
if (isLikelyHtmlResponse(response.headers, text)) {
|
|
2322
|
-
|
|
2323
|
-
this.
|
|
2721
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2722
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2723
|
+
this.recordRouterError('upstream_html_maintenance', requestId, { model: key })
|
|
2324
2724
|
this.addRequestLog({ request_id: requestId, model: key, status: 503, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_html_maintenance' })
|
|
2325
|
-
return { done: false, failoverToNext: true, reason:
|
|
2725
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2326
2726
|
}
|
|
2327
2727
|
|
|
2328
2728
|
if (response.ok) {
|
|
2329
2729
|
const parsed = parseJsonResult(text)
|
|
2330
2730
|
if (!parsed.ok || !parsed.value || typeof parsed.value !== 'object') {
|
|
2331
|
-
|
|
2332
|
-
this.
|
|
2731
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.INVALID_JSON })
|
|
2732
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream invalid json', statusCode: 502, meta: upstreamMeta })
|
|
2733
|
+
this.recordRouterError('upstream_invalid_json', requestId, { model: key })
|
|
2333
2734
|
this.addRequestLog({ request_id: requestId, model: key, status: 502, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_invalid_json' })
|
|
2334
|
-
return { done: false, failoverToNext: true, reason:
|
|
2735
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 502, latencyMs }
|
|
2736
|
+
}
|
|
2737
|
+
// 📖 THE v2 content gate: a 200 only counts as success when the
|
|
2738
|
+
// payload holds real content. Empty choices, embedded error objects
|
|
2739
|
+
// and content-less answers fail over (v1 served those as successes).
|
|
2740
|
+
if (settings.contentValidation !== 'off') {
|
|
2741
|
+
const gate = validateChatCompletionPayload(parsed.value, { mode: settings.contentValidation })
|
|
2742
|
+
if (!gate.ok) {
|
|
2743
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS[gateReasonToKind(gate.reason)] || FAILURE_KINDS.INVALID_JSON })
|
|
2744
|
+
this.applyFailureVerdict(key, verdict, { detail: `gate: ${gate.reason}${gate.detail ? ` (${gate.detail})` : ''}`, statusCode: 200, meta: upstreamMeta })
|
|
2745
|
+
this.recordRouterError('gate_reject', requestId, { model: key, reason: gate.reason })
|
|
2746
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 200, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `gate_${gate.reason}` })
|
|
2747
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2748
|
+
}
|
|
2335
2749
|
}
|
|
2750
|
+
|
|
2336
2751
|
this.markSuccess(key, latencyMs)
|
|
2337
2752
|
const usage = extractUsage(parsed.value)
|
|
2338
2753
|
this.tokenTracker.record(candidate.provider, candidate.model, usage)
|
|
2339
|
-
|
|
2340
|
-
// 📖 real-world score can rank models by what *actually* works.
|
|
2341
|
-
this.recordRuntimeCall({
|
|
2342
|
-
providerKey: candidate.provider,
|
|
2343
|
-
modelId: candidate.model,
|
|
2344
|
-
success: true,
|
|
2345
|
-
latencyMs,
|
|
2346
|
-
usage,
|
|
2347
|
-
})
|
|
2754
|
+
this.recordRuntimeCall({ providerKey: candidate.provider, modelId: candidate.model, success: true, latencyMs, usage })
|
|
2348
2755
|
this.totalRequestsRouted += 1
|
|
2349
|
-
|
|
2756
|
+
trace.served_model = key
|
|
2757
|
+
trace.tokens = usage?.total_tokens || 0
|
|
2350
2758
|
if (this.totalRequestsRouted % 10 === 0) {
|
|
2351
2759
|
void sendUsageTelemetry(this.config, {}, {
|
|
2352
2760
|
event: 'app_router_use',
|
|
2353
2761
|
mode: 'daemon',
|
|
2354
|
-
properties: {
|
|
2355
|
-
total_requests: this.totalRequestsRouted,
|
|
2356
|
-
active_set: this.routerConfig().activeSet,
|
|
2357
|
-
},
|
|
2762
|
+
properties: { total_requests: this.totalRequestsRouted, active_set: this.routerConfig().activeSet },
|
|
2358
2763
|
})
|
|
2359
2764
|
}
|
|
2360
2765
|
this.addRequestLog({
|
|
@@ -2366,121 +2771,110 @@ class RouterRuntime {
|
|
|
2366
2771
|
failover: attemptIndex > 0,
|
|
2367
2772
|
})
|
|
2368
2773
|
this.logger.info(`Routed to ${key} - ${latencyMs}ms`, { request_id: requestId, status: response.status })
|
|
2369
|
-
// 📖
|
|
2774
|
+
// 📖 Record the winning attempt BEFORE the response head is written so
|
|
2775
|
+
// the decision header shows the final status of this model.
|
|
2776
|
+
const winningAttempt = trace.attempts[trace.attempts.length - 1]
|
|
2777
|
+
if (winningAttempt) {
|
|
2778
|
+
winningAttempt.status = response.status
|
|
2779
|
+
winningAttempt.latency_ms = latencyMs
|
|
2780
|
+
}
|
|
2781
|
+
// 📖 Fix #124: normalize malformed tool_calls (finish_reason
|
|
2782
|
+
// tool_calls without a tool_calls array).
|
|
2370
2783
|
let responseText = text
|
|
2371
2784
|
try {
|
|
2372
|
-
if (
|
|
2785
|
+
if (protocol === 'anthropic') {
|
|
2786
|
+
const translated = translateOpenAIToAnthropicResponse(parsed.value, { model: key })
|
|
2787
|
+
responseText = translated.ok ? JSON.stringify(translated.body) : text
|
|
2788
|
+
} else if (normalizeToolCallsResponse(parsed.value)) {
|
|
2373
2789
|
responseText = JSON.stringify(parsed.value)
|
|
2374
2790
|
}
|
|
2375
2791
|
} catch {}
|
|
2376
2792
|
if (!res.writableEnded) {
|
|
2377
2793
|
res.writeHead(response.status, {
|
|
2378
2794
|
...headerEntries(response.headers),
|
|
2379
|
-
|
|
2380
|
-
'x-
|
|
2795
|
+
...this.decisionHeaders(trace),
|
|
2796
|
+
...(isLastResort ? { 'x-fcm-v2-last-resort': 'true' } : {}),
|
|
2381
2797
|
})
|
|
2382
2798
|
res.end(responseText)
|
|
2383
2799
|
}
|
|
2384
|
-
return { done: true }
|
|
2385
|
-
}
|
|
2386
|
-
|
|
2387
|
-
if (AUTH_STATUS_CODES.has(response.status)) {
|
|
2388
|
-
this.markAuthError(key, `HTTP ${response.status}`)
|
|
2389
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'auth_error' })
|
|
2390
|
-
this.recordRuntimeCall({
|
|
2391
|
-
providerKey: candidate.provider, modelId: candidate.model,
|
|
2392
|
-
success: false, latencyMs, error: `auth_${response.status}`,
|
|
2393
|
-
})
|
|
2394
|
-
return { done: false, failoverToNext: true, reason: `auth_${response.status}`, authFailure: true }
|
|
2395
|
-
}
|
|
2396
|
-
|
|
2397
|
-
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
2398
|
-
this.markFailure(key, `HTTP ${response.status}`, response.status, upstreamMeta)
|
|
2399
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}` })
|
|
2400
|
-
this.recordRuntimeCall({
|
|
2401
|
-
providerKey: candidate.provider, modelId: candidate.model,
|
|
2402
|
-
success: false, latencyMs, error: `http_${response.status}`,
|
|
2403
|
-
})
|
|
2404
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2800
|
+
return { done: true, status: response.status, latencyMs }
|
|
2405
2801
|
}
|
|
2406
2802
|
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
}
|
|
2421
|
-
|
|
2422
|
-
if (!res.writableEnded) {
|
|
2423
|
-
res.writeHead(response.status, {
|
|
2424
|
-
...headerEntries(response.headers),
|
|
2425
|
-
'x-fcm-router-model': key,
|
|
2426
|
-
'x-request-id': requestId,
|
|
2427
|
-
})
|
|
2428
|
-
res.end(text)
|
|
2429
|
-
}
|
|
2430
|
-
return { done: true }
|
|
2803
|
+
const verdict = classifyFailure({ status: response.status, retryAfterMs: upstreamMeta.retryAfterMs })
|
|
2804
|
+
this.applyFailureVerdict(key, verdict, {
|
|
2805
|
+
detail: `HTTP ${response.status}`,
|
|
2806
|
+
statusCode: response.status,
|
|
2807
|
+
meta: upstreamMeta,
|
|
2808
|
+
})
|
|
2809
|
+
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: verdict.kind })
|
|
2810
|
+
this.recordRuntimeCall({
|
|
2811
|
+
providerKey: candidate.provider, modelId: candidate.model,
|
|
2812
|
+
success: false, latencyMs, error: verdict.kind,
|
|
2813
|
+
})
|
|
2814
|
+
this.recordRouterError(verdict.kind, requestId, { model: key, status: response.status })
|
|
2815
|
+
return { done: false, failoverToNext: verdict.failover, reason: verdict.kind, verdict, status: response.status, latencyMs }
|
|
2431
2816
|
} catch (error) {
|
|
2817
|
+
// 📖 Blame attribution (v2): a client disconnect is never an upstream
|
|
2818
|
+
// failure and must not damage the model's health.
|
|
2432
2819
|
if (clientAbort.aborted) {
|
|
2433
2820
|
this.logger.info(`Client disconnected before upstream response from ${key}`, { request_id: requestId })
|
|
2434
|
-
|
|
2821
|
+
trace.outcome = 'client_aborted'
|
|
2822
|
+
return { done: true, reason: 'client_aborted' }
|
|
2435
2823
|
}
|
|
2436
|
-
const
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2824
|
+
const isBodyReadTimeout = error?.name === 'BodyReadTimeoutError'
|
|
2825
|
+
const verdict = isBodyReadTimeout || error.name === 'AbortError'
|
|
2826
|
+
? classifyFailure({ kind: FAILURE_KINDS.TIMEOUT })
|
|
2827
|
+
: classifyFailure({ kind: FAILURE_KINDS.NETWORK })
|
|
2828
|
+
const detail = isBodyReadTimeout ? 'body read timeout' : (error.name === 'AbortError' ? 'timeout' : (error.message || String(error)))
|
|
2829
|
+
this.applyFailureVerdict(key, verdict, { detail })
|
|
2830
|
+
this.recordRouterError('upstream_transport_error', requestId, { model: key, reason: detail })
|
|
2831
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: detail })
|
|
2832
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2441
2833
|
} finally {
|
|
2442
2834
|
clearTimeout(timeout)
|
|
2443
2835
|
clientAbort.dispose()
|
|
2444
2836
|
}
|
|
2445
2837
|
}
|
|
2446
2838
|
|
|
2447
|
-
async proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex }) {
|
|
2839
|
+
async proxyStreamingRequest({ req, res, body, candidate, requestId, attemptIndex, protocol, trace, isLastResort = false, anthropicModelName = null }) {
|
|
2448
2840
|
const key = candidate.key
|
|
2449
2841
|
const activeReq = this.activeRequests.get(requestId)
|
|
2450
2842
|
if (activeReq) {
|
|
2451
2843
|
activeReq.current_model = key
|
|
2452
|
-
|
|
2844
|
+
activeReq.last_activity_at = Date.now()
|
|
2453
2845
|
}
|
|
2454
2846
|
const apiKey = this.getApiKeyForProvider(candidate.provider)
|
|
2455
|
-
// 📖 Guard: bail early if provider URL cannot be resolved
|
|
2456
2847
|
const providerUrl = resolveProviderUrl(candidate.provider)
|
|
2457
2848
|
if (!providerUrl) {
|
|
2458
|
-
|
|
2849
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.PROVIDER_URL })
|
|
2850
|
+
this.applyFailureVerdict(key, verdict, { detail: 'provider URL unresolvable' })
|
|
2459
2851
|
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'provider_url_unresolvable', stream: true })
|
|
2460
|
-
return { done: false, failoverToNext: true, reason:
|
|
2852
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2461
2853
|
}
|
|
2462
2854
|
const controller = new AbortController()
|
|
2463
2855
|
const started = performance.now()
|
|
2464
|
-
|
|
2465
|
-
// 📖
|
|
2466
|
-
//
|
|
2467
|
-
const
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
const bodyNormalized = normalizeRequestBody(bodyWithPrePrompt, candidate.provider)
|
|
2471
|
-
const upstreamBody = {
|
|
2472
|
-
...bodyNormalized,
|
|
2473
|
-
model: getApiModelId(candidate.provider, candidate.model),
|
|
2474
|
-
stream: true,
|
|
2475
|
-
}
|
|
2476
|
-
// 📖 Some providers/models fail if we send custom internal params, so strip them
|
|
2477
|
-
if (upstreamBody.add_generation_prompt !== undefined) delete upstreamBody.add_generation_prompt
|
|
2478
|
-
if (upstreamBody.continue_final_message !== undefined) delete upstreamBody.continue_final_message
|
|
2479
|
-
if (upstreamBody.tools?.length === 0) delete upstreamBody.tools
|
|
2480
|
-
|
|
2856
|
+
const upstreamBody = this.buildUpstreamBody(body, candidate, true)
|
|
2857
|
+
// 📖 Anthropic clients receive Anthropic SSE events: every byte written
|
|
2858
|
+
// to the client goes through the transformer sink instead of raw.
|
|
2859
|
+
const sink = protocol === 'anthropic'
|
|
2860
|
+
? createAnthropicStreamTransformer({ model: anthropicModelName || key })
|
|
2861
|
+
: null
|
|
2481
2862
|
const timeout = setTimeout(() => controller.abort(), this.routerConfig().failover.requestTimeoutMs)
|
|
2482
2863
|
let sentToClient = false
|
|
2483
2864
|
const clientAbort = attachClientAbort(req, res, controller)
|
|
2865
|
+
|
|
2866
|
+
const writeToClient = (text) => {
|
|
2867
|
+
if (res.writableEnded) return
|
|
2868
|
+
if (sink) res.write(sink.write(text))
|
|
2869
|
+
else res.write(Buffer.isBuffer(text) ? text : Buffer.from(text))
|
|
2870
|
+
}
|
|
2871
|
+
const endClientStream = () => {
|
|
2872
|
+
try {
|
|
2873
|
+
if (sink && !res.writableEnded) res.write(sink.end())
|
|
2874
|
+
} catch {}
|
|
2875
|
+
try { if (!res.writableEnded) res.end() } catch {}
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2484
2878
|
try {
|
|
2485
2879
|
const response = await fetch(providerUrl, {
|
|
2486
2880
|
method: 'POST',
|
|
@@ -2495,164 +2889,205 @@ class RouterRuntime {
|
|
|
2495
2889
|
const latencyMs = Math.round(performance.now() - started)
|
|
2496
2890
|
const upstreamMeta = buildUpstreamMeta(response, '', candidate.provider)
|
|
2497
2891
|
if (isLikelyHtmlResponse(response.headers)) {
|
|
2498
|
-
|
|
2499
|
-
this.
|
|
2892
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2893
|
+
this.applyFailureVerdict(key, verdict, { detail: 'upstream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2894
|
+
this.recordRouterError('upstream_html_maintenance', requestId, { model: key, stream: true })
|
|
2500
2895
|
this.addRequestLog({ request_id: requestId, model: key, status: 503, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: 'upstream_html_maintenance', stream: true })
|
|
2501
|
-
return { done: false, failoverToNext: true, reason:
|
|
2896
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2502
2897
|
}
|
|
2503
2898
|
if (!response.ok) {
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
}
|
|
2509
|
-
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
2510
|
-
this.markFailure(key, `HTTP ${response.status}`, response.status, upstreamMeta)
|
|
2511
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}`, stream: true })
|
|
2512
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2513
|
-
}
|
|
2514
|
-
|
|
2515
|
-
// 📖 Provide failover fallback for non-retryable errors from the provider (like 400 Bad Request)
|
|
2516
|
-
// when they are caused by format idiosyncrasies (e.g. empty tools array that another model might accept)
|
|
2517
|
-
if (response.status >= 400 && response.status < 500) {
|
|
2518
|
-
// 📖 Telemetry keeps structural fields only: upstream response bodies
|
|
2519
|
-
// 📖 must never leave the machine (they can embed user code/prompts).
|
|
2520
|
-
this.recordRouterError(`http_${response.status}`, requestId, { provider: candidate.provider, model: key, status: response.status, stream: true })
|
|
2521
|
-
this.markFailure(key, `HTTP ${response.status}`)
|
|
2522
|
-
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: `http_${response.status}`, stream: true })
|
|
2523
|
-
return { done: false, failoverToNext: true, reason: `http_${response.status}` }
|
|
2524
|
-
}
|
|
2525
|
-
|
|
2526
|
-
if (!res.writableEnded) {
|
|
2527
|
-
res.writeHead(response.status, {
|
|
2528
|
-
...headerEntries(response.headers),
|
|
2529
|
-
'x-fcm-router-model': key,
|
|
2530
|
-
'x-request-id': requestId,
|
|
2531
|
-
})
|
|
2532
|
-
try { res.end(await response.text()) } catch {}
|
|
2533
|
-
}
|
|
2534
|
-
return { done: true }
|
|
2899
|
+
const verdict = classifyFailure({ status: response.status, retryAfterMs: upstreamMeta.retryAfterMs })
|
|
2900
|
+
this.applyFailureVerdict(key, verdict, { detail: `HTTP ${response.status}`, statusCode: response.status, meta: upstreamMeta })
|
|
2901
|
+
this.recordRouterError(verdict.kind, requestId, { model: key, status: response.status, stream: true })
|
|
2902
|
+
this.addRequestLog({ request_id: requestId, model: key, status: response.status, latency_ms: latencyMs, tokens: 0, failover: attemptIndex > 0, error: verdict.kind, stream: true })
|
|
2903
|
+
return { done: false, failoverToNext: verdict.failover, reason: verdict.kind, verdict, status: response.status, latencyMs }
|
|
2535
2904
|
}
|
|
2536
2905
|
|
|
2537
2906
|
const reader = response.body?.getReader()
|
|
2538
2907
|
if (!reader) {
|
|
2539
|
-
|
|
2540
|
-
|
|
2908
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.EMPTY_STREAM })
|
|
2909
|
+
this.applyFailureVerdict(key, verdict, { detail: 'empty stream' })
|
|
2910
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: 'empty_stream', stream: true })
|
|
2911
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2541
2912
|
}
|
|
2542
2913
|
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2914
|
+
// 📖 v2 readiness gate: hold early chunks until the tracker sees useful
|
|
2915
|
+
// content, an upstream error frame (fail over BEFORE the client gets
|
|
2916
|
+
// bytes), or the hold cap overflows (weird provider: pass through).
|
|
2917
|
+
const tracker = createStreamReadinessTracker()
|
|
2918
|
+
const holdBuffer = []
|
|
2919
|
+
let forwarded = false
|
|
2920
|
+
let forwardedChars = 0
|
|
2921
|
+
let upstreamErrorAfterForward = false
|
|
2922
|
+
|
|
2923
|
+
const flushHold = () => {
|
|
2924
|
+
// 📖 Record this attempt as the serving one before the head is
|
|
2925
|
+
// written so the decision header reflects the streaming model.
|
|
2926
|
+
const winningAttempt = trace.attempts[trace.attempts.length - 1]
|
|
2927
|
+
if (winningAttempt) {
|
|
2928
|
+
winningAttempt.status = 200
|
|
2929
|
+
winningAttempt.latency_ms = latencyMs
|
|
2930
|
+
}
|
|
2931
|
+
// 📖 Issue #137: when a previous model already sent partial data the
|
|
2932
|
+
// headers are on the wire; append with an SSE comment marker instead.
|
|
2933
|
+
if (!res.headersSent) {
|
|
2934
|
+
res.writeHead(200, {
|
|
2935
|
+
...headerEntries(response.headers),
|
|
2936
|
+
// 📖 A forwarded stream is SSE by definition: override whatever
|
|
2937
|
+
// content-type the upstream declared (json/html/...).
|
|
2938
|
+
'Content-Type': 'text/event-stream',
|
|
2939
|
+
'Cache-Control': 'no-cache',
|
|
2940
|
+
Connection: 'keep-alive',
|
|
2941
|
+
...this.decisionHeaders(trace),
|
|
2942
|
+
...(isLastResort ? { 'x-fcm-v2-last-resort': 'true' } : {}),
|
|
2943
|
+
})
|
|
2944
|
+
} else {
|
|
2945
|
+
try { res.write(`: fcm-router-failover-from=${key}\n\n`) } catch {}
|
|
2946
|
+
}
|
|
2947
|
+
for (const text of holdBuffer) {
|
|
2948
|
+
writeToClient(text)
|
|
2949
|
+
forwardedChars += text.length
|
|
2950
|
+
}
|
|
2951
|
+
holdBuffer.length = 0
|
|
2952
|
+
sentToClient = true
|
|
2953
|
+
forwarded = true
|
|
2573
2954
|
}
|
|
2574
|
-
sentToClient = true
|
|
2575
|
-
res.write(firstChunkBuffer)
|
|
2576
2955
|
|
|
2577
|
-
while (
|
|
2956
|
+
while (true) {
|
|
2578
2957
|
const chunk = await this.readStreamChunkWithTimeout(reader)
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2958
|
+
const text = chunk.done || !chunk.value
|
|
2959
|
+
? null
|
|
2960
|
+
: (Buffer.isBuffer(chunk.value) ? chunk.value.toString('utf8') : Buffer.from(chunk.value).toString('utf8'))
|
|
2961
|
+
if (text === null) break
|
|
2962
|
+
tracker.observe(text)
|
|
2583
2963
|
if (activeReq) {
|
|
2584
|
-
|
|
2585
|
-
activeReq.tokens += 1
|
|
2964
|
+
activeReq.last_activity_at = Date.now()
|
|
2965
|
+
activeReq.tokens += 1
|
|
2586
2966
|
}
|
|
2967
|
+
if (!forwarded) {
|
|
2968
|
+
if (tracker.errorPayload) {
|
|
2969
|
+
try { controller.abort() } catch {}
|
|
2970
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.ERROR_PAYLOAD })
|
|
2971
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream error payload before content', statusCode: 200, meta: upstreamMeta })
|
|
2972
|
+
this.recordRouterError('gate_reject', requestId, { model: key, stream: true, reason: 'error_payload' })
|
|
2973
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
2974
|
+
}
|
|
2975
|
+
if (tracker.useful) {
|
|
2976
|
+
holdBuffer.push(text)
|
|
2977
|
+
flushHold()
|
|
2978
|
+
} else if (tracker.bytesSeen > tracker.maxHoldBytes) {
|
|
2979
|
+
// 📖 Huge non-JSON preamble: pass it through rather than stall.
|
|
2980
|
+
holdBuffer.push(text)
|
|
2981
|
+
flushHold()
|
|
2982
|
+
} else if (isLikelyHtmlText(text)) {
|
|
2983
|
+
try { controller.abort() } catch {}
|
|
2984
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.HTML })
|
|
2985
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream html maintenance', statusCode: 503, meta: upstreamMeta })
|
|
2986
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 503, latencyMs }
|
|
2987
|
+
} else {
|
|
2988
|
+
holdBuffer.push(text)
|
|
2989
|
+
}
|
|
2990
|
+
} else {
|
|
2991
|
+
writeToClient(text)
|
|
2992
|
+
forwardedChars += text.length
|
|
2993
|
+
if (tracker.errorPayload) {
|
|
2994
|
+
// 📖 Upstream errored AFTER real content: keep the partial output,
|
|
2995
|
+
// close cleanly, and record a real failure (v1 marked success).
|
|
2996
|
+
upstreamErrorAfterForward = true
|
|
2997
|
+
break
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
}
|
|
3001
|
+
|
|
3002
|
+
if (!forwarded) {
|
|
3003
|
+
// 📖 The stream closed without ever producing useful content. v1 only
|
|
3004
|
+
// caught the zero-chunk case; the gate also fails over a stream that
|
|
3005
|
+
// sent only framing garbage. Nothing reached the client, so failover
|
|
3006
|
+
// is safe.
|
|
3007
|
+
try { controller.abort() } catch {}
|
|
3008
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.EMPTY_STREAM })
|
|
3009
|
+
this.applyFailureVerdict(key, verdict, { detail: `stream closed without content (${tracker.describe()})`, statusCode: 200, meta: upstreamMeta })
|
|
3010
|
+
this.recordRouterError('gate_reject', requestId, { model: key, stream: true, reason: 'empty_stream' })
|
|
3011
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict, status: 200, latencyMs }
|
|
3012
|
+
}
|
|
3013
|
+
|
|
3014
|
+
if (upstreamErrorAfterForward) {
|
|
3015
|
+
const verdict = classifyFailure({ kind: FAILURE_KINDS.ERROR_PAYLOAD })
|
|
3016
|
+
this.applyFailureVerdict(key, verdict, { detail: 'stream error payload after content', statusCode: 200, meta: upstreamMeta })
|
|
3017
|
+
endClientStream()
|
|
3018
|
+
return { done: true, status: 200, latencyMs }
|
|
2587
3019
|
}
|
|
2588
3020
|
|
|
2589
3021
|
this.markSuccess(key, latencyMs)
|
|
3022
|
+
const completionTokens = estimateTokens(forwardedChars)
|
|
3023
|
+
this.tokenTracker.record(candidate.provider, candidate.model, {
|
|
3024
|
+
prompt_tokens: 0,
|
|
3025
|
+
completion_tokens: completionTokens,
|
|
3026
|
+
total_tokens: completionTokens,
|
|
3027
|
+
})
|
|
3028
|
+
this.recordRuntimeCall({
|
|
3029
|
+
providerKey: candidate.provider, modelId: candidate.model,
|
|
3030
|
+
success: true, latencyMs,
|
|
3031
|
+
usage: { prompt_tokens: 0, completion_tokens: completionTokens, total_tokens: completionTokens },
|
|
3032
|
+
})
|
|
2590
3033
|
this.totalRequestsRouted += 1
|
|
3034
|
+
trace.served_model = key
|
|
3035
|
+
trace.tokens = completionTokens
|
|
2591
3036
|
this.addRequestLog({
|
|
2592
3037
|
request_id: requestId,
|
|
2593
3038
|
model: key,
|
|
2594
|
-
status:
|
|
3039
|
+
status: 200,
|
|
2595
3040
|
latency_ms: latencyMs,
|
|
2596
|
-
tokens:
|
|
3041
|
+
tokens: completionTokens,
|
|
2597
3042
|
failover: attemptIndex > 0,
|
|
2598
3043
|
stream: true,
|
|
2599
3044
|
})
|
|
2600
|
-
|
|
2601
|
-
return { done: true }
|
|
3045
|
+
endClientStream()
|
|
3046
|
+
return { done: true, status: 200, latencyMs }
|
|
2602
3047
|
} catch (error) {
|
|
2603
3048
|
try { controller.abort() } catch {}
|
|
2604
3049
|
if (clientAbort.aborted) {
|
|
2605
3050
|
this.logger.info(`Client disconnected during streaming response from ${key}`, { request_id: requestId })
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
// 📖 Issue #137: stream-stall timeouts get a special tag so we can
|
|
2610
|
-
// 📖 distinguish them from generic upstream errors below. Only stalls
|
|
2611
|
-
// 📖 should trigger failover after a partial response — generic errors
|
|
2612
|
-
// 📖 (malformed JSON, network reset, etc.) usually mean the partial
|
|
2613
|
-
// 📖 data is invalid anyway, so closing cleanly is safer.
|
|
2614
|
-
const isStall = reason === 'stream_stall_timeout' || reason === 'timeout'
|
|
2615
|
-
this.markFailure(key, reason)
|
|
2616
|
-
if (reason !== 'timeout') {
|
|
2617
|
-
this.recordRouterError('upstream_stream_error', requestId, { model: key, reason, partial: sentToClient })
|
|
2618
|
-
} else {
|
|
2619
|
-
this.recordRouterError('timeout', requestId, { model: key, reason, partial: sentToClient })
|
|
3051
|
+
trace.outcome = 'client_aborted'
|
|
3052
|
+
endClientStream()
|
|
3053
|
+
return { done: true, reason: 'client_aborted' }
|
|
2620
3054
|
}
|
|
2621
|
-
|
|
3055
|
+
const isStall = error?.message === 'stream_stall_timeout' || error.name === 'AbortError'
|
|
3056
|
+
const kind = error?.message === 'stream_stall_timeout'
|
|
3057
|
+
? FAILURE_KINDS.STREAM_STALL
|
|
3058
|
+
: (error.name === 'AbortError' ? FAILURE_KINDS.TIMEOUT : FAILURE_KINDS.NETWORK)
|
|
3059
|
+
const detail = error?.message === 'stream_stall_timeout'
|
|
3060
|
+
? 'stream stall timeout'
|
|
3061
|
+
: (error.name === 'AbortError' ? 'timeout' : (error.message || String(error)))
|
|
3062
|
+
const verdict = classifyFailure({ kind })
|
|
3063
|
+
this.applyFailureVerdict(key, verdict, { detail })
|
|
3064
|
+
this.recordRouterError(isStall ? 'timeout' : 'upstream_stream_error', requestId, { model: key, reason: detail, partial: sentToClient, stream: true })
|
|
3065
|
+
this.addRequestLog({ request_id: requestId, model: key, status: 'ERR', latency_ms: null, tokens: 0, failover: attemptIndex > 0, error: detail, stream: true })
|
|
2622
3066
|
if (sentToClient) {
|
|
2623
3067
|
if (isStall) {
|
|
2624
|
-
// 📖 Issue #137:
|
|
2625
|
-
//
|
|
2626
|
-
//
|
|
2627
|
-
|
|
2628
|
-
// 📖 upstream connection; its chunks are appended to the same
|
|
2629
|
-
// 📖 response object so the client sees one continuous stream.
|
|
2630
|
-
this.logger.warn(`Stream stall after partial response from ${key}, attempting failover`, { request_id: requestId, reason })
|
|
3068
|
+
// 📖 Issue #137: fail over even after partial output. OpenAI
|
|
3069
|
+
// clients get a synthetic caution delta; Anthropic clients get an
|
|
3070
|
+
// SSE error event before the stream closes.
|
|
3071
|
+
this.logger.warn(`Stream stall after partial response from ${key}, attempting failover`, { request_id: requestId, reason: detail })
|
|
2631
3072
|
if (!res.writableEnded) {
|
|
2632
3073
|
try {
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
index: 0,
|
|
2640
|
-
delta: { content: failoverMsg },
|
|
2641
|
-
finish_reason: null,
|
|
2642
|
-
}],
|
|
2643
|
-
})
|
|
2644
|
-
res.write(`data: ${deltaPayload}\n\n`)
|
|
3074
|
+
if (sink) {
|
|
3075
|
+
res.write(sink.write(`data: ${JSON.stringify({ error: { message: `stream truncated by router (${detail}); failing over to next model`, type: 'api_error' } })}\n\n`))
|
|
3076
|
+
} else {
|
|
3077
|
+
const failoverMsg = `\n\n> [!CAUTION]\n> Stream truncated by router due to upstream ${detail}; failing over to next model.\n\n`
|
|
3078
|
+
res.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: failoverMsg }, finish_reason: null }] })}\n\n`)
|
|
3079
|
+
}
|
|
2645
3080
|
} catch { /* best-effort */ }
|
|
2646
3081
|
}
|
|
2647
|
-
return { done: false, failoverToNext: true, reason: `
|
|
3082
|
+
return { done: false, failoverToNext: true, reason: `stream_stall`, verdict }
|
|
2648
3083
|
}
|
|
2649
|
-
// 📖 Non-stall errors after partial output:
|
|
2650
|
-
//
|
|
2651
|
-
this.logger.warn(`Streaming failure after partial response from ${key}`, { request_id: requestId, reason })
|
|
2652
|
-
|
|
3084
|
+
// 📖 Non-stall errors after partial output: close cleanly, no
|
|
3085
|
+
// failover, to avoid sending malformed data (v1 behavior kept).
|
|
3086
|
+
this.logger.warn(`Streaming failure after partial response from ${key}`, { request_id: requestId, reason: detail })
|
|
3087
|
+
endClientStream()
|
|
2653
3088
|
return { done: true }
|
|
2654
3089
|
}
|
|
2655
|
-
return { done: false, failoverToNext: true, reason }
|
|
3090
|
+
return { done: false, failoverToNext: true, reason: verdict.kind, verdict }
|
|
2656
3091
|
} finally {
|
|
2657
3092
|
clearTimeout(timeout)
|
|
2658
3093
|
clientAbort.dispose()
|
|
@@ -2677,6 +3112,41 @@ class RouterRuntime {
|
|
|
2677
3112
|
])
|
|
2678
3113
|
}
|
|
2679
3114
|
|
|
3115
|
+
// ─── Anthropic /v1/messages (v2 protocol support) ─────────────────────────
|
|
3116
|
+
|
|
3117
|
+
async handleAnthropicMessages(req, res, requestId) {
|
|
3118
|
+
if (!isAuthorizedForV1(req)) {
|
|
3119
|
+
sendJson(res, 401, anthropicErrorPayload('authentication_error', 'Missing or invalid router token'), { 'x-request-id': requestId })
|
|
3120
|
+
return
|
|
3121
|
+
}
|
|
3122
|
+
let body
|
|
3123
|
+
try {
|
|
3124
|
+
body = await readJsonBody(req)
|
|
3125
|
+
} catch (error) {
|
|
3126
|
+
if (error.code === 'BODY_TOO_LARGE') {
|
|
3127
|
+
sendJson(res, 413, anthropicErrorPayload('request_too_large', 'Request body too large'), { 'x-request-id': requestId })
|
|
3128
|
+
return
|
|
3129
|
+
}
|
|
3130
|
+
sendJson(res, 400, anthropicErrorPayload('invalid_request_error', 'Invalid JSON body'), { 'x-request-id': requestId })
|
|
3131
|
+
return
|
|
3132
|
+
}
|
|
3133
|
+
const translated = translateAnthropicToOpenAI(body)
|
|
3134
|
+
if (!translated.ok) {
|
|
3135
|
+
sendJson(res, 400, anthropicErrorPayload('invalid_request_error', translated.error), { 'x-request-id': requestId })
|
|
3136
|
+
return
|
|
3137
|
+
}
|
|
3138
|
+
const openaiBody = { ...translated.body, stream: body.stream === true }
|
|
3139
|
+
await this.routeRequest({
|
|
3140
|
+
req,
|
|
3141
|
+
res,
|
|
3142
|
+
body: openaiBody,
|
|
3143
|
+
setName: null,
|
|
3144
|
+
requestId,
|
|
3145
|
+
protocol: 'anthropic',
|
|
3146
|
+
anthropicModelName: typeof body.model === 'string' ? body.model : null,
|
|
3147
|
+
})
|
|
3148
|
+
}
|
|
3149
|
+
|
|
2680
3150
|
async handleSetsRequest(req, res, url, requestId) {
|
|
2681
3151
|
// 📖 Hoisted same-origin guard: covers both the canonical /sets routes and
|
|
2682
3152
|
// 📖 the /api/router/sets alias so no set-mutating path skips the check.
|
|
@@ -3022,6 +3492,12 @@ class RouterRuntime {
|
|
|
3022
3492
|
const requestId = typeof rawRequestId === 'string' && rawRequestId.trim()
|
|
3023
3493
|
? rawRequestId.trim().slice(0, 64)
|
|
3024
3494
|
: `req-${randomUUID()}`
|
|
3495
|
+
applyCors(req, res)
|
|
3496
|
+
if (req.method === 'OPTIONS') {
|
|
3497
|
+
res.writeHead(204)
|
|
3498
|
+
res.end()
|
|
3499
|
+
return
|
|
3500
|
+
}
|
|
3025
3501
|
// 📖 DNS-rebinding guard: reject requests whose Host header is not the
|
|
3026
3502
|
// 📖 loopback (or the configured FCM_HOST) before any routing happens.
|
|
3027
3503
|
if (!isAllowedHostHeader(req.headers.host, this.port, this.boundHost)) {
|
|
@@ -3478,6 +3954,80 @@ class RouterRuntime {
|
|
|
3478
3954
|
serveWebStaticFile(res, url.pathname, requestId)
|
|
3479
3955
|
return
|
|
3480
3956
|
}
|
|
3957
|
+
// ─── Anthropic-compatible routing surface (v2) ──────────────────────
|
|
3958
|
+
if (url.pathname === '/v1/messages') {
|
|
3959
|
+
if (req.method !== 'POST') {
|
|
3960
|
+
sendJson(res, 405, anthropicErrorPayload('invalid_request_error', 'Method not allowed, use POST'), { 'x-request-id': requestId })
|
|
3961
|
+
return
|
|
3962
|
+
}
|
|
3963
|
+
await this.handleAnthropicMessages(req, res, requestId)
|
|
3964
|
+
return
|
|
3965
|
+
}
|
|
3966
|
+
|
|
3967
|
+
// ─── Router v2 dashboard API (beta overlays + web page) ──────────────
|
|
3968
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/status') {
|
|
3969
|
+
sendJson(res, 200, { ...this.statusPayload(), router: 'v2', beta: true }, { 'x-request-id': requestId })
|
|
3970
|
+
return
|
|
3971
|
+
}
|
|
3972
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/stats') {
|
|
3973
|
+
sendJson(res, 200, { ...this.statsPayload(), router: 'v2', beta: true }, { 'x-request-id': requestId })
|
|
3974
|
+
return
|
|
3975
|
+
}
|
|
3976
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/history') {
|
|
3977
|
+
const limitRaw = Number.parseInt(url.searchParams.get('limit') || '50', 10)
|
|
3978
|
+
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(1, limitRaw), 500) : 50
|
|
3979
|
+
sendJson(res, 200, { entries: this.history.recent(limit), stats: this.history.stats() }, { 'x-request-id': requestId })
|
|
3980
|
+
return
|
|
3981
|
+
}
|
|
3982
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/traces') {
|
|
3983
|
+
const limitRaw = Number.parseInt(url.searchParams.get('limit') || '20', 10)
|
|
3984
|
+
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(1, limitRaw), 50) : 20
|
|
3985
|
+
sendJson(res, 200, { traces: this.recentTraces.slice(-limit).reverse() }, { 'x-request-id': requestId })
|
|
3986
|
+
return
|
|
3987
|
+
}
|
|
3988
|
+
if (url.pathname === '/api/router-v2/history' && req.method === 'DELETE') {
|
|
3989
|
+
if (!isSameOriginOrLocal(req)) {
|
|
3990
|
+
sendError(res, 403, 'Forbidden cross-origin request', 'invalid_request_error', 'forbidden_origin', requestId)
|
|
3991
|
+
return
|
|
3992
|
+
}
|
|
3993
|
+
this.history.clear()
|
|
3994
|
+
this.recentTraces = []
|
|
3995
|
+
sendJson(res, 200, { ok: true }, { 'x-request-id': requestId })
|
|
3996
|
+
return
|
|
3997
|
+
}
|
|
3998
|
+
if (url.pathname === '/api/router-v2/test' && req.method === 'POST') {
|
|
3999
|
+
if (!isSameOriginOrLocal(req)) {
|
|
4000
|
+
sendError(res, 403, 'Forbidden cross-origin request', 'invalid_request_error', 'forbidden_origin', requestId)
|
|
4001
|
+
return
|
|
4002
|
+
}
|
|
4003
|
+
const body = await readJsonBody(req)
|
|
4004
|
+
const provider = typeof body.provider === 'string' ? body.provider.trim() : ''
|
|
4005
|
+
const model = typeof body.model === 'string' ? body.model.trim() : ''
|
|
4006
|
+
if (!provider || !model) {
|
|
4007
|
+
sendError(res, 400, 'Both `provider` and `model` are required', 'invalid_request_error', 'missing_model_fields', requestId)
|
|
4008
|
+
return
|
|
4009
|
+
}
|
|
4010
|
+
const { testModelViaRouter } = await import('./router-v2/bench.js')
|
|
4011
|
+
const result = await testModelViaRouter({ port: this.port, provider, model })
|
|
4012
|
+
sendJson(res, 200, result, { 'x-request-id': requestId })
|
|
4013
|
+
return
|
|
4014
|
+
}
|
|
4015
|
+
if (req.method === 'GET' && url.pathname === '/api/router-v2/events') {
|
|
4016
|
+
if (!this.tryOpenSseConnection(req, res, requestId)) return
|
|
4017
|
+
res.writeHead(200, {
|
|
4018
|
+
'Content-Type': 'text/event-stream',
|
|
4019
|
+
'Cache-Control': 'no-cache',
|
|
4020
|
+
Connection: 'keep-alive',
|
|
4021
|
+
'x-request-id': requestId,
|
|
4022
|
+
})
|
|
4023
|
+
res.flushHeaders?.()
|
|
4024
|
+
res.write(': connected\n\n')
|
|
4025
|
+
res.write(`event: hello\ndata: ${JSON.stringify(this.statusPayload())}\n\n`)
|
|
4026
|
+
this.sseClients.add(res)
|
|
4027
|
+
req.on('close', () => this.sseClients.delete(res))
|
|
4028
|
+
return
|
|
4029
|
+
}
|
|
4030
|
+
|
|
3481
4031
|
if (url.pathname === '/v1/chat/completions' || url.pathname.match(/^\/v1\/sets\/[^/]+\/chat\/completions$/)) {
|
|
3482
4032
|
if (req.method !== 'POST') {
|
|
3483
4033
|
sendError(res, 405, 'Method not allowed', 'invalid_request_error', 'method_not_allowed', requestId, { allowed: ['POST'] })
|
|
@@ -3556,6 +4106,7 @@ class RouterRuntime {
|
|
|
3556
4106
|
this.shuttingDown = true
|
|
3557
4107
|
this.logger.info('Router daemon stopping')
|
|
3558
4108
|
if (this.probeTimer) clearInterval(this.probeTimer)
|
|
4109
|
+
if (this.probeWatchdog) clearInterval(this.probeWatchdog)
|
|
3559
4110
|
if (this.configReloadTimer) clearInterval(this.configReloadTimer)
|
|
3560
4111
|
if (this.tokenFlushTimer) clearInterval(this.tokenFlushTimer)
|
|
3561
4112
|
if (this.probeCacheFlushTimer) clearInterval(this.probeCacheFlushTimer)
|
|
@@ -3566,6 +4117,8 @@ class RouterRuntime {
|
|
|
3566
4117
|
await sleep(100)
|
|
3567
4118
|
}
|
|
3568
4119
|
this.tokenTracker.flush({ force: true })
|
|
4120
|
+
this.breakers.flush()
|
|
4121
|
+
this.history.flush()
|
|
3569
4122
|
flushProbeCache() // 📖 t1: persist any pending probe-cache deltas before exit
|
|
3570
4123
|
if (this.runtimeTelemetryDirty) flushRuntimeTelemetryStore() // 📖 t3
|
|
3571
4124
|
try { this.server?.close() } catch {}
|
|
@@ -3587,7 +4140,7 @@ class RouterRuntime {
|
|
|
3587
4140
|
// 📖 Pinned picks: only used as a *tie-breaker* when multiple models have
|
|
3588
4141
|
// 📖 identical (tier, sweScore, latency) - never a hard requirement, so
|
|
3589
4142
|
// 📖 a user whose NVIDIA key is dead still gets a working set.
|
|
3590
|
-
const PREFERRED_DEFAULT_MODELS = [
|
|
4143
|
+
export const PREFERRED_DEFAULT_MODELS = [
|
|
3591
4144
|
{ provider: 'groq', model: 'openai/gpt-oss-120b' },
|
|
3592
4145
|
{ provider: 'groq', model: 'qwen/qwen3.6-27b' },
|
|
3593
4146
|
{ provider: 'cerebras', model: 'llama3.1-70b' },
|
|
@@ -3755,7 +4308,7 @@ export async function buildDefaultRouterSet(config = {}, maxModels, options = {}
|
|
|
3755
4308
|
}
|
|
3756
4309
|
}
|
|
3757
4310
|
|
|
3758
|
-
export function createRouterRuntimeForTest({ config, port = 0, logger = null, tokenPath = ROUTER_TOKENS_PATH } = {}) {
|
|
4311
|
+
export function createRouterRuntimeForTest({ config, port = 0, logger = null, tokenPath = ROUTER_TOKENS_PATH, breakersPath = null, historyPath = null } = {}) {
|
|
3759
4312
|
const testLogger = logger || {
|
|
3760
4313
|
level: 'error',
|
|
3761
4314
|
error() {},
|
|
@@ -3767,12 +4320,19 @@ export function createRouterRuntimeForTest({ config, port = 0, logger = null, to
|
|
|
3767
4320
|
// 📖 fake providers without spawning a daemon or touching user token files.
|
|
3768
4321
|
// 📖 Router config persistence is disabled here so set/probe-mode endpoint
|
|
3769
4322
|
// 📖 tests cannot write fixture router sets into ~/.free-coding-models.json.
|
|
4323
|
+
// 📖 v2: breaker/history state also lands in unique tmp files, otherwise
|
|
4324
|
+
// 📖 tests would inherit the machine's real persisted breaker state.
|
|
4325
|
+
const testId = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`
|
|
3770
4326
|
return new RouterRuntime({
|
|
3771
4327
|
config: config || {},
|
|
3772
4328
|
port,
|
|
3773
4329
|
logger: testLogger,
|
|
3774
4330
|
tokenPath,
|
|
3775
4331
|
persistConfig: false,
|
|
4332
|
+
paths: {
|
|
4333
|
+
breakers: breakersPath || join(tmpdir(), `fcm-router-test-breakers-${testId}.json`),
|
|
4334
|
+
history: historyPath || join(tmpdir(), `fcm-router-test-history-${testId}.json`),
|
|
4335
|
+
},
|
|
3776
4336
|
})
|
|
3777
4337
|
}
|
|
3778
4338
|
|
|
@@ -3792,7 +4352,7 @@ export function createRouterRuntimeForTest({ config, port = 0, logger = null, to
|
|
|
3792
4352
|
*
|
|
3793
4353
|
* @returns {(entry: { provider: string, model: string }) => Promise<{ ok: boolean, code: string|number, latencyMs: number }>}
|
|
3794
4354
|
*/
|
|
3795
|
-
function createDefaultProbeFn(apiKeys) {
|
|
4355
|
+
export function createDefaultProbeFn(apiKeys) {
|
|
3796
4356
|
return async (entry) => {
|
|
3797
4357
|
const { provider, model } = entry
|
|
3798
4358
|
if (!isRouteableProvider(provider, sources)) return { ok: false, code: 'NOT_ROUTEABLE', latencyMs: 0 }
|
|
@@ -3839,7 +4399,7 @@ function createDefaultProbeFn(apiKeys) {
|
|
|
3839
4399
|
}
|
|
3840
4400
|
}
|
|
3841
4401
|
|
|
3842
|
-
function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
4402
|
+
export function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
3843
4403
|
// 📖 Synchronous fallback used when async probing isn't available (e.g.
|
|
3844
4404
|
// 📖 routerConfig() getter, which is on the hot path). Falls back to the
|
|
3845
4405
|
// 📖 static tier-based ordering. The async probed version is the one
|
|
@@ -3882,7 +4442,7 @@ function buildDefaultRouterSetSync(config = {}, maxModels = 5) {
|
|
|
3882
4442
|
}
|
|
3883
4443
|
}
|
|
3884
4444
|
|
|
3885
|
-
async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
4445
|
+
export async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
3886
4446
|
// 📖 Preserve existing named sets (e.g., created by sync-set) to avoid overwriting
|
|
3887
4447
|
// 📖 user-created configurations. Only rebuild from favorites/defaults when no
|
|
3888
4448
|
// 📖 sets exist at all (fresh install).
|
|
@@ -3924,7 +4484,7 @@ async function ensureRouterConfigForDaemon(config, skipSave = false) {
|
|
|
3924
4484
|
* 📖 Each favorite "providerKey/modelId" is resolved to its source model entry.
|
|
3925
4485
|
* 📖 Falls back to buildDefaultRouterSet if no favorites exist.
|
|
3926
4486
|
*/
|
|
3927
|
-
function buildRouterSetFromFavorites(config) {
|
|
4487
|
+
export function buildRouterSetFromFavorites(config) {
|
|
3928
4488
|
const favorites = config.favorites
|
|
3929
4489
|
if (!Array.isArray(favorites) || favorites.length === 0) return null
|
|
3930
4490
|
const models = []
|
|
@@ -3952,7 +4512,7 @@ function buildRouterSetFromFavorites(config) {
|
|
|
3952
4512
|
}
|
|
3953
4513
|
}
|
|
3954
4514
|
|
|
3955
|
-
function listenOnPort(server, port, host = '127.0.0.1') {
|
|
4515
|
+
export function listenOnPort(server, port, host = '127.0.0.1') {
|
|
3956
4516
|
return new Promise((resolve, reject) => {
|
|
3957
4517
|
const onError = (error) => {
|
|
3958
4518
|
server.off('error', onError)
|
|
@@ -3968,7 +4528,7 @@ function listenOnPort(server, port, host = '127.0.0.1') {
|
|
|
3968
4528
|
})
|
|
3969
4529
|
}
|
|
3970
4530
|
|
|
3971
|
-
async function listenWithFallback(server, preferredPort, logger, host = '127.0.0.1') {
|
|
4531
|
+
export async function listenWithFallback(server, preferredPort, logger, host = '127.0.0.1') {
|
|
3972
4532
|
const { defaultPort, maxPort } = getRouterPortRange()
|
|
3973
4533
|
const start = Math.max(1, preferredPort || defaultPort)
|
|
3974
4534
|
const candidates = []
|
|
@@ -3989,9 +4549,47 @@ async function listenWithFallback(server, preferredPort, logger, host = '127.0.0
|
|
|
3989
4549
|
throw lastError || new Error('No router ports available')
|
|
3990
4550
|
}
|
|
3991
4551
|
|
|
4552
|
+
// 📖 v2: a set counts as usable when the active set holds at least one model.
|
|
4553
|
+
function hasUsableActiveSet(config) {
|
|
4554
|
+
const router = config?.router
|
|
4555
|
+
if (!router || typeof router !== 'object') return false
|
|
4556
|
+
const activeSet = router.activeSet || DEFAULT_ROUTER_SETTINGS.activeSet
|
|
4557
|
+
const set = router.sets?.[activeSet]
|
|
4558
|
+
return Boolean(set && Array.isArray(set.models) && set.models.length > 0)
|
|
4559
|
+
}
|
|
4560
|
+
|
|
3992
4561
|
export async function runRouterDaemon() {
|
|
3993
4562
|
const config = loadConfig()
|
|
3994
|
-
|
|
4563
|
+
// 📖 v2: listen FIRST. v1 awaited a 24-candidate probe sweep before the
|
|
4564
|
+
// server socket opened, leaving first boots with a ~36s black hole. Build
|
|
4565
|
+
// a fast static set when none exists, serve immediately, and upgrade to
|
|
4566
|
+
// the probe-driven set in the background.
|
|
4567
|
+
let needsProbedSetUpgrade = false
|
|
4568
|
+
if (!hasUsableActiveSet(config)) {
|
|
4569
|
+
const favSet = buildRouterSetFromFavorites(config)
|
|
4570
|
+
if (favSet) {
|
|
4571
|
+
config.router = normalizeRouterConfig({
|
|
4572
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4573
|
+
enabled: true,
|
|
4574
|
+
onboardingSeen: true,
|
|
4575
|
+
activeSet: favSet.name,
|
|
4576
|
+
sets: { [favSet.name]: favSet },
|
|
4577
|
+
})
|
|
4578
|
+
saveConfig(config)
|
|
4579
|
+
} else {
|
|
4580
|
+
const syncSet = buildDefaultRouterSetSync(config, 5)
|
|
4581
|
+
config.router = normalizeRouterConfig({
|
|
4582
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4583
|
+
enabled: true,
|
|
4584
|
+
onboardingSeen: true,
|
|
4585
|
+
activeSet: syncSet.name,
|
|
4586
|
+
sets: { [syncSet.name]: syncSet },
|
|
4587
|
+
})
|
|
4588
|
+
saveConfig(config)
|
|
4589
|
+
needsProbedSetUpgrade = true
|
|
4590
|
+
}
|
|
4591
|
+
}
|
|
4592
|
+
const router = config.router
|
|
3995
4593
|
// 📖 In dev mode, override the saved port with the dev default so a local
|
|
3996
4594
|
// 📖 checkout doesn't clash with a production install on the same machine.
|
|
3997
4595
|
// 📖 The saved config has port: 19280 (production); dev should use 29280.
|
|
@@ -4034,8 +4632,39 @@ export async function runRouterDaemon() {
|
|
|
4034
4632
|
})
|
|
4035
4633
|
runtime.configReloadTimer = setInterval(() => runtime.reloadConfigFromDisk(), CONFIG_RELOAD_INTERVAL_MS)
|
|
4036
4634
|
runtime.tokenFlushTimer = setInterval(() => runtime.tokenTracker.flush(), TOKEN_FLUSH_INTERVAL_MS)
|
|
4037
|
-
|
|
4038
|
-
|
|
4635
|
+
// 📖 v2: probe-driven default set upgrade (when the static one above was
|
|
4636
|
+
// just created) happens in the background, after listen().
|
|
4637
|
+
void (async () => {
|
|
4638
|
+
try {
|
|
4639
|
+
if (needsProbedSetUpgrade) {
|
|
4640
|
+
// 📖 The static tier-ordered pick can contain models the user's key
|
|
4641
|
+
// cannot actually call. Replace it once with the probe-driven set so
|
|
4642
|
+
// the router starts on models that really answer.
|
|
4643
|
+
const fresh = loadConfig()
|
|
4644
|
+
const probed = await buildDefaultRouterSet(fresh, 5, {
|
|
4645
|
+
probeFn: createDefaultProbeFn(fresh.apiKeys || {}),
|
|
4646
|
+
probeTimeoutMs: 1500,
|
|
4647
|
+
probeBudget: 24,
|
|
4648
|
+
})
|
|
4649
|
+
if (probed && Array.isArray(probed.models) && probed.models.length > 0) {
|
|
4650
|
+
fresh.router = normalizeRouterConfig({
|
|
4651
|
+
...DEFAULT_ROUTER_SETTINGS,
|
|
4652
|
+
enabled: true,
|
|
4653
|
+
onboardingSeen: true,
|
|
4654
|
+
activeSet: probed.name,
|
|
4655
|
+
sets: { [probed.name]: probed },
|
|
4656
|
+
})
|
|
4657
|
+
saveConfig(fresh)
|
|
4658
|
+
runtime.config = fresh
|
|
4659
|
+
runtime.refreshRouteState()
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4662
|
+
} catch (error) {
|
|
4663
|
+
logger.debug('Background router set upgrade skipped', { error: error?.message })
|
|
4664
|
+
}
|
|
4665
|
+
void runtime.runProbeBurst()
|
|
4666
|
+
runtime.scheduleProbeLoop()
|
|
4667
|
+
})()
|
|
4039
4668
|
// 📖 Auto-heal: wait for the first probe burst to populate health data,
|
|
4040
4669
|
// 📖 then swap any broken models (AUTH_ERROR / STALE) for working
|
|
4041
4670
|
// 📖 alternatives. This is the M6 promise: the Playground and Router
|
|
@@ -4125,7 +4754,7 @@ export async function startRouterDaemonBackground() {
|
|
|
4125
4754
|
// 📖 Best-effort process command lookup used to verify a PID file before
|
|
4126
4755
|
// 📖 signalling. Returns null when `ps` is unavailable (e.g. Windows) so the
|
|
4127
4756
|
// 📖 caller can keep the previous behavior instead of failing hard.
|
|
4128
|
-
function getProcessCommand(pid) {
|
|
4757
|
+
export function getProcessCommand(pid) {
|
|
4129
4758
|
try {
|
|
4130
4759
|
return execFileSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8' }).trim()
|
|
4131
4760
|
} catch {
|