mslxdff 0.1.39 → 0.1.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/routes.js CHANGED
@@ -1,1316 +1,3 @@
1
- import { timingSafeEqual, createHash } from "node:crypto";
2
- import { performance } from "node:perf_hooks";
3
- import { injectReasoningContent, normalizeModel } from "./reasoning.js";
4
- import { isAutoModel } from "./auto.js";
5
- import { DEFAULT_MAX_HOPS } from "./peers.js";
6
- import { loadGroupsJoined } from "./state.js";
7
-
8
- export const errMsg = (err) => String(err?.message || err);
9
-
10
- // In-memory relay queues for broadband members (polling-based, no WS dependency)
11
- // pendingByTarget: Map<`${group}::${targetUrl}`, Array<{reqId, body, hops, resolve, reject, timer}>>
12
- const relayPending = new Map();
13
- const relayPendingByReqId = new Map();
14
-
15
- function enqueueRelay({ group, target, reqId, body, hops }) {
16
- const key = `${group}::${target}`;
17
- const list = relayPending.get(key) || [];
18
- return new Promise((resolve, reject) => {
19
- const timer = setTimeout(() => {
20
- const idx = list.findIndex((e) => e.reqId === reqId);
21
- if (idx >= 0) list.splice(idx, 1);
22
- relayPendingByReqId.delete(reqId);
23
- reject(new Error("relay timeout"));
24
- }, 30_000);
25
- timer.unref?.();
26
- const entry = { reqId, body, hops, resolve, reject, timer };
27
- list.push(entry);
28
- relayPending.set(key, list);
29
- relayPendingByReqId.set(reqId, entry);
30
- });
31
- }
32
-
33
- function dequeueRelayForPoll({ group, target, limit = 10 }) {
34
- const key = `${group}::${target}`;
35
- const list = relayPending.get(key) || [];
36
- const batch = list.splice(0, limit);
37
- for (const e of batch) {
38
- // keep timer, but remove from pending list; result will resolve via relayResult
39
- // we keep entry in relayPendingByReqId until result arrives
40
- }
41
- if (list.length) relayPending.set(key, list);
42
- else relayPending.delete(key);
43
- return batch.map((e) => ({ reqId: e.reqId, body: e.body, hops: e.hops }));
44
- }
45
-
46
- function resolveRelay(reqId, result) {
47
- const entry = relayPendingByReqId.get(reqId);
48
- if (!entry) return false;
49
- clearTimeout(entry.timer);
50
- relayPendingByReqId.delete(reqId);
51
- entry.resolve(result);
52
- return true;
53
- }
54
-
55
- async function tryBroadbandRelay({ groups, token: myToken, model, body, hops, bus, logs, reqId, evt, res, mark, perf0, stages }) {
56
- try {
57
- const joined = loadGroupsJoined();
58
- const broadbandGroups = joined.filter((g) => g.kind === "broadband" || g.myUrl?.startsWith("relay://"));
59
- // Also consider leader-owned broadband members directly
60
- const allCandidates = [];
61
- // 1) If this node is leader, check its own groups for broadband members
62
- if (groups) {
63
- const localGroups = groups.list();
64
- for (const [gName, g] of Object.entries(localGroups)) {
65
- for (const [id, m] of Object.entries(g.members || {})) {
66
- if (id === "leader") continue;
67
- const isBb = m?.kind === "broadband" || String(m?.url || "").startsWith("relay://");
68
- if (!isBb) continue;
69
- const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
70
- if (typeof m.lastSeen === "number" && Date.now() - m.lastSeen > staleMs) continue;
71
- // local leader: enqueue directly
72
- allCandidates.push({ group: gName, target: m.url, member: m, via: "local-leader", leaderUrl: null });
73
- }
74
- }
75
- }
76
- // 2) For member nodes, check leader's broadband members (need leaderUrl)
77
- for (const g of joined) {
78
- if (!g.leaderUrl) continue; // already handled as leader
79
- try {
80
- // fetch fresh members from leader (re-use refreshGroupMembers logic but direct fetch)
81
- const controller = new AbortController();
82
- const timer = setTimeout(() => controller.abort(), 5000);
83
- const r = await fetch(`${g.leaderUrl}/v1/groups/join`, {
84
- method: "POST",
85
- headers: { "Content-Type": "application/json", "Authorization": `Bearer ${myToken}` },
86
- body: JSON.stringify({ name: g.name, memberName: g.memberName, url: g.myUrl, token: myToken }),
87
- signal: controller.signal,
88
- });
89
- clearTimeout(timer);
90
- if (!r.ok) continue;
91
- const data = await r.json().catch(() => ({}));
92
- const members = data.members || {};
93
- for (const [id, m] of Object.entries(members)) {
94
- if (id === "leader") continue;
95
- const isBb = m?.kind === "broadband" || String(m?.url || "").startsWith("relay://");
96
- if (!isBb) continue;
97
- if (m.url === g.myUrl) continue; // skip self
98
- const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
99
- if (typeof m.lastSeen === "number" && Date.now() - m.lastSeen > staleMs) continue;
100
- allCandidates.push({ group: g.name, target: m.url, member: m, via: "via-leader", leaderUrl: g.leaderUrl });
101
- }
102
- } catch {}
103
- }
104
- if (!allCandidates.length) return null;
105
- // Try each candidate via relay forward (sequential, first success wins)
106
- for (const cand of allCandidates) {
107
- try {
108
- evt?.("relay-try", { reqId, model, via: cand.via, target: cand.target, group: cand.group });
109
- if (!cand.leaderUrl) {
110
- // local leader: enqueue and wait for broadband poll (same as forward handler's local path)
111
- const fwdBody = { model, ...body, model };
112
- const reqIdLocal = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
113
- const promise = enqueueRelay({ group: cand.group, target: cand.target, reqId: reqIdLocal, body: fwdBody, hops });
114
- // also trigger a dummy poll wait: we can't directly wait for poll without D, so timeout quickly
115
- // For local leader, the D will poll; we wait for result with timeout 30s
116
- const result = await promise;
117
- if (result && result.status) {
118
- return { via: "broadband-local", result, target: cand.target, group: cand.group };
119
- }
120
- } else {
121
- // forward via leader's relay/forward endpoint
122
- const ctrl = new AbortController();
123
- const t = setTimeout(() => ctrl.abort(), 35_000);
124
- const r = await fetch(`${cand.leaderUrl}/v1/groups/relay/forward`, {
125
- method: "POST",
126
- headers: { "Content-Type": "application/json", "Authorization": `Bearer ${myToken}`, "x-mslxdff-hops": String(hops + 1) },
127
- body: JSON.stringify({ group: cand.group, target: cand.target, body: { ...body, model }, hops: hops + 1, reqId }),
128
- signal: ctrl.signal,
129
- });
130
- clearTimeout(t);
131
- if (!r.ok) {
132
- const txt = await r.text().catch(() => "");
133
- evt?.("relay-fail", { reqId, model, via: cand.via, target: cand.target, status: r.status, message: txt.slice(0,200) });
134
- continue;
135
- }
136
- // leader's forward returns the upstream response (maybe SSE)
137
- // For simplicity, we treat it as direct response to relay back to client
138
- // We can stream it back: but for now, we just return the Response
139
- return { via: "broadband-via-leader", result: r, target: cand.target, group: cand.group };
140
- }
141
- } catch (err) {
142
- evt?.("relay-fail", { reqId, model, via: cand.via, target: cand.target, message: String(err?.message || err).slice(0,200) });
143
- continue;
144
- }
145
- }
146
- return null;
147
- } catch {
148
- return null;
149
- }
150
- }
151
-
152
- export function createRouter({ token, upstream, models, auto, logs, peers, maxHops = DEFAULT_MAX_HOPS, groups, bans, bus }) {
153
- return async function router(req, res) {
154
- const method = req.method || "GET";
155
- const path = (req.url || "").split("?")[0];
156
-
157
- const route = ROUTES.find((r) => r.method === method && r.path === path);
158
-
159
- if (!route) return notFound(res);
160
-
161
- if (route.requiresAuth && !authorized(req, token)) {
162
- res.statusCode = 401;
163
- res.setHeader("WWW-Authenticate", "Bearer");
164
- return json(res, 401, { error: "Unauthorized" });
165
- }
166
-
167
- await route.handler({ req, res, upstream, models, auto, logs, peers, maxHops, groups, bans, token, bus });
168
- };
169
- }
170
-
171
- function clientIp(req) {
172
- const fwd = req.headers["x-forwarded-for"];
173
- const head = typeof fwd === "string" ? fwd.split(",")[0].trim() : "";
174
- const raw = String(head || req.socket.remoteAddress || "");
175
- return raw.replace(/^::ffff:/, "").replace(/^::1$/, "127.0.0.1") || null;
176
- }
177
-
178
- function authorized(req, token) {
179
- const header = req.headers["authorization"] || "";
180
- const match = /^Bearer (.+)$/.exec(header);
181
- if (!match) return false;
182
- const digests = (s) => createHash("sha256").update(s).digest();
183
- return timingSafeEqual(digests(match[1]), digests(token));
184
- }
185
-
186
- function json(res, status, body) {
187
- res.statusCode = status;
188
- res.setHeader("Content-Type", "application/json");
189
- res.end(JSON.stringify(body));
190
- }
191
-
192
- function notFound(res) {
193
- return json(res, 404, { error: "Not Found" });
194
- }
195
-
196
- // --- fallback 显式提示(巧妙不破兼容)---
197
- // 机器可读:x-mslxdff-* headers;人类可读:mslxdff 字段 + SSE comment
198
- function fallbackReason(lastErr) {
199
- if (!lastErr) return "cooldown";
200
- const s = Number(lastErr.status);
201
- if (s === 429) return "rate_limited";
202
- if (lastErr.message && /timeout/i.test(String(lastErr.message))) return "timeout";
203
- if (s === 502 || s === 503 || s === 504) return "upstream_error";
204
- if (s >= 400) return "upstream_error";
205
- return "fallback";
206
- }
207
-
208
- function buildFallbackInfo({ requested, actual, lastErr, via, useAuto, lockModel }) {
209
- if (!requested || !actual) return null;
210
- const alwaysHeaders = {
211
- requested_model: requested,
212
- actual_model: actual,
213
- via: via || "local",
214
- };
215
- // auto / lock 仍告知 actual,但不算 fallback
216
- if (useAuto || lockModel) {
217
- return { ...alwaysHeaders, fallback: false, reason: null, notice: null };
218
- }
219
- const isFallback = requested !== actual;
220
- if (!isFallback) {
221
- return { ...alwaysHeaders, fallback: false, reason: null, notice: null };
222
- }
223
- const reason = fallbackReason(lastErr);
224
- const reasonZh = reason === "rate_limited" ? "限流" : reason === "timeout" ? "超时" : reason === "cooldown" ? "冷却中" : "不可用";
225
- const notice = `${requested} ${reasonZh},已由 ${actual} 代答`;
226
- return { ...alwaysHeaders, fallback: true, reason, notice };
227
- }
228
-
229
- function applyFallbackHeaders(res, info) {
230
- if (!info) return;
231
- // 始终告知实际与请求,客户端对比即知
232
- if (info.requested_model) res.setHeader("x-mslxdff-requested-model", info.requested_model);
233
- if (info.actual_model) res.setHeader("x-mslxdff-actual-model", info.actual_model);
234
- if (info.via) res.setHeader("x-mslxdff-via", info.via);
235
- if (info.fallback) {
236
- res.setHeader("x-mslxdff-fallback", "1");
237
- if (info.reason) res.setHeader("x-mslxdff-fallback-reason", info.reason);
238
- // 人类 curl 可见
239
- if (info.notice) res.setHeader("x-mslxdff-notice", encodeURIComponent(info.notice));
240
- }
241
- }
242
-
243
- function enrichNonStreamJson(obj, info) {
244
- if (!info || typeof obj !== "object" || obj === null) return obj;
245
- // 仅当 fallback 时才注入顶层 mslxdff,避免噪音;但始终可通过 header 拿到 actual
246
- if (!info.fallback) return obj;
247
- if (obj.mslxdff) return obj;
248
- return {
249
- ...obj,
250
- mslxdff: {
251
- fallback: true,
252
- requested_model: info.requested_model,
253
- actual_model: info.actual_model,
254
- reason: info.reason,
255
- via: info.via,
256
- notice: info.notice,
257
- },
258
- };
259
- }
260
-
261
- function enrichSseChunkText(text, info) {
262
- if (!info?.fallback) return text;
263
- // 行级注入:对每行 data: {json} 尝试注入 mslxdff
264
- const lines = text.split("\n");
265
- let changed = false;
266
- for (let i = 0; i < lines.length; i++) {
267
- const line = lines[i];
268
- const m = /^data:\s*(\{.*\})\s*$/.exec(line);
269
- if (!m) continue;
270
- try {
271
- const obj = JSON.parse(m[1]);
272
- if (obj && typeof obj === "object" && !obj.mslxdff) {
273
- obj.mslxdff = {
274
- fallback: true,
275
- requested_model: info.requested_model,
276
- actual_model: info.actual_model,
277
- reason: info.reason,
278
- via: info.via,
279
- notice: info.notice,
280
- };
281
- lines[i] = `data: ${JSON.stringify(obj)}`;
282
- changed = true;
283
- break; // 仅注入首个 JSON 行
284
- }
285
- } catch {
286
- continue;
287
- }
288
- }
289
- return changed ? lines.join("\n") : text;
290
- }
291
-
292
- function readBody(req) {
293
- return new Promise((resolve, reject) => {
294
- let data = "";
295
- req.on("data", (c) => (data += c));
296
- req.on("end", () => {
297
- try {
298
- resolve(data ? JSON.parse(data) : {});
299
- } catch (err) {
300
- reject(err);
301
- }
302
- });
303
- req.on("error", reject);
304
- });
305
- }
306
-
307
- // Relay an upstream response to the client. Returns { status, ttfMs, aborted, interrupted, detail }
308
- // detail carries byte/chunk/sawDone diagnostics so a truncated deep-think
309
- // stream can be told apart from a clean EOF vs our stall/max vs client abort.
310
- async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort, streamTimeoutMs = STREAM_TIMEOUT_MS, fallback } = {}) {
311
- const t0 = performance.now();
312
- const contentType = upRes.headers.get("content-type") || "";
313
- const isStream = Boolean(body?.stream) || contentType.includes("text/event-stream");
314
- res.statusCode = upRes.status;
315
- if (fallback) applyFallbackHeaders(res, fallback);
316
-
317
- let ttf = null;
318
- let interrupted = false;
319
- let finishedNormally = false;
320
- const detail = {
321
- receivedChunks: 0,
322
- receivedBytes: 0,
323
- wroteChunks: 0,
324
- wroteBytes: 0,
325
- sawDone: false,
326
- sawFinishReason: null,
327
- lastChunkAtMs: null,
328
- lastChunkGapMs: null,
329
- maxGapMs: 0,
330
- stallHits: 0, // chunks where gap > SCORE_STALL_MS (quality signal, never cuts)
331
- exitReason: null, // normal | first-timeout | stall | max | upstream-error | downstream-close | empty-body
332
- upstreamError: null,
333
- downstreamClosed: false,
334
- };
335
- let prevChunkAt = t0;
336
- const onClose = () => {
337
- detail.downstreamClosed = true;
338
- if (!finishedNormally && onDownstreamAbort) onDownstreamAbort();
339
- };
340
- res.on("close", onClose);
341
-
342
- if (isStream) {
343
- res.setHeader("Content-Type", "text/event-stream");
344
- res.setHeader("Cache-Control", "no-cache");
345
- res.setHeader("Connection", "keep-alive");
346
- // SSE 注释:curl -N 可见,EventSource/SDK 自动忽略,不污染 content
347
- if (fallback?.fallback) {
348
- try {
349
- res.write(`: mslxdff fallback ${fallback.requested_model} -> ${fallback.actual_model} (${fallback.reason})\n`);
350
- res.write(`: notice ${fallback.notice}\n\n`);
351
- } catch {}
352
- }
353
- if (upRes.body) {
354
- let first = true;
355
- let wroteAny = false;
356
- let timedOut = false;
357
- let stalled = false;
358
- let tooLong = false;
359
- let stallTimer = null;
360
- const armStall = () => {
361
- if (stallTimer) clearTimeout(stallTimer);
362
- stallTimer = STALL_TIMEOUT_MS
363
- ? setTimeout(() => {
364
- stalled = true;
365
- detail.exitReason = "stall";
366
- if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
367
- }, STALL_TIMEOUT_MS)
368
- : null;
369
- };
370
- let firstTimer = setTimeout(() => {
371
- timedOut = true;
372
- detail.exitReason = "first-timeout";
373
- if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
374
- }, streamTimeoutMs);
375
- const maxTimer = MAX_STREAM_MS
376
- ? setTimeout(() => {
377
- tooLong = true;
378
- detail.exitReason = "max";
379
- if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
380
- }, MAX_STREAM_MS)
381
- : null;
382
- try {
383
- for await (const chunk of upRes.body) {
384
- const now = performance.now();
385
- detail.receivedChunks += 1;
386
- const len = chunk?.length ?? chunk?.byteLength ?? 0;
387
- detail.receivedBytes += len;
388
- const gap = Math.round(now - prevChunkAt);
389
- detail.lastChunkAtMs = Math.round(now - t0);
390
- detail.lastChunkGapMs = gap;
391
- if (gap > detail.maxGapMs) detail.maxGapMs = gap;
392
- if (gap > SCORE_STALL_MS) detail.stallHits += 1;
393
- prevChunkAt = now;
394
- // cheap inspection for diagnostics (no full parse)
395
- try {
396
- const txt = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : typeof chunk === "string" ? chunk : "";
397
- if (txt.includes("[DONE]")) detail.sawDone = true;
398
- const m = txt.match(/"finish_reason"\s*:\s*"([^"]+)"/);
399
- if (m) detail.sawFinishReason = m[1];
400
- } catch { /* ignore */ }
401
- if (timedOut || stalled || tooLong) break;
402
- if (first) {
403
- first = false;
404
- ttf = Math.round(now - t0);
405
- onFirstChunk?.(ttf);
406
- if (firstTimer) { clearTimeout(firstTimer); firstTimer = null; }
407
- }
408
- // 首块注入 mslxdff 字段(仅 fallback 时),SDK 解析 JSON 可直接发现
409
- let outChunk = chunk;
410
- if (first === false && fallback?.fallback && wroteAny === false) {
411
- try {
412
- let txt = "";
413
- if (Buffer.isBuffer(chunk)) txt = chunk.toString("utf8");
414
- else if (chunk instanceof Uint8Array) txt = Buffer.from(chunk).toString("utf8");
415
- else if (typeof chunk === "string") txt = chunk;
416
- if (txt.includes("data:")) {
417
- const enriched = enrichSseChunkText(txt, fallback);
418
- if (enriched !== txt) outChunk = Buffer.from(enriched, "utf8");
419
- }
420
- } catch {}
421
- }
422
- wroteAny = true;
423
- detail.wroteChunks += 1;
424
- detail.wroteBytes += Buffer.isBuffer(outChunk) ? outChunk.length : (outChunk?.length ?? len);
425
- res.write(outChunk);
426
- armStall(); // no-op when STALL_TIMEOUT_MS=0; scoring uses SCORE_STALL_MS gap above
427
- }
428
- if (!detail.exitReason) detail.exitReason = "normal";
429
- } catch (err) {
430
- detail.upstreamError = String(err?.message || err).slice(0, 300);
431
- detail.exitReason = "upstream-error";
432
- if (!wroteAny) timedOut = true;
433
- else stalled = true;
434
- } finally {
435
- if (firstTimer) clearTimeout(firstTimer);
436
- if (maxTimer) clearTimeout(maxTimer);
437
- if (stallTimer) clearTimeout(stallTimer);
438
- }
439
- if (timedOut && !wroteAny) {
440
- res.removeListener("close", onClose);
441
- return { status: STREAM_TIMEOUT_MS, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true, interrupted: false, detail };
442
- }
443
- if ((stalled || tooLong) && wroteAny) {
444
- interrupted = true;
445
- detail.exitReason = detail.exitReason || (stalled ? "stall" : "max");
446
- res.removeListener("close", onClose);
447
- try { res.end(); } catch { /* ignore */ }
448
- return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted, detail };
449
- }
450
- } else {
451
- detail.exitReason = "empty-body";
452
- }
453
- const totalMs = Math.round(performance.now() - t0);
454
- if (!detail.exitReason) detail.exitReason = "normal";
455
- finishedNormally = true;
456
- res.removeListener("close", onClose);
457
- try { res.end(); } catch { /* ignore */ }
458
- return { status: 200, ttfMs: ttf, totalMs, aborted: false, interrupted: false, detail };
459
- }
460
-
461
- finishedNormally = true;
462
- res.removeListener("close", onClose);
463
- const text = await upRes.text();
464
- detail.receivedBytes = Buffer.byteLength(text);
465
- detail.exitReason = "normal-non-stream";
466
- try {
467
- const parsed = JSON.parse(text);
468
- const enriched = enrichNonStreamJson(parsed, fallback);
469
- json(res, upRes.status, enriched);
470
- } catch {
471
- res.statusCode = upRes.status;
472
- res.setHeader("Content-Type", contentType || "text/plain");
473
- res.end(text);
474
- }
475
- return { status: upRes.status, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted: false, detail };
476
- }
477
-
478
- const PEER_TIMEOUT_MS = 30_000;
479
- const PEER_STATUS_TIMEOUT_MS = 2_000;
480
-
481
- // Ask a peer which of its models are healthy (status normal or never
482
- // failed). Returns model ids ordered as the peer listed them; empty when the
483
- // peer is unreachable, unauthorized, or has no healthy model.
484
- export async function peerHealthyModels(peer, { timeoutMs = PEER_STATUS_TIMEOUT_MS, fetchImpl = fetch } = {}) {
485
- try {
486
- const res = await fetchImpl(`${peer.url}/v1/models/status`, {
487
- headers: {
488
- "Authorization": `Bearer ${peer.token || ""}`,
489
- "Accept": "application/json",
490
- },
491
- signal: AbortSignal.timeout(timeoutMs),
492
- });
493
- if (!res.ok) return [];
494
- const json = await res.json().catch(() => ({}));
495
- return (json.data || [])
496
- .filter((m) => m && typeof m.id === "string" && m.status === "normal")
497
- .map((m) => m.id);
498
- } catch {
499
- return [];
500
- }
501
- }
502
-
503
- export const PROMPT_MAX_LEN = 160;
504
-
505
- // Human-debuggable summary of the request body: the last non-empty message
506
- // text (multi-modal parts joined), whitespace-flattened and truncated.
507
- export function summarizePrompt(body) {
508
- const msgs = body?.messages;
509
- if (!Array.isArray(msgs) || !msgs.length) return "";
510
- const msg = msgs[msgs.length - 1];
511
- const c = msg?.content;
512
- let text = "";
513
- if (typeof c === "string") text = c;
514
- else if (Array.isArray(c)) {
515
- text = c
516
- .map((p) => (typeof p === "string" ? p : p && typeof p.text === "string" ? p.text : ""))
517
- .join(" ");
518
- }
519
- text = String(text || "").replace(/\s+/g, " ").trim();
520
- return text.length > PROMPT_MAX_LEN ? text.slice(0, PROMPT_MAX_LEN) + "…" : text;
521
- }
522
-
523
- function parseHops(header) {
524
- const n = Number(header);
525
- return Number.isInteger(n) && n >= 0 ? n : 0;
526
- }
527
-
528
- async function forwardToPeer(peer, body, model, hops) {
529
- const controller = new AbortController();
530
- const timer = setTimeout(() => controller.abort(), PEER_TIMEOUT_MS);
531
- try {
532
- return await fetch(`${peer.url}/v1/chat/completions`, {
533
- method: "POST",
534
- headers: {
535
- "Content-Type": "application/json",
536
- "Authorization": `Bearer ${peer.token}`,
537
- "x-mslxdff-hops": String(hops + 1),
538
- "x-mslxdff-model-lock": model,
539
- "Accept": "text/event-stream",
540
- },
541
- body: JSON.stringify({ ...body, model }),
542
- signal: controller.signal,
543
- });
544
- } catch (err) {
545
- return err;
546
- } finally {
547
- clearTimeout(timer);
548
- }
549
- }
550
-
551
- // Resolve the model a peer should serve for this request.
552
- // 原设计严格语义:显式指定模型时,永远用该模型去试 peer,不因 peer 的本地 healthy 状态而偷换成 hy3。
553
- // 只有 auto 模式才走 healthy 探测与择优。
554
- // Returns { peer, target } or null when the peer is unusable.
555
- async function resolvePeerTarget(ctx, peer) {
556
- const prevModel = ctx.peers.stat(peer.url)?.model;
557
- const hot = ctx.peers.isHot(peer.url) && prevModel === ctx.model;
558
- if (hot) return { peer, target: prevModel };
559
- // 显式模型:严格用请求模型,不做 healthy 偷换(B/D 必须以 deepseek 去试,失败才算该模型在该 peer 不可用)
560
- const isExplicit = !!ctx.model && !isAutoModel(ctx.model);
561
- if (isExplicit) {
562
- // 仅做可达性探测:轻量 ping /v1/models/status 判断 peer 是否活着,不因模型状态过滤
563
- const healthy = await peerHealthyModels(peer);
564
- if (!healthy.length) {
565
- // 无法探活也仍尝试:让 forward 去试,失败会由 race 逻辑记错;但为保持原有“全不健康则跳过”行为,仍标记
566
- // 这里改为:即使 healthy 为空,也返回 target=ctx.model,让上游去判 429,而不是直接丢弃 peer
567
- // 只有当 fetch 本身异常(healthy=[] 来自网络错)才视为 peer 不可用,需区分
568
- // peerHealthyModels 在网络错时返回 [],此时应视为 peer 不可用
569
- // 我们通过再次轻量探测区分:若 peer 完全不可达,healthy=[] 且 peer 曾无成功记录,则跳过
570
- // 简化:若 healthy 为空,直接尝试目标模型,失败再记错(更符合“严格”)
571
- ctx.evt("peer-health", { peer: peer.url, healthy: [], count: 0, strict: true });
572
- return { peer, target: ctx.model };
573
- }
574
- ctx.evt("peer-health", { peer: peer.url, healthy, count: healthy.length, strict: true });
575
- return { peer, target: ctx.model };
576
- }
577
- // auto 模式:走原有择优逻辑
578
- const healthy = await peerHealthyModels(peer);
579
- if (!healthy.length) {
580
- await ctx.peers.recordError(peer.url);
581
- ctx.logError(ctx.model, 0, `peer ${peer.url} has no healthy models`);
582
- ctx.evt("peer-health", { peer: peer.url, healthy: [], count: 0 });
583
- return null;
584
- }
585
- ctx.evt("peer-health", { peer: peer.url, healthy, count: healthy.length });
586
- return { peer, target: healthy.includes(ctx.model) ? ctx.model : healthy[0] };
587
- }
588
-
589
- // Race a batch of candidates: up to PEER_RACE_LIMIT at a time, first success
590
- // wins. Uses ctx.model/body/hops/peers to resolve targets and forward. Retries
591
- // remaining candidates in subsequent batches. Returns the winning
592
- // { peer, target, res, latencyMs } or null when everyone failed.
593
- export const PEER_RACE_LIMIT = Number(process.env.MSLXDFF_PEER_RACE_LIMIT) > 0
594
- ? Number(process.env.MSLXDFF_PEER_RACE_LIMIT)
595
- : 3;
596
-
597
- // A model whose whole request takes longer than this wall-clock duration is
598
- // remembered as slow and demoted, so a fast model is preferred next request.
599
- // Set MSLXDFF_SLOW_TOTAL_MS=0 to disable.
600
- export const SLOW_TOTAL_MS = (() => {
601
- const n = Number(process.env.MSLXDFF_SLOW_TOTAL_MS);
602
- return Number.isInteger(n) && n > 0 ? n : 20_000;
603
- })();
604
-
605
- // How long to wait for the first chunk of a streamed response before giving up
606
- // on that model (nothing has been written yet, so we can fail over cleanly).
607
- // Set MSLXDFF_STREAM_TIMEOUT_MS=0 to disable the circuit breaker.
608
- export const STREAM_TIMEOUT_MS = (() => {
609
- const n = Number(process.env.MSLXDFF_STREAM_TIMEOUT_MS);
610
- return Number.isInteger(n) && n > 0 ? n : 25_000;
611
- })();
612
-
613
- // Stall / max ceilings — disabled by default for relays (we never cut a
614
- // stream that has already started; different models have different verbosity,
615
- // that's normal). Stall is kept only as a *quality* signal for ranking.
616
- // Set MSLXDFF_STALL_TIMEOUT_MS=15000 to re-enable cutting (not recommended),
617
- // or tune MSLXDFF_SCORE_STALL_MS for scoring.
618
- export const STALL_TIMEOUT_MS = (() => {
619
- const n = Number(process.env.MSLXDFF_STALL_TIMEOUT_MS);
620
- return Number.isInteger(n) && n > 0 ? n : 0;
621
- })();
622
-
623
- export const SCORE_STALL_MS = (() => {
624
- const raw = process.env.MSLXDFF_SCORE_STALL_MS ?? process.env.MSLXDFF_STALL_TIMEOUT_MS;
625
- const n = Number(raw);
626
- return Number.isInteger(n) && n > 0 ? n : 15_000;
627
- })();
628
-
629
- export const MAX_STREAM_MS = (() => {
630
- const n = Number(process.env.MSLXDFF_MAX_STREAM_MS);
631
- return Number.isInteger(n) && n > 0 ? n : 0;
632
- })();
633
-
634
- async function racePeerCandidates(candidates, ctx) {
635
- for (let i = 0; i < candidates.length; i += PEER_RACE_LIMIT) {
636
- const batch = candidates.slice(i, i + PEER_RACE_LIMIT);
637
- // resolve targets for this batch first (may probe), then fire them together
638
- const prepared = (await Promise.all(batch.map((peer) => resolvePeerTarget(ctx, peer)))).filter(Boolean);
639
- if (!prepared.length) continue;
640
- // fire every peer in the batch concurrently and record completion order —
641
- // the first one to succeed wins the race
642
- const completed = await new Promise((resolve) => {
643
- const order = [];
644
- const total = prepared.length;
645
- for (const { peer, target } of prepared) {
646
- ctx.evt("peer-request", { peer: peer.url, model: target, hops: ctx.hops + 1 });
647
- const t0 = performance.now();
648
- forwardToPeer(peer, ctx.body, target, ctx.hops).then((res) => {
649
- const latencyMs = Math.round(performance.now() - t0);
650
- const failed = res instanceof Error || res.status >= 400;
651
- ctx.evt("peer-forward", { peer: peer.url, model: target, hops: ctx.hops + 1, latencyMs, ok: !failed });
652
- if (failed) {
653
- const status = res instanceof Error ? 502 : res.status;
654
- ctx.logError(ctx.model,
655
- status,
656
- res instanceof Error ? errMsg(res) : `peer ${status}`);
657
- ctx.evt("peer-error", {
658
- peer: peer.url,
659
- model: target,
660
- status,
661
- message: res instanceof Error ? errMsg(res) : null,
662
- });
663
- order.push({ ok: false, peer, target, res, status });
664
- } else {
665
- order.push({ ok: true, peer, target, res, latencyMs });
666
- }
667
- if (order.length === total) resolve(order);
668
- });
669
- }
670
- });
671
- const winner = completed.find((o) => o.ok);
672
- if (winner) {
673
- // every other responder also gets its memory cleared and its stats warmed
674
- // so it becomes a candidate next time too
675
- for (const o of completed) {
676
- if (o === winner) continue;
677
- if (!o.ok) {
678
- await ctx.peers.recordError(o.peer.url);
679
- await ctx.peers.recordResult(o.peer.url, { ok: false });
680
- } else {
681
- await ctx.peers.recordResult(o.peer.url, { ok: true, latencyMs: o.latencyMs, model: o.target });
682
- }
683
- }
684
- return { peer: winner.peer, target: winner.target, res: winner.res, latencyMs: winner.latencyMs };
685
- }
686
- for (const o of completed) {
687
- await ctx.peers.recordError(o.peer.url);
688
- await ctx.peers.recordResult(o.peer.url, { ok: false });
689
- }
690
- }
691
- return null;
692
- }
693
-
694
- const ROUTES = [
695
- {
696
- method: "GET",
697
- path: "/health",
698
- handler: ({ res }) => json(res, 200, { status: "ok" }),
699
- },
700
- {
701
- method: "POST",
702
- path: "/v1/chat/completions",
703
- requiresAuth: true,
704
- handler: async ({ req, res, upstream, auto, logs, peers, maxHops, groups, bus }) => {
705
- let body;
706
- try {
707
- body = await readBody(req);
708
- } catch {
709
- return json(res, 400, { error: "Invalid JSON body" });
710
- }
711
-
712
- const startedAt = Date.now();
713
- const reqId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
714
- const perf0 = performance.now();
715
- const stages = [];
716
- const mark = (name) => stages.push([name, Math.round(performance.now() - perf0)]);
717
- const hops = parseHops(req.headers["x-mslxdff-hops"]);
718
- const lockModel = req.headers["x-mslxdff-model-lock"] || "";
719
- const rawModel = body.model || "";
720
- const requested = normalizeModel(lockModel || rawModel || "");
721
- const useAuto = isAutoModel(requested);
722
- mark("parsed");
723
-
724
- let order;
725
- if (lockModel) {
726
- order = [requested];
727
- } else if (useAuto) {
728
- order = auto ? await auto.candidates() : [""];
729
- } else {
730
- order = auto ? await auto.candidatesFor(requested) : [requested];
731
- }
732
- if (!order.length) order = [""];
733
- const canFallback = order.length > 1;
734
- const canForwardPeers = Boolean(peers) && hops < maxHops;
735
- mark("ordered");
736
-
737
- const logCall = (model, status) =>
738
- logs?.appendCall({ reqId, model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream), stages });
739
- const logError = (model, status, message) =>
740
- logs?.appendError({ reqId, model, auto: useAuto, status, message, stages });
741
- const evt = (type, data) => {
742
- const entry = { ts: Date.now(), reqId, type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
743
- if (bus) bus.emit(entry);
744
- logs?.appendEvent?.(entry);
745
- };
746
- evt("request", { reqId, hops, ip: clientIp(req), stream: Boolean(body.stream), prompt: summarizePrompt(body), rawModel, requested, lockModel: lockModel || null });
747
- evt("ordered", { reqId, order, canFallback, canForwardPeers, useAuto, statuses: auto?.statuses?.() ?? null });
748
-
749
- // Shared context for the peer race helpers below (each model iteration
750
- // reuses it; `model` is bound per iteration call).
751
- const handlerCtx = {
752
- model: null,
753
- body,
754
- hops,
755
- peers,
756
- evt,
757
- logError,
758
- logCall,
759
- };
760
-
761
- let lastErr = null;
762
- for (let idx = 0; idx < order.length; idx++) {
763
- const model = order[idx];
764
- handlerCtx.model = model;
765
- evt("model-try", { reqId, model, idx, remaining: order.length - idx });
766
- let upRes = null;
767
- const forwarded = { ...injectReasoningContent(model, body), model };
768
- const tUp = performance.now();
769
- evt("upstream-try", { reqId, model, attempt: idx + 1 });
770
- try {
771
- upRes = await upstream.chat(forwarded);
772
- evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
773
- } catch (err) {
774
- if (auto) await auto.recordError(model, { message: errMsg(err) });
775
- lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
776
- logError(model, 502, errMsg(err));
777
- evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
778
- }
779
- mark(`up-${model}`);
780
- if (upRes && upRes.status >= 400) {
781
- if (auto) await auto.recordError(model, { status: upRes.status });
782
- lastErr = { model, upstream: upRes, status: upRes.status, message: null };
783
- logError(model, upRes.status, `upstream ${upRes.status}`);
784
- evt("upstream-error", { reqId, model, status: upRes.status, message: null, timing: upRes._t ?? null });
785
- upRes = null;
786
- }
787
- if (upRes) {
788
- logCall(model, upRes.status);
789
- const fallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "local", useAuto, lockModel });
790
- if (fallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: fallback.reason, notice: fallback.notice, via: "local" });
791
- evt("relay-start", { reqId, model, via: "local", isStream: Boolean(body.stream), fallback });
792
- const out = await relay(res, upRes, body, {
793
- fallback,
794
- onFirstChunk: (delta) => {
795
- mark(`ttf-${model}`);
796
- evt("relay-first-chunk", { reqId, model, ttfMs: delta });
797
- },
798
- onDownstreamAbort: () => {
799
- evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] });
800
- },
801
- });
802
- evt("relay-done", { reqId, model, via: "local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
803
- if (out.status === STREAM_TIMEOUT_MS) {
804
- if (auto) await auto.recordError(model, { status: 502, slow: true, note: `stream timeout ${STREAM_TIMEOUT_MS}ms` });
805
- lastErr = { model, upstream: null, status: 502, message: `stream timed out after ${STREAM_TIMEOUT_MS}ms` };
806
- logError(model, 502, `stream timeout ${STREAM_TIMEOUT_MS}ms`);
807
- evt("upstream-error", { reqId, model, status: 502, message: "stream timeout", timing: null });
808
- evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: "stream timeout" });
809
- upRes = null;
810
- continue;
811
- }
812
- if (out.interrupted) {
813
- if (auto) {
814
- await auto.recordError(model, { status: 200, slow: true, note: `stall ${STALL_TIMEOUT_MS}ms` });
815
- await auto.recordLatency(model, out.totalMs ?? (Date.now() - startedAt));
816
- }
817
- evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: STALL_TIMEOUT_MS, interrupted: true, detail: out.detail ?? null });
818
- logCall(model, 200);
819
- evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true, detail: out.detail ?? null, fallback, requested, actual: model });
820
- evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, interrupted: true, reqId });
821
- return;
822
- }
823
- const elapsed = Date.now() - startedAt;
824
- const latencyMs = out.totalMs ?? elapsed;
825
- let scoredSlow = false;
826
- if (SLOW_TOTAL_MS && auto && elapsed > SLOW_TOTAL_MS && out.status === 200) {
827
- void auto.recordError(model, { status: 200, slow: true, note: `slow ${elapsed}ms` });
828
- void auto.recordLatency(model, latencyMs);
829
- evt("slow-model", { model, elapsedMs: elapsed, threshold: SLOW_TOTAL_MS, reason: "total", detail: out.detail ?? null });
830
- scoredSlow = true;
831
- }
832
- if (out.detail?.stallHits > 0 && auto && out.status === 200) {
833
- void auto.recordError(model, { status: 200, slow: true, note: `stall ${out.detail.stallHits}x gap>${SCORE_STALL_MS}ms maxGap ${out.detail.maxGapMs}ms` });
834
- void auto.recordLatency(model, latencyMs);
835
- evt("slow-model", { model, elapsedMs: elapsed, threshold: SCORE_STALL_MS, reason: "stall", stallHits: out.detail.stallHits, maxGapMs: out.detail.maxGapMs, detail: out.detail ?? null });
836
- scoredSlow = true;
837
- }
838
- if (!scoredSlow && auto && out.status === 200) {
839
- await auto.recordOk(model, { latencyMs });
840
- } else if (!scoredSlow && auto) {
841
- // still update latency for non-200? keep for completeness
842
- await auto.recordLatency(model, latencyMs);
843
- } else if (scoredSlow && out.detail) {
844
- // already recorded slow+latency above, still ensure latency EMA is updated for slow case (done)
845
- }
846
- evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback, requested, actual: model });
847
- evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, reqId });
848
- return;
849
- }
850
-
851
- // local failed for this model: race the peers — send the request to
852
- // up to N of them in parallel, first success wins (the winner's model
853
- // is remembered for the next call). If every candidate fails, retry
854
- // once ordered by recovery time (earliest failure first), which
855
- // favours the peer that has had the longest to come back.
856
- if (canForwardPeers) {
857
- evt("peer-race-start", { reqId, model, peers: peers.ordered().length });
858
- const win =
859
- (await racePeerCandidates(peers.ordered(), handlerCtx)) ||
860
- (await racePeerCandidates(peers.orderedByLastError(), handlerCtx));
861
- if (win) {
862
- evt("peer-race-win", { reqId, model, winPeer: win.peer.url, winTarget: win.target, latencyMs: win.latencyMs });
863
- await peers.recordResult(win.peer.url, { ok: true, latencyMs: win.latencyMs, model: win.target });
864
- logCall(win.target, win.res.status);
865
- const peerFallback = buildFallbackInfo({ requested, actual: win.target, lastErr, via: "peer", useAuto, lockModel });
866
- if (peerFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: win.target, reason: peerFallback.reason, notice: peerFallback.notice, via: "peer" });
867
- evt("relay-start", { reqId, model: win.target, via: "peer", isStream: Boolean(body.stream), fallback: peerFallback });
868
- const out = await relay(res, win.res, body, {
869
- fallback: peerFallback,
870
- onFirstChunk: (d) => mark(`ttf-peer-${win.target}`),
871
- onDownstreamAbort: () => evt("client-abort", { reqId, model: win.target, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
872
- });
873
- evt("relay-done", { reqId, model: win.target, via: "peer", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
874
- if (auto && out.status === 200) {
875
- const latencyMs = out.totalMs ?? win.latencyMs;
876
- if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
877
- void auto.recordError(win.target, { status: 200, slow: true, note: `peer slow ${latencyMs}ms` });
878
- void auto.recordLatency(win.target, latencyMs);
879
- } else {
880
- await auto.recordOk(win.target, { latencyMs });
881
- }
882
- }
883
- evt("result", { model: win.target, status: out.status, via: "peer", timing: win.res._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: peerFallback, requested, actual: win.target });
884
- evt("client-response", { requested, actual: win.target, via: "peer", fallback: peerFallback, status: out.status, reqId });
885
- return;
886
- }
887
- evt("peer-race-lose", { reqId, model });
888
- }
889
-
890
- // Broadband relay: try to forward via leader to a broadband member's quota
891
- if (groups) {
892
- const bb = await tryBroadbandRelay({ groups, token, model, body, hops, bus, logs, reqId, evt, res, mark, perf0, stages });
893
- if (bb) {
894
- const isResponse = bb.result && typeof bb.result.status === "number" && typeof bb.result.headers?.get === "function";
895
- if (isResponse) {
896
- // streaming response from leader's forward (which waited for broadband)
897
- const bbFallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "broadband", useAuto, lockModel });
898
- if (bbFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: bbFallback.reason, notice: bbFallback.notice, via: "broadband" });
899
- evt("relay-start", { reqId, model, via: "broadband", target: bb.target, group: bb.group, fallback: bbFallback });
900
- const out = await relay(res, bb.result, body, {
901
- fallback: bbFallback,
902
- onFirstChunk: (d) => mark(`ttf-bb-${model}`),
903
- onDownstreamAbort: () => evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
904
- });
905
- evt("relay-done", { reqId, model, via: "broadband", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
906
- if (auto && out.status === 200) {
907
- const latencyMs = out.totalMs ?? 0;
908
- if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
909
- void auto.recordError(model, { status: 200, slow: true, note: `broadband slow ${latencyMs}ms` });
910
- void auto.recordLatency(model, latencyMs);
911
- } else {
912
- await auto.recordOk(model, { latencyMs });
913
- }
914
- }
915
- evt("result", { model, status: out.status, via: "broadband", timing: bb.result._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: bbFallback, requested, actual: model });
916
- evt("client-response", { requested, actual: model, via: "broadband", fallback: bbFallback, status: out.status, reqId });
917
- return;
918
- } else if (bb.result && typeof bb.result.status === "number") {
919
- // buffered result from local leader enqueue
920
- const fakeRes = {
921
- status: bb.result.status,
922
- headers: { get: (k) => bb.result.headers?.[k] || bb.result.headers?.[k.toLowerCase()] || null },
923
- text: async () => typeof bb.result.body === "string" ? bb.result.body : JSON.stringify(bb.result.body),
924
- body: (() => {
925
- const b = bb.result.body || "";
926
- const str = typeof b === "string" ? b : JSON.stringify(b);
927
- const isSSE = bb.result.headers?.["Content-Type"]?.includes("text/event-stream");
928
- if (isSSE) {
929
- return (async function* () { yield Buffer.from(str); })();
930
- }
931
- return null;
932
- })(),
933
- };
934
- const bbLocalFallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "broadband", useAuto, lockModel });
935
- if (bbLocalFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: bbLocalFallback.reason, notice: bbLocalFallback.notice, via: "broadband" });
936
- evt("relay-start", { reqId, model, via: "broadband-local", target: bb.target, group: bb.group, fallback: bbLocalFallback });
937
- const out = await relay(res, fakeRes, body, {
938
- fallback: bbLocalFallback,
939
- onFirstChunk: (d) => mark(`ttf-bb-${model}`),
940
- onDownstreamAbort: () => evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
941
- });
942
- evt("relay-done", { reqId, model, via: "broadband-local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
943
- evt("result", { model, status: out.status, via: "broadband", timing: null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: bbLocalFallback, requested, actual: model });
944
- evt("client-response", { requested, actual: model, via: "broadband", fallback: bbLocalFallback, status: out.status, reqId });
945
- return;
946
- }
947
- }
948
- evt("relay-miss", { reqId, model });
949
- }
950
-
951
- if (canFallback) {
952
- evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` });
953
- continue;
954
- }
955
- evt("exhausted-local", { reqId, lastModel: lastErr?.model ?? model, lastStatus: lastErr?.status ?? 502, order });
956
- logCall(lastErr?.model ?? model, lastErr?.status ?? 502);
957
- if (lastErr?.upstream) {
958
- evt("relay-start", { reqId, model: lastErr.model, via: "local-exhausted", isStream: Boolean(body.stream) });
959
- const out = await relay(res, lastErr.upstream, body, {
960
- onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
961
- onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
962
- });
963
- evt("relay-done", { reqId, model: lastErr.model, via: "local-exhausted", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
964
- evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
965
- return;
966
- }
967
- evt("result", { reqId, model, status: lastErr?.status ?? 502, via: "none", timing: null });
968
- return json(res, 502, { error: lastErr?.message || "all auto models failed" });
969
- }
970
-
971
- evt("exhausted-all", { reqId, lastModel: lastErr?.model ?? requested, lastStatus: lastErr?.status ?? 502, order });
972
- logCall(lastErr?.model ?? requested, lastErr?.status ?? 502);
973
- if (lastErr?.upstream) {
974
- evt("relay-start", { reqId, model: lastErr.model, via: "local-final", isStream: Boolean(body.stream) });
975
- const out = await relay(res, lastErr.upstream, body, {
976
- onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
977
- onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
978
- });
979
- evt("relay-done", { reqId, model: lastErr.model, via: "local-final", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
980
- evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
981
- return;
982
- }
983
- evt("result", { reqId, model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none", timing: null });
984
- return json(res, 502, { error: lastErr?.message || "all auto models failed" });
985
- },
986
- },
987
- {
988
- method: "POST",
989
- path: "/v1/groups/join",
990
- requiresAuth: false,
991
- handler: async ({ req, res, groups, token, bans }) => {
992
- if (!groups) return json(res, 501, { error: "Groups service not configured" });
993
- const ip = clientIp(req);
994
- const banned = bans?.isBanned(ip);
995
- if (banned) {
996
- return json(res, 403, { error: `banned until ${new Date(banned.until).toISOString()}` });
997
- }
998
- let body;
999
- try {
1000
- body = await readBody(req);
1001
- } catch {
1002
- return json(res, 400, { error: "Invalid JSON body" });
1003
- }
1004
- if (!body?.name) return json(res, 400, { error: "group name is required" });
1005
-
1006
- const fail = (msg) => {
1007
- try {
1008
- if (bans) {
1009
- const b = bans.recordFailure(ip);
1010
- if (b) console.error(`${ip} banned (${bans.threshold} failed joins)`);
1011
- }
1012
- } catch {
1013
- // ban bookkeeping must never break the join endpoint
1014
- }
1015
- return json(res, 403, { error: msg });
1016
- };
1017
-
1018
- // Already-registered members re-register (sync) using their bearer token;
1019
- // new members must present the group name (which IS the password).
1020
- if (!body.key) {
1021
- const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
1022
- const hit = auth && groups.membersForToken(body.name, auth[1]);
1023
- if (!hit) return fail("invalid member token");
1024
- try {
1025
- const isBroadbandRe = String(body.url || hit.member?.url || "").startsWith("relay://") || body.kind === "broadband" || hit.member?.kind === "broadband";
1026
- const extra = {};
1027
- if (isBroadbandRe) {
1028
- extra.kind = "broadband";
1029
- extra.publicIp = ip;
1030
- extra.lastSeen = Date.now();
1031
- if (body.url) extra.url = String(body.url);
1032
- }
1033
- const refreshed = groups.upsertMember(body.name, {
1034
- memberName: body.memberName,
1035
- url: body.url || hit.member.url,
1036
- token: body.token || hit.member.token,
1037
- kind: extra.kind,
1038
- publicIp: extra.publicIp,
1039
- lastSeen: extra.lastSeen,
1040
- });
1041
- // also ensure broadband's publicIp/lastSeen updated even if url same
1042
- if (isBroadbandRe && refreshed) {
1043
- const targetId = Object.keys(refreshed).find((k) => refreshed[k].url === (body.url || hit.member.url));
1044
- if (targetId) {
1045
- refreshed[targetId].publicIp = ip;
1046
- refreshed[targetId].lastSeen = Date.now();
1047
- refreshed[targetId].kind = "broadband";
1048
- }
1049
- }
1050
- return json(res, 200, { object: "group", name: body.name, members: refreshed });
1051
- } catch (err) {
1052
- return json(res, 400, { error: errMsg(err) });
1053
- }
1054
- }
1055
-
1056
- try {
1057
- const youPort = Number(body.myPort);
1058
- const youUrl = Number.isInteger(youPort) && youPort > 0 ? `http://${ip}:${youPort}` : "";
1059
- let memberUrl = String(body.url || youUrl);
1060
- if (!memberUrl) throw new Error("member url is required");
1061
- const isBroadband = String(memberUrl).startsWith("relay://") || body.kind === "broadband";
1062
- if (isBroadband) {
1063
- // broadband: keep relay:// url, record publicIp from source ip
1064
- memberUrl = String(body.url || memberUrl);
1065
- }
1066
- const members = groups.addMember(body.name, {
1067
- key: body.key,
1068
- memberName: body.memberName,
1069
- url: memberUrl,
1070
- token: body.token,
1071
- kind: isBroadband ? "broadband" : "static",
1072
- publicIp: isBroadband ? ip : undefined,
1073
- lastSeen: isBroadband ? Date.now() : undefined,
1074
- });
1075
- // 5 wrong passwords bans the source IP (48h) — see createBansService.
1076
- if (bans) bans.clear(ip);
1077
- // First join seeds the leader's own entry using the addr the joiner saw,
1078
- // so -creategroup needs no address argument.
1079
- if (!members.leader) {
1080
- const leaderUrl = String(body.leaderUrl || "").replace(/\/+$/, "");
1081
- if (leaderUrl) {
1082
- groups.upsertMember(body.name, { memberName: "leader", url: leaderUrl, token });
1083
- Object.assign(members, { leader: { url: leaderUrl, token } });
1084
- }
1085
- }
1086
- // Tell the joiner the url we registered them under (source IP + their port),
1087
- // so they can exclude themselves from their own peer list.
1088
- json(res, 200, { object: "group", name: body.name, members, you: { url: memberUrl } });
1089
- } catch (err) {
1090
- return fail(errMsg(err));
1091
- }
1092
- },
1093
- },
1094
- {
1095
- method: "POST",
1096
- path: "/v1/groups/leave",
1097
- requiresAuth: false,
1098
- handler: async ({ req, res, groups }) => {
1099
- if (!groups) return json(res, 501, { error: "Groups service not configured" });
1100
- let body;
1101
- try {
1102
- body = await readBody(req);
1103
- } catch {
1104
- return json(res, 400, { error: "Invalid JSON body" });
1105
- }
1106
- if (!body?.name) return json(res, 400, { error: "group name is required" });
1107
- const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
1108
- if (!auth) return json(res, 401, { error: "bearer token required" });
1109
- const group = groups.list()[body.name];
1110
- if (!group) return json(res, 404, { error: `group "${body.name}" not found` });
1111
- const hit = groups.membersForToken(body.name, auth[1]);
1112
- if (!hit) return json(res, 403, { error: "invalid member token" });
1113
- try {
1114
- const removed = groups.removeMember(body.name, { url: hit.member.url });
1115
- return json(res, 200, {
1116
- object: "group",
1117
- name: body.name,
1118
- removed: removed?.removed ?? null,
1119
- members: groups.list()[body.name]?.members ?? {},
1120
- });
1121
- } catch (err) {
1122
- return json(res, 400, { error: errMsg(err) });
1123
- }
1124
- },
1125
- },
1126
- {
1127
- method: "POST",
1128
- path: "/v1/groups/relay/heartbeat",
1129
- requiresAuth: false,
1130
- handler: async ({ req, res, groups, bus, logs }) => {
1131
- const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
1132
- if (!auth) return json(res, 401, { error: "bearer token required" });
1133
- let body;
1134
- try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
1135
- const groupName = body?.name || body?.group;
1136
- if (!groupName) return json(res, 400, { error: "group name is required" });
1137
- const hit = groups?.membersForToken(groupName, auth[1]);
1138
- if (!hit) return json(res, 403, { error: "invalid member token" });
1139
- const ip = clientIp(req);
1140
- try {
1141
- const memberUrl = hit.member?.url;
1142
- const members = groups.list()[groupName]?.members || {};
1143
- const targetId = Object.keys(members).find((k) => members[k].url === memberUrl) || hit.member?.url;
1144
- // update lastSeen/publicIp for broadband
1145
- if (hit.member?.kind === "broadband" || String(memberUrl).startsWith("relay://")) {
1146
- const m = members[targetId] || hit.member;
1147
- if (m) {
1148
- m.publicIp = ip;
1149
- m.lastSeen = Date.now();
1150
- // persist via groups service (upsert)
1151
- try { groups.upsertMember(groupName, { memberName: targetId, url: m.url, token: m.token, kind: "broadband", publicIp: ip, lastSeen: m.lastSeen }); } catch {}
1152
- }
1153
- const evtData = { ts: Date.now(), type: "relay-heartbeat", member: targetId, ip, lastSeen: m?.lastSeen, group: groupName };
1154
- if (bus) bus.emit(evtData);
1155
- logs?.appendEvent?.(evtData);
1156
- }
1157
- return json(res, 200, { object: "heartbeat", ok: true, ip, lastSeen: Date.now() });
1158
- } catch (err) {
1159
- return json(res, 400, { error: errMsg(err) });
1160
- }
1161
- },
1162
- },
1163
- {
1164
- method: "POST",
1165
- path: "/v1/groups/relay/poll",
1166
- requiresAuth: false,
1167
- handler: async ({ req, res, groups }) => {
1168
- const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
1169
- if (!auth) return json(res, 401, { error: "bearer token required" });
1170
- let body;
1171
- try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
1172
- const groupName = body?.name || body?.group;
1173
- if (!groupName) return json(res, 400, { error: "group name is required" });
1174
- const hit = groups?.membersForToken(groupName, auth[1]);
1175
- if (!hit) return json(res, 403, { error: "invalid member token" });
1176
- const targetUrl = hit.member?.url;
1177
- if (!targetUrl) return json(res, 400, { error: "member url not found" });
1178
- const batch = dequeueRelayForPoll({ group: groupName, target: targetUrl, limit: 10 });
1179
- return json(res, 200, { object: "poll", data: batch });
1180
- },
1181
- },
1182
- {
1183
- method: "POST",
1184
- path: "/v1/groups/relay/result",
1185
- requiresAuth: false,
1186
- handler: async ({ req, res, groups }) => {
1187
- const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
1188
- if (!auth) return json(res, 401, { error: "bearer token required" });
1189
- let body;
1190
- try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
1191
- const groupName = body?.name || body?.group;
1192
- const reqId = body?.reqId;
1193
- if (!groupName || !reqId) return json(res, 400, { error: "group and reqId required" });
1194
- const hit = groups?.membersForToken(groupName, auth[1]);
1195
- if (!hit) return json(res, 403, { error: "invalid member token" });
1196
- const ok = resolveRelay(reqId, body.result || body);
1197
- if (!ok) return json(res, 404, { error: "pending request not found or timed out" });
1198
- return json(res, 200, { object: "result", ok: true });
1199
- },
1200
- },
1201
- {
1202
- method: "POST",
1203
- path: "/v1/groups/relay/forward",
1204
- requiresAuth: true,
1205
- handler: async ({ req, res, groups, bus, logs }) => {
1206
- let body;
1207
- try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
1208
- const groupName = body?.group || body?.name;
1209
- const target = body?.target || body?.url;
1210
- const hops = parseHops(req.headers["x-mslxdff-hops"] || body?.hops);
1211
- if (!groupName || !target) return json(res, 400, { error: "group and target required" });
1212
- if (hops >= DEFAULT_MAX_HOPS) return json(res, 429, { error: "max hops exceeded" });
1213
- const members = groups?.list()[groupName]?.members || {};
1214
- const targetMember = Object.values(members).find((m) => m.url === target) || Object.entries(members).find(([id]) => id === target)?.[1];
1215
- if (!targetMember) return json(res, 404, { error: `target ${target} not found in group ${groupName}` });
1216
- const isBb = targetMember.kind === "broadband" || String(targetMember.url).startsWith("relay://");
1217
- if (!isBb) {
1218
- // static: direct fetch (should have been handled by peer race, but support via relay as well)
1219
- try {
1220
- const fwdBody = body.body || body;
1221
- const r = await fetch(`${targetMember.url}/v1/chat/completions`, {
1222
- method: "POST",
1223
- headers: { "Content-Type": "application/json", "Authorization": `Bearer ${targetMember.token || ""}`, "x-mslxdff-hops": String(hops + 1), "x-mslxdff-model-lock": fwdBody.model || "", "Accept": "text/event-stream" },
1224
- body: JSON.stringify(fwdBody),
1225
- });
1226
- const evtData = { ts: Date.now(), type: "relay-forward", target, via: "direct", group: groupName, hops };
1227
- if (bus) bus.emit(evtData);
1228
- logs?.appendEvent?.(evtData);
1229
- res.statusCode = r.status;
1230
- if (r.headers.get("content-type")?.includes("text/event-stream")) {
1231
- res.setHeader("Content-Type", "text/event-stream");
1232
- if (r.body) for await (const c of r.body) res.write(c);
1233
- res.end();
1234
- } else {
1235
- const txt = await r.text();
1236
- res.setHeader("Content-Type", r.headers.get("content-type") || "application/json");
1237
- res.end(txt);
1238
- }
1239
- return;
1240
- } catch (err) {
1241
- return json(res, 502, { error: errMsg(err) });
1242
- }
1243
- }
1244
- // broadband: enqueue and wait for poll/result
1245
- const reqId = body.reqId || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
1246
- const fwdBody = body.body || { model: body.model, messages: body.messages, stream: body.stream };
1247
- const evtData = { ts: Date.now(), type: "relay-forward", target, via: "leader", group: groupName, hops, reqId, model: fwdBody.model };
1248
- if (bus) bus.emit(evtData);
1249
- logs?.appendEvent?.(evtData);
1250
- // check stale
1251
- const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
1252
- if (typeof targetMember.lastSeen === "number" && Date.now() - targetMember.lastSeen > staleMs) {
1253
- return json(res, 502, { error: "broadband member stale (no heartbeat)" });
1254
- }
1255
- try {
1256
- const resultPromise = enqueueRelay({ group: groupName, target: targetMember.url, reqId, body: fwdBody, hops });
1257
- // race with timeout already in enqueueRelay (30s)
1258
- const result = await resultPromise;
1259
- // result is expected to be { status, headers, body } from broadband
1260
- if (result && typeof result.status === "number") {
1261
- res.statusCode = result.status;
1262
- if (result.headers) for (const [k, v] of Object.entries(result.headers)) res.setHeader(k, v);
1263
- if (result.body) {
1264
- if (typeof result.body === "string") res.end(result.body);
1265
- else res.end(JSON.stringify(result.body));
1266
- } else res.end();
1267
- return;
1268
- }
1269
- // if result is raw upstream response body
1270
- return json(res, 200, result);
1271
- } catch (err) {
1272
- return json(res, 504, { error: errMsg(err) || "relay timeout" });
1273
- }
1274
- },
1275
- },
1276
- {
1277
- method: "GET",
1278
- path: "/v1/models",
1279
- requiresAuth: true,
1280
- handler: async ({ res, models }) => {
1281
- if (!models) return json(res, 501, { error: "Models service not configured" });
1282
- try {
1283
- const data = await models.get();
1284
- json(res, 200, data);
1285
- } catch (err) {
1286
- json(res, 502, { error: errMsg(err) });
1287
- }
1288
- },
1289
- },
1290
- {
1291
- method: "GET",
1292
- path: "/v1/models/status",
1293
- requiresAuth: true,
1294
- handler: async ({ res, models, auto }) => {
1295
- const statuses = auto?.statuses?.() || {};
1296
- let ids = [];
1297
- try {
1298
- ids = (await models?.get?.())?.data?.map((m) => m.id) || [];
1299
- } catch {
1300
- // models list unavailable; fall back to status records only
1301
- }
1302
- const seen = new Set();
1303
- const data = [];
1304
- for (const id of [...ids, ...Object.keys(statuses)]) {
1305
- if (seen.has(id)) continue;
1306
- seen.add(id);
1307
- const e = statuses[id];
1308
- const entry = typeof e === "number"
1309
- ? { id, status: "error", at: e }
1310
- : { id, status: e?.status || "normal", at: e?.at ?? null, code: e?.code ?? null };
1311
- data.push(entry);
1312
- }
1313
- json(res, 200, { object: "list", data });
1314
- },
1315
- },
1316
- ];
1
+ // Facade keeps legacy import path `from "./routes.js"` working.
2
+ // New code should import from "./routes/index.js" directly.
3
+ export * from "./routes/index.js";