mslxdff 0.1.39 → 0.1.42

package/bin/mslxdff.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { execFile } from "node:child_process";
-import { readFileSync, existsSync, statSync, rmSync } from "node:fs";
+import { readFileSync, existsSync, statSync, rmSync, writeFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { dirname, join, basename } from "node:path";
 import { startServer, resolvePort } from "../src/server.js";
@@ -192,14 +192,24 @@ if (args.includes("-model") || args.includes("-models")) {
 // event bus — no filesystem polling. Ctrl+C / SIGTERM restarts the daemon in
 // the background, then exits. See the daemon body below for the stream wiring.
 if (args.includes("-debug") || args.includes("--debug")) {
-  const recent = recentEvents(100);
-  if (recent.length) {
-    console.log(`--- last ${recent.length} event(s) ---`);
-    for (const e of recent) console.log(fmtEvent(e));
-  }
-  console.log("--- live (Ctrl+C: stop debugging and restore background daemon) ---");
+  // 先停旧 daemon，再清日志，保持 debug 输出干净（仅本次会话）
   const { stopped, pid } = stopDaemon();
   if (stopped) console.log(`[debug] stopped background daemon (pid ${pid})`);
+  try {
+    const dir = logDir();
+    const toClear = [eventsFile(), callsFile(), errorsFile(), logFile()];
+    let cleared = 0;
+    for (const f of toClear) {
+      try {
+        if (existsSync(f)) {
+          writeFileSync(f, "");
+          cleared++;
+        }
+      } catch {}
+    }
+    console.log(`[debug] 已清理旧日志 ${cleared} 个文件 (${dir})，本次会话干净输出`);
+  } catch {}
+  console.log("--- live (Ctrl+C: stop debugging and restore background daemon) ---");
   process.env.MSLXDFF_DEBUG = "1";
   process.env.MSLXDFF_DAEMON = "1";
   // fall through to the daemon body below — no process.exit() here
@@ -692,6 +702,19 @@ if (isDebug) {
 }
 
 await srv.ready();
+
+// 上游 Keep-Alive 预热：首条 TCP+TLS 暖好，100ms 后异步触发，不阻塞 ready
+setTimeout(() => {
+  upstream.preheat().then((r) => {
+    const entry = { ts: Date.now(), type: "upstream-preheat", ...r, baseUrl };
+    try { bus.emit(entry); } catch {}
+    try { logs.appendEvent(entry); } catch {}
+    if (r.skipped) console.log(`[preheat] skipped (MSLXDFF_PREHEAT disabled)`);
+    else if (r.ok) console.log(`[preheat] opencode models ok ${r.status} ${r.ms}ms`);
+    else console.log(`[preheat] opencode models failed ${r.error || r.status || ""} ${r.ms || 0}ms`);
+  }).catch(() => {});
+}, 100).unref?.();
+
 models.startAutoRefresh();
 if (process.env.MSLXDFF_DAEMON) {
   writePid(process.pid, VERSION);
@@ -1179,7 +1202,7 @@ function fmtEvent(e) {
   const t = e?.ts ? new Date(e.ts).toISOString().slice(11, 19) : "--:--:--";
   const head = `[${t}]`;
   const m = (x) => x || "-";
-  const fallbackTag = e.fallback ? ` fallback=${e.fallback.requested_model || e.requested}->${e.fallback.actual_model || e.actual}(${e.fallback.reason || e.reason})` : "";
+  const fallbackTag = e.fallback?.fallback ? ` fallback=${e.fallback.requested_model || e.requested}->${e.fallback.actual_model || e.actual}(${e.fallback.reason || e.reason})` : "";
   switch (e?.type) {
     case "request":
       return `${head} request       client-> ${m(e.requested || e.model)}${e.auto ? " (auto)" : ""} raw=${m(e.rawModel)} lock=${m(e.lockModel)} hops=${e.hops} from ${e.ip || "?"}${e.stream ? " stream" : ""}${e.prompt ? ` content="${e.prompt}"` : ""} reqId=${e.reqId || ""}`;
@@ -1195,6 +1218,9 @@ function fmtEvent(e) {
       return `${head} upstream ok   ${m(e.model)} HTTP ${e.status}${e.timing ? ` total=${e.timing.totalMs}ms` : ""}`;
     case "upstream-error":
       return `${head} upstream err  ${m(e.model)} ${e.status ? `HTTP ${e.status}` : "network"}: ${m(e.message)}`;
+    case "upstream-preheat":
+      if (e.skipped) return `${head} preheat       跳过 (MSLXDFF_PREHEAT disabled)`;
+      return `${head} preheat       预热 opencode models ${e.ok ? "ok" : "fail"} ${e.status ? `HTTP ${e.status}` : e.error || ""} ${e.ms ? `${e.ms}ms` : ""}`;
     case "peer-race-start":
       return `${head} peer race     开始并发给组员 model=${m(e.model)} peers=${e.peers}`;
     case "peer-health":
@@ -1228,7 +1254,7 @@ function fmtEvent(e) {
     case "result":
       return `${head} result        返回客户端 status=${e.status} model=${m(e.model)} via=${e.via} 响应耗时 ${fmtDur(e.durationMs)}${fallbackTag}`;
     case "client-response":
-      return `${head} client res    返回客户端 客户端请求 ${m(e.requested)} 实际返回 ${m(e.actual)} via=${m(e.via)}${e.fallback ? ` fallback=${e.fallback.requested_model}->${e.fallback.actual_model}(${e.fallback.reason})` : " 无fallback"} status=${e.status}`;
+      return `${head} client res    返回客户端 客户端请求 ${m(e.requested)} 实际返回 ${m(e.actual)} via=${m(e.via)}${e.fallback?.fallback ? ` fallback=${e.fallback.requested_model}->${e.fallback.actual_model}(${e.fallback.reason})` : " 无fallback"} status=${e.status}`;
     case "auto-update-enabled":
       return `${head} auto-update   enabled every ${Math.round((e.intervalMs||0)/60000)}m current=${e.current}`;
     case "auto-update-disabled":

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.39",
+  "version": "0.1.42",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {
@@ -23,5 +23,8 @@
     "demo",
     "placeholder"
   ],
-  "license": "MIT"
+  "license": "MIT",
+  "dependencies": {
+    "undici": "^8.10.0"
+  }
 }

package/src/routes/chat.js

@@ -0,0 +1,278 @@
+import { performance } from "node:perf_hooks";
+import { injectReasoningContent, normalizeModel } from "../reasoning.js";
+import { isAutoModel } from "../auto.js";
+import { clientIp, json, readBody, parseHops, summarizePrompt, errMsg } from "./helpers.js";
+import { buildFallbackInfo } from "./fallback.js";
+import { relay, SLOW_TOTAL_MS, STREAM_TIMEOUT_MS, STALL_TIMEOUT_MS, SCORE_STALL_MS } from "./stream.js";
+import { racePeerCandidates } from "./peers.js";
+import { tryBroadbandRelay } from "./relay-queue.js";
+
+export async function chatHandler({ req, res, upstream, auto, logs, peers, maxHops, groups, bus, token }) {
+  let body;
+  try {
+    body = await readBody(req);
+  } catch {
+    return json(res, 400, { error: "Invalid JSON body" });
+  }
+
+  const startedAt = Date.now();
+  const reqId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
+  const perf0 = performance.now();
+  const stages = [];
+  const mark = (name) => stages.push([name, Math.round(performance.now() - perf0)]);
+  const hops = parseHops(req.headers["x-mslxdff-hops"]);
+  const lockModel = req.headers["x-mslxdff-model-lock"] || "";
+  const rawModel = body.model || "";
+  const requested = normalizeModel(lockModel || rawModel || "");
+  const useAuto = isAutoModel(requested);
+  mark("parsed");
+
+  let order;
+  if (lockModel) {
+    order = [requested];
+  } else if (useAuto) {
+    order = auto ? await auto.candidates() : [""];
+  } else {
+    order = auto ? await auto.candidatesFor(requested) : [requested];
+  }
+  if (!order.length) order = [""];
+  const canFallback = order.length > 1;
+  const canForwardPeers = Boolean(peers) && hops < maxHops;
+  mark("ordered");
+
+  const logCall = (model, status) =>
+    logs?.appendCall({ reqId, model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream), stages });
+  const logError = (model, status, message) =>
+    logs?.appendError({ reqId, model, auto: useAuto, status, message, stages });
+  const evt = (type, data) => {
+    const entry = { ts: Date.now(), reqId, type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
+    if (bus) bus.emit(entry);
+    logs?.appendEvent?.(entry);
+  };
+  evt("request", { reqId, hops, ip: clientIp(req), stream: Boolean(body.stream), prompt: summarizePrompt(body), rawModel, requested, lockModel: lockModel || null });
+  evt("ordered", { reqId, order, canFallback, canForwardPeers, useAuto, statuses: auto?.statuses?.() ?? null });
+
+  const handlerCtx = {
+    model: null,
+    body,
+    hops,
+    peers,
+    evt,
+    logError,
+    logCall,
+  };
+
+  let lastErr = null;
+  for (let idx = 0; idx < order.length; idx++) {
+    const model = order[idx];
+    handlerCtx.model = model;
+    evt("model-try", { reqId, model, idx, remaining: order.length - idx });
+    let upRes = null;
+    const forwarded = { ...injectReasoningContent(model, body), model };
+    const tUp = performance.now();
+    evt("upstream-try", { reqId, model, attempt: idx + 1 });
+    try {
+      upRes = await upstream.chat(forwarded);
+      evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
+    } catch (err) {
+      if (auto) await auto.recordError(model, { message: errMsg(err) });
+      lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
+      logError(model, 502, errMsg(err));
+      evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
+    }
+    mark(`up-${model}`);
+    if (upRes && upRes.status >= 400) {
+      if (auto) await auto.recordError(model, { status: upRes.status });
+      lastErr = { model, upstream: upRes, status: upRes.status, message: null };
+      logError(model, upRes.status, `upstream ${upRes.status}`);
+      evt("upstream-error", { reqId, model, status: upRes.status, message: null, timing: upRes._t ?? null });
+      upRes = null;
+    }
+    if (upRes) {
+      logCall(model, upRes.status);
+      const fallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "local", useAuto, lockModel });
+      if (fallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: fallback.reason, notice: fallback.notice, via: "local" });
+      evt("relay-start", { reqId, model, via: "local", isStream: Boolean(body.stream), fallback });
+      const out = await relay(res, upRes, body, {
+        fallback,
+        onFirstChunk: (delta) => {
+          mark(`ttf-${model}`);
+          evt("relay-first-chunk", { reqId, model, ttfMs: delta });
+        },
+        onDownstreamAbort: () => {
+          evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] });
+        },
+      });
+      evt("relay-done", { reqId, model, via: "local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+      if (out.status === STREAM_TIMEOUT_MS) {
+        if (auto) await auto.recordError(model, { status: 502, slow: true, note: `stream timeout ${STREAM_TIMEOUT_MS}ms` });
+        lastErr = { model, upstream: null, status: 502, message: `stream timed out after ${STREAM_TIMEOUT_MS}ms` };
+        logError(model, 502, `stream timeout ${STREAM_TIMEOUT_MS}ms`);
+        evt("upstream-error", { reqId, model, status: 502, message: "stream timeout", timing: null });
+        evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: "stream timeout" });
+        upRes = null;
+        continue;
+      }
+      if (out.interrupted) {
+        if (auto) {
+          await auto.recordError(model, { status: 200, slow: true, note: `stall ${STALL_TIMEOUT_MS}ms` });
+          await auto.recordLatency(model, out.totalMs ?? (Date.now() - startedAt));
+        }
+        evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: STALL_TIMEOUT_MS, interrupted: true, detail: out.detail ?? null });
+        logCall(model, 200);
+        evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true, detail: out.detail ?? null, fallback, requested, actual: model });
+        evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, interrupted: true, reqId });
+        return;
+      }
+      const elapsed = Date.now() - startedAt;
+      const latencyMs = out.totalMs ?? elapsed;
+      let scoredSlow = false;
+      if (SLOW_TOTAL_MS && auto && elapsed > SLOW_TOTAL_MS && out.status === 200) {
+        void auto.recordError(model, { status: 200, slow: true, note: `slow ${elapsed}ms` });
+        void auto.recordLatency(model, latencyMs);
+        evt("slow-model", { model, elapsedMs: elapsed, threshold: SLOW_TOTAL_MS, reason: "total", detail: out.detail ?? null });
+        scoredSlow = true;
+      }
+      if (out.detail?.stallHits > 0 && auto && out.status === 200) {
+        void auto.recordError(model, { status: 200, slow: true, note: `stall ${out.detail.stallHits}x gap>${SCORE_STALL_MS}ms maxGap ${out.detail.maxGapMs}ms` });
+        void auto.recordLatency(model, latencyMs);
+        evt("slow-model", { model, elapsedMs: elapsed, threshold: SCORE_STALL_MS, reason: "stall", stallHits: out.detail.stallHits, maxGapMs: out.detail.maxGapMs, detail: out.detail ?? null });
+        scoredSlow = true;
+      }
+      if (!scoredSlow && auto && out.status === 200) {
+        await auto.recordOk(model, { latencyMs });
+      } else if (!scoredSlow && auto) {
+        await auto.recordLatency(model, latencyMs);
+      }
+      evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback, requested, actual: model });
+      evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, reqId });
+      return;
+    }
+
+    if (canForwardPeers) {
+      evt("peer-race-start", { reqId, model, peers: peers.ordered().length });
+      const win =
+        (await racePeerCandidates(peers.ordered(), handlerCtx)) ||
+        (await racePeerCandidates(peers.orderedByLastError(), handlerCtx));
+      if (win) {
+        evt("peer-race-win", { reqId, model, winPeer: win.peer.url, winTarget: win.target, latencyMs: win.latencyMs });
+        await peers.recordResult(win.peer.url, { ok: true, latencyMs: win.latencyMs, model: win.target });
+        logCall(win.target, win.res.status);
+        const peerFallback = buildFallbackInfo({ requested, actual: win.target, lastErr, via: "peer", useAuto, lockModel });
+        if (peerFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: win.target, reason: peerFallback.reason, notice: peerFallback.notice, via: "peer" });
+        evt("relay-start", { reqId, model: win.target, via: "peer", isStream: Boolean(body.stream), fallback: peerFallback });
+        const out = await relay(res, win.res, body, {
+          fallback: peerFallback,
+          onFirstChunk: (d) => mark(`ttf-peer-${win.target}`),
+          onDownstreamAbort: () => evt("client-abort", { reqId, model: win.target, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+        });
+        evt("relay-done", { reqId, model: win.target, via: "peer", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+        if (auto && out.status === 200) {
+          const latencyMs = out.totalMs ?? win.latencyMs;
+          if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
+            void auto.recordError(win.target, { status: 200, slow: true, note: `peer slow ${latencyMs}ms` });
+            void auto.recordLatency(win.target, latencyMs);
+          } else {
+            await auto.recordOk(win.target, { latencyMs });
+          }
+        }
+        evt("result", { model: win.target, status: out.status, via: "peer", timing: win.res._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: peerFallback, requested, actual: win.target });
+        evt("client-response", { requested, actual: win.target, via: "peer", fallback: peerFallback, status: out.status, reqId });
+        return;
+      }
+      evt("peer-race-lose", { reqId, model });
+    }
+
+    if (groups) {
+      const bb = await tryBroadbandRelay({ groups, token, model, body, hops, bus, logs, reqId, evt, res, mark, perf0, stages });
+      if (bb) {
+        const isResponse = bb.result && typeof bb.result.status === "number" && typeof bb.result.headers?.get === "function";
+        if (isResponse) {
+          const bbFallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "broadband", useAuto, lockModel });
+          if (bbFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: bbFallback.reason, notice: bbFallback.notice, via: "broadband" });
+          evt("relay-start", { reqId, model, via: "broadband", target: bb.target, group: bb.group, fallback: bbFallback });
+          const out = await relay(res, bb.result, body, {
+            fallback: bbFallback,
+            onFirstChunk: (d) => mark(`ttf-bb-${model}`),
+            onDownstreamAbort: () => evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+          });
+          evt("relay-done", { reqId, model, via: "broadband", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+          if (auto && out.status === 200) {
+            const latencyMs = out.totalMs ?? 0;
+            if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
+              void auto.recordError(model, { status: 200, slow: true, note: `broadband slow ${latencyMs}ms` });
+              void auto.recordLatency(model, latencyMs);
+            } else {
+              await auto.recordOk(model, { latencyMs });
+            }
+          }
+          evt("result", { model, status: out.status, via: "broadband", timing: bb.result._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: bbFallback, requested, actual: model });
+          evt("client-response", { requested, actual: model, via: "broadband", fallback: bbFallback, status: out.status, reqId });
+          return;
+        } else if (bb.result && typeof bb.result.status === "number") {
+          const fakeRes = {
+            status: bb.result.status,
+            headers: { get: (k) => bb.result.headers?.[k] || bb.result.headers?.[k.toLowerCase()] || null },
+            text: async () => typeof bb.result.body === "string" ? bb.result.body : JSON.stringify(bb.result.body),
+            body: (() => {
+              const b = bb.result.body || "";
+              const str = typeof b === "string" ? b : JSON.stringify(b);
+              const isSSE = bb.result.headers?.["Content-Type"]?.includes("text/event-stream");
+              if (isSSE) {
+                return (async function* () { yield Buffer.from(str); })();
+              }
+              return null;
+            })(),
+          };
+          const bbLocalFallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "broadband", useAuto, lockModel });
+          if (bbLocalFallback?.fallback) evt("fallback-notice", { reqId, requested, actual: model, reason: bbLocalFallback.reason, notice: bbLocalFallback.notice, via: "broadband" });
+          evt("relay-start", { reqId, model, via: "broadband-local", target: bb.target, group: bb.group, fallback: bbLocalFallback });
+          const out = await relay(res, fakeRes, body, {
+            fallback: bbLocalFallback,
+            onFirstChunk: (d) => mark(`ttf-bb-${model}`),
+            onDownstreamAbort: () => evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+          });
+          evt("relay-done", { reqId, model, via: "broadband-local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+          evt("result", { model, status: out.status, via: "broadband", timing: null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: bbLocalFallback, requested, actual: model });
+          evt("client-response", { requested, actual: model, via: "broadband", fallback: bbLocalFallback, status: out.status, reqId });
+          return;
+        }
+      }
+      evt("relay-miss", { reqId, model });
+    }
+
+    if (canFallback) {
+      evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` });
+      continue;
+    }
+    evt("exhausted-local", { reqId, lastModel: lastErr?.model ?? model, lastStatus: lastErr?.status ?? 502, order });
+    logCall(lastErr?.model ?? model, lastErr?.status ?? 502);
+    if (lastErr?.upstream) {
+      evt("relay-start", { reqId, model: lastErr.model, via: "local-exhausted", isStream: Boolean(body.stream) });
+      const out = await relay(res, lastErr.upstream, body, {
+        onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
+        onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+      });
+      evt("relay-done", { reqId, model: lastErr.model, via: "local-exhausted", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+      evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
+      return;
+    }
+    evt("result", { reqId, model, status: lastErr?.status ?? 502, via: "none", timing: null });
+    return json(res, 502, { error: lastErr?.message || "all auto models failed" });
+  }
+
+  evt("exhausted-all", { reqId, lastModel: lastErr?.model ?? requested, lastStatus: lastErr?.status ?? 502, order });
+  logCall(lastErr?.model ?? requested, lastErr?.status ?? 502);
+  if (lastErr?.upstream) {
+    evt("relay-start", { reqId, model: lastErr.model, via: "local-final", isStream: Boolean(body.stream) });
+    const out = await relay(res, lastErr.upstream, body, {
+      onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
+      onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+    });
+    evt("relay-done", { reqId, model: lastErr.model, via: "local-final", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+    evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
+    return;
+  }
+  evt("result", { reqId, model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none", timing: null });
+  return json(res, 502, { error: lastErr?.message || "all auto models failed" });
+}

package/src/routes/fallback.js

@@ -0,0 +1,88 @@
+function fallbackReason(lastErr) {
+  if (!lastErr) return "cooldown";
+  const s = Number(lastErr.status);
+  if (s === 429) return "rate_limited";
+  if (lastErr.message && /timeout/i.test(String(lastErr.message))) return "timeout";
+  if (s === 502 || s === 503 || s === 504) return "upstream_error";
+  if (s >= 400) return "upstream_error";
+  return "fallback";
+}
+
+export function buildFallbackInfo({ requested, actual, lastErr, via, useAuto, lockModel }) {
+  if (!requested || !actual) return null;
+  const alwaysHeaders = {
+    requested_model: requested,
+    actual_model: actual,
+    via: via || "local",
+  };
+  if (useAuto || lockModel) {
+    return { ...alwaysHeaders, fallback: false, reason: null, notice: null };
+  }
+  const isFallback = requested !== actual;
+  if (!isFallback) {
+    return { ...alwaysHeaders, fallback: false, reason: null, notice: null };
+  }
+  const reason = fallbackReason(lastErr);
+  const reasonZh = reason === "rate_limited" ? "限流" : reason === "timeout" ? "超时" : reason === "cooldown" ? "冷却中" : "不可用";
+  const notice = `${requested} ${reasonZh}，已由 ${actual} 代答`;
+  return { ...alwaysHeaders, fallback: true, reason, notice };
+}
+
+export function applyFallbackHeaders(res, info) {
+  if (!info) return;
+  if (info.requested_model) res.setHeader("x-mslxdff-requested-model", info.requested_model);
+  if (info.actual_model) res.setHeader("x-mslxdff-actual-model", info.actual_model);
+  if (info.via) res.setHeader("x-mslxdff-via", info.via);
+  if (info.fallback) {
+    res.setHeader("x-mslxdff-fallback", "1");
+    if (info.reason) res.setHeader("x-mslxdff-fallback-reason", info.reason);
+    if (info.notice) res.setHeader("x-mslxdff-notice", encodeURIComponent(info.notice));
+  }
+}
+
+export function enrichNonStreamJson(obj, info) {
+  if (!info || typeof obj !== "object" || obj === null) return obj;
+  if (!info.fallback) return obj;
+  if (obj.mslxdff) return obj;
+  return {
+    ...obj,
+    mslxdff: {
+      fallback: true,
+      requested_model: info.requested_model,
+      actual_model: info.actual_model,
+      reason: info.reason,
+      via: info.via,
+      notice: info.notice,
+    },
+  };
+}
+
+export function enrichSseChunkText(text, info) {
+  if (!info?.fallback) return text;
+  const lines = text.split("\n");
+  let changed = false;
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const m = /^data:\s*(\{.*\})\s*$/.exec(line);
+    if (!m) continue;
+    try {
+      const obj = JSON.parse(m[1]);
+      if (obj && typeof obj === "object" && !obj.mslxdff) {
+        obj.mslxdff = {
+          fallback: true,
+          requested_model: info.requested_model,
+          actual_model: info.actual_model,
+          reason: info.reason,
+          via: info.via,
+          notice: info.notice,
+        };
+        lines[i] = `data: ${JSON.stringify(obj)}`;
+        changed = true;
+        break;
+      }
+    } catch {
+      continue;
+    }
+  }
+  return changed ? lines.join("\n") : text;
+}

package/src/routes/groups-relay.js

@@ -0,0 +1,129 @@
+import { clientIp, json, readBody, parseHops, errMsg } from "./helpers.js";
+import { DEFAULT_MAX_HOPS } from "../peers.js";
+import { enqueueRelay, dequeueRelayForPoll, resolveRelay } from "./relay-queue.js";
+
+export async function heartbeatHandler({ req, res, groups, bus, logs }) {
+  const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
+  if (!auth) return json(res, 401, { error: "bearer token required" });
+  let body;
+  try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
+  const groupName = body?.name || body?.group;
+  if (!groupName) return json(res, 400, { error: "group name is required" });
+  const hit = groups?.membersForToken(groupName, auth[1]);
+  if (!hit) return json(res, 403, { error: "invalid member token" });
+  const ip = clientIp(req);
+  try {
+    const memberUrl = hit.member?.url;
+    const members = groups.list()[groupName]?.members || {};
+    const targetId = Object.keys(members).find((k) => members[k].url === memberUrl) || hit.member?.url;
+    if (hit.member?.kind === "broadband" || String(memberUrl).startsWith("relay://")) {
+      const m = members[targetId] || hit.member;
+      if (m) {
+        m.publicIp = ip;
+        m.lastSeen = Date.now();
+        try { groups.upsertMember(groupName, { memberName: targetId, url: m.url, token: m.token, kind: "broadband", publicIp: ip, lastSeen: m.lastSeen }); } catch {}
+      }
+      const evtData = { ts: Date.now(), type: "relay-heartbeat", member: targetId, ip, lastSeen: m?.lastSeen, group: groupName };
+      if (bus) bus.emit(evtData);
+      logs?.appendEvent?.(evtData);
+    }
+    return json(res, 200, { object: "heartbeat", ok: true, ip, lastSeen: Date.now() });
+  } catch (err) {
+    return json(res, 400, { error: errMsg(err) });
+  }
+}
+
+export async function pollHandler({ req, res, groups }) {
+  const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
+  if (!auth) return json(res, 401, { error: "bearer token required" });
+  let body;
+  try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
+  const groupName = body?.name || body?.group;
+  if (!groupName) return json(res, 400, { error: "group name is required" });
+  const hit = groups?.membersForToken(groupName, auth[1]);
+  if (!hit) return json(res, 403, { error: "invalid member token" });
+  const targetUrl = hit.member?.url;
+  if (!targetUrl) return json(res, 400, { error: "member url not found" });
+  const batch = dequeueRelayForPoll({ group: groupName, target: targetUrl, limit: 10 });
+  return json(res, 200, { object: "poll", data: batch });
+}
+
+export async function resultHandler({ req, res, groups }) {
+  const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
+  if (!auth) return json(res, 401, { error: "bearer token required" });
+  let body;
+  try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
+  const groupName = body?.name || body?.group;
+  const reqId = body?.reqId;
+  if (!groupName || !reqId) return json(res, 400, { error: "group and reqId required" });
+  const hit = groups?.membersForToken(groupName, auth[1]);
+  if (!hit) return json(res, 403, { error: "invalid member token" });
+  const ok = resolveRelay(reqId, body.result || body);
+  if (!ok) return json(res, 404, { error: "pending request not found or timed out" });
+  return json(res, 200, { object: "result", ok: true });
+}
+
+export async function forwardHandler({ req, res, groups, bus, logs }) {
+  let body;
+  try { body = await readBody(req); } catch { return json(res, 400, { error: "Invalid JSON body" }); }
+  const groupName = body?.group || body?.name;
+  const target = body?.target || body?.url;
+  const hops = parseHops(req.headers["x-mslxdff-hops"] || body?.hops);
+  if (!groupName || !target) return json(res, 400, { error: "group and target required" });
+  if (hops >= DEFAULT_MAX_HOPS) return json(res, 429, { error: "max hops exceeded" });
+  const members = groups?.list()[groupName]?.members || {};
+  const targetMember = Object.values(members).find((m) => m.url === target) || Object.entries(members).find(([id]) => id === target)?.[1];
+  if (!targetMember) return json(res, 404, { error: `target ${target} not found in group ${groupName}` });
+  const isBb = targetMember.kind === "broadband" || String(targetMember.url).startsWith("relay://");
+  if (!isBb) {
+    try {
+      const fwdBody = body.body || body;
+      const r = await fetch(`${targetMember.url}/v1/chat/completions`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": `Bearer ${targetMember.token || ""}`, "x-mslxdff-hops": String(hops + 1), "x-mslxdff-model-lock": fwdBody.model || "", "Accept": "text/event-stream" },
+        body: JSON.stringify(fwdBody),
+      });
+      const evtData = { ts: Date.now(), type: "relay-forward", target, via: "direct", group: groupName, hops };
+      if (bus) bus.emit(evtData);
+      logs?.appendEvent?.(evtData);
+      res.statusCode = r.status;
+      if (r.headers.get("content-type")?.includes("text/event-stream")) {
+        res.setHeader("Content-Type", "text/event-stream");
+        if (r.body) for await (const c of r.body) res.write(c);
+        res.end();
+      } else {
+        const txt = await r.text();
+        res.setHeader("Content-Type", r.headers.get("content-type") || "application/json");
+        res.end(txt);
+      }
+      return;
+    } catch (err) {
+      return json(res, 502, { error: errMsg(err) });
+    }
+  }
+  const reqId = body.reqId || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
+  const fwdBody = body.body || { model: body.model, messages: body.messages, stream: body.stream };
+  const evtData = { ts: Date.now(), type: "relay-forward", target, via: "leader", group: groupName, hops, reqId, model: fwdBody.model };
+  if (bus) bus.emit(evtData);
+  logs?.appendEvent?.(evtData);
+  const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
+  if (typeof targetMember.lastSeen === "number" && Date.now() - targetMember.lastSeen > staleMs) {
+    return json(res, 502, { error: "broadband member stale (no heartbeat)" });
+  }
+  try {
+    const resultPromise = enqueueRelay({ group: groupName, target: targetMember.url, reqId, body: fwdBody, hops });
+    const result = await resultPromise;
+    if (result && typeof result.status === "number") {
+      res.statusCode = result.status;
+      if (result.headers) for (const [k, v] of Object.entries(result.headers)) res.setHeader(k, v);
+      if (result.body) {
+        if (typeof result.body === "string") res.end(result.body);
+        else res.end(JSON.stringify(result.body));
+      } else res.end();
+      return;
+    }
+    return json(res, 200, result);
+  } catch (err) {
+    return json(res, 504, { error: errMsg(err) || "relay timeout" });
+  }
+}