mslxdff 0.1.121 → 0.1.123

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.121",
+  "version": "0.1.123",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/chat-pipeline/serial-trial.js

@@ -107,9 +107,18 @@ export async function runSerialTrial(ctx, deps = {}) {
         return json(res, 403, errBody);
       }
       if (auto) await auto.recordError(model, { status: upRes.status });
-      lastErr = { model, upstream: upRes, status: upRes.status, message: null };
-      logError(model, upRes.status, `upstream ${upRes.status}`);
-      evt("upstream-error", { reqId, model, status: upRes.status, message: null, timing: upRes._t ?? null });
+      // 读失败响应体（clone 不影响后续 relay 转发原响应；1s 上限防流式错误体拖慢）
+      let upBody = "";
+      try {
+        upBody = String(await Promise.race([
+          upRes.clone().text(),
+          new Promise((r) => { const t = setTimeout(() => r(""), 1000); t.unref?.(); }),
+        ])).replace(/\s+/g, " ").slice(0, 400);
+      } catch {}
+      const upMsg = upBody || `upstream ${upRes.status}`;
+      lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
+      logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}`);
+      evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null });
       upRes = null;
     }
     if (upRes) {
@@ -138,6 +147,7 @@ export async function runSerialTrial(ctx, deps = {}) {
       } else {
         const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, stages, startedAt, plugins, res });
         if (pr.handled) return { done: true };
+        if (pr.lastErr) lastErr = pr.lastErr;
       }
     }
     if (groups) {

package/src/cli/commands/daemon.js

@@ -95,8 +95,29 @@ export async function handleDaemonFlag(args, VERSION) {
 
 export async function handleDebug(args) {
   if (!(args.includes("-debug") || args.includes("--debug"))) return false;
+  // systemd 自启协同：若后台 daemon 由 user service（Restart=always, RestartSec=3）托管，
+  // 仅 stopDaemon() 会让 systemd 3 秒后拉起新实例 → 抢 8989 → EADDRINUSE 自愈反杀本 debug 前台。
+  // 必须先 systemctl stop（主动停止不会被 Restart 拉起）。
+  if (process.platform === "linux") {
+    try {
+      const { execFile } = await import("node:child_process");
+      const active = await new Promise((res) => execFile("systemctl", ["--user", "is-active", "mslxdff"], { windowsHide: true, timeout: 4000 }, (e, so) => res(String(so || "").trim())));
+      if (active === "active") {
+        await new Promise((res) => execFile("systemctl", ["--user", "stop", "mslxdff"], { windowsHide: true, timeout: 6000 }, () => res()));
+        console.log("[debug] stopped systemd user service (mslxdff) — it will be restarted on exit");
+      }
+    } catch {}
+  }
   const { stopped, pid } = stopDaemon();
   if (stopped) console.log(`[debug] stopped background daemon (pid ${pid})`);
+  // 等旧 daemon 真正退出再抢端口（Windows 端口释放有延迟，否则 EADDRINUSE 会让 debug 立即崩）
+  if (stopped && pid) {
+    const t0 = Date.now();
+    while (isPidAlive(pid) && Date.now() - t0 < 4000) {
+      await new Promise((r) => setTimeout(r, 100));
+    }
+    await new Promise((r) => setTimeout(r, 150));
+  }
   try {
     const dir = logDir();
     const toClear = [eventsFile(), callsFile(), errorsFile(), logFile()];

package/src/cli/commands/provider/add.js

@@ -101,6 +101,7 @@ export async function handleProviderAdd(id, sub, rest) {
   console.log(`  share: ${loadProviderShareKeys(nid) ? "ON" : "off"}   (mslxdff -provider ${nid} share on|off)`);
   console.log(`  allowAny: OFF (secure, empty allowlist = 403 block before upstream) — enable via: mslxdff -provider ${nid} allowAny on`);
   console.log(`  use as: ${nid}/<model-id>  — restart daemon to activate`);
+  if (allowedModels.length) console.log(`  opencode 中使用: mslxdff -setto opencode ${nid}/${allowedModels[0]}  (同步进 opencode.json，会一并设为默认模型)`);
   console.log(`  NOTE: empty allowlist = 403 before upstream, no cost — must set allowlist to use`);
   process.exit(0);
 }

package/src/cli/policy.js

@@ -89,6 +89,10 @@ export function peerCooldownMs() {
   const n = Number(process.env.MSLXDFF_PEER_COOLDOWN_MS);
   return Number.isInteger(n) && n > 0 ? n : 30_000;
 }
+export function peerLimitCooldownMs() {
+  const n = Number(process.env.MSLXDFF_PEER_LIMIT_COOLDOWN_MS);
+  return Number.isInteger(n) && n > 0 ? n : 5 * 60_000;
+}
 export function peerHeatMs() {
   const n = Number(process.env.MSLXDFF_PEER_HEAT_MS);
   return Number.isInteger(n) && n > 0 ? n : 5 * 60_000;

package/src/peers.js

@@ -1,6 +1,7 @@
 import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
 
 export const DEFAULT_PEER_COOLDOWN_MS = 30_000;
+export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 5 * 60_000;
 export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
 export const DEFAULT_MAX_HOPS = 3;
 export const DEFAULT_BROADBAND_STALE_MS = 90_000;
@@ -28,6 +29,7 @@ export function createPeersService({
   file,
   now = () => Date.now(),
   cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
+  limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
   heatMs = DEFAULT_PEER_HEAT_MS,
   peers: seedPeers,
   errors: seedErrors,
@@ -75,10 +77,18 @@ export function createPeersService({
     return before - list.length;
   }
 
+  // 分级冷却：连续 429（streak >= 2）→ limitCooldownMs（限流恢复慢，5min 内不再考虑）；
+  // 其他失败（网络抖动、5xx）→ cooldownMs（可能几秒恢复，给快速重试机会）。
+  function coolingWindowMs(url) {
+    const streak = stats[url]?.streak429 ?? 0;
+    return streak >= 2 ? limitCooldownMs : cooldownMs;
+  }
+
   function isCooling(url) {
-    if (!cooldownMs) return false;
+    const win = coolingWindowMs(url);
+    if (!win) return false;
     const err = lastErrorAt[url];
-    if (typeof err === "number" && now() - err < cooldownMs) return true;
+    if (typeof err === "number" && now() - err < win) return true;
     const peer = list.find((p) => p.url === url);
     if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
     return false;
@@ -148,6 +158,21 @@ export function createPeersService({
 
   let cursor = 0;
 
+  // 兜底重试集合：仅"因错误冷却中"的 peer（按最早失败优先）。
+  // 场景：唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时，兜底给它一次机会；
+  // 成功会被 recordResult 清除错误记录、回归热路径，避免"明明能用的节点被 30s 冷却全灭"。
+  function coolingByLastError() {
+    const t = now();
+    return list
+      .filter((p) => {
+        const err = lastErrorAt[p.url];
+        if (typeof err !== "number") return false;
+        if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却：兜底也不考虑
+        return t - err < cooldownMs;
+      })
+      .sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
+  }
+
   function next() {
     const avail = available();
     if (!avail.length) return null;
@@ -158,10 +183,16 @@ export function createPeersService({
   // Long-lived error memory: a peer keeps its last-error timestamp until a
   // subsequent success resets it (success clears the failure record) or the
   // error is no longer in the persist store on next load.
-  async function recordError(url) {
+  async function recordError(url, { status } = {}) {
     if (!url) return;
     lastErrorAt[url] = now();
     await persistErrors({ ...lastErrorAt });
+    // 连续 429 计数（仅成功清零）：第 2 次起进入长冷却——该 peer 大概率真被限流，不再频繁试它
+    if (Number(status) === 429) {
+      const prev = stats[url] || {};
+      stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
+      await persistStats({ ...stats });
+    }
   }
 
   // Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
@@ -182,6 +213,7 @@ export function createPeersService({
           : (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
         fails: 0,
         model: model || prev.model || "",
+        streak429: 0,
       };
     } else {
       const prev = stats[url] || {};
@@ -194,7 +226,7 @@ export function createPeersService({
     all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
       const peer = list.find((p) => p.url === url);
       return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
-    }, isHot, stat, ordered, orderedByLastError, available, next,
+    }, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
     recordError, recordResult, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
   };
 }

package/src/routes/chat/peer-handler.js

@@ -24,12 +24,20 @@ export async function handlePeerRelay({
   res,
 }) {
   evt("peer-race-start", { reqId: handlerCtx.reqId, model, peers: peers.ordered().length });
+  const peerErrors = [];
+  const pctx = { ...handlerCtx, peerErrors };
   const win =
-    (await racePeerCandidates(peers.ordered(), handlerCtx)) ||
-    (await racePeerCandidates(peers.orderedByLastError(), handlerCtx));
+    (await racePeerCandidates(peers.ordered(), pctx)) ||
+    (await racePeerCandidates(peers.orderedByLastError(), pctx)) ||
+    (await racePeerCandidates(peers.coolingByLastError(), pctx));
   if (!win) {
     evt("peer-race-lose", { reqId: handlerCtx.reqId, model });
-    return { handled: false };
+    // 组员全失败：把每个组员的真实返回汇总给调用者（否则只剩一个无信息的 429/502）
+    const detail = peerErrors.length
+      ? peerErrors.map((e) => `${e.peer} -> ${e.status}${e.message ? ` (${String(e.message).slice(0, 160)})` : ""}`).join("; ")
+      : "no peers available";
+    const allLimit = peerErrors.length > 0 && peerErrors.every((e) => e.status === 429);
+    return { handled: false, lastErr: { model, upstream: null, status: allLimit ? 429 : 502, message: `peers failed: ${detail}` } };
   }
   evt("peer-race-win", { reqId: handlerCtx.reqId, model, winPeer: win.peer.url, winTarget: win.target, latencyMs: win.latencyMs });
   await peers.recordResult(win.peer.url, { ok: true, latencyMs: win.latencyMs, model: win.target });

package/src/routes/chat/via-route-handler.js

@@ -104,7 +104,7 @@ export async function handleViaRoute({
     try { bodyText = await upRes.clone().text(); } catch {}
     const msg = bodyText.slice(0, 300) || errMsg(upRes) || `peer ${status}`;
     evt("via-route-peer-error", { reqId: handlerCtx.reqId, peer: peer.url, model, status, message: msg.slice(0, 200) });
-    try { await peers.recordError(peer.url); } catch {}
+    try { await peers.recordError(peer.url, { status }); } catch {}
     try { await peers.recordResult(peer.url, { ok: false }); } catch {}
     // 502/429 等可 fallback 到 direct
     return { handled: false, lastErr: { model, upstream: upRes, status, message: msg } };

package/src/routes/hedge.js

@@ -136,7 +136,8 @@ export async function hedgedFirstChunkRace({
   try {
     peerWin =
       (await racePeerCandidates(candidates, handlerCtx)) ||
-      (await racePeerCandidates(handlerCtx.peers.orderedByLastError(), handlerCtx));
+      (await racePeerCandidates(handlerCtx.peers.orderedByLastError(), handlerCtx)) ||
+      (await racePeerCandidates(handlerCtx.peers.coolingByLastError(), handlerCtx));
   } catch (_) {
     peerWin = null;
   }

package/src/routes/peers.js

@@ -3,10 +3,53 @@ import { isAutoModel } from "../auto.js";
 import { errMsg } from "./helpers.js";
 import { runHook } from "../plugins.js";
 import { buildShareKeysHeader, SHARE_KEYS_HEADER } from "../providers/share-keys.js";
-import { compatFetch, timeoutSignal } from "../compat.js";
+import { compatFetch, timeoutSignal, getUndici } from "../compat.js";
 
 const PEER_TIMEOUT_MS = 30_000;
 const PEER_STATUS_TIMEOUT_MS = 2_000;
+const DEFAULT_PEER_CONNECT_TIMEOUT_MS = 3_000;
+
+function peerConnectTimeoutMs() {
+  const n = Number(process.env.MSLXDFF_PEER_CONNECT_TIMEOUT_MS);
+  return Number.isInteger(n) && n > 0 ? n : DEFAULT_PEER_CONNECT_TIMEOUT_MS;
+}
+
+// 连接级超时（DNS + TCP/TLS 握手）：黑洞节点（端口挂起）必须快速失败，
+// 不能占用 30s 的响应超时（曾把一次请求拖到 92s）。仅 undici 可用时生效。
+let peerDispatcher = null;
+function getPeerDispatcher() {
+  if (peerDispatcher) return peerDispatcher;
+  const { Agent } = getUndici();
+  if (!Agent) return null;
+  try {
+    peerDispatcher = new Agent({
+      connect: { timeout: peerConnectTimeoutMs() },
+      headersTimeout: PEER_TIMEOUT_MS,
+      bodyTimeout: PEER_TIMEOUT_MS,
+      keepAliveTimeout: 30_000,
+      keepAliveMaxTimeout: 60_000,
+    });
+  } catch { peerDispatcher = null; }
+  return peerDispatcher;
+}
+
+// 跨 caller 中继必须剥离 reasoning 加密态：encrypted_content 由上游按 caller（出口）签发，
+// 换个节点转发会被拒（400 "reasoning encrypted_content was not issued to this caller"）。
+// 明文 reasoning_content 保留（不绑定 caller，thinking 模式需要）。
+export function stripCallerBoundReasoning(body) {
+  const messages = body?.messages;
+  if (!Array.isArray(messages)) return body;
+  let changed = false;
+  const out = messages.map((m) => {
+    if (!m || m.role !== "assistant") return m;
+    if (!Array.isArray(m.reasoning_items) || !m.reasoning_items.length) return m;
+    changed = true;
+    const copy = { ...m };
+    delete copy.reasoning_items;
+    return copy;
+  });
+  return changed ? { ...body, messages: out } : body;
+}
 
 function peerHealthTtlMs() {
   const n = Number(process.env.MSLXDFF_PEER_HEALTH_TTL_MS);
@@ -75,11 +118,13 @@ async function forwardToPeer(peer, body, model, hops) {
     // ADR-0008：该模型命中的供应商若开启 share → 附带瞬时 key 给组员借用（opencode 恒排除）
     const shareHeader = buildShareKeysHeader(model);
     if (shareHeader) headers[SHARE_KEYS_HEADER] = shareHeader;
+    const dispatcher = getPeerDispatcher();
     return await compatFetch(`${peer.url}/v1/chat/completions`, {
       method: "POST",
       headers,
-      body: JSON.stringify({ ...body, model }),
+      body: JSON.stringify({ ...stripCallerBoundReasoning(body), model }),
       signal: controller.signal,
+      ...(dispatcher ? { dispatcher } : {}),
     });
   } catch (err) {
     return err;
@@ -117,15 +162,30 @@ export const PEER_RACE_LIMIT = Number(process.env.MSLXDFF_PEER_RACE_LIMIT) > 0
   ? Number(process.env.MSLXDFF_PEER_RACE_LIMIT)
   : 3;
 
+// 串行记账队列：失败/迟到成功的记录不阻塞赢家返回，同时避免并发写盘互相覆盖。
+let peerRecordChain = Promise.resolve();
+function recordLater(fn) {
+  peerRecordChain = peerRecordChain.then(fn).catch(() => {});
+}
+
 export async function racePeerCandidates(candidates, ctx) {
-  for (let i = 0; i < candidates.length; i += PEER_RACE_LIMIT) {
-    const batch = candidates.slice(i, i + PEER_RACE_LIMIT);
+  const tried = (ctx.triedUrls ??= new Set());
+  const fresh = candidates.filter((p) => !tried.has(p.url));
+  for (let i = 0; i < fresh.length; i += PEER_RACE_LIMIT) {
+    const batch = fresh.slice(i, i + PEER_RACE_LIMIT);
     const prepared = (await Promise.all(batch.map((peer) => resolvePeerTarget(ctx, peer)))).filter(Boolean);
     if (!prepared.length) continue;
     const completed = await new Promise((resolve) => {
       const order = [];
       const total = prepared.length;
+      let settled = false;
+      const finish = (winner) => {
+        if (settled) return;
+        settled = true;
+        resolve({ list: order, winner });
+      };
       for (const { peer, target } of prepared) {
+        tried.add(peer.url);
         ctx.evt("peer-request", { peer: peer.url, model: target, hops: ctx.hops + 1 });
         // 插件 hook：peer:beforeForward — 转发给组员前观察
         if (ctx.plugins?.length) {
@@ -142,33 +202,48 @@ export async function racePeerCandidates(candidates, ctx) {
           }
           if (failed) {
             const status = res instanceof Error ? 502 : res.status;
-            ctx.logError(ctx.model, status, res instanceof Error ? errMsg(res) : `peer ${status}`);
+            const failRec = { peer: peer.url, status, message: res instanceof Error ? errMsg(res) : null };
+            if (Array.isArray(ctx.peerErrors)) ctx.peerErrors.push(failRec);
+            ctx.logError(ctx.model, status, res instanceof Error ? `peer ${peer.url} ${errMsg(res)}` : `peer ${peer.url} ${status}`);
             ctx.evt("peer-error", { peer: peer.url, model: target, status, message: res instanceof Error ? errMsg(res) : null });
             order.push({ ok: false, peer, target, res, status });
+            recordLater(() => ctx.peers.recordError(peer.url, { status }));
+            recordLater(() => ctx.peers.recordResult(peer.url, { ok: false }));
+            if (order.length === total) finish(null);
           } else {
-            order.push({ ok: true, peer, target, res, latencyMs });
+            const entry = { ok: true, peer, target, res, latencyMs };
+            order.push(entry);
+            if (!settled) {
+              // 第一个成功立即返回：不等慢/黑洞候选（迟到者的记账由各分支自理）
+              finish(entry);
+            } else {
+              recordLater(() => ctx.peers.recordResult(peer.url, { ok: true, latencyMs, model: target }));
+            }
           }
-          if (order.length === total) resolve(order);
         });
       }
     });
-    const winner = completed.find((o) => o.ok);
+    const winner = completed.winner;
     if (winner) {
-      for (const o of completed) {
-        if (o === winner) continue;
-        if (!o.ok) {
-          await ctx.peers.recordError(o.peer.url);
-          await ctx.peers.recordResult(o.peer.url, { ok: false });
-        } else {
-          await ctx.peers.recordResult(o.peer.url, { ok: true, latencyMs: o.latencyMs, model: o.target });
-        }
-      }
       return { peer: winner.peer, target: winner.target, res: winner.res, latencyMs: winner.latencyMs };
     }
-    for (const o of completed) {
-      await ctx.peers.recordError(o.peer.url);
-      await ctx.peers.recordResult(o.peer.url, { ok: false });
-    }
+    // 全失败：补读失败响应体（诊断 + 调用者详情）。仅在"本轮无 winner"时执行，
+    // 不拖慢成功路径；并行读、每 peer 上限 600ms，body 里才有 400/429 的真实原因。
+    await Promise.all(completed.list.map(async (o) => {
+      if (o.ok || !o.res || typeof o.res !== "object" || o.res instanceof Error) return;
+      try {
+        const snip = String(await Promise.race([
+          o.res.clone().text(),
+          new Promise((r) => { const t = setTimeout(() => r(""), 600); t.unref?.(); }),
+        ])).replace(/\s+/g, " ").slice(0, 300);
+        if (!snip) return;
+        if (Array.isArray(ctx.peerErrors)) {
+          const rec = ctx.peerErrors.find((x) => x.peer === o.peer.url && x.status === o.status && !x.message);
+          if (rec) rec.message = snip;
+        }
+        ctx.logError(ctx.model, o.status, `peer ${o.peer.url} ${o.status} body=${snip}`);
+      } catch {}
+    }));
   }
   return null;
 }

package/src/runtime/auto-update.js

@@ -11,7 +11,12 @@ export function setupAutoUpdate({ VERSION, bus, logs }) {
     const line = `[auto-update] ${type} ${JSON.stringify(data)}`;
     console.log(line);
   }
-  if (autoUpdateMs) {
+  // debug 会话不自动升级：debug 前台会把自己的 pid 写入 daemon.pid，
+  // auto-update 的 stopDaemon() 会把它自己停掉（现象：-debug 跑一会儿就"自己退出"）
+  if (autoUpdateMs && process.env.MSLXDFF_DEBUG === "1") {
+    console.log(`auto-update: skipped (debug session)`);
+    emitAutoUpdate("auto-update-skipped", { intervalMs: autoUpdateMs, current: VERSION, reason: "debug" });
+  } else if (autoUpdateMs) {
     console.log(`auto-update enabled: checking every ${Math.round(autoUpdateMs / 60000)}m`);
     emitAutoUpdate("auto-update-enabled", { intervalMs: autoUpdateMs, current: VERSION });
     setTimeout(() => {

package/src/runtime/providers-setup.js

@@ -11,7 +11,7 @@ import { logDir, appendEvent } from "../logs.js";
 import { loadPlugins, runHook, resolvePluginDirs } from "../plugins.js";
 import { createOpenCodeProvider } from "../providers/opencode.js";
 import { loadProviderKeys, loadProviderAuths, loadProviderConfigs } from "../state.js";
-import { refreshIntervalMs, modelCooldownMs, slowCooldownMs, peerCooldownMs, peerHeatMs, banWindowMs, banThreshold } from "../cli/policy.js";
+import { refreshIntervalMs, modelCooldownMs, slowCooldownMs, peerCooldownMs, peerLimitCooldownMs, peerHeatMs, banWindowMs, banThreshold } from "../cli/policy.js";
 import { errMsg } from "../cli/util.js";
 
 /**
@@ -140,7 +140,7 @@ export async function setupProviders() {
       }
     },
   });
-  const peers = createPeersService({ cooldownMs: peerCooldownMs(), heatMs: peerHeatMs() });
+  const peers = createPeersService({ cooldownMs: peerCooldownMs(), limitCooldownMs: peerLimitCooldownMs(), heatMs: peerHeatMs() });
   const groups = createGroupsService({});
   const bans = createBansService({ windowMs: banWindowMs(), threshold: banThreshold() });
 

package/src/runtime/server-lifecycle.js

@@ -37,8 +37,16 @@ export async function startServerLifecycle({ VERSION, token, created, upstream,
       } catch {}
     });
     const { startDaemon: sd } = await import("../daemon.js");
-    const restore2 = () => {
+    const restore2 = async () => {
       console.log("\n[debug] restoring background daemon...");
+      try { await srv.close(); } catch {} // 先释放端口，避免恢复的 daemon 抢端口互杀
+      if (process.platform === "linux") {
+        try {
+          const { execFile } = await import("node:child_process");
+          const ok = await new Promise((res) => execFile("systemctl", ["--user", "start", "mslxdff"], { windowsHide: true, timeout: 8000 }, (e) => res(!e)));
+          if (ok) { console.log("[debug] systemd user service restarted (mslxdff)"); setTimeout(() => process.exit(0), 300); return; }
+        } catch {}
+      }
       try {
         const restoredPid = sd([]);
         console.log(`[debug] daemon restored (pid ${restoredPid})`);