mslxdff 0.1.213 → 0.1.214

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.213",
+  "version": "0.1.214",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/cli/format.js

@@ -76,6 +76,14 @@ export function fmtEvent(e) {
       return `${head} fallback!     客户端请求 ${m(e.requested)} 实际返回 ${m(e.actual)} 原因=${m(e.reason)} via=${m(e.via)} notice=${m(e.notice)}`;
     case "relay-try":
       return `${head} relay try     给宽带中继请求 target=${e.target} model=${m(e.model)} via=${e.via}`;
+    case "relay-miss":
+      return `${head} relay miss    宽带中继无候选（${e.reason || "-"}${e.candidates != null ? ` candidates=${e.candidates}` : ""}）`;
+    case "relay-received":
+      return `${head} relay recv    ${e.via === "broadband" ? "经中继" : "直连"}转给 target=${e.target} group=${e.group} model=${m(e.model)} msgs=${e.messages ?? "-"} tools=${e.tools ?? "-"} maxTok=${e.maxTokens ?? "-"}`;
+    case "relay-queued":
+      return `${head} relay queued  已入队交成员 target=${e.target} reqId=${e.reqId || "-"}`;
+    case "relay-member-result":
+      return `${head} relay result  成员回 target=${e.target} ${e.ok ? "ok" : "FAIL"} ${e.status ? `HTTP ${e.status}` : ""} ${e.ms ? `${e.ms}ms` : ""}${e.respChars ? ` resp=${e.respChars}B` : ""} ${m(e.error || e.respBody || "")}`;
     case "relay-fail":
       return `${head} relay fail    ${e.target} ${e.status ? `HTTP ${e.status}` : ""} ${m(e.message)}`;
     case "relay-ip-change":

package/src/model-trace.js

@@ -87,6 +87,10 @@ const DECISION_FIELDS = [
   "heldOpen", "terminalForm", "requestWaitedMs", "budgetMs", "budgetOut",
   // 终局判决（ADR-0058）：取证时第一眼要看的三件——判的哪种、命中哪条判据、是否因此没问组员
   "kind", "marker", "peersSkipped",
+  // 中继/组员目标（2026-10-19）：跨机链路排障第一眼要的是**这一跳打给了谁**——target= 目标成员
+  // （url 或 relay://id）、group= 组、leaderUrl= 经谁中继；缺了这几项，relay-try/fail 只剩状态码，
+  // 看日志根本认不出是哪个组员出的问题（用户实证要求「日志要全面」）。
+  "target", "group", "leaderUrl", "candidates", "tried",
 ];
 
 function decisionKv(data = {}) {
@@ -141,6 +145,22 @@ export function formatModelTrace({ type, reqId, model, data = {}, request = null
     detail = `upstream_response status=${data.status ?? "-"} finish=${d.sawFinishReason || "-"} chunks=${d.receivedChunks ?? "-"} bytes=${d.receivedBytes ?? "-"} tools=${d.toolCalls ?? 0} chars=${d.chars ?? 0} ${usageTxt} elapsed=${data.totalMs ?? "-"}ms${data.upstream ? ` upstream=${data.upstream}` : ""}${data.account ? ` account=${data.account}` : ""}${data.pick ? ` pick=${data.pick}` : ""}`;
   } else if (type === "result" || type === "client-response") {
     detail = `client status=${data.status ?? "-"} via=${data.via || "-"} actual=${data.actual || model || "-"} fallback=${data.fallback?.fallback ? 1 : 0} total=${totalMs ?? data.durationMs ?? "-"}ms${data.upstream ? ` upstream=${data.upstream}` : ""}${data.account ? ` account=${data.account}` : ""}${data.pick ? ` pick=${data.pick}` : ""}`;
+  } else if (type === "relay-try") {
+    detail = `指定宽带中继 target=${data.target || "?"} group=${data.group || "-"} via=${data.via || "-"}`;
+  } else if (type === "relay-received") {
+    detail = `中继收到 转给 target=${data.target || "?"} group=${data.group || "-"} via=${data.via || "-"} msgs=${data.messages ?? "-"} tools=${data.tools ?? "-"} maxTok=${data.maxTokens ?? "-"}`;
+  } else if (type === "relay-queued") {
+    detail = `已入队交成员 target=${data.target || "?"} reqId=${data.reqId || "-"}`;
+  } else if (type === "relay-member-result") {
+    detail = `成员回 target=${data.target || "?"} ${data.ok ? "ok" : "FAIL"}${data.status ? ` status=${data.status}` : ""}${data.ms ? ` ${data.ms}ms` : ""}${data.respChars ? ` resp=${data.respChars}B` : ""} ${safeText(data.error || data.respBody || "", 200)}`;
+  } else if (type === "relay-fail") {
+    detail = `中继失败 target=${data.target || "?"} group=${data.group || "-"} via=${data.via || "-"}${data.status ? ` status=${data.status}` : ""} ${safeText(data.message || "")}`;
+  } else if (type === "relay-miss") {
+    detail = `宽带中继无候选可试（全部候选未通/无存活成员，交回本地兜底）`;
+  } else if (type === "peer-race-start") {
+    detail = `组员竞速开始 peers=${data.peers ?? "-"}`;
+  } else if (type === "peer-budget-out") {
+    detail = `组员竞速预算用尽 reason=${safeText(data.reason || "", 80)} budgetMs=${data.budgetMs ?? "-"} remaining=${data.remaining ?? "-"}`;
   } else detail = safeText(`${decisionKv(data)} ${data.message || data.error || ""}`, 200);
   return `[${base.time}] [req=${base.req}] [model=${base.model}] [stage=${base.stage}] ${detail}`;
 }

package/src/routes/groups-relay.js

@@ -3,6 +3,20 @@ import { DEFAULT_MAX_HOPS } from "../peers.js";
 import { enqueueRelay, dequeueRelayForPoll, resolveRelay, subscribeStream, unsubscribeStream } from "./relay-queue.js";
 import { compatFetch } from "../compat.js";
 
+/** 请求规模摘要（给中继日志用）：消息数/角色分布/工具数/maxTokens/model —— 排障第一眼要看清"转了什么"。 */
+function relayReqSummary(b = {}) {
+  const msgs = Array.isArray(b?.messages) ? b.messages : [];
+  const roles = {};
+  for (const m of msgs) roles[String(m?.role || "?")] = (roles[String(m?.role || "?")] || 0) + 1;
+  return { model: b?.model || "-", stream: b?.stream === true, messages: msgs.length, roles, tools: Array.isArray(b?.tools) ? b.tools.length : 0, maxTokens: Number(b?.max_tokens ?? b?.max_completion_tokens) || null };
+}
+
+/** 响应正文摘要（供中继日志）：单行化 + 截断，只为让人看出上游回的是什么。 */
+function snippet(v, n = 300) {
+  const s = typeof v === "string" ? v : JSON.stringify(v);
+  return String(s || "").replace(/\s+/g, " ").slice(0, n);
+}
+
 export async function heartbeatHandler({ req, res, groups, bus, logs }) {
   const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
   if (!auth) return json(res, 401, { error: "bearer token required" });
@@ -169,10 +183,15 @@ export async function forwardHandler({ req, res, groups, bus, logs, token }) {
   const targetMember = Object.values(members).find((m) => m.url === target) || Object.entries(members).find(([id]) => id === target)?.[1];
   if (!targetMember) return json(res, 404, { error: `target ${target} not found in group ${groupName}` });
   const isBb = targetMember.kind === "broadband" || String(targetMember.url).startsWith("relay://");
+  const reqSummary = relayReqSummary(body.body || body);
+  const fwdEvt = (type, extra) => { const e = { ts: Date.now(), type, group: groupName, target, hops, ...extra }; try { logs?.appendEvent?.(e); } catch {} try { bus?.emit?.(e); } catch {} };
+  // 组长「收到」：谁让我转给谁、转的是什么规模 —— 中继全流程的第一条日志
+  fwdEvt("relay-received", { via: isBb ? "broadband" : "direct", reqId: body.reqId || null, ...reqSummary });
   if (!isBb) {
     // 标记走直连：**在 fetch 之前**就设，这样直连失败（catch 里的 502）也带着它——
     // 那正是最需要区分的一跳（组长直连该成员不通 vs 宽带成员回报失败，两者的 body 可能一模一样）。
     res.setHeader("x-mslxdff-relay-via", "direct");
+    const t0 = Date.now();
     try {
       const fwdBody = body.body || body;
       const r = await compatFetch(`${targetMember.url}/v1/chat/completions`, {
@@ -180,41 +199,49 @@ export async function forwardHandler({ req, res, groups, bus, logs, token }) {
         headers: { "Content-Type": "application/json", "Authorization": `Bearer ${targetMember.token || ""}`, "x-mslxdff-hops": String(hops + 1), "x-mslxdff-model-lock": fwdBody.model || "", "Accept": "text/event-stream" },
         body: JSON.stringify(fwdBody),
       });
-      const evtData = { ts: Date.now(), type: "relay-forward", target, via: "direct", group: groupName, hops };
-      if (bus) bus.emit(evtData);
-      logs?.appendEvent?.(evtData);
-      res.statusCode = r.status;
-      if (r.headers.get("content-type")?.includes("text/event-stream")) {
+      const isSSE = r.headers.get("content-type")?.includes("text/event-stream");
+      if (isSSE) {
+        // 流式直连：没法先读全 body 再转发，就记「已建立」+ 状态码
+        fwdEvt("relay-forward", { via: "direct", ok: r.status < 400, status: r.status, ms: Date.now() - t0, detail: "stream 直连建立" });
+        res.statusCode = r.status;
         res.setHeader("Content-Type", "text/event-stream");
         if (r.body) for await (const c of r.body) res.write(c);
         res.end();
       } else {
         const txt = await r.text();
+        // 直连非流式：把上游回的正文摘要也记下（这就是「组员回了什么」）
+        fwdEvt("relay-forward", { via: "direct", ok: r.status < 400, status: r.status, ms: Date.now() - t0, respChars: txt.length, respBody: snippet(txt) });
+        res.statusCode = r.status;
         res.setHeader("Content-Type", r.headers.get("content-type") || "application/json");
         res.end(txt);
       }
       return;
     } catch (err) {
+      fwdEvt("relay-forward", { via: "direct", ok: false, status: 502, ms: Date.now() - t0, error: errMsg(err) });
       return json(res, 502, { error: errMsg(err) });
     }
   }
   const reqId = body.reqId || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
   const fwdBody = body.body || { model: body.model, messages: body.messages, stream: body.stream };
-  const evtData = { ts: Date.now(), type: "relay-forward", target, via: "leader", group: groupName, hops, reqId, model: fwdBody.model };
-  if (bus) bus.emit(evtData);
-  logs?.appendEvent?.(evtData);
   const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
   if (typeof targetMember.lastSeen === "number" && Date.now() - targetMember.lastSeen > staleMs) {
     res.setHeader("x-mslxdff-relay-via", "broadband");
+    fwdEvt("relay-member-result", { reqId, ok: false, status: 502, error: "broadband member stale (no heartbeat)" });
     return json(res, 502, { error: "broadband member stale (no heartbeat)" });
   }
   // 标记走的是宽带中继：下游据此一眼分清「组长直连失败(502 fetch failed)」vs「宽带成员回报的失败」，
   // 两者响应体可能都是 {"error":"fetch failed"}，只有这个头能区分（成员侧已把真实原因写进 errors.log）。
   res.setHeader("x-mslxdff-relay-via", "broadband");
+  const tFwd = Date.now();
+  // 组长「已交给成员」：入队即记 —— 成员有没有真正收到，由成员侧那条 relay-task-recv 佐证
+  fwdEvt("relay-queued", { reqId, via: "broadband" });
   try {
     const resultPromise = enqueueRelay({ group: groupName, target: targetMember.url, reqId, body: fwdBody, hops });
     const result = await resultPromise;
     if (result && typeof result.status === "number") {
+      const rb = result.body == null ? "" : (typeof result.body === "string" ? result.body : JSON.stringify(result.body));
+      // 组长「收到成员回什么」：状态码 + 正文摘要 + 耗时 —— 中继全流程的收尾日志
+      fwdEvt("relay-member-result", { reqId, ok: result.status < 400, status: result.status, ms: Date.now() - tFwd, respChars: rb.length, respBody: snippet(rb) });
       res.statusCode = result.status;
       if (result.status >= 500) res.setHeader("x-mslxdff-relay-member-status", String(result.status));
       if (result.headers) for (const [k, v] of Object.entries(result.headers)) res.setHeader(k, v);
@@ -224,8 +251,10 @@ export async function forwardHandler({ req, res, groups, bus, logs, token }) {
       } else res.end();
       return;
     }
+    fwdEvt("relay-member-result", { reqId, ok: true, status: 200, ms: Date.now() - tFwd, respBody: snippet(result) });
     return json(res, 200, result);
   } catch (err) {
+    fwdEvt("relay-member-result", { reqId, ok: false, status: 504, ms: Date.now() - tFwd, error: errMsg(err) || "relay timeout" });
     return json(res, 504, { error: errMsg(err) || "relay timeout" });
   }
 }

package/src/routes/relay-queue.js

@@ -140,7 +140,10 @@ export async function tryBroadbandRelay({ groups, token: myToken, model, body, h
         }
       } catch {}
     }
-    if (!allCandidates.length) return null;
+    if (!allCandidates.length) {
+      evt?.("relay-miss", { reqId, model, reason: "no-candidates", candidates: 0 });
+      return null;
+    }
     for (const cand of allCandidates) {
       try {
         evt?.("relay-try", { reqId, model, via: cand.via, target: cand.target, group: cand.group });

package/src/runtime/broadband.js

@@ -24,6 +24,11 @@ export function createBroadbandExec({ token, upstream, logs, log = console.log,
     const t0 = Date.now();
     const model = String(body?.model || "");
     const fl = withOutputFloor(body); // 首发输出额度托底：leader 转来的 body 不经 pipeline，成员侧自托（评审 P1）
+    // 成员「收到任务」：谁转来的、是什么规模、转给谁 —— 中继全流程在**成员这一跳**的入口日志。
+    // 没有它，组长侧只知道"入队了"，无从确认成员是否真收到、收到的是什么（用户实证要求「日志要全面」）。
+    const nMsgs = Array.isArray(body?.messages) ? body.messages.length : 0;
+    const nTools = Array.isArray(body?.tools) ? body.tools.length : 0;
+    log(`broadband task recv: group=${g.name} req=${reqId} from leader=${g.leaderUrl} model=${model} messages=${nMsgs} tools=${nTools}`);
     if (fl !== body) log(`output-floor(broadband): req=${reqId} ${(body?.max_tokens ?? body?.max_completion_tokens)}→${(fl.max_tokens ?? fl.max_completion_tokens)}`);
     try {
       const upRes = await upstream.chat(fl);