mslxdff 0.1.210 → 0.1.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/group-test-cli.js +1 -0
- package/src/cli/group-test.js +15 -4
- package/src/net-diag.js +125 -0
- package/src/routes/groups-relay.js +8 -0
- package/src/runtime/bootstrap.js +1 -1
- package/src/runtime/broadband-stream.js +12 -6
- package/src/runtime/broadband.js +65 -23
package/package.json
CHANGED
|
@@ -60,6 +60,7 @@ export function renderGroupTest(r, { json = false } = {}) {
|
|
|
60
60
|
upstream: "成员打上游失败",
|
|
61
61
|
}[r.stage] || r.stage;
|
|
62
62
|
console.log(` ❌ 没通 — 断在:**${stageZh}**${r.httpStatus ? `(HTTP ${r.httpStatus})` : ""}`);
|
|
63
|
+
if (r.relayVia) console.log(` 路径:组长 ${r.relayVia === "broadband" ? "经中继转发给成员" : "直连成员"}${r.memberStatus ? `(成员回报 HTTP ${r.memberStatus})` : ""}`);
|
|
63
64
|
if (r.error) console.log(` 原因:${String(r.error).slice(0, 300)}`);
|
|
64
65
|
if (r.hint) console.log(` 下一步:${r.hint}`);
|
|
65
66
|
process.exitCode = 1;
|
package/src/cli/group-test.js
CHANGED
|
@@ -102,10 +102,21 @@ export async function runGroupLinkTest({
|
|
|
102
102
|
if (res.status !== 200) {
|
|
103
103
|
const isFetchFail = /fetch failed/i.test(txt);
|
|
104
104
|
const isRelay = String(target).startsWith("relay://");
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
105
|
+
// 归因头(0.1.211+ 组长才有):把「组长直连失败」与「宽带成员回报的失败」分开——
|
|
106
|
+
// 两者响应体可能都长成 {"error":"fetch failed"},只有这个头能区分断在哪一跳。
|
|
107
|
+
const relayVia = res.headers?.get?.("x-mslxdff-relay-via") || "";
|
|
108
|
+
const memberStatus = res.headers?.get?.("x-mslxdff-relay-member-status") || "";
|
|
109
|
+
let hint;
|
|
110
|
+
if (relayVia === "broadband") {
|
|
111
|
+
hint = `组长已把请求转给该成员(宽带中继正常),失败发生在**成员那一跳**:成员执行自己的上游时抛错并把错原样回报(该成员侧 errors.log 有真正原因:连不上/DNS/超时)。判据:同组其他 relay 成员若正常,即证是这一台的网络/上游问题。`;
|
|
112
|
+
} else if (relayVia === "direct") {
|
|
113
|
+
hint = `组长是**直连**该成员失败的(它离线或端口不通)—— 不是中继链路问题。`;
|
|
114
|
+
} else if (isFetchFail && isRelay) {
|
|
115
|
+
hint = `对 relay 成员,502 fetch failed 通常是**该成员自己**打上游失败(组长只转发,不碰上游)。判据:同组其他 relay 成员若正常,即证是这一台的网络/上游问题。组长版本 <0.1.211 时无归因头,升级后可直接判「直连 or 中继」。`;
|
|
116
|
+
} else if (isFetchFail) {
|
|
117
|
+
hint = `组长直连该静态成员失败(它离线或端口不通)。`;
|
|
118
|
+
}
|
|
119
|
+
return { status: "error", stage: "forward-failed", httpStatus: res.status, target, leaderUrl: lead.leaderUrl, group: groupName, ms, error: txt.slice(0, 300), relayVia, memberStatus, hint };
|
|
109
120
|
}
|
|
110
121
|
|
|
111
122
|
let reply = null;
|
package/src/net-diag.js
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// 网络错误诊断 —— 把 undici 那句没头没尾的 `fetch failed` 展开成「哪种故障、打谁、卡在哪」。
|
|
2
|
+
//
|
|
3
|
+
// 为什么需要它:undici 抛 `TypeError: fetch failed` 时,真正的原因(ECONNREFUSED / ENOTFOUND /
|
|
4
|
+
// headers timeout / TLS …)藏在 `err.cause`,再深一次还可能藏在 `cause.cause`。而 `errMsg()`
|
|
5
|
+
// 只取 `err.message`,于是全链路日志里只剩 `fetch failed` —— 这正是「组员 B 回报 502 fetch failed、
|
|
6
|
+
// 却谁也说不清它为什么打不到上游」的死因(member 侧 broadband execAndPost 原样把 errMsg 回给组长)。
|
|
7
|
+
//
|
|
8
|
+
// 本模块是纯函数、零依赖:把 err 连同 cause 链压成 { name, code, message, addr, syscall, summary },
|
|
9
|
+
// 让成员侧、组长侧、CLI 三处都能写同一口径的一句话。
|
|
10
|
+
|
|
11
|
+
/** 检测 fetch 网络错误标志位的有序列表:先 code,再 name,再看 message。 */
|
|
12
|
+
const KNOWN_CODES = new Set([
|
|
13
|
+
"ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT", "ECONNRESET", "EPIPE",
|
|
14
|
+
"ENETUNREACH", "EHOSTUNREACH", "EADDRNOTAVAIL", "ENETDOWN", "ECONNABORTED",
|
|
15
|
+
"UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT",
|
|
16
|
+
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_ABORTED", "UND_ERR_RESPONSE_STATUS_CODE",
|
|
17
|
+
"CERT_HAS_EXPIRED", "ERR_TLS_CERT_ALTNAME_INVALID", "UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
|
18
|
+
"DEPTH_ZERO_SELF_SIGNED_CERT", "SELF_SIGNED_CERT_IN_CHAIN",
|
|
19
|
+
]);
|
|
20
|
+
|
|
21
|
+
/** code/name → 人话短语。查不到就回落成 code 本身。 */
|
|
22
|
+
const CODE_ZH = {
|
|
23
|
+
ECONNREFUSED: "连接被拒绝",
|
|
24
|
+
ENOTFOUND: "DNS 解析失败",
|
|
25
|
+
EAI_AGAIN: "DNS 暂时失败",
|
|
26
|
+
ETIMEDOUT: "连接超时",
|
|
27
|
+
ECONNRESET: "连接被重置",
|
|
28
|
+
EPIPE: "管道已断开",
|
|
29
|
+
ENETUNREACH: "网络不可达",
|
|
30
|
+
EHOSTUNREACH: "主机不可达",
|
|
31
|
+
EADDRNOTAVAIL: "本地地址不可用(出口绑定失败)",
|
|
32
|
+
ECONNABORTED: "连接被中断",
|
|
33
|
+
UND_ERR_CONNECT_TIMEOUT: "建连超时",
|
|
34
|
+
UND_ERR_HEADERS_TIMEOUT: "等响应头超时",
|
|
35
|
+
UND_ERR_BODY_TIMEOUT: "读响应体超时",
|
|
36
|
+
UND_ERR_SOCKET: "socket 异常",
|
|
37
|
+
UND_ERR_CLOSED: "连接已被关闭",
|
|
38
|
+
UND_ERR_ABORTED: "请求被取消",
|
|
39
|
+
CERT_HAS_EXPIRED: "TLS 证书过期",
|
|
40
|
+
ERR_TLS_CERT_ALTNAME_INVALID: "TLS 证书域名不匹配",
|
|
41
|
+
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "TLS 证书链无法验证",
|
|
42
|
+
DEPTH_ZERO_SELF_SIGNED_CERT: "TLS 自签证书",
|
|
43
|
+
SELF_SIGNED_CERT_IN_CHAIN: "TLS 证书链含自签证书",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** 从一层的 err/cause 对象上读 code:优先 `.code`,其次 `.name`(undici 超时类常用 name)。 */
|
|
47
|
+
function pickCode(node) {
|
|
48
|
+
if (!node) return "";
|
|
49
|
+
const c = node.code;
|
|
50
|
+
if (typeof c === "string" && c) return c;
|
|
51
|
+
const n = node.name;
|
|
52
|
+
if (typeof n === "string" && n && n !== "Error" && n !== "TypeError") return n;
|
|
53
|
+
return "";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** 把 message 里可能带的 URL 提炼出来(`fetch failed to http://…` 少见,但 socket 错误常带地址)。 */
|
|
57
|
+
function pickAddr(node) {
|
|
58
|
+
const a = node?.address || node?.host || node?.hostname;
|
|
59
|
+
if (typeof a === "string" && a) return a;
|
|
60
|
+
const m = /(?:https?:\/\/)?(\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?)/.exec(String(node?.message || ""));
|
|
61
|
+
return m ? m[1] : "";
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 把任意 thrown 值展开成诊断对象。
|
|
66
|
+
* @param {unknown} err
|
|
67
|
+
* @returns {{ name: string, code: string, message: string, addr: string, syscall: string, summary: string }}
|
|
68
|
+
*/
|
|
69
|
+
export function describeError(err) {
|
|
70
|
+
const chain = [];
|
|
71
|
+
let node = err;
|
|
72
|
+
for (let depth = 0; node && depth < 5; depth++) {
|
|
73
|
+
chain.push(node);
|
|
74
|
+
node = node.cause;
|
|
75
|
+
}
|
|
76
|
+
const first = chain[0] || {};
|
|
77
|
+
const name = String(first.name || "Error");
|
|
78
|
+
|
|
79
|
+
// 选定「有信息量」的那层:第一层带已知 code 的层;都没有就用最深的非空 code,再退回第一层。
|
|
80
|
+
let chosen = null;
|
|
81
|
+
for (const n of chain) {
|
|
82
|
+
const c = pickCode(n);
|
|
83
|
+
if (c && KNOWN_CODES.has(c)) { chosen = { node: n, code: c }; break; }
|
|
84
|
+
}
|
|
85
|
+
if (!chosen) {
|
|
86
|
+
for (let i = chain.length - 1; i >= 0; i--) {
|
|
87
|
+
const c = pickCode(chain[i]);
|
|
88
|
+
if (c) { chosen = { node: chain[i], code: c }; break; }
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const code = chosen?.code || "";
|
|
92
|
+
const addr = pickAddr(chosen?.node || first);
|
|
93
|
+
const syscall = String(chosen?.node?.syscall || first.syscall || "");
|
|
94
|
+
|
|
95
|
+
// message 取「最深一条非空 message」—— 那通常是最贴近 syscall 的一句;都没有就用第一层。
|
|
96
|
+
let message = "";
|
|
97
|
+
for (let i = chain.length - 1; i >= 0; i--) {
|
|
98
|
+
const m = String(chain[i]?.message || "").trim();
|
|
99
|
+
if (m) { message = m; break; }
|
|
100
|
+
}
|
|
101
|
+
if (!message) message = String(first.message || err || "");
|
|
102
|
+
|
|
103
|
+
const zh = CODE_ZH[code] || "";
|
|
104
|
+
const parts = [];
|
|
105
|
+
parts.push(code ? `${code}${zh ? `(${zh})` : ""}` : name);
|
|
106
|
+
if (addr) parts.push(`peer=${addr}`);
|
|
107
|
+
if (syscall) parts.push(`syscall=${syscall}`);
|
|
108
|
+
const head = parts.join(" ");
|
|
109
|
+
// message 已含 code 时不重复(如 "connect ECONNREFUSED …" 与 code=ECONNREFUSED);
|
|
110
|
+
// 注意 code 为空时不能建空正则——空 pattern 匹配一切,会把 message 误判成冗余。
|
|
111
|
+
const redundant = code && new RegExp(escapeRegExp(code)).test(message);
|
|
112
|
+
const summary = message && message !== code && !redundant
|
|
113
|
+
? `${head} — ${message}`
|
|
114
|
+
: head;
|
|
115
|
+
return { name, code, message, addr, syscall, summary };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** 一句话诊断,供日志/回包直接用。 */
|
|
119
|
+
export function describeErrorText(err) {
|
|
120
|
+
return describeError(err).summary;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function escapeRegExp(s) {
|
|
124
|
+
return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
125
|
+
}
|
|
@@ -161,6 +161,9 @@ export async function forwardHandler({ req, res, groups, bus, logs }) {
|
|
|
161
161
|
if (!targetMember) return json(res, 404, { error: `target ${target} not found in group ${groupName}` });
|
|
162
162
|
const isBb = targetMember.kind === "broadband" || String(targetMember.url).startsWith("relay://");
|
|
163
163
|
if (!isBb) {
|
|
164
|
+
// 标记走直连:**在 fetch 之前**就设,这样直连失败(catch 里的 502)也带着它——
|
|
165
|
+
// 那正是最需要区分的一跳(组长直连该成员不通 vs 宽带成员回报失败,两者的 body 可能一模一样)。
|
|
166
|
+
res.setHeader("x-mslxdff-relay-via", "direct");
|
|
164
167
|
try {
|
|
165
168
|
const fwdBody = body.body || body;
|
|
166
169
|
const r = await compatFetch(`${targetMember.url}/v1/chat/completions`, {
|
|
@@ -193,13 +196,18 @@ export async function forwardHandler({ req, res, groups, bus, logs }) {
|
|
|
193
196
|
logs?.appendEvent?.(evtData);
|
|
194
197
|
const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
|
|
195
198
|
if (typeof targetMember.lastSeen === "number" && Date.now() - targetMember.lastSeen > staleMs) {
|
|
199
|
+
res.setHeader("x-mslxdff-relay-via", "broadband");
|
|
196
200
|
return json(res, 502, { error: "broadband member stale (no heartbeat)" });
|
|
197
201
|
}
|
|
202
|
+
// 标记走的是宽带中继:下游据此一眼分清「组长直连失败(502 fetch failed)」vs「宽带成员回报的失败」,
|
|
203
|
+
// 两者响应体可能都是 {"error":"fetch failed"},只有这个头能区分(成员侧已把真实原因写进 errors.log)。
|
|
204
|
+
res.setHeader("x-mslxdff-relay-via", "broadband");
|
|
198
205
|
try {
|
|
199
206
|
const resultPromise = enqueueRelay({ group: groupName, target: targetMember.url, reqId, body: fwdBody, hops });
|
|
200
207
|
const result = await resultPromise;
|
|
201
208
|
if (result && typeof result.status === "number") {
|
|
202
209
|
res.statusCode = result.status;
|
|
210
|
+
if (result.status >= 500) res.setHeader("x-mslxdff-relay-member-status", String(result.status));
|
|
203
211
|
if (result.headers) for (const [k, v] of Object.entries(result.headers)) res.setHeader(k, v);
|
|
204
212
|
if (result.body) {
|
|
205
213
|
if (typeof result.body === "string") res.end(result.body);
|
package/src/runtime/bootstrap.js
CHANGED
|
@@ -35,7 +35,7 @@ export async function startDaemonMain(VERSION) {
|
|
|
35
35
|
const { srv, bus } = await startServerLifecycle({ VERSION, ...ctx, logs });
|
|
36
36
|
void srv;
|
|
37
37
|
startGroupSync({ peers: ctx.peers, groups: ctx.groups, logs });
|
|
38
|
-
startBroadband({ token: ctx.token, upstream: ctx.upstream });
|
|
38
|
+
startBroadband({ token: ctx.token, upstream: ctx.upstream, logs });
|
|
39
39
|
|
|
40
40
|
// 保热心跳:每 ≤120s 经 **chat 同池**访问每个组员(/health) + 每个 picks 供应商(preheat),
|
|
41
41
|
// 防「久未使用 → 首发 fetch failed / 上游冷启动」(现网 2026-10-18 16:16 三组员同时 fetch failed)。
|
|
@@ -5,7 +5,7 @@ import { compatFetch } from "../compat.js";
|
|
|
5
5
|
* 宽带 SSE 长连 — 单组成员订阅 + 指数退避重连 + ensure 定时器。
|
|
6
6
|
* execAndPost / broadbandGroups 由 broadband.js 注入。
|
|
7
7
|
*/
|
|
8
|
-
export function startBroadbandStream({ token, upstream, execAndPost, broadbandGroups }) {
|
|
8
|
+
export function startBroadbandStream({ token, upstream, execAndPost, broadbandGroups, log = console.log }) {
|
|
9
9
|
const streamManagers = new Map();
|
|
10
10
|
const startStreamForGroup = (g) => {
|
|
11
11
|
if (streamManagers.has(g.name)) return;
|
|
@@ -25,7 +25,7 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
25
25
|
if (!res.ok) throw new Error(`stream ${res.status}`);
|
|
26
26
|
if (!res.body) throw new Error("no body");
|
|
27
27
|
attempts = 0;
|
|
28
|
-
|
|
28
|
+
log(`broadband relay: connected ${g.name} via ${g.leaderUrl} (SSE)`);
|
|
29
29
|
let buf = "";
|
|
30
30
|
const decodeChunk = (c) => {
|
|
31
31
|
if (typeof c === "string") return c;
|
|
@@ -50,8 +50,14 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
50
50
|
try {
|
|
51
51
|
const parsed = JSON.parse(data);
|
|
52
52
|
const { reqId, body } = parsed;
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
// 不再 `.catch(()=>{})` 静默吞:execAndPost 内部已把执行/回传失败写 errors.log,
|
|
54
|
+
// 这里只兜底一句,防它真抛出(如 body 畸变)时整条 SSE 循环被吞掉无痕。
|
|
55
|
+
if (reqId && body) {
|
|
56
|
+
execAndPost(g, reqId, body).catch((e) => log(`broadband exec ${g.name} req=${reqId}: failed — ${errMsg(e)}`));
|
|
57
|
+
}
|
|
58
|
+
} catch (e) {
|
|
59
|
+
log(`broadband stream ${g.name}: bad relay frame — ${errMsg(e)}`);
|
|
60
|
+
}
|
|
55
61
|
}
|
|
56
62
|
}
|
|
57
63
|
}
|
|
@@ -59,7 +65,7 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
59
65
|
} catch (err) {
|
|
60
66
|
if (stopped) return;
|
|
61
67
|
const msg = errMsg(err);
|
|
62
|
-
if (!String(msg).includes("abort") && !String(msg).includes("Abort"))
|
|
68
|
+
if (!String(msg).includes("abort") && !String(msg).includes("Abort")) log(`broadband stream ${g.name}: ${msg} — reconnecting`);
|
|
63
69
|
attempts++;
|
|
64
70
|
const delay = Math.min(30_000, 1000 * Math.pow(2, attempts - 1) + Math.random() * 500);
|
|
65
71
|
await new Promise((r) => setTimeout(r, delay));
|
|
@@ -75,5 +81,5 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
75
81
|
}, 10_000);
|
|
76
82
|
ensureTimer.unref();
|
|
77
83
|
const n = broadbandGroups().length;
|
|
78
|
-
|
|
84
|
+
log(n ? `broadband relay: stream (SSE) + ping 25s for ${n} group(s)` : "broadband relay: idle (0 group) — waiting for a group join, checked every 10s");
|
|
79
85
|
}
|
package/src/runtime/broadband.js
CHANGED
|
@@ -1,26 +1,30 @@
|
|
|
1
1
|
import { loadGroupsJoined } from "../state.js";
|
|
2
2
|
import { errMsg } from "../cli/util.js";
|
|
3
|
+
import { describeErrorText } from "../net-diag.js";
|
|
3
4
|
import { withOutputFloor } from "../chat-pipeline/empty-turn.js";
|
|
4
5
|
import { startBroadbandStream } from "./broadband-stream.js";
|
|
5
6
|
import { compatFetch, timeoutSignal } from "../compat.js";
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* 造一个「成员侧执行 + 回传结果」的执行器(可注入依赖,便于单测)。
|
|
10
|
+
*
|
|
11
|
+
* 为什么要单独抽出来并**带日志**:成员 B 用 upstream.chat 打自己的上游失败时,原实现只造了个
|
|
12
|
+
* `{status:502, body:{"error":"fetch failed"}}` 回传组长,**本机一行日志都没留** —— 于是组长侧
|
|
13
|
+
* 看到 `502 fetch failed`、成员侧看到空日志,全链路都没人能说出 B 为什么打不到上游。
|
|
14
|
+
* 现在:执行抛错记 errors.log(带 cause 链诊断),上游返回 4xx/5xx 也记,回传失败同样记。
|
|
15
|
+
*
|
|
16
|
+
* @param {{ token: string, upstream: {chat: Function}, logs?: {appendError?: Function, appendEvent?: Function}, log?: Function, fetchImpl?: Function }} opts
|
|
17
|
+
* @returns {(g: {name: string, leaderUrl: string}, reqId: string, body: object) => Promise<void>}
|
|
12
18
|
*/
|
|
13
|
-
export function
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
if (v === "0" || v === "false" || v === "off") return false;
|
|
18
|
-
return true;
|
|
19
|
-
})();
|
|
20
|
-
const execAndPost = async (g, reqId, body) => {
|
|
19
|
+
export function createBroadbandExec({ token, upstream, logs, log = console.log, fetchImpl = compatFetch } = {}) {
|
|
20
|
+
const appendError = (entry) => { try { logs?.appendError?.(entry); } catch { /* 日志失败不拖垮执行 */ } };
|
|
21
|
+
const appendEvent = (entry) => { try { logs?.appendEvent?.(entry); } catch {} };
|
|
22
|
+
return async function execAndPost(g, reqId, body) {
|
|
21
23
|
let result;
|
|
24
|
+
const t0 = Date.now();
|
|
25
|
+
const model = String(body?.model || "");
|
|
22
26
|
const fl = withOutputFloor(body); // 首发输出额度托底:leader 转来的 body 不经 pipeline,成员侧自托(评审 P1)
|
|
23
|
-
if (fl !== body)
|
|
27
|
+
if (fl !== body) log(`output-floor(broadband): req=${reqId} ${(body?.max_tokens ?? body?.max_completion_tokens)}→${(fl.max_tokens ?? fl.max_completion_tokens)}`);
|
|
24
28
|
try {
|
|
25
29
|
const upRes = await upstream.chat(fl);
|
|
26
30
|
const ct = upRes.headers.get("content-type") || "";
|
|
@@ -40,33 +44,69 @@ export function startBroadband({ token, upstream }) {
|
|
|
40
44
|
try { parsed = JSON.parse(txt); } catch { parsed = txt; }
|
|
41
45
|
result = { status: upRes.status, headers: { "Content-Type": upRes.headers.get("content-type") || "application/json" }, body: typeof parsed === "string" ? parsed : JSON.stringify(parsed) };
|
|
42
46
|
}
|
|
47
|
+
appendEvent({ ts: Date.now(), type: "broadband-task", reqId, group: g.name, model, status: result.status, ms: Date.now() - t0, chars: String(result.body || "").length });
|
|
48
|
+
if (result.status >= 400) {
|
|
49
|
+
// 成员成功连上上游、但上游回错 —— 也要留痕,否则组长只见状态码不知原因
|
|
50
|
+
const snippet = String(result.body || "").slice(0, 300);
|
|
51
|
+
log(`broadband task: ${g.name} req=${reqId} 成员上游返回 HTTP ${result.status} — ${snippet}`);
|
|
52
|
+
appendError({ api: "broadband", reqId, group: g.name, model, status: result.status, message: `组员执行上游返回 HTTP ${result.status}:${snippet}` });
|
|
53
|
+
}
|
|
43
54
|
} catch (err) {
|
|
44
|
-
|
|
55
|
+
// 成员打自己上游**抛错**(连不上/DNS/超时/TLS)—— 这是 `502 fetch failed` 的真正来源,展开 cause 链
|
|
56
|
+
const diag = describeErrorText(err);
|
|
57
|
+
result = { status: 502, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: diag }) };
|
|
58
|
+
log(`broadband task: ${g.name} req=${reqId} 成员打上游失败 — ${diag}`);
|
|
59
|
+
appendError({ api: "broadband", reqId, group: g.name, model, status: 502, message: `组员打上游请求抛错(成员侧失败,非组长):${diag}` });
|
|
45
60
|
}
|
|
46
61
|
try {
|
|
47
|
-
await
|
|
62
|
+
const r = await fetchImpl(`${g.leaderUrl}/v1/groups/relay/result`, {
|
|
48
63
|
method: "POST",
|
|
49
|
-
headers: { "Content-Type": "application/json",
|
|
64
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
50
65
|
body: JSON.stringify({ name: g.name, group: g.name, reqId, result }),
|
|
51
66
|
signal: timeoutSignal(5000),
|
|
52
67
|
});
|
|
53
|
-
|
|
68
|
+
if (!r.ok) {
|
|
69
|
+
const txt = await r.text().catch(() => "");
|
|
70
|
+
log(`broadband result post: ${g.name} req=${reqId} HTTP ${r.status} ${txt.slice(0, 120)}`);
|
|
71
|
+
appendError({ api: "broadband", reqId, group: g.name, status: r.status, message: `结果回传组长失败 HTTP ${r.status}:${txt.slice(0, 200)}` });
|
|
72
|
+
}
|
|
73
|
+
} catch (err) {
|
|
74
|
+
const diag = describeErrorText(err);
|
|
75
|
+
log(`broadband result post: ${g.name} req=${reqId} failed — ${diag}`);
|
|
76
|
+
appendError({ api: "broadband", reqId, group: g.name, status: 0, message: `结果回传组长请求抛错:${diag}` });
|
|
77
|
+
}
|
|
54
78
|
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* 宽带中继 — poll 模式(heartbeat + poll)与 stream 模式分发。
|
|
83
|
+
* stream 长连下沉 broadband-stream.js,本文件保留任务执行与降级分支。
|
|
84
|
+
* 空组不再早退:加组后由 ensure(stream 10s / poll 每次读组)自动接上,无需重启 daemon。
|
|
85
|
+
* // Note: 空组不早退与向导默认 broadband 的取舍 — 见 .agents/notes/implemented/feature/2026-09-16-mobile-broadband-join-wizard.md
|
|
86
|
+
*/
|
|
87
|
+
export function startBroadband({ token, upstream, logs, log = console.log }) {
|
|
88
|
+
const broadbandGroups = () => loadGroupsJoined().filter((g) => g.kind === "broadband" && g.leaderUrl);
|
|
89
|
+
const streamEnabled = (() => {
|
|
90
|
+
const v = process.env.MSLXDFF_BROADBAND_STREAM;
|
|
91
|
+
if (v === "0" || v === "false" || v === "off") return false;
|
|
92
|
+
return true;
|
|
93
|
+
})();
|
|
94
|
+
const execAndPost = createBroadbandExec({ token, upstream, logs, log });
|
|
55
95
|
const doHeartbeat = async () => {
|
|
56
96
|
for (const g of broadbandGroups()) {
|
|
57
97
|
try {
|
|
58
98
|
const res = await compatFetch(`${g.leaderUrl}/v1/groups/relay/heartbeat`, {
|
|
59
99
|
method: "POST",
|
|
60
|
-
headers: { "Content-Type": "application/json",
|
|
100
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
61
101
|
body: JSON.stringify({ name: g.name, group: g.name }),
|
|
62
102
|
signal: timeoutSignal(5000),
|
|
63
103
|
});
|
|
64
104
|
if (!res.ok) {
|
|
65
105
|
const txt = await res.text().catch(() => "");
|
|
66
|
-
|
|
106
|
+
log(`broadband heartbeat ${g.name}: ${res.status} ${txt.slice(0, 100)}`);
|
|
67
107
|
}
|
|
68
108
|
} catch (err) {
|
|
69
|
-
|
|
109
|
+
log(`broadband heartbeat ${g.name}: failed — ${errMsg(err)}`);
|
|
70
110
|
}
|
|
71
111
|
}
|
|
72
112
|
};
|
|
@@ -86,7 +126,9 @@ export function startBroadband({ token, upstream }) {
|
|
|
86
126
|
const { reqId, body } = item;
|
|
87
127
|
await execAndPost(g, reqId, body);
|
|
88
128
|
}
|
|
89
|
-
} catch {
|
|
129
|
+
} catch (err) {
|
|
130
|
+
log(`broadband poll ${g.name}: failed — ${describeErrorText(err)}`);
|
|
131
|
+
}
|
|
90
132
|
}
|
|
91
133
|
};
|
|
92
134
|
if (!streamEnabled) {
|
|
@@ -96,8 +138,8 @@ export function startBroadband({ token, upstream }) {
|
|
|
96
138
|
const pollTimer = setInterval(doPoll, 1000);
|
|
97
139
|
pollTimer.unref();
|
|
98
140
|
const n = broadbandGroups().length;
|
|
99
|
-
|
|
141
|
+
log(n ? `broadband relay: heartbeat 30s + poll 1s for ${n} group(s) [poll mode]` : "broadband relay: idle (0 group) — waiting for a group join [poll mode]");
|
|
100
142
|
return;
|
|
101
143
|
}
|
|
102
|
-
startBroadbandStream({ token, upstream, execAndPost, broadbandGroups });
|
|
144
|
+
startBroadbandStream({ token, upstream, execAndPost, broadbandGroups, log });
|
|
103
145
|
}
|