mslxdff 0.1.209 → 0.1.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/group-test-cli.js +1 -0
- package/src/cli/group-test.js +15 -4
- package/src/daemon.js +16 -2
- package/src/net-diag.js +125 -0
- package/src/routes/groups-relay.js +8 -0
- package/src/runtime/bootstrap.js +1 -1
- package/src/runtime/broadband-stream.js +12 -6
- package/src/runtime/broadband.js +65 -23
- package/src/runtime/proc-info.js +11 -0
- package/src/runtime/server-lifecycle.js +15 -1
package/package.json
CHANGED
|
@@ -60,6 +60,7 @@ export function renderGroupTest(r, { json = false } = {}) {
|
|
|
60
60
|
upstream: "成员打上游失败",
|
|
61
61
|
}[r.stage] || r.stage;
|
|
62
62
|
console.log(` ❌ 没通 — 断在:**${stageZh}**${r.httpStatus ? `(HTTP ${r.httpStatus})` : ""}`);
|
|
63
|
+
if (r.relayVia) console.log(` 路径:组长 ${r.relayVia === "broadband" ? "经中继转发给成员" : "直连成员"}${r.memberStatus ? `(成员回报 HTTP ${r.memberStatus})` : ""}`);
|
|
63
64
|
if (r.error) console.log(` 原因:${String(r.error).slice(0, 300)}`);
|
|
64
65
|
if (r.hint) console.log(` 下一步:${r.hint}`);
|
|
65
66
|
process.exitCode = 1;
|
package/src/cli/group-test.js
CHANGED
|
@@ -102,10 +102,21 @@ export async function runGroupLinkTest({
|
|
|
102
102
|
if (res.status !== 200) {
|
|
103
103
|
const isFetchFail = /fetch failed/i.test(txt);
|
|
104
104
|
const isRelay = String(target).startsWith("relay://");
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
105
|
+
// 归因头(0.1.211+ 组长才有):把「组长直连失败」与「宽带成员回报的失败」分开——
|
|
106
|
+
// 两者响应体可能都长成 {"error":"fetch failed"},只有这个头能区分断在哪一跳。
|
|
107
|
+
const relayVia = res.headers?.get?.("x-mslxdff-relay-via") || "";
|
|
108
|
+
const memberStatus = res.headers?.get?.("x-mslxdff-relay-member-status") || "";
|
|
109
|
+
let hint;
|
|
110
|
+
if (relayVia === "broadband") {
|
|
111
|
+
hint = `组长已把请求转给该成员(宽带中继正常),失败发生在**成员那一跳**:成员执行自己的上游时抛错并把错原样回报(该成员侧 errors.log 有真正原因:连不上/DNS/超时)。判据:同组其他 relay 成员若正常,即证是这一台的网络/上游问题。`;
|
|
112
|
+
} else if (relayVia === "direct") {
|
|
113
|
+
hint = `组长是**直连**该成员失败的(它离线或端口不通)—— 不是中继链路问题。`;
|
|
114
|
+
} else if (isFetchFail && isRelay) {
|
|
115
|
+
hint = `对 relay 成员,502 fetch failed 通常是**该成员自己**打上游失败(组长只转发,不碰上游)。判据:同组其他 relay 成员若正常,即证是这一台的网络/上游问题。组长版本 <0.1.211 时无归因头,升级后可直接判「直连 or 中继」。`;
|
|
116
|
+
} else if (isFetchFail) {
|
|
117
|
+
hint = `组长直连该静态成员失败(它离线或端口不通)。`;
|
|
118
|
+
}
|
|
119
|
+
return { status: "error", stage: "forward-failed", httpStatus: res.status, target, leaderUrl: lead.leaderUrl, group: groupName, ms, error: txt.slice(0, 300), relayVia, memberStatus, hint };
|
|
109
120
|
}
|
|
110
121
|
|
|
111
122
|
let reply = null;
|
package/src/daemon.js
CHANGED
|
@@ -102,12 +102,26 @@ export function isPidAlive(pid) {
|
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
export function stopDaemon({ reason = "stop", kill = process.kill, taskkill = spawnSync } = {}) {
|
|
105
|
+
export function stopDaemon({ reason = "stop", kill = process.kill, taskkill = spawnSync, isAlive = isPidAlive } = {}) {
|
|
106
106
|
const pid = readPid();
|
|
107
107
|
if (!pid) return { stopped: false, reason: "no pid file" };
|
|
108
|
+
// 自杀护栏(最高优先级):pid 文件里写着**本进程**时,杀它就是杀自己。
|
|
109
|
+
// 这不是假想情形 —— daemon 自己 `writePid(process.pid)`(server-lifecycle.js),而 `enableAutostart()`
|
|
110
|
+
// 会在 systemd 托管下被 daemon 自己调起(server-lifecycle 的 autostart 自检块):systemd **系统** unit
|
|
111
|
+
// 的 daemon 去查 **user** unit 必然判「未启用」⇒ 走进 enableAutostart ⇒ stopDaemon ⇒ SIGTERM 自己
|
|
112
|
+
// ⇒ 优雅关停 exit 0 ⇒ systemd Restart=always 拉起 ⇒ 每 2.5 秒一轮的无限重启
|
|
113
|
+
//(真机 2026-10-19:0.1.209 + 系统 unit,每进程活 2~3s 被 SIGTERM;0.1.160 不炸只因当时
|
|
114
|
+
// `enableAutostart` 里 import 路径写错、这段从未执行过,2026-10-17 c7bf8af 修正路径后暴露)。
|
|
115
|
+
if (pid === process.pid) {
|
|
116
|
+
try {
|
|
117
|
+
appendFileSync(logFile(), `[lifecycle] stopDaemon refused: pid file points at self (pid=${pid}, reason=${reason})\n`);
|
|
118
|
+
} catch {}
|
|
119
|
+
return { stopped: false, pid, reason: "self — pid 文件写的就是本进程,拒绝自杀" };
|
|
120
|
+
}
|
|
121
|
+
|
|
108
122
|
// 杀者牌(验尸官的另一半证据):动手前写、自带死者身份。只给在世者留牌——给尸体留牌
|
|
109
123
|
// 会让下次验尸把"早已静默死亡"误判成"被本次杀死"。
|
|
110
|
-
const victimAlive =
|
|
124
|
+
const victimAlive = isAlive(pid);
|
|
111
125
|
// 杀者留痕(写 daemon.log):Windows 的 SIGTERM 是 TerminateProcess 强杀,被杀的 daemon
|
|
112
126
|
// 在 JS 层收不到任何事件(无法自记),故由"杀者"记录调用方 pid——
|
|
113
127
|
// 没有这行 = 非我方所杀(外部 taskkill/任务管理器/崩溃),配合最后一条 heartbeat 定位死亡时刻。
|
package/src/net-diag.js
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// 网络错误诊断 —— 把 undici 那句没头没尾的 `fetch failed` 展开成「哪种故障、打谁、卡在哪」。
|
|
2
|
+
//
|
|
3
|
+
// 为什么需要它:undici 抛 `TypeError: fetch failed` 时,真正的原因(ECONNREFUSED / ENOTFOUND /
|
|
4
|
+
// headers timeout / TLS …)藏在 `err.cause`,再深一次还可能藏在 `cause.cause`。而 `errMsg()`
|
|
5
|
+
// 只取 `err.message`,于是全链路日志里只剩 `fetch failed` —— 这正是「组员 B 回报 502 fetch failed、
|
|
6
|
+
// 却谁也说不清它为什么打不到上游」的死因(member 侧 broadband execAndPost 原样把 errMsg 回给组长)。
|
|
7
|
+
//
|
|
8
|
+
// 本模块是纯函数、零依赖:把 err 连同 cause 链压成 { name, code, message, addr, syscall, summary },
|
|
9
|
+
// 让成员侧、组长侧、CLI 三处都能写同一口径的一句话。
|
|
10
|
+
|
|
11
|
+
/** 检测 fetch 网络错误标志位的有序列表:先 code,再 name,再看 message。 */
|
|
12
|
+
const KNOWN_CODES = new Set([
|
|
13
|
+
"ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT", "ECONNRESET", "EPIPE",
|
|
14
|
+
"ENETUNREACH", "EHOSTUNREACH", "EADDRNOTAVAIL", "ENETDOWN", "ECONNABORTED",
|
|
15
|
+
"UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT",
|
|
16
|
+
"UND_ERR_SOCKET", "UND_ERR_CLOSED", "UND_ERR_ABORTED", "UND_ERR_RESPONSE_STATUS_CODE",
|
|
17
|
+
"CERT_HAS_EXPIRED", "ERR_TLS_CERT_ALTNAME_INVALID", "UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
|
18
|
+
"DEPTH_ZERO_SELF_SIGNED_CERT", "SELF_SIGNED_CERT_IN_CHAIN",
|
|
19
|
+
]);
|
|
20
|
+
|
|
21
|
+
/** code/name → 人话短语。查不到就回落成 code 本身。 */
|
|
22
|
+
const CODE_ZH = {
|
|
23
|
+
ECONNREFUSED: "连接被拒绝",
|
|
24
|
+
ENOTFOUND: "DNS 解析失败",
|
|
25
|
+
EAI_AGAIN: "DNS 暂时失败",
|
|
26
|
+
ETIMEDOUT: "连接超时",
|
|
27
|
+
ECONNRESET: "连接被重置",
|
|
28
|
+
EPIPE: "管道已断开",
|
|
29
|
+
ENETUNREACH: "网络不可达",
|
|
30
|
+
EHOSTUNREACH: "主机不可达",
|
|
31
|
+
EADDRNOTAVAIL: "本地地址不可用(出口绑定失败)",
|
|
32
|
+
ECONNABORTED: "连接被中断",
|
|
33
|
+
UND_ERR_CONNECT_TIMEOUT: "建连超时",
|
|
34
|
+
UND_ERR_HEADERS_TIMEOUT: "等响应头超时",
|
|
35
|
+
UND_ERR_BODY_TIMEOUT: "读响应体超时",
|
|
36
|
+
UND_ERR_SOCKET: "socket 异常",
|
|
37
|
+
UND_ERR_CLOSED: "连接已被关闭",
|
|
38
|
+
UND_ERR_ABORTED: "请求被取消",
|
|
39
|
+
CERT_HAS_EXPIRED: "TLS 证书过期",
|
|
40
|
+
ERR_TLS_CERT_ALTNAME_INVALID: "TLS 证书域名不匹配",
|
|
41
|
+
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "TLS 证书链无法验证",
|
|
42
|
+
DEPTH_ZERO_SELF_SIGNED_CERT: "TLS 自签证书",
|
|
43
|
+
SELF_SIGNED_CERT_IN_CHAIN: "TLS 证书链含自签证书",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** 从一层的 err/cause 对象上读 code:优先 `.code`,其次 `.name`(undici 超时类常用 name)。 */
|
|
47
|
+
function pickCode(node) {
|
|
48
|
+
if (!node) return "";
|
|
49
|
+
const c = node.code;
|
|
50
|
+
if (typeof c === "string" && c) return c;
|
|
51
|
+
const n = node.name;
|
|
52
|
+
if (typeof n === "string" && n && n !== "Error" && n !== "TypeError") return n;
|
|
53
|
+
return "";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** 把 message 里可能带的 URL 提炼出来(`fetch failed to http://…` 少见,但 socket 错误常带地址)。 */
|
|
57
|
+
function pickAddr(node) {
|
|
58
|
+
const a = node?.address || node?.host || node?.hostname;
|
|
59
|
+
if (typeof a === "string" && a) return a;
|
|
60
|
+
const m = /(?:https?:\/\/)?(\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?)/.exec(String(node?.message || ""));
|
|
61
|
+
return m ? m[1] : "";
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 把任意 thrown 值展开成诊断对象。
|
|
66
|
+
* @param {unknown} err
|
|
67
|
+
* @returns {{ name: string, code: string, message: string, addr: string, syscall: string, summary: string }}
|
|
68
|
+
*/
|
|
69
|
+
export function describeError(err) {
|
|
70
|
+
const chain = [];
|
|
71
|
+
let node = err;
|
|
72
|
+
for (let depth = 0; node && depth < 5; depth++) {
|
|
73
|
+
chain.push(node);
|
|
74
|
+
node = node.cause;
|
|
75
|
+
}
|
|
76
|
+
const first = chain[0] || {};
|
|
77
|
+
const name = String(first.name || "Error");
|
|
78
|
+
|
|
79
|
+
// 选定「有信息量」的那层:第一层带已知 code 的层;都没有就用最深的非空 code,再退回第一层。
|
|
80
|
+
let chosen = null;
|
|
81
|
+
for (const n of chain) {
|
|
82
|
+
const c = pickCode(n);
|
|
83
|
+
if (c && KNOWN_CODES.has(c)) { chosen = { node: n, code: c }; break; }
|
|
84
|
+
}
|
|
85
|
+
if (!chosen) {
|
|
86
|
+
for (let i = chain.length - 1; i >= 0; i--) {
|
|
87
|
+
const c = pickCode(chain[i]);
|
|
88
|
+
if (c) { chosen = { node: chain[i], code: c }; break; }
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const code = chosen?.code || "";
|
|
92
|
+
const addr = pickAddr(chosen?.node || first);
|
|
93
|
+
const syscall = String(chosen?.node?.syscall || first.syscall || "");
|
|
94
|
+
|
|
95
|
+
// message 取「最深一条非空 message」—— 那通常是最贴近 syscall 的一句;都没有就用第一层。
|
|
96
|
+
let message = "";
|
|
97
|
+
for (let i = chain.length - 1; i >= 0; i--) {
|
|
98
|
+
const m = String(chain[i]?.message || "").trim();
|
|
99
|
+
if (m) { message = m; break; }
|
|
100
|
+
}
|
|
101
|
+
if (!message) message = String(first.message || err || "");
|
|
102
|
+
|
|
103
|
+
const zh = CODE_ZH[code] || "";
|
|
104
|
+
const parts = [];
|
|
105
|
+
parts.push(code ? `${code}${zh ? `(${zh})` : ""}` : name);
|
|
106
|
+
if (addr) parts.push(`peer=${addr}`);
|
|
107
|
+
if (syscall) parts.push(`syscall=${syscall}`);
|
|
108
|
+
const head = parts.join(" ");
|
|
109
|
+
// message 已含 code 时不重复(如 "connect ECONNREFUSED …" 与 code=ECONNREFUSED);
|
|
110
|
+
// 注意 code 为空时不能建空正则——空 pattern 匹配一切,会把 message 误判成冗余。
|
|
111
|
+
const redundant = code && new RegExp(escapeRegExp(code)).test(message);
|
|
112
|
+
const summary = message && message !== code && !redundant
|
|
113
|
+
? `${head} — ${message}`
|
|
114
|
+
: head;
|
|
115
|
+
return { name, code, message, addr, syscall, summary };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** 一句话诊断,供日志/回包直接用。 */
|
|
119
|
+
export function describeErrorText(err) {
|
|
120
|
+
return describeError(err).summary;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function escapeRegExp(s) {
|
|
124
|
+
return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
125
|
+
}
|
|
@@ -161,6 +161,9 @@ export async function forwardHandler({ req, res, groups, bus, logs }) {
|
|
|
161
161
|
if (!targetMember) return json(res, 404, { error: `target ${target} not found in group ${groupName}` });
|
|
162
162
|
const isBb = targetMember.kind === "broadband" || String(targetMember.url).startsWith("relay://");
|
|
163
163
|
if (!isBb) {
|
|
164
|
+
// 标记走直连:**在 fetch 之前**就设,这样直连失败(catch 里的 502)也带着它——
|
|
165
|
+
// 那正是最需要区分的一跳(组长直连该成员不通 vs 宽带成员回报失败,两者的 body 可能一模一样)。
|
|
166
|
+
res.setHeader("x-mslxdff-relay-via", "direct");
|
|
164
167
|
try {
|
|
165
168
|
const fwdBody = body.body || body;
|
|
166
169
|
const r = await compatFetch(`${targetMember.url}/v1/chat/completions`, {
|
|
@@ -193,13 +196,18 @@ export async function forwardHandler({ req, res, groups, bus, logs }) {
|
|
|
193
196
|
logs?.appendEvent?.(evtData);
|
|
194
197
|
const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
|
|
195
198
|
if (typeof targetMember.lastSeen === "number" && Date.now() - targetMember.lastSeen > staleMs) {
|
|
199
|
+
res.setHeader("x-mslxdff-relay-via", "broadband");
|
|
196
200
|
return json(res, 502, { error: "broadband member stale (no heartbeat)" });
|
|
197
201
|
}
|
|
202
|
+
// 标记走的是宽带中继:下游据此一眼分清「组长直连失败(502 fetch failed)」vs「宽带成员回报的失败」,
|
|
203
|
+
// 两者响应体可能都是 {"error":"fetch failed"},只有这个头能区分(成员侧已把真实原因写进 errors.log)。
|
|
204
|
+
res.setHeader("x-mslxdff-relay-via", "broadband");
|
|
198
205
|
try {
|
|
199
206
|
const resultPromise = enqueueRelay({ group: groupName, target: targetMember.url, reqId, body: fwdBody, hops });
|
|
200
207
|
const result = await resultPromise;
|
|
201
208
|
if (result && typeof result.status === "number") {
|
|
202
209
|
res.statusCode = result.status;
|
|
210
|
+
if (result.status >= 500) res.setHeader("x-mslxdff-relay-member-status", String(result.status));
|
|
203
211
|
if (result.headers) for (const [k, v] of Object.entries(result.headers)) res.setHeader(k, v);
|
|
204
212
|
if (result.body) {
|
|
205
213
|
if (typeof result.body === "string") res.end(result.body);
|
package/src/runtime/bootstrap.js
CHANGED
|
@@ -35,7 +35,7 @@ export async function startDaemonMain(VERSION) {
|
|
|
35
35
|
const { srv, bus } = await startServerLifecycle({ VERSION, ...ctx, logs });
|
|
36
36
|
void srv;
|
|
37
37
|
startGroupSync({ peers: ctx.peers, groups: ctx.groups, logs });
|
|
38
|
-
startBroadband({ token: ctx.token, upstream: ctx.upstream });
|
|
38
|
+
startBroadband({ token: ctx.token, upstream: ctx.upstream, logs });
|
|
39
39
|
|
|
40
40
|
// 保热心跳:每 ≤120s 经 **chat 同池**访问每个组员(/health) + 每个 picks 供应商(preheat),
|
|
41
41
|
// 防「久未使用 → 首发 fetch failed / 上游冷启动」(现网 2026-10-18 16:16 三组员同时 fetch failed)。
|
|
@@ -5,7 +5,7 @@ import { compatFetch } from "../compat.js";
|
|
|
5
5
|
* 宽带 SSE 长连 — 单组成员订阅 + 指数退避重连 + ensure 定时器。
|
|
6
6
|
* execAndPost / broadbandGroups 由 broadband.js 注入。
|
|
7
7
|
*/
|
|
8
|
-
export function startBroadbandStream({ token, upstream, execAndPost, broadbandGroups }) {
|
|
8
|
+
export function startBroadbandStream({ token, upstream, execAndPost, broadbandGroups, log = console.log }) {
|
|
9
9
|
const streamManagers = new Map();
|
|
10
10
|
const startStreamForGroup = (g) => {
|
|
11
11
|
if (streamManagers.has(g.name)) return;
|
|
@@ -25,7 +25,7 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
25
25
|
if (!res.ok) throw new Error(`stream ${res.status}`);
|
|
26
26
|
if (!res.body) throw new Error("no body");
|
|
27
27
|
attempts = 0;
|
|
28
|
-
|
|
28
|
+
log(`broadband relay: connected ${g.name} via ${g.leaderUrl} (SSE)`);
|
|
29
29
|
let buf = "";
|
|
30
30
|
const decodeChunk = (c) => {
|
|
31
31
|
if (typeof c === "string") return c;
|
|
@@ -50,8 +50,14 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
50
50
|
try {
|
|
51
51
|
const parsed = JSON.parse(data);
|
|
52
52
|
const { reqId, body } = parsed;
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
// 不再 `.catch(()=>{})` 静默吞:execAndPost 内部已把执行/回传失败写 errors.log,
|
|
54
|
+
// 这里只兜底一句,防它真抛出(如 body 畸变)时整条 SSE 循环被吞掉无痕。
|
|
55
|
+
if (reqId && body) {
|
|
56
|
+
execAndPost(g, reqId, body).catch((e) => log(`broadband exec ${g.name} req=${reqId}: failed — ${errMsg(e)}`));
|
|
57
|
+
}
|
|
58
|
+
} catch (e) {
|
|
59
|
+
log(`broadband stream ${g.name}: bad relay frame — ${errMsg(e)}`);
|
|
60
|
+
}
|
|
55
61
|
}
|
|
56
62
|
}
|
|
57
63
|
}
|
|
@@ -59,7 +65,7 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
59
65
|
} catch (err) {
|
|
60
66
|
if (stopped) return;
|
|
61
67
|
const msg = errMsg(err);
|
|
62
|
-
if (!String(msg).includes("abort") && !String(msg).includes("Abort"))
|
|
68
|
+
if (!String(msg).includes("abort") && !String(msg).includes("Abort")) log(`broadband stream ${g.name}: ${msg} — reconnecting`);
|
|
63
69
|
attempts++;
|
|
64
70
|
const delay = Math.min(30_000, 1000 * Math.pow(2, attempts - 1) + Math.random() * 500);
|
|
65
71
|
await new Promise((r) => setTimeout(r, delay));
|
|
@@ -75,5 +81,5 @@ export function startBroadbandStream({ token, upstream, execAndPost, broadbandGr
|
|
|
75
81
|
}, 10_000);
|
|
76
82
|
ensureTimer.unref();
|
|
77
83
|
const n = broadbandGroups().length;
|
|
78
|
-
|
|
84
|
+
log(n ? `broadband relay: stream (SSE) + ping 25s for ${n} group(s)` : "broadband relay: idle (0 group) — waiting for a group join, checked every 10s");
|
|
79
85
|
}
|
package/src/runtime/broadband.js
CHANGED
|
@@ -1,26 +1,30 @@
|
|
|
1
1
|
import { loadGroupsJoined } from "../state.js";
|
|
2
2
|
import { errMsg } from "../cli/util.js";
|
|
3
|
+
import { describeErrorText } from "../net-diag.js";
|
|
3
4
|
import { withOutputFloor } from "../chat-pipeline/empty-turn.js";
|
|
4
5
|
import { startBroadbandStream } from "./broadband-stream.js";
|
|
5
6
|
import { compatFetch, timeoutSignal } from "../compat.js";
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* 造一个「成员侧执行 + 回传结果」的执行器(可注入依赖,便于单测)。
|
|
10
|
+
*
|
|
11
|
+
* 为什么要单独抽出来并**带日志**:成员 B 用 upstream.chat 打自己的上游失败时,原实现只造了个
|
|
12
|
+
* `{status:502, body:{"error":"fetch failed"}}` 回传组长,**本机一行日志都没留** —— 于是组长侧
|
|
13
|
+
* 看到 `502 fetch failed`、成员侧看到空日志,全链路都没人能说出 B 为什么打不到上游。
|
|
14
|
+
* 现在:执行抛错记 errors.log(带 cause 链诊断),上游返回 4xx/5xx 也记,回传失败同样记。
|
|
15
|
+
*
|
|
16
|
+
* @param {{ token: string, upstream: {chat: Function}, logs?: {appendError?: Function, appendEvent?: Function}, log?: Function, fetchImpl?: Function }} opts
|
|
17
|
+
* @returns {(g: {name: string, leaderUrl: string}, reqId: string, body: object) => Promise<void>}
|
|
12
18
|
*/
|
|
13
|
-
export function
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
if (v === "0" || v === "false" || v === "off") return false;
|
|
18
|
-
return true;
|
|
19
|
-
})();
|
|
20
|
-
const execAndPost = async (g, reqId, body) => {
|
|
19
|
+
export function createBroadbandExec({ token, upstream, logs, log = console.log, fetchImpl = compatFetch } = {}) {
|
|
20
|
+
const appendError = (entry) => { try { logs?.appendError?.(entry); } catch { /* 日志失败不拖垮执行 */ } };
|
|
21
|
+
const appendEvent = (entry) => { try { logs?.appendEvent?.(entry); } catch {} };
|
|
22
|
+
return async function execAndPost(g, reqId, body) {
|
|
21
23
|
let result;
|
|
24
|
+
const t0 = Date.now();
|
|
25
|
+
const model = String(body?.model || "");
|
|
22
26
|
const fl = withOutputFloor(body); // 首发输出额度托底:leader 转来的 body 不经 pipeline,成员侧自托(评审 P1)
|
|
23
|
-
if (fl !== body)
|
|
27
|
+
if (fl !== body) log(`output-floor(broadband): req=${reqId} ${(body?.max_tokens ?? body?.max_completion_tokens)}→${(fl.max_tokens ?? fl.max_completion_tokens)}`);
|
|
24
28
|
try {
|
|
25
29
|
const upRes = await upstream.chat(fl);
|
|
26
30
|
const ct = upRes.headers.get("content-type") || "";
|
|
@@ -40,33 +44,69 @@ export function startBroadband({ token, upstream }) {
|
|
|
40
44
|
try { parsed = JSON.parse(txt); } catch { parsed = txt; }
|
|
41
45
|
result = { status: upRes.status, headers: { "Content-Type": upRes.headers.get("content-type") || "application/json" }, body: typeof parsed === "string" ? parsed : JSON.stringify(parsed) };
|
|
42
46
|
}
|
|
47
|
+
appendEvent({ ts: Date.now(), type: "broadband-task", reqId, group: g.name, model, status: result.status, ms: Date.now() - t0, chars: String(result.body || "").length });
|
|
48
|
+
if (result.status >= 400) {
|
|
49
|
+
// 成员成功连上上游、但上游回错 —— 也要留痕,否则组长只见状态码不知原因
|
|
50
|
+
const snippet = String(result.body || "").slice(0, 300);
|
|
51
|
+
log(`broadband task: ${g.name} req=${reqId} 成员上游返回 HTTP ${result.status} — ${snippet}`);
|
|
52
|
+
appendError({ api: "broadband", reqId, group: g.name, model, status: result.status, message: `组员执行上游返回 HTTP ${result.status}:${snippet}` });
|
|
53
|
+
}
|
|
43
54
|
} catch (err) {
|
|
44
|
-
|
|
55
|
+
// 成员打自己上游**抛错**(连不上/DNS/超时/TLS)—— 这是 `502 fetch failed` 的真正来源,展开 cause 链
|
|
56
|
+
const diag = describeErrorText(err);
|
|
57
|
+
result = { status: 502, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: diag }) };
|
|
58
|
+
log(`broadband task: ${g.name} req=${reqId} 成员打上游失败 — ${diag}`);
|
|
59
|
+
appendError({ api: "broadband", reqId, group: g.name, model, status: 502, message: `组员打上游请求抛错(成员侧失败,非组长):${diag}` });
|
|
45
60
|
}
|
|
46
61
|
try {
|
|
47
|
-
await
|
|
62
|
+
const r = await fetchImpl(`${g.leaderUrl}/v1/groups/relay/result`, {
|
|
48
63
|
method: "POST",
|
|
49
|
-
headers: { "Content-Type": "application/json",
|
|
64
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
50
65
|
body: JSON.stringify({ name: g.name, group: g.name, reqId, result }),
|
|
51
66
|
signal: timeoutSignal(5000),
|
|
52
67
|
});
|
|
53
|
-
|
|
68
|
+
if (!r.ok) {
|
|
69
|
+
const txt = await r.text().catch(() => "");
|
|
70
|
+
log(`broadband result post: ${g.name} req=${reqId} HTTP ${r.status} ${txt.slice(0, 120)}`);
|
|
71
|
+
appendError({ api: "broadband", reqId, group: g.name, status: r.status, message: `结果回传组长失败 HTTP ${r.status}:${txt.slice(0, 200)}` });
|
|
72
|
+
}
|
|
73
|
+
} catch (err) {
|
|
74
|
+
const diag = describeErrorText(err);
|
|
75
|
+
log(`broadband result post: ${g.name} req=${reqId} failed — ${diag}`);
|
|
76
|
+
appendError({ api: "broadband", reqId, group: g.name, status: 0, message: `结果回传组长请求抛错:${diag}` });
|
|
77
|
+
}
|
|
54
78
|
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* 宽带中继 — poll 模式(heartbeat + poll)与 stream 模式分发。
|
|
83
|
+
* stream 长连下沉 broadband-stream.js,本文件保留任务执行与降级分支。
|
|
84
|
+
* 空组不再早退:加组后由 ensure(stream 10s / poll 每次读组)自动接上,无需重启 daemon。
|
|
85
|
+
* // Note: 空组不早退与向导默认 broadband 的取舍 — 见 .agents/notes/implemented/feature/2026-09-16-mobile-broadband-join-wizard.md
|
|
86
|
+
*/
|
|
87
|
+
export function startBroadband({ token, upstream, logs, log = console.log }) {
|
|
88
|
+
const broadbandGroups = () => loadGroupsJoined().filter((g) => g.kind === "broadband" && g.leaderUrl);
|
|
89
|
+
const streamEnabled = (() => {
|
|
90
|
+
const v = process.env.MSLXDFF_BROADBAND_STREAM;
|
|
91
|
+
if (v === "0" || v === "false" || v === "off") return false;
|
|
92
|
+
return true;
|
|
93
|
+
})();
|
|
94
|
+
const execAndPost = createBroadbandExec({ token, upstream, logs, log });
|
|
55
95
|
const doHeartbeat = async () => {
|
|
56
96
|
for (const g of broadbandGroups()) {
|
|
57
97
|
try {
|
|
58
98
|
const res = await compatFetch(`${g.leaderUrl}/v1/groups/relay/heartbeat`, {
|
|
59
99
|
method: "POST",
|
|
60
|
-
headers: { "Content-Type": "application/json",
|
|
100
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
61
101
|
body: JSON.stringify({ name: g.name, group: g.name }),
|
|
62
102
|
signal: timeoutSignal(5000),
|
|
63
103
|
});
|
|
64
104
|
if (!res.ok) {
|
|
65
105
|
const txt = await res.text().catch(() => "");
|
|
66
|
-
|
|
106
|
+
log(`broadband heartbeat ${g.name}: ${res.status} ${txt.slice(0, 100)}`);
|
|
67
107
|
}
|
|
68
108
|
} catch (err) {
|
|
69
|
-
|
|
109
|
+
log(`broadband heartbeat ${g.name}: failed — ${errMsg(err)}`);
|
|
70
110
|
}
|
|
71
111
|
}
|
|
72
112
|
};
|
|
@@ -86,7 +126,9 @@ export function startBroadband({ token, upstream }) {
|
|
|
86
126
|
const { reqId, body } = item;
|
|
87
127
|
await execAndPost(g, reqId, body);
|
|
88
128
|
}
|
|
89
|
-
} catch {
|
|
129
|
+
} catch (err) {
|
|
130
|
+
log(`broadband poll ${g.name}: failed — ${describeErrorText(err)}`);
|
|
131
|
+
}
|
|
90
132
|
}
|
|
91
133
|
};
|
|
92
134
|
if (!streamEnabled) {
|
|
@@ -96,8 +138,8 @@ export function startBroadband({ token, upstream }) {
|
|
|
96
138
|
const pollTimer = setInterval(doPoll, 1000);
|
|
97
139
|
pollTimer.unref();
|
|
98
140
|
const n = broadbandGroups().length;
|
|
99
|
-
|
|
141
|
+
log(n ? `broadband relay: heartbeat 30s + poll 1s for ${n} group(s) [poll mode]` : "broadband relay: idle (0 group) — waiting for a group join [poll mode]");
|
|
100
142
|
return;
|
|
101
143
|
}
|
|
102
|
-
startBroadbandStream({ token, upstream, execAndPost, broadbandGroups });
|
|
144
|
+
startBroadbandStream({ token, upstream, execAndPost, broadbandGroups, log });
|
|
103
145
|
}
|
package/src/runtime/proc-info.js
CHANGED
|
@@ -92,6 +92,17 @@ export function isDaemonArgv(argv) {
|
|
|
92
92
|
return list.includes("--daemon") && list.some((a) => /(^|[\\/])mslxdff(\.js)?$/i.test(String(a)));
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
/**
|
|
96
|
+
* 是否跑在 systemd 托管下(system 或 user unit 都算)。
|
|
97
|
+
* 判据:systemd 给被管服务注入 `INVOCATION_ID`(每个 invocation 唯一)与 `JOURNAL_STREAM`(stdout 连接信息)。
|
|
98
|
+
* 为什么要问:托管时 supervisor 已保证单实例与重启策略,daemon 再自己「杀旧进程/杀端口占用者」是职责重叠,
|
|
99
|
+
* 会与 systemd 的拉起时序打架成死循环(真机 2026-10-19:0.1.209 + 系统 unit 每 2.5s 自杀一轮)。
|
|
100
|
+
* @returns {boolean}
|
|
101
|
+
*/
|
|
102
|
+
export function underSystemd(env = process.env) {
|
|
103
|
+
return Boolean(env.INVOCATION_ID || env.JOURNAL_STREAM);
|
|
104
|
+
}
|
|
105
|
+
|
|
95
106
|
/** 「已运行 6h12m」这种人话时长(取证日志用)。 */
|
|
96
107
|
export function fmtAlive(sec) {
|
|
97
108
|
if (!Number.isFinite(sec)) return "运行时长未知";
|
|
@@ -5,6 +5,7 @@ import { runHook } from "../plugins.js";
|
|
|
5
5
|
import { effectiveHost, maxHopsValue } from "../cli/policy.js";
|
|
6
6
|
import { fmtEvent } from "../cli/format.js";
|
|
7
7
|
import { writePid } from "../daemon.js";
|
|
8
|
+
import { underSystemd } from "./proc-info.js";
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* 服务生命周期 — router 装配 → server 创建/ready-EADDRINUSE 自愈 →
|
|
@@ -66,6 +67,13 @@ export async function startServerLifecycle({ VERSION, token, created, upstream,
|
|
|
66
67
|
const msg = String(err?.message || err);
|
|
67
68
|
const code = err?.code || "";
|
|
68
69
|
if (code === "EADDRINUSE" || msg.includes("EADDRINUSE")) {
|
|
70
|
+
// systemd 托管下不许杀端口占用者:supervisor 保证单实例,去杀「占用者」是越权,且会与
|
|
71
|
+
// systemd 的拉起时序互杀成循环(报告的另一半)。非零退出,让 systemd 按 Restart/RestartSec
|
|
72
|
+
// 退避重试——这才是托管语义下正确的收口。
|
|
73
|
+
if (underSystemd()) {
|
|
74
|
+
console.error(`[lifecycle] EADDRINUSE on port ${resolvePort()} under systemd — exit(1), let systemd backoff-retry(不杀任何进程)`);
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
69
77
|
console.log(`[lifecycle] EADDRINUSE on port ${resolvePort()} — freeing stale holder and retrying...`);
|
|
70
78
|
try {
|
|
71
79
|
const { execFile } = await import("node:child_process");
|
|
@@ -166,7 +174,13 @@ export async function startServerLifecycle({ VERSION, token, created, upstream,
|
|
|
166
174
|
} catch {}
|
|
167
175
|
|
|
168
176
|
// best-effort: ensure autostart on Linux (so daemon survives reboot/SSH disconnect without manual cmd)
|
|
169
|
-
|
|
177
|
+
// **systemd 托管下必须跳过**(2026-10-19 真机无限重启的根因):
|
|
178
|
+
// - 托管时 supervisor 已经保证单实例+开机自启,这里再 enable 一次多余;
|
|
179
|
+
// - 本块查的是 **user** unit(`~/.config/systemd/user/mslxdff.service`),而 systemd **系统**
|
|
180
|
+
// unit 托管的 daemon 查它必然得 enabled=false ⇒ 走 enableAutostart() ⇒ linuxEnable() ⇒ stopDaemon()。
|
|
181
|
+
// 那条路径里 pid 文件写的就是**本进程**,过去等于每 2.5s 自杀一次(见 daemon.js 的自杀护栏)。
|
|
182
|
+
// systemd 会给被管进程注入 INVOCATION_ID / JOURNAL_STREAM,判据可靠、无需用户配置。
|
|
183
|
+
if (process.platform === "linux" && !process.env.MSLXDFF_NO_AUTOSTART && !underSystemd()) {
|
|
170
184
|
setTimeout(async () => {
|
|
171
185
|
try {
|
|
172
186
|
const { getAutostartStatus, enableAutostart } = await import("../autostart.js");
|