mslxdff 0.1.179 → 0.1.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/warp-pool/health.js +50 -9
- package/src/warp-pool/index.js +74 -30
- package/src/warp-pool/ports.js +45 -7
- package/src/warp-pool/report.js +149 -0
- package/src/warp-pool/rotate.js +1 -1
- package/src/warp-pool/wireproxy.js +11 -6
package/package.json
CHANGED
package/src/warp-pool/health.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// warp-pool · 节点健康:主动探活(经隧道打上游 models,不烧 token)+ 失败冷却 + 连续失败触发重拨换 IP。
|
|
2
2
|
import { probeTunnel } from "./wireproxy.js";
|
|
3
|
+
import { fmtHealth, fmtHeartbeat, dur, MARK } from "./report.js";
|
|
3
4
|
|
|
4
5
|
export function createHealthMonitor({
|
|
5
6
|
dispatcher,
|
|
@@ -7,41 +8,81 @@ export function createHealthMonitor({
|
|
|
7
8
|
probePath = "/zen/v1/models",
|
|
8
9
|
unhealthyAfter = 3,
|
|
9
10
|
intervalMs = 60_000,
|
|
11
|
+
heartbeatEvery = 10,
|
|
12
|
+
infoOf = null,
|
|
13
|
+
cooldownMs = 600_000,
|
|
10
14
|
onExhausted = null,
|
|
11
15
|
probe = probeTunnel,
|
|
12
16
|
headers = {},
|
|
17
|
+
now = Date.now,
|
|
13
18
|
log = () => {},
|
|
14
19
|
} = {}) {
|
|
15
20
|
let timer = null;
|
|
16
21
|
let rounds = 0;
|
|
22
|
+
let ticks = 0;
|
|
23
|
+
let lastAllDownAt = "";
|
|
24
|
+
/** @type {Map<number, boolean>} 每端口上一轮通不通(用来只在「状态变化」时打日志) */
|
|
25
|
+
const lastOk = new Map();
|
|
26
|
+
const stamp = () => new Date(now()).toISOString().slice(11, 19);
|
|
17
27
|
|
|
18
|
-
/**
|
|
28
|
+
/**
|
|
29
|
+
* 跑一轮全池探活。日志纪律:**只在「状态有变化」「全池不通」「心跳点」时打**——每分钟一行会把日志淹掉。
|
|
30
|
+
* 每个不通的节点都要报全:HTTP 状态、错误原文、耗时、连续第几次、冷却多久(这样才不用靠猜)。
|
|
31
|
+
*/
|
|
19
32
|
async function tick() {
|
|
20
33
|
if (!dispatcher) return { checked: 0, healthy: 0 };
|
|
21
34
|
const list = dispatcher.entries();
|
|
22
35
|
let healthy = 0;
|
|
36
|
+
let slowestMs = 0;
|
|
37
|
+
const problems = [];
|
|
38
|
+
const changes = [];
|
|
23
39
|
for (const e of list) {
|
|
24
40
|
const r = await probe({ port: e.port, baseUrl, path: probePath, headers }).catch((err) => ({
|
|
25
41
|
ok: false,
|
|
26
42
|
error: String(err?.message || err),
|
|
27
43
|
}));
|
|
44
|
+
if (Number(r.ms) > slowestMs) slowestMs = Number(r.ms);
|
|
28
45
|
if (r.ok) {
|
|
29
46
|
dispatcher.markOk(e.index);
|
|
30
47
|
healthy += 1;
|
|
31
|
-
|
|
32
|
-
|
|
48
|
+
if (lastOk.get(e.port) === false) changes.push(`:${e.port} 恢复(HTTP ${r.status},${dur(r.ms)})`);
|
|
49
|
+
lastOk.set(e.port, true);
|
|
50
|
+
continue;
|
|
33
51
|
}
|
|
52
|
+
dispatcher.markFailure(e.index);
|
|
53
|
+
if (lastOk.get(e.port) !== false) {
|
|
54
|
+
changes.push(`:${e.port} 掉线(HTTP ${r.status || 0}${r.error ? ` ${String(r.error).slice(0, 80)}` : ""})`);
|
|
55
|
+
}
|
|
56
|
+
lastOk.set(e.port, false);
|
|
57
|
+
problems.push({ port: e.port, status: r.status, error: r.error, ms: r.ms, fails: Number(e.fails || 0) + 1, cooldownMs });
|
|
34
58
|
}
|
|
35
|
-
|
|
59
|
+
const allDown = list.length > 0 && healthy === 0;
|
|
60
|
+
ticks += 1;
|
|
61
|
+
if (allDown) {
|
|
36
62
|
rounds += 1;
|
|
37
|
-
|
|
38
|
-
if (rounds >= 2 && typeof onExhausted === "function") {
|
|
39
|
-
rounds = 0;
|
|
40
|
-
await onExhausted().catch((e) => log(`[warp-pool] 重拨失败:${String(e?.message || e)}`));
|
|
41
|
-
}
|
|
63
|
+
lastAllDownAt = stamp();
|
|
42
64
|
} else {
|
|
43
65
|
rounds = 0;
|
|
44
66
|
}
|
|
67
|
+
if (changes.length || allDown || ticks % heartbeatEvery === 0) {
|
|
68
|
+
log(fmtHealth({ healthy, total: list.length, slowestMs, problems }));
|
|
69
|
+
}
|
|
70
|
+
for (const c of changes) log(`[warp-pool] ${MARK.warn} 状态变化:${c}`);
|
|
71
|
+
if (ticks % heartbeatEvery === 0) {
|
|
72
|
+
const extra = typeof infoOf === "function" ? infoOf() || {} : {};
|
|
73
|
+
log(fmtHeartbeat({
|
|
74
|
+
alive: extra.alive ?? list.length,
|
|
75
|
+
total: list.length,
|
|
76
|
+
pooled: extra.pooled ?? list.length,
|
|
77
|
+
exits: extra.exits || [],
|
|
78
|
+
cooling: list.filter((e) => Number(e.cooldownUntil) > now()).length,
|
|
79
|
+
lastAllDown: lastAllDownAt,
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
if (allDown && rounds >= 2 && typeof onExhausted === "function") {
|
|
83
|
+
rounds = 0;
|
|
84
|
+
await onExhausted().catch((e) => log(`[warp-pool] 重拨失败:${String(e?.message || e)}`));
|
|
85
|
+
}
|
|
45
86
|
return { checked: list.length, healthy };
|
|
46
87
|
}
|
|
47
88
|
|
package/src/warp-pool/index.js
CHANGED
|
@@ -8,7 +8,8 @@ import { ensureDir, loadNodes, removeNode, saveNode } from "./nodes.js";
|
|
|
8
8
|
import { registerWarpDevice } from "./register.js";
|
|
9
9
|
import { ensureBinary, probeTunnel, resolveAsset, startTunnel, writeTunnelConf } from "./wireproxy.js";
|
|
10
10
|
import { createReviver, labelExits, tailReason } from "./revive.js";
|
|
11
|
-
import { portInUse, reclaimPort } from "./ports.js";
|
|
11
|
+
import { portInUse, portOwner, reclaimPort, waitPortFree } from "./ports.js";
|
|
12
|
+
import { fmtConf, fmtExit, fmtPool, fmtPoolEmpty, fmtProbe, fmtProbeGiveUp, fmtPortBlocked, fmtSpawn, startupReport } from "./report.js";
|
|
12
13
|
import { wireRotation } from "./identity.js";
|
|
13
14
|
|
|
14
15
|
/** 重拨节流:连续失败指数退避。网络不通的机器绝不能让重注册把 CF 刷到 429。 */
|
|
@@ -38,6 +39,9 @@ export function createWarpPool({
|
|
|
38
39
|
ensureBin = ensureBinary,
|
|
39
40
|
start = startTunnel,
|
|
40
41
|
probe = probeTunnel,
|
|
42
|
+
reclaim = reclaimPort,
|
|
43
|
+
inUse = portInUse,
|
|
44
|
+
waitFree = waitPortFree,
|
|
41
45
|
now = Date.now,
|
|
42
46
|
rnd = Math.random,
|
|
43
47
|
wait = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
@@ -75,7 +79,12 @@ export function createWarpPool({
|
|
|
75
79
|
let redialGapMs = REDIAL_MIN_GAP_MS;
|
|
76
80
|
|
|
77
81
|
// 兜底清理:Node 退出不会回收 spawn 的子进程,daemon 突然退出时同步杀掉隧道。
|
|
82
|
+
// 同时置 closed:否则「杀子进程 → onExit → 立刻复活」会在退出过程中再 spawn 出一批孤儿(真机:每次重启漏一批)。
|
|
78
83
|
process.once("exit", () => {
|
|
84
|
+
closed = true;
|
|
85
|
+
try {
|
|
86
|
+
reviver.stop();
|
|
87
|
+
} catch {}
|
|
79
88
|
for (const t of tunnels.values()) {
|
|
80
89
|
try {
|
|
81
90
|
t.stop();
|
|
@@ -83,28 +92,30 @@ export function createWarpPool({
|
|
|
83
92
|
}
|
|
84
93
|
});
|
|
85
94
|
|
|
86
|
-
/**
|
|
95
|
+
/** 起一条隧道的完整动作:确认不是重复起 → 清残留 → 写 conf → 等端口真空出来 → spawn → 记表。 */
|
|
87
96
|
async function bringUp(node) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
97
|
+
if (tunnels.has(node.port)) return; // 已归我们管:绝不再清残留(reclaim 会把自己这条隧道当残留杀掉)
|
|
98
|
+
await reclaim({ dir: cfg.dir, port: node.port, log }); // 清上一代孤儿(daemon 被强杀时子进程不会自己退)
|
|
99
|
+
const { conf, bytes } = writeTunnelConf(cfg.dir, node);
|
|
100
|
+
log(fmtConf({ port: node.port, conf, bytes }));
|
|
101
|
+
// SIGTERM 之后套接字要几百毫秒才释放,这期间连得上 ≠ 别人的进程占着——先等它真空出来再判
|
|
102
|
+
if (!(await waitFree(node.port, { inUse }))) {
|
|
103
|
+
const owner = await portOwner({ dir: cfg.dir, port: node.port }).catch(() => ({ ours: [], others: [] }));
|
|
104
|
+
throw new Error(fmtPortBlocked({ port: node.port, ...owner }));
|
|
93
105
|
}
|
|
94
106
|
const t = start({
|
|
95
107
|
binPath,
|
|
96
108
|
dir: cfg.dir,
|
|
97
109
|
port: node.port,
|
|
98
110
|
log,
|
|
99
|
-
onExit: ({ port, code, signal, tail }) => {
|
|
111
|
+
onExit: ({ port, code, signal, tail, pid, aliveMs }) => {
|
|
100
112
|
tunnels.delete(port);
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
void reviver.onExit(port);
|
|
113
|
+
log(fmtExit({ port, code, signal, pid, aliveMs, tail: tailReason(tail) }));
|
|
114
|
+
if (!closed) void reviver.onExit(port); // 收摊中就别复活了,否则退出过程里又 spawn 出一批孤儿
|
|
104
115
|
},
|
|
105
116
|
});
|
|
106
117
|
tunnels.set(node.port, t);
|
|
107
|
-
log(
|
|
118
|
+
log(fmtSpawn({ port: node.port, pid: t.pid, binPath, conf }));
|
|
108
119
|
}
|
|
109
120
|
|
|
110
121
|
// 429 = 上游按出口 IP 计的名额耗尽 ⇒ 换一台设备的身份换出口(换身份=换桶)。
|
|
@@ -117,6 +128,38 @@ export function createWarpPool({
|
|
|
117
128
|
log,
|
|
118
129
|
});
|
|
119
130
|
|
|
131
|
+
/** 探活闸门参数:刚起的隧道要等 WireGuard 握手完成才通(真机:一起来就探 ⇒ 4 条全没通 ⇒ 池为空、WARP 白开)。 */
|
|
132
|
+
const PROBE_SETTLE_MS = 2_000;
|
|
133
|
+
const PROBE_ATTEMPTS = 2;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* 入池前的实测闸门:给握手留时间 → 逐个探活 → 探不通的绝不进池。
|
|
137
|
+
* 为什么必须有:wireproxy 进程活着但隧道转不动流量时,请求会挂到超时(比直连更糟)。
|
|
138
|
+
* @returns {Promise<Array<object>>} 通过探活的节点
|
|
139
|
+
*/
|
|
140
|
+
async function probeGate(list) {
|
|
141
|
+
const live = [];
|
|
142
|
+
await wait(PROBE_SETTLE_MS);
|
|
143
|
+
for (const n of list) {
|
|
144
|
+
if (!tunnels.has(n.port)) continue;
|
|
145
|
+
const url = `${String(baseUrl).replace(/\/+$/, "")}${cfg.probePath}`;
|
|
146
|
+
let r = { ok: false };
|
|
147
|
+
for (let i = 1; i <= PROBE_ATTEMPTS; i++) {
|
|
148
|
+
const t0 = now();
|
|
149
|
+
r = await probe({ port: n.port, baseUrl, path: cfg.probePath });
|
|
150
|
+
const ms = Number.isFinite(r.ms) ? r.ms : now() - t0;
|
|
151
|
+
log(fmtProbe({ port: n.port, attempt: i, attempts: PROBE_ATTEMPTS, url, ok: r.ok, status: r.status, error: r.error, ms }));
|
|
152
|
+
if (r.ok) break;
|
|
153
|
+
if (i < PROBE_ATTEMPTS) await wait(PROBE_SETTLE_MS);
|
|
154
|
+
}
|
|
155
|
+
if (r.ok) {
|
|
156
|
+
live.push(n);
|
|
157
|
+
} else {
|
|
158
|
+
log(fmtProbeGiveUp({ port: n.port, attempts: PROBE_ATTEMPTS, url }));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return live;
|
|
162
|
+
}
|
|
120
163
|
async function bootstrap() {
|
|
121
164
|
if (closed) return;
|
|
122
165
|
ensureDir(cfg.dir);
|
|
@@ -133,7 +176,7 @@ export function createWarpPool({
|
|
|
133
176
|
|
|
134
177
|
// 1. 复用磁盘上已注册的节点
|
|
135
178
|
let nodes = loadNodes(cfg.dir, cfg.ports);
|
|
136
|
-
|
|
179
|
+
await startupReport({ cfg, nodes, binPath, ownerOf: (port) => portOwner({ dir: cfg.dir, port }), log });
|
|
137
180
|
|
|
138
181
|
// 2. 补足:逐个注册(拉长间隔,避免 CF 注册限流与同网段撞车)
|
|
139
182
|
const used = new Set(nodes.map((n) => n.port));
|
|
@@ -162,23 +205,17 @@ export function createWarpPool({
|
|
|
162
205
|
}
|
|
163
206
|
}
|
|
164
207
|
|
|
165
|
-
// 入池前实测探活:WARP
|
|
166
|
-
const live =
|
|
167
|
-
for (const n of nodes) {
|
|
168
|
-
if (!tunnels.has(n.port)) continue;
|
|
169
|
-
const r = await probe({ port: n.port, baseUrl, path: cfg.probePath });
|
|
170
|
-
if (r.ok) live.push(n);
|
|
171
|
-
else log(`[warp-pool] :${n.port} 隧道没探通(${r.status || r.error || "?"}),不进池:进池会把请求挂到超时,比直连更糟`);
|
|
172
|
-
}
|
|
208
|
+
// 入池前实测探活:WARP 握手没完成时 wireproxy 在跑却转不动流量(请求会挂到超时),那种节点进池会黑洞全部请求
|
|
209
|
+
const live = await probeGate(nodes);
|
|
173
210
|
if (live.length) {
|
|
174
211
|
await dispatcher.setNodes(live.map((n) => ({ port: n.port, uri: `http://127.0.0.1:${n.port}` })));
|
|
175
|
-
|
|
212
|
+
// 出口取证:一次拿到每个端口的真实出口,既喂给轮换器做「换没换动」的比对,也直接写进池结论那一行
|
|
213
|
+
const exits = await labelExits({ ports: live.map((n) => n.port), log });
|
|
214
|
+
rotator.noteExits(exits);
|
|
215
|
+
log(fmtPool({ pooled: live.length, total: nodes.length, exits }));
|
|
176
216
|
} else {
|
|
177
|
-
log(
|
|
217
|
+
log(fmtPoolEmpty({ total: nodes.length, ports: cfg.ports }));
|
|
178
218
|
}
|
|
179
|
-
|
|
180
|
-
// 5. 出口取证:把每台设备的真实出口地址打进日志(免费版 WARP 的 IPv4 是共享 NAT,N 台设备很可能折叠成同一个出口)
|
|
181
|
-
if (live.length) rotator.noteExits(await labelExits({ ports: live.map((n) => n.port), log }));
|
|
182
219
|
}
|
|
183
220
|
|
|
184
221
|
const monitor = createHealthMonitor({
|
|
@@ -186,6 +223,8 @@ export function createWarpPool({
|
|
|
186
223
|
baseUrl,
|
|
187
224
|
probePath: cfg.probePath,
|
|
188
225
|
log,
|
|
226
|
+
infoOf: () => ({ alive: tunnels.size, pooled: dispatcher.size, exits: rotator ? rotator.stats() : [] }),
|
|
227
|
+
cooldownMs: cfg.cooldownMs,
|
|
189
228
|
onExhausted: async () => {
|
|
190
229
|
// 重拨 = 重新注册换出口。只重拨「进程已死」的隧道:全活却全挂 = 网络路径问题,重注册只会把 CF 刷到 429
|
|
191
230
|
const plan = redialPlan({ ports: cfg.ports, alivePorts: [...tunnels.keys()], lastRedialAt, redialGapMs, now });
|
|
@@ -215,10 +254,15 @@ export function createWarpPool({
|
|
|
215
254
|
}
|
|
216
255
|
await wait(gapMs(cfg, rnd));
|
|
217
256
|
}
|
|
218
|
-
if (!closed &&
|
|
219
|
-
//
|
|
220
|
-
|
|
221
|
-
|
|
257
|
+
if (!closed && tunnels.size) {
|
|
258
|
+
// 重拨起来的隧道同样要过闸门:没过探活的隧道进池 = 把请求挂到超时(原先这里直接 setNodes,等于绕过闸门)
|
|
259
|
+
const alive = await probeGate([...tunnels.keys()].map((port) => ({ port })));
|
|
260
|
+
if (alive.length) {
|
|
261
|
+
redialGapMs = REDIAL_MIN_GAP_MS; // 确实有隧道能通 ⇒ 退避复位
|
|
262
|
+
await dispatcher.setNodes(alive.map((n) => ({ port: n.port, uri: `http://127.0.0.1:${n.port}` })));
|
|
263
|
+
} else {
|
|
264
|
+
log("[warp-pool] 重拨后仍没有一条隧道探通,保持直连(等下一轮健康检查)");
|
|
265
|
+
}
|
|
222
266
|
}
|
|
223
267
|
},
|
|
224
268
|
});
|
package/src/warp-pool/ports.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import { readFileSync, readdirSync } from "node:fs";
|
|
8
8
|
import { connect } from "node:net";
|
|
9
9
|
import { nodePaths } from "./nodes.js";
|
|
10
|
+
import { fmtReclaim } from "./report.js";
|
|
10
11
|
|
|
11
12
|
/** /proc/<pid>/cmdline 是 NUL 分隔的字节串;切成参数数组(丢掉结尾空串)。 */
|
|
12
13
|
export function parseCmdline(buf) {
|
|
@@ -69,13 +70,13 @@ export async function reclaimPort({
|
|
|
69
70
|
} = {}) {
|
|
70
71
|
if (platform !== "linux") return [];
|
|
71
72
|
const conf = nodePaths(dir, port).conf;
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
73
|
+
const all = pids();
|
|
74
|
+
const targets = all.filter((pid) => isOurWireproxy(cmdline(pid), conf));
|
|
75
|
+
const startedAt = Date.now();
|
|
76
|
+
if (!targets.length) {
|
|
77
|
+
log(fmtReclaim({ port, scanned: all.length, ours: [], others: [], waitedMs: 0, killed: false }));
|
|
78
|
+
return [];
|
|
79
|
+
}
|
|
79
80
|
for (const pid of targets) {
|
|
80
81
|
try {
|
|
81
82
|
kill(pid, "SIGTERM");
|
|
@@ -87,6 +88,8 @@ export async function reclaimPort({
|
|
|
87
88
|
kill(pid, "SIGKILL");
|
|
88
89
|
} catch {}
|
|
89
90
|
}
|
|
91
|
+
const gone = !targets.some((p) => pidAlive(p, kill));
|
|
92
|
+
log(fmtReclaim({ port, scanned: all.length, ours: targets, waitedMs: Date.now() - startedAt, killed: gone }));
|
|
90
93
|
return targets;
|
|
91
94
|
}
|
|
92
95
|
|
|
@@ -108,3 +111,38 @@ export function portInUse(port, { host = "127.0.0.1", timeoutMs = 800, connectFn
|
|
|
108
111
|
sock.once("error", () => finish(false));
|
|
109
112
|
});
|
|
110
113
|
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* 等端口真正空出来(SIGTERM 之后套接字要几百毫秒才释放,这期间连得上 ≠ 别人的进程占着)。
|
|
117
|
+
* 返回 true = 空了;false = 等了 rounds 轮还是被占。
|
|
118
|
+
*/
|
|
119
|
+
export async function waitPortFree(port, { inUse = portInUse, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), rounds = 12, stepMs = 250 } = {}) {
|
|
120
|
+
for (let i = 0; i < rounds; i++) {
|
|
121
|
+
if (!(await inUse(port))) return true;
|
|
122
|
+
await sleep(stepMs);
|
|
123
|
+
}
|
|
124
|
+
return !(await inUse(port));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* 端口占用者取证:谁在监听——本池 wireproxy 残留?别人的进程?还是空闲。
|
|
129
|
+
* 「孤儿占端口」这类事故唯一能一眼看出结论的读数:只报「占用 / 拒绝再起」是查不出问题的(第 2 轮返工的教训)。
|
|
130
|
+
* @returns {Promise<{busy:boolean, ours:number[], others:Array<{pid:number,name:string}>}>}
|
|
131
|
+
*/
|
|
132
|
+
export async function portOwner({ dir, port, platform = process.platform, pids = defaultPids, cmdline = defaultCmdline, inUse = portInUse } = {}) {
|
|
133
|
+
const busy = await inUse(port);
|
|
134
|
+
const out = { busy, ours: [], others: [] };
|
|
135
|
+
if (!busy || platform !== "linux") return out;
|
|
136
|
+
const conf = nodePaths(dir, port).conf;
|
|
137
|
+
for (const pid of pids()) {
|
|
138
|
+
const argv = cmdline(pid);
|
|
139
|
+
if (!argv.length) continue;
|
|
140
|
+
if (isOurWireproxy(argv, conf)) {
|
|
141
|
+
out.ours.push(pid);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// 非 root 看不到别人的 listen 表,退一步:命令行里带着这个端口号的进程也值得报出来
|
|
145
|
+
if (argv.some((a) => String(a).includes(`:${port}`))) out.others.push({ pid, name: String(argv[0]).split("/").pop() });
|
|
146
|
+
}
|
|
147
|
+
return out;
|
|
148
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// warp-pool · 日志与自检报告(2026-10-16 四轮返工后新增)。
|
|
2
|
+
//
|
|
3
|
+
// 为什么单独一个模块(四轮返工的教训):日志若只说「失败了」、不说「谁干的、为什么」,就只能靠猜——
|
|
4
|
+
// 第 1 轮猜孤儿占端口、第 2 轮分不清旧 daemon 子进程与刚被杀还没释放的套接字、第 3 轮分不清握手未完成与上游拒绝。
|
|
5
|
+
// 故把可判定信息一次摆全:配置快照、端口占用者(pid+命令行+是否本池残留)、身份文件、每条隧道完整生命周期
|
|
6
|
+
// (清理→写配置→spawn pid→探活耗时/状态/错误→退出码/信号/存活时长/末行)、池子结论与「下一步敲什么命令」。
|
|
7
|
+
export const MARK = { step: "▸", ok: "✓", fail: "✗", warn: "⚠" };
|
|
8
|
+
|
|
9
|
+
const KB = (n) => (n >= 1024 * 1024 ? `${(n / 1024 / 1024).toFixed(1)}MB` : `${Math.round(n / 1024)}KB`);
|
|
10
|
+
|
|
11
|
+
/** 人类可读时长:412ms / 1.8s / 2m3s。 */
|
|
12
|
+
export function dur(ms) {
|
|
13
|
+
const n = Number(ms);
|
|
14
|
+
if (!Number.isFinite(n) || n < 0) return "?";
|
|
15
|
+
if (n < 1000) return `${Math.round(n)}ms`;
|
|
16
|
+
if (n < 60_000) return `${(n / 1000).toFixed(1)}s`;
|
|
17
|
+
return `${Math.floor(n / 60_000)}m${Math.round((n % 60_000) / 1000)}s`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** 配置快照:一行说清这次到底按什么参数在跑。 */
|
|
21
|
+
export function fmtConfig(cfg) {
|
|
22
|
+
return `[warp-pool] ${MARK.step} 启动自检:开关=${cfg.enabled ? "开" : "关"} 模式=${cfg.mode} 端口=${cfg.ports.join(",")} 策略=${cfg.strategy} 轮换=${cfg.rotate ? "开" : "关"} 冷却=${Math.round(cfg.cooldownMs / 1000)}s`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** 目录与二进制(带体积,顺手能看出二进制是不是被下坏了)。 */
|
|
26
|
+
export function fmtEnv({ dir, binPath, binSize }) {
|
|
27
|
+
const bin = binPath ? `${binPath}(${binSize ? KB(binSize) : "体积未知"})` : "未就绪";
|
|
28
|
+
return `[warp-pool] ${MARK.step} 目录=${dir} 二进制=${bin}`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 身份文件盘点:哪个端口有 id+token+私钥,哪个缺。 */
|
|
32
|
+
export function fmtIdentities(nodes, ports) {
|
|
33
|
+
const have = new Set(nodes.map((n) => n.port));
|
|
34
|
+
const parts = ports.map((p) => `:${p}${have.has(p) ? " ✓" : " ✗缺"}`);
|
|
35
|
+
return `[warp-pool] ${MARK.step} 身份文件 ${nodes.length}/${ports.length}:${parts.join(" ")}(✓=有 id+token+私钥,✗缺=现场注册)`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** 单端口占用者:区分「本池残留」「别人的进程」「空闲」——第 1 轮返工缺的就是这一行。 */
|
|
39
|
+
export function fmtPortState({ port, busy, ours = [], others = [] }) {
|
|
40
|
+
if (!busy) return `[warp-pool] :${port} 空闲`;
|
|
41
|
+
if (ours.length) return `[warp-pool] :${port} 被 pid ${ours.join(",")} 占:本池 wireproxy 残留(上一代 daemon 被强杀留下的孤儿)→ 起隧道前清理`;
|
|
42
|
+
const who = others.length ? others.map((o) => `pid ${o.pid} ${o.name}`).join("、") : "未知进程";
|
|
43
|
+
return `[warp-pool] :${port} 被 ${who} 占:非本池进程 → 不碰它,这个端口本轮跳过(查明:ss -ltnp | grep :${port})`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 清理残留的结果:扫了多少进程、命中谁、等了多久、清掉没——第 2 轮返工缺的就是「谁在占」。 */
|
|
47
|
+
export function fmtReclaim({ port, scanned, ours = [], others = [], waitedMs, killed }) {
|
|
48
|
+
if (!ours.length) {
|
|
49
|
+
const extra = others.length ? "(端口另有别的进程占着,不是本池的,不动它)" : "";
|
|
50
|
+
return `[warp-pool] :${port} ${MARK.step} 清理残留:扫 ${scanned} 个进程,无本池残留${extra}`;
|
|
51
|
+
}
|
|
52
|
+
return `[warp-pool] :${port} ${MARK.step} 清理残留:扫 ${scanned} 个进程,命中本池残留 pid ${ours.join(",")} → ${killed ? "已清掉" : "清不掉"}(等了 ${dur(waitedMs)})`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 写出的 WireGuard 配置(体积能看出是不是缺字段:缺私钥那次就是 338B vs 373B)。 */
|
|
56
|
+
export function fmtConf({ port, conf, bytes }) {
|
|
57
|
+
return `[warp-pool] :${port} ${MARK.step} 写配置 ${conf}(${bytes}B,0600)`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** 起隧道:pid + 完整命令行,事后可 `ps -p <pid>` 对账。 */
|
|
61
|
+
export function fmtSpawn({ port, pid, binPath, conf }) {
|
|
62
|
+
return `[warp-pool] :${port} ${MARK.step} 起隧道 pid=${pid ?? "?"}:${binPath} -c ${conf}`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** 探活单次:URL、状态、耗时、错误原文——第 3 轮返工缺的就是「到底哪一步不通」。 */
|
|
66
|
+
export function fmtProbe({ port, attempt, attempts, url, ok, status, error, ms }) {
|
|
67
|
+
const head = `[warp-pool] :${port} ${ok ? MARK.ok : MARK.fail} 探活 ${attempt}/${attempts}:GET ${url} 经 127.0.0.1:${port}`;
|
|
68
|
+
const tail = ok
|
|
69
|
+
? `HTTP ${status}(${dur(ms)})→ 进池`
|
|
70
|
+
: `HTTP ${status || 0}(${dur(ms)})${error ? ` 错误=${String(error).slice(0, 160)}` : ""} → 稍后重试`;
|
|
71
|
+
return `${head} → ${tail}`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** 探活彻底失败:说清「不进池」的后果与自检命令,而不是只丢一句「没探通」。 */
|
|
75
|
+
export function fmtProbeGiveUp({ port, attempts, url }) {
|
|
76
|
+
return `[warp-pool] :${port} ${MARK.fail} 探活 ${attempts} 次都不通 ⇒ 不进池(进程可能在跑但隧道转不动流量,进池会把请求挂到超时,比直连更糟)。自检:curl --proxy http://127.0.0.1:${port} ${url}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 隧道退出:退出码/信号/存活时长/pid/是否外部终止/stderr 末行(含「谁杀的」这一层推断)。 */
|
|
80
|
+
export function fmtExit({ port, code, signal, pid, aliveMs, tail }) {
|
|
81
|
+
const how = signal ? `信号 ${signal}` : `退出码 ${code}`;
|
|
82
|
+
const killed = signal ? "本进程未调用 stop ⇒ 外部终止(别人杀了它 / 系统回收)" : "进程自行退出(多为配置被拒或端口被占)";
|
|
83
|
+
return `[warp-pool] :${port} ${MARK.fail} 隧道退出:${how}|存活 ${aliveMs == null ? "?" : dur(aliveMs)}|pid ${pid ?? "?"}|${killed}|末行="${tail || "没有输出"}"`;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** 端口最终被非本池进程占着、拒绝硬起时,把占用者与「套接字还没释放」两种情形分开说。 */
|
|
87
|
+
export function fmtPortBlocked({ port, ours = [], others = [] }) {
|
|
88
|
+
const oursTxt = ours.length ? `本池残留 pid ${ours.join(",")}(SIGTERM/SIGKILL 都没清掉:可能僵尸或权限不足)` : "";
|
|
89
|
+
const othTxt = others.length ? others.map((o) => `pid ${o.pid} ${o.name}`).join("、") : "";
|
|
90
|
+
const who = [oursTxt, othTxt].filter(Boolean).join(";") || "未知进程(已等套接字释放仍连得上)";
|
|
91
|
+
return `[warp-pool] :${port} ${MARK.fail} 端口被 ${who} 占着,拒绝再起(硬起只会 bind 失败、无限重启刷屏)。查明:ss -ltnp | grep :${port}`;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** 池子结论:入池几个、各端口出口是什么、几种不同地址——一句话判断「WARP 到底有没有在用」。 */
|
|
95
|
+
export function fmtPool({ pooled, total, exits }) {
|
|
96
|
+
const head = `[warp-pool] ${MARK.step} 出口池就绪:${pooled}/${total} 条入池`;
|
|
97
|
+
const ipOf = new Map();
|
|
98
|
+
for (const g of exits || []) for (const p of g.nodes || []) if (g.ip) ipOf.set(p, g.ip);
|
|
99
|
+
if (!ipOf.size) return `${head}|出口地址未知(CF trace 取证失败,不影响使用)`;
|
|
100
|
+
const table = [...ipOf.entries()].map(([p, ip]) => `:${p}=${ip}`).join(" ");
|
|
101
|
+
return `${head}|出口:${table}|不同地址 ${new Set(ipOf.values()).size} 个`;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** 池空告警:最容易被误读成「功能在跑」,必须把结论与逐条自检命令一起说。 */
|
|
105
|
+
export function fmtPoolEmpty({ total, ports }) {
|
|
106
|
+
return `[warp-pool] ${MARK.warn} 没有一条隧道探通 ⇒ 池为空,全部请求照旧直连(不影响可用性,但 WARP 出口等于没启用)。逐条自检:for p in ${ports.join(" ")}; do echo -n ":$p "; curl -s --max-time 8 --proxy http://127.0.0.1:$p https://cloudflare.com/cdn-cgi/trace | grep '^ip='; done`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** 健康检查一轮:只在状态变化/心跳时打(每分钟一行会把日志淹掉)。 */
|
|
110
|
+
export function fmtHealth({ healthy, total, slowestMs, problems = [] }) {
|
|
111
|
+
if (!problems.length) return `[warp-pool] ${MARK.ok} 健康检查:${healthy}/${total} 通(最慢 ${dur(slowestMs)})`;
|
|
112
|
+
const detail = problems
|
|
113
|
+
.map((p) => `:${p.port} 第 ${p.fails} 次不通(HTTP ${p.status || 0}${p.error ? ` ${String(p.error).slice(0, 80)}` : ""},${dur(p.ms)})冷却 ${Math.round(p.cooldownMs / 1000)}s`)
|
|
114
|
+
.join(";");
|
|
115
|
+
return `[warp-pool] ${MARK.warn} 健康检查:${healthy}/${total} 通|不通:${detail}`;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** 心跳:一行给全部关键状态,省得「日志翻半天不知道现在啥情况」。 */
|
|
119
|
+
export function fmtHeartbeat({ alive, total, pooled, exits, cooling, lastAllDown }) {
|
|
120
|
+
const ips = [...new Set((exits || []).map((e) => e.ip).filter(Boolean))];
|
|
121
|
+
return `[warp-pool] ${MARK.step} 心跳:隧道 ${alive}/${total} 活|入池 ${pooled}|出口 ${ips.length ? `${ips.length} 种(${ips.join(",")})` : "未知"}|冷却中 ${cooling}|全池不通 ${lastAllDown || "未发生"}`;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* 启动自检块:一次性把「这次按什么参数跑、二进制在哪多大、磁盘上有几个身份、端口分别被谁占着」打全。
|
|
126
|
+
* 第 1 轮返工只看到「退出 code=1 / 已复活」循环,就是缺这一块——真因(孤儿占端口)当时只能靠猜。
|
|
127
|
+
* @param {{ cfg: object, nodes: Array<object>, binPath?: string, binSize?: number,
|
|
128
|
+
* ownerOf: (port:number)=>Promise<{busy:boolean, ours:number[], others:Array<{pid:number,name:string}>}>,
|
|
129
|
+
* log?: Function }} opts
|
|
130
|
+
*/
|
|
131
|
+
export async function startupReport({ cfg, nodes, binPath = "", binSize = 0, ownerOf, log = () => {} }) {
|
|
132
|
+
log(fmtConfig(cfg));
|
|
133
|
+
log(fmtEnv({ dir: cfg.dir, binPath, binSize }));
|
|
134
|
+
log(fmtIdentities(nodes, cfg.ports));
|
|
135
|
+
const states = [];
|
|
136
|
+
for (const port of cfg.ports) {
|
|
137
|
+
try {
|
|
138
|
+
states.push({ port, ...(await ownerOf(port)) });
|
|
139
|
+
} catch (e) {
|
|
140
|
+
states.push({ port, busy: false, ours: [], others: [], error: String(e?.message || e) });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const busy = states.filter((s) => s.busy);
|
|
144
|
+
log(`[warp-pool] ${MARK.step} 端口体检:${busy.length} 个被占 / ${states.length} 个端口`);
|
|
145
|
+
for (const s of states) log(fmtPortState(s));
|
|
146
|
+
const orphans = busy.filter((s) => (s.ours || []).length).length;
|
|
147
|
+
if (orphans) log(`[warp-pool] ${MARK.warn} 有 ${orphans} 个端口被本池残留占着(上一代 daemon 被强杀留下的孤儿)——起隧道前逐个清理,日志会写清 pid`);
|
|
148
|
+
return states;
|
|
149
|
+
}
|
package/src/warp-pool/rotate.js
CHANGED
|
@@ -78,7 +78,7 @@ export function createRotatingDispatcher({
|
|
|
78
78
|
if (!e) return;
|
|
79
79
|
e.fails += 1;
|
|
80
80
|
e.cooldownUntil = now() + cooldownMs;
|
|
81
|
-
log(`[warp-pool] 节点 :${e.port} 连续失败 ${e.fails} 次,冷却 ${Math.round(cooldownMs / 1000)}s
|
|
81
|
+
if (e.fails === 1 || e.fails % 10 === 0) log(`[warp-pool] 节点 :${e.port} 连续失败 ${e.fails} 次,冷却 ${Math.round(cooldownMs / 1000)}s(中间次数静默:健康探活每分钟一轮,一个不通的节点曾把日志刷成 12 行/分钟)`);
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
function markOk(index) {
|
|
@@ -93,16 +93,19 @@ function runCommand(cmd, args, { log = () => {} } = {}) {
|
|
|
93
93
|
export function writeTunnelConf(dir, node) {
|
|
94
94
|
if (!node?.privateKey) throw new Error(`节点 :${node?.port} 缺 WireGuard 私钥,拒绝写出一份跑不起来的配置`);
|
|
95
95
|
const { conf } = nodePaths(dir, node.port);
|
|
96
|
-
|
|
96
|
+
const text = renderWireproxyConf({ ...node, port: node.port });
|
|
97
|
+
writeSecret(conf, text);
|
|
98
|
+
return { conf, bytes: Buffer.byteLength(text) };
|
|
97
99
|
}
|
|
98
100
|
|
|
99
101
|
/**
|
|
100
102
|
* 启动一条隧道。返回 { port, conf, stop() }。
|
|
101
103
|
* 进程退出即视为隧道失效(不自动重启:由 health 层决策重拨换 IP)。
|
|
102
104
|
*/
|
|
103
|
-
export function startTunnel({ binPath: exe, dir, port, log = () => {}, onExit = () => {} } = {}) {
|
|
105
|
+
export function startTunnel({ binPath: exe, dir, port, log = () => {}, onExit = () => {}, now = Date.now } = {}) {
|
|
104
106
|
const { conf } = nodePaths(dir, port);
|
|
105
107
|
const child = spawn(exe, ["-c", conf], { stdio: ["ignore", "pipe", "pipe"] });
|
|
108
|
+
const startedAt = now();
|
|
106
109
|
let tail = "";
|
|
107
110
|
const keepTail = (d) => {
|
|
108
111
|
tail = (tail + String(d)).slice(-800);
|
|
@@ -112,7 +115,7 @@ export function startTunnel({ binPath: exe, dir, port, log = () => {}, onExit =
|
|
|
112
115
|
|
|
113
116
|
let stopped = false;
|
|
114
117
|
child.on("exit", (code, signal) => {
|
|
115
|
-
if (!stopped) onExit({ port, code, signal, tail });
|
|
118
|
+
if (!stopped) onExit({ port, code, signal, tail, pid: child.pid, aliveMs: now() - startedAt });
|
|
116
119
|
});
|
|
117
120
|
|
|
118
121
|
return {
|
|
@@ -139,18 +142,20 @@ export function startTunnel({ binPath: exe, dir, port, log = () => {}, onExit =
|
|
|
139
142
|
export async function probeTunnel({ port, baseUrl = "https://opencode.ai", path = "/zen/v1/models", fetchImpl = compatFetch, timeoutMs = 8000, headers = {}, wantText = false } = {}) {
|
|
140
143
|
const { ProxyAgent } = getUndici();
|
|
141
144
|
const agent = new ProxyAgent({ uri: `http://127.0.0.1:${port}`, connections: 2, keepAliveTimeout: 10_000 });
|
|
145
|
+
const url = `${String(baseUrl).replace(/\/+$/, "")}${path}`;
|
|
146
|
+
const t0 = Date.now();
|
|
142
147
|
try {
|
|
143
|
-
const res = await fetchImpl(
|
|
148
|
+
const res = await fetchImpl(url, {
|
|
144
149
|
headers,
|
|
145
150
|
signal: AbortSignal.timeout(timeoutMs),
|
|
146
151
|
dispatcher: agent,
|
|
147
152
|
});
|
|
148
|
-
const base = { ok: res.ok, status: res.status };
|
|
153
|
+
const base = { ok: res.ok, status: res.status, ms: Date.now() - t0, url };
|
|
149
154
|
if (!wantText) return base;
|
|
150
155
|
const text = await res.text().catch(() => "");
|
|
151
156
|
return { ...base, text: String(text).slice(0, 2000) };
|
|
152
157
|
} catch (e) {
|
|
153
|
-
return { ok: false, status: 0, error: String(e?.message || e) };
|
|
158
|
+
return { ok: false, status: 0, error: String(e?.message || e), ms: Date.now() - t0, url };
|
|
154
159
|
} finally {
|
|
155
160
|
try { await agent.close(); } catch {}
|
|
156
161
|
}
|