mslxdff 0.1.226 → 0.1.228
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/chat-pipeline/index.js +6 -2
- package/src/chat-pipeline/serial-trial.js +3 -1
- package/src/cli/format.js +2 -0
- package/src/peers.js +345 -307
- package/src/routes/peer-probe.js +53 -0
- package/src/routes/peer-serial.js +56 -6
- package/src/routes/peers.js +8 -0
- package/src/runtime/keep-warm.js +135 -120
package/package.json
CHANGED
|
@@ -101,7 +101,11 @@ export function createChatPipeline({ upstream, auto, logs, peers, groups, bus, t
|
|
|
101
101
|
}
|
|
102
102
|
if (!order.length) order = [""];
|
|
103
103
|
const canFallback = order.length > 1;
|
|
104
|
-
|
|
104
|
+
// hi 探针(x-mslxdff-probe:1)只问「这台机器此刻能不能打通上游」,不代表真实业务:
|
|
105
|
+
// 它绝不能再触发本机的组员/宽带接力——否则探针会跟着走完整条接力链(最坏 45s 预算),
|
|
106
|
+
// 而调用方只看它几百毫秒内活不活。组员侧就地直连,成败如实回给调用方。
|
|
107
|
+
const isProbe = String(req?.headers?.["x-mslxdff-probe"] || "") === "1";
|
|
108
|
+
const canForwardPeers = !isProbe && Boolean(peers) && hops < (maxHops ?? 3);
|
|
105
109
|
mark("ordered");
|
|
106
110
|
|
|
107
111
|
const reqIp = clientIp(req); // 一次解析两处用:request 事件的 ip 字段 + 回路语料的 clientIp(routes/helpers.js 取 x-forwarded-for 首段)
|
|
@@ -125,7 +129,7 @@ export function createChatPipeline({ upstream, auto, logs, peers, groups, bus, t
|
|
|
125
129
|
const clientSession = String(req?.headers?.["x-session-affinity"] || req?.headers?.["x-session-id"] || "").trim() || null;
|
|
126
130
|
// triedUrls 挂请求级并**按引用**共享:同一请求内"同一组员不重复试"的契约跨候选生效
|
|
127
131
|
// (peer 阶段各分支会 spread 出上下文副本,集合在这里建一次才不会各存一份)。
|
|
128
|
-
const handlerCtx = { reqId, api, model: null, body: req?.body, hops, peers, plugins, evt, logError, logCall, logs, workbuddyUid, sessionId: clientSession, clientIp: reqIp, triedUrls: new Set() };
|
|
132
|
+
const handlerCtx = { reqId, api, model: null, body: req?.body, hops, peers, plugins, evt, logError, logCall, logs, workbuddyUid, sessionId: clientSession, clientIp: reqIp, triedUrls: new Set(), isProbe };
|
|
129
133
|
if (clientSession) evt("client-session", { reqId, sessionId: clientSession.slice(0, 24) });
|
|
130
134
|
|
|
131
135
|
await engine.run({
|
|
@@ -100,7 +100,9 @@ export async function runSerialTrial(ctx, deps = {}) {
|
|
|
100
100
|
const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
|
|
101
101
|
// 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
|
|
102
102
|
// 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
|
|
103
|
-
|
|
103
|
+
// 探针(x-mslxdff-probe:1)不重试:调用方只要一个「活不活」的答案,重试阶梯会把它吊 40s。
|
|
104
|
+
const isProbe = handlerCtx?.isProbe === true;
|
|
105
|
+
const emptyCfg = isProbe ? { max: 0, steps: null, delayMs: 0 } : emptyRetryCfg();
|
|
104
106
|
const raiseCap = emptyRaiseCap();
|
|
105
107
|
const nudgeCfg = emptyNudgeCfg();
|
|
106
108
|
let emptyRetried = 0;
|
package/src/cli/format.js
CHANGED
|
@@ -62,6 +62,8 @@ export function fmtEvent(e) {
|
|
|
62
62
|
return `${head} peer check ${e.peer} -> ${e.count ? e.healthy.join(", ") : "no healthy models"}${e.strict ? " (strict)" : ""}`;
|
|
63
63
|
case "peer-request":
|
|
64
64
|
return `${head} peer req 并发发出 peer=${e.peer} model=${m(e.model)} hops=${e.hops}`;
|
|
65
|
+
case "peer-probe":
|
|
66
|
+
return `${head} peer hi 备选探活 peer=${e.peer} ${e.ok ? "ok" : `fail HTTP ${e.status ?? "network"}`} model=${m(e.model)} ${e.latencyMs ? `${e.latencyMs}ms` : ""}${e.action === "skip" ? " → 跳过换下一个" : ""}`;
|
|
65
67
|
case "peer-forward":
|
|
66
68
|
return `${head} peer resp 收到响应 peer=${e.peer} model=${m(e.model)} ${e.ok ? "ok" : "fail"} ${e.latencyMs ? `${e.latencyMs}ms` : ""} hops=${e.hops}${e.retry ? " (retry)" : ""}`;
|
|
67
69
|
case "peer-error":
|
package/src/peers.js
CHANGED
|
@@ -1,307 +1,345 @@
|
|
|
1
|
-
import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
|
|
2
|
-
|
|
3
|
-
// 组员失败冷却分级(2026-10-18 用户拍板):
|
|
4
|
-
// 非额度失败(502/5xx/超时,可能几秒就恢复)→ 短冷却 10s,到点自动重新纳入候选、继续挨个重试;
|
|
5
|
-
// 额度类(连续 2 次 429)→ 长冷却 10min(额度当天不会因快重试恢复,反复撞只会白烧配额)。
|
|
6
|
-
export const DEFAULT_PEER_COOLDOWN_MS = 10_000;
|
|
7
|
-
export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 10 * 60_000;
|
|
8
|
-
export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
|
|
9
|
-
export const DEFAULT_MAX_HOPS = 3;
|
|
10
|
-
export const DEFAULT_BROADBAND_STALE_MS = 90_000;
|
|
11
|
-
// 连续几次连接级失败即视为"这个地址没服务",进长冷却(不再靠 30s 短冷却反复捞回)。
|
|
12
|
-
export const DEAD_FAIL_THRESHOLD = 2;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
//
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
//
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if (
|
|
171
|
-
|
|
172
|
-
return
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
//
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
return
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
const prev = stats[url] || {};
|
|
282
|
-
stats[url] = {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
}
|
|
1
|
+
import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
|
|
2
|
+
|
|
3
|
+
// 组员失败冷却分级(2026-10-18 用户拍板):
|
|
4
|
+
// 非额度失败(502/5xx/超时,可能几秒就恢复)→ 短冷却 10s,到点自动重新纳入候选、继续挨个重试;
|
|
5
|
+
// 额度类(连续 2 次 429)→ 长冷却 10min(额度当天不会因快重试恢复,反复撞只会白烧配额)。
|
|
6
|
+
export const DEFAULT_PEER_COOLDOWN_MS = 10_000;
|
|
7
|
+
export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 10 * 60_000;
|
|
8
|
+
export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
|
|
9
|
+
export const DEFAULT_MAX_HOPS = 3;
|
|
10
|
+
export const DEFAULT_BROADBAND_STALE_MS = 90_000;
|
|
11
|
+
// 连续几次连接级失败即视为"这个地址没服务",进长冷却(不再靠 30s 短冷却反复捞回)。
|
|
12
|
+
export const DEAD_FAIL_THRESHOLD = 2;
|
|
13
|
+
// 探测失败(GET /health 连不上)视为"这个节点当前不可用",10 分钟内不再当候选。
|
|
14
|
+
// 与请求失败账分开:health 探测是**主动、零额度**的可用性判断,比"这一发不顺"更可信——
|
|
15
|
+
// 现网 2026-10-20 13:27 req mvg8r7cz-b8qn:152.67.222.233:62028 早已 health timeout,
|
|
16
|
+
// CLI/保热都探到了,但选路不知道,仍把它排进候选并吃满 20s 单次上限(总 27s)。
|
|
17
|
+
export const DEFAULT_PEER_PROBE_DEAD_MS = 10 * 60_000;
|
|
18
|
+
|
|
19
|
+
export function peerProbeDeadMs() {
|
|
20
|
+
const n = Number(process.env.MSLXDFF_PEER_PROBE_DEAD_MS);
|
|
21
|
+
return Number.isInteger(n) && n >= 0 ? n : DEFAULT_PEER_PROBE_DEAD_MS;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const EMA_ALPHA = 0.3;
|
|
25
|
+
|
|
26
|
+
export function broadbandStaleMs() {
|
|
27
|
+
const n = Number(process.env.MSLXDFF_BROADBAND_STALE_MS);
|
|
28
|
+
return Number.isInteger(n) && n > 0 ? n : DEFAULT_BROADBAND_STALE_MS;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function isBroadbandStale(peer, nowMs = Date.now(), staleMs = broadbandStaleMs()) {
|
|
32
|
+
const kind = peer?.kind || (String(peer?.url || "").startsWith("relay://") ? "broadband" : "static");
|
|
33
|
+
if (kind !== "broadband") return false;
|
|
34
|
+
const lastSeen = peer?.lastSeen;
|
|
35
|
+
if (typeof lastSeen !== "number") return true; // no heartbeat yet => stale
|
|
36
|
+
return nowMs - lastSeen > staleMs;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function normalizePeerUrl(url) {
|
|
40
|
+
return String(url || "").trim().replace(/\/+$/, "");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function createPeersService({
|
|
44
|
+
file,
|
|
45
|
+
now = () => Date.now(),
|
|
46
|
+
cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
|
|
47
|
+
limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
|
|
48
|
+
heatMs = DEFAULT_PEER_HEAT_MS,
|
|
49
|
+
peers: seedPeers,
|
|
50
|
+
errors: seedErrors,
|
|
51
|
+
stats: seedStats,
|
|
52
|
+
persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
|
|
53
|
+
persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
|
|
54
|
+
persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
|
|
55
|
+
} = {}) {
|
|
56
|
+
const list = (seedPeers ?? loadPeers(file ? { file } : {}))
|
|
57
|
+
.map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
|
|
58
|
+
.filter((p) => p && p.url);
|
|
59
|
+
const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
|
|
60
|
+
const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
|
|
61
|
+
|
|
62
|
+
function all() {
|
|
63
|
+
return [...list];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function add(peer) {
|
|
67
|
+
const url = normalizePeerUrl(peer?.url);
|
|
68
|
+
if (!url) return false;
|
|
69
|
+
const existing = list.find((p) => p.url === url);
|
|
70
|
+
const entry = { ...peer, url };
|
|
71
|
+
if (existing) Object.assign(existing, entry);
|
|
72
|
+
else list.push(entry);
|
|
73
|
+
persistPeers([...list]);
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function remove(url) {
|
|
78
|
+
const target = normalizePeerUrl(url);
|
|
79
|
+
const idx = list.findIndex((p) => p.url === target);
|
|
80
|
+
if (idx < 0) return false;
|
|
81
|
+
list.splice(idx, 1);
|
|
82
|
+
persistPeers([...list]);
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function removeByGroup(group) {
|
|
87
|
+
const before = list.length;
|
|
88
|
+
for (let i = list.length - 1; i >= 0; i--) {
|
|
89
|
+
if (list[i].group === group) list.splice(i, 1);
|
|
90
|
+
}
|
|
91
|
+
if (list.length !== before) persistPeers([...list]);
|
|
92
|
+
return before - list.length;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 分级冷却:连续 429(streak >= 2)→ limitCooldownMs(限流恢复慢,5min 内不再考虑);
|
|
96
|
+
// 连接级永久失败(ECONNREFUSED/ENOTFOUND,deadFails >= 2)同样进长冷却;
|
|
97
|
+
// 其他失败(网络抖动、5xx)→ cooldownMs(可能几秒恢复,给快速重试机会)。
|
|
98
|
+
function coolingWindowMs(url) {
|
|
99
|
+
// 探测死亡(health 连不上):10min 内不再考虑——比请求失败更可信的"没服务"证据
|
|
100
|
+
if (stats[url]?.probeDeadAt !== undefined && now() - stats[url].probeDeadAt < peerProbeDeadMs()) return limitCooldownMs;
|
|
101
|
+
if ((stats[url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return limitCooldownMs;
|
|
102
|
+
const streak = stats[url]?.streak429 ?? 0;
|
|
103
|
+
return streak >= 2 ? limitCooldownMs : cooldownMs;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 连接级失败(端口没服务/DNS 解析不了)与"这一发不顺"是两回事:前者换多少次都一样,
|
|
107
|
+
// 现网 node-b(127.0.0.1:3500,本机根本没服务)靠 30s 短冷却 + 兜底,被反复竞速 1569 次。
|
|
108
|
+
function isConnLevelFailure(message) {
|
|
109
|
+
const s = String(message || "");
|
|
110
|
+
return /ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ECONNRESET|fetch failed|connect (?:timeout|ETIMEDOUT)|socket hang up/i.test(s);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function isCooling(url) {
|
|
114
|
+
// 探测死亡(health 探不通):10min 内不再当候选——比请求失败更可信的"没服务"证据
|
|
115
|
+
const pDead = stats[url]?.probeDeadAt;
|
|
116
|
+
if (pDead !== undefined && now() - pDead < peerProbeDeadMs()) return true;
|
|
117
|
+
const win = coolingWindowMs(url);
|
|
118
|
+
if (!win) return false;
|
|
119
|
+
const err = lastErrorAt[url];
|
|
120
|
+
if (typeof err === "number" && now() - err < win) return true;
|
|
121
|
+
const peer = list.find((p) => p.url === url);
|
|
122
|
+
if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function isBroadbandCooling(url) {
|
|
127
|
+
const peer = list.find((p) => p.url === url);
|
|
128
|
+
if (!peer) return false;
|
|
129
|
+
return isBroadbandStale(peer, now(), broadbandStaleMs());
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function available() {
|
|
133
|
+
return list.filter((p) => !isCooling(p.url));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// A peer counts as "hot" when it succeeded recently: reuse its last model
|
|
137
|
+
// without a fresh health probe to keep the fast path fast.
|
|
138
|
+
function isHot(url, t = now()) {
|
|
139
|
+
if (isCooling(url)) return false;
|
|
140
|
+
const s = stats[url];
|
|
141
|
+
return Boolean(s?.okAt) && t - s.okAt < heatMs;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function stat(url) {
|
|
145
|
+
return stats[url] ? { ...stats[url] } : null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function rankScore(p, t) {
|
|
149
|
+
const s = stats[p.url];
|
|
150
|
+
const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
|
|
151
|
+
const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
|
|
152
|
+
const fails = s?.fails ?? 0;
|
|
153
|
+
return hot * 1_000_000_000 + latency * 1_000 + fails;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// 可用组员排序(用户 2026-10-19 决定):**延迟最小优先**,不再「热优先」。
|
|
157
|
+
// 语义 = 「组员的额度用完后,用跟组长延迟最小的组员」——延迟是首要判据。
|
|
158
|
+
// 延迟来源(优先级从高到低):
|
|
159
|
+
// ① stats 里的真实请求 EMA(recordResult 记的 latencyMs)——最可信;
|
|
160
|
+
// ② peer.heartbeatLatencyMs —— 组员**定时心跳**上报的 rtt(没跑过真实请求时也能排序);
|
|
161
|
+
// 未测过延迟的排在有数据之后;同延迟按「最早失败优先」;都没失败过的排最后。
|
|
162
|
+
// 组长识别:本机是组员时,peers 里有 name==="leader" 的记录(cli/group-test.js 同口径)。
|
|
163
|
+
// 组长自己节点视角没有这条记录 → isLeaderPeer 恒 false,退化为纯延迟排序(正是想要的行为)。
|
|
164
|
+
function isLeaderPeer(p) {
|
|
165
|
+
return String(p?.name || "").toLowerCase() === "leader";
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function peerLatency(p) {
|
|
169
|
+
const s = stats[p.url]?.latencyMs;
|
|
170
|
+
if (s != null) return s;
|
|
171
|
+
const hb = list.find((x) => x.url === p.url)?.heartbeatLatencyMs;
|
|
172
|
+
return Number.isFinite(Number(hb)) ? Number(hb) : null;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// 组员延迟升序(组长不参与),供两处复用:ordered 的第二档 + standbyChain。
|
|
176
|
+
function byLatencyAsc(a, b) {
|
|
177
|
+
const la = peerLatency(a);
|
|
178
|
+
const lb = peerLatency(b);
|
|
179
|
+
const ha = la != null, hb = lb != null;
|
|
180
|
+
if (ha !== hb) return ha ? -1 : 1; // 有延迟数据的优先
|
|
181
|
+
if (ha && hb && la !== lb) return la - lb; // 延迟小的先上
|
|
182
|
+
const ea = lastErrorAt[a.url];
|
|
183
|
+
const eb = lastErrorAt[b.url];
|
|
184
|
+
if (ea == null && eb == null) return 0;
|
|
185
|
+
if (ea == null) return 1;
|
|
186
|
+
if (eb == null) return -1;
|
|
187
|
+
return ea - eb;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// 用户 2026-10-20 约定:**组长恒定第一**(先榨干组长额度),组长用完再按延迟升序上组员。
|
|
191
|
+
// 之前是纯延迟排序,「9/9 走组长」只是组长恰好延迟最低的巧合(AGENTS.md 已记差距)。
|
|
192
|
+
function ordered(t = now()) {
|
|
193
|
+
return available().sort((a, b) => {
|
|
194
|
+
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
195
|
+
if (la !== lb) return la ? -1 : 1; // 组长恒排第一
|
|
196
|
+
return byLatencyAsc(a, b);
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 常驻接力链(用户 2026-10-20:永远准备一个备选,429 立即切)。
|
|
202
|
+
* IPv4 阶段:组长 → 组员(延迟升序);IPv4 全用完才进 IPv6 阶段:先本机/组长 v6,
|
|
203
|
+
* 再按延迟升序找 hasV6 的组员。
|
|
204
|
+
* @returns {{ipv4: object[], ipv6: object[]}} 两段链,各自已排序
|
|
205
|
+
*/
|
|
206
|
+
function standbyChain(t = now()) {
|
|
207
|
+
const ipv4 = available().sort((a, b) => {
|
|
208
|
+
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
209
|
+
if (la !== lb) return la ? -1 : 1;
|
|
210
|
+
return byLatencyAsc(a, b);
|
|
211
|
+
});
|
|
212
|
+
// IPv6 阶段:只保留心跳自报 hasV6 的节点,同样组长优先、其余延迟升序
|
|
213
|
+
const ipv6 = ipv4.filter((p) => p.hasV6 === true);
|
|
214
|
+
return { ipv4, ipv6 };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** 下一个备选:从链里取第一个还没试过的。这就是「永远准备好的那个」。 */
|
|
218
|
+
function nextStandby(tried, t = now()) {
|
|
219
|
+
const { ipv4, ipv6 } = standbyChain(t);
|
|
220
|
+
const skip = tried instanceof Set ? tried : new Set(tried || []);
|
|
221
|
+
return {
|
|
222
|
+
ipv4: ipv4.find((p) => !skip.has(p.url)) || null,
|
|
223
|
+
ipv6: ipv6.find((p) => !skip.has(p.url)) || null,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Available peers ordered specifically for recovery-time retry: earliest
|
|
228
|
+
// failure first (it has had the longest to come back), never-failed last.
|
|
229
|
+
function orderedByLastError(t = now()) {
|
|
230
|
+
return available().sort((a, b) => {
|
|
231
|
+
const ea = lastErrorAt[a.url];
|
|
232
|
+
const eb = lastErrorAt[b.url];
|
|
233
|
+
if (ea == null && eb == null) return 0;
|
|
234
|
+
if (ea == null) return 1; // never-failed peers go after any that failed
|
|
235
|
+
if (eb == null) return -1;
|
|
236
|
+
return ea - eb; // earliest failure first
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
let cursor = 0;
|
|
241
|
+
|
|
242
|
+
// 兜底重试集合:仅"因错误冷却中"的 peer(按最早失败优先)。
|
|
243
|
+
// 场景:唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时,兜底给它一次机会;
|
|
244
|
+
// 成功会被 recordResult 清除错误记录、回归热路径,避免"明明能用的节点被 30s 冷却全灭"。
|
|
245
|
+
function coolingByLastError() {
|
|
246
|
+
const t = now();
|
|
247
|
+
return list
|
|
248
|
+
.filter((p) => {
|
|
249
|
+
const err = lastErrorAt[p.url];
|
|
250
|
+
if (typeof err !== "number") return false;
|
|
251
|
+
if (stats[p.url]?.probeDeadAt !== undefined && now() - stats[p.url].probeDeadAt < peerProbeDeadMs()) return false; // 探测死亡:兜底也不考虑(否则每次白等一次拒连)
|
|
252
|
+
if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却:兜底也不考虑
|
|
253
|
+
if ((stats[p.url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return false; // 连接级永久失败:兜底也不考虑(否则每次白等一次拒连)
|
|
254
|
+
return t - err < cooldownMs;
|
|
255
|
+
})
|
|
256
|
+
.sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function next() {
|
|
260
|
+
const avail = available();
|
|
261
|
+
if (!avail.length) return null;
|
|
262
|
+
cursor = cursor % avail.length;
|
|
263
|
+
return avail[cursor++];
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Long-lived error memory: a peer keeps its last-error timestamp until a
|
|
267
|
+
// subsequent success resets it (success clears the failure record) or the
|
|
268
|
+
// error is no longer in the persist store on next load.
|
|
269
|
+
async function recordError(url, { status, message } = {}) {
|
|
270
|
+
if (!url) return;
|
|
271
|
+
lastErrorAt[url] = now();
|
|
272
|
+
await persistErrors({ ...lastErrorAt });
|
|
273
|
+
// 连续 429 计数(仅成功清零):第 2 次起进入长冷却——该 peer 大概率真被限流,不再频繁试它
|
|
274
|
+
if (Number(status) === 429) {
|
|
275
|
+
const prev = stats[url] || {};
|
|
276
|
+
stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
|
|
277
|
+
await persistStats({ ...stats });
|
|
278
|
+
}
|
|
279
|
+
// 连接级失败计数(仅成功清零):第 2 次起进长冷却,且不再被 coolingByLastError 兜底捞回
|
|
280
|
+
if (isConnLevelFailure(message)) {
|
|
281
|
+
const prev = stats[url] || {};
|
|
282
|
+
stats[url] = { ...prev, deadFails: (prev.deadFails || 0) + 1 };
|
|
283
|
+
await persistStats({ ...stats });
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
|
|
288
|
+
// last successful model) and clears the error memory; failures keep the
|
|
289
|
+
// error timestamp so the retry pass can order by recovery time.
|
|
290
|
+
async function recordResult(url, { ok, latencyMs, model } = {}) {
|
|
291
|
+
if (!url) return;
|
|
292
|
+
if (ok) {
|
|
293
|
+
if (lastErrorAt[url] != null) {
|
|
294
|
+
delete lastErrorAt[url];
|
|
295
|
+
await persistErrors({ ...lastErrorAt });
|
|
296
|
+
}
|
|
297
|
+
const prev = stats[url] || {};
|
|
298
|
+
const next = {
|
|
299
|
+
okAt: now(),
|
|
300
|
+
latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
|
|
301
|
+
? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
|
|
302
|
+
: (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
|
|
303
|
+
fails: 0,
|
|
304
|
+
model: model || prev.model || "",
|
|
305
|
+
streak429: 0,
|
|
306
|
+
deadFails: 0,
|
|
307
|
+
};
|
|
308
|
+
// 探测死亡标记:请求成功 = 节点活着,冷却窗口立即解除。
|
|
309
|
+
// next 本身不含 probeDeadAt 键,天然清除;这里只兜底确保没有残留。
|
|
310
|
+
stats[url] = next;
|
|
311
|
+
delete stats[url].probeDeadAt;
|
|
312
|
+
} else {
|
|
313
|
+
const prev = stats[url] || {};
|
|
314
|
+
stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
|
|
315
|
+
}
|
|
316
|
+
await persistStats({ ...stats });
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// 探测死亡标记(health 连不上):probeDeadAt 置为此刻,10min 内冷却。不反复探。
|
|
320
|
+
// recordProbeOk 在探测成功时清标记(节点回来了,冷却窗口立刻结束)。
|
|
321
|
+
async function recordProbeDead(url) {
|
|
322
|
+
if (!url) return;
|
|
323
|
+
const prev = stats[url] || {};
|
|
324
|
+
stats[url] = { ...prev, probeDeadAt: now() };
|
|
325
|
+
await persistStats({ ...stats });
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
async function recordProbeOk(url) {
|
|
329
|
+
if (!url) return;
|
|
330
|
+
const prev = stats[url] || {};
|
|
331
|
+
if (!("probeDeadAt" in prev)) return;
|
|
332
|
+
delete prev.probeDeadAt;
|
|
333
|
+
stats[url] = prev;
|
|
334
|
+
await persistStats({ ...stats });
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return {
|
|
338
|
+
all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
|
|
339
|
+
const peer = list.find((p) => p.url === url);
|
|
340
|
+
return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
|
|
341
|
+
}, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
|
|
342
|
+
standbyChain, nextStandby, isLeaderPeer,
|
|
343
|
+
recordError, recordResult, recordProbeDead, recordProbeOk, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
|
|
344
|
+
};
|
|
345
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// 组员「打一个 hi」探针:备选时打一次、轮到它时再打一次,回不来就当它不行。
|
|
2
|
+
//
|
|
3
|
+
// 为什么要打 hi 而不是 /health:额度只体现在真实 chat 调用上。实测(2026-10-20,本机出口已知 429)
|
|
4
|
+
// GET /health 与 GET /zen/v1/models **都照样 200**,只有 POST chat 才回 429 FreeUsageLimitError。
|
|
5
|
+
// 为什么要挑在「备选」和「轮到」两个时刻:
|
|
6
|
+
// 备选是上一次切换时定的,中间上一个节点可能已在大 body 上撞墙几十秒(实测 1.5MB 撞 429 要
|
|
7
|
+
// 28~62s),那时有额度不代表现在还有 ⇒ 轮到它时必须再确认一次。
|
|
8
|
+
// 开销:hi + max_tokens=8,拿响应头即掐断,不等正文,也不触发对端空轮重试。
|
|
9
|
+
import { compatFetch, timeoutSignal } from "../compat.js";
|
|
10
|
+
import { buildShareKeysHeader, SHARE_KEYS_HEADER } from "../providers/share-keys.js";
|
|
11
|
+
|
|
12
|
+
// 等 hi 的上限。超时按「没判过」处理(交主流程照常试),绝不因为探针慢就错杀一个能用的节点。
|
|
13
|
+
export function peerProbeTimeoutMs(env = process.env) {
|
|
14
|
+
const n = Number(env.MSLXDFF_PEER_PROBE_TIMEOUT_MS);
|
|
15
|
+
return Number.isInteger(n) && n > 0 ? n : 10_000;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 给一个组员打一句 hi,只关心它活不活。
|
|
20
|
+
* @returns {Promise<{ok: boolean, status: number, ms: number, unknown?: boolean}>} 永不抛
|
|
21
|
+
* ok=false 只在「明确不该再试」时给:拿到 4xx/5xx 响应。
|
|
22
|
+
* 连不上/超时一律 ok=true + unknown=true(那是主流程 forwardToPeer 的账,探针不越权判死)。
|
|
23
|
+
*/
|
|
24
|
+
export async function probePeer(peer, { model, timeoutMs = peerProbeTimeoutMs(), dispatcher, fetchImpl = compatFetch } = {}) {
|
|
25
|
+
const t0 = Date.now();
|
|
26
|
+
if (!peer?.url || !model) return { ok: true, status: 0, ms: 0, unknown: true };
|
|
27
|
+
const headers = {
|
|
28
|
+
"Content-Type": "application/json",
|
|
29
|
+
"Authorization": `Bearer ${peer.token || ""}`,
|
|
30
|
+
"x-mslxdff-hops": "1",
|
|
31
|
+
"x-mslxdff-model-lock": model,
|
|
32
|
+
// 探针必须可识别:组员侧据此知道这是「体检」不是真实业务(不落 usage、不参与空轮重试),
|
|
33
|
+
// 日志/测试也能把它与真实转发分开,否则「一次请求打了几次组员」这类断言会被探针污染。
|
|
34
|
+
"x-mslxdff-probe": "1",
|
|
35
|
+
"Accept": "text/event-stream",
|
|
36
|
+
};
|
|
37
|
+
const share = buildShareKeysHeader(model);
|
|
38
|
+
if (share) headers[SHARE_KEYS_HEADER] = share;
|
|
39
|
+
try {
|
|
40
|
+
const res = await fetchImpl(`${String(peer.url).replace(/\/+$/, "")}/v1/chat/completions`, {
|
|
41
|
+
method: "POST",
|
|
42
|
+
headers,
|
|
43
|
+
body: JSON.stringify({ model, stream: true, max_tokens: 8, messages: [{ role: "user", content: "hi" }] }),
|
|
44
|
+
signal: timeoutSignal(timeoutMs),
|
|
45
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
46
|
+
});
|
|
47
|
+
// 拿到响应头就够判了:立刻掐断,不等正文
|
|
48
|
+
try { const c = res.body?.cancel?.(); if (c && typeof c.catch === "function") c.catch(() => {}); } catch {}
|
|
49
|
+
return { ok: res.status < 400, status: res.status, ms: Date.now() - t0 };
|
|
50
|
+
} catch {
|
|
51
|
+
return { ok: true, status: 0, ms: Date.now() - t0, unknown: true };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -45,7 +45,7 @@ export function warmPeer(peer, deps = {}) {
|
|
|
45
45
|
export async function serialTryPeers({ candidates, v6Candidates = [], ctx, tried, deps }) {
|
|
46
46
|
const {
|
|
47
47
|
resolvePeerTarget, forwardToPeer, peerName, recordLater, summarizeRequest, runHook, errMsg,
|
|
48
|
-
perAttemptCap, deadline, budgetMs,
|
|
48
|
+
perAttemptCap, deadline, budgetMs, probePeer,
|
|
49
49
|
} = deps;
|
|
50
50
|
const warm = deps.warmPeer || warmPeer;
|
|
51
51
|
const failed = [];
|
|
@@ -71,22 +71,51 @@ export async function serialTryPeers({ candidates, v6Candidates = [], ctx, tried
|
|
|
71
71
|
const ipv4Chain = filterSlow(fresh(candidates));
|
|
72
72
|
const ipv6Chain = filterSlow(fresh(v6Candidates));
|
|
73
73
|
|
|
74
|
-
// ——
|
|
74
|
+
// —— 预取 + 探针(用户 2026-10-20 口径)——
|
|
75
|
+
// 「某个节点当备选的时候打一个 hi,准备轮到它了也打一个 hi;hi 都回不来就当它不行,直接换。」
|
|
76
|
+
// 两个 hi 都打,不做结果复用:备选是上一次切换时定的,中间可能已过了几十秒
|
|
77
|
+
// (上一个节点在大 body 上撞墙,实测 1.5MB 要 28~62s),那时活不代表现在活。
|
|
78
|
+
// 备选那个 hi 的作用是暖连接 + 提前暴露死节点,轮到时的那个才决定去留。
|
|
75
79
|
let pending = null;
|
|
76
|
-
|
|
80
|
+
// resolvePeerTarget 会打 /v1/models/status,别为探针白测两次:这里记住解析结果给下一轮复用。
|
|
81
|
+
const resolved = new Map(); // url -> Promise<{peer,target}|null>
|
|
82
|
+
const resolveOnce = (peer) => {
|
|
83
|
+
const hit = resolved.get(peer.url);
|
|
84
|
+
if (hit) return hit;
|
|
85
|
+
const p = Promise.resolve()
|
|
86
|
+
.then(() => resolvePeerTarget(ctx, peer))
|
|
87
|
+
.catch(() => null);
|
|
88
|
+
resolved.set(peer.url, p);
|
|
89
|
+
return p;
|
|
90
|
+
};
|
|
91
|
+
const prime = (peer, target) => {
|
|
92
|
+
if (!peer || !target || typeof probePeer !== "function") return;
|
|
93
|
+
try {
|
|
94
|
+
const p = probePeer(peer, target);
|
|
95
|
+
if (p && typeof p.then === "function") p.catch(() => {});
|
|
96
|
+
} catch { /* 预热失败不影响主流程 */ }
|
|
97
|
+
};
|
|
98
|
+
const primeAt = (peer) => {
|
|
77
99
|
if (!peer) return;
|
|
78
|
-
|
|
100
|
+
resolveOnce(peer)
|
|
101
|
+
.then((r) => { if (r) prime(r.peer, r.target); })
|
|
102
|
+
.catch(() => {});
|
|
79
103
|
};
|
|
80
104
|
const takeNext = (stage) => {
|
|
81
105
|
const chain = stage === "ipv6" ? ipv6Chain : ipv4Chain;
|
|
82
106
|
const p = chain.shift() || null;
|
|
83
|
-
|
|
107
|
+
// 立刻给「再下一个」打 hi。模型名要现在解析——下一轮循环该它时 ctx.model 可能已变。
|
|
108
|
+
if (p) primeAt(chain[0] || null);
|
|
84
109
|
return p;
|
|
85
110
|
};
|
|
86
111
|
|
|
87
112
|
let stage = "ipv4";
|
|
88
113
|
let emptyIpv4Announced = false;
|
|
89
114
|
|
|
115
|
+
// 链头从没当过「备选」(它一上来就是主角),进入循环前先给它也打一个 hi ——
|
|
116
|
+
// 此时 ctx.model 还是本机刚失败的那个模型,解析目标与后面轮到时同源。
|
|
117
|
+
primeAt(ipv4Chain[0] || null);
|
|
118
|
+
|
|
90
119
|
for (;;) {
|
|
91
120
|
if (deadline > 0 && Date.now() >= deadline) {
|
|
92
121
|
const remaining = ipv4Chain.length + ipv6Chain.length;
|
|
@@ -110,12 +139,33 @@ export async function serialTryPeers({ candidates, v6Candidates = [], ctx, tried
|
|
|
110
139
|
}
|
|
111
140
|
if (!pending) break;
|
|
112
141
|
|
|
113
|
-
const prepared = await
|
|
142
|
+
const prepared = await resolveOnce(pending);
|
|
114
143
|
if (!prepared) continue;
|
|
115
144
|
const { peer, target } = prepared;
|
|
116
145
|
tried.add(peer.url);
|
|
117
146
|
const ctrl = new AbortController();
|
|
118
147
|
const name = peerName(peer);
|
|
148
|
+
|
|
149
|
+
// 轮到它了:再打一个 hi。回不来就当这个备选不行,直接换下一个。
|
|
150
|
+
if (typeof probePeer === "function") {
|
|
151
|
+
const t = performance.now();
|
|
152
|
+
let r;
|
|
153
|
+
try { r = await probePeer(peer, target); } catch { r = { ok: false }; }
|
|
154
|
+
const probeMs = Math.round(performance.now() - t);
|
|
155
|
+
if (r && r.ok === false) {
|
|
156
|
+
ctx.evt("peer-probe", { peer: peer.url, name, model: target, phase: "turn", ok: false, status: r.status ?? null, latencyMs: probeMs, action: "skip" });
|
|
157
|
+
failed.push({ ok: false, peer, target, res: null, status: r.status || 502 });
|
|
158
|
+
recordLater(() => ctx.peers.recordError(peer.url, { status: r.status || 502 }));
|
|
159
|
+
recordLater(() => ctx.peers.recordResult(peer.url, { ok: false }));
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
ctx.evt("peer-probe", { peer: peer.url, name, model: target, phase: "turn", ok: true, status: r?.status ?? null, latencyMs: probeMs });
|
|
163
|
+
if (deadline > 0 && Date.now() >= deadline) {
|
|
164
|
+
ctx.evt("peer-budget-out", { peer: null, reason: `peer 阶段总预算 ${budgetMs}ms 用尽(探针后)`, budgetMs, remaining: ipv4Chain.length + ipv6Chain.length });
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
119
169
|
ctx.evt("peer-request", { peer: peer.url, name, model: target, hops: ctx.hops + 1, payload: summarizeRequest(ctx.body), serial: 1, stage });
|
|
120
170
|
if (ctx.plugins?.length) {
|
|
121
171
|
runHook(ctx.plugins, "peer:beforeForward", { reqId: ctx.reqId, peer: peer.url, model: target, hops: ctx.hops + 1 }).catch(() => {});
|
package/src/routes/peers.js
CHANGED
|
@@ -5,6 +5,7 @@ import { runHook } from "../plugins.js";
|
|
|
5
5
|
import { buildShareKeysHeader, SHARE_KEYS_HEADER } from "../providers/share-keys.js";
|
|
6
6
|
import { compatFetch, timeoutSignal, getUndici } from "../compat.js";
|
|
7
7
|
import { serialTryPeers, PEER_SERIAL } from "./peer-serial.js";
|
|
8
|
+
import { probePeer, peerProbeTimeoutMs } from "./peer-probe.js";
|
|
8
9
|
import { overForwardLimit, contextGuardMessage } from "../chat-pipeline/context-guard.js";
|
|
9
10
|
|
|
10
11
|
// peer 阶段总预算:三批候选(ordered / orderedByLastError / coolingByLastError)共享同一个
|
|
@@ -252,6 +253,13 @@ export async function racePeerCandidates(candidates, ctx) {
|
|
|
252
253
|
perAttemptCap: peerAttemptTimeoutMs(), deadline, budgetMs,
|
|
253
254
|
// 连接预热:对「下一个备选」提前打 /health,复用 chat 的同一连接池(见 warmPeerHealth 注释)
|
|
254
255
|
warmPeer: (peer) => { if (peer?.url) warmPeerHealth(peer.url, { timeoutMs: 1500 }).catch(() => {}); },
|
|
256
|
+
// 用户 2026-10-20:备选时打一个 hi、轮到它时再打一个 hi,hi 回不来就换下一个。
|
|
257
|
+
// 走 getPeerDispatcher 的**同一条连接池**(与 chat 转发复用 socket,裸 fetch 热的是另一条等于白热)。
|
|
258
|
+
probePeer: (peer, model) => probePeer(peer, {
|
|
259
|
+
model: model || null,
|
|
260
|
+
timeoutMs: peerProbeTimeoutMs(),
|
|
261
|
+
dispatcher: getPeerDispatcher(),
|
|
262
|
+
}),
|
|
255
263
|
},
|
|
256
264
|
});
|
|
257
265
|
}
|
package/src/runtime/keep-warm.js
CHANGED
|
@@ -1,120 +1,135 @@
|
|
|
1
|
-
// 保热心跳(用户 2026-10-18 拍板):每 ≤120s 主动访问一次「每个组员 + 每个在 picks 的上游供应商」,
|
|
2
|
-
// 保持连接热度,防「久未使用 → 首发 fetch failed / 上游冷启动 26s」。
|
|
3
|
-
//
|
|
4
|
-
// ⚠ 必须用**与 chat 完全相同的那条连接池**,否则白热(用户 2026-10-18 纠正):
|
|
5
|
-
// - 组员 → warmPeerHealth()(getPeerDispatcher,正是 peer 转发复用的池);
|
|
6
|
-
// - 供应商 → provider.preheat()(各家 preheat 经各自 agent/extDispatcher 打自己的 models 端点,
|
|
7
|
-
// 与 chat 同池:opencode 走 v6/warp 池、generic/custom 走各自 agent)。
|
|
8
|
-
// 绝不能裸 compatFetch / directProbe——那热的是全局池的 socket,真流量复用的是另一条,等于没热。
|
|
9
|
-
//
|
|
10
|
-
// 为什么需要它(现网取证 2026-10-18 16:16,req mvdjwol5-aysc):三个组员同时 `fetch failed`(TCP 都没建立),
|
|
11
|
-
// 根因是两端 keep-alive 打架——服务端 Node 默认 5s 掐闲置连接,而组长 undici 池账面 30s 内以为 socket 还活着,
|
|
12
|
-
// 复用死 socket 即失败。已把服务端 keepAliveTimeout 提到 65s(server.js)治本;本心跳是**补强**:
|
|
13
|
-
// ① 间隔 120s < 家里 NAT/防火墙更短的空闲回收窗(很多路由 60~120s 就丢映射,单靠 65s keep-alive 挡不住);
|
|
14
|
-
// ② 顺带让上游对我们出口保持"温热"(opencode 实测固定 session 1.2s、冷机器 26s+)。
|
|
15
|
-
//
|
|
16
|
-
// 零额度消耗(全部经各家 preheat:GET models / GET /health,不打 chat)。永不抛,只记事件。
|
|
17
|
-
import { warmPeerHealth } from "../routes/peers.js";
|
|
18
|
-
|
|
19
|
-
export function keepWarmIntervalMs(env = process.env) {
|
|
20
|
-
const n = Number(env.MSLXDFF_KEEP_WARM_MS);
|
|
21
|
-
return Number.isInteger(n) && n >= 0 ? n : 120_000; // 0 = 关闭
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* 从 picks 里算出要保热的「供应商头」集合:裸 id(无 `/`)归 opencode(免费池);带前缀取首段。
|
|
26
|
-
* 只保留出现过的供应商,天然覆盖"每个在 picks 的接口"。opencode 恒排第一(免费池是主力、最该保热)。
|
|
27
|
-
* @param {string[]} picks
|
|
28
|
-
* @returns {string[]} 供应商 id 列表(含 "opencode"),已去重、opencode 置顶
|
|
29
|
-
*/
|
|
30
|
-
export function providerHeadsFromPicks(picks = []) {
|
|
31
|
-
const heads = new Set();
|
|
32
|
-
for (const id of picks) {
|
|
33
|
-
const s = String(id || "").trim();
|
|
34
|
-
if (!s) continue;
|
|
35
|
-
heads.add(s.includes("/") ? s.slice(0, s.indexOf("/")) : "opencode");
|
|
36
|
-
}
|
|
37
|
-
const out = [...heads];
|
|
38
|
-
if (out.includes("opencode")) return ["opencode", ...out.filter((h) => h !== "opencode")];
|
|
39
|
-
return out;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* 单轮保热:组员各 warmPeerHealth、picks 供应商各 preheat。返回汇总(供事件取证)。绝不抛。
|
|
44
|
-
* @param {{peers:{url:string}[], picks:string[], byId?:Map<string,{preheat?:Function}>, upstream?:{preheat?:Function},
|
|
45
|
-
* warmPeer?:Function, evt?:Function}} deps
|
|
46
|
-
*/
|
|
47
|
-
export async function warmOnce({
|
|
48
|
-
peers = [],
|
|
49
|
-
picks = [],
|
|
50
|
-
byId = null,
|
|
51
|
-
upstream = null,
|
|
52
|
-
warmPeer = warmPeerHealth,
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
try {
|
|
59
|
-
const
|
|
60
|
-
if (
|
|
61
|
-
} catch {
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
peers,
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
1
|
+
// 保热心跳(用户 2026-10-18 拍板):每 ≤120s 主动访问一次「每个组员 + 每个在 picks 的上游供应商」,
|
|
2
|
+
// 保持连接热度,防「久未使用 → 首发 fetch failed / 上游冷启动 26s」。
|
|
3
|
+
//
|
|
4
|
+
// ⚠ 必须用**与 chat 完全相同的那条连接池**,否则白热(用户 2026-10-18 纠正):
|
|
5
|
+
// - 组员 → warmPeerHealth()(getPeerDispatcher,正是 peer 转发复用的池);
|
|
6
|
+
// - 供应商 → provider.preheat()(各家 preheat 经各自 agent/extDispatcher 打自己的 models 端点,
|
|
7
|
+
// 与 chat 同池:opencode 走 v6/warp 池、generic/custom 走各自 agent)。
|
|
8
|
+
// 绝不能裸 compatFetch / directProbe——那热的是全局池的 socket,真流量复用的是另一条,等于没热。
|
|
9
|
+
//
|
|
10
|
+
// 为什么需要它(现网取证 2026-10-18 16:16,req mvdjwol5-aysc):三个组员同时 `fetch failed`(TCP 都没建立),
|
|
11
|
+
// 根因是两端 keep-alive 打架——服务端 Node 默认 5s 掐闲置连接,而组长 undici 池账面 30s 内以为 socket 还活着,
|
|
12
|
+
// 复用死 socket 即失败。已把服务端 keepAliveTimeout 提到 65s(server.js)治本;本心跳是**补强**:
|
|
13
|
+
// ① 间隔 120s < 家里 NAT/防火墙更短的空闲回收窗(很多路由 60~120s 就丢映射,单靠 65s keep-alive 挡不住);
|
|
14
|
+
// ② 顺带让上游对我们出口保持"温热"(opencode 实测固定 session 1.2s、冷机器 26s+)。
|
|
15
|
+
//
|
|
16
|
+
// 零额度消耗(全部经各家 preheat:GET models / GET /health,不打 chat)。永不抛,只记事件。
|
|
17
|
+
import { warmPeerHealth } from "../routes/peers.js";
|
|
18
|
+
|
|
19
|
+
export function keepWarmIntervalMs(env = process.env) {
|
|
20
|
+
const n = Number(env.MSLXDFF_KEEP_WARM_MS);
|
|
21
|
+
return Number.isInteger(n) && n >= 0 ? n : 120_000; // 0 = 关闭
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* 从 picks 里算出要保热的「供应商头」集合:裸 id(无 `/`)归 opencode(免费池);带前缀取首段。
|
|
26
|
+
* 只保留出现过的供应商,天然覆盖"每个在 picks 的接口"。opencode 恒排第一(免费池是主力、最该保热)。
|
|
27
|
+
* @param {string[]} picks
|
|
28
|
+
* @returns {string[]} 供应商 id 列表(含 "opencode"),已去重、opencode 置顶
|
|
29
|
+
*/
|
|
30
|
+
export function providerHeadsFromPicks(picks = []) {
|
|
31
|
+
const heads = new Set();
|
|
32
|
+
for (const id of picks) {
|
|
33
|
+
const s = String(id || "").trim();
|
|
34
|
+
if (!s) continue;
|
|
35
|
+
heads.add(s.includes("/") ? s.slice(0, s.indexOf("/")) : "opencode");
|
|
36
|
+
}
|
|
37
|
+
const out = [...heads];
|
|
38
|
+
if (out.includes("opencode")) return ["opencode", ...out.filter((h) => h !== "opencode")];
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* 单轮保热:组员各 warmPeerHealth、picks 供应商各 preheat。返回汇总(供事件取证)。绝不抛。
|
|
44
|
+
* @param {{peers:{url:string}[], picks:string[], byId?:Map<string,{preheat?:Function}>, upstream?:{preheat?:Function},
|
|
45
|
+
* warmPeer?:Function, evt?:Function}} deps
|
|
46
|
+
*/
|
|
47
|
+
export async function warmOnce({
|
|
48
|
+
peers = [],
|
|
49
|
+
picks = [],
|
|
50
|
+
byId = null,
|
|
51
|
+
upstream = null,
|
|
52
|
+
warmPeer = warmPeerHealth,
|
|
53
|
+
markProbe = null, // (url, ok) => void:探测结果写进 peer 冷却账(health 探不通 ⇒ 10min 不当候选)
|
|
54
|
+
evt = () => {},
|
|
55
|
+
} = {}) {
|
|
56
|
+
const staticPeers = peers.filter((p) => p?.url && !String(p.url).startsWith("relay://")); // broadband 走 relay 队列,不直连
|
|
57
|
+
const safeMark = (url, ok) => {
|
|
58
|
+
try {
|
|
59
|
+
const p = markProbe?.(url, ok);
|
|
60
|
+
if (p && typeof p.catch === "function") p.catch(() => {});
|
|
61
|
+
} catch { /* 记账失败不影响保热 */ }
|
|
62
|
+
};
|
|
63
|
+
let peerOk = 0, peerFail = 0;
|
|
64
|
+
for (const p of staticPeers) {
|
|
65
|
+
try {
|
|
66
|
+
const r = await warmPeer(String(p.url));
|
|
67
|
+
if (r?.ok) { peerOk += 1; safeMark(String(p.url), true); }
|
|
68
|
+
else { peerFail += 1; safeMark(String(p.url), false); }
|
|
69
|
+
} catch { peerFail += 1; safeMark(String(p.url), false); }
|
|
70
|
+
}
|
|
71
|
+
const heads = providerHeadsFromPicks(picks);
|
|
72
|
+
let provOk = 0, provFail = 0, provSkip = 0;
|
|
73
|
+
for (const head of heads) {
|
|
74
|
+
try {
|
|
75
|
+
// 供应商预热一律走它自己的 preheat(chat 同池)。opencode 特判:**多 provider 装配**下
|
|
76
|
+
// providers 数组里含 opencode,直接 byId 取;单 provider 模式下 byId 为空,退回 upstream.preheat。
|
|
77
|
+
const inst = byId?.get?.(head);
|
|
78
|
+
const preheat = inst?.preheat || (head === "opencode" ? upstream?.preheat : null);
|
|
79
|
+
if (typeof preheat !== "function") { provSkip += 1; continue; } // 本地号型/无域名供应商:本就没有可保热的域
|
|
80
|
+
const r = await preheat.call(inst || upstream);
|
|
81
|
+
if (r?.ok) provOk += 1; else provFail += 1;
|
|
82
|
+
} catch { provFail += 1; }
|
|
83
|
+
}
|
|
84
|
+
evt("keep-warm", { peers: staticPeers.length, peerOk, peerFail, providers: heads.length, provOk, provFail, provSkip });
|
|
85
|
+
return { peers: staticPeers.length, peerOk, peerFail, providers: heads.length, provOk, provFail, provSkip };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* daemon 装配:起一个 unref 的定时器,每 keepWarmIntervalMs 跑一轮 warmOnce。
|
|
90
|
+
* 关(=0)返回 null。依赖注入以便测试;生产用 setupProviders 的 ctx(providers/upstream/peers)。
|
|
91
|
+
*/
|
|
92
|
+
export function startKeepWarm({
|
|
93
|
+
peers,
|
|
94
|
+
providers = [],
|
|
95
|
+
upstream,
|
|
96
|
+
env = process.env,
|
|
97
|
+
evt = () => {},
|
|
98
|
+
logs,
|
|
99
|
+
loadModelPicks,
|
|
100
|
+
warmPeer = warmPeerHealth,
|
|
101
|
+
} = {}) {
|
|
102
|
+
const interval = keepWarmIntervalMs(env);
|
|
103
|
+
if (interval === 0) return null;
|
|
104
|
+
const byId = new Map((providers || []).map((p) => [p.id, p]));
|
|
105
|
+
const emit = (type, data) => {
|
|
106
|
+
const entry = { ts: Date.now(), type, ...data };
|
|
107
|
+
try { evt(type, data); } catch {}
|
|
108
|
+
try { logs?.appendEvent?.(entry); } catch {}
|
|
109
|
+
};
|
|
110
|
+
const runTick = async () => {
|
|
111
|
+
try {
|
|
112
|
+
const picks = loadModelPicks ? loadModelPicks() : [];
|
|
113
|
+
await warmOnce({
|
|
114
|
+
peers: peers?.all?.() || [],
|
|
115
|
+
picks: Array.isArray(picks) ? picks : [],
|
|
116
|
+
byId,
|
|
117
|
+
upstream,
|
|
118
|
+
warmPeer,
|
|
119
|
+
// 探测结果写进冷却账:health 连不上 ⇒ 10min 不当候选(防"探到了但选路不知道"——现网 2026-10-20 13:27)
|
|
120
|
+
markProbe: async (url, ok) => {
|
|
121
|
+
try {
|
|
122
|
+
if (ok) await peers?.recordProbeOk?.(url);
|
|
123
|
+
else await peers?.recordProbeDead?.(url);
|
|
124
|
+
} catch { /* 记账失败不影响保热 */ }
|
|
125
|
+
},
|
|
126
|
+
evt: emit,
|
|
127
|
+
});
|
|
128
|
+
} catch { /* 保热永不影响主链路 */ }
|
|
129
|
+
};
|
|
130
|
+
console.log(`[keep-warm] enabled: every ${Math.round(interval / 1000)}s 经 chat 同池访问每个组员(/health) + 每个 picks 供应商(preheat) 保热,设 MSLXDFF_KEEP_WARM_MS=0 关`);
|
|
131
|
+
void runTick(); // 启动即热一次,别等第一个间隔
|
|
132
|
+
const timer = setInterval(() => { void runTick(); }, interval);
|
|
133
|
+
timer.unref?.();
|
|
134
|
+
return { stop: () => clearInterval(timer), interval, tick: runTick };
|
|
135
|
+
}
|