mslxdff 0.1.231 → 0.1.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mslxdff.js +3 -3
- package/package.json +1 -1
- package/src/auto.js +254 -254
- package/src/bench/cline-bench.js +42 -42
- package/src/bench/probe.js +70 -70
- package/src/bench/report.js +162 -162
- package/src/bench/runner.js +77 -77
- package/src/bench/via-probe.js +124 -124
- package/src/bench/via-routes.js +87 -87
- package/src/bench/workbuddy-bench.js +54 -54
- package/src/chat/engine.js +160 -160
- package/src/chat/gateway.js +163 -163
- package/src/chat/orchestrator.js +234 -234
- package/src/chat/prompt.js +70 -70
- package/src/chat/repl.js +88 -88
- package/src/chat/terminal.js +135 -135
- package/src/chat/tools.js +306 -306
- package/src/chat-pipeline/index.js +145 -145
- package/src/chat-pipeline/policy.js +84 -84
- package/src/cli/commands/group.js +266 -266
- package/src/cli/commands/model/picks.js +50 -50
- package/src/cli/commands/provider/bench-via.js +247 -247
- package/src/cli/commands/provider/bench.js +141 -141
- package/src/cli/commands/provider/index.js +139 -139
- package/src/cli/commands/provider/models.js +149 -149
- package/src/cli/commands/provider/qwenwork-login.js +119 -119
- package/src/cli/commands/provider/raccoon-quota.js +156 -156
- package/src/cli/commands/sync.js +310 -310
- package/src/cli/status.js +279 -279
- package/src/daemon.js +246 -246
- package/src/model-capabilities/enrich.js +86 -86
- package/src/model-capabilities/index.js +183 -183
- package/src/model-capabilities/parse.js +70 -70
- package/src/models.js +225 -225
- package/src/peers.js +345 -345
- package/src/providers/agentrouter.js +238 -238
- package/src/providers/base.js +164 -164
- package/src/providers/cline/auth.js +228 -228
- package/src/providers/cline/chat.js +307 -307
- package/src/providers/cline.js +2 -2
- package/src/providers/codearts/chat.js +207 -207
- package/src/providers/dispatcher.js +147 -147
- package/src/providers/globalqwenwork/cosy.js +123 -123
- package/src/providers/globalqwenwork/crypto.js +218 -218
- package/src/providers/globalqwenwork/http.js +20 -20
- package/src/providers/globalqwenwork/rsa.js +54 -54
- package/src/providers/globalqwenwork/sse.js +268 -268
- package/src/providers/globalqwenwork/stream.js +130 -130
- package/src/providers/globalqwenwork/upstream.js +122 -122
- package/src/providers/keyring.js +60 -60
- package/src/providers/qoder/chat.js +203 -203
- package/src/providers/qoder/index.js +249 -249
- package/src/providers/qoder/sse.js +145 -145
- package/src/providers/qwenwork/account-store.js +133 -133
- package/src/providers/qwenwork/constants.js +67 -67
- package/src/providers/qwenwork/cosy.js +120 -120
- package/src/providers/qwenwork/crypto.js +218 -218
- package/src/providers/qwenwork/http.js +20 -20
- package/src/providers/qwenwork/index.js +337 -337
- package/src/providers/qwenwork/payload.js +144 -144
- package/src/providers/qwenwork/rsa.js +54 -54
- package/src/providers/qwenwork/sse.js +268 -268
- package/src/providers/qwenwork/stream.js +130 -130
- package/src/providers/qwenwork/upstream.js +120 -120
- package/src/providers/qwenwork.js +1 -1
- package/src/providers/raccoon/const.js +202 -202
- package/src/providers/raccoon/credits.js +285 -285
- package/src/providers/registry.js +89 -89
- package/src/providers/workbuddy/auth.js +16 -0
- package/src/providers/workbuddy/chat.js +298 -264
- package/src/providers/workbuddy/client-headers.js +148 -0
- package/src/providers/workbuddy/desktop-info.js +17 -1
- package/src/providers/workbuddy/models.js +8 -20
- package/src/providers/workbuddy/reshape.js +152 -152
- package/src/providers/workbuddy.js +2 -2
- package/src/reasoning.js +32 -32
- package/src/routes/chat/broadband-handler.js +80 -80
- package/src/routes/chat/gateway.js +46 -46
- package/src/routes/chat/relay-pipeline.js +282 -282
- package/src/routes/chat/via-route-handler.js +156 -156
- package/src/routes/hedge.js +260 -260
- package/src/routes/models-route.js +167 -167
- package/src/routes/peers.js +373 -373
- package/src/routes/stream.js +347 -347
- package/src/runtime/bootstrap.js +75 -75
- package/src/runtime/keep-warm.js +135 -135
- package/src/runtime/provider-gate.js +75 -75
- package/src/runtime/providers-setup.js +187 -187
- package/src/runtime/raccoon-checkin.js +175 -175
- package/src/server.js +85 -85
- package/src/state/schemas/allowlist.js +92 -92
- package/src/sync-opencode.js +280 -280
- package/src/transport/index.js +244 -244
- package/src/transport/pool.js +56 -56
- package/src/transport/retry.js +24 -24
- package/src/transport/sse.js +93 -93
- package/src/upstream-probe/display.js +52 -52
- package/src/upstream-probe/probe.js +49 -49
- package/src/upstream-probe/rotate.js +110 -110
- package/src/upstream-probe/start.js +45 -45
- package/src/upstream.js +355 -355
- package/docs/cli_help_mini.md +0 -148
- package/src/providers/AGENTS.md +0 -49
- package/src/routes/AGENTS.md +0 -37
package/src/peers.js
CHANGED
|
@@ -1,345 +1,345 @@
|
|
|
1
|
-
import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
|
|
2
|
-
|
|
3
|
-
// 组员失败冷却分级(2026-10-18 用户拍板):
|
|
4
|
-
// 非额度失败(502/5xx/超时,可能几秒就恢复)→ 短冷却 10s,到点自动重新纳入候选、继续挨个重试;
|
|
5
|
-
// 额度类(连续 2 次 429)→ 长冷却 10min(额度当天不会因快重试恢复,反复撞只会白烧配额)。
|
|
6
|
-
export const DEFAULT_PEER_COOLDOWN_MS = 10_000;
|
|
7
|
-
export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 10 * 60_000;
|
|
8
|
-
export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
|
|
9
|
-
export const DEFAULT_MAX_HOPS = 3;
|
|
10
|
-
export const DEFAULT_BROADBAND_STALE_MS = 90_000;
|
|
11
|
-
// 连续几次连接级失败即视为"这个地址没服务",进长冷却(不再靠 30s 短冷却反复捞回)。
|
|
12
|
-
export const DEAD_FAIL_THRESHOLD = 2;
|
|
13
|
-
// 探测失败(GET /health 连不上)视为"这个节点当前不可用",10 分钟内不再当候选。
|
|
14
|
-
// 与请求失败账分开:health 探测是**主动、零额度**的可用性判断,比"这一发不顺"更可信——
|
|
15
|
-
// 现网 2026-10-20 13:27 req mvg8r7cz-b8qn:152.67.222.233:62028 早已 health timeout,
|
|
16
|
-
// CLI/保热都探到了,但选路不知道,仍把它排进候选并吃满 20s 单次上限(总 27s)。
|
|
17
|
-
export const DEFAULT_PEER_PROBE_DEAD_MS = 10 * 60_000;
|
|
18
|
-
|
|
19
|
-
export function peerProbeDeadMs() {
|
|
20
|
-
const n = Number(process.env.MSLXDFF_PEER_PROBE_DEAD_MS);
|
|
21
|
-
return Number.isInteger(n) && n >= 0 ? n : DEFAULT_PEER_PROBE_DEAD_MS;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const EMA_ALPHA = 0.3;
|
|
25
|
-
|
|
26
|
-
export function broadbandStaleMs() {
|
|
27
|
-
const n = Number(process.env.MSLXDFF_BROADBAND_STALE_MS);
|
|
28
|
-
return Number.isInteger(n) && n > 0 ? n : DEFAULT_BROADBAND_STALE_MS;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export function isBroadbandStale(peer, nowMs = Date.now(), staleMs = broadbandStaleMs()) {
|
|
32
|
-
const kind = peer?.kind || (String(peer?.url || "").startsWith("relay://") ? "broadband" : "static");
|
|
33
|
-
if (kind !== "broadband") return false;
|
|
34
|
-
const lastSeen = peer?.lastSeen;
|
|
35
|
-
if (typeof lastSeen !== "number") return true; // no heartbeat yet => stale
|
|
36
|
-
return nowMs - lastSeen > staleMs;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export function normalizePeerUrl(url) {
|
|
40
|
-
return String(url || "").trim().replace(/\/+$/, "");
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
export function createPeersService({
|
|
44
|
-
file,
|
|
45
|
-
now = () => Date.now(),
|
|
46
|
-
cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
|
|
47
|
-
limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
|
|
48
|
-
heatMs = DEFAULT_PEER_HEAT_MS,
|
|
49
|
-
peers: seedPeers,
|
|
50
|
-
errors: seedErrors,
|
|
51
|
-
stats: seedStats,
|
|
52
|
-
persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
|
|
53
|
-
persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
|
|
54
|
-
persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
|
|
55
|
-
} = {}) {
|
|
56
|
-
const list = (seedPeers ?? loadPeers(file ? { file } : {}))
|
|
57
|
-
.map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
|
|
58
|
-
.filter((p) => p && p.url);
|
|
59
|
-
const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
|
|
60
|
-
const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
|
|
61
|
-
|
|
62
|
-
function all() {
|
|
63
|
-
return [...list];
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
function add(peer) {
|
|
67
|
-
const url = normalizePeerUrl(peer?.url);
|
|
68
|
-
if (!url) return false;
|
|
69
|
-
const existing = list.find((p) => p.url === url);
|
|
70
|
-
const entry = { ...peer, url };
|
|
71
|
-
if (existing) Object.assign(existing, entry);
|
|
72
|
-
else list.push(entry);
|
|
73
|
-
persistPeers([...list]);
|
|
74
|
-
return true;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
function remove(url) {
|
|
78
|
-
const target = normalizePeerUrl(url);
|
|
79
|
-
const idx = list.findIndex((p) => p.url === target);
|
|
80
|
-
if (idx < 0) return false;
|
|
81
|
-
list.splice(idx, 1);
|
|
82
|
-
persistPeers([...list]);
|
|
83
|
-
return true;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function removeByGroup(group) {
|
|
87
|
-
const before = list.length;
|
|
88
|
-
for (let i = list.length - 1; i >= 0; i--) {
|
|
89
|
-
if (list[i].group === group) list.splice(i, 1);
|
|
90
|
-
}
|
|
91
|
-
if (list.length !== before) persistPeers([...list]);
|
|
92
|
-
return before - list.length;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// 分级冷却:连续 429(streak >= 2)→ limitCooldownMs(限流恢复慢,5min 内不再考虑);
|
|
96
|
-
// 连接级永久失败(ECONNREFUSED/ENOTFOUND,deadFails >= 2)同样进长冷却;
|
|
97
|
-
// 其他失败(网络抖动、5xx)→ cooldownMs(可能几秒恢复,给快速重试机会)。
|
|
98
|
-
function coolingWindowMs(url) {
|
|
99
|
-
// 探测死亡(health 连不上):10min 内不再考虑——比请求失败更可信的"没服务"证据
|
|
100
|
-
if (stats[url]?.probeDeadAt !== undefined && now() - stats[url].probeDeadAt < peerProbeDeadMs()) return limitCooldownMs;
|
|
101
|
-
if ((stats[url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return limitCooldownMs;
|
|
102
|
-
const streak = stats[url]?.streak429 ?? 0;
|
|
103
|
-
return streak >= 2 ? limitCooldownMs : cooldownMs;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// 连接级失败(端口没服务/DNS 解析不了)与"这一发不顺"是两回事:前者换多少次都一样,
|
|
107
|
-
// 现网 node-b(127.0.0.1:3500,本机根本没服务)靠 30s 短冷却 + 兜底,被反复竞速 1569 次。
|
|
108
|
-
function isConnLevelFailure(message) {
|
|
109
|
-
const s = String(message || "");
|
|
110
|
-
return /ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ECONNRESET|fetch failed|connect (?:timeout|ETIMEDOUT)|socket hang up/i.test(s);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function isCooling(url) {
|
|
114
|
-
// 探测死亡(health 探不通):10min 内不再当候选——比请求失败更可信的"没服务"证据
|
|
115
|
-
const pDead = stats[url]?.probeDeadAt;
|
|
116
|
-
if (pDead !== undefined && now() - pDead < peerProbeDeadMs()) return true;
|
|
117
|
-
const win = coolingWindowMs(url);
|
|
118
|
-
if (!win) return false;
|
|
119
|
-
const err = lastErrorAt[url];
|
|
120
|
-
if (typeof err === "number" && now() - err < win) return true;
|
|
121
|
-
const peer = list.find((p) => p.url === url);
|
|
122
|
-
if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
|
|
123
|
-
return false;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function isBroadbandCooling(url) {
|
|
127
|
-
const peer = list.find((p) => p.url === url);
|
|
128
|
-
if (!peer) return false;
|
|
129
|
-
return isBroadbandStale(peer, now(), broadbandStaleMs());
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
function available() {
|
|
133
|
-
return list.filter((p) => !isCooling(p.url));
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// A peer counts as "hot" when it succeeded recently: reuse its last model
|
|
137
|
-
// without a fresh health probe to keep the fast path fast.
|
|
138
|
-
function isHot(url, t = now()) {
|
|
139
|
-
if (isCooling(url)) return false;
|
|
140
|
-
const s = stats[url];
|
|
141
|
-
return Boolean(s?.okAt) && t - s.okAt < heatMs;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function stat(url) {
|
|
145
|
-
return stats[url] ? { ...stats[url] } : null;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
function rankScore(p, t) {
|
|
149
|
-
const s = stats[p.url];
|
|
150
|
-
const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
|
|
151
|
-
const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
|
|
152
|
-
const fails = s?.fails ?? 0;
|
|
153
|
-
return hot * 1_000_000_000 + latency * 1_000 + fails;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// 可用组员排序(用户 2026-10-19 决定):**延迟最小优先**,不再「热优先」。
|
|
157
|
-
// 语义 = 「组员的额度用完后,用跟组长延迟最小的组员」——延迟是首要判据。
|
|
158
|
-
// 延迟来源(优先级从高到低):
|
|
159
|
-
// ① stats 里的真实请求 EMA(recordResult 记的 latencyMs)——最可信;
|
|
160
|
-
// ② peer.heartbeatLatencyMs —— 组员**定时心跳**上报的 rtt(没跑过真实请求时也能排序);
|
|
161
|
-
// 未测过延迟的排在有数据之后;同延迟按「最早失败优先」;都没失败过的排最后。
|
|
162
|
-
// 组长识别:本机是组员时,peers 里有 name==="leader" 的记录(cli/group-test.js 同口径)。
|
|
163
|
-
// 组长自己节点视角没有这条记录 → isLeaderPeer 恒 false,退化为纯延迟排序(正是想要的行为)。
|
|
164
|
-
function isLeaderPeer(p) {
|
|
165
|
-
return String(p?.name || "").toLowerCase() === "leader";
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
function peerLatency(p) {
|
|
169
|
-
const s = stats[p.url]?.latencyMs;
|
|
170
|
-
if (s != null) return s;
|
|
171
|
-
const hb = list.find((x) => x.url === p.url)?.heartbeatLatencyMs;
|
|
172
|
-
return Number.isFinite(Number(hb)) ? Number(hb) : null;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// 组员延迟升序(组长不参与),供两处复用:ordered 的第二档 + standbyChain。
|
|
176
|
-
function byLatencyAsc(a, b) {
|
|
177
|
-
const la = peerLatency(a);
|
|
178
|
-
const lb = peerLatency(b);
|
|
179
|
-
const ha = la != null, hb = lb != null;
|
|
180
|
-
if (ha !== hb) return ha ? -1 : 1; // 有延迟数据的优先
|
|
181
|
-
if (ha && hb && la !== lb) return la - lb; // 延迟小的先上
|
|
182
|
-
const ea = lastErrorAt[a.url];
|
|
183
|
-
const eb = lastErrorAt[b.url];
|
|
184
|
-
if (ea == null && eb == null) return 0;
|
|
185
|
-
if (ea == null) return 1;
|
|
186
|
-
if (eb == null) return -1;
|
|
187
|
-
return ea - eb;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
// 用户 2026-10-20 约定:**组长恒定第一**(先榨干组长额度),组长用完再按延迟升序上组员。
|
|
191
|
-
// 之前是纯延迟排序,「9/9 走组长」只是组长恰好延迟最低的巧合(AGENTS.md 已记差距)。
|
|
192
|
-
function ordered(t = now()) {
|
|
193
|
-
return available().sort((a, b) => {
|
|
194
|
-
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
195
|
-
if (la !== lb) return la ? -1 : 1; // 组长恒排第一
|
|
196
|
-
return byLatencyAsc(a, b);
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* 常驻接力链(用户 2026-10-20:永远准备一个备选,429 立即切)。
|
|
202
|
-
* IPv4 阶段:组长 → 组员(延迟升序);IPv4 全用完才进 IPv6 阶段:先本机/组长 v6,
|
|
203
|
-
* 再按延迟升序找 hasV6 的组员。
|
|
204
|
-
* @returns {{ipv4: object[], ipv6: object[]}} 两段链,各自已排序
|
|
205
|
-
*/
|
|
206
|
-
function standbyChain(t = now()) {
|
|
207
|
-
const ipv4 = available().sort((a, b) => {
|
|
208
|
-
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
209
|
-
if (la !== lb) return la ? -1 : 1;
|
|
210
|
-
return byLatencyAsc(a, b);
|
|
211
|
-
});
|
|
212
|
-
// IPv6 阶段:只保留心跳自报 hasV6 的节点,同样组长优先、其余延迟升序
|
|
213
|
-
const ipv6 = ipv4.filter((p) => p.hasV6 === true);
|
|
214
|
-
return { ipv4, ipv6 };
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/** 下一个备选:从链里取第一个还没试过的。这就是「永远准备好的那个」。 */
|
|
218
|
-
function nextStandby(tried, t = now()) {
|
|
219
|
-
const { ipv4, ipv6 } = standbyChain(t);
|
|
220
|
-
const skip = tried instanceof Set ? tried : new Set(tried || []);
|
|
221
|
-
return {
|
|
222
|
-
ipv4: ipv4.find((p) => !skip.has(p.url)) || null,
|
|
223
|
-
ipv6: ipv6.find((p) => !skip.has(p.url)) || null,
|
|
224
|
-
};
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Available peers ordered specifically for recovery-time retry: earliest
|
|
228
|
-
// failure first (it has had the longest to come back), never-failed last.
|
|
229
|
-
function orderedByLastError(t = now()) {
|
|
230
|
-
return available().sort((a, b) => {
|
|
231
|
-
const ea = lastErrorAt[a.url];
|
|
232
|
-
const eb = lastErrorAt[b.url];
|
|
233
|
-
if (ea == null && eb == null) return 0;
|
|
234
|
-
if (ea == null) return 1; // never-failed peers go after any that failed
|
|
235
|
-
if (eb == null) return -1;
|
|
236
|
-
return ea - eb; // earliest failure first
|
|
237
|
-
});
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
let cursor = 0;
|
|
241
|
-
|
|
242
|
-
// 兜底重试集合:仅"因错误冷却中"的 peer(按最早失败优先)。
|
|
243
|
-
// 场景:唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时,兜底给它一次机会;
|
|
244
|
-
// 成功会被 recordResult 清除错误记录、回归热路径,避免"明明能用的节点被 30s 冷却全灭"。
|
|
245
|
-
function coolingByLastError() {
|
|
246
|
-
const t = now();
|
|
247
|
-
return list
|
|
248
|
-
.filter((p) => {
|
|
249
|
-
const err = lastErrorAt[p.url];
|
|
250
|
-
if (typeof err !== "number") return false;
|
|
251
|
-
if (stats[p.url]?.probeDeadAt !== undefined && now() - stats[p.url].probeDeadAt < peerProbeDeadMs()) return false; // 探测死亡:兜底也不考虑(否则每次白等一次拒连)
|
|
252
|
-
if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却:兜底也不考虑
|
|
253
|
-
if ((stats[p.url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return false; // 连接级永久失败:兜底也不考虑(否则每次白等一次拒连)
|
|
254
|
-
return t - err < cooldownMs;
|
|
255
|
-
})
|
|
256
|
-
.sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
function next() {
|
|
260
|
-
const avail = available();
|
|
261
|
-
if (!avail.length) return null;
|
|
262
|
-
cursor = cursor % avail.length;
|
|
263
|
-
return avail[cursor++];
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
// Long-lived error memory: a peer keeps its last-error timestamp until a
|
|
267
|
-
// subsequent success resets it (success clears the failure record) or the
|
|
268
|
-
// error is no longer in the persist store on next load.
|
|
269
|
-
async function recordError(url, { status, message } = {}) {
|
|
270
|
-
if (!url) return;
|
|
271
|
-
lastErrorAt[url] = now();
|
|
272
|
-
await persistErrors({ ...lastErrorAt });
|
|
273
|
-
// 连续 429 计数(仅成功清零):第 2 次起进入长冷却——该 peer 大概率真被限流,不再频繁试它
|
|
274
|
-
if (Number(status) === 429) {
|
|
275
|
-
const prev = stats[url] || {};
|
|
276
|
-
stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
|
|
277
|
-
await persistStats({ ...stats });
|
|
278
|
-
}
|
|
279
|
-
// 连接级失败计数(仅成功清零):第 2 次起进长冷却,且不再被 coolingByLastError 兜底捞回
|
|
280
|
-
if (isConnLevelFailure(message)) {
|
|
281
|
-
const prev = stats[url] || {};
|
|
282
|
-
stats[url] = { ...prev, deadFails: (prev.deadFails || 0) + 1 };
|
|
283
|
-
await persistStats({ ...stats });
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
// Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
|
|
288
|
-
// last successful model) and clears the error memory; failures keep the
|
|
289
|
-
// error timestamp so the retry pass can order by recovery time.
|
|
290
|
-
async function recordResult(url, { ok, latencyMs, model } = {}) {
|
|
291
|
-
if (!url) return;
|
|
292
|
-
if (ok) {
|
|
293
|
-
if (lastErrorAt[url] != null) {
|
|
294
|
-
delete lastErrorAt[url];
|
|
295
|
-
await persistErrors({ ...lastErrorAt });
|
|
296
|
-
}
|
|
297
|
-
const prev = stats[url] || {};
|
|
298
|
-
const next = {
|
|
299
|
-
okAt: now(),
|
|
300
|
-
latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
|
|
301
|
-
? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
|
|
302
|
-
: (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
|
|
303
|
-
fails: 0,
|
|
304
|
-
model: model || prev.model || "",
|
|
305
|
-
streak429: 0,
|
|
306
|
-
deadFails: 0,
|
|
307
|
-
};
|
|
308
|
-
// 探测死亡标记:请求成功 = 节点活着,冷却窗口立即解除。
|
|
309
|
-
// next 本身不含 probeDeadAt 键,天然清除;这里只兜底确保没有残留。
|
|
310
|
-
stats[url] = next;
|
|
311
|
-
delete stats[url].probeDeadAt;
|
|
312
|
-
} else {
|
|
313
|
-
const prev = stats[url] || {};
|
|
314
|
-
stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
|
|
315
|
-
}
|
|
316
|
-
await persistStats({ ...stats });
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
// 探测死亡标记(health 连不上):probeDeadAt 置为此刻,10min 内冷却。不反复探。
|
|
320
|
-
// recordProbeOk 在探测成功时清标记(节点回来了,冷却窗口立刻结束)。
|
|
321
|
-
async function recordProbeDead(url) {
|
|
322
|
-
if (!url) return;
|
|
323
|
-
const prev = stats[url] || {};
|
|
324
|
-
stats[url] = { ...prev, probeDeadAt: now() };
|
|
325
|
-
await persistStats({ ...stats });
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
async function recordProbeOk(url) {
|
|
329
|
-
if (!url) return;
|
|
330
|
-
const prev = stats[url] || {};
|
|
331
|
-
if (!("probeDeadAt" in prev)) return;
|
|
332
|
-
delete prev.probeDeadAt;
|
|
333
|
-
stats[url] = prev;
|
|
334
|
-
await persistStats({ ...stats });
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
return {
|
|
338
|
-
all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
|
|
339
|
-
const peer = list.find((p) => p.url === url);
|
|
340
|
-
return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
|
|
341
|
-
}, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
|
|
342
|
-
standbyChain, nextStandby, isLeaderPeer,
|
|
343
|
-
recordError, recordResult, recordProbeDead, recordProbeOk, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
|
|
344
|
-
};
|
|
345
|
-
}
|
|
1
|
+
import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
|
|
2
|
+
|
|
3
|
+
// 组员失败冷却分级(2026-10-18 用户拍板):
|
|
4
|
+
// 非额度失败(502/5xx/超时,可能几秒就恢复)→ 短冷却 10s,到点自动重新纳入候选、继续挨个重试;
|
|
5
|
+
// 额度类(连续 2 次 429)→ 长冷却 10min(额度当天不会因快重试恢复,反复撞只会白烧配额)。
|
|
6
|
+
export const DEFAULT_PEER_COOLDOWN_MS = 10_000;
|
|
7
|
+
export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 10 * 60_000;
|
|
8
|
+
export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
|
|
9
|
+
export const DEFAULT_MAX_HOPS = 3;
|
|
10
|
+
export const DEFAULT_BROADBAND_STALE_MS = 90_000;
|
|
11
|
+
// 连续几次连接级失败即视为"这个地址没服务",进长冷却(不再靠 30s 短冷却反复捞回)。
|
|
12
|
+
export const DEAD_FAIL_THRESHOLD = 2;
|
|
13
|
+
// 探测失败(GET /health 连不上)视为"这个节点当前不可用",10 分钟内不再当候选。
|
|
14
|
+
// 与请求失败账分开:health 探测是**主动、零额度**的可用性判断,比"这一发不顺"更可信——
|
|
15
|
+
// 现网 2026-10-20 13:27 req mvg8r7cz-b8qn:152.67.222.233:62028 早已 health timeout,
|
|
16
|
+
// CLI/保热都探到了,但选路不知道,仍把它排进候选并吃满 20s 单次上限(总 27s)。
|
|
17
|
+
export const DEFAULT_PEER_PROBE_DEAD_MS = 10 * 60_000;
|
|
18
|
+
|
|
19
|
+
export function peerProbeDeadMs() {
|
|
20
|
+
const n = Number(process.env.MSLXDFF_PEER_PROBE_DEAD_MS);
|
|
21
|
+
return Number.isInteger(n) && n >= 0 ? n : DEFAULT_PEER_PROBE_DEAD_MS;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const EMA_ALPHA = 0.3;
|
|
25
|
+
|
|
26
|
+
export function broadbandStaleMs() {
|
|
27
|
+
const n = Number(process.env.MSLXDFF_BROADBAND_STALE_MS);
|
|
28
|
+
return Number.isInteger(n) && n > 0 ? n : DEFAULT_BROADBAND_STALE_MS;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function isBroadbandStale(peer, nowMs = Date.now(), staleMs = broadbandStaleMs()) {
|
|
32
|
+
const kind = peer?.kind || (String(peer?.url || "").startsWith("relay://") ? "broadband" : "static");
|
|
33
|
+
if (kind !== "broadband") return false;
|
|
34
|
+
const lastSeen = peer?.lastSeen;
|
|
35
|
+
if (typeof lastSeen !== "number") return true; // no heartbeat yet => stale
|
|
36
|
+
return nowMs - lastSeen > staleMs;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function normalizePeerUrl(url) {
|
|
40
|
+
return String(url || "").trim().replace(/\/+$/, "");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function createPeersService({
|
|
44
|
+
file,
|
|
45
|
+
now = () => Date.now(),
|
|
46
|
+
cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
|
|
47
|
+
limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
|
|
48
|
+
heatMs = DEFAULT_PEER_HEAT_MS,
|
|
49
|
+
peers: seedPeers,
|
|
50
|
+
errors: seedErrors,
|
|
51
|
+
stats: seedStats,
|
|
52
|
+
persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
|
|
53
|
+
persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
|
|
54
|
+
persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
|
|
55
|
+
} = {}) {
|
|
56
|
+
const list = (seedPeers ?? loadPeers(file ? { file } : {}))
|
|
57
|
+
.map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
|
|
58
|
+
.filter((p) => p && p.url);
|
|
59
|
+
const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
|
|
60
|
+
const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
|
|
61
|
+
|
|
62
|
+
function all() {
|
|
63
|
+
return [...list];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function add(peer) {
|
|
67
|
+
const url = normalizePeerUrl(peer?.url);
|
|
68
|
+
if (!url) return false;
|
|
69
|
+
const existing = list.find((p) => p.url === url);
|
|
70
|
+
const entry = { ...peer, url };
|
|
71
|
+
if (existing) Object.assign(existing, entry);
|
|
72
|
+
else list.push(entry);
|
|
73
|
+
persistPeers([...list]);
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function remove(url) {
|
|
78
|
+
const target = normalizePeerUrl(url);
|
|
79
|
+
const idx = list.findIndex((p) => p.url === target);
|
|
80
|
+
if (idx < 0) return false;
|
|
81
|
+
list.splice(idx, 1);
|
|
82
|
+
persistPeers([...list]);
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function removeByGroup(group) {
|
|
87
|
+
const before = list.length;
|
|
88
|
+
for (let i = list.length - 1; i >= 0; i--) {
|
|
89
|
+
if (list[i].group === group) list.splice(i, 1);
|
|
90
|
+
}
|
|
91
|
+
if (list.length !== before) persistPeers([...list]);
|
|
92
|
+
return before - list.length;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 分级冷却:连续 429(streak >= 2)→ limitCooldownMs(限流恢复慢,5min 内不再考虑);
|
|
96
|
+
// 连接级永久失败(ECONNREFUSED/ENOTFOUND,deadFails >= 2)同样进长冷却;
|
|
97
|
+
// 其他失败(网络抖动、5xx)→ cooldownMs(可能几秒恢复,给快速重试机会)。
|
|
98
|
+
function coolingWindowMs(url) {
|
|
99
|
+
// 探测死亡(health 连不上):10min 内不再考虑——比请求失败更可信的"没服务"证据
|
|
100
|
+
if (stats[url]?.probeDeadAt !== undefined && now() - stats[url].probeDeadAt < peerProbeDeadMs()) return limitCooldownMs;
|
|
101
|
+
if ((stats[url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return limitCooldownMs;
|
|
102
|
+
const streak = stats[url]?.streak429 ?? 0;
|
|
103
|
+
return streak >= 2 ? limitCooldownMs : cooldownMs;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 连接级失败(端口没服务/DNS 解析不了)与"这一发不顺"是两回事:前者换多少次都一样,
|
|
107
|
+
// 现网 node-b(127.0.0.1:3500,本机根本没服务)靠 30s 短冷却 + 兜底,被反复竞速 1569 次。
|
|
108
|
+
function isConnLevelFailure(message) {
|
|
109
|
+
const s = String(message || "");
|
|
110
|
+
return /ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ECONNRESET|fetch failed|connect (?:timeout|ETIMEDOUT)|socket hang up/i.test(s);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function isCooling(url) {
|
|
114
|
+
// 探测死亡(health 探不通):10min 内不再当候选——比请求失败更可信的"没服务"证据
|
|
115
|
+
const pDead = stats[url]?.probeDeadAt;
|
|
116
|
+
if (pDead !== undefined && now() - pDead < peerProbeDeadMs()) return true;
|
|
117
|
+
const win = coolingWindowMs(url);
|
|
118
|
+
if (!win) return false;
|
|
119
|
+
const err = lastErrorAt[url];
|
|
120
|
+
if (typeof err === "number" && now() - err < win) return true;
|
|
121
|
+
const peer = list.find((p) => p.url === url);
|
|
122
|
+
if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function isBroadbandCooling(url) {
|
|
127
|
+
const peer = list.find((p) => p.url === url);
|
|
128
|
+
if (!peer) return false;
|
|
129
|
+
return isBroadbandStale(peer, now(), broadbandStaleMs());
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function available() {
|
|
133
|
+
return list.filter((p) => !isCooling(p.url));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// A peer counts as "hot" when it succeeded recently: reuse its last model
|
|
137
|
+
// without a fresh health probe to keep the fast path fast.
|
|
138
|
+
function isHot(url, t = now()) {
|
|
139
|
+
if (isCooling(url)) return false;
|
|
140
|
+
const s = stats[url];
|
|
141
|
+
return Boolean(s?.okAt) && t - s.okAt < heatMs;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function stat(url) {
|
|
145
|
+
return stats[url] ? { ...stats[url] } : null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function rankScore(p, t) {
|
|
149
|
+
const s = stats[p.url];
|
|
150
|
+
const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
|
|
151
|
+
const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
|
|
152
|
+
const fails = s?.fails ?? 0;
|
|
153
|
+
return hot * 1_000_000_000 + latency * 1_000 + fails;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// 可用组员排序(用户 2026-10-19 决定):**延迟最小优先**,不再「热优先」。
|
|
157
|
+
// 语义 = 「组员的额度用完后,用跟组长延迟最小的组员」——延迟是首要判据。
|
|
158
|
+
// 延迟来源(优先级从高到低):
|
|
159
|
+
// ① stats 里的真实请求 EMA(recordResult 记的 latencyMs)——最可信;
|
|
160
|
+
// ② peer.heartbeatLatencyMs —— 组员**定时心跳**上报的 rtt(没跑过真实请求时也能排序);
|
|
161
|
+
// 未测过延迟的排在有数据之后;同延迟按「最早失败优先」;都没失败过的排最后。
|
|
162
|
+
// 组长识别:本机是组员时,peers 里有 name==="leader" 的记录(cli/group-test.js 同口径)。
|
|
163
|
+
// 组长自己节点视角没有这条记录 → isLeaderPeer 恒 false,退化为纯延迟排序(正是想要的行为)。
|
|
164
|
+
function isLeaderPeer(p) {
|
|
165
|
+
return String(p?.name || "").toLowerCase() === "leader";
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function peerLatency(p) {
|
|
169
|
+
const s = stats[p.url]?.latencyMs;
|
|
170
|
+
if (s != null) return s;
|
|
171
|
+
const hb = list.find((x) => x.url === p.url)?.heartbeatLatencyMs;
|
|
172
|
+
return Number.isFinite(Number(hb)) ? Number(hb) : null;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// 组员延迟升序(组长不参与),供两处复用:ordered 的第二档 + standbyChain。
|
|
176
|
+
function byLatencyAsc(a, b) {
|
|
177
|
+
const la = peerLatency(a);
|
|
178
|
+
const lb = peerLatency(b);
|
|
179
|
+
const ha = la != null, hb = lb != null;
|
|
180
|
+
if (ha !== hb) return ha ? -1 : 1; // 有延迟数据的优先
|
|
181
|
+
if (ha && hb && la !== lb) return la - lb; // 延迟小的先上
|
|
182
|
+
const ea = lastErrorAt[a.url];
|
|
183
|
+
const eb = lastErrorAt[b.url];
|
|
184
|
+
if (ea == null && eb == null) return 0;
|
|
185
|
+
if (ea == null) return 1;
|
|
186
|
+
if (eb == null) return -1;
|
|
187
|
+
return ea - eb;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// 用户 2026-10-20 约定:**组长恒定第一**(先榨干组长额度),组长用完再按延迟升序上组员。
|
|
191
|
+
// 之前是纯延迟排序,「9/9 走组长」只是组长恰好延迟最低的巧合(AGENTS.md 已记差距)。
|
|
192
|
+
function ordered(t = now()) {
|
|
193
|
+
return available().sort((a, b) => {
|
|
194
|
+
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
195
|
+
if (la !== lb) return la ? -1 : 1; // 组长恒排第一
|
|
196
|
+
return byLatencyAsc(a, b);
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 常驻接力链(用户 2026-10-20:永远准备一个备选,429 立即切)。
|
|
202
|
+
* IPv4 阶段:组长 → 组员(延迟升序);IPv4 全用完才进 IPv6 阶段:先本机/组长 v6,
|
|
203
|
+
* 再按延迟升序找 hasV6 的组员。
|
|
204
|
+
* @returns {{ipv4: object[], ipv6: object[]}} 两段链,各自已排序
|
|
205
|
+
*/
|
|
206
|
+
function standbyChain(t = now()) {
|
|
207
|
+
const ipv4 = available().sort((a, b) => {
|
|
208
|
+
const la = isLeaderPeer(a), lb = isLeaderPeer(b);
|
|
209
|
+
if (la !== lb) return la ? -1 : 1;
|
|
210
|
+
return byLatencyAsc(a, b);
|
|
211
|
+
});
|
|
212
|
+
// IPv6 阶段:只保留心跳自报 hasV6 的节点,同样组长优先、其余延迟升序
|
|
213
|
+
const ipv6 = ipv4.filter((p) => p.hasV6 === true);
|
|
214
|
+
return { ipv4, ipv6 };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** 下一个备选:从链里取第一个还没试过的。这就是「永远准备好的那个」。 */
|
|
218
|
+
function nextStandby(tried, t = now()) {
|
|
219
|
+
const { ipv4, ipv6 } = standbyChain(t);
|
|
220
|
+
const skip = tried instanceof Set ? tried : new Set(tried || []);
|
|
221
|
+
return {
|
|
222
|
+
ipv4: ipv4.find((p) => !skip.has(p.url)) || null,
|
|
223
|
+
ipv6: ipv6.find((p) => !skip.has(p.url)) || null,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Available peers ordered specifically for recovery-time retry: earliest
|
|
228
|
+
// failure first (it has had the longest to come back), never-failed last.
|
|
229
|
+
function orderedByLastError(t = now()) {
|
|
230
|
+
return available().sort((a, b) => {
|
|
231
|
+
const ea = lastErrorAt[a.url];
|
|
232
|
+
const eb = lastErrorAt[b.url];
|
|
233
|
+
if (ea == null && eb == null) return 0;
|
|
234
|
+
if (ea == null) return 1; // never-failed peers go after any that failed
|
|
235
|
+
if (eb == null) return -1;
|
|
236
|
+
return ea - eb; // earliest failure first
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
let cursor = 0;
|
|
241
|
+
|
|
242
|
+
// 兜底重试集合:仅"因错误冷却中"的 peer(按最早失败优先)。
|
|
243
|
+
// 场景:唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时,兜底给它一次机会;
|
|
244
|
+
// 成功会被 recordResult 清除错误记录、回归热路径,避免"明明能用的节点被 30s 冷却全灭"。
|
|
245
|
+
function coolingByLastError() {
|
|
246
|
+
const t = now();
|
|
247
|
+
return list
|
|
248
|
+
.filter((p) => {
|
|
249
|
+
const err = lastErrorAt[p.url];
|
|
250
|
+
if (typeof err !== "number") return false;
|
|
251
|
+
if (stats[p.url]?.probeDeadAt !== undefined && now() - stats[p.url].probeDeadAt < peerProbeDeadMs()) return false; // 探测死亡:兜底也不考虑(否则每次白等一次拒连)
|
|
252
|
+
if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却:兜底也不考虑
|
|
253
|
+
if ((stats[p.url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return false; // 连接级永久失败:兜底也不考虑(否则每次白等一次拒连)
|
|
254
|
+
return t - err < cooldownMs;
|
|
255
|
+
})
|
|
256
|
+
.sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function next() {
|
|
260
|
+
const avail = available();
|
|
261
|
+
if (!avail.length) return null;
|
|
262
|
+
cursor = cursor % avail.length;
|
|
263
|
+
return avail[cursor++];
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Long-lived error memory: a peer keeps its last-error timestamp until a
|
|
267
|
+
// subsequent success resets it (success clears the failure record) or the
|
|
268
|
+
// error is no longer in the persist store on next load.
|
|
269
|
+
async function recordError(url, { status, message } = {}) {
|
|
270
|
+
if (!url) return;
|
|
271
|
+
lastErrorAt[url] = now();
|
|
272
|
+
await persistErrors({ ...lastErrorAt });
|
|
273
|
+
// 连续 429 计数(仅成功清零):第 2 次起进入长冷却——该 peer 大概率真被限流,不再频繁试它
|
|
274
|
+
if (Number(status) === 429) {
|
|
275
|
+
const prev = stats[url] || {};
|
|
276
|
+
stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
|
|
277
|
+
await persistStats({ ...stats });
|
|
278
|
+
}
|
|
279
|
+
// 连接级失败计数(仅成功清零):第 2 次起进长冷却,且不再被 coolingByLastError 兜底捞回
|
|
280
|
+
if (isConnLevelFailure(message)) {
|
|
281
|
+
const prev = stats[url] || {};
|
|
282
|
+
stats[url] = { ...prev, deadFails: (prev.deadFails || 0) + 1 };
|
|
283
|
+
await persistStats({ ...stats });
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
|
|
288
|
+
// last successful model) and clears the error memory; failures keep the
|
|
289
|
+
// error timestamp so the retry pass can order by recovery time.
|
|
290
|
+
async function recordResult(url, { ok, latencyMs, model } = {}) {
|
|
291
|
+
if (!url) return;
|
|
292
|
+
if (ok) {
|
|
293
|
+
if (lastErrorAt[url] != null) {
|
|
294
|
+
delete lastErrorAt[url];
|
|
295
|
+
await persistErrors({ ...lastErrorAt });
|
|
296
|
+
}
|
|
297
|
+
const prev = stats[url] || {};
|
|
298
|
+
const next = {
|
|
299
|
+
okAt: now(),
|
|
300
|
+
latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
|
|
301
|
+
? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
|
|
302
|
+
: (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
|
|
303
|
+
fails: 0,
|
|
304
|
+
model: model || prev.model || "",
|
|
305
|
+
streak429: 0,
|
|
306
|
+
deadFails: 0,
|
|
307
|
+
};
|
|
308
|
+
// 探测死亡标记:请求成功 = 节点活着,冷却窗口立即解除。
|
|
309
|
+
// next 本身不含 probeDeadAt 键,天然清除;这里只兜底确保没有残留。
|
|
310
|
+
stats[url] = next;
|
|
311
|
+
delete stats[url].probeDeadAt;
|
|
312
|
+
} else {
|
|
313
|
+
const prev = stats[url] || {};
|
|
314
|
+
stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
|
|
315
|
+
}
|
|
316
|
+
await persistStats({ ...stats });
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// 探测死亡标记(health 连不上):probeDeadAt 置为此刻,10min 内冷却。不反复探。
|
|
320
|
+
// recordProbeOk 在探测成功时清标记(节点回来了,冷却窗口立刻结束)。
|
|
321
|
+
async function recordProbeDead(url) {
|
|
322
|
+
if (!url) return;
|
|
323
|
+
const prev = stats[url] || {};
|
|
324
|
+
stats[url] = { ...prev, probeDeadAt: now() };
|
|
325
|
+
await persistStats({ ...stats });
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
async function recordProbeOk(url) {
|
|
329
|
+
if (!url) return;
|
|
330
|
+
const prev = stats[url] || {};
|
|
331
|
+
if (!("probeDeadAt" in prev)) return;
|
|
332
|
+
delete prev.probeDeadAt;
|
|
333
|
+
stats[url] = prev;
|
|
334
|
+
await persistStats({ ...stats });
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return {
|
|
338
|
+
all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
|
|
339
|
+
const peer = list.find((p) => p.url === url);
|
|
340
|
+
return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
|
|
341
|
+
}, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
|
|
342
|
+
standbyChain, nextStandby, isLeaderPeer,
|
|
343
|
+
recordError, recordResult, recordProbeDead, recordProbeOk, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
|
|
344
|
+
};
|
|
345
|
+
}
|