mslxdff 0.1.214 → 0.1.216
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/format.js +2 -0
- package/src/groups.js +4 -1
- package/src/model-trace.js +2 -0
- package/src/peers.js +20 -10
- package/src/providers/codearts/concurrency.js +17 -2
- package/src/providers/codearts/index.js +49 -2
- package/src/routes/groups-relay.js +13 -2
- package/src/routes/peer-serial.js +99 -0
- package/src/routes/peers.js +23 -0
- package/src/runtime/broadband.js +11 -1
- package/src/upstream-engine/index.js +16 -4
- package/src/upstream.js +14 -4
package/package.json
CHANGED
package/src/cli/format.js
CHANGED
|
@@ -70,6 +70,8 @@ export function fmtEvent(e) {
|
|
|
70
70
|
return `${head} peer win 选中 peer=${e.winPeer} model=${m(e.winTarget)} latency=${e.latencyMs}ms`;
|
|
71
71
|
case "peer-race-lose":
|
|
72
72
|
return `${head} peer lose 组员全部失败 model=${m(e.model)}`;
|
|
73
|
+
case "peer-skip-slow":
|
|
74
|
+
return `${head} peer skip 跳过过慢组员 peer=${e.peer} 历史延迟=${e.latencyMs ?? "-"}ms > ${e.capMs ?? "-"}ms`;
|
|
73
75
|
case "fallback":
|
|
74
76
|
return `${head} fallback ${m(e.from)} -> ${m(e.to)} reason=${m(e.reason)}`;
|
|
75
77
|
case "fallback-notice":
|
package/src/groups.js
CHANGED
|
@@ -202,7 +202,10 @@ export function syncPeersFromMembers({ peers, members, myUrl, group, skipIds = [
|
|
|
202
202
|
if (kind === "broadband" || String(m?.url || "").startsWith("relay://")) continue;
|
|
203
203
|
const url = normalizePeerUrl(m?.url);
|
|
204
204
|
if (!url || url === self) continue;
|
|
205
|
-
|
|
205
|
+
// heartbeatLatencyMs:组长在**心跳**里量的成员延迟(成员上报 rttMs)。选路 `ordered()` 在没有
|
|
206
|
+
// 真实请求 EMA 时回落到它——这就是「组员定时上报的过程中就知道谁延迟更低」。
|
|
207
|
+
const hbLat = Number.isFinite(Number(m?.latencyMs)) && Number(m.latencyMs) >= 0 ? Number(m.latencyMs) : null;
|
|
208
|
+
if (peers.add({ name: id, url, token: m?.token || "", group, ...(hbLat != null ? { heartbeatLatencyMs: hbLat } : {}) })) added++;
|
|
206
209
|
}
|
|
207
210
|
return { removed, added, total: Object.keys(members || {}).length };
|
|
208
211
|
}
|
package/src/model-trace.js
CHANGED
|
@@ -161,6 +161,8 @@ export function formatModelTrace({ type, reqId, model, data = {}, request = null
|
|
|
161
161
|
detail = `组员竞速开始 peers=${data.peers ?? "-"}`;
|
|
162
162
|
} else if (type === "peer-budget-out") {
|
|
163
163
|
detail = `组员竞速预算用尽 reason=${safeText(data.reason || "", 80)} budgetMs=${data.budgetMs ?? "-"} remaining=${data.remaining ?? "-"}`;
|
|
164
|
+
} else if (type === "peer-skip-slow") {
|
|
165
|
+
detail = `跳过过慢组员 peer=${data.peer || "?"} 历史延迟=${data.latencyMs ?? "-"}ms > 上限=${data.capMs ?? "-"}ms`;
|
|
164
166
|
} else detail = safeText(`${decisionKv(data)} ${data.message || data.error || ""}`, 200);
|
|
165
167
|
return `[${base.time}] [req=${base.req}] [model=${base.model}] [stage=${base.stage}] ${detail}`;
|
|
166
168
|
}
|
package/src/peers.js
CHANGED
|
@@ -138,22 +138,32 @@ export function createPeersService({
|
|
|
138
138
|
return hot * 1_000_000_000 + latency * 1_000 + fails;
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
//
|
|
145
|
-
//
|
|
141
|
+
// 可用组员排序(用户 2026-10-19 决定):**延迟最小优先**,不再「热优先」。
|
|
142
|
+
// 语义 = 「组员的额度用完后,用跟组长延迟最小的组员」——延迟是首要判据。
|
|
143
|
+
// 延迟来源(优先级从高到低):
|
|
144
|
+
// ① stats 里的真实请求 EMA(recordResult 记的 latencyMs)——最可信;
|
|
145
|
+
// ② peer.heartbeatLatencyMs —— 组员**定时心跳**上报的 rtt(没跑过真实请求时也能排序);
|
|
146
|
+
// 未测过延迟的排在有数据之后;同延迟按「最早失败优先」;都没失败过的排最后。
|
|
147
|
+
function peerLatency(p) {
|
|
148
|
+
const s = stats[p.url]?.latencyMs;
|
|
149
|
+
if (s != null) return s;
|
|
150
|
+
const hb = list.find((x) => x.url === p.url)?.heartbeatLatencyMs;
|
|
151
|
+
return Number.isFinite(Number(hb)) ? Number(hb) : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
146
154
|
function ordered(t = now()) {
|
|
147
155
|
return available().sort((a, b) => {
|
|
148
|
-
const
|
|
149
|
-
const
|
|
150
|
-
|
|
156
|
+
const la = peerLatency(a);
|
|
157
|
+
const lb = peerLatency(b);
|
|
158
|
+
const ha = la != null, hb = lb != null;
|
|
159
|
+
if (ha !== hb) return ha ? -1 : 1; // 有延迟数据的优先
|
|
160
|
+
if (ha && hb && la !== lb) return la - lb; // 延迟小的先上
|
|
151
161
|
const ea = lastErrorAt[a.url];
|
|
152
162
|
const eb = lastErrorAt[b.url];
|
|
153
163
|
if (ea == null && eb == null) return 0;
|
|
154
|
-
if (ea == null) return 1;
|
|
164
|
+
if (ea == null) return 1;
|
|
155
165
|
if (eb == null) return -1;
|
|
156
|
-
return ea - eb;
|
|
166
|
+
return ea - eb;
|
|
157
167
|
});
|
|
158
168
|
}
|
|
159
169
|
|
|
@@ -10,7 +10,11 @@ export function createGate({ limit = 3 } = {}) {
|
|
|
10
10
|
if (w) w();
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
// 取得一个槽位,返回「释放一次」的幂等函数。超限时 FIFO 排队(并回调 onWait)。
|
|
14
|
+
// 关键:释放的时机交给调用方——流式请求必须**等流真正读完/取消**才释放,
|
|
15
|
+
// 否则 runChat 一返回(流还没消费)就放行,上游其实仍占着会话 ⇒ 第 4 路被上游拒
|
|
16
|
+
// (TM.00001041 并发会话数已达上限)。见 index.js 的 chat 接线。
|
|
17
|
+
async function acquire(onWait) {
|
|
14
18
|
if (active >= cap) {
|
|
15
19
|
const t0 = Date.now();
|
|
16
20
|
const queued = waiters.length + 1;
|
|
@@ -18,15 +22,26 @@ export function createGate({ limit = 3 } = {}) {
|
|
|
18
22
|
if (onWait) { try { onWait({ waitedMs: Date.now() - t0, queued, limit: cap }); } catch {} }
|
|
19
23
|
}
|
|
20
24
|
active++;
|
|
25
|
+
let released = false;
|
|
26
|
+
return function releaseOnce() {
|
|
27
|
+
if (released) return;
|
|
28
|
+
released = true;
|
|
29
|
+
release();
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function run(fn, onWait) {
|
|
34
|
+
const rel = await acquire(onWait);
|
|
21
35
|
try {
|
|
22
36
|
return await fn();
|
|
23
37
|
} finally {
|
|
24
|
-
|
|
38
|
+
rel();
|
|
25
39
|
}
|
|
26
40
|
}
|
|
27
41
|
|
|
28
42
|
return {
|
|
29
43
|
run,
|
|
44
|
+
acquire,
|
|
30
45
|
stats: () => ({ active, queued: waiters.length, limit: cap }),
|
|
31
46
|
get active() { return active; },
|
|
32
47
|
get queued() { return waiters.length; },
|
|
@@ -82,9 +82,48 @@ export function createCodeartsProvider({
|
|
|
82
82
|
const onWait = (info) => {
|
|
83
83
|
try { appendEvent({ type: "codearts-queue", ...info }); } catch {}
|
|
84
84
|
};
|
|
85
|
+
// 流式请求必须在**流真正读完/取消后**才释放并发槽——否则 runChat 一返回(body 还没消费)
|
|
86
|
+
// 就 release,上游其实仍占着会话 ⇒ 第 4 路被上游拒(TM.00001041 并发会话数已达上限)。
|
|
87
|
+
// 非流式(JSON)在 runChat 内部已聚合完,可立即释放。返回的 Response 保持原 body 引用,
|
|
88
|
+
// 只包一层 tee 触发 release;出错也保证释放一次(releaseOnce 幂等)。
|
|
89
|
+
function holdUntilBodyDone(res, releaseOnce) {
|
|
90
|
+
const ct = res?.headers?.get?.("content-type") || "";
|
|
91
|
+
if (!ct.includes("text/event-stream") || !res.body) {
|
|
92
|
+
releaseOnce();
|
|
93
|
+
return res;
|
|
94
|
+
}
|
|
95
|
+
let done = false;
|
|
96
|
+
const fin = () => { if (!done) { done = true; releaseOnce(); } };
|
|
97
|
+
const onErr = () => fin();
|
|
98
|
+
try {
|
|
99
|
+
const reader = res.body.getReader();
|
|
100
|
+
const stream = new ReadableStream({
|
|
101
|
+
async pull(controller) {
|
|
102
|
+
try {
|
|
103
|
+
const { done: d, value } = await reader.read();
|
|
104
|
+
if (d) { fin(); controller.close(); return; }
|
|
105
|
+
controller.enqueue(value);
|
|
106
|
+
} catch (e) { onErr(); controller.error(e); }
|
|
107
|
+
},
|
|
108
|
+
cancel(reason) { fin(); try { return reader.cancel(reason); } catch { /* ignore */ } },
|
|
109
|
+
});
|
|
110
|
+
return new Response(stream, { status: res.status, statusText: res.statusText, headers: res.headers });
|
|
111
|
+
} catch {
|
|
112
|
+
fin();
|
|
113
|
+
return res;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
85
116
|
async function chat(body, opts = {}) {
|
|
86
117
|
if (!ring.size) throw new Error("codearts: missing credential — run: mslxdff -provider codearts login");
|
|
87
|
-
|
|
118
|
+
const rel = await gate.acquire(onWait);
|
|
119
|
+
let res;
|
|
120
|
+
try {
|
|
121
|
+
res = await chatSvc.runChat(body, ring, opts);
|
|
122
|
+
} catch (err) {
|
|
123
|
+
rel();
|
|
124
|
+
throw err;
|
|
125
|
+
}
|
|
126
|
+
return holdUntilBodyDone(res, rel);
|
|
88
127
|
}
|
|
89
128
|
async function chatWithKeys(body, keys, opts = {}) {
|
|
90
129
|
const blobs = (keys || []).filter((k) => typeof k === "string" && k.trim().startsWith("{"));
|
|
@@ -92,7 +131,15 @@ export function createCodeartsProvider({
|
|
|
92
131
|
const tmpRing = createKeyRing(blobs, { cooldownMs });
|
|
93
132
|
const tmpPool = createAuthPool({ keys: blobs, fetchImpl, stsHost, clock }); // 借用凭据不写回本节点
|
|
94
133
|
const tmpSvc = createChatService({ id, baseUrl: chatSvcBase, chatPath, fetchImpl, dispatcher: null, authPool: tmpPool, catalog, connectTimeoutMs, clock });
|
|
95
|
-
|
|
134
|
+
const rel = await gate.acquire(onWait);
|
|
135
|
+
let res;
|
|
136
|
+
try {
|
|
137
|
+
res = await tmpSvc.runChat(body, tmpRing, opts);
|
|
138
|
+
} catch (err) {
|
|
139
|
+
rel();
|
|
140
|
+
throw err;
|
|
141
|
+
}
|
|
142
|
+
return holdUntilBodyDone(res, rel);
|
|
96
143
|
}
|
|
97
144
|
|
|
98
145
|
// 三路发现(主账号);结果写目录(benefit 判定按账号隔离)
|
|
@@ -18,6 +18,7 @@ function snippet(v, n = 300) {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export async function heartbeatHandler({ req, res, groups, bus, logs }) {
|
|
21
|
+
const tRecv = Date.now();
|
|
21
22
|
const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
|
|
22
23
|
if (!auth) return json(res, 401, { error: "bearer token required" });
|
|
23
24
|
let body;
|
|
@@ -36,9 +37,19 @@ export async function heartbeatHandler({ req, res, groups, bus, logs }) {
|
|
|
36
37
|
if (m) {
|
|
37
38
|
m.publicIp = ip;
|
|
38
39
|
m.lastSeen = Date.now();
|
|
39
|
-
|
|
40
|
+
// 延迟采样(心跳即可得,无需真实请求):
|
|
41
|
+
// rttMs = 成员自己量的「上一轮心跳往返」(它到组长的网络路径)
|
|
42
|
+
// serverMs = 组长这一侧从收到到处理完的耗时
|
|
43
|
+
// 两者取有值的那个写进成员记录,供选路按「延迟最小优先」排序(routes/peers.js ordered)。
|
|
44
|
+
const reportedRtt = Number(body?.rttMs);
|
|
45
|
+
const memberRtt = Number.isFinite(reportedRtt) && reportedRtt >= 0 ? Math.round(reportedRtt) : null;
|
|
46
|
+
if (memberRtt != null) m.latencyMs = memberRtt;
|
|
47
|
+
try {
|
|
48
|
+
groups.upsertMember(groupName, { memberName: targetId, url: m.url, token: m.token, kind: "broadband", publicIp: ip, lastSeen: m.lastSeen, ...(memberRtt != null ? { latencyMs: memberRtt } : {}) });
|
|
49
|
+
} catch {}
|
|
40
50
|
}
|
|
41
|
-
const
|
|
51
|
+
const serverMs = Date.now() - tRecv;
|
|
52
|
+
const evtData = { ts: Date.now(), type: "relay-heartbeat", member: targetId, ip, lastSeen: m?.lastSeen, rttMs: Number.isFinite(Number(body?.rttMs)) ? Number(body.rttMs) : null, serverMs, group: groupName };
|
|
42
53
|
if (bus) bus.emit(evtData);
|
|
43
54
|
logs?.appendEvent?.(evtData);
|
|
44
55
|
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// 组员串行选路(用户 2026-10-19 决定:不竞速)。
|
|
2
|
+
//
|
|
3
|
+
// 为什么改:并发赛跑会同时占用多个组员的额度/连接,慢节点还会拖住整批;而用户要的语义是
|
|
4
|
+
// 「本机额度用完 → 用延迟最小的组员」,延迟小的先上、不行再下一个。串行让这条优先级真正生效。
|
|
5
|
+
//
|
|
6
|
+
// 本模块只做「按序试」这件事,其余(画目标、转发、记账、事件名)从 peers.js 注入,避免循环依赖。
|
|
7
|
+
export const PEER_SERIAL = true;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* 串行试候选:一次只打一个,成功即返回;失败顺位试下一个。
|
|
11
|
+
*
|
|
12
|
+
* @param {object} p
|
|
13
|
+
* @param {Array} p.candidates 已按「延迟最小优先」排好的组员
|
|
14
|
+
* @param {object} p.ctx 请求上下文(evt/peers/body/hops/plugins/peerErrors/triedUrls…)
|
|
15
|
+
* @param {Set} p.tried 已试过的 url 集合
|
|
16
|
+
* @param {object} p.deps { resolvePeerTarget, forwardToPeer, peerName, recordLater,
|
|
17
|
+
* summarizeRequest, runHook, errMsg, perAttemptCap, deadline, budgetMs }
|
|
18
|
+
* @returns {Promise<null | {peer, target, res, latencyMs}>}
|
|
19
|
+
*/
|
|
20
|
+
export async function serialTryPeers({ candidates, ctx, tried, deps }) {
|
|
21
|
+
const {
|
|
22
|
+
resolvePeerTarget, forwardToPeer, peerName, recordLater, summarizeRequest, runHook, errMsg,
|
|
23
|
+
perAttemptCap, deadline, budgetMs,
|
|
24
|
+
} = deps;
|
|
25
|
+
const failed = [];
|
|
26
|
+
// 延迟上限过滤:历史延迟超过「单次尝试上限」的组员直接跳过(它大概率又超时)。
|
|
27
|
+
// 只在**还有别的候选**时才跳过,避免把唯一可用的节点也跳没了。
|
|
28
|
+
const ranked = candidates.filter((p) => {
|
|
29
|
+
let lat = null;
|
|
30
|
+
try {
|
|
31
|
+
// 与 ordered() 同源:先看真实请求 EMA,再回落到心跳上报的 rtt
|
|
32
|
+
lat = ctx.peers?.stat?.(p.url)?.latencyMs ?? p.heartbeatLatencyMs ?? null;
|
|
33
|
+
if (lat != null) lat = Number(lat);
|
|
34
|
+
} catch { lat = null; }
|
|
35
|
+
if (lat == null || !Number.isFinite(lat) || lat <= perAttemptCap) return true;
|
|
36
|
+
if (candidates.length <= 1) return true; // 只剩它 → 仍然试
|
|
37
|
+
ctx.evt("peer-skip-slow", { peer: p.url, name: peerName(p), latencyMs: lat, capMs: perAttemptCap });
|
|
38
|
+
return false;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
for (let i = 0; i < ranked.length; i++) {
|
|
42
|
+
if (deadline > 0 && Date.now() >= deadline) {
|
|
43
|
+
ctx.evt("peer-budget-out", { peer: null, reason: `peer 阶段总预算 ${budgetMs}ms 用尽`, budgetMs, remaining: ranked.length - i });
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
const prepared = await resolvePeerTarget(ctx, ranked[i]);
|
|
47
|
+
if (!prepared) continue;
|
|
48
|
+
const { peer, target } = prepared;
|
|
49
|
+
tried.add(peer.url);
|
|
50
|
+
const ctrl = new AbortController();
|
|
51
|
+
const name = peerName(peer);
|
|
52
|
+
ctx.evt("peer-request", { peer: peer.url, name, model: target, hops: ctx.hops + 1, payload: summarizeRequest(ctx.body), serial: i + 1 });
|
|
53
|
+
if (ctx.plugins?.length) {
|
|
54
|
+
runHook(ctx.plugins, "peer:beforeForward", { reqId: ctx.reqId, peer: peer.url, model: target, hops: ctx.hops + 1 }).catch(() => {});
|
|
55
|
+
}
|
|
56
|
+
const remainingMs = deadline > 0 ? deadline - Date.now() : 0;
|
|
57
|
+
// 单个组员的等待上限 = min(阶段剩余预算, 单次尝试上限)。单次上限保证黑洞节点
|
|
58
|
+
// 不会吃掉整个阶段(否则延迟最小的好组员轮不到)。
|
|
59
|
+
const waitMs = deadline > 0 ? Math.max(1, Math.min(perAttemptCap, remainingMs)) : perAttemptCap;
|
|
60
|
+
const t0 = performance.now();
|
|
61
|
+
const res = await forwardToPeer(peer, ctx.body, target, ctx.hops, ctrl, waitMs);
|
|
62
|
+
const latencyMs = Math.round(performance.now() - t0);
|
|
63
|
+
const isErr = res instanceof Error;
|
|
64
|
+
const status = isErr ? 502 : res.status;
|
|
65
|
+
if (ctx.plugins?.length) {
|
|
66
|
+
runHook(ctx.plugins, "peer:result", { reqId: ctx.reqId, peer: peer.url, model: target, ok: !(isErr || status >= 400), status, latencyMs }).catch(() => {});
|
|
67
|
+
}
|
|
68
|
+
if (isErr || status >= 400) {
|
|
69
|
+
const msg = isErr ? errMsg(res).slice(0, 200) : null;
|
|
70
|
+
if (Array.isArray(ctx.peerErrors)) ctx.peerErrors.push({ peer: peer.url, status, message: msg });
|
|
71
|
+
ctx.evt("peer-attempt", { peer: peer.url, name, model: target, hops: ctx.hops + 1, latencyMs, ok: false, status, message: msg });
|
|
72
|
+
ctx.evt("peer-error", { peer: peer.url, name, model: target, status, message: msg });
|
|
73
|
+
failed.push({ ok: false, peer, target, res, status });
|
|
74
|
+
recordLater(() => ctx.peers.recordError(peer.url, { status, message: msg }));
|
|
75
|
+
recordLater(() => ctx.peers.recordResult(peer.url, { ok: false }));
|
|
76
|
+
continue; // 串行:试下一个
|
|
77
|
+
}
|
|
78
|
+
ctx.evt("peer-forward", { peer: peer.url, name, model: target, hops: ctx.hops + 1, latencyMs, ok: true, status });
|
|
79
|
+
return { peer, target, res, latencyMs };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// 全败:补读失败响应体(诊断 + 调用者详情)
|
|
83
|
+
await Promise.all(failed.map(async (o) => {
|
|
84
|
+
if (!o.res || typeof o.res !== "object" || o.res instanceof Error) return;
|
|
85
|
+
try {
|
|
86
|
+
const snip = String(await Promise.race([
|
|
87
|
+
o.res.clone().text(),
|
|
88
|
+
new Promise((r) => { const t = setTimeout(() => r(""), 600); t.unref?.(); }),
|
|
89
|
+
])).replace(/\s+/g, " ").slice(0, 300);
|
|
90
|
+
if (!snip) return;
|
|
91
|
+
if (Array.isArray(ctx.peerErrors)) {
|
|
92
|
+
const rec = ctx.peerErrors.find((x) => x.peer === o.peer.url && x.status === o.status && !x.message);
|
|
93
|
+
if (rec) rec.message = snip;
|
|
94
|
+
}
|
|
95
|
+
ctx.evt("peer-error", { peer: o.peer.url, name: peerName(o.peer), model: o.target, status: o.status, message: `body=${snip}` });
|
|
96
|
+
} catch {}
|
|
97
|
+
}));
|
|
98
|
+
return null;
|
|
99
|
+
}
|
package/src/routes/peers.js
CHANGED
|
@@ -4,6 +4,7 @@ import { errMsg } from "./helpers.js";
|
|
|
4
4
|
import { runHook } from "../plugins.js";
|
|
5
5
|
import { buildShareKeysHeader, SHARE_KEYS_HEADER } from "../providers/share-keys.js";
|
|
6
6
|
import { compatFetch, timeoutSignal, getUndici } from "../compat.js";
|
|
7
|
+
import { serialTryPeers, PEER_SERIAL } from "./peer-serial.js";
|
|
7
8
|
|
|
8
9
|
// peer 阶段总预算:三批候选(ordered / orderedByLastError / coolingByLastError)共享同一个
|
|
9
10
|
// 截止时间,而不是每批各吃满一次等头超时。实测最坏路径是三批串行各 30s(≈90s),
|
|
@@ -34,6 +35,15 @@ function peerConnectTimeoutMs() {
|
|
|
34
35
|
return Number.isInteger(n) && n > 0 ? n : DEFAULT_PEER_CONNECT_TIMEOUT_MS;
|
|
35
36
|
}
|
|
36
37
|
|
|
38
|
+
// 单组员等待上限(串行模式下逐个体生效,MSLXDFF_PEER_ATTEMPT_TIMEOUT_MS,缺省 20s)。
|
|
39
|
+
// 为什么必须限:串行是「一个失败才试下一个」,若一个黑洞节点吃满 120s(bodyTimeout),
|
|
40
|
+
// 后面延迟更小的好组员根本轮不到——实测 172.93.221.187 常 38~44s 才回/被 abort。
|
|
41
|
+
// 20s 足以覆盖正常首包,又不会让一个坏节点吃掉整个阶段预算;可用 env 调。
|
|
42
|
+
function peerAttemptTimeoutMs() {
|
|
43
|
+
const n = Number(process.env.MSLXDFF_PEER_ATTEMPT_TIMEOUT_MS);
|
|
44
|
+
return Number.isInteger(n) && n > 0 ? n : 20_000;
|
|
45
|
+
}
|
|
46
|
+
|
|
37
47
|
// peer 阶段总预算(MSLXDFF_PEER_RACE_BUDGET_MS,0=不限):一次 peer 阶段(同批或若干批候选)
|
|
38
48
|
// 共享同一个截止时刻。调用方用 peerStageCtx() 起一个阶段,别把 deadline 挂在整个请求上下文上——
|
|
39
49
|
// 否则前一个候选把预算吃光,后一个候选一进 peer 阶段就直接放弃。
|
|
@@ -249,6 +259,10 @@ function peerName(peer) {
|
|
|
249
259
|
return String(peer?.name || "").trim() || null;
|
|
250
260
|
}
|
|
251
261
|
|
|
262
|
+
// 组员试法:**串行**(用户 2026-10-19 决定:不竞速)。实现在 ./peer-serial.js,
|
|
263
|
+
// 本文件只保留调度与并发版(PEER_SERIAL=false 时的旧路径)。
|
|
264
|
+
export { PEER_SERIAL };
|
|
265
|
+
|
|
252
266
|
export async function racePeerCandidates(candidates, ctx) {
|
|
253
267
|
const tried = (ctx.triedUrls ??= new Set());
|
|
254
268
|
const fresh = candidates.filter((p) => !tried.has(p.url));
|
|
@@ -256,6 +270,15 @@ export async function racePeerCandidates(candidates, ctx) {
|
|
|
256
270
|
// 没传的阶段(单批调用)不带预算——保持旧行为,不外溢到整条请求。
|
|
257
271
|
const budgetMs = ctx.peerRaceBudgetMs ?? 0;
|
|
258
272
|
const deadline = Number(ctx.peerRaceDeadline) || 0;
|
|
273
|
+
|
|
274
|
+
// 串行:按 candidates 的既有次序(调用方已按「延迟最小优先」排好)逐个试。
|
|
275
|
+
if (PEER_SERIAL) {
|
|
276
|
+
return serialTryPeers({
|
|
277
|
+
candidates: fresh, ctx, tried,
|
|
278
|
+
deps: { resolvePeerTarget, forwardToPeer, peerName, recordLater, summarizeRequest, runHook, errMsg, perAttemptCap: peerAttemptTimeoutMs(), deadline, budgetMs },
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
|
|
259
282
|
for (let i = 0; i < fresh.length; i += PEER_RACE_LIMIT) {
|
|
260
283
|
if (deadline > 0 && Date.now() >= deadline) {
|
|
261
284
|
ctx.evt("peer-budget-out", { peer: null, reason: `peer 阶段总预算 ${budgetMs}ms 用尽`, budgetMs, remaining: fresh.length - i });
|
package/src/runtime/broadband.js
CHANGED
|
@@ -97,20 +97,30 @@ export function startBroadband({ token, upstream, logs, log = console.log }) {
|
|
|
97
97
|
return true;
|
|
98
98
|
})();
|
|
99
99
|
const execAndPost = createBroadbandExec({ token, upstream, logs, log });
|
|
100
|
+
// 心跳往返延迟:成员侧自己量「上一轮 heartbeat 的 RTT」,随下一次心跳上报给组长。
|
|
101
|
+
// 组长侧据此排序「谁的延迟更低」(选路=延迟最小优先,见 routes/peers.js 的 ordered)。
|
|
102
|
+
// 用上一轮而不是本轮的原因:本轮 RTT 要等响应回来才知道,来不及带在这次请求里。
|
|
103
|
+
const hbRtt = new Map(); // group name -> ms
|
|
100
104
|
const doHeartbeat = async () => {
|
|
101
105
|
for (const g of broadbandGroups()) {
|
|
102
106
|
try {
|
|
107
|
+
const t0 = Date.now();
|
|
103
108
|
const res = await compatFetch(`${g.leaderUrl}/v1/groups/relay/heartbeat`, {
|
|
104
109
|
method: "POST",
|
|
105
110
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
106
|
-
body: JSON.stringify({ name: g.name, group: g.name }),
|
|
111
|
+
body: JSON.stringify({ name: g.name, group: g.name, rttMs: hbRtt.get(g.name) ?? null }),
|
|
107
112
|
signal: timeoutSignal(5000),
|
|
108
113
|
});
|
|
114
|
+
const rtt = Date.now() - t0;
|
|
115
|
+
hbRtt.set(g.name, rtt);
|
|
109
116
|
if (!res.ok) {
|
|
110
117
|
const txt = await res.text().catch(() => "");
|
|
111
118
|
log(`broadband heartbeat ${g.name}: ${res.status} ${txt.slice(0, 100)}`);
|
|
119
|
+
} else {
|
|
120
|
+
log(`broadband heartbeat ${g.name}: ok rtt=${rtt}ms`);
|
|
112
121
|
}
|
|
113
122
|
} catch (err) {
|
|
123
|
+
hbRtt.delete(g.name);
|
|
114
124
|
log(`broadband heartbeat ${g.name}: failed — ${errMsg(err)}`);
|
|
115
125
|
}
|
|
116
126
|
}
|
|
@@ -46,10 +46,22 @@ export function createUpstreamEngine(opts = {}) {
|
|
|
46
46
|
const via = useResponses ? "@ai-sdk/openai · responses" : "@ai-sdk/openai-compatible · chat";
|
|
47
47
|
try { console.error(`[upstream-engine] sdk 引擎生效(${via})`); } catch {}
|
|
48
48
|
}
|
|
49
|
-
// 保留 legacy 的匿名重试语义(仅 !anonFirst 配置下存在);
|
|
50
|
-
// 也走这条委派——重试环在 legacy.chat
|
|
51
|
-
//
|
|
52
|
-
|
|
49
|
+
// 保留 legacy 的匿名重试语义(仅 !anonFirst 配置下存在);free-429 的**请求内 v6 换出口**
|
|
50
|
+
// 也走这条委派——重试环在 legacy.chat 里。
|
|
51
|
+
// ⚠ 顺序契约(用户 2026-10-19):先借人(组员/组长,用对方 IPv4 出口),全都用完才换出口。
|
|
52
|
+
// 故 v6 换出口**默认关**,只有显式 MSLXDFF_V6_INREQUEST_ROTATE=1 才委派;
|
|
53
|
+
// 默认把 429 原样返回,交给 chat-pipeline 去试组员/组长。
|
|
54
|
+
const v6RotateOn = (() => {
|
|
55
|
+
const raw = process.env.MSLXDFF_V6_INREQUEST_ROTATE;
|
|
56
|
+
if (raw === undefined || raw === null || raw === "") return false;
|
|
57
|
+
const s = String(raw).trim().toLowerCase();
|
|
58
|
+
return !(s === "0" || s === "false" || s === "off" || s === "no");
|
|
59
|
+
})();
|
|
60
|
+
const canRotate = v6RotateOn && typeof dispatcher?.rotateFor === "function";
|
|
61
|
+
// 原语义:撞 429 且是 free 模型,且(非匿名首试 或 有 v6 池可换址)⇒ 委派 legacy 重试环。
|
|
62
|
+
// 新语义:**只有显式开启 v6 请求内换址时才委派**;默认 429 原样返回,交给上层去借人
|
|
63
|
+
// (组员/组长,用对方 IPv4 出口),符合用户 2026-10-19 的「先借人再换出口」顺序。
|
|
64
|
+
if (res.status === 429 && isFreeModel(body?.model) && (anonFirst ? canRotate : v6RotateOn)) return legacy.chat(body);
|
|
53
65
|
return res;
|
|
54
66
|
} catch (e) {
|
|
55
67
|
if (e && e._sdkLoadFailed) {
|
package/src/upstream.js
CHANGED
|
@@ -188,14 +188,24 @@ export function createUpstreamClient({
|
|
|
188
188
|
try { console.error(`[free-lane] resp model=${body?.model} status=${res.status} ct=${res.headers?.get?.("content-type") || ""}`); } catch {}
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
//
|
|
192
|
-
//
|
|
193
|
-
//
|
|
191
|
+
// 请求内换出口(v6 池):撞 429 时在同一请求里换源地址重发。
|
|
192
|
+
//
|
|
193
|
+
// ⚠ 顺序契约(用户 2026-10-19 明确):**先借人,再换出口**。
|
|
194
|
+
// 429 ⇒ ① 先走组员/组长接力(用**对方的 IPv4 出口**);② 全都没得用/都失败,
|
|
195
|
+
// 才轮到 WARP / IPv6 换出口拿"新的自然额度"。WARP/IPv6 是另一层兜底,不该抢在借人之前。
|
|
196
|
+
// 故本功能**默认关闭**,需要时才显式开:MSLXDFF_V6_INREQUEST_ROTATE=1(原 ADR-0061 行为)。
|
|
197
|
+
// 关闭时 429 原样交回上层 → chat-pipeline 去试组员/组长,符合用户要的顺序。
|
|
194
198
|
// 边界(探测实证过的硬约束):① 仅头部即 429 才换(不碰半开的 200 流);② 429 错误体已被
|
|
195
199
|
// transport 非流分支读尽,无连接泄漏;③ 每轮换的是 session 粘的那个槽(swap 原地换 addr/e.agent,
|
|
196
200
|
// 粘性映射不失效);④ 次数上限防 429 风暴空烧认领地址;⑤ 无 rotateFor 的 dispatcher(直连/WARP)
|
|
197
201
|
// 条件短路——WARP 换出口=重注册设备 6s~600s,在请求内做只会白等,**设计上排除**。
|
|
198
|
-
|
|
202
|
+
const inRequestRotateOn = (() => {
|
|
203
|
+
const raw = process.env.MSLXDFF_V6_INREQUEST_ROTATE;
|
|
204
|
+
if (raw === undefined || raw === null || raw === "") return false; // 缺省关:先借人
|
|
205
|
+
const s = String(raw).trim().toLowerCase();
|
|
206
|
+
return !(s === "0" || s === "false" || s === "off" || s === "no");
|
|
207
|
+
})();
|
|
208
|
+
if (inRequestRotateOn && freeLane && res.status === 429 && typeof transport.dispatcher?.rotateFor === "function") {
|
|
199
209
|
const maxRot = envIntOrZero("MSLXDFF_V6_INREQUEST_RETRIES", 3); // 0 = 关掉请求内换出口
|
|
200
210
|
let rotated = 0;
|
|
201
211
|
while (res.status === 429 && rotated < maxRot) {
|