mslxdff 0.1.159 → 0.1.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/cli_help_mini.md +3 -1
- package/package.json +1 -1
- package/src/autostart.js +3 -0
- package/src/chat-pipeline/auto-race.js +1 -1
- package/src/chat-pipeline/empty-turn.js +119 -0
- package/src/chat-pipeline/index.js +4 -1
- package/src/chat-pipeline/serial-trial.js +244 -210
- package/src/cli/commands/daemon.js +4 -4
- package/src/cli/commands/model/list-providers.js +1 -1
- package/src/cli/commands/provider/globalqwenwork-login.js +126 -0
- package/src/cli/commands/provider/index.js +2 -0
- package/src/cli/commands/provider/models.js +4 -1
- package/src/cli/commands/stats.js +20 -5
- package/src/cli/commands/system.js +2 -2
- package/src/cli/format.js +6 -0
- package/src/cli/policy.js +2 -2
- package/src/cli/provider-row.js +2 -2
- package/src/daemon.js +7 -2
- package/src/logs.js +14 -1
- package/src/model-trace.js +5 -2
- package/src/providers/AGENTS.md +46 -0
- package/src/providers/classify.js +1 -1
- package/src/providers/globalqwenwork/account-store.js +141 -0
- package/src/providers/globalqwenwork/constants.js +73 -0
- package/src/providers/globalqwenwork/cosy.js +123 -0
- package/src/providers/globalqwenwork/crypto.js +220 -0
- package/src/providers/globalqwenwork/http.js +22 -0
- package/src/providers/globalqwenwork/index.js +331 -0
- package/src/providers/globalqwenwork/payload.js +145 -0
- package/src/providers/globalqwenwork/rsa.js +56 -0
- package/src/providers/globalqwenwork/sse.js +270 -0
- package/src/providers/globalqwenwork/stream.js +132 -0
- package/src/providers/globalqwenwork/upstream.js +122 -0
- package/src/providers/globalqwenwork.js +1 -0
- package/src/providers/registry.js +8 -0
- package/src/providers/zcode/chat.js +11 -10
- package/src/providers/zcode/const.js +1 -1
- package/src/providers/zcode/context-shape.js +162 -0
- package/src/providers/zcode/headers.js +38 -0
- package/src/providers/zcode/sse.js +19 -2
- package/src/routes/AGENTS.md +37 -0
- package/src/routes/chat/broadband-handler.js +3 -2
- package/src/routes/chat/exhausted-handler.js +23 -8
- package/src/routes/chat/hedge-handler.js +3 -0
- package/src/routes/chat/local-handler.js +5 -1
- package/src/routes/chat/peer-handler.js +2 -0
- package/src/routes/chat/relay-pipeline.js +40 -14
- package/src/routes/chat/via-route-handler.js +2 -0
- package/src/routes/helpers.js +20 -4
- package/src/routes/stream-hold.js +138 -0
- package/src/routes/stream-scan.js +278 -0
- package/src/routes/stream.js +106 -158
- package/src/runtime/lifecycle-forensics.js +116 -0
- package/src/runtime/lifecycle-log.js +23 -0
- package/src/runtime/provider-gate.js +4 -3
- package/src/talk-log.js +226 -0
- package/src/timeline.js +5 -2
- package/src/usage/record.js +8 -1
- package/src/usage/report.js +39 -2
|
@@ -1,210 +1,244 @@
|
|
|
1
|
-
import { performance } from "node:perf_hooks";
|
|
2
|
-
import { injectReasoningContent } from "../reasoning.js";
|
|
3
|
-
import { runHook } from "../plugins.js";
|
|
4
|
-
import { errMsg, json } from "../routes/helpers.js";
|
|
5
|
-
import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
|
|
6
|
-
import { handleHedge } from "../routes/chat/hedge-handler.js";
|
|
7
|
-
import { handleLocalRelay } from "../routes/chat/local-handler.js";
|
|
8
|
-
import { handlePeerRelay } from "../routes/chat/peer-handler.js";
|
|
9
|
-
import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
|
|
10
|
-
import { handleViaRoute } from "../routes/chat/via-route-handler.js";
|
|
11
|
-
import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
|
|
12
|
-
import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
|
|
13
|
-
import { summarizeRequest, upstreamEcho } from "../model-trace.js";
|
|
14
|
-
import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
evt("upstream-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
1
|
+
import { performance } from "node:perf_hooks";
|
|
2
|
+
import { injectReasoningContent } from "../reasoning.js";
|
|
3
|
+
import { runHook } from "../plugins.js";
|
|
4
|
+
import { errMsg, json } from "../routes/helpers.js";
|
|
5
|
+
import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
|
|
6
|
+
import { handleHedge } from "../routes/chat/hedge-handler.js";
|
|
7
|
+
import { handleLocalRelay } from "../routes/chat/local-handler.js";
|
|
8
|
+
import { handlePeerRelay } from "../routes/chat/peer-handler.js";
|
|
9
|
+
import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
|
|
10
|
+
import { handleViaRoute } from "../routes/chat/via-route-handler.js";
|
|
11
|
+
import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
|
|
12
|
+
import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
|
|
13
|
+
import { summarizeRequest, upstreamEcho } from "../model-trace.js";
|
|
14
|
+
import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
|
|
15
|
+
import { emptyRetryCfg, emptyRaiseCap, withRaisedMaxTokens, emptyNudgeCfg, withEmptyNudge, computeNextDelay, emptyTurnBudgetMs, emptyTurnMinRaiseTo } from "./empty-turn.js";
|
|
16
|
+
|
|
17
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
18
|
+
|
|
19
|
+
function groupSkipReason(model) {
|
|
20
|
+
if (isHardLocalOnly(model)) return "provider local-only(禁组员,仅本机直连)";
|
|
21
|
+
if (isKeyProviderDirectOnly(model)) return "key provider default direct(仅本机直连,MSLXDFF_USE_GROUP_KEYS=1 可开组员)";
|
|
22
|
+
return "useGroup=off";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* 串行 trial — 从 engine.js 抽出的第二段:via-route 单路径 → 串行 trial →
|
|
27
|
+
* hedge/local/peer/broadband/exhausted。恒终结,返回 { done:true }。
|
|
28
|
+
*/
|
|
29
|
+
export async function runSerialTrial(ctx, deps = {}) {
|
|
30
|
+
const {
|
|
31
|
+
viaRoute = handleViaRoute,
|
|
32
|
+
hedge = handleHedge,
|
|
33
|
+
localRelay = handleLocalRelay,
|
|
34
|
+
peerRelay = handlePeerRelay,
|
|
35
|
+
broadbandRelay = handleBroadbandRelay,
|
|
36
|
+
exhaustedLocal = handleExhaustedLocal,
|
|
37
|
+
exhaustedAll = handleExhaustedAll,
|
|
38
|
+
} = deps;
|
|
39
|
+
const {
|
|
40
|
+
order, reqId, requested, body, hops, useAuto, lockModel, plugins,
|
|
41
|
+
auto, upstream, peers, groups, bus, token, canFallback, canForwardPeers,
|
|
42
|
+
perf0, stages, mark, evt, logCall, logError, done, handlerCtx,
|
|
43
|
+
res, startedAt, logs,
|
|
44
|
+
} = ctx;
|
|
45
|
+
const shareKeys = ctx.shareKeys ?? ctx.policy?.shareKeys ?? {};
|
|
46
|
+
const workbuddyUid = ctx.workbuddyUid ?? ctx.policy?.workbuddyUid ?? null;
|
|
47
|
+
|
|
48
|
+
let viaRouteLastErr = null;
|
|
49
|
+
if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers && shouldUseGroupForModel(requested)) {
|
|
50
|
+
try {
|
|
51
|
+
const vr = await viaRoute({ model: requested, body, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res, requested, useAuto, lockModel, auto });
|
|
52
|
+
if (vr.handled) return { done: true };
|
|
53
|
+
if (vr.lastErr) viaRouteLastErr = vr.lastErr;
|
|
54
|
+
} catch (e) {
|
|
55
|
+
evt("via-route-exception", { reqId, model: requested, error: errMsg(e) });
|
|
56
|
+
}
|
|
57
|
+
} else if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers) {
|
|
58
|
+
evt("group-skip", { reqId, model: requested, reason: `${groupSkipReason(requested)} (via-route)` });
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let lastErr = viaRouteLastErr;
|
|
62
|
+
// 请求级空轮等待累计(跨候选):阶梯是「每发候选」各算的,不设顶就会 N×(2+8+30) 把客户端吊在门外
|
|
63
|
+
let requestWaitedMs = 0;
|
|
64
|
+
candidate: for (let idx = 0; idx < order.length; idx++) {
|
|
65
|
+
const model = order[idx];
|
|
66
|
+
handlerCtx.model = model;
|
|
67
|
+
handlerCtx.orderLen = order.length;
|
|
68
|
+
handlerCtx.idx = idx;
|
|
69
|
+
evt("model-try", { reqId, model, idx, remaining: order.length - idx });
|
|
70
|
+
if (plugins?.length) {
|
|
71
|
+
const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
|
|
72
|
+
for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
|
|
73
|
+
if (bt.value === false || bt.value?.skip === true) { evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model }); continue; }
|
|
74
|
+
}
|
|
75
|
+
let upRes = null;
|
|
76
|
+
let forwarded = { ...injectReasoningContent(model, body), model };
|
|
77
|
+
if (plugins?.length) {
|
|
78
|
+
const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
|
|
79
|
+
for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
|
|
80
|
+
if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
|
|
81
|
+
}
|
|
82
|
+
const chatOpts = {};
|
|
83
|
+
if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
|
|
84
|
+
if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
|
|
85
|
+
if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
|
|
86
|
+
// reqId = 本次客户端请求的身份:供应商据此"同请求粘号"(重试不换号,只有 401/403/429/5xx 冷却才换)
|
|
87
|
+
if (reqId) chatOpts.reqId = reqId;
|
|
88
|
+
const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
|
|
89
|
+
// 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
|
|
90
|
+
// 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
|
|
91
|
+
const emptyCfg = emptyRetryCfg();
|
|
92
|
+
const raiseCap = emptyRaiseCap();
|
|
93
|
+
const nudgeCfg = emptyNudgeCfg();
|
|
94
|
+
let emptyRetried = 0;
|
|
95
|
+
let emptyWaitedMs = 0;
|
|
96
|
+
for (;;) {
|
|
97
|
+
const tUp = performance.now();
|
|
98
|
+
evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
|
|
99
|
+
try {
|
|
100
|
+
upRes = await upstream.chat(forwarded, chatOptsArg);
|
|
101
|
+
evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null, ...upstreamEcho(upRes) });
|
|
102
|
+
} catch (err) {
|
|
103
|
+
if (auto) await auto.recordError(model, { message: errMsg(err) });
|
|
104
|
+
lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
|
|
105
|
+
logError(model, 502, errMsg(err));
|
|
106
|
+
evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
|
|
107
|
+
upRes = null;
|
|
108
|
+
}
|
|
109
|
+
if (plugins?.length) {
|
|
110
|
+
runHook(plugins, "upstream:response", {
|
|
111
|
+
reqId, requested, model,
|
|
112
|
+
status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
|
|
113
|
+
ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
|
|
114
|
+
error: upRes instanceof Error ? errMsg(upRes) : null,
|
|
115
|
+
timing: upRes?._t ?? null,
|
|
116
|
+
}).catch(() => {});
|
|
117
|
+
}
|
|
118
|
+
mark(`up-${model}`);
|
|
119
|
+
if (upRes && upRes.status >= 400) {
|
|
120
|
+
const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
|
|
121
|
+
if (isAllowlistBlock) {
|
|
122
|
+
let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
|
|
123
|
+
let errBody = { error: `model not allowed for provider` };
|
|
124
|
+
try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
|
|
125
|
+
if (useAuto) {
|
|
126
|
+
logError(model, 403, errBody.error || "model not allowed");
|
|
127
|
+
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
|
|
128
|
+
lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
|
|
129
|
+
if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
|
|
130
|
+
return json(res, 403, errBody);
|
|
131
|
+
}
|
|
132
|
+
logError(model, 403, errBody.error || "model not allowed");
|
|
133
|
+
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
|
|
134
|
+
return json(res, 403, errBody);
|
|
135
|
+
}
|
|
136
|
+
if (auto) await auto.recordError(model, { status: upRes.status });
|
|
137
|
+
// 读失败响应体(clone 不影响后续 relay 转发原响应;1s 上限防流式错误体拖慢)
|
|
138
|
+
let upBody = "";
|
|
139
|
+
try {
|
|
140
|
+
upBody = String(await Promise.race([
|
|
141
|
+
upRes.clone().text(),
|
|
142
|
+
new Promise((r) => { const t = setTimeout(() => r(""), 1000); t.unref?.(); }),
|
|
143
|
+
])).replace(/\s+/g, " ").slice(0, 400);
|
|
144
|
+
} catch {}
|
|
145
|
+
const upMsg = upBody || `upstream ${upRes.status}`;
|
|
146
|
+
lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
|
|
147
|
+
logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}`);
|
|
148
|
+
evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null, ...upstreamEcho(upRes) });
|
|
149
|
+
upRes = null;
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
if (upRes) {
|
|
153
|
+
const isStream = Boolean(body.stream);
|
|
154
|
+
const d = hedgeDelayMs();
|
|
155
|
+
const hasPeers = Boolean(peers) && peers.ordered().length > 0;
|
|
156
|
+
const canUseGroup = shouldUseGroupForModel(model);
|
|
157
|
+
const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
|
|
158
|
+
if (doHedge) {
|
|
159
|
+
const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res, hedgeDelayMs: d });
|
|
160
|
+
if (hr.handled) return { done: true };
|
|
161
|
+
if (hr.lastErr) lastErr = hr.lastErr;
|
|
162
|
+
if (hr.upRes === null) upRes = null;
|
|
163
|
+
else if (hr.upRes) upRes = hr.upRes;
|
|
164
|
+
}
|
|
165
|
+
if (upRes) {
|
|
166
|
+
const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res });
|
|
167
|
+
if (lr.handled) {
|
|
168
|
+
// 空转重试后真拿到输出 = 降级但成功(WARN 语义):必须交代"第几次救回来的、白等了多久",
|
|
169
|
+
// 否则用户只看到"这一发变慢了",无从判断是重试在兜底还是上游真的死了。
|
|
170
|
+
// 「救回」必须有送达证据:handled:true 也可能来自下游已断开/零输出路径(见 ADR-0043),
|
|
171
|
+
// 只认 relay 报上来的 wrotePayload —— 没有正文到下游就不许记成功,否则日志在骗排障的人。
|
|
172
|
+
if (emptyRetried > 0 && lr.wrotePayload === true) {
|
|
173
|
+
evt("empty-turn-recovered", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, ...upstreamEcho(upRes) });
|
|
174
|
+
}
|
|
175
|
+
return { done: true };
|
|
176
|
+
}
|
|
177
|
+
if (lr.lastErr && isEmptyTurnError(lr.lastErr)) {
|
|
178
|
+
// 空转判据原文(finish_reason / 零正文 / 上游错误摘要)随事件落盘,排障不必再翻第二个文件
|
|
179
|
+
const _why = String(lr.lastErr.message || "").replace(/\s+/g, " ").trim().slice(0, 220);
|
|
180
|
+
const budgetMs = emptyTurnBudgetMs();
|
|
181
|
+
const budgetLeftMs = budgetMs - requestWaitedMs;
|
|
182
|
+
if (emptyRetried < emptyCfg.max && budgetLeftMs > 0) {
|
|
183
|
+
// 口径(用户定):正文为空就重试,最多 emptyCfg.max 次——不设"能不能送达"的前提,
|
|
184
|
+
// 大不了两次都空,反正不是无限重试。抬额度只加不减且有顶(默认 16384):
|
|
185
|
+
// 思考刷满 max_tokens 是零正文的主因,同参重拉必然复现,抬一次才算换了打法。
|
|
186
|
+
emptyRetried++;
|
|
187
|
+
const _before = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens) || null;
|
|
188
|
+
// 客户端没设额度也兜底发明一次:现网主流空轮就是「思考吃满 max_tokens、正文为零」,同参重拉必复现
|
|
189
|
+
const _raised = withRaisedMaxTokens(forwarded, raiseCap, emptyTurnMinRaiseTo());
|
|
190
|
+
let _after = null;
|
|
191
|
+
if (_raised !== forwarded) { forwarded = _raised; _after = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens); }
|
|
192
|
+
const _isLast = emptyRetried >= emptyCfg.max;
|
|
193
|
+
let _nudged = false;
|
|
194
|
+
if (nudgeCfg.enabled && _isLast) {
|
|
195
|
+
const _next = withEmptyNudge(forwarded, nudgeCfg.text);
|
|
196
|
+
if (_next !== forwarded) { forwarded = _next; _nudged = true; }
|
|
197
|
+
}
|
|
198
|
+
// 事件在 sleep **之前**发:对着日志能立刻看到"正在暂停 Nms 重拉",而不是等结果
|
|
199
|
+
// 带上"刚空转的是哪个号/哪个站":切号是重试驱动的,日志必须能自证
|
|
200
|
+
const stepIdx = emptyCfg.steps ? (emptyRetried - 1) % emptyCfg.steps.length : 0;
|
|
201
|
+
const rawDelayMs = computeNextDelay(emptyRetried - 1, emptyCfg.steps, lr.lastErr);
|
|
202
|
+
const delayMs = Math.min(rawDelayMs, budgetLeftMs); // 末次等待不越过请求级预算
|
|
203
|
+
emptyWaitedMs += delayMs;
|
|
204
|
+
requestWaitedMs += delayMs;
|
|
205
|
+
evt("empty-turn-retry", { reqId, model, retry: emptyRetried, step: stepIdx, max: emptyCfg.max, delayMs, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs, nudged: _nudged ? 1 : undefined, raiseFrom: _after != null ? _before : undefined, raiseTo: _after ?? undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
206
|
+
await sleep(delayMs);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
// 次数用尽(或被 MSLXDFF_EMPTY_TURN_RETRIES=0 关掉)→ 记一行"不再重试"再交回 failover
|
|
210
|
+
const _budgetOut = emptyRetried < emptyCfg.max && emptyTurnBudgetMs() - requestWaitedMs <= 0;
|
|
211
|
+
evt("empty-turn-exhausted", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs: emptyTurnBudgetMs(), budgetOut: _budgetOut ? 1 : undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
212
|
+
try { logError(model, 502, `空转重试 ${emptyRetried}/${emptyCfg.max} 后仍无输出${emptyCfg.max === 0 ? "(MSLXDFF_EMPTY_TURN_RETRIES=0 已关闭重试)" : _budgetOut ? `(请求级等待预算 ${emptyTurnBudgetMs()}ms 用尽)` : ""}:${_why}`); } catch {}
|
|
213
|
+
}
|
|
214
|
+
if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
|
|
215
|
+
return { done: true };
|
|
216
|
+
}
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
if (canForwardPeers) {
|
|
222
|
+
if (!shouldUseGroupForModel(model)) {
|
|
223
|
+
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (peer)` });
|
|
224
|
+
} else {
|
|
225
|
+
const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res });
|
|
226
|
+
if (pr.handled) return { done: true };
|
|
227
|
+
if (pr.lastErr) lastErr = pr.lastErr;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (groups) {
|
|
231
|
+
if (!shouldUseGroupForModel(model)) {
|
|
232
|
+
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (broadband)` });
|
|
233
|
+
} else {
|
|
234
|
+
const br = await broadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, attemptStartMs: performance.now(), stages, res, startedAt, plugins });
|
|
235
|
+
if (br.handled) return { done: true };
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (canFallback) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` }); continue; }
|
|
239
|
+
await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
|
|
240
|
+
return { done: true };
|
|
241
|
+
}
|
|
242
|
+
await exhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
|
|
243
|
+
return { done: true };
|
|
244
|
+
}
|
|
@@ -23,11 +23,11 @@ export async function handleRestart(args, VERSION) {
|
|
|
23
23
|
const alive = pid ? isPidAlive(pid) : false;
|
|
24
24
|
if (alive) {
|
|
25
25
|
console.log(`restarting daemon (pid ${pid})...`);
|
|
26
|
-
stopDaemon();
|
|
26
|
+
stopDaemon({ reason: "restart" });
|
|
27
27
|
await new Promise((r) => setTimeout(r, 300));
|
|
28
28
|
} else if (pid) {
|
|
29
29
|
console.log(`daemon pid ${pid} is stale (not running) — starting fresh...`);
|
|
30
|
-
try { stopDaemon(); } catch {}
|
|
30
|
+
try { stopDaemon({ reason: "restart" }); } catch {}
|
|
31
31
|
} else {
|
|
32
32
|
console.log(`daemon not running — starting...`);
|
|
33
33
|
}
|
|
@@ -58,7 +58,7 @@ export async function handlePort(args) {
|
|
|
58
58
|
setPort(port);
|
|
59
59
|
const daemon = readPid();
|
|
60
60
|
if (daemon) {
|
|
61
|
-
stopDaemon();
|
|
61
|
+
stopDaemon({ reason: "restart" });
|
|
62
62
|
startDaemon(["-port", String(port)]);
|
|
63
63
|
await waitForHealth(port, 4000);
|
|
64
64
|
console.log(`mslxdff restarted on port ${port} (pid ${readPid()})`);
|
|
@@ -108,7 +108,7 @@ export async function handleDebug(args) {
|
|
|
108
108
|
}
|
|
109
109
|
} catch {}
|
|
110
110
|
}
|
|
111
|
-
const { stopped, pid } = stopDaemon();
|
|
111
|
+
const { stopped, pid } = stopDaemon({ reason: "debug-takeover" });
|
|
112
112
|
if (stopped) console.log(`[debug] stopped background daemon (pid ${pid})`);
|
|
113
113
|
// 等旧 daemon 真正退出再抢端口(Windows 端口释放有延迟,否则 EADDRINUSE 会让 debug 立即崩)
|
|
114
114
|
if (stopped && pid) {
|
|
@@ -10,7 +10,7 @@ export async function renderOtherProviders({ pickedIds, ids, fullAliases }) {
|
|
|
10
10
|
try { _la2(); } catch {}
|
|
11
11
|
const configs = loadProviderConfigs();
|
|
12
12
|
const otherIds = Object.keys(configs).filter((k) => String(k).toLowerCase() !== "opencode");
|
|
13
|
-
const order2 = ["workbuddy", "cline", "openrouter", "bai", "qoder", "qwenwork", "traework", "zcode", "codearts"];
|
|
13
|
+
const order2 = ["workbuddy", "cline", "openrouter", "bai", "qoder", "qwenwork", "globalqwenwork", "traework", "zcode", "codearts"];
|
|
14
14
|
otherIds.sort((a, b) => {
|
|
15
15
|
const ia = order2.indexOf(a), ib = order2.indexOf(b);
|
|
16
16
|
if (ia !== -1 || ib !== -1) {
|