mslxdff 0.1.203 → 0.1.207

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (330) hide show
  1. package/README.md +3 -3
  2. package/bin/mslxdff.js +1 -1
  3. package/docs/cli_help_mini.md +146 -0
  4. package/package.json +38 -38
  5. package/plugins/README.md +14 -14
  6. package/plugins/prefer-model.mjs.example +14 -14
  7. package/src/anthropic/canonical-json.js +55 -0
  8. package/src/anthropic/count-tokens.js +61 -61
  9. package/src/anthropic/request-body.js +183 -126
  10. package/src/anthropic/request-shape.js +60 -0
  11. package/src/anthropic/session-id.js +58 -0
  12. package/src/anthropic/shared.js +18 -18
  13. package/src/anthropic/sse-aggregate.js +56 -56
  14. package/src/anthropic/tool-media.js +75 -0
  15. package/src/anthropic/translate.js +296 -287
  16. package/src/anthropic/web-search.js +361 -361
  17. package/src/api-kind.js +31 -31
  18. package/src/autostart.js +288 -288
  19. package/src/bench/probe.js +1 -1
  20. package/src/bench/runner.js +1 -1
  21. package/src/bench/via-probe.js +1 -1
  22. package/src/bench/via.js +144 -144
  23. package/src/bench/workbuddy-bench.js +1 -1
  24. package/src/chat/config.js +11 -11
  25. package/src/chat/cooling.js +81 -81
  26. package/src/chat/direct.js +58 -58
  27. package/src/chat/index.js +6 -6
  28. package/src/chat/spinner.js +38 -38
  29. package/src/chat/sse.js +69 -69
  30. package/src/chat/stats.js +247 -247
  31. package/src/chat/store.js +48 -48
  32. package/src/chat/tool-handlers.js +67 -67
  33. package/src/chat/tools.js +1 -1
  34. package/src/chat/upstream.js +59 -59
  35. package/src/chat-pipeline/auto-race.js +158 -158
  36. package/src/chat-pipeline/empty-turn.js +147 -147
  37. package/src/chat-pipeline/engine.js +17 -17
  38. package/src/chat-pipeline/index.js +8 -40
  39. package/src/chat-pipeline/serial-trial.js +279 -283
  40. package/src/chat-pipeline/terminal-verdict.js +132 -147
  41. package/src/chooser.js +54 -54
  42. package/src/claude-desktop/apply.js +101 -101
  43. package/src/claude-desktop/context1m.js +38 -38
  44. package/src/claude-desktop/doctor.js +146 -146
  45. package/src/claude-desktop/slots.js +152 -152
  46. package/src/cli/bootstrap.js +1 -1
  47. package/src/cli/commands/claude-desktop.js +284 -284
  48. package/src/cli/commands/daemon.js +231 -231
  49. package/src/cli/commands/model/list-live.js +22 -22
  50. package/src/cli/commands/model/list-providers.js +76 -76
  51. package/src/cli/commands/model/list-render.js +77 -77
  52. package/src/cli/commands/model/list-sort.js +52 -52
  53. package/src/cli/commands/model/list-tty.js +76 -76
  54. package/src/cli/commands/model/list.js +166 -166
  55. package/src/cli/commands/model/live-models.js +24 -24
  56. package/src/cli/commands/model/stats.js +43 -43
  57. package/src/cli/commands/model/status.js +47 -47
  58. package/src/cli/commands/model.js +51 -51
  59. package/src/cli/commands/provider/add.js +108 -108
  60. package/src/cli/commands/provider/allowlist.js +99 -99
  61. package/src/cli/commands/provider/cline-free.js +99 -99
  62. package/src/cli/commands/provider/cline-login.js +218 -218
  63. package/src/cli/commands/provider/cline-quota.js +65 -65
  64. package/src/cli/commands/provider/codearts-login.js +45 -45
  65. package/src/cli/commands/provider/config.js +119 -119
  66. package/src/cli/commands/provider/globalqwenwork-login.js +126 -126
  67. package/src/cli/commands/provider/keys-guard.js +50 -50
  68. package/src/cli/commands/provider/keys.js +161 -161
  69. package/src/cli/commands/provider/qoder-checkin.js +81 -81
  70. package/src/cli/commands/provider/qoder-login.js +102 -102
  71. package/src/cli/commands/provider/raccoon-login.js +180 -180
  72. package/src/cli/commands/provider/raccoon-password.js +162 -162
  73. package/src/cli/commands/provider/raccoon-quota.js +109 -109
  74. package/src/cli/commands/provider/traework-checkin.js +144 -144
  75. package/src/cli/commands/provider/traework-login.js +164 -164
  76. package/src/cli/commands/provider/workbuddy-login.js +153 -153
  77. package/src/cli/commands/provider/zcode-login.js +77 -77
  78. package/src/cli/commands/provider/zcode-quota.js +55 -55
  79. package/src/cli/commands/provider.js +1 -1
  80. package/src/cli/commands/stats.js +158 -149
  81. package/src/cli/commands/system.js +261 -261
  82. package/src/cli/commands/timezone.js +47 -47
  83. package/src/cli/commands/use-group.js +60 -60
  84. package/src/cli/commands/workbuddy.js +189 -189
  85. package/src/cli/format.js +141 -141
  86. package/src/cli/group-helpers.js +70 -70
  87. package/src/cli/help.js +80 -80
  88. package/src/cli/index.js +70 -70
  89. package/src/cli/interactive.js +121 -121
  90. package/src/cli/join-core.js +99 -99
  91. package/src/cli/join-wizard.js +175 -175
  92. package/src/cli/policy.js +136 -136
  93. package/src/cli/provider-row.js +126 -126
  94. package/src/cli/util.js +24 -24
  95. package/src/compat.js +53 -53
  96. package/src/events.js +38 -38
  97. package/src/free-lane.js +131 -131
  98. package/src/free-watcher.js +63 -63
  99. package/src/groups.js +262 -262
  100. package/src/logs.js +195 -195
  101. package/src/metrics.js +71 -62
  102. package/src/model-capabilities/context-lookup.js +65 -65
  103. package/src/model-capabilities/merge.js +142 -142
  104. package/src/model-trace.js +187 -187
  105. package/src/models.js +1 -1
  106. package/src/opencode-identity.js +72 -72
  107. package/src/peers.js +254 -254
  108. package/src/plugins.js +107 -107
  109. package/src/providers/AGENTS.md +49 -0
  110. package/src/providers/base.js +164 -164
  111. package/src/providers/classify.js +22 -22
  112. package/src/providers/cline/allowlist-sync.js +93 -93
  113. package/src/providers/cline/free-catalog.js +71 -71
  114. package/src/providers/cline/headers.js +37 -37
  115. package/src/providers/cline/index.js +130 -130
  116. package/src/providers/cline/models.js +150 -150
  117. package/src/providers/cline/usage.js +274 -274
  118. package/src/providers/codearts/auth-pool.js +148 -148
  119. package/src/providers/codearts/concurrency.js +34 -34
  120. package/src/providers/codearts/const.js +65 -65
  121. package/src/providers/codearts/dpop.js +95 -95
  122. package/src/providers/codearts/index.js +175 -175
  123. package/src/providers/codearts/login.js +214 -214
  124. package/src/providers/codearts/models.js +182 -182
  125. package/src/providers/codearts/sign.js +96 -96
  126. package/src/providers/codearts/sse.js +204 -204
  127. package/src/providers/codearts/stream.js +209 -209
  128. package/src/providers/codearts/sts.js +124 -124
  129. package/src/providers/codearts.js +1 -1
  130. package/src/providers/dispatcher.js +146 -146
  131. package/src/providers/generic.js +156 -156
  132. package/src/providers/globalqwenwork/account-store.js +141 -141
  133. package/src/providers/globalqwenwork/constants.js +73 -73
  134. package/src/providers/globalqwenwork/crypto.js +2 -2
  135. package/src/providers/globalqwenwork/http.js +2 -2
  136. package/src/providers/globalqwenwork/index.js +331 -331
  137. package/src/providers/globalqwenwork/payload.js +147 -147
  138. package/src/providers/globalqwenwork/rsa.js +2 -2
  139. package/src/providers/globalqwenwork/sse.js +2 -2
  140. package/src/providers/globalqwenwork/stream.js +2 -2
  141. package/src/providers/globalqwenwork.js +1 -1
  142. package/src/providers/globalworkbuddy/account-store.js +198 -198
  143. package/src/providers/globalworkbuddy/auth.js +97 -97
  144. package/src/providers/globalworkbuddy/catalog.js +138 -138
  145. package/src/providers/globalworkbuddy/constants.js +73 -73
  146. package/src/providers/globalworkbuddy/device-auth.js +120 -120
  147. package/src/providers/globalworkbuddy/errors.js +98 -98
  148. package/src/providers/globalworkbuddy/headers.js +89 -89
  149. package/src/providers/globalworkbuddy/models.js +96 -96
  150. package/src/providers/globalworkbuddy/payload.js +80 -80
  151. package/src/providers/index.js +5 -5
  152. package/src/providers/model-id.js +172 -172
  153. package/src/providers/opencode.js +19 -19
  154. package/src/providers/openrouter.js +31 -31
  155. package/src/providers/qoder/account-store.js +106 -106
  156. package/src/providers/qoder/aggregate.js +60 -60
  157. package/src/providers/qoder/baseprompt.json +591 -591
  158. package/src/providers/qoder/checkin.js +160 -160
  159. package/src/providers/qoder/constants.js +44 -44
  160. package/src/providers/qoder/encode.js +41 -41
  161. package/src/providers/qoder/fingerprint.js +38 -38
  162. package/src/providers/qoder/models.js +77 -77
  163. package/src/providers/qoder/oauth.js +84 -84
  164. package/src/providers/qoder/payload.js +162 -162
  165. package/src/providers/qoder/request.js +27 -27
  166. package/src/providers/qoder/session.js +134 -134
  167. package/src/providers/qoder/sticky.js +38 -38
  168. package/src/providers/qoder/stream.js +116 -116
  169. package/src/providers/qoder.js +1 -1
  170. package/src/providers/raccoon/account-store.js +216 -216
  171. package/src/providers/raccoon/auth.js +67 -67
  172. package/src/providers/raccoon/chat.js +228 -228
  173. package/src/providers/raccoon/const.js +180 -180
  174. package/src/providers/raccoon/credits.js +185 -185
  175. package/src/providers/raccoon/crypto.js +43 -43
  176. package/src/providers/raccoon/envelope.js +31 -31
  177. package/src/providers/raccoon/headers.js +45 -45
  178. package/src/providers/raccoon/index.js +216 -216
  179. package/src/providers/raccoon/login.js +279 -279
  180. package/src/providers/raccoon/models.js +160 -160
  181. package/src/providers/raccoon/password-login.js +102 -102
  182. package/src/providers/raccoon/qr-codewords.js +115 -115
  183. package/src/providers/raccoon/qr.js +236 -236
  184. package/src/providers/raccoon/sse.js +135 -135
  185. package/src/providers/responses-channel.js +120 -120
  186. package/src/providers/share-keys.js +53 -53
  187. package/src/providers/traework/account-store.js +234 -234
  188. package/src/providers/traework/chat.js +217 -217
  189. package/src/providers/traework/checkin.js +77 -77
  190. package/src/providers/traework/constants.js +41 -41
  191. package/src/providers/traework/errors.js +52 -52
  192. package/src/providers/traework/headers.js +79 -79
  193. package/src/providers/traework/index.js +132 -132
  194. package/src/providers/traework/market-identity.js +116 -116
  195. package/src/providers/traework/models.js +98 -98
  196. package/src/providers/traework/payload.js +86 -86
  197. package/src/providers/traework/sse.js +184 -184
  198. package/src/providers/traework/token.js +52 -52
  199. package/src/providers/traework.js +2 -2
  200. package/src/providers/workbuddy/account-store.js +186 -186
  201. package/src/providers/workbuddy/auth.js +158 -158
  202. package/src/providers/workbuddy/balance.js +86 -86
  203. package/src/providers/workbuddy/cat-travel.js +125 -125
  204. package/src/providers/workbuddy/checkin.js +75 -75
  205. package/src/providers/workbuddy/desktop-info.js +78 -78
  206. package/src/providers/workbuddy/growth-api.js +67 -67
  207. package/src/providers/workbuddy/growth-plans.js +63 -63
  208. package/src/providers/workbuddy/growth.js +181 -181
  209. package/src/providers/workbuddy/index.js +217 -217
  210. package/src/providers/workbuddy/models.js +126 -126
  211. package/src/providers/workbuddy/payload.js +141 -141
  212. package/src/providers/workbuddy/rotation-log.js +83 -83
  213. package/src/providers/workbuddy/sanitize-tools.js +61 -61
  214. package/src/providers/workbuddy/sdk-chat.js +26 -26
  215. package/src/providers/workbuddy-balance.js +78 -78
  216. package/src/providers/zcode/account-store.js +129 -129
  217. package/src/providers/zcode/auth.js +28 -28
  218. package/src/providers/zcode/chat.js +179 -179
  219. package/src/providers/zcode/const.js +54 -54
  220. package/src/providers/zcode/context-shape.js +162 -162
  221. package/src/providers/zcode/headers.js +93 -93
  222. package/src/providers/zcode/index.js +124 -124
  223. package/src/providers/zcode/models.js +65 -65
  224. package/src/providers/zcode/oauth.js +120 -120
  225. package/src/providers/zcode/quota.js +176 -176
  226. package/src/providers/zcode/sse.js +196 -196
  227. package/src/readline-compat.js +25 -25
  228. package/src/responses/translate.js +311 -311
  229. package/src/routes/AGENTS.md +37 -0
  230. package/src/routes/chat/broadband-handler.js +72 -72
  231. package/src/routes/chat/empty-turn-report.js +31 -31
  232. package/src/routes/chat/exhausted-handler.js +87 -87
  233. package/src/routes/chat/hedge-handler.js +130 -130
  234. package/src/routes/chat/index.js +1 -1
  235. package/src/routes/chat/local-handler.js +63 -63
  236. package/src/routes/chat/peer-handler.js +81 -81
  237. package/src/routes/chat.js +3 -3
  238. package/src/routes/fallback.js +88 -88
  239. package/src/routes/groups-relay.js +214 -214
  240. package/src/routes/groups.js +122 -122
  241. package/src/routes/helpers.js +93 -83
  242. package/src/routes/index.js +147 -147
  243. package/src/routes/messages-route.js +375 -365
  244. package/src/routes/peers.js +10 -2
  245. package/src/routes/relay-queue.js +181 -181
  246. package/src/routes/relay.js +60 -60
  247. package/src/routes/responses-route.js +213 -213
  248. package/src/routes/stream-body.js +84 -84
  249. package/src/routes/stream-hold.js +337 -337
  250. package/src/routes/stream-scan.js +366 -366
  251. package/src/routes.js +3 -3
  252. package/src/runtime/auto-update.js +113 -113
  253. package/src/runtime/broadband-stream.js +79 -79
  254. package/src/runtime/broadband.js +103 -103
  255. package/src/runtime/group-sync.js +81 -81
  256. package/src/runtime/keep-warm.js +120 -120
  257. package/src/runtime/lifecycle-forensics.js +116 -116
  258. package/src/runtime/lifecycle-log.js +87 -87
  259. package/src/runtime/log-stamp.js +73 -73
  260. package/src/runtime/proc-info.js +137 -137
  261. package/src/runtime/qoder-checkin.js +93 -93
  262. package/src/runtime/raccoon-checkin.js +126 -126
  263. package/src/runtime/server-lifecycle.js +189 -189
  264. package/src/runtime/traework-checkin.js +118 -118
  265. package/src/runtime/workbuddy-checkin.js +120 -120
  266. package/src/runtime/workbuddy-growth.js +133 -133
  267. package/src/state/facade.js +58 -58
  268. package/src/state/memory.js +162 -162
  269. package/src/state/merge.js +25 -25
  270. package/src/state/migrations/cline-unify.js +94 -94
  271. package/src/state/migrations.js +33 -33
  272. package/src/state/persist.js +42 -42
  273. package/src/state/provider-config.js +142 -142
  274. package/src/state/schemas/group.js +31 -31
  275. package/src/state/schemas/model.js +94 -94
  276. package/src/state/schemas/peer.js +31 -31
  277. package/src/state/schemas/port.js +10 -10
  278. package/src/state/schemas/provider.js +202 -202
  279. package/src/state/schemas/timezone.js +70 -70
  280. package/src/state/schemas/token.js +48 -48
  281. package/src/state/schemas/use-group.js +102 -102
  282. package/src/state/store.js +176 -176
  283. package/src/state.js +1 -1
  284. package/src/sync-claude-desktop.js +234 -234
  285. package/src/sync-claude.js +152 -152
  286. package/src/sync-codex.js +116 -116
  287. package/src/sync-workbuddy.js +158 -158
  288. package/src/talk-full-redact.js +129 -129
  289. package/src/talk-full.js +131 -131
  290. package/src/talk-log.js +257 -257
  291. package/src/text-tokens.js +52 -52
  292. package/src/thinking-effort.js +67 -67
  293. package/src/time.js +106 -106
  294. package/src/timeline.js +26 -26
  295. package/src/transport/index.js +1 -1
  296. package/src/upstream-engine/index.js +74 -74
  297. package/src/upstream-engine/mode.js +15 -15
  298. package/src/upstream-engine/sdk/attempt.js +221 -221
  299. package/src/upstream-engine/sdk/chat.js +28 -28
  300. package/src/upstream-engine/sdk/convert.js +192 -192
  301. package/src/upstream-engine/sdk/diagnose.js +62 -62
  302. package/src/upstream-engine/sdk/dispatch.js +55 -55
  303. package/src/upstream-engine/sdk/responses.js +176 -176
  304. package/src/upstream-engine/sdk/sse.js +166 -166
  305. package/src/upstream-responses.js +305 -305
  306. package/src/usage/cache-fields.js +58 -0
  307. package/src/usage/record.js +109 -106
  308. package/src/usage/report.js +209 -195
  309. package/src/v6pool/claim.js +157 -157
  310. package/src/v6pool/config.js +65 -65
  311. package/src/v6pool/detect.js +130 -130
  312. package/src/v6pool/index.js +152 -152
  313. package/src/v6pool/prefix.js +140 -140
  314. package/src/v6pool/probe.js +40 -40
  315. package/src/v6pool/report.js +104 -104
  316. package/src/v6pool/rotate.js +263 -263
  317. package/src/warp-pool/config.js +74 -74
  318. package/src/warp-pool/health.js +106 -106
  319. package/src/warp-pool/identity.js +141 -141
  320. package/src/warp-pool/index.js +358 -358
  321. package/src/warp-pool/keys.js +29 -29
  322. package/src/warp-pool/nodes.js +95 -95
  323. package/src/warp-pool/ports.js +160 -160
  324. package/src/warp-pool/register.js +113 -113
  325. package/src/warp-pool/report.js +232 -232
  326. package/src/warp-pool/revive.js +168 -168
  327. package/src/warp-pool/rotate.js +214 -214
  328. package/src/warp-pool/strategy.js +43 -43
  329. package/src/warp-pool/verdict.js +69 -69
  330. package/src/warp-pool/wireproxy.js +162 -162
@@ -1,283 +1,279 @@
1
- import { performance } from "node:perf_hooks";
2
- import { injectReasoningContent } from "../reasoning.js";
3
- import { runHook } from "../plugins.js";
4
- import { errMsg, json } from "../routes/helpers.js";
5
- import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
6
- import { handleHedge } from "../routes/chat/hedge-handler.js";
7
- import { handleLocalRelay } from "../routes/chat/local-handler.js";
8
- import { handlePeerRelay } from "../routes/chat/peer-handler.js";
9
- import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
10
- import { handleViaRoute } from "../routes/chat/via-route-handler.js";
11
- import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
12
- import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
13
- import { summarizeRequest, upstreamEcho } from "../model-trace.js";
14
- import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
15
- import { emptyRetryCfg, emptyRaiseCap, withRaisedMaxTokens, withOutputFloor, emptyNudgeCfg, withEmptyNudge, computeNextDelay, emptyTurnBudgetMs, emptyTurnMinRaiseTo } from "./empty-turn.js";
16
- import { classifyTerminalUpstream, readVerdictText, terminalShortcircuitOff, isPoolSwitchWorthy } from "./terminal-verdict.js";
17
-
18
- const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
19
-
20
- function groupSkipReason(model) {
21
- if (isHardLocalOnly(model)) return "provider local-only(禁组员,仅本机直连)";
22
- if (isKeyProviderDirectOnly(model)) return "key provider default direct(仅本机直连,MSLXDFF_USE_GROUP_KEYS=1 可开组员)";
23
- return "useGroup=off";
24
- }
25
-
26
- /**
27
- * 串行 trial — 从 engine.js 抽出的第二段:via-route 单路径 → 串行 trial →
28
- * hedge/local/peer/broadband/exhausted。恒终结,返回 { done:true }。
29
- */
30
- export async function runSerialTrial(ctx, deps = {}) {
31
- const {
32
- viaRoute = handleViaRoute,
33
- hedge = handleHedge,
34
- localRelay = handleLocalRelay,
35
- peerRelay = handlePeerRelay,
36
- broadbandRelay = handleBroadbandRelay,
37
- exhaustedLocal = handleExhaustedLocal,
38
- exhaustedAll = handleExhaustedAll,
39
- } = deps;
40
- const {
41
- order, reqId, requested, body, hops, useAuto, lockModel, plugins,
42
- auto, upstream, peers, groups, bus, token, canFallback, canForwardPeers, poolFallback,
43
- perf0, stages, mark, evt, logCall, logError, done, handlerCtx,
44
- res, startedAt, logs,
45
- } = ctx;
46
- const shareKeys = ctx.shareKeys ?? ctx.policy?.shareKeys ?? {};
47
- const workbuddyUid = ctx.workbuddyUid ?? ctx.policy?.workbuddyUid ?? null;
48
-
49
- let viaRouteLastErr = null;
50
- if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers && shouldUseGroupForModel(requested)) {
51
- try {
52
- const vr = await viaRoute({ model: requested, body, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res, requested, useAuto, lockModel, auto });
53
- if (vr.handled) return { done: true };
54
- if (vr.lastErr) viaRouteLastErr = vr.lastErr;
55
- } catch (e) {
56
- evt("via-route-exception", { reqId, model: requested, error: errMsg(e) });
57
- }
58
- } else if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers) {
59
- evt("group-skip", { reqId, model: requested, reason: `${groupSkipReason(requested)} (via-route)` });
60
- }
61
-
62
- let lastErr = viaRouteLastErr;
63
- // BUG-42 同池互备判据:poolWorthy 记「本模型直连时被上游判过额度/终局」,接力揉的 502 冲不掉它(详见 ADR-0061)。
64
- let poolWorthy = false;
65
- const canFailover = (err) =>
66
- canFallback && (!poolFallback || isPoolSwitchWorthy(err ?? lastErr) || poolWorthy);
67
- // 请求级空轮等待累计(跨候选):阶梯是「每发候选」各算的,不设顶就会 N×(2+8+30) 把客户端吊在门外
68
- let requestWaitedMs = 0;
69
- candidate: for (let idx = 0; idx < order.length; idx++) {
70
- const model = order[idx];
71
- poolWorthy = false; // 每候选独立记账:上一发 429 不代表这一发也 429
72
- handlerCtx.model = model;
73
- handlerCtx.orderLen = order.length;
74
- handlerCtx.idx = idx;
75
- evt("model-try", { reqId, model, idx, remaining: order.length - idx });
76
- if (plugins?.length) {
77
- const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
78
- for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
79
- if (bt.value === false || bt.value?.skip === true) { evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model }); continue; }
80
- }
81
- let upRes = null;
82
- let forwarded = { ...injectReasoningContent(model, body), model };
83
- // 首发输出额度托底(plugin hook 前——plugin 有最终发言权,硬底线只约束客户端直发 body)
84
- const _floored = withOutputFloor(forwarded);
85
- if (_floored !== forwarded) {
86
- const _fk = ["max_tokens", "max_completion_tokens"].find((k) => _floored[k] !== forwarded[k]) || "max_tokens";
87
- evt("output-floor", { reqId, model, key: _fk, raiseFrom: Number(forwarded[_fk]), raiseTo: Number(_floored[_fk]) });
88
- forwarded = _floored;
89
- }
90
- if (plugins?.length) {
91
- const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
92
- for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
93
- if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
94
- }
95
- const chatOpts = {};
96
- if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
97
- if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
98
- if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
99
- // reqId = 本次客户端请求的身份:供应商据此"同请求粘号"(重试不换号,只有 401/403/429/5xx 冷却才换)
100
- if (reqId) chatOpts.reqId = reqId;
101
- const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
102
- // 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
103
- // 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
104
- const emptyCfg = emptyRetryCfg();
105
- const raiseCap = emptyRaiseCap();
106
- const nudgeCfg = emptyNudgeCfg();
107
- let emptyRetried = 0;
108
- let emptyWaitedMs = 0;
109
- for (;;) {
110
- const tUp = performance.now();
111
- evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
112
- try {
113
- upRes = await upstream.chat(forwarded, chatOptsArg);
114
- evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null, ...upstreamEcho(upRes) });
115
- } catch (err) {
116
- if (auto) await auto.recordError(model, { message: errMsg(err) });
117
- lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
118
- logError(model, 502, errMsg(err));
119
- evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
120
- upRes = null;
121
- }
122
- if (plugins?.length) {
123
- runHook(plugins, "upstream:response", {
124
- reqId, requested, model,
125
- status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
126
- ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
127
- error: upRes instanceof Error ? errMsg(upRes) : null,
128
- timing: upRes?._t ?? null,
129
- }).catch(() => {});
130
- }
131
- mark(`up-${model}`);
132
- if (upRes && upRes.status >= 400) {
133
- const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
134
- if (isAllowlistBlock) {
135
- let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
136
- let errBody = { error: `model not allowed for provider` };
137
- try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
138
- if (useAuto) {
139
- logError(model, 403, errBody.error || "model not allowed");
140
- evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
141
- lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
142
- if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
143
- return json(res, 403, errBody);
144
- }
145
- logError(model, 403, errBody.error || "model not allowed");
146
- evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
147
- return json(res, 403, errBody);
148
- }
149
- if (auto) await auto.recordError(model, { status: upRes.status });
150
- // 读失败响应体(clone 不影响后续 relay 转发原响应;1s 上限防流式错误体拖慢)。
151
- // 两份文本分工写死:`upRaw` 喂判据(≤4KB,反向护栏不能被 400 字摘要喂不到——多 provider 明细体
152
- // 如 OpenRouter 的 `metadata.raw[]` 会把 rate limit 排在 model not found 之后),`upBody` 供日志摘要。
153
- let upRaw = "";
154
- let upBody = "";
155
- upRaw = await readVerdictText(upRes);
156
- upBody = upRaw.replace(/\s+/g, " ").slice(0, 400);
157
- const upMsg = upBody || `upstream ${upRes.status}`;
158
- lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
159
- // 终局判决短路(判据与现网取证见 terminal-verdict.js)
160
- const verdict = classifyTerminalUpstream(upRes.status, upRaw);
161
- const terminalOn = verdict.terminal && !terminalShortcircuitOff();
162
- // BUG-42 记账点:本机直连时上游判过额度/终局 ⇒ 换兄弟有账可依(接力冲不掉,详见 ADR-0061)
163
- if (isPoolSwitchWorthy({ status: upRes.status, terminal: verdict.terminal })) poolWorthy = true;
164
- // 一行话把「判决」与「原文」一起落 errors.log(不分两行:同一发失败在 errors.log 里只该有一条)
165
- logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}${terminalOn ? ` · 终局判决 ${verdict.kind}(跳过组员接力)` : ""}`);
166
- evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null, ...upstreamEcho(upRes) });
167
- if (terminalOn) {
168
- evt("upstream-terminal", { reqId, model, status: upRes.status, kind: verdict.kind, marker: verdict.marker, peersSkipped: true });
169
- // ASCII 具名头。**不**复用 x-mslxdff-notice:那头的既定语义是「已被别的模型代答」(与
170
- // x-mslxdff-fallback 成组,src/routes/fallback.js:27-39),拿它报「没代答、直接拒」会把按头
171
- // 判是否换模型的下游 UI 读反。头值只填 ASCII 枚举,中文留在 evt/logError/timeline(文件是 UTF-8)。
172
- try { if (!res.headersSent && typeof res.setHeader === "function") res.setHeader("x-mslxdff-terminal", verdict.kind); } catch {}
173
- if (canFallback && idx < order.length - 1) {
174
- // 判决只对**这个模型**成立 → 换下一候选。这里刻意不挂 upstream:这条 Response 的 body 没被
175
- // 消费,挂在 lastErr 上带到收场会白占连接(收场只 relay 最后一发),故直接 cancel 后只留文本。
176
- lastErr = { model, upstream: null, status: upRes.status, message: upMsg, terminal: verdict.kind };
177
- try { const cx = upRes.body?.cancel?.(); if (cx && typeof cx.catch === "function") cx.catch(() => {}); } catch {}
178
- evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `terminal ${verdict.kind}: ${upMsg.slice(0, 80)}` });
179
- continue candidate;
180
- }
181
- // 收场交回既有出口:exhaustedLocal 见 lastErr.upstream 会 relay **原响应**(原状态码 + 原字节 +
182
- // 原头),并自带 logCall / result / client-response / talk 语料落盘。自己 json() 反而劣化:
183
- // 只能给被截到 400 字的二手摘要、JSON.parse 失败时退成字符串 error(SDK 读 err.error.message
184
- // 拿到 undefined),且 SSE 已 flush 时 helpers 的 in-band 错误帧会把对象写成 [object Object]。
185
- await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
186
- return { done: true };
187
- }
188
- upRes = null;
189
- break;
190
- }
191
- if (upRes) {
192
- const isStream = Boolean(body.stream);
193
- const d = hedgeDelayMs();
194
- const hasPeers = Boolean(peers) && peers.ordered().length > 0;
195
- const canUseGroup = shouldUseGroupForModel(model);
196
- const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
197
- if (doHedge) {
198
- const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res, hedgeDelayMs: d });
199
- if (hr.handled) return { done: true };
200
- if (hr.lastErr) lastErr = hr.lastErr;
201
- if (hr.upRes === null) upRes = null;
202
- else if (hr.upRes) upRes = hr.upRes;
203
- }
204
- if (upRes) {
205
- const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res });
206
- if (lr.handled) {
207
- // 空转重试后真拿到输出 = 降级但成功(WARN 语义):必须交代"第几次救回来的、白等了多久",
208
- // 否则用户只看到"这一发变慢了",无从判断是重试在兜底还是上游真的死了。
209
- // 「救回」必须有送达证据:handled:true 也可能来自下游已断开/零输出路径(见 ADR-0043),
210
- // 只认 relay 报上来的 wrotePayload —— 没有正文到下游就不许记成功,否则日志在骗排障的人。
211
- if (emptyRetried > 0 && lr.wrotePayload === true) {
212
- evt("empty-turn-recovered", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, ...upstreamEcho(upRes) });
213
- }
214
- return { done: true };
215
- }
216
- if (lr.lastErr && isEmptyTurnError(lr.lastErr)) {
217
- // 空转判据原文(finish_reason / 零正文 / 上游错误摘要)随事件落盘,排障不必再翻第二个文件
218
- const _why = String(lr.lastErr.message || "").replace(/\s+/g, " ").trim().slice(0, 220);
219
- const budgetMs = emptyTurnBudgetMs();
220
- const budgetLeftMs = budgetMs - requestWaitedMs;
221
- if (emptyRetried < emptyCfg.max && budgetLeftMs > 0) {
222
- // 口径(用户定):正文为空就重试,最多 emptyCfg.max 次——不设"能不能送达"的前提,
223
- // 大不了两次都空,反正不是无限重试。抬额度只加不减且有顶(默认 16384):
224
- // 思考刷满 max_tokens 是零正文的主因,同参重拉必然复现,抬一次才算换了打法。
225
- emptyRetried++;
226
- const _before = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens) || null;
227
- // 客户端没设额度也兜底发明一次:现网主流空轮就是「思考吃满 max_tokens、正文为零」,同参重拉必复现
228
- const _raised = withRaisedMaxTokens(forwarded, raiseCap, emptyTurnMinRaiseTo());
229
- let _after = null;
230
- if (_raised !== forwarded) { forwarded = _raised; _after = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens); }
231
- const _isLast = emptyRetried >= emptyCfg.max;
232
- let _nudged = false;
233
- if (nudgeCfg.enabled && _isLast) {
234
- const _next = withEmptyNudge(forwarded, nudgeCfg.text);
235
- if (_next !== forwarded) { forwarded = _next; _nudged = true; }
236
- }
237
- // 事件在 sleep **之前**发:对着日志能立刻看到"正在暂停 Nms 重拉",而不是等结果
238
- // 带上"刚空转的是哪个号/哪个站":切号是重试驱动的,日志必须能自证
239
- const stepIdx = emptyCfg.steps ? (emptyRetried - 1) % emptyCfg.steps.length : 0;
240
- const rawDelayMs = computeNextDelay(emptyRetried - 1, emptyCfg.steps, lr.lastErr);
241
- const delayMs = Math.min(rawDelayMs, budgetLeftMs); // 末次等待不越过请求级预算
242
- emptyWaitedMs += delayMs;
243
- requestWaitedMs += delayMs;
244
- evt("empty-turn-retry", { reqId, model, retry: emptyRetried, step: stepIdx, max: emptyCfg.max, delayMs, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs, nudged: _nudged ? 1 : undefined, raiseFrom: _after != null ? _before : undefined, raiseTo: _after ?? undefined, reason: _why, ...upstreamEcho(upRes) });
245
- await sleep(delayMs);
246
- continue;
247
- }
248
- // 次数用尽(或被 MSLXDFF_EMPTY_TURN_RETRIES=0 关掉)→ 记一行"不再重试"再交回 failover
249
- const _budgetOut = emptyRetried < emptyCfg.max && emptyTurnBudgetMs() - requestWaitedMs <= 0;
250
- evt("empty-turn-exhausted", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs: emptyTurnBudgetMs(), budgetOut: _budgetOut ? 1 : undefined, reason: _why, ...upstreamEcho(upRes) });
251
- try { logError(model, 502, `空转重试 ${emptyRetried}/${emptyCfg.max} 后仍无输出${emptyCfg.max === 0 ? "(MSLXDFF_EMPTY_TURN_RETRIES=0 已关闭重试)" : _budgetOut ? `(请求级等待预算 ${emptyTurnBudgetMs()}ms 用尽)` : ""}:${_why}`); } catch {}
252
- }
253
- if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
254
- return { done: true };
255
- }
256
- break;
257
- }
258
- break;
259
- }
260
- if (canForwardPeers) {
261
- if (!shouldUseGroupForModel(model)) {
262
- evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (peer)` });
263
- } else {
264
- const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res });
265
- if (pr.handled) return { done: true };
266
- if (pr.lastErr) lastErr = pr.lastErr;
267
- }
268
- }
269
- if (groups) {
270
- if (!shouldUseGroupForModel(model)) {
271
- evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (broadband)` });
272
- } else {
273
- const br = await broadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, attemptStartMs: performance.now(), stages, res, startedAt, plugins });
274
- if (br.handled) return { done: true };
275
- }
276
- }
277
- if (canFailover(lastErr)) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, pool: poolFallback || undefined, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` }); continue; }
278
- await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
279
- return { done: true };
280
- }
281
- await exhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
282
- return { done: true };
283
- }
1
+ import { performance } from "node:perf_hooks";
2
+ import { injectReasoningContent } from "../reasoning.js";
3
+ import { runHook } from "../plugins.js";
4
+ import { errMsg, json } from "../routes/helpers.js";
5
+ import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
6
+ import { handleHedge } from "../routes/chat/hedge-handler.js";
7
+ import { handleLocalRelay } from "../routes/chat/local-handler.js";
8
+ import { handlePeerRelay } from "../routes/chat/peer-handler.js";
9
+ import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
10
+ import { handleViaRoute } from "../routes/chat/via-route-handler.js";
11
+ import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
12
+ import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
13
+ import { summarizeRequest, upstreamEcho } from "../model-trace.js";
14
+ import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
15
+ import { emptyRetryCfg, emptyRaiseCap, withRaisedMaxTokens, withOutputFloor, emptyNudgeCfg, withEmptyNudge, computeNextDelay, emptyTurnBudgetMs, emptyTurnMinRaiseTo } from "./empty-turn.js";
16
+ import { classifyTerminalUpstream, readVerdictText, terminalShortcircuitOff } from "./terminal-verdict.js";
17
+
18
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
19
+
20
+ function groupSkipReason(model) {
21
+ if (isHardLocalOnly(model)) return "provider local-only(禁组员,仅本机直连)";
22
+ if (isKeyProviderDirectOnly(model)) return "key provider default direct(仅本机直连,MSLXDFF_USE_GROUP_KEYS=1 可开组员)";
23
+ return "useGroup=off";
24
+ }
25
+
26
+ /**
27
+ * 串行 trial — 从 engine.js 抽出的第二段:via-route 单路径 → 串行 trial →
28
+ * hedge/local/peer/broadband/exhausted。恒终结,返回 { done:true }。
29
+ */
30
+ export async function runSerialTrial(ctx, deps = {}) {
31
+ const {
32
+ viaRoute = handleViaRoute,
33
+ hedge = handleHedge,
34
+ localRelay = handleLocalRelay,
35
+ peerRelay = handlePeerRelay,
36
+ broadbandRelay = handleBroadbandRelay,
37
+ exhaustedLocal = handleExhaustedLocal,
38
+ exhaustedAll = handleExhaustedAll,
39
+ } = deps;
40
+ const {
41
+ order, reqId, requested, body, hops, useAuto, lockModel, plugins,
42
+ auto, upstream, peers, groups, bus, token, canFallback, canForwardPeers,
43
+ perf0, stages, mark, evt, logCall, logError, done, handlerCtx,
44
+ res, startedAt, logs,
45
+ } = ctx;
46
+ const shareKeys = ctx.shareKeys ?? ctx.policy?.shareKeys ?? {};
47
+ const workbuddyUid = ctx.workbuddyUid ?? ctx.policy?.workbuddyUid ?? null;
48
+
49
+ let viaRouteLastErr = null;
50
+ if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers && shouldUseGroupForModel(requested)) {
51
+ try {
52
+ const vr = await viaRoute({ model: requested, body, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res, requested, useAuto, lockModel, auto });
53
+ if (vr.handled) return { done: true };
54
+ if (vr.lastErr) viaRouteLastErr = vr.lastErr;
55
+ } catch (e) {
56
+ evt("via-route-exception", { reqId, model: requested, error: errMsg(e) });
57
+ }
58
+ } else if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers) {
59
+ evt("group-skip", { reqId, model: requested, reason: `${groupSkipReason(requested)} (via-route)` });
60
+ }
61
+
62
+ let lastErr = viaRouteLastErr;
63
+ // 指定模型只走单候选(order=[requested]),组员/组长接力在同模型内进行;候选间不换模型。
64
+ // canFallback 仅对 auto 的多候选生效。poolFallback/"同池互备"已移除:指定模型绝不漂移到别的模型。
65
+ const canFailover = () => canFallback;
66
+ // 请求级空轮等待累计(跨候选):阶梯是「每发候选」各算的,不设顶就会 N×(2+8+30) 把客户端吊在门外
67
+ let requestWaitedMs = 0;
68
+ candidate: for (let idx = 0; idx < order.length; idx++) {
69
+ const model = order[idx];
70
+ handlerCtx.model = model;
71
+ handlerCtx.orderLen = order.length;
72
+ handlerCtx.idx = idx;
73
+ evt("model-try", { reqId, model, idx, remaining: order.length - idx });
74
+ if (plugins?.length) {
75
+ const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
76
+ for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
77
+ if (bt.value === false || bt.value?.skip === true) { evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model }); continue; }
78
+ }
79
+ let upRes = null;
80
+ let forwarded = { ...injectReasoningContent(model, body), model };
81
+ // 首发输出额度托底(plugin hook 前——plugin 有最终发言权,硬底线只约束客户端直发 body)
82
+ const _floored = withOutputFloor(forwarded);
83
+ if (_floored !== forwarded) {
84
+ const _fk = ["max_tokens", "max_completion_tokens"].find((k) => _floored[k] !== forwarded[k]) || "max_tokens";
85
+ evt("output-floor", { reqId, model, key: _fk, raiseFrom: Number(forwarded[_fk]), raiseTo: Number(_floored[_fk]) });
86
+ forwarded = _floored;
87
+ }
88
+ if (plugins?.length) {
89
+ const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
90
+ for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
91
+ if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
92
+ }
93
+ const chatOpts = {};
94
+ if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
95
+ if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
96
+ if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
97
+ // reqId = 本次客户端请求的身份:供应商据此"同请求粘号"(重试不换号,只有 401/403/429/5xx 冷却才换)
98
+ if (reqId) chatOpts.reqId = reqId;
99
+ const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
100
+ // 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
101
+ // 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
102
+ const emptyCfg = emptyRetryCfg();
103
+ const raiseCap = emptyRaiseCap();
104
+ const nudgeCfg = emptyNudgeCfg();
105
+ let emptyRetried = 0;
106
+ let emptyWaitedMs = 0;
107
+ for (;;) {
108
+ const tUp = performance.now();
109
+ evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
110
+ try {
111
+ upRes = await upstream.chat(forwarded, chatOptsArg);
112
+ evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null, ...upstreamEcho(upRes) });
113
+ } catch (err) {
114
+ if (auto) await auto.recordError(model, { message: errMsg(err) });
115
+ lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
116
+ logError(model, 502, errMsg(err));
117
+ evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
118
+ upRes = null;
119
+ }
120
+ if (plugins?.length) {
121
+ runHook(plugins, "upstream:response", {
122
+ reqId, requested, model,
123
+ status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
124
+ ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
125
+ error: upRes instanceof Error ? errMsg(upRes) : null,
126
+ timing: upRes?._t ?? null,
127
+ }).catch(() => {});
128
+ }
129
+ mark(`up-${model}`);
130
+ if (upRes && upRes.status >= 400) {
131
+ const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
132
+ if (isAllowlistBlock) {
133
+ let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
134
+ let errBody = { error: `model not allowed for provider` };
135
+ try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
136
+ if (useAuto) {
137
+ logError(model, 403, errBody.error || "model not allowed");
138
+ evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
139
+ lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
140
+ if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
141
+ return json(res, 403, errBody);
142
+ }
143
+ logError(model, 403, errBody.error || "model not allowed");
144
+ evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
145
+ return json(res, 403, errBody);
146
+ }
147
+ if (auto) await auto.recordError(model, { status: upRes.status });
148
+ // 读失败响应体(clone 不影响后续 relay 转发原响应;1s 上限防流式错误体拖慢)。
149
+ // 两份文本分工写死:`upRaw` 喂判据(≤4KB,反向护栏不能被 400 字摘要喂不到——多 provider 明细体
150
+ // 如 OpenRouter 的 `metadata.raw[]` 会把 rate limit 排在 model not found 之后),`upBody` 供日志摘要。
151
+ let upRaw = "";
152
+ let upBody = "";
153
+ upRaw = await readVerdictText(upRes);
154
+ upBody = upRaw.replace(/\s+/g, " ").slice(0, 400);
155
+ const upMsg = upBody || `upstream ${upRes.status}`;
156
+ lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
157
+ // 终局判决短路(判据与现网取证见 terminal-verdict.js)
158
+ const verdict = classifyTerminalUpstream(upRes.status, upRaw);
159
+ const terminalOn = verdict.terminal && !terminalShortcircuitOff();
160
+ // 一行话把「判决」与「原文」一起落 errors.log(不分两行:同一发失败在 errors.log 里只该有一条)
161
+ logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}${terminalOn ? ` · 终局判决 ${verdict.kind}(跳过组员接力)` : ""}`);
162
+ evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null, ...upstreamEcho(upRes) });
163
+ if (terminalOn) {
164
+ evt("upstream-terminal", { reqId, model, status: upRes.status, kind: verdict.kind, marker: verdict.marker, peersSkipped: true });
165
+ // ASCII 具名头。**不**复用 x-mslxdff-notice:那头的既定语义是「已被别的模型代答」(与
166
+ // x-mslxdff-fallback 成组,src/routes/fallback.js:27-39),拿它报「没代答、直接拒」会把按头
167
+ // 判是否换模型的下游 UI 读反。头值只填 ASCII 枚举,中文留在 evt/logError/timeline(文件是 UTF-8)。
168
+ try { if (!res.headersSent && typeof res.setHeader === "function") res.setHeader("x-mslxdff-terminal", verdict.kind); } catch {}
169
+ if (canFallback && idx < order.length - 1) {
170
+ // 判决只对**这个模型**成立 → 换下一候选。这里刻意不挂 upstream:这条 Response 的 body 没被
171
+ // 消费,挂在 lastErr 上带到收场会白占连接(收场只 relay 最后一发),故直接 cancel 后只留文本。
172
+ lastErr = { model, upstream: null, status: upRes.status, message: upMsg, terminal: verdict.kind };
173
+ try { const cx = upRes.body?.cancel?.(); if (cx && typeof cx.catch === "function") cx.catch(() => {}); } catch {}
174
+ evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `terminal ${verdict.kind}: ${upMsg.slice(0, 80)}` });
175
+ continue candidate;
176
+ }
177
+ // 收场交回既有出口:exhaustedLocal 见 lastErr.upstream 会 relay **原响应**(原状态码 + 原字节 +
178
+ // 原头),并自带 logCall / result / client-response / talk 语料落盘。自己 json() 反而劣化:
179
+ // 只能给被截到 400 字的二手摘要、JSON.parse 失败时退成字符串 error(SDK 读 err.error.message
180
+ // 拿到 undefined),且 SSE 已 flush 时 helpers 的 in-band 错误帧会把对象写成 [object Object]。
181
+ await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
182
+ return { done: true };
183
+ }
184
+ upRes = null;
185
+ break;
186
+ }
187
+ if (upRes) {
188
+ const isStream = Boolean(body.stream);
189
+ const d = hedgeDelayMs();
190
+ const hasPeers = Boolean(peers) && peers.ordered().length > 0;
191
+ const canUseGroup = shouldUseGroupForModel(model);
192
+ const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
193
+ if (doHedge) {
194
+ const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res, hedgeDelayMs: d });
195
+ if (hr.handled) return { done: true };
196
+ if (hr.lastErr) lastErr = hr.lastErr;
197
+ if (hr.upRes === null) upRes = null;
198
+ else if (hr.upRes) upRes = hr.upRes;
199
+ }
200
+ if (upRes) {
201
+ const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res });
202
+ if (lr.handled) {
203
+ // 空转重试后真拿到输出 = 降级但成功(WARN 语义):必须交代"第几次救回来的、白等了多久",
204
+ // 否则用户只看到"这一发变慢了",无从判断是重试在兜底还是上游真的死了。
205
+ // 「救回」必须有送达证据:handled:true 也可能来自下游已断开/零输出路径(见 ADR-0043),
206
+ // 只认 relay 报上来的 wrotePayload —— 没有正文到下游就不许记成功,否则日志在骗排障的人。
207
+ if (emptyRetried > 0 && lr.wrotePayload === true) {
208
+ evt("empty-turn-recovered", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, ...upstreamEcho(upRes) });
209
+ }
210
+ return { done: true };
211
+ }
212
+ if (lr.lastErr && isEmptyTurnError(lr.lastErr)) {
213
+ // 空转判据原文(finish_reason / 零正文 / 上游错误摘要)随事件落盘,排障不必再翻第二个文件
214
+ const _why = String(lr.lastErr.message || "").replace(/\s+/g, " ").trim().slice(0, 220);
215
+ const budgetMs = emptyTurnBudgetMs();
216
+ const budgetLeftMs = budgetMs - requestWaitedMs;
217
+ if (emptyRetried < emptyCfg.max && budgetLeftMs > 0) {
218
+ // 口径(用户定):正文为空就重试,最多 emptyCfg.max 次——不设"能不能送达"的前提,
219
+ // 大不了两次都空,反正不是无限重试。抬额度只加不减且有顶(默认 16384):
220
+ // 思考刷满 max_tokens 是零正文的主因,同参重拉必然复现,抬一次才算换了打法。
221
+ emptyRetried++;
222
+ const _before = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens) || null;
223
+ // 客户端没设额度也兜底发明一次:现网主流空轮就是「思考吃满 max_tokens、正文为零」,同参重拉必复现
224
+ const _raised = withRaisedMaxTokens(forwarded, raiseCap, emptyTurnMinRaiseTo());
225
+ let _after = null;
226
+ if (_raised !== forwarded) { forwarded = _raised; _after = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens); }
227
+ const _isLast = emptyRetried >= emptyCfg.max;
228
+ let _nudged = false;
229
+ if (nudgeCfg.enabled && _isLast) {
230
+ const _next = withEmptyNudge(forwarded, nudgeCfg.text);
231
+ if (_next !== forwarded) { forwarded = _next; _nudged = true; }
232
+ }
233
+ // 事件在 sleep **之前**发:对着日志能立刻看到"正在暂停 Nms 重拉",而不是等结果
234
+ // 带上"刚空转的是哪个号/哪个站":切号是重试驱动的,日志必须能自证
235
+ const stepIdx = emptyCfg.steps ? (emptyRetried - 1) % emptyCfg.steps.length : 0;
236
+ const rawDelayMs = computeNextDelay(emptyRetried - 1, emptyCfg.steps, lr.lastErr);
237
+ const delayMs = Math.min(rawDelayMs, budgetLeftMs); // 末次等待不越过请求级预算
238
+ emptyWaitedMs += delayMs;
239
+ requestWaitedMs += delayMs;
240
+ evt("empty-turn-retry", { reqId, model, retry: emptyRetried, step: stepIdx, max: emptyCfg.max, delayMs, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs, nudged: _nudged ? 1 : undefined, raiseFrom: _after != null ? _before : undefined, raiseTo: _after ?? undefined, reason: _why, ...upstreamEcho(upRes) });
241
+ await sleep(delayMs);
242
+ continue;
243
+ }
244
+ // 次数用尽(或被 MSLXDFF_EMPTY_TURN_RETRIES=0 关掉)→ 记一行"不再重试"再交回 failover
245
+ const _budgetOut = emptyRetried < emptyCfg.max && emptyTurnBudgetMs() - requestWaitedMs <= 0;
246
+ evt("empty-turn-exhausted", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs: emptyTurnBudgetMs(), budgetOut: _budgetOut ? 1 : undefined, reason: _why, ...upstreamEcho(upRes) });
247
+ try { logError(model, 502, `空转重试 ${emptyRetried}/${emptyCfg.max} 后仍无输出${emptyCfg.max === 0 ? "(MSLXDFF_EMPTY_TURN_RETRIES=0 已关闭重试)" : _budgetOut ? `(请求级等待预算 ${emptyTurnBudgetMs()}ms 用尽)` : ""}:${_why}`); } catch {}
248
+ }
249
+ if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
250
+ return { done: true };
251
+ }
252
+ break;
253
+ }
254
+ break;
255
+ }
256
+ if (canForwardPeers) {
257
+ if (!shouldUseGroupForModel(model)) {
258
+ evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (peer)` });
259
+ } else {
260
+ const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res });
261
+ if (pr.handled) return { done: true };
262
+ if (pr.lastErr) lastErr = pr.lastErr;
263
+ }
264
+ }
265
+ if (groups) {
266
+ if (!shouldUseGroupForModel(model)) {
267
+ evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (broadband)` });
268
+ } else {
269
+ const br = await broadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, attemptStartMs: performance.now(), stages, res, startedAt, plugins });
270
+ if (br.handled) return { done: true };
271
+ }
272
+ }
273
+ if (canFailover()) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` }); continue; }
274
+ await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
275
+ return { done: true };
276
+ }
277
+ await exhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
278
+ return { done: true };
279
+ }