mslxdff 0.1.199 → 0.1.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin/mslxdff.js +1 -1
- package/package.json +38 -38
- package/plugins/README.md +14 -14
- package/plugins/prefer-model.mjs.example +14 -14
- package/src/anthropic/count-tokens.js +61 -61
- package/src/anthropic/request-body.js +126 -126
- package/src/anthropic/shared.js +18 -18
- package/src/anthropic/sse-aggregate.js +56 -56
- package/src/anthropic/translate.js +287 -287
- package/src/anthropic/web-search.js +361 -361
- package/src/api-kind.js +31 -31
- package/src/autostart.js +288 -288
- package/src/bench/probe.js +1 -1
- package/src/bench/runner.js +1 -1
- package/src/bench/via-probe.js +1 -1
- package/src/bench/via.js +144 -144
- package/src/bench/workbuddy-bench.js +1 -1
- package/src/chat/config.js +11 -11
- package/src/chat/cooling.js +81 -81
- package/src/chat/direct.js +58 -58
- package/src/chat/index.js +6 -6
- package/src/chat/spinner.js +38 -38
- package/src/chat/sse.js +69 -69
- package/src/chat/stats.js +247 -247
- package/src/chat/store.js +48 -48
- package/src/chat/tool-handlers.js +67 -67
- package/src/chat/tools.js +1 -1
- package/src/chat/upstream.js +59 -59
- package/src/chat-pipeline/auto-race.js +158 -158
- package/src/chat-pipeline/empty-turn.js +147 -147
- package/src/chat-pipeline/engine.js +17 -17
- package/src/chat-pipeline/serial-trial.js +283 -283
- package/src/chat-pipeline/terminal-verdict.js +147 -147
- package/src/chooser.js +54 -54
- package/src/claude-desktop/apply.js +101 -101
- package/src/claude-desktop/context1m.js +38 -38
- package/src/claude-desktop/doctor.js +146 -146
- package/src/claude-desktop/slots.js +152 -152
- package/src/cli/bootstrap.js +1 -1
- package/src/cli/commands/claude-desktop.js +284 -284
- package/src/cli/commands/daemon.js +231 -231
- package/src/cli/commands/model/list-live.js +22 -22
- package/src/cli/commands/model/list-providers.js +76 -76
- package/src/cli/commands/model/list-render.js +77 -77
- package/src/cli/commands/model/list-sort.js +52 -52
- package/src/cli/commands/model/list-tty.js +76 -76
- package/src/cli/commands/model/list.js +166 -166
- package/src/cli/commands/model/live-models.js +24 -24
- package/src/cli/commands/model/stats.js +43 -43
- package/src/cli/commands/model/status.js +47 -47
- package/src/cli/commands/model.js +51 -51
- package/src/cli/commands/provider/add.js +108 -108
- package/src/cli/commands/provider/allowlist.js +99 -99
- package/src/cli/commands/provider/cline-free.js +99 -99
- package/src/cli/commands/provider/cline-login.js +218 -218
- package/src/cli/commands/provider/cline-quota.js +65 -65
- package/src/cli/commands/provider/codearts-login.js +45 -45
- package/src/cli/commands/provider/config.js +119 -119
- package/src/cli/commands/provider/globalqwenwork-login.js +126 -126
- package/src/cli/commands/provider/keys-guard.js +50 -50
- package/src/cli/commands/provider/keys.js +161 -161
- package/src/cli/commands/provider/qoder-checkin.js +81 -81
- package/src/cli/commands/provider/qoder-login.js +102 -102
- package/src/cli/commands/provider/raccoon-login.js +180 -180
- package/src/cli/commands/provider/raccoon-password.js +162 -162
- package/src/cli/commands/provider/raccoon-quota.js +109 -109
- package/src/cli/commands/provider/traework-checkin.js +144 -144
- package/src/cli/commands/provider/traework-login.js +164 -164
- package/src/cli/commands/provider/workbuddy-login.js +153 -153
- package/src/cli/commands/provider/zcode-login.js +77 -77
- package/src/cli/commands/provider/zcode-quota.js +55 -55
- package/src/cli/commands/provider.js +1 -1
- package/src/cli/commands/stats.js +149 -149
- package/src/cli/commands/system.js +261 -261
- package/src/cli/commands/timezone.js +47 -47
- package/src/cli/commands/use-group.js +60 -60
- package/src/cli/commands/workbuddy.js +189 -189
- package/src/cli/format.js +141 -141
- package/src/cli/group-helpers.js +70 -70
- package/src/cli/help.js +80 -80
- package/src/cli/index.js +70 -70
- package/src/cli/interactive.js +121 -121
- package/src/cli/join-core.js +99 -99
- package/src/cli/join-wizard.js +175 -175
- package/src/cli/policy.js +136 -136
- package/src/cli/provider-row.js +126 -126
- package/src/cli/util.js +24 -24
- package/src/compat.js +53 -53
- package/src/events.js +38 -38
- package/src/free-lane.js +131 -131
- package/src/free-watcher.js +63 -63
- package/src/groups.js +262 -262
- package/src/logs.js +195 -195
- package/src/metrics.js +62 -62
- package/src/model-capabilities/context-lookup.js +65 -65
- package/src/model-capabilities/merge.js +142 -142
- package/src/model-trace.js +187 -187
- package/src/models.js +1 -1
- package/src/opencode-identity.js +72 -72
- package/src/peers.js +254 -251
- package/src/plugins.js +107 -107
- package/src/providers/base.js +164 -164
- package/src/providers/classify.js +22 -22
- package/src/providers/cline/allowlist-sync.js +93 -93
- package/src/providers/cline/free-catalog.js +71 -71
- package/src/providers/cline/headers.js +37 -37
- package/src/providers/cline/index.js +130 -130
- package/src/providers/cline/models.js +150 -150
- package/src/providers/cline/usage.js +274 -274
- package/src/providers/codearts/auth-pool.js +148 -148
- package/src/providers/codearts/concurrency.js +34 -34
- package/src/providers/codearts/const.js +65 -65
- package/src/providers/codearts/dpop.js +95 -95
- package/src/providers/codearts/index.js +175 -175
- package/src/providers/codearts/login.js +214 -214
- package/src/providers/codearts/models.js +182 -182
- package/src/providers/codearts/sign.js +96 -96
- package/src/providers/codearts/sse.js +204 -204
- package/src/providers/codearts/stream.js +209 -209
- package/src/providers/codearts/sts.js +124 -124
- package/src/providers/codearts.js +1 -1
- package/src/providers/dispatcher.js +146 -146
- package/src/providers/generic.js +156 -156
- package/src/providers/globalqwenwork/account-store.js +141 -141
- package/src/providers/globalqwenwork/constants.js +73 -73
- package/src/providers/globalqwenwork/crypto.js +2 -2
- package/src/providers/globalqwenwork/http.js +2 -2
- package/src/providers/globalqwenwork/index.js +331 -331
- package/src/providers/globalqwenwork/payload.js +147 -147
- package/src/providers/globalqwenwork/rsa.js +2 -2
- package/src/providers/globalqwenwork/sse.js +2 -2
- package/src/providers/globalqwenwork/stream.js +2 -2
- package/src/providers/globalqwenwork.js +1 -1
- package/src/providers/globalworkbuddy/account-store.js +198 -198
- package/src/providers/globalworkbuddy/auth.js +97 -97
- package/src/providers/globalworkbuddy/catalog.js +138 -138
- package/src/providers/globalworkbuddy/constants.js +73 -73
- package/src/providers/globalworkbuddy/device-auth.js +120 -120
- package/src/providers/globalworkbuddy/errors.js +98 -98
- package/src/providers/globalworkbuddy/headers.js +89 -89
- package/src/providers/globalworkbuddy/models.js +96 -96
- package/src/providers/globalworkbuddy/payload.js +80 -80
- package/src/providers/index.js +5 -5
- package/src/providers/model-id.js +172 -172
- package/src/providers/opencode.js +19 -19
- package/src/providers/openrouter.js +31 -31
- package/src/providers/qoder/account-store.js +106 -106
- package/src/providers/qoder/aggregate.js +60 -60
- package/src/providers/qoder/baseprompt.json +591 -591
- package/src/providers/qoder/checkin.js +160 -160
- package/src/providers/qoder/constants.js +44 -44
- package/src/providers/qoder/encode.js +41 -41
- package/src/providers/qoder/fingerprint.js +38 -38
- package/src/providers/qoder/models.js +77 -77
- package/src/providers/qoder/oauth.js +84 -84
- package/src/providers/qoder/payload.js +162 -162
- package/src/providers/qoder/request.js +27 -27
- package/src/providers/qoder/session.js +134 -134
- package/src/providers/qoder/sticky.js +38 -38
- package/src/providers/qoder/stream.js +116 -116
- package/src/providers/qoder.js +1 -1
- package/src/providers/raccoon/account-store.js +216 -216
- package/src/providers/raccoon/auth.js +67 -67
- package/src/providers/raccoon/chat.js +228 -228
- package/src/providers/raccoon/const.js +180 -165
- package/src/providers/raccoon/credits.js +185 -175
- package/src/providers/raccoon/crypto.js +43 -0
- package/src/providers/raccoon/envelope.js +31 -31
- package/src/providers/raccoon/headers.js +45 -45
- package/src/providers/raccoon/index.js +216 -216
- package/src/providers/raccoon/login.js +279 -279
- package/src/providers/raccoon/models.js +160 -160
- package/src/providers/raccoon/password-login.js +102 -87
- package/src/providers/raccoon/qr-codewords.js +115 -115
- package/src/providers/raccoon/qr.js +236 -236
- package/src/providers/raccoon/sse.js +135 -135
- package/src/providers/responses-channel.js +120 -120
- package/src/providers/share-keys.js +53 -53
- package/src/providers/traework/account-store.js +234 -234
- package/src/providers/traework/chat.js +217 -217
- package/src/providers/traework/checkin.js +77 -77
- package/src/providers/traework/constants.js +41 -41
- package/src/providers/traework/errors.js +52 -52
- package/src/providers/traework/headers.js +79 -79
- package/src/providers/traework/index.js +132 -132
- package/src/providers/traework/market-identity.js +116 -116
- package/src/providers/traework/models.js +98 -98
- package/src/providers/traework/payload.js +86 -86
- package/src/providers/traework/sse.js +184 -184
- package/src/providers/traework/token.js +52 -52
- package/src/providers/traework.js +2 -2
- package/src/providers/workbuddy/account-store.js +186 -186
- package/src/providers/workbuddy/auth.js +158 -158
- package/src/providers/workbuddy/balance.js +86 -86
- package/src/providers/workbuddy/cat-travel.js +125 -125
- package/src/providers/workbuddy/checkin.js +75 -75
- package/src/providers/workbuddy/desktop-info.js +78 -78
- package/src/providers/workbuddy/growth-api.js +67 -67
- package/src/providers/workbuddy/growth-plans.js +63 -63
- package/src/providers/workbuddy/growth.js +181 -181
- package/src/providers/workbuddy/index.js +217 -217
- package/src/providers/workbuddy/models.js +126 -126
- package/src/providers/workbuddy/payload.js +141 -141
- package/src/providers/workbuddy/rotation-log.js +83 -83
- package/src/providers/workbuddy/sanitize-tools.js +61 -61
- package/src/providers/workbuddy/sdk-chat.js +26 -26
- package/src/providers/workbuddy-balance.js +78 -78
- package/src/providers/zcode/account-store.js +129 -129
- package/src/providers/zcode/auth.js +28 -28
- package/src/providers/zcode/chat.js +179 -179
- package/src/providers/zcode/const.js +54 -54
- package/src/providers/zcode/context-shape.js +162 -162
- package/src/providers/zcode/headers.js +93 -93
- package/src/providers/zcode/index.js +124 -124
- package/src/providers/zcode/models.js +65 -65
- package/src/providers/zcode/oauth.js +120 -120
- package/src/providers/zcode/quota.js +176 -176
- package/src/providers/zcode/sse.js +196 -196
- package/src/readline-compat.js +25 -25
- package/src/responses/translate.js +311 -311
- package/src/routes/chat/broadband-handler.js +72 -72
- package/src/routes/chat/empty-turn-report.js +31 -31
- package/src/routes/chat/exhausted-handler.js +87 -87
- package/src/routes/chat/hedge-handler.js +130 -130
- package/src/routes/chat/index.js +1 -1
- package/src/routes/chat/local-handler.js +63 -63
- package/src/routes/chat/peer-handler.js +81 -81
- package/src/routes/chat.js +3 -3
- package/src/routes/fallback.js +88 -88
- package/src/routes/groups-relay.js +214 -214
- package/src/routes/groups.js +122 -122
- package/src/routes/helpers.js +83 -83
- package/src/routes/index.js +147 -147
- package/src/routes/messages-route.js +365 -365
- package/src/routes/peers.js +23 -1
- package/src/routes/relay-queue.js +181 -181
- package/src/routes/relay.js +60 -60
- package/src/routes/responses-route.js +213 -213
- package/src/routes/stream-body.js +84 -84
- package/src/routes/stream-hold.js +337 -337
- package/src/routes/stream-scan.js +366 -366
- package/src/routes.js +3 -3
- package/src/runtime/auto-update.js +113 -113
- package/src/runtime/bootstrap.js +8 -0
- package/src/runtime/broadband-stream.js +79 -79
- package/src/runtime/broadband.js +103 -103
- package/src/runtime/group-sync.js +81 -81
- package/src/runtime/keep-warm.js +120 -0
- package/src/runtime/lifecycle-forensics.js +116 -116
- package/src/runtime/lifecycle-log.js +87 -87
- package/src/runtime/log-stamp.js +73 -73
- package/src/runtime/proc-info.js +137 -137
- package/src/runtime/qoder-checkin.js +93 -93
- package/src/runtime/raccoon-checkin.js +126 -126
- package/src/runtime/server-lifecycle.js +189 -189
- package/src/runtime/traework-checkin.js +118 -118
- package/src/runtime/workbuddy-checkin.js +120 -120
- package/src/runtime/workbuddy-growth.js +133 -133
- package/src/server.js +7 -0
- package/src/state/facade.js +58 -58
- package/src/state/memory.js +162 -162
- package/src/state/merge.js +25 -25
- package/src/state/migrations/cline-unify.js +94 -94
- package/src/state/migrations.js +33 -33
- package/src/state/persist.js +42 -42
- package/src/state/provider-config.js +142 -142
- package/src/state/schemas/group.js +31 -31
- package/src/state/schemas/model.js +94 -94
- package/src/state/schemas/peer.js +31 -31
- package/src/state/schemas/port.js +10 -10
- package/src/state/schemas/provider.js +202 -202
- package/src/state/schemas/timezone.js +70 -70
- package/src/state/schemas/token.js +48 -48
- package/src/state/schemas/use-group.js +102 -102
- package/src/state/store.js +176 -176
- package/src/state.js +1 -1
- package/src/sync-claude-desktop.js +234 -234
- package/src/sync-claude.js +152 -152
- package/src/sync-codex.js +116 -116
- package/src/sync-workbuddy.js +158 -158
- package/src/talk-full-redact.js +129 -129
- package/src/talk-full.js +131 -131
- package/src/talk-log.js +257 -257
- package/src/text-tokens.js +52 -52
- package/src/thinking-effort.js +67 -67
- package/src/time.js +106 -106
- package/src/timeline.js +26 -26
- package/src/transport/index.js +1 -1
- package/src/upstream-engine/index.js +74 -72
- package/src/upstream-engine/mode.js +15 -15
- package/src/upstream-engine/sdk/attempt.js +221 -221
- package/src/upstream-engine/sdk/chat.js +28 -28
- package/src/upstream-engine/sdk/convert.js +192 -192
- package/src/upstream-engine/sdk/diagnose.js +62 -62
- package/src/upstream-engine/sdk/dispatch.js +55 -55
- package/src/upstream-engine/sdk/responses.js +176 -176
- package/src/upstream-engine/sdk/sse.js +166 -166
- package/src/upstream-responses.js +305 -305
- package/src/upstream.js +53 -1
- package/src/usage/record.js +106 -106
- package/src/usage/report.js +195 -195
- package/src/v6pool/claim.js +157 -157
- package/src/v6pool/config.js +65 -65
- package/src/v6pool/detect.js +130 -130
- package/src/v6pool/index.js +152 -152
- package/src/v6pool/prefix.js +140 -140
- package/src/v6pool/probe.js +40 -40
- package/src/v6pool/report.js +104 -104
- package/src/v6pool/rotate.js +263 -239
- package/src/warp-pool/config.js +74 -74
- package/src/warp-pool/health.js +106 -106
- package/src/warp-pool/identity.js +141 -141
- package/src/warp-pool/index.js +358 -358
- package/src/warp-pool/keys.js +29 -29
- package/src/warp-pool/nodes.js +95 -95
- package/src/warp-pool/ports.js +160 -160
- package/src/warp-pool/register.js +113 -113
- package/src/warp-pool/report.js +232 -232
- package/src/warp-pool/revive.js +168 -168
- package/src/warp-pool/rotate.js +214 -214
- package/src/warp-pool/strategy.js +43 -43
- package/src/warp-pool/verdict.js +69 -69
- package/src/warp-pool/wireproxy.js +162 -162
- package/docs/cli_help_mini.md +0 -146
- package/src/providers/AGENTS.md +0 -49
- package/src/routes/AGENTS.md +0 -37
|
@@ -1,283 +1,283 @@
|
|
|
1
|
-
import { performance } from "node:perf_hooks";
|
|
2
|
-
import { injectReasoningContent } from "../reasoning.js";
|
|
3
|
-
import { runHook } from "../plugins.js";
|
|
4
|
-
import { errMsg, json } from "../routes/helpers.js";
|
|
5
|
-
import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
|
|
6
|
-
import { handleHedge } from "../routes/chat/hedge-handler.js";
|
|
7
|
-
import { handleLocalRelay } from "../routes/chat/local-handler.js";
|
|
8
|
-
import { handlePeerRelay } from "../routes/chat/peer-handler.js";
|
|
9
|
-
import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
|
|
10
|
-
import { handleViaRoute } from "../routes/chat/via-route-handler.js";
|
|
11
|
-
import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
|
|
12
|
-
import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
|
|
13
|
-
import { summarizeRequest, upstreamEcho } from "../model-trace.js";
|
|
14
|
-
import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
|
|
15
|
-
import { emptyRetryCfg, emptyRaiseCap, withRaisedMaxTokens, withOutputFloor, emptyNudgeCfg, withEmptyNudge, computeNextDelay, emptyTurnBudgetMs, emptyTurnMinRaiseTo } from "./empty-turn.js";
|
|
16
|
-
import { classifyTerminalUpstream, readVerdictText, terminalShortcircuitOff, isPoolSwitchWorthy } from "./terminal-verdict.js";
|
|
17
|
-
|
|
18
|
-
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
19
|
-
|
|
20
|
-
function groupSkipReason(model) {
|
|
21
|
-
if (isHardLocalOnly(model)) return "provider local-only(禁组员,仅本机直连)";
|
|
22
|
-
if (isKeyProviderDirectOnly(model)) return "key provider default direct(仅本机直连,MSLXDFF_USE_GROUP_KEYS=1 可开组员)";
|
|
23
|
-
return "useGroup=off";
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* 串行 trial — 从 engine.js 抽出的第二段:via-route 单路径 → 串行 trial →
|
|
28
|
-
* hedge/local/peer/broadband/exhausted。恒终结,返回 { done:true }。
|
|
29
|
-
*/
|
|
30
|
-
export async function runSerialTrial(ctx, deps = {}) {
|
|
31
|
-
const {
|
|
32
|
-
viaRoute = handleViaRoute,
|
|
33
|
-
hedge = handleHedge,
|
|
34
|
-
localRelay = handleLocalRelay,
|
|
35
|
-
peerRelay = handlePeerRelay,
|
|
36
|
-
broadbandRelay = handleBroadbandRelay,
|
|
37
|
-
exhaustedLocal = handleExhaustedLocal,
|
|
38
|
-
exhaustedAll = handleExhaustedAll,
|
|
39
|
-
} = deps;
|
|
40
|
-
const {
|
|
41
|
-
order, reqId, requested, body, hops, useAuto, lockModel, plugins,
|
|
42
|
-
auto, upstream, peers, groups, bus, token, canFallback, canForwardPeers, poolFallback,
|
|
43
|
-
perf0, stages, mark, evt, logCall, logError, done, handlerCtx,
|
|
44
|
-
res, startedAt, logs,
|
|
45
|
-
} = ctx;
|
|
46
|
-
const shareKeys = ctx.shareKeys ?? ctx.policy?.shareKeys ?? {};
|
|
47
|
-
const workbuddyUid = ctx.workbuddyUid ?? ctx.policy?.workbuddyUid ?? null;
|
|
48
|
-
|
|
49
|
-
let viaRouteLastErr = null;
|
|
50
|
-
if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers && shouldUseGroupForModel(requested)) {
|
|
51
|
-
try {
|
|
52
|
-
const vr = await viaRoute({ model: requested, body, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res, requested, useAuto, lockModel, auto });
|
|
53
|
-
if (vr.handled) return { done: true };
|
|
54
|
-
if (vr.lastErr) viaRouteLastErr = vr.lastErr;
|
|
55
|
-
} catch (e) {
|
|
56
|
-
evt("via-route-exception", { reqId, model: requested, error: errMsg(e) });
|
|
57
|
-
}
|
|
58
|
-
} else if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers) {
|
|
59
|
-
evt("group-skip", { reqId, model: requested, reason: `${groupSkipReason(requested)} (via-route)` });
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
let lastErr = viaRouteLastErr;
|
|
63
|
-
// BUG-42 同池互备判据:poolWorthy 记「本模型直连时被上游判过额度/终局」,接力揉的 502 冲不掉它(详见 ADR-0061)。
|
|
64
|
-
let poolWorthy = false;
|
|
65
|
-
const canFailover = (err) =>
|
|
66
|
-
canFallback && (!poolFallback || isPoolSwitchWorthy(err ?? lastErr) || poolWorthy);
|
|
67
|
-
// 请求级空轮等待累计(跨候选):阶梯是「每发候选」各算的,不设顶就会 N×(2+8+30) 把客户端吊在门外
|
|
68
|
-
let requestWaitedMs = 0;
|
|
69
|
-
candidate: for (let idx = 0; idx < order.length; idx++) {
|
|
70
|
-
const model = order[idx];
|
|
71
|
-
poolWorthy = false; // 每候选独立记账:上一发 429 不代表这一发也 429
|
|
72
|
-
handlerCtx.model = model;
|
|
73
|
-
handlerCtx.orderLen = order.length;
|
|
74
|
-
handlerCtx.idx = idx;
|
|
75
|
-
evt("model-try", { reqId, model, idx, remaining: order.length - idx });
|
|
76
|
-
if (plugins?.length) {
|
|
77
|
-
const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
|
|
78
|
-
for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
|
|
79
|
-
if (bt.value === false || bt.value?.skip === true) { evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model }); continue; }
|
|
80
|
-
}
|
|
81
|
-
let upRes = null;
|
|
82
|
-
let forwarded = { ...injectReasoningContent(model, body), model };
|
|
83
|
-
// 首发输出额度托底(plugin hook 前——plugin 有最终发言权,硬底线只约束客户端直发 body)
|
|
84
|
-
const _floored = withOutputFloor(forwarded);
|
|
85
|
-
if (_floored !== forwarded) {
|
|
86
|
-
const _fk = ["max_tokens", "max_completion_tokens"].find((k) => _floored[k] !== forwarded[k]) || "max_tokens";
|
|
87
|
-
evt("output-floor", { reqId, model, key: _fk, raiseFrom: Number(forwarded[_fk]), raiseTo: Number(_floored[_fk]) });
|
|
88
|
-
forwarded = _floored;
|
|
89
|
-
}
|
|
90
|
-
if (plugins?.length) {
|
|
91
|
-
const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
|
|
92
|
-
for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
|
|
93
|
-
if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
|
|
94
|
-
}
|
|
95
|
-
const chatOpts = {};
|
|
96
|
-
if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
|
|
97
|
-
if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
|
|
98
|
-
if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
|
|
99
|
-
// reqId = 本次客户端请求的身份:供应商据此"同请求粘号"(重试不换号,只有 401/403/429/5xx 冷却才换)
|
|
100
|
-
if (reqId) chatOpts.reqId = reqId;
|
|
101
|
-
const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
|
|
102
|
-
// 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
|
|
103
|
-
// 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
|
|
104
|
-
const emptyCfg = emptyRetryCfg();
|
|
105
|
-
const raiseCap = emptyRaiseCap();
|
|
106
|
-
const nudgeCfg = emptyNudgeCfg();
|
|
107
|
-
let emptyRetried = 0;
|
|
108
|
-
let emptyWaitedMs = 0;
|
|
109
|
-
for (;;) {
|
|
110
|
-
const tUp = performance.now();
|
|
111
|
-
evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
|
|
112
|
-
try {
|
|
113
|
-
upRes = await upstream.chat(forwarded, chatOptsArg);
|
|
114
|
-
evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null, ...upstreamEcho(upRes) });
|
|
115
|
-
} catch (err) {
|
|
116
|
-
if (auto) await auto.recordError(model, { message: errMsg(err) });
|
|
117
|
-
lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
|
|
118
|
-
logError(model, 502, errMsg(err));
|
|
119
|
-
evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
|
|
120
|
-
upRes = null;
|
|
121
|
-
}
|
|
122
|
-
if (plugins?.length) {
|
|
123
|
-
runHook(plugins, "upstream:response", {
|
|
124
|
-
reqId, requested, model,
|
|
125
|
-
status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
|
|
126
|
-
ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
|
|
127
|
-
error: upRes instanceof Error ? errMsg(upRes) : null,
|
|
128
|
-
timing: upRes?._t ?? null,
|
|
129
|
-
}).catch(() => {});
|
|
130
|
-
}
|
|
131
|
-
mark(`up-${model}`);
|
|
132
|
-
if (upRes && upRes.status >= 400) {
|
|
133
|
-
const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
|
|
134
|
-
if (isAllowlistBlock) {
|
|
135
|
-
let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
|
|
136
|
-
let errBody = { error: `model not allowed for provider` };
|
|
137
|
-
try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
|
|
138
|
-
if (useAuto) {
|
|
139
|
-
logError(model, 403, errBody.error || "model not allowed");
|
|
140
|
-
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
|
|
141
|
-
lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
|
|
142
|
-
if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
|
|
143
|
-
return json(res, 403, errBody);
|
|
144
|
-
}
|
|
145
|
-
logError(model, 403, errBody.error || "model not allowed");
|
|
146
|
-
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
|
|
147
|
-
return json(res, 403, errBody);
|
|
148
|
-
}
|
|
149
|
-
if (auto) await auto.recordError(model, { status: upRes.status });
|
|
150
|
-
// 读失败响应体(clone 不影响后续 relay 转发原响应;1s 上限防流式错误体拖慢)。
|
|
151
|
-
// 两份文本分工写死:`upRaw` 喂判据(≤4KB,反向护栏不能被 400 字摘要喂不到——多 provider 明细体
|
|
152
|
-
// 如 OpenRouter 的 `metadata.raw[]` 会把 rate limit 排在 model not found 之后),`upBody` 供日志摘要。
|
|
153
|
-
let upRaw = "";
|
|
154
|
-
let upBody = "";
|
|
155
|
-
upRaw = await readVerdictText(upRes);
|
|
156
|
-
upBody = upRaw.replace(/\s+/g, " ").slice(0, 400);
|
|
157
|
-
const upMsg = upBody || `upstream ${upRes.status}`;
|
|
158
|
-
lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
|
|
159
|
-
// 终局判决短路(判据与现网取证见 terminal-verdict.js)
|
|
160
|
-
const verdict = classifyTerminalUpstream(upRes.status, upRaw);
|
|
161
|
-
const terminalOn = verdict.terminal && !terminalShortcircuitOff();
|
|
162
|
-
// BUG-42 记账点:本机直连时上游判过额度/终局 ⇒ 换兄弟有账可依(接力冲不掉,详见 ADR-0061)
|
|
163
|
-
if (isPoolSwitchWorthy({ status: upRes.status, terminal: verdict.terminal })) poolWorthy = true;
|
|
164
|
-
// 一行话把「判决」与「原文」一起落 errors.log(不分两行:同一发失败在 errors.log 里只该有一条)
|
|
165
|
-
logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}${terminalOn ? ` · 终局判决 ${verdict.kind}(跳过组员接力)` : ""}`);
|
|
166
|
-
evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null, ...upstreamEcho(upRes) });
|
|
167
|
-
if (terminalOn) {
|
|
168
|
-
evt("upstream-terminal", { reqId, model, status: upRes.status, kind: verdict.kind, marker: verdict.marker, peersSkipped: true });
|
|
169
|
-
// ASCII 具名头。**不**复用 x-mslxdff-notice:那头的既定语义是「已被别的模型代答」(与
|
|
170
|
-
// x-mslxdff-fallback 成组,src/routes/fallback.js:27-39),拿它报「没代答、直接拒」会把按头
|
|
171
|
-
// 判是否换模型的下游 UI 读反。头值只填 ASCII 枚举,中文留在 evt/logError/timeline(文件是 UTF-8)。
|
|
172
|
-
try { if (!res.headersSent && typeof res.setHeader === "function") res.setHeader("x-mslxdff-terminal", verdict.kind); } catch {}
|
|
173
|
-
if (canFallback && idx < order.length - 1) {
|
|
174
|
-
// 判决只对**这个模型**成立 → 换下一候选。这里刻意不挂 upstream:这条 Response 的 body 没被
|
|
175
|
-
// 消费,挂在 lastErr 上带到收场会白占连接(收场只 relay 最后一发),故直接 cancel 后只留文本。
|
|
176
|
-
lastErr = { model, upstream: null, status: upRes.status, message: upMsg, terminal: verdict.kind };
|
|
177
|
-
try { const cx = upRes.body?.cancel?.(); if (cx && typeof cx.catch === "function") cx.catch(() => {}); } catch {}
|
|
178
|
-
evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `terminal ${verdict.kind}: ${upMsg.slice(0, 80)}` });
|
|
179
|
-
continue candidate;
|
|
180
|
-
}
|
|
181
|
-
// 收场交回既有出口:exhaustedLocal 见 lastErr.upstream 会 relay **原响应**(原状态码 + 原字节 +
|
|
182
|
-
// 原头),并自带 logCall / result / client-response / talk 语料落盘。自己 json() 反而劣化:
|
|
183
|
-
// 只能给被截到 400 字的二手摘要、JSON.parse 失败时退成字符串 error(SDK 读 err.error.message
|
|
184
|
-
// 拿到 undefined),且 SSE 已 flush 时 helpers 的 in-band 错误帧会把对象写成 [object Object]。
|
|
185
|
-
await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
|
|
186
|
-
return { done: true };
|
|
187
|
-
}
|
|
188
|
-
upRes = null;
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
|
-
if (upRes) {
|
|
192
|
-
const isStream = Boolean(body.stream);
|
|
193
|
-
const d = hedgeDelayMs();
|
|
194
|
-
const hasPeers = Boolean(peers) && peers.ordered().length > 0;
|
|
195
|
-
const canUseGroup = shouldUseGroupForModel(model);
|
|
196
|
-
const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
|
|
197
|
-
if (doHedge) {
|
|
198
|
-
const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res, hedgeDelayMs: d });
|
|
199
|
-
if (hr.handled) return { done: true };
|
|
200
|
-
if (hr.lastErr) lastErr = hr.lastErr;
|
|
201
|
-
if (hr.upRes === null) upRes = null;
|
|
202
|
-
else if (hr.upRes) upRes = hr.upRes;
|
|
203
|
-
}
|
|
204
|
-
if (upRes) {
|
|
205
|
-
const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res });
|
|
206
|
-
if (lr.handled) {
|
|
207
|
-
// 空转重试后真拿到输出 = 降级但成功(WARN 语义):必须交代"第几次救回来的、白等了多久",
|
|
208
|
-
// 否则用户只看到"这一发变慢了",无从判断是重试在兜底还是上游真的死了。
|
|
209
|
-
// 「救回」必须有送达证据:handled:true 也可能来自下游已断开/零输出路径(见 ADR-0043),
|
|
210
|
-
// 只认 relay 报上来的 wrotePayload —— 没有正文到下游就不许记成功,否则日志在骗排障的人。
|
|
211
|
-
if (emptyRetried > 0 && lr.wrotePayload === true) {
|
|
212
|
-
evt("empty-turn-recovered", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, ...upstreamEcho(upRes) });
|
|
213
|
-
}
|
|
214
|
-
return { done: true };
|
|
215
|
-
}
|
|
216
|
-
if (lr.lastErr && isEmptyTurnError(lr.lastErr)) {
|
|
217
|
-
// 空转判据原文(finish_reason / 零正文 / 上游错误摘要)随事件落盘,排障不必再翻第二个文件
|
|
218
|
-
const _why = String(lr.lastErr.message || "").replace(/\s+/g, " ").trim().slice(0, 220);
|
|
219
|
-
const budgetMs = emptyTurnBudgetMs();
|
|
220
|
-
const budgetLeftMs = budgetMs - requestWaitedMs;
|
|
221
|
-
if (emptyRetried < emptyCfg.max && budgetLeftMs > 0) {
|
|
222
|
-
// 口径(用户定):正文为空就重试,最多 emptyCfg.max 次——不设"能不能送达"的前提,
|
|
223
|
-
// 大不了两次都空,反正不是无限重试。抬额度只加不减且有顶(默认 16384):
|
|
224
|
-
// 思考刷满 max_tokens 是零正文的主因,同参重拉必然复现,抬一次才算换了打法。
|
|
225
|
-
emptyRetried++;
|
|
226
|
-
const _before = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens) || null;
|
|
227
|
-
// 客户端没设额度也兜底发明一次:现网主流空轮就是「思考吃满 max_tokens、正文为零」,同参重拉必复现
|
|
228
|
-
const _raised = withRaisedMaxTokens(forwarded, raiseCap, emptyTurnMinRaiseTo());
|
|
229
|
-
let _after = null;
|
|
230
|
-
if (_raised !== forwarded) { forwarded = _raised; _after = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens); }
|
|
231
|
-
const _isLast = emptyRetried >= emptyCfg.max;
|
|
232
|
-
let _nudged = false;
|
|
233
|
-
if (nudgeCfg.enabled && _isLast) {
|
|
234
|
-
const _next = withEmptyNudge(forwarded, nudgeCfg.text);
|
|
235
|
-
if (_next !== forwarded) { forwarded = _next; _nudged = true; }
|
|
236
|
-
}
|
|
237
|
-
// 事件在 sleep **之前**发:对着日志能立刻看到"正在暂停 Nms 重拉",而不是等结果
|
|
238
|
-
// 带上"刚空转的是哪个号/哪个站":切号是重试驱动的,日志必须能自证
|
|
239
|
-
const stepIdx = emptyCfg.steps ? (emptyRetried - 1) % emptyCfg.steps.length : 0;
|
|
240
|
-
const rawDelayMs = computeNextDelay(emptyRetried - 1, emptyCfg.steps, lr.lastErr);
|
|
241
|
-
const delayMs = Math.min(rawDelayMs, budgetLeftMs); // 末次等待不越过请求级预算
|
|
242
|
-
emptyWaitedMs += delayMs;
|
|
243
|
-
requestWaitedMs += delayMs;
|
|
244
|
-
evt("empty-turn-retry", { reqId, model, retry: emptyRetried, step: stepIdx, max: emptyCfg.max, delayMs, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs, nudged: _nudged ? 1 : undefined, raiseFrom: _after != null ? _before : undefined, raiseTo: _after ?? undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
245
|
-
await sleep(delayMs);
|
|
246
|
-
continue;
|
|
247
|
-
}
|
|
248
|
-
// 次数用尽(或被 MSLXDFF_EMPTY_TURN_RETRIES=0 关掉)→ 记一行"不再重试"再交回 failover
|
|
249
|
-
const _budgetOut = emptyRetried < emptyCfg.max && emptyTurnBudgetMs() - requestWaitedMs <= 0;
|
|
250
|
-
evt("empty-turn-exhausted", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs: emptyTurnBudgetMs(), budgetOut: _budgetOut ? 1 : undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
251
|
-
try { logError(model, 502, `空转重试 ${emptyRetried}/${emptyCfg.max} 后仍无输出${emptyCfg.max === 0 ? "(MSLXDFF_EMPTY_TURN_RETRIES=0 已关闭重试)" : _budgetOut ? `(请求级等待预算 ${emptyTurnBudgetMs()}ms 用尽)` : ""}:${_why}`); } catch {}
|
|
252
|
-
}
|
|
253
|
-
if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
|
|
254
|
-
return { done: true };
|
|
255
|
-
}
|
|
256
|
-
break;
|
|
257
|
-
}
|
|
258
|
-
break;
|
|
259
|
-
}
|
|
260
|
-
if (canForwardPeers) {
|
|
261
|
-
if (!shouldUseGroupForModel(model)) {
|
|
262
|
-
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (peer)` });
|
|
263
|
-
} else {
|
|
264
|
-
const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res });
|
|
265
|
-
if (pr.handled) return { done: true };
|
|
266
|
-
if (pr.lastErr) lastErr = pr.lastErr;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
if (groups) {
|
|
270
|
-
if (!shouldUseGroupForModel(model)) {
|
|
271
|
-
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (broadband)` });
|
|
272
|
-
} else {
|
|
273
|
-
const br = await broadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, attemptStartMs: performance.now(), stages, res, startedAt, plugins });
|
|
274
|
-
if (br.handled) return { done: true };
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
if (canFailover(lastErr)) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, pool: poolFallback || undefined, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` }); continue; }
|
|
278
|
-
await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
|
|
279
|
-
return { done: true };
|
|
280
|
-
}
|
|
281
|
-
await exhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
|
|
282
|
-
return { done: true };
|
|
283
|
-
}
|
|
1
|
+
import { performance } from "node:perf_hooks";
|
|
2
|
+
import { injectReasoningContent } from "../reasoning.js";
|
|
3
|
+
import { runHook } from "../plugins.js";
|
|
4
|
+
import { errMsg, json } from "../routes/helpers.js";
|
|
5
|
+
import { hedgeDelayMs, shouldHedge } from "../routes/hedge.js";
|
|
6
|
+
import { handleHedge } from "../routes/chat/hedge-handler.js";
|
|
7
|
+
import { handleLocalRelay } from "../routes/chat/local-handler.js";
|
|
8
|
+
import { handlePeerRelay } from "../routes/chat/peer-handler.js";
|
|
9
|
+
import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
|
|
10
|
+
import { handleViaRoute } from "../routes/chat/via-route-handler.js";
|
|
11
|
+
import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
|
|
12
|
+
import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
|
|
13
|
+
import { summarizeRequest, upstreamEcho } from "../model-trace.js";
|
|
14
|
+
import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
|
|
15
|
+
import { emptyRetryCfg, emptyRaiseCap, withRaisedMaxTokens, withOutputFloor, emptyNudgeCfg, withEmptyNudge, computeNextDelay, emptyTurnBudgetMs, emptyTurnMinRaiseTo } from "./empty-turn.js";
|
|
16
|
+
import { classifyTerminalUpstream, readVerdictText, terminalShortcircuitOff, isPoolSwitchWorthy } from "./terminal-verdict.js";
|
|
17
|
+
|
|
18
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
19
|
+
|
|
20
|
+
function groupSkipReason(model) {
|
|
21
|
+
if (isHardLocalOnly(model)) return "provider local-only(禁组员,仅本机直连)";
|
|
22
|
+
if (isKeyProviderDirectOnly(model)) return "key provider default direct(仅本机直连,MSLXDFF_USE_GROUP_KEYS=1 可开组员)";
|
|
23
|
+
return "useGroup=off";
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 串行 trial — 从 engine.js 抽出的第二段:via-route 单路径 → 串行 trial →
|
|
28
|
+
* hedge/local/peer/broadband/exhausted。恒终结,返回 { done:true }。
|
|
29
|
+
*/
|
|
30
|
+
export async function runSerialTrial(ctx, deps = {}) {
|
|
31
|
+
const {
|
|
32
|
+
viaRoute = handleViaRoute,
|
|
33
|
+
hedge = handleHedge,
|
|
34
|
+
localRelay = handleLocalRelay,
|
|
35
|
+
peerRelay = handlePeerRelay,
|
|
36
|
+
broadbandRelay = handleBroadbandRelay,
|
|
37
|
+
exhaustedLocal = handleExhaustedLocal,
|
|
38
|
+
exhaustedAll = handleExhaustedAll,
|
|
39
|
+
} = deps;
|
|
40
|
+
const {
|
|
41
|
+
order, reqId, requested, body, hops, useAuto, lockModel, plugins,
|
|
42
|
+
auto, upstream, peers, groups, bus, token, canFallback, canForwardPeers, poolFallback,
|
|
43
|
+
perf0, stages, mark, evt, logCall, logError, done, handlerCtx,
|
|
44
|
+
res, startedAt, logs,
|
|
45
|
+
} = ctx;
|
|
46
|
+
const shareKeys = ctx.shareKeys ?? ctx.policy?.shareKeys ?? {};
|
|
47
|
+
const workbuddyUid = ctx.workbuddyUid ?? ctx.policy?.workbuddyUid ?? null;
|
|
48
|
+
|
|
49
|
+
let viaRouteLastErr = null;
|
|
50
|
+
if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers && shouldUseGroupForModel(requested)) {
|
|
51
|
+
try {
|
|
52
|
+
const vr = await viaRoute({ model: requested, body, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res, requested, useAuto, lockModel, auto });
|
|
53
|
+
if (vr.handled) return { done: true };
|
|
54
|
+
if (vr.lastErr) viaRouteLastErr = vr.lastErr;
|
|
55
|
+
} catch (e) {
|
|
56
|
+
evt("via-route-exception", { reqId, model: requested, error: errMsg(e) });
|
|
57
|
+
}
|
|
58
|
+
} else if (!useAuto && requested && requested.includes("/") && canForwardPeers && !lockModel && peers) {
|
|
59
|
+
evt("group-skip", { reqId, model: requested, reason: `${groupSkipReason(requested)} (via-route)` });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
let lastErr = viaRouteLastErr;
|
|
63
|
+
// BUG-42 同池互备判据:poolWorthy 记「本模型直连时被上游判过额度/终局」,接力揉的 502 冲不掉它(详见 ADR-0061)。
|
|
64
|
+
let poolWorthy = false;
|
|
65
|
+
const canFailover = (err) =>
|
|
66
|
+
canFallback && (!poolFallback || isPoolSwitchWorthy(err ?? lastErr) || poolWorthy);
|
|
67
|
+
// 请求级空轮等待累计(跨候选):阶梯是「每发候选」各算的,不设顶就会 N×(2+8+30) 把客户端吊在门外
|
|
68
|
+
let requestWaitedMs = 0;
|
|
69
|
+
candidate: for (let idx = 0; idx < order.length; idx++) {
|
|
70
|
+
const model = order[idx];
|
|
71
|
+
poolWorthy = false; // 每候选独立记账:上一发 429 不代表这一发也 429
|
|
72
|
+
handlerCtx.model = model;
|
|
73
|
+
handlerCtx.orderLen = order.length;
|
|
74
|
+
handlerCtx.idx = idx;
|
|
75
|
+
evt("model-try", { reqId, model, idx, remaining: order.length - idx });
|
|
76
|
+
if (plugins?.length) {
|
|
77
|
+
const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
|
|
78
|
+
for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
|
|
79
|
+
if (bt.value === false || bt.value?.skip === true) { evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model }); continue; }
|
|
80
|
+
}
|
|
81
|
+
let upRes = null;
|
|
82
|
+
let forwarded = { ...injectReasoningContent(model, body), model };
|
|
83
|
+
// 首发输出额度托底(plugin hook 前——plugin 有最终发言权,硬底线只约束客户端直发 body)
|
|
84
|
+
const _floored = withOutputFloor(forwarded);
|
|
85
|
+
if (_floored !== forwarded) {
|
|
86
|
+
const _fk = ["max_tokens", "max_completion_tokens"].find((k) => _floored[k] !== forwarded[k]) || "max_tokens";
|
|
87
|
+
evt("output-floor", { reqId, model, key: _fk, raiseFrom: Number(forwarded[_fk]), raiseTo: Number(_floored[_fk]) });
|
|
88
|
+
forwarded = _floored;
|
|
89
|
+
}
|
|
90
|
+
if (plugins?.length) {
|
|
91
|
+
const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
|
|
92
|
+
for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
|
|
93
|
+
if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
|
|
94
|
+
}
|
|
95
|
+
const chatOpts = {};
|
|
96
|
+
if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
|
|
97
|
+
if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
|
|
98
|
+
if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
|
|
99
|
+
// reqId = 本次客户端请求的身份:供应商据此"同请求粘号"(重试不换号,只有 401/403/429/5xx 冷却才换)
|
|
100
|
+
if (reqId) chatOpts.reqId = reqId;
|
|
101
|
+
const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
|
|
102
|
+
// 空转 200(模型无输出)同模型暂停重试:默认 3 次、阶梯 [2s,8s,30s];仅 EMPTY_MODEL_RESPONSE,
|
|
103
|
+
// 429/403/500 与 fetch 异常走原有切号/failover(防烧额度)。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
|
|
104
|
+
const emptyCfg = emptyRetryCfg();
|
|
105
|
+
const raiseCap = emptyRaiseCap();
|
|
106
|
+
const nudgeCfg = emptyNudgeCfg();
|
|
107
|
+
let emptyRetried = 0;
|
|
108
|
+
let emptyWaitedMs = 0;
|
|
109
|
+
for (;;) {
|
|
110
|
+
const tUp = performance.now();
|
|
111
|
+
evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
|
|
112
|
+
try {
|
|
113
|
+
upRes = await upstream.chat(forwarded, chatOptsArg);
|
|
114
|
+
evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null, ...upstreamEcho(upRes) });
|
|
115
|
+
} catch (err) {
|
|
116
|
+
if (auto) await auto.recordError(model, { message: errMsg(err) });
|
|
117
|
+
lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
|
|
118
|
+
logError(model, 502, errMsg(err));
|
|
119
|
+
evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
|
|
120
|
+
upRes = null;
|
|
121
|
+
}
|
|
122
|
+
if (plugins?.length) {
|
|
123
|
+
runHook(plugins, "upstream:response", {
|
|
124
|
+
reqId, requested, model,
|
|
125
|
+
status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
|
|
126
|
+
ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
|
|
127
|
+
error: upRes instanceof Error ? errMsg(upRes) : null,
|
|
128
|
+
timing: upRes?._t ?? null,
|
|
129
|
+
}).catch(() => {});
|
|
130
|
+
}
|
|
131
|
+
mark(`up-${model}`);
|
|
132
|
+
if (upRes && upRes.status >= 400) {
|
|
133
|
+
const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
|
|
134
|
+
if (isAllowlistBlock) {
|
|
135
|
+
let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
|
|
136
|
+
let errBody = { error: `model not allowed for provider` };
|
|
137
|
+
try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
|
|
138
|
+
if (useAuto) {
|
|
139
|
+
logError(model, 403, errBody.error || "model not allowed");
|
|
140
|
+
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
|
|
141
|
+
lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
|
|
142
|
+
if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
|
|
143
|
+
return json(res, 403, errBody);
|
|
144
|
+
}
|
|
145
|
+
logError(model, 403, errBody.error || "model not allowed");
|
|
146
|
+
evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
|
|
147
|
+
return json(res, 403, errBody);
|
|
148
|
+
}
|
|
149
|
+
if (auto) await auto.recordError(model, { status: upRes.status });
|
|
150
|
+
// 读失败响应体(clone 不影响后续 relay 转发原响应;1s 上限防流式错误体拖慢)。
|
|
151
|
+
// 两份文本分工写死:`upRaw` 喂判据(≤4KB,反向护栏不能被 400 字摘要喂不到——多 provider 明细体
|
|
152
|
+
// 如 OpenRouter 的 `metadata.raw[]` 会把 rate limit 排在 model not found 之后),`upBody` 供日志摘要。
|
|
153
|
+
let upRaw = "";
|
|
154
|
+
let upBody = "";
|
|
155
|
+
upRaw = await readVerdictText(upRes);
|
|
156
|
+
upBody = upRaw.replace(/\s+/g, " ").slice(0, 400);
|
|
157
|
+
const upMsg = upBody || `upstream ${upRes.status}`;
|
|
158
|
+
lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
|
|
159
|
+
// 终局判决短路(判据与现网取证见 terminal-verdict.js)
|
|
160
|
+
const verdict = classifyTerminalUpstream(upRes.status, upRaw);
|
|
161
|
+
const terminalOn = verdict.terminal && !terminalShortcircuitOff();
|
|
162
|
+
// BUG-42 记账点:本机直连时上游判过额度/终局 ⇒ 换兄弟有账可依(接力冲不掉,详见 ADR-0061)
|
|
163
|
+
if (isPoolSwitchWorthy({ status: upRes.status, terminal: verdict.terminal })) poolWorthy = true;
|
|
164
|
+
// 一行话把「判决」与「原文」一起落 errors.log(不分两行:同一发失败在 errors.log 里只该有一条)
|
|
165
|
+
logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}${terminalOn ? ` · 终局判决 ${verdict.kind}(跳过组员接力)` : ""}`);
|
|
166
|
+
evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null, ...upstreamEcho(upRes) });
|
|
167
|
+
if (terminalOn) {
|
|
168
|
+
evt("upstream-terminal", { reqId, model, status: upRes.status, kind: verdict.kind, marker: verdict.marker, peersSkipped: true });
|
|
169
|
+
// ASCII 具名头。**不**复用 x-mslxdff-notice:那头的既定语义是「已被别的模型代答」(与
|
|
170
|
+
// x-mslxdff-fallback 成组,src/routes/fallback.js:27-39),拿它报「没代答、直接拒」会把按头
|
|
171
|
+
// 判是否换模型的下游 UI 读反。头值只填 ASCII 枚举,中文留在 evt/logError/timeline(文件是 UTF-8)。
|
|
172
|
+
try { if (!res.headersSent && typeof res.setHeader === "function") res.setHeader("x-mslxdff-terminal", verdict.kind); } catch {}
|
|
173
|
+
if (canFallback && idx < order.length - 1) {
|
|
174
|
+
// 判决只对**这个模型**成立 → 换下一候选。这里刻意不挂 upstream:这条 Response 的 body 没被
|
|
175
|
+
// 消费,挂在 lastErr 上带到收场会白占连接(收场只 relay 最后一发),故直接 cancel 后只留文本。
|
|
176
|
+
lastErr = { model, upstream: null, status: upRes.status, message: upMsg, terminal: verdict.kind };
|
|
177
|
+
try { const cx = upRes.body?.cancel?.(); if (cx && typeof cx.catch === "function") cx.catch(() => {}); } catch {}
|
|
178
|
+
evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `terminal ${verdict.kind}: ${upMsg.slice(0, 80)}` });
|
|
179
|
+
continue candidate;
|
|
180
|
+
}
|
|
181
|
+
// 收场交回既有出口:exhaustedLocal 见 lastErr.upstream 会 relay **原响应**(原状态码 + 原字节 +
|
|
182
|
+
// 原头),并自带 logCall / result / client-response / talk 语料落盘。自己 json() 反而劣化:
|
|
183
|
+
// 只能给被截到 400 字的二手摘要、JSON.parse 失败时退成字符串 error(SDK 读 err.error.message
|
|
184
|
+
// 拿到 undefined),且 SSE 已 flush 时 helpers 的 in-band 错误帧会把对象写成 [object Object]。
|
|
185
|
+
await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
|
|
186
|
+
return { done: true };
|
|
187
|
+
}
|
|
188
|
+
upRes = null;
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
if (upRes) {
|
|
192
|
+
const isStream = Boolean(body.stream);
|
|
193
|
+
const d = hedgeDelayMs();
|
|
194
|
+
const hasPeers = Boolean(peers) && peers.ordered().length > 0;
|
|
195
|
+
const canUseGroup = shouldUseGroupForModel(model);
|
|
196
|
+
const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
|
|
197
|
+
if (doHedge) {
|
|
198
|
+
const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res, hedgeDelayMs: d });
|
|
199
|
+
if (hr.handled) return { done: true };
|
|
200
|
+
if (hr.lastErr) lastErr = hr.lastErr;
|
|
201
|
+
if (hr.upRes === null) upRes = null;
|
|
202
|
+
else if (hr.upRes) upRes = hr.upRes;
|
|
203
|
+
}
|
|
204
|
+
if (upRes) {
|
|
205
|
+
const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, attemptStartMs: tUp, stages, startedAt, plugins, res });
|
|
206
|
+
if (lr.handled) {
|
|
207
|
+
// 空转重试后真拿到输出 = 降级但成功(WARN 语义):必须交代"第几次救回来的、白等了多久",
|
|
208
|
+
// 否则用户只看到"这一发变慢了",无从判断是重试在兜底还是上游真的死了。
|
|
209
|
+
// 「救回」必须有送达证据:handled:true 也可能来自下游已断开/零输出路径(见 ADR-0043),
|
|
210
|
+
// 只认 relay 报上来的 wrotePayload —— 没有正文到下游就不许记成功,否则日志在骗排障的人。
|
|
211
|
+
if (emptyRetried > 0 && lr.wrotePayload === true) {
|
|
212
|
+
evt("empty-turn-recovered", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, ...upstreamEcho(upRes) });
|
|
213
|
+
}
|
|
214
|
+
return { done: true };
|
|
215
|
+
}
|
|
216
|
+
if (lr.lastErr && isEmptyTurnError(lr.lastErr)) {
|
|
217
|
+
// 空转判据原文(finish_reason / 零正文 / 上游错误摘要)随事件落盘,排障不必再翻第二个文件
|
|
218
|
+
const _why = String(lr.lastErr.message || "").replace(/\s+/g, " ").trim().slice(0, 220);
|
|
219
|
+
const budgetMs = emptyTurnBudgetMs();
|
|
220
|
+
const budgetLeftMs = budgetMs - requestWaitedMs;
|
|
221
|
+
if (emptyRetried < emptyCfg.max && budgetLeftMs > 0) {
|
|
222
|
+
// 口径(用户定):正文为空就重试,最多 emptyCfg.max 次——不设"能不能送达"的前提,
|
|
223
|
+
// 大不了两次都空,反正不是无限重试。抬额度只加不减且有顶(默认 16384):
|
|
224
|
+
// 思考刷满 max_tokens 是零正文的主因,同参重拉必然复现,抬一次才算换了打法。
|
|
225
|
+
emptyRetried++;
|
|
226
|
+
const _before = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens) || null;
|
|
227
|
+
// 客户端没设额度也兜底发明一次:现网主流空轮就是「思考吃满 max_tokens、正文为零」,同参重拉必复现
|
|
228
|
+
const _raised = withRaisedMaxTokens(forwarded, raiseCap, emptyTurnMinRaiseTo());
|
|
229
|
+
let _after = null;
|
|
230
|
+
if (_raised !== forwarded) { forwarded = _raised; _after = Number(forwarded.max_tokens ?? forwarded.max_completion_tokens); }
|
|
231
|
+
const _isLast = emptyRetried >= emptyCfg.max;
|
|
232
|
+
let _nudged = false;
|
|
233
|
+
if (nudgeCfg.enabled && _isLast) {
|
|
234
|
+
const _next = withEmptyNudge(forwarded, nudgeCfg.text);
|
|
235
|
+
if (_next !== forwarded) { forwarded = _next; _nudged = true; }
|
|
236
|
+
}
|
|
237
|
+
// 事件在 sleep **之前**发:对着日志能立刻看到"正在暂停 Nms 重拉",而不是等结果
|
|
238
|
+
// 带上"刚空转的是哪个号/哪个站":切号是重试驱动的,日志必须能自证
|
|
239
|
+
const stepIdx = emptyCfg.steps ? (emptyRetried - 1) % emptyCfg.steps.length : 0;
|
|
240
|
+
const rawDelayMs = computeNextDelay(emptyRetried - 1, emptyCfg.steps, lr.lastErr);
|
|
241
|
+
const delayMs = Math.min(rawDelayMs, budgetLeftMs); // 末次等待不越过请求级预算
|
|
242
|
+
emptyWaitedMs += delayMs;
|
|
243
|
+
requestWaitedMs += delayMs;
|
|
244
|
+
evt("empty-turn-retry", { reqId, model, retry: emptyRetried, step: stepIdx, max: emptyCfg.max, delayMs, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs, nudged: _nudged ? 1 : undefined, raiseFrom: _after != null ? _before : undefined, raiseTo: _after ?? undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
245
|
+
await sleep(delayMs);
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
// 次数用尽(或被 MSLXDFF_EMPTY_TURN_RETRIES=0 关掉)→ 记一行"不再重试"再交回 failover
|
|
249
|
+
const _budgetOut = emptyRetried < emptyCfg.max && emptyTurnBudgetMs() - requestWaitedMs <= 0;
|
|
250
|
+
evt("empty-turn-exhausted", { reqId, model, retries: emptyRetried, max: emptyCfg.max, waitedMs: emptyWaitedMs, requestWaitedMs, budgetMs: emptyTurnBudgetMs(), budgetOut: _budgetOut ? 1 : undefined, reason: _why, ...upstreamEcho(upRes) });
|
|
251
|
+
try { logError(model, 502, `空转重试 ${emptyRetried}/${emptyCfg.max} 后仍无输出${emptyCfg.max === 0 ? "(MSLXDFF_EMPTY_TURN_RETRIES=0 已关闭重试)" : _budgetOut ? `(请求级等待预算 ${emptyTurnBudgetMs()}ms 用尽)` : ""}:${_why}`); } catch {}
|
|
252
|
+
}
|
|
253
|
+
if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
|
|
254
|
+
return { done: true };
|
|
255
|
+
}
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
if (canForwardPeers) {
|
|
261
|
+
if (!shouldUseGroupForModel(model)) {
|
|
262
|
+
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (peer)` });
|
|
263
|
+
} else {
|
|
264
|
+
const pr = await peerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, attemptStartMs: performance.now(), stages, startedAt, plugins, res });
|
|
265
|
+
if (pr.handled) return { done: true };
|
|
266
|
+
if (pr.lastErr) lastErr = pr.lastErr;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
if (groups) {
|
|
270
|
+
if (!shouldUseGroupForModel(model)) {
|
|
271
|
+
evt("group-skip", { reqId, model, reason: `${groupSkipReason(model)} (broadband)` });
|
|
272
|
+
} else {
|
|
273
|
+
const br = await broadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, attemptStartMs: performance.now(), stages, res, startedAt, plugins });
|
|
274
|
+
if (br.handled) return { done: true };
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (canFailover(lastErr)) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, pool: poolFallback || undefined, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` }); continue; }
|
|
278
|
+
await exhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
|
|
279
|
+
return { done: true };
|
|
280
|
+
}
|
|
281
|
+
await exhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
|
|
282
|
+
return { done: true };
|
|
283
|
+
}
|