maxpool 1.5.47 → 1.5.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +29 -0
- package/src/index.js +46 -3
- package/src/reload-protocol.js +7 -0
- package/src/server.js +207 -14
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -1881,6 +1881,35 @@ export class AccountManager {
|
|
|
1881
1881
|
this.sessionPolicies.set(sessionKey, { ...existing, thinkingContaminated: true });
|
|
1882
1882
|
}
|
|
1883
1883
|
|
|
1884
|
+
/** Release a session pinned provider-only by an earlier turn. The incompatible latch is
|
|
1885
|
+
* deliberately sticky (it never downgrades), which is right while the transcript really
|
|
1886
|
+
* is unrepairable — but a repaired body IS replayable on Claude, so the pin must lift
|
|
1887
|
+
* or every previously-broken session stays exiled forever. */
|
|
1888
|
+
clearSessionIncompatible(sessionKey) {
|
|
1889
|
+
if (!sessionKey) return;
|
|
1890
|
+
const existing = this.sessionPolicies.get(sessionKey);
|
|
1891
|
+
if (!existing?.anthropicIncompatible) return;
|
|
1892
|
+
this.sessionPolicies.set(sessionKey, { ...existing, anthropicIncompatible: false });
|
|
1893
|
+
console.log(`[Maxpool] Session "${sessionKey}" repaired — Claude routes re-enabled`);
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
/** Remember the effort level a MODEL actually accepted for this session, so later turns
|
|
1897
|
+
* apply it up front. The client resends the same rejected setting every turn, so without
|
|
1898
|
+
* this each turn pays a rejected round-trip AND charges a consecutive-failure to whichever
|
|
1899
|
+
* account ate it — deprioritising a perfectly healthy account for a client-side setting.
|
|
1900
|
+
* Keyed by model: opus-4-5 takes 'high' while opus-4-1 takes no effort at all. */
|
|
1901
|
+
markSessionEffort(sessionKey, model, effort) {
|
|
1902
|
+
if (!sessionKey || !model) return;
|
|
1903
|
+
const existing = this.sessionPolicies.get(sessionKey) || {};
|
|
1904
|
+
this.sessionPolicies.set(sessionKey, { ...existing, effortFix: { model, effort } });
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
getSessionEffort(sessionKey, model) {
|
|
1908
|
+
if (!sessionKey || !model) return undefined;
|
|
1909
|
+
const fix = this.sessionPolicies.get(sessionKey)?.effortFix;
|
|
1910
|
+
return fix && fix.model === model ? fix : undefined;
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1884
1913
|
isSessionThinkingContaminated(sessionKey) {
|
|
1885
1914
|
if (!sessionKey) return false;
|
|
1886
1915
|
return Boolean(this.sessionPolicies.get(sessionKey)?.thinkingContaminated);
|
package/src/index.js
CHANGED
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
runReloadBaton,
|
|
18
18
|
RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
|
|
19
19
|
MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
|
|
20
|
-
MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY,
|
|
20
|
+
MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
|
|
21
21
|
} from './reload-protocol.js';
|
|
22
22
|
|
|
23
23
|
const args = process.argv.slice(2);
|
|
@@ -626,7 +626,9 @@ async function serverWorkerCommand() {
|
|
|
626
626
|
// before (or without) the npm update check. The cold worker's update check below
|
|
627
627
|
// fills in latest/hasUpdate.
|
|
628
628
|
getCurrentVersion()
|
|
629
|
-
.then(v => {
|
|
629
|
+
.then(v => {
|
|
630
|
+
accountManager.versionInfo ||= { current: v, latest: null, hasUpdate: false, checkedAt: null };
|
|
631
|
+
})
|
|
630
632
|
.catch(() => {});
|
|
631
633
|
|
|
632
634
|
// Persist refreshed tokens back to config. Defense-in-depth: the updater reads
|
|
@@ -727,6 +729,10 @@ async function serverWorkerCommand() {
|
|
|
727
729
|
// Self-heal timer for a seamless reload that rolls back (new worker fails to boot,
|
|
728
730
|
// we're never released). Cleared on MSG_RELEASE; fires cancelRestart otherwise.
|
|
729
731
|
let reloadWatchdog = null;
|
|
732
|
+
// One cold-restart fallback per reload request — see the rollback watchdog below.
|
|
733
|
+
let coldFallbackUsed = false;
|
|
734
|
+
let reloadIsForUpdate = false;
|
|
735
|
+
let lastRollbackReason = null;
|
|
730
736
|
|
|
731
737
|
// Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
|
|
732
738
|
// a bare process.exit, a crash) so the user's shell is never left in raw mode
|
|
@@ -857,6 +863,8 @@ async function serverWorkerCommand() {
|
|
|
857
863
|
|| process.env.MAXPOOL_TUI_COLD_RESTART === '1';
|
|
858
864
|
if (!forceCold && reloadStrategy({ supervised }) === 'seamless') {
|
|
859
865
|
try {
|
|
866
|
+
coldFallbackUsed = false; // fresh reload → fresh cold-fallback budget
|
|
867
|
+
lastRollbackReason = null;
|
|
860
868
|
process.send({ type: MSG_RELOAD_REQUEST });
|
|
861
869
|
// Arm the rollback self-heal: restartController already latched
|
|
862
870
|
// pending/restarting + paused admission (in _restart, before we got here).
|
|
@@ -868,7 +876,29 @@ async function serverWorkerCommand() {
|
|
|
868
876
|
reloadWatchdog = null;
|
|
869
877
|
if (draining) return; // MSG_RELEASE already arrived → a real reload, not a rollback
|
|
870
878
|
if (restartController?.cancelRestart()) {
|
|
871
|
-
console.log('[Maxpool] Reload rolled back (new worker never took over)
|
|
879
|
+
console.log('[Maxpool] Reload rolled back (new worker never took over).');
|
|
880
|
+
}
|
|
881
|
+
// FALL BACK TO A COLD RESTART — but ONLY when a newer build is actually on disk
|
|
882
|
+
// waiting to be picked up. Resuming the old worker is what made "press u → c"
|
|
883
|
+
// look like nothing happened: the graceful swap failed under machine load and
|
|
884
|
+
// the update was abandoned, forever. Gating on a real pending update preserves
|
|
885
|
+
// the safety property that a plain failed reload leaves a HEALTHY worker serving
|
|
886
|
+
// (never kill a working process for nothing), while an update still lands.
|
|
887
|
+
// One attempt per reload request, so a build that cannot boot can't crash-loop.
|
|
888
|
+
// Cold-restart ONLY when the new build was merely SLOW to signal ready (a
|
|
889
|
+
// loaded machine — the reported case). If it reported failure or died before
|
|
890
|
+
// ready, the build cannot boot: restarting into it would leave the user with no
|
|
891
|
+
// working proxy at all, strictly worse than the bug being fixed. Absent reason
|
|
892
|
+
// (older supervisor) is treated as unsafe.
|
|
893
|
+
const swapWasSlowNotBroken = lastRollbackReason === 'timeout';
|
|
894
|
+
if (reloadIsForUpdate && swapWasSlowNotBroken && !coldFallbackUsed) {
|
|
895
|
+
coldFallbackUsed = true;
|
|
896
|
+
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
897
|
+
restartWorkerNow();
|
|
898
|
+
} else if (reloadIsForUpdate && !swapWasSlowNotBroken) {
|
|
899
|
+
console.log('[Maxpool] The new version failed to start — resumed serving on the current version. Update NOT applied.');
|
|
900
|
+
} else {
|
|
901
|
+
console.log('[Maxpool] Rollback complete — resumed serving on the current version.');
|
|
872
902
|
}
|
|
873
903
|
}, RELOAD_ROLLBACK_SELFHEAL_MS);
|
|
874
904
|
reloadWatchdog.unref?.();
|
|
@@ -1060,6 +1090,11 @@ async function serverWorkerCommand() {
|
|
|
1060
1090
|
if (r?.applicable && config?.autoApply && restartController) {
|
|
1061
1091
|
markApplied(r.installedVersion);
|
|
1062
1092
|
notifyUpdate('Applying update — seamless reload…');
|
|
1093
|
+
// Mark this reload as UPDATE-driven: if the seamless swap rolls back (machine
|
|
1094
|
+
// under load), the watchdog cold-restarts so the update actually lands instead of
|
|
1095
|
+
// silently resuming the old build. A plain 'r' restart never does that — there a
|
|
1096
|
+
// rollback should leave the healthy worker serving.
|
|
1097
|
+
reloadIsForUpdate = true;
|
|
1063
1098
|
restartController.requestRestart();
|
|
1064
1099
|
}
|
|
1065
1100
|
};
|
|
@@ -1071,6 +1106,11 @@ async function serverWorkerCommand() {
|
|
|
1071
1106
|
if (r?.applicable && restartController) {
|
|
1072
1107
|
markApplied(r.installedVersion);
|
|
1073
1108
|
notifyUpdate('Applying update — seamless reload…');
|
|
1109
|
+
// Mark this reload as UPDATE-driven: if the seamless swap rolls back (machine
|
|
1110
|
+
// under load), the watchdog cold-restarts so the update actually lands instead of
|
|
1111
|
+
// silently resuming the old build. A plain 'r' restart never does that — there a
|
|
1112
|
+
// rollback should leave the healthy worker serving.
|
|
1113
|
+
reloadIsForUpdate = true;
|
|
1074
1114
|
restartController.requestRestart();
|
|
1075
1115
|
}
|
|
1076
1116
|
};
|
|
@@ -1284,6 +1324,9 @@ async function serverWorkerCommand() {
|
|
|
1284
1324
|
// sends MSG_PRIMARY (the baton waits on it).
|
|
1285
1325
|
await listenOnHandle(handle);
|
|
1286
1326
|
await becomePrimary({ viaTakeover: true });
|
|
1327
|
+
} else if (msg?.type === MSG_ROLLED_BACK) {
|
|
1328
|
+
// Why the swap failed decides whether a cold restart is safe (see the watchdog).
|
|
1329
|
+
lastRollbackReason = msg.reason || null;
|
|
1287
1330
|
} else if (msg?.type === MSG_RELEASE) {
|
|
1288
1331
|
// Baton release: stop accepting NEW (keep in-flight), retire keep-alive
|
|
1289
1332
|
// sockets with Connection: close, stop writing, flush once, drop TUI.
|
package/src/reload-protocol.js
CHANGED
|
@@ -14,6 +14,11 @@
|
|
|
14
14
|
|
|
15
15
|
// Supervisor → worker
|
|
16
16
|
export const MSG_LISTEN = 'listen'; // (with handle) accept on this socket + take TUI/lease
|
|
17
|
+
// Tells the surviving old worker WHY a reload rolled back. 'timeout' = the new worker
|
|
18
|
+
// was merely slow to signal ready (a loaded machine) — safe to retry with a cold
|
|
19
|
+
// restart. 'failed' = it reported failure or died before ready — a cold restart there
|
|
20
|
+
// would commit to a build that cannot boot and leave NO working proxy.
|
|
21
|
+
export const MSG_ROLLED_BACK = 'rolled-back';
|
|
17
22
|
export const MSG_RELEASE = 'release'; // stop accepting, stop writing, flush, give up TUI
|
|
18
23
|
export const MSG_TAKEOVER = 'takeover'; // (with handle) start accepting + acquire writer lease + TUI
|
|
19
24
|
export const MSG_PROBE_READY = 'probe-ready'; // ask a headless worker to confirm it booted OK
|
|
@@ -70,11 +75,13 @@ export async function runReloadBaton({
|
|
|
70
75
|
ready = await newWorker.waitFor([MSG_READY, MSG_FAILED], readyTimeoutMs);
|
|
71
76
|
} catch (err) {
|
|
72
77
|
log(`reload: readiness wait failed (${err.message}); rolling back`);
|
|
78
|
+
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'timeout' }); } catch { /* old worker gone */ }
|
|
73
79
|
return RELOAD_ROLLED_BACK;
|
|
74
80
|
}
|
|
75
81
|
// (b) New worker did not come up cleanly → roll back fully, old stays primary.
|
|
76
82
|
if (!ready || ready.type === MSG_FAILED) {
|
|
77
83
|
log(`reload: new worker reported failed (${ready?.reason || 'no ready'}); rolling back`);
|
|
84
|
+
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'failed' }); } catch { /* old worker gone */ }
|
|
78
85
|
return RELOAD_ROLLED_BACK;
|
|
79
86
|
}
|
|
80
87
|
|
package/src/server.js
CHANGED
|
@@ -357,12 +357,42 @@ async function forwardRequest(
|
|
|
357
357
|
// PRE-STRIP a session already known to carry provider-authored thinking. The client
|
|
358
358
|
// resends the whole poisoned history every turn, so without this each turn pays another
|
|
359
359
|
// rejected round-trip before the reactive repair kicks in. Latched by the first repair.
|
|
360
|
+
// Apply the effort level this session's model already proved it accepts, so later turns
|
|
361
|
+
// skip the rejected round-trip entirely (the client resends the same setting every turn).
|
|
362
|
+
if (retryCount === 0 && !requestInfo.effortRepaired) {
|
|
363
|
+
const latched = accountManager.getSessionEffort?.(requestInfo.sessionKey, requestInfo.model);
|
|
364
|
+
if (latched) {
|
|
365
|
+
const pre = repairEffort(body, latched.effort ? 'downgrade' : 'drop',
|
|
366
|
+
latched.effort ? `Supported levels: ${latched.effort}` : '');
|
|
367
|
+
if (pre.body) {
|
|
368
|
+
body = pre.body;
|
|
369
|
+
requestInfo = { ...requestInfo, effortRepaired: true };
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// ALSO repairs a transcript maxpool predicts Anthropic will reject (a provider web
|
|
375
|
+
// search). That prediction happens BEFORE any request is sent and bars every Claude
|
|
376
|
+
// account, so the reactive repair in the 4xx handler could never be reached for it —
|
|
377
|
+
// the session stayed exiled to GLM/Kimi, or got NO ROUTE AT ALL when they're disabled
|
|
378
|
+
// (8 healthy Claude accounts idle while the user is told nothing is available).
|
|
379
|
+
// Repairing here, ahead of routing, is what makes that case recoverable.
|
|
360
380
|
if (retryCount === 0 && !requestInfo.thinkingStripped
|
|
361
|
-
&&
|
|
381
|
+
&& (requestInfo.anthropicIncompatible
|
|
382
|
+
|| accountManager.isSessionThinkingContaminated?.(requestInfo.sessionKey))) {
|
|
362
383
|
const pre = stripForeignThinkingBlocks(body);
|
|
363
384
|
if (pre.body) {
|
|
364
385
|
body = pre.body;
|
|
365
386
|
requestInfo = { ...requestInfo, thinkingStripped: true };
|
|
387
|
+
if (pre.converted) {
|
|
388
|
+
// The body now replays cleanly on Claude, so drop the predictive verdict —
|
|
389
|
+
// otherwise routing still exiles it and _noteRequestPolicy latches it sticky.
|
|
390
|
+
requestInfo = { ...requestInfo, anthropicIncompatible: false };
|
|
391
|
+
// And un-latch a session pinned by an EARLIER turn: the sticky policy is ORed in
|
|
392
|
+
// and never downgrades, so without this the user's already-broken sessions stay
|
|
393
|
+
// pinned forever even though we can now repair them.
|
|
394
|
+
accountManager.clearSessionIncompatible?.(requestInfo.sessionKey);
|
|
395
|
+
}
|
|
366
396
|
}
|
|
367
397
|
}
|
|
368
398
|
|
|
@@ -915,13 +945,32 @@ async function forwardRequest(
|
|
|
915
945
|
// fixes it. Still deterministic — never the fuzzy isAnthropicIncompatBody heuristic.
|
|
916
946
|
const isSignatureRejection = account.type !== 'provider'
|
|
917
947
|
&& (/invalid `signature` in `thinking`/i.test(errorBody)
|
|
918
|
-
|| /content\.\d+\.thinking\.signature/i.test(errorBody)
|
|
948
|
+
|| /content\.\d+\.thinking\.signature/i.test(errorBody)
|
|
949
|
+
// A provider web search: `server_tool_use.id: String should match pattern
|
|
950
|
+
// '^srvtoolu_…'`. Repairable by converting the pair to text (verified 200 OK) —
|
|
951
|
+
// it used to fall through to a PERMANENT provider pin.
|
|
952
|
+
|| /server_tool_use\.id: String should match pattern/i.test(errorBody));
|
|
953
|
+
// LOG THE ACTUAL REASON. Previously a 4xx recorded only "HTTP 400" and the upstream
|
|
954
|
+
// message was never written anywhere, so a whole class of failures (e.g. a rejected
|
|
955
|
+
// effort level breaking every web search) was invisible in the log — you could not
|
|
956
|
+
// even grep for it. Truncated so a huge validation dump can't wall the file.
|
|
957
|
+
if (upstreamRes.status >= 400 && upstreamRes.status !== 429) {
|
|
958
|
+
const why = (() => {
|
|
959
|
+
try { return JSON.parse(errorBody)?.error?.message || errorBody; } catch { return errorBody; }
|
|
960
|
+
})();
|
|
961
|
+
console.log(`[Maxpool] ${upstreamRes.status} from "${account.name}": ${String(why).slice(0, 300)}`);
|
|
962
|
+
}
|
|
919
963
|
const errorType = errorBody.includes('Invalid `signature` in `thinking` block')
|
|
920
964
|
? 'invalid_thinking_signature'
|
|
921
965
|
: anthropicIncompat ? 'anthropic_incompatible_transcript'
|
|
922
966
|
: providerTooSmall ? 'provider_context_too_small'
|
|
923
967
|
: `HTTP ${upstreamRes.status}`;
|
|
924
|
-
|
|
968
|
+
const effortMode = classifyEffortRejection(errorBody);
|
|
969
|
+
// A rejected effort level is a REQUEST-shaped fault, not an account-health signal —
|
|
970
|
+
// release neutral so it never charges a consecutive-failure to a healthy account.
|
|
971
|
+
accountManager.releaseAccount(lease, effortMode
|
|
972
|
+
? { status: upstreamRes.status, error: errorType, neutral: true }
|
|
973
|
+
: { status: upstreamRes.status, error: errorType });
|
|
925
974
|
|
|
926
975
|
if (logDir) {
|
|
927
976
|
logSections.push(`=== RESPONSE ${upstreamRes.status} — non-retryable client error from "${account.name}" ===\n${formatHeaders(upstreamRes.headers)}`);
|
|
@@ -950,6 +999,25 @@ async function forwardRequest(
|
|
|
950
999
|
}
|
|
951
1000
|
}
|
|
952
1001
|
|
|
1002
|
+
// EFFORT REPAIR. A rejected output_config.effort is a hard failure of whatever the
|
|
1003
|
+
// client was doing (a web search, a tool call) — worth healing rather than surfacing.
|
|
1004
|
+
if (effortMode && !requestInfo.effortRepaired
|
|
1005
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !res.headersSent) {
|
|
1006
|
+
const fix = repairEffort(body, effortMode, errorBody);
|
|
1007
|
+
if (fix.body) {
|
|
1008
|
+
// Latch it so LATER turns apply the working level up front: the client resends the
|
|
1009
|
+
// same rejected setting every turn, and each rejection would otherwise charge a
|
|
1010
|
+
// consecutive-failure to a healthy account and deprioritise it in the router.
|
|
1011
|
+
accountManager.markSessionEffort?.(requestInfo.sessionKey, requestInfo.model, fix.effort);
|
|
1012
|
+
console.log(`[Maxpool] "${requestInfo.model || 'model'}" rejected effort "${requestInfo.effort || 'xhigh'}" (via ${account.name}); retrying with ${fix.effort ? `effort "${fix.effort}"` : 'the effort setting removed'}`);
|
|
1013
|
+
return forwardRequest(
|
|
1014
|
+
req, res, fix.body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1015
|
+
retryConfig, queueConfig, { ...requestInfo, effortRepaired: true },
|
|
1016
|
+
canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
1017
|
+
);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
953
1021
|
// RECOVER-ON-CLAUDE (preferred over the provider pin below): the transcript
|
|
954
1022
|
// carries provider-authored thinking blocks whose signature Anthropic rejects.
|
|
955
1023
|
// Strip exactly those blocks and retry on Claude, so a session that took even one
|
|
@@ -959,12 +1027,12 @@ async function forwardRequest(
|
|
|
959
1027
|
// still fails, the provider pin below is the fallback.
|
|
960
1028
|
if (isSignatureRejection && !requestInfo.thinkingStripped
|
|
961
1029
|
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !res.headersSent) {
|
|
962
|
-
const { body: cleanBody, removed } = stripForeignThinkingBlocks(body);
|
|
1030
|
+
const { body: cleanBody, removed, converted } = stripForeignThinkingBlocks(body);
|
|
963
1031
|
if (cleanBody) {
|
|
964
1032
|
// Latch it so EVERY later turn is stripped up front instead of re-paying this
|
|
965
1033
|
// rejected round-trip (the client resends the full poisoned history each turn).
|
|
966
1034
|
accountManager.markSessionThinkingContaminated?.(requestInfo.sessionKey);
|
|
967
|
-
console.log(`[Maxpool] Recovering session on Claude: stripped ${removed} provider
|
|
1035
|
+
console.log(`[Maxpool] Recovering session on Claude: stripped ${removed} provider thinking block(s), converted ${converted} provider search block(s) to text`);
|
|
968
1036
|
return forwardRequest(
|
|
969
1037
|
req, res, cleanBody, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
970
1038
|
retryConfig, queueConfig, { ...requestInfo, thinkingStripped: true },
|
|
@@ -1030,8 +1098,8 @@ async function forwardRequest(
|
|
|
1030
1098
|
// ran; without it we found nothing provider-shaped, so claiming we stripped —
|
|
1031
1099
|
// or blaming GLM/Kimi — would misdirect the user.
|
|
1032
1100
|
const what = requestInfo.thinkingStripped
|
|
1033
|
-
? 'This session ran on GLM/Kimi earlier and Anthropic
|
|
1034
|
-
: "Anthropic rejected
|
|
1101
|
+
? 'This session ran on GLM/Kimi earlier, and Anthropic will not accept parts of what they wrote. Maxpool repaired what it could and retried on Claude, but Anthropic still rejected the history.'
|
|
1102
|
+
: "Anthropic rejected part of this session's history that maxpool could not repair automatically.";
|
|
1035
1103
|
const hint = provs.length === 0
|
|
1036
1104
|
? ''
|
|
1037
1105
|
: provs.every(a => a.enabled === false)
|
|
@@ -1355,7 +1423,7 @@ function isContextLengthError(errorBody) {
|
|
|
1355
1423
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
1356
1424
|
}
|
|
1357
1425
|
|
|
1358
|
-
export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, streamResponse, startIdleRequestReaper };
|
|
1426
|
+
export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, streamResponse, startIdleRequestReaper };
|
|
1359
1427
|
|
|
1360
1428
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
1361
1429
|
if (!upstreamRes.body) return '';
|
|
@@ -1556,22 +1624,147 @@ function isStrippableThinkingBlock(block) {
|
|
|
1556
1624
|
*
|
|
1557
1625
|
* Returns { body, removed } — `body` is null when nothing needed stripping.
|
|
1558
1626
|
*/
|
|
1627
|
+
// A provider's web search leaves a `server_tool_use` id Anthropic rejects (it demands
|
|
1628
|
+
// ^srvtoolu_). Verified 2026-07-26 against the live API: renaming the id is NOT enough —
|
|
1629
|
+
// a second gate rejects the result's `encrypted_content`, which only Anthropic can mint.
|
|
1630
|
+
// But converting the pair into plain TEXT is accepted (200 OK) and keeps what the search
|
|
1631
|
+
// actually found, so the session survives with its information intact. This is what made
|
|
1632
|
+
// the web-search case look permanently unrepairable.
|
|
1633
|
+
// Claude Code can send an `output_config.effort` the target model won't take — usually
|
|
1634
|
+
// after the session's model changes (a resume/fallback) while the effort setting stays.
|
|
1635
|
+
// It is a HARD error: the tool call just fails, which is what killed the user's web
|
|
1636
|
+
// searches. Three shapes seen live 2026-07-26, all repairable:
|
|
1637
|
+
// "does not support effort level 'xhigh'. Supported levels: high, low, medium" -> downgrade
|
|
1638
|
+
// "'xhigh' is not supported when thinking is disabled … Use effort 'high' or below" -> downgrade
|
|
1639
|
+
// "does not support the effort parameter." -> drop it
|
|
1640
|
+
function classifyEffortRejection(errorBody) {
|
|
1641
|
+
if (!/effort/i.test(errorBody)) return null;
|
|
1642
|
+
if (/does not support the effort parameter/i.test(errorBody)) return 'drop';
|
|
1643
|
+
if (/does not support effort level|is not supported when thinking is disabled/i.test(errorBody)
|
|
1644
|
+
|| /output_config\.effort: Input should be/i.test(errorBody)) { // invalid value from the client
|
|
1645
|
+
return 'downgrade';
|
|
1646
|
+
}
|
|
1647
|
+
return null;
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
/** Rewrite the request's effort so the model accepts it. 'downgrade' picks the best level
|
|
1651
|
+
* the error itself advertises (falling back to 'high'); 'drop' removes the field. */
|
|
1652
|
+
function repairEffort(body, mode, errorBody = '') {
|
|
1653
|
+
try {
|
|
1654
|
+
const json = JSON.parse(Buffer.isBuffer(body) ? body.toString('utf8') : String(body));
|
|
1655
|
+
const cur = json?.output_config?.effort;
|
|
1656
|
+
if (!cur) return { body: null, effort: null };
|
|
1657
|
+
if (mode === 'drop') {
|
|
1658
|
+
delete json.output_config.effort;
|
|
1659
|
+
if (Object.keys(json.output_config).length === 0) delete json.output_config;
|
|
1660
|
+
return { body: Buffer.from(JSON.stringify(json)), effort: null };
|
|
1661
|
+
}
|
|
1662
|
+
// Prefer a level the error explicitly lists, else 'high' (what the message recommends).
|
|
1663
|
+
const rank = ['max', 'xhigh', 'high', 'medium', 'low'];
|
|
1664
|
+
const listed = /supported levels:\s*([a-z, ']+)/i.exec(errorBody)?.[1];
|
|
1665
|
+
let allowed = listed ? listed.split(',').map(x => x.trim().replace(/'/g, '').toLowerCase()).filter(Boolean) : [];
|
|
1666
|
+
// The other real shape names a ceiling instead of a list: "Use effort 'high' or below".
|
|
1667
|
+
const ceiling = /use effort '([a-z]+)' or below/i.exec(errorBody)?.[1]?.toLowerCase();
|
|
1668
|
+
if (!allowed.length && ceiling) allowed = rank.slice(rank.indexOf(ceiling)).filter(Boolean);
|
|
1669
|
+
let next = rank.find(r => allowed.includes(r));
|
|
1670
|
+
// Nothing usable advertised (or it names the level we already sent) — step strictly
|
|
1671
|
+
// BELOW the current level rather than giving up, so we never retry the same value.
|
|
1672
|
+
if (!next || next === cur) {
|
|
1673
|
+
const below = rank.slice(rank.indexOf(cur) + 1);
|
|
1674
|
+
next = below.find(r => !allowed.length || allowed.includes(r)) || below[0];
|
|
1675
|
+
}
|
|
1676
|
+
if (!next || next === cur) return { body: null, effort: null };
|
|
1677
|
+
json.output_config.effort = next;
|
|
1678
|
+
return { body: Buffer.from(JSON.stringify(json)), effort: next };
|
|
1679
|
+
} catch {
|
|
1680
|
+
return { body: null, effort: null };
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
/** Collect foreign server-tool ids across the WHOLE transcript first — a call sits on the
|
|
1685
|
+
* assistant turn but its result is often carried on the FOLLOWING user turn, so a
|
|
1686
|
+
* per-message scan would leave that result behind (and it alone still 400s). */
|
|
1687
|
+
function collectForeignServerToolIds(messages) {
|
|
1688
|
+
const ids = new Set();
|
|
1689
|
+
const clientIds = new Set();
|
|
1690
|
+
const walk = (blocks) => {
|
|
1691
|
+
if (!Array.isArray(blocks)) return;
|
|
1692
|
+
for (const b of blocks) {
|
|
1693
|
+
if (b?.type === 'tool_use' && b.id) clientIds.add(b.id);
|
|
1694
|
+
if (b?.type === 'server_tool_use' && !ANTHROPIC_TOOL_ID.test(String(b.id || ''))) ids.add(b.id);
|
|
1695
|
+
if (Array.isArray(b?.content)) walk(b.content); // nested, mirrors detectTranscriptOrigin
|
|
1696
|
+
}
|
|
1697
|
+
};
|
|
1698
|
+
for (const msg of messages) walk(msg?.content);
|
|
1699
|
+
// NEVER treat an id that a real client tool_use also owns as foreign: converting its
|
|
1700
|
+
// tool_result would orphan that tool_use and Anthropic 400s ("tool_use ids without
|
|
1701
|
+
// tool_result"), which the sticky pre-strip would then re-inflict every turn — a
|
|
1702
|
+
// permanent loop. Providers with per-turn counters (call_0, call_1) make the collision
|
|
1703
|
+
// likely, not theoretical.
|
|
1704
|
+
for (const id of clientIds) ids.delete(id);
|
|
1705
|
+
return ids;
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
function convertForeignServerTools(content, foreignIds) {
|
|
1709
|
+
if (!foreignIds || foreignIds.size === 0) return { content, converted: 0 };
|
|
1710
|
+
let converted = 0;
|
|
1711
|
+
const out = [];
|
|
1712
|
+
for (const b of content) {
|
|
1713
|
+
if (b?.type === 'server_tool_use' && foreignIds.has(b.id)) {
|
|
1714
|
+
const q = b.input?.query;
|
|
1715
|
+
out.push({ type: 'text', text: q ? `[searched the web for: ${q}]` : `[used ${b.name || 'a tool'}]` });
|
|
1716
|
+
converted++;
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
// Any result block referring to a foreign call — web_search_tool_result and friends.
|
|
1720
|
+
if (b?.tool_use_id && foreignIds.has(b.tool_use_id)) {
|
|
1721
|
+
const rows = Array.isArray(b.content) ? b.content : [];
|
|
1722
|
+
// Fall back through the shapes a PROVIDER may use — the point of converting rather
|
|
1723
|
+
// than dropping is to keep what the search found; assuming Anthropic's {title,url}
|
|
1724
|
+
// would silently discard a GLM row that carries text instead.
|
|
1725
|
+
const found = rows
|
|
1726
|
+
.map(r => [r?.title, r?.url].filter(Boolean).join(' — ')
|
|
1727
|
+
|| (typeof r === 'string' ? r : (r?.text ?? r?.content ?? r?.snippet ?? '')))
|
|
1728
|
+
.map(t => (typeof t === 'string' ? t.slice(0, 300) : ''))
|
|
1729
|
+
.filter(Boolean).slice(0, 10);
|
|
1730
|
+
out.push({ type: 'text', text: found.length ? `[search results: ${found.join(' | ')}]` : '[search returned no usable results]' });
|
|
1731
|
+
converted++;
|
|
1732
|
+
continue;
|
|
1733
|
+
}
|
|
1734
|
+
out.push(b);
|
|
1735
|
+
}
|
|
1736
|
+
return { content: out, converted };
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1559
1739
|
function stripForeignThinkingBlocks(body) {
|
|
1560
1740
|
try {
|
|
1561
1741
|
const json = JSON.parse(Buffer.isBuffer(body) ? body.toString('utf8') : String(body));
|
|
1562
1742
|
if (!Array.isArray(json?.messages)) return { body: null, removed: 0 };
|
|
1563
1743
|
let removed = 0;
|
|
1744
|
+
let converted = 0;
|
|
1745
|
+
const foreignToolIds = collectForeignServerToolIds(json.messages);
|
|
1564
1746
|
const messages = [];
|
|
1565
1747
|
for (const msg of json.messages) {
|
|
1566
|
-
if (
|
|
1748
|
+
if (!Array.isArray(msg?.content)) { messages.push(msg); continue; }
|
|
1749
|
+
// Foreign server-tool pairs are converted on EVERY role: the call sits on the
|
|
1750
|
+
// assistant turn but its result can be carried on the following user turn.
|
|
1751
|
+
const tools = convertForeignServerTools(msg.content, foreignToolIds);
|
|
1752
|
+
converted += tools.converted;
|
|
1753
|
+
if (msg.role !== 'assistant') {
|
|
1754
|
+
messages.push(tools.converted ? { ...msg, content: tools.content } : msg);
|
|
1755
|
+
continue;
|
|
1756
|
+
}
|
|
1567
1757
|
let localRemoved = 0;
|
|
1568
|
-
const kept =
|
|
1758
|
+
const kept = tools.content.filter(block => {
|
|
1569
1759
|
if (!isStrippableThinkingBlock(block)) return true;
|
|
1570
1760
|
localRemoved++;
|
|
1571
1761
|
return false;
|
|
1572
1762
|
});
|
|
1573
1763
|
removed += localRemoved;
|
|
1574
|
-
if (!localRemoved) {
|
|
1764
|
+
if (!localRemoved) {
|
|
1765
|
+
messages.push(tools.converted ? { ...msg, content: tools.content } : msg);
|
|
1766
|
+
continue;
|
|
1767
|
+
}
|
|
1575
1768
|
// A turn that stripping empties is DROPPED, not left with its poisoned block:
|
|
1576
1769
|
// leaving it would resend the exact body that just 400'd while reporting success
|
|
1577
1770
|
// and burning the single recovery attempt. Verified against the live API — the
|
|
@@ -1579,11 +1772,11 @@ function stripForeignThinkingBlocks(body) {
|
|
|
1579
1772
|
if (kept.length === 0) continue;
|
|
1580
1773
|
messages.push({ ...msg, content: kept });
|
|
1581
1774
|
}
|
|
1582
|
-
if (!removed) return { body: null, removed: 0 };
|
|
1775
|
+
if (!removed && !converted) return { body: null, removed: 0, converted: 0 };
|
|
1583
1776
|
json.messages = messages;
|
|
1584
|
-
return { body: Buffer.from(JSON.stringify(json)), removed };
|
|
1777
|
+
return { body: Buffer.from(JSON.stringify(json)), removed, converted };
|
|
1585
1778
|
} catch {
|
|
1586
|
-
return { body: null, removed: 0 }; // non-JSON / unparseable → no rewrite
|
|
1779
|
+
return { body: null, removed: 0, converted: 0 }; // non-JSON / unparseable → no rewrite
|
|
1587
1780
|
}
|
|
1588
1781
|
}
|
|
1589
1782
|
|