maxpool 1.24.100 → 1.24.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/server.js +42 -2
- package/src/thread-gate.js +68 -7
package/package.json
CHANGED
package/src/server.js
CHANGED
|
@@ -4,7 +4,7 @@ import { normalizeClientHeaders, rewriteMetadataUserIdRaw } from './client-ident
|
|
|
4
4
|
import { appendEventLog } from './event-log.js';
|
|
5
5
|
import { atomicConfigUpdate } from './config.js';
|
|
6
6
|
import { HOP_HEADER, nextHopCount, safeText } from './pool-link.js';
|
|
7
|
-
import { ThreadOwners, readThreadIntent, threadRefusalBody, isThreadlessAccount } from './thread-gate.js';
|
|
7
|
+
import { ThreadOwners, readThreadIntent, threadRefusalBody, isThreadlessAccount, isThreadedIntent } from './thread-gate.js';
|
|
8
8
|
import { ProviderThinkingScrubber, scrubProviderResponseBody, scrubEnabledFor, foreignToolBlockText, describeScrub } from './provider-thinking-scrub.js';
|
|
9
9
|
import { writeFile, mkdir } from 'node:fs/promises';
|
|
10
10
|
import { readFileSync } from 'node:fs';
|
|
@@ -653,6 +653,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
653
653
|
const queueConfig = { ...DEFAULT_QUEUE, ...(config.queue || {}) };
|
|
654
654
|
const canRetryBufferedBody = body.length <= retryConfig.maxRetryBufferBytes;
|
|
655
655
|
requestInfo = describeRequest(req, body);
|
|
656
|
+
// THREAD REPLY SINK: one object per client request, shared BY REFERENCE through
|
|
657
|
+
// every `{ ...requestInfo }` copy the repair/retry paths make, so the reply id the
|
|
658
|
+
// serving attempt captures reaches this handler's finally (a field written on a
|
|
659
|
+
// copy never would — every turn of an effort-latched session is served on one).
|
|
660
|
+
requestInfo._threadSink = { attemptAccount: null, replyId: null, account: null };
|
|
656
661
|
// Request clock: the direct client-gone line ("Client left after Xs" — request
|
|
657
662
|
// age) and the retry-same patience rule read it. (The link path reports time on
|
|
658
663
|
// THIS account, from lease.startedAt.) Was never set, so every direct-path
|
|
@@ -730,6 +735,15 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
730
735
|
}
|
|
731
736
|
} finally {
|
|
732
737
|
clearInterval(reaperTimer);
|
|
738
|
+
// THREAD OWNERSHIP, recorded at terminal release: the reply id captured during
|
|
739
|
+
// the response (message_start for streaming, body id for non-streaming) belongs
|
|
740
|
+
// to the account that SERVED the turn — pin it so the next continue is judged
|
|
741
|
+
// by thread identity. Only a served (2xx-5xx upstream answer) turn reaches here
|
|
742
|
+
// with _threadReplyId set; refusals return before the fetch.
|
|
743
|
+
const sink = requestInfo?._threadSink;
|
|
744
|
+
if (sink?.replyId && sink.account) {
|
|
745
|
+
threadOwners.noteThreadId(sink.replyId, sink.account);
|
|
746
|
+
}
|
|
733
747
|
accountManager.noteClientOutcome?.(requestInfo, ctx.status);
|
|
734
748
|
hooks.onRequestEnd?.(reqId, {
|
|
735
749
|
method: req.method, path: req.url,
|
|
@@ -1540,7 +1554,15 @@ async function forwardRequest(
|
|
|
1540
1554
|
}
|
|
1541
1555
|
// This account is about to serve the turn, so it holds the thread from here on.
|
|
1542
1556
|
// Recorded optimistically: if the turn fails, the next one is refused anyway.
|
|
1557
|
+
// The reply's own message id is captured during the response (message_start / body
|
|
1558
|
+
// head — see parseSSEEvent and the non-streaming read) and recorded at release, so
|
|
1559
|
+
// the next continue is judged by THREAD identity rather than by terminal.
|
|
1543
1560
|
threadOwners.noteServed(requestInfo.sessionKey, account.name, threadIntent);
|
|
1561
|
+
// Arm reply capture for THIS attempt only when the turn is threaded: a plain turn's
|
|
1562
|
+
// id names no thread, and recording it would push live thread holders out of the map.
|
|
1563
|
+
if (requestInfo._threadSink) {
|
|
1564
|
+
requestInfo._threadSink.attemptAccount = isThreadedIntent(threadIntent) ? account.name : null;
|
|
1565
|
+
}
|
|
1544
1566
|
|
|
1545
1567
|
// Build log sections
|
|
1546
1568
|
const logSections = [];
|
|
@@ -3269,7 +3291,7 @@ function isContextLengthError(errorBody) {
|
|
|
3269
3291
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
3270
3292
|
}
|
|
3271
3293
|
|
|
3272
|
-
export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3294
|
+
export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, parseSSEEvent, extractUsageFromBody, threadOwners: () => threadOwners, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3273
3295
|
|
|
3274
3296
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
3275
3297
|
if (!upstreamRes.body) return '';
|
|
@@ -5296,12 +5318,28 @@ function tapLinkUsage(chunk, requestInfo) {
|
|
|
5296
5318
|
}
|
|
5297
5319
|
}
|
|
5298
5320
|
|
|
5321
|
+
/** Record a threaded reply's id against the account serving THIS attempt (armed at the
|
|
5322
|
+
* gate). Id and account are stored together, so a retry that moves accounts after an
|
|
5323
|
+
* earlier attempt captured an id can never pin that id to the wrong account. */
|
|
5324
|
+
function captureThreadReply(requestInfo, id) {
|
|
5325
|
+
const sink = requestInfo?._threadSink;
|
|
5326
|
+
if (!sink?.attemptAccount) return;
|
|
5327
|
+
sink.replyId = id;
|
|
5328
|
+
sink.account = sink.attemptAccount;
|
|
5329
|
+
}
|
|
5330
|
+
|
|
5299
5331
|
function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
|
|
5300
5332
|
const dataLine = event.split('\n').find(l => l.startsWith('data: '));
|
|
5301
5333
|
if (!dataLine) return;
|
|
5302
5334
|
|
|
5303
5335
|
try {
|
|
5304
5336
|
const data = JSON.parse(dataLine.slice(6));
|
|
5337
|
+
// THREAD OWNERSHIP: message_start carries the reply's own id — the id the client
|
|
5338
|
+
// will continue from (previous_message_id). Capture it on the SERVING account so
|
|
5339
|
+
// the gate can judge the next continue by thread identity instead of by terminal.
|
|
5340
|
+
if (data?.type === 'message_start' && typeof data.message?.id === 'string') {
|
|
5341
|
+
captureThreadReply(requestInfo, data.message.id);
|
|
5342
|
+
}
|
|
5305
5343
|
// CAPACITY LEDGER (2026-08-22): stream-level usage feeds the per-cycle ledger.
|
|
5306
5344
|
// M3: Anthropic interim message_delta usage is CUMULATIVE — add-semantics inflates
|
|
5307
5345
|
// output on long generations. Track a per-stream RUNNING MAX, accrue once at end.
|
|
@@ -5361,6 +5399,8 @@ function classifyHeldStreamPrefix(prefix) {
|
|
|
5361
5399
|
function extractUsageFromBody(buffer, accountIndex, accountManager, requestInfo = {}) {
|
|
5362
5400
|
try {
|
|
5363
5401
|
const json = JSON.parse(buffer.toString());
|
|
5402
|
+
// THREAD OWNERSHIP: a non-streaming reply's own id is the next continue's anchor.
|
|
5403
|
+
if (typeof json.id === 'string') captureThreadReply(requestInfo, json.id);
|
|
5364
5404
|
if (json.usage) {
|
|
5365
5405
|
// Per-account / per-caller totals book once per request at the terminal release
|
|
5366
5406
|
// (finding 3, 2026-11-17) — off the running-max capacity figures booked here.
|
package/src/thread-gate.js
CHANGED
|
@@ -27,7 +27,10 @@ export function isThreadlessAccount(account) {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
/** What kind of thread intent a request body carries. Cheap: only the head of the body
|
|
30
|
-
* is JSON-parsed, and a non-JSON body is simply 'none'.
|
|
30
|
+
* is JSON-parsed, and a non-JSON body is simply 'none'. A `continue` also carries the
|
|
31
|
+
* id of the reply it follows — the ONE identifier that names the thread itself, so the
|
|
32
|
+
* gate returns it (session keys are per terminal and sibling agents inside one terminal
|
|
33
|
+
* hold threads on different accounts; 2026-11-17: 60 of 109 thread 404s). */
|
|
31
34
|
export function readThreadIntent(body) {
|
|
32
35
|
try {
|
|
33
36
|
const j = JSON.parse(body.toString('utf8'));
|
|
@@ -36,7 +39,10 @@ export function readThreadIntent(body) {
|
|
|
36
39
|
// `previous_message_id` can also ride in `diagnostics`; that alone is not a thread.
|
|
37
40
|
return { kind: 'none' };
|
|
38
41
|
}
|
|
39
|
-
if (t.type === 'continue')
|
|
42
|
+
if (t.type === 'continue') {
|
|
43
|
+
const previousMessageId = typeof t.previous_message_id === 'string' ? t.previous_message_id : null;
|
|
44
|
+
return { kind: 'continue', previousMessageId };
|
|
45
|
+
}
|
|
40
46
|
if (t.type === 'create') return { kind: 'create' };
|
|
41
47
|
return { kind: 'none' };
|
|
42
48
|
} catch {
|
|
@@ -44,6 +50,11 @@ export function readThreadIntent(body) {
|
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
52
|
|
|
53
|
+
/** Whether a turn carries thread state at all — only these replies name a thread. */
|
|
54
|
+
export function isThreadedIntent(intent) {
|
|
55
|
+
return intent?.kind === 'create' || intent?.kind === 'continue';
|
|
56
|
+
}
|
|
57
|
+
|
|
47
58
|
/** The exact body the client's classifier reads. `details.error_code` is the field it
|
|
48
59
|
* keys on; the message is free text and is never shown to a person. */
|
|
49
60
|
export function threadRefusalBody(accountName) {
|
|
@@ -60,6 +71,11 @@ export function threadRefusalBody(accountName) {
|
|
|
60
71
|
// Sessions whose last threaded turn we served, and how many times we have refused them.
|
|
61
72
|
// Two short strings per entry; bounded and LRU-evicted.
|
|
62
73
|
const MAX_SESSIONS = 500;
|
|
74
|
+
// Reply ids are one per threaded TURN (every superseded id is kept: a client retry or a
|
|
75
|
+
// rewind may still continue it), so the thread map needs a turn-volume bound, not a
|
|
76
|
+
// session bound: 500 was exhausted in ~10-20 min on the shared cloud pool and a busy
|
|
77
|
+
// thread's holder fell back to the per-terminal guess. ~100 bytes an entry → ~2 MB.
|
|
78
|
+
const MAX_THREADS = 20000;
|
|
63
79
|
// A session that keeps sending threaded turns after being refused is one whose client
|
|
64
80
|
// did NOT take the downgrade (a different agent id, a model switch, an older build).
|
|
65
81
|
// Refusing forever would double its request volume, so stop and forward instead.
|
|
@@ -69,9 +85,16 @@ const MAX_CONSECUTIVE_REFUSALS = 2;
|
|
|
69
85
|
const NO_SESSION_PREFIX = '\u0000nosession:';
|
|
70
86
|
|
|
71
87
|
export class ThreadOwners {
|
|
72
|
-
constructor({ maxSessions = MAX_SESSIONS, maxRefusals = MAX_CONSECUTIVE_REFUSALS } = {}) {
|
|
88
|
+
constructor({ maxSessions = MAX_SESSIONS, maxThreads = MAX_THREADS, maxRefusals = MAX_CONSECUTIVE_REFUSALS } = {}) {
|
|
73
89
|
this.map = new Map(); // sessionKey -> { owner, refusals }
|
|
90
|
+
// reply message id -> owning account name. The THREAD is keyed by the reply id the
|
|
91
|
+
// client sends back as previous_message_id — a session key is per TERMINAL, and
|
|
92
|
+
// sibling agents inside one terminal hold threads on different accounts, so the
|
|
93
|
+
// per-terminal entry both mis-serves siblings and never expires per thread.
|
|
94
|
+
// Bounded LRU, same lifetime class as the per-session map above it.
|
|
95
|
+
this.threads = new Map();
|
|
74
96
|
this.maxSessions = maxSessions;
|
|
97
|
+
this.maxThreads = maxThreads;
|
|
75
98
|
this.maxRefusals = maxRefusals;
|
|
76
99
|
}
|
|
77
100
|
|
|
@@ -87,6 +110,23 @@ export class ThreadOwners {
|
|
|
87
110
|
while (this.map.size > this.maxSessions) this.map.delete(this.map.keys().next().value);
|
|
88
111
|
}
|
|
89
112
|
|
|
113
|
+
/** Record which account produced a threaded reply (the id the client will continue
|
|
114
|
+
* from). Streaming: the message_start event's message.id; non-streaming: the body's
|
|
115
|
+
* leading id. Called on every served turn that carried thread intent — a `create`
|
|
116
|
+
* starts a thread, a `continue` extends it onto whoever served the follow-up. */
|
|
117
|
+
noteThreadId(threadId, accountName) {
|
|
118
|
+
if (!threadId || !accountName) return;
|
|
119
|
+
this.threads.delete(threadId);
|
|
120
|
+
this.threads.set(threadId, accountName);
|
|
121
|
+
while (this.threads.size > this.maxThreads) this.threads.delete(this.threads.keys().next().value);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The account that holds `threadId`, if we saw its reply. Null = unknown. */
|
|
125
|
+
threadOwner(threadId) {
|
|
126
|
+
const v = threadId ? this.threads.get(threadId) : undefined;
|
|
127
|
+
return v ?? null;
|
|
128
|
+
}
|
|
129
|
+
|
|
90
130
|
/** Decide AFTER routing has chosen. Returns true only for a `continue` turn that the
|
|
91
131
|
* chosen account cannot serve, and only while refusals are still under the bound.
|
|
92
132
|
*
|
|
@@ -106,14 +146,32 @@ export class ThreadOwners {
|
|
|
106
146
|
shouldRefuse(sessionKey, accountName, intent, isProvider = false) {
|
|
107
147
|
if (!accountName) return false;
|
|
108
148
|
if (intent?.kind !== 'continue') return false; // `create` carries the full transcript
|
|
109
|
-
if (isProvider) return true; // can never serve an Anthropic thread
|
|
149
|
+
if (isProvider) return true; // can never serve an Anthropic thread — even as the recorded holder
|
|
150
|
+
// THREAD identity first: the previous_message_id names the exact thread being
|
|
151
|
+
// continued. When we know its holder, the answer is exact (the holder passes,
|
|
152
|
+
// everyone else is refused) and never falls through to the terminal-keyed guess —
|
|
153
|
+
// which is what mis-served sibling agents inside one terminal (2026-11-17).
|
|
154
|
+
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|
|
155
|
+
if (intent?.previousMessageId) {
|
|
156
|
+
const holder = this.threads.get(intent.previousMessageId);
|
|
157
|
+
if (holder != null) {
|
|
158
|
+
this.noteThreadId(intent.previousMessageId, holder); // LRU bump: a live thread stays
|
|
159
|
+
if (holder === accountName) return false;
|
|
160
|
+
// A known non-holder is refused under the SAME per-session bound as an unknown
|
|
161
|
+
// owner. Past it we forward: the real 404 makes the client replay with
|
|
162
|
+
// thread:create and KEEP threading, where our 400 would downgrade that agent to
|
|
163
|
+
// stateless for the whole session (Claude Code 2.1.325) — unbounded refusals
|
|
164
|
+
// would downgrade every sibling after a terminal's account moves.
|
|
165
|
+
const entry = this._touch(key);
|
|
166
|
+
return !(entry && entry.refusals >= this.maxRefusals);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
110
169
|
// A request with NO session header is invisible to ownership tracking, and measured
|
|
111
170
|
// 2026-09-11 those are the majority of traffic — 14 of 20 consecutive /v1/messages
|
|
112
171
|
// lines carried no `[sess …]`. Skipping them left threaded turns reaching GLM and
|
|
113
172
|
// failing exactly as before the gate existed. We cannot know the owner, so treat it
|
|
114
173
|
// as an unknown session (refuse, the safe direction) and bucket the refusal COUNT
|
|
115
174
|
// per account so the storm bound still applies.
|
|
116
|
-
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|
|
117
175
|
const entry = this._touch(key);
|
|
118
176
|
if (entry && entry.owner === accountName) return false; // the account that holds it
|
|
119
177
|
if (entry && entry.refusals >= this.maxRefusals) return false; // bounded fail-open
|
|
@@ -129,10 +187,13 @@ export class ThreadOwners {
|
|
|
129
187
|
}
|
|
130
188
|
|
|
131
189
|
/** Record that `accountName` served a threaded turn for this session — it now holds
|
|
132
|
-
* the thread. Any refusal streak ends here.
|
|
133
|
-
|
|
190
|
+
* the thread. Any refusal streak ends here. The reply's own message id (when the
|
|
191
|
+
* caller captured it from the response) is recorded as the newest link of the
|
|
192
|
+
* thread, so the NEXT continue is judged by thread identity, not by terminal. */
|
|
193
|
+
noteServed(sessionKey, accountName, intent, replyMessageId = null) {
|
|
134
194
|
if (!accountName) return;
|
|
135
195
|
if (intent?.kind !== 'create' && intent?.kind !== 'continue') return;
|
|
196
|
+
if (replyMessageId) this.noteThreadId(replyMessageId, accountName);
|
|
136
197
|
// Without a session header there is no conversation to attribute ownership to; only
|
|
137
198
|
// clear the per-account refusal streak so a served turn re-arms the bound.
|
|
138
199
|
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|