maxpool 1.24.94 → 1.24.96

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/server.js CHANGED
@@ -9,7 +9,8 @@ import { ProviderThinkingScrubber, scrubProviderResponseBody, scrubEnabledFor, f
9
9
  import { writeFile, mkdir } from 'node:fs/promises';
10
10
  import { readFileSync } from 'node:fs';
11
11
  import { join } from 'node:path';
12
- import { modelFamily } from './oauth.js';
12
+ import { modelFamily, organizationUnknownMatch, sameAccountIdentity, sameUser } from './oauth.js';
13
+ import { resolveStickyBusyBudget } from './account-manager.js';
13
14
 
14
15
 
15
16
  const HOP_BY_HOP_HEADERS = new Set([
@@ -80,6 +81,18 @@ const posMsFromEnv = (name, dflt) => {
80
81
  const n = Number(process.env[name]);
81
82
  return Number.isFinite(n) && n > 0 ? n : dflt;
82
83
  };
84
+ // STICKY BUSY WAIT (2026-11-15) — a warm Claude conversation whose cloud account says
85
+ // "busy" waits for it (bounded) instead of moving and re-writing its prompt cache.
86
+ const PIN_BUSY_MAX_RETRY_AFTER_S = 10; // a header-less 429/503 asking ≤ this is busy-class
87
+ const STICKY_BUSY_MAX_ATTEMPTS = posMsFromEnv('MAXPOOL_STICKY_BUSY_MAX_ATTEMPTS', 6);
88
+ const STICKY_BUSY_MIN_BYTES = posMsFromEnv('MAXPOOL_PIN_BUSY_MIN_BYTES', BIG_PROMPT_BYTES);
89
+ const STICKY_BUSY_WAIT_CAP_MS = 15_000; // per-wait cap for the growing waits (nextStickyBusyWaitMs)
90
+ /** The total wait budget, read LIVE per request (TUI-editable `stickyBusyBudgetMs`);
91
+ * env MAXPOOL_STICKY_BUSY_BUDGET_MS overrides it only when set. Same resolver as the
92
+ * TUI setting, so the screen always shows the value in force. */
93
+ function stickyBusyBudgetMs(accountManager) {
94
+ return resolveStickyBusyBudget(accountManager?.scheduler).ms;
95
+ }
83
96
  /** Live knob values (env read per call so a two-pool test can inject a small cap). */
84
97
  function deadlineKnobs() {
85
98
  return {
@@ -460,6 +473,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
460
473
  try { body = JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { body = null; }
461
474
  const acct = body?.account;
462
475
  if (acct && typeof acct.name === 'string') acct.name = safeText(acct.name);
476
+ if (acct && typeof acct.organizationUuid !== 'string') delete acct.organizationUuid;
463
477
  if (!acct || acct.type !== 'oauth' || typeof acct.accountUuid !== 'string' || !acct.accountUuid ||
464
478
  typeof acct.refreshToken !== 'string' || !acct.refreshToken || typeof acct.accessToken !== 'string' || !acct.accessToken) {
465
479
  res.writeHead(400, { 'Content-Type': 'application/json' });
@@ -467,7 +481,18 @@ export function createProxyServer(accountManager, config, hooks = {}) {
467
481
  return;
468
482
  }
469
483
  const who = caller.name || 'legacy-key';
470
- const amAcct = accountManager.accounts.find(a => a.accountUuid === acct.accountUuid);
484
+ // Exact match (user + org), else the ONE same-user row with no org yet: a
485
+ // login to a pool saved before organizations were recorded re-authenticates
486
+ // that row and stamps the organization onto it, exactly like the local path.
487
+ // Healing, BOTH directions: (a) an org-bearing login into a pool whose row
488
+ // predates orgs re-authenticates that row and stamps the org onto it; (b) an
489
+ // org-less login from an OLDER client re-authenticates the user's single row
490
+ // (any org) without touching its org. Two rows of the user → ambiguous: add.
491
+ const sameUserRows = accountManager.accounts.filter(a => sameUser(a, acct));
492
+ const amAcct = accountManager.accounts.find(a => sameAccountIdentity(a, acct))
493
+ || (acct.organizationUuid ? organizationUnknownMatch(accountManager.accounts, acct)
494
+ : (sameUserRows.length === 1 ? sameUserRows[0] : null));
495
+ if (amAcct && acct.organizationUuid) amAcct.organizationUuid = acct.organizationUuid;
471
496
  const tokenFields = {
472
497
  accessToken: acct.accessToken,
473
498
  refreshToken: acct.refreshToken,
@@ -485,26 +510,36 @@ export function createProxyServer(accountManager, config, hooks = {}) {
485
510
  if (amAcct.status === 'error') amAcct.status = 'active';
486
511
  appendEventLog(`[Maxpool] Remote re-auth: "${amAcct.name}" (uuid ${acct.accountUuid.slice(0, 8)}) by ${who}`);
487
512
  } else {
488
- const entry = { name: acct.name || acct.accountUuid.slice(0, 8), type: 'oauth', accountUuid: acct.accountUuid, enabled: true, ...tokenFields };
513
+ const entry = newInjectedEntry(acct, tokenFields, accountManager.accounts);
514
+ acct.name = entry.name; // the disk row and config mirror below take the same unique name
489
515
  accountManager.addAccount(entry);
490
516
  appendEventLog(`[Maxpool] Remote add: "${entry.name}" (uuid ${acct.accountUuid.slice(0, 8)}) by ${who}`);
491
517
  }
492
518
  try {
493
519
  await atomicConfigUpdate(diskConfig => {
494
- const onDisk = diskConfig.accounts.find(a => a.accountUuid === acct.accountUuid);
520
+ let onDisk = diskConfig.accounts.find(a => sameAccountIdentity(a, acct));
521
+ if (!onDisk && acct.organizationUuid) onDisk = organizationUnknownMatch(diskConfig.accounts, acct);
522
+ if (!onDisk && !acct.organizationUuid) onDisk = diskConfig.accounts.filter(a => sameUser(a, acct)).length === 1
523
+ ? diskConfig.accounts.find(a => sameUser(a, acct)) : null;
495
524
  if (onDisk) {
496
525
  Object.assign(onDisk, tokenFields);
526
+ if (acct.organizationUuid) onDisk.organizationUuid = acct.organizationUuid;
497
527
  } else {
498
- diskConfig.accounts.push({ name: acct.name || acct.accountUuid.slice(0, 8), type: 'oauth', accountUuid: acct.accountUuid, enabled: true, ...tokenFields });
528
+ diskConfig.accounts.push(newInjectedEntry(acct, tokenFields, diskConfig.accounts));
499
529
  }
500
530
  });
501
531
  // Keep the in-memory config mirror in step so a later TUI saveConfig
502
532
  // can't clobber the fresh tokens with stale ones.
503
533
  // A NEW row must enter the mirror too: the disk sync adds any disk row whose
504
534
  // uuid the mirror lacks, so a missing entry became a second live row.
505
- const mem = (config.accounts || []).find(a => a.accountUuid === acct.accountUuid);
506
- if (mem) Object.assign(mem, tokenFields);
507
- else if (Array.isArray(config.accounts)) config.accounts.push({ name: acct.name || acct.accountUuid.slice(0, 8), type: 'oauth', accountUuid: acct.accountUuid, enabled: true, ...tokenFields });
535
+ let mem = (config.accounts || []).find(a => sameAccountIdentity(a, acct));
536
+ if (!mem && acct.organizationUuid) mem = organizationUnknownMatch(config.accounts || [], acct);
537
+ if (!mem && !acct.organizationUuid) mem = (config.accounts || []).filter(a => sameUser(a, acct)).length === 1
538
+ ? (config.accounts || []).find(a => sameUser(a, acct)) : null;
539
+ if (mem) {
540
+ Object.assign(mem, tokenFields);
541
+ if (acct.organizationUuid) mem.organizationUuid = acct.organizationUuid;
542
+ } else if (Array.isArray(config.accounts)) config.accounts.push(newInjectedEntry(acct, tokenFields, config.accounts));
508
543
  } catch (err) {
509
544
  // A single-use credential write is only "ok" when it is DURABLE: the browser
510
545
  // login may have invalidated the previous refresh token, so memory-only means
@@ -695,6 +730,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
695
730
  }
696
731
  } finally {
697
732
  clearInterval(reaperTimer);
733
+ accountManager.noteClientOutcome?.(requestInfo, ctx.status);
698
734
  hooks.onRequestEnd?.(reqId, {
699
735
  method: req.method, path: req.url,
700
736
  account: ctx.account, status: ctx.status,
@@ -712,6 +748,10 @@ export function createProxyServer(accountManager, config, hooks = {}) {
712
748
  // F9: only the message-creating call (POST /v1/messages) has a streaming shape — count_tokens and GETs
713
749
  // must not read as "non-streaming" and swamp the share.
714
750
  stream: requestInfo?.method === 'POST' && /^\/v1\/messages(?:\?|$)/.test(String(req.url || '')) ? Boolean(requestInfo.stream) : null,
751
+ // The served attempt's cache figures, for the `cache r… w… in…` line tag.
752
+ cacheUsage: requestInfo && !requestInfo.isCountTokens && (requestInfo._capacityInput || requestInfo._capacityCacheRead || requestInfo._capacityCacheCreation)
753
+ ? { input_tokens: requestInfo._capacityInput || 0, cache_read_input_tokens: requestInfo._capacityCacheRead || 0, cache_creation_input_tokens: requestInfo._capacityCacheCreation || 0 }
754
+ : null,
715
755
  });
716
756
  }
717
757
  } catch (err) {
@@ -755,6 +795,23 @@ export function createProxyServer(accountManager, config, hooks = {}) {
755
795
  /**
756
796
  * Relay a request to upstream with no header rewriting — pure passthrough.
757
797
  */
798
+ /** The account a remote login adds: one Claude user in one organization, plus its tokens. */
799
+ function newInjectedEntry(acct, tokenFields, existing = []) {
800
+ // Two organizations of one user must never share a name: name-keyed lookups
801
+ // (disk sync, config writer) would write one org's tokens over the other's.
802
+ let name = acct.name || acct.accountUuid.slice(0, 8);
803
+ if (existing.some(a => !sameAccountIdentity(a, acct) && a.name === name)) {
804
+ const type = String(acct.orgType || '').replace(/^claude_/, '');
805
+ const suffix = type || '2';
806
+ let n = 2;
807
+ while (existing.some(a => a.name === `${name} (${suffix}${n > 2 ? ' ' + n : ''})`)) n++;
808
+ name = `${name} (${suffix}${n > 2 ? ' ' + n : ''})`;
809
+ }
810
+ const entry = { name, type: 'oauth', accountUuid: acct.accountUuid, enabled: true, ...tokenFields };
811
+ if (acct.organizationUuid) entry.organizationUuid = acct.organizationUuid;
812
+ return entry;
813
+ }
814
+
758
815
  async function relayRaw(req, res, upstream) {
759
816
  const bodyChunks = [];
760
817
  for await (const chunk of req) bodyChunks.push(chunk);
@@ -880,8 +937,23 @@ async function forwardRequest(
880
937
  // actually happened (acquireAccount consumes the one-shot hint either way).
881
938
  const pendingSame = requestInfo.pendingRetrySame || null;
882
939
  requestInfo.pendingRetrySame = null;
940
+ // A cloud row this request already gave up waiting on stays out while it is still
941
+ // busy — including after a queue resume, whose fresh exclusion set would re-pick it.
942
+ // (Gated on the same pinBusyUntil hasAvailableRoute reads, so the two never disagree.)
943
+ if (requestInfo._busyExcluded) {
944
+ const now = Date.now();
945
+ for (const a of accountManager.accounts) {
946
+ if (requestInfo._busyExcluded.has(a.name) && a.pinBusyUntil > now) excludedIndexes.add(a.index);
947
+ }
948
+ }
883
949
  const lease = accountManager.acquireAccount(requestInfo, excludedIndexes);
884
950
  const account = lease?.account;
951
+ requestInfo._hop = retryCount;
952
+ // The sticky-wait retry pin is good for exactly this one acquire.
953
+ if (requestInfo._stickyPin) {
954
+ requestInfo.pinnedAccountName = requestInfo._stickyPinPrev ?? null;
955
+ requestInfo._stickyPin = false;
956
+ }
885
957
  if (pendingSame && account) {
886
958
  if (account.name === pendingSame.account) logFailedAttempt(pendingSame.account, pendingSame.kind, pendingSame.detail, 'retry-same');
887
959
  else logFailedAttempt(pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, served by "${account.name}"`, 'moved');
@@ -899,7 +971,16 @@ async function forwardRequest(
899
971
  ctx.status = 503;
900
972
  const busy = Boolean(requestInfo.pinRefusedBusy);
901
973
  ctx.account = `(pinned ${requestInfo.pinAccount} ${busy ? 'busy' : 'unavailable'})`;
902
- try { res.setHeader('retry-after', busy ? '2' : '30'); } catch { /* headers sent */ }
974
+ try {
975
+ res.setHeader('retry-after', busy ? '2' : '30');
976
+ // Machine-readable cause + depth for the linked pool (additive; old pools ignore them).
977
+ res.setHeader('x-maxpool-pin-refusal', requestInfo.pinRefusal || 'unavailable');
978
+ const pinned = accountManager.accounts.find(a => !a.remote && a.name === requestInfo.pinAccount);
979
+ if (pinned) {
980
+ res.setHeader('x-maxpool-pin-inflight', String(pinned.inFlight || 0));
981
+ res.setHeader('x-maxpool-pin-limit', String(accountManager._pinBusyLimit(pinned)));
982
+ }
983
+ } catch { /* headers sent */ }
903
984
  sendErrorResponse(res, requestInfo, 503, { type: 'error', error: {
904
985
  type: busy ? 'pinned_account_busy' : 'pinned_account_unavailable',
905
986
  message: `Account "${requestInfo.pinAccount}" is ${busy ? 'busy' : 'not available'} right now` } });
@@ -1175,14 +1256,66 @@ async function forwardRequest(
1175
1256
  // streamed: cool THIS mirror row and fail over through the normal path to the
1176
1257
  // next-best account, local or cloud — exactly like a local account's 429.
1177
1258
  if (account.remote && (upRes.status === 429 || upRes.status === 503) && !clientHasRealBytes(res, requestInfo)) {
1178
- try { await upRes.body?.cancel(); } catch { /* drained */ }
1179
- const ra = Number(upRes.headers.get('retry-after'));
1180
- // P3: "busy" is a 2s back-off on this row only; "unavailable"/429 cools longer.
1181
- account.cooldownUntil = Date.now() + (Number.isFinite(ra) && ra > 0 ? Math.min(ra, 300) * 1000 : 30_000);
1182
- finishLink({ status: upRes.status, error: 'remote_pinned_unavailable' });
1259
+ const refusal = await readPinRefusal(upRes);
1260
+ accountManager.noteRemoteDepth?.(account.name, refusal.inflight, refusal.limit);
1261
+ if (refusal.cause === 'busy') {
1262
+ // BUSY is seconds, not an account fault: a bounce (no cooldown, no failure
1263
+ // count), and the row repels new placements for a moment.
1264
+ const raMs = (refusal.retryAfterS ?? 1) * 1000;
1265
+ finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_busy', refusal: 'busy' });
1266
+ accountManager.markPinBusy(account.index, raMs);
1267
+ const waitMs = nextStickyBusyWaitMs(requestInfo._busyWaits || 0, raMs, requestInfo._busyWaitMs || 0, stickyBusyBudgetMs(accountManager));
1268
+ if (waitMs > 0 && stickyBusyWaitEligible(account, requestInfo, res, queueConfig, accountManager, waitMs)) {
1269
+ requestInfo._busyWaits = (requestInfo._busyWaits || 0) + 1;
1270
+ requestInfo._busyWaitMs = (requestInfo._busyWaitMs || 0) + waitMs;
1271
+ detachLinkClose();
1272
+ // KEEP-ALIVE through the wait (a wait can run 15 s+, the last one longer): re-arm the stream-forward
1273
+ // grace around EVERY sleep. Its deadline is anchored once per request (default 10 s
1274
+ // after arrival), so it fires mid-sleep and commits the SSE stream + the heartbeat
1275
+ // interval (a ping every queue.heartbeatMs, 10 s default) — the client's idle
1276
+ // watchdog is fed for the whole wait and every later hop, until the first real
1277
+ // chunk stops it. Only the grace TIMER is cleared after the sleep; a committed
1278
+ // heartbeat keeps running. No lease and no queue ticket is held while waiting.
1279
+ const grace = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
1280
+ const stayed = await sleepUnlessClientGone(waitMs, req, res);
1281
+ if (grace) clearTimeout(grace);
1282
+ if (!stayed) {
1283
+ clearQueueHeartbeat(requestInfo);
1284
+ // The CLIENT cancelled mid-wait: not a pool error. ctx.status still holds
1285
+ // the refusal's 503 — 499 (client closed) keeps the request-end log honest.
1286
+ requestInfo._clientCancelled = true;
1287
+ ctx.status = 499;
1288
+ logFailedAttempt(account.name, `remote-${upRes.status}`, 'linked pool busy; client left during the wait', 'released-cancel');
1289
+ return;
1290
+ }
1291
+ if (resendAllowed(requestInfo)) {
1292
+ // Retry the SAME row: a hard one-shot pin (selection cannot outscore it).
1293
+ requestInfo._stickyPinPrev = requestInfo.pinnedAccountName ?? null;
1294
+ requestInfo.pinnedAccountName = account.name;
1295
+ requestInfo._stickyPin = true;
1296
+ requestInfo.pendingRetrySame = { account: account.name, kind: `remote-${upRes.status}`,
1297
+ detail: `linked pool busy; waited ${(waitMs / 1000).toFixed(1)}s for the account holding this conversation` };
1298
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
1299
+ }
1300
+ }
1301
+ (requestInfo._busyExcluded ||= new Set()).add(account.name);
1302
+ requestInfo._moveCause = 'busy';
1303
+ } else if (refusal.cause === 'throttled') {
1304
+ // The linked pool's egress is held: every row behind it is out — one move.
1305
+ finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_throttled', refusal: 'throttled' });
1306
+ const cooled = accountManager.markLinkMirrorsCooldown(account, Math.min((refusal.retryAfterS || 5) * 1000, 30_000));
1307
+ for (const a of cooled) excludedIndexes.add(a.index);
1308
+ requestInfo._moveCause = 'wall';
1309
+ } else {
1310
+ const ra = refusal.retryAfterS;
1311
+ // "unavailable"/429 cools this row; booked as a bounce, not a failure.
1312
+ account.cooldownUntil = Date.now() + (ra != null ? Math.min(ra, 300) * 1000 : 30_000);
1313
+ finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_unavailable', refusal: 'unavailable' });
1314
+ requestInfo._moveCause = 'wall';
1315
+ }
1183
1316
  excludedIndexes.add(account.index);
1184
1317
  if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
1185
- logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte`, 'moved');
1318
+ logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte (${refusal.cause})`, 'moved');
1186
1319
  detachLinkClose();
1187
1320
  return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
1188
1321
  }
@@ -1278,9 +1411,16 @@ async function forwardRequest(
1278
1411
  // a conditional clear let pings land inside the real stream when the grace
1279
1412
  // timer had already fired).
1280
1413
  if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
1414
+ if (upRes.status < 300) {
1415
+ // The conversation now lives on this row (first REAL byte), and its usage
1416
+ // is read off the relayed bytes — figures only, never a booking.
1417
+ accountManager.noteConversationServed?.(requestInfo, account);
1418
+ tapLinkUsage(value, requestInfo);
1419
+ }
1281
1420
  res.write(Buffer.from(value));
1282
1421
  }
1283
1422
  } finally { try { await reader.cancel(); } catch { /* already done */ } }
1423
+ if (upRes.status < 300) tapLinkUsage(null, requestInfo); // flush the non-stream tail
1284
1424
  }
1285
1425
  res.end(); // an empty 2xx body: the heartbeat reaps itself on res.writableEnded
1286
1426
  finishLink({ success: ctx.status < 500, status: ctx.status });
@@ -1299,6 +1439,7 @@ async function forwardRequest(
1299
1439
  if (linkClientGone && !clientHasRealBytes(res, requestInfo)) {
1300
1440
  finishLink({ success: false, neutral: true, error: 'client_cancelled' });
1301
1441
  clearQueueHeartbeat(requestInfo);
1442
+ requestInfo._clientCancelled = true;
1302
1443
  console.log(`[Maxpool] Client left after ${heldS}s on "${account.name}" (link, still waiting) — released, not retried`);
1303
1444
  logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
1304
1445
  return;
@@ -1331,6 +1472,7 @@ async function forwardRequest(
1331
1472
  finishLink({ status: 502, error: linkKind, scope: 'request', linkFault: true });
1332
1473
  }
1333
1474
  excludedIndexes.add(account.index); // P4b: never re-pick a row that just failed
1475
+ if (!linkClientGone) requestInfo._moveCause = 'link';
1334
1476
  const streamTag = requestInfo.stream ? 'streaming' : 'non-streaming';
1335
1477
  // A string code (UND_ERR_SOCKET, ECONNRESET) names the fault; DOMException
1336
1478
  // codes are numbers (AbortError = 20), so fall back to the cause, then the name.
@@ -1478,6 +1620,8 @@ async function forwardRequest(
1478
1620
  // Log-only: no setting reads it.
1479
1621
  requestInfo._upstreamForwardStartAt = upstreamForwardStartAt;
1480
1622
  requestInfo._firstUpstreamByteAt = Date.now();
1623
+ // A 2xx answer: the conversation now lives on this account (never on a refusal).
1624
+ if (upstreamRes.status < 300) accountManager.noteConversationServed?.(requestInfo, account);
1481
1625
  // Response arrived — the pre-response leak window is over. Stop guarding for
1482
1626
  // client-disconnect via abort (streamResponse handles mid-stream disconnects).
1483
1627
  res.off('close', onClientClose);
@@ -1581,6 +1725,9 @@ async function forwardRequest(
1581
1725
  && !clientHasRealBytes(res, requestInfo)
1582
1726
  && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
1583
1727
  ) {
1728
+ // Forced move (review 2026-11-16, 5b): label it, so the TUI breakdown and the
1729
+ // forced-moves signal see it instead of an unlabelled 'other'.
1730
+ requestInfo._moveCause = 'wall';
1584
1731
  logFailedAttempt(account.name, 'upstream_429', `server-side throttle, retry-after ${retryAfter}s`, 'moved');
1585
1732
  return forwardRequest(
1586
1733
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -1699,6 +1846,7 @@ async function forwardRequest(
1699
1846
  !clientHasRealBytes(res, requestInfo) &&
1700
1847
  hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
1701
1848
  ) {
1849
+ requestInfo._moveCause = 'wall';
1702
1850
  logFailedAttempt(account.name, rlKind, `HTTP 429, benched ${retryAfter}s`, 'moved');
1703
1851
  return forwardRequest(
1704
1852
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -1806,6 +1954,7 @@ async function forwardRequest(
1806
1954
  && !clientHasRealBytes(res, requestInfo)
1807
1955
  && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
1808
1956
  ) {
1957
+ requestInfo._moveCause = 'wall';
1809
1958
  logFailedAttempt(account.name, 'upstream_529', `overloaded, retry-after ${retryAfter}s`, 'moved');
1810
1959
  return forwardRequest(
1811
1960
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -1862,6 +2011,7 @@ async function forwardRequest(
1862
2011
 
1863
2012
  if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
1864
2013
  && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
2014
+ requestInfo._moveCause = 'link';
1865
2015
  logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
1866
2016
  return forwardRequest(
1867
2017
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -2506,6 +2656,7 @@ async function forwardRequest(
2506
2656
  console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" `
2507
2657
  + `(${gotOutput ? 'mid-response — output already sent' : 'still waiting, nothing sent'})`);
2508
2658
  logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (${gotOutput ? 'mid-response' : 'still waiting'})`, 'released-cancel');
2659
+ requestInfo._clientCancelled = true;
2509
2660
  releaseOnClientGone();
2510
2661
  return;
2511
2662
  }
@@ -2515,6 +2666,7 @@ async function forwardRequest(
2515
2666
  const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
2516
2667
  console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" (during the response body read) — released, not retried`);
2517
2668
  logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (body read)`, 'released-cancel');
2669
+ requestInfo._clientCancelled = true;
2518
2670
  accountManager.relinquishUpstreamProbe?.(lease);
2519
2671
  accountManager.releaseAccount(lease, { neutral: true, error: 'client_cancelled', kind: 'client_cancelled' });
2520
2672
  clearQueueHeartbeat(requestInfo);
@@ -2631,6 +2783,7 @@ async function forwardRequest(
2631
2783
  // answer with a misleading quota message).
2632
2784
  if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2633
2785
  && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
2786
+ requestInfo._moveCause = 'link';
2634
2787
  logFailedAttempt(account.name, transientKind, `transient after ${heldSeconds(lease)}s`, 'moved');
2635
2788
  return forwardRequest(
2636
2789
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -2681,6 +2834,7 @@ async function forwardRequest(
2681
2834
  && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
2682
2835
  account.status = 'error';
2683
2836
  excludedIndexes.add(account.index);
2837
+ requestInfo._moveCause = 'link';
2684
2838
  logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
2685
2839
  return forwardRequest(
2686
2840
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -2762,6 +2916,76 @@ function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
2762
2916
  return !clientHasRealBytes(res, requestInfo);
2763
2917
  }
2764
2918
 
2919
+ /** The next sticky wait (ms) for 0-based attempt `attempt`, or 0 = stop waiting and move.
2920
+ * The 6-attempt cap bounds body re-uploads, so the waits GROW to still cover the budget:
2921
+ * min(15 s, retry-after × 2^attempt) × full jitter (0.5-1.5), and the LAST attempt waits
2922
+ * whatever is left. Every wait is clipped to the budget left, so the total equals the
2923
+ * setting and never exceeds it. At retry-after 2 s, jitter midpoint:
2924
+ * 15 s → 2+4+8+1 30 s → 2+4+8+15+1 60 s → 2+4+8+15+15+16 120 s → 2+4+8+15+15+76
2925
+ * Budget 0 ('off') → never waits (moves at once, as before). */
2926
+ function nextStickyBusyWaitMs(attempt, retryAfterMs, spentMs, budgetMs, rand = Math.random) {
2927
+ const remaining = budgetMs - spentMs;
2928
+ if (attempt >= STICKY_BUSY_MAX_ATTEMPTS || remaining <= 0) return 0;
2929
+ if (attempt === STICKY_BUSY_MAX_ATTEMPTS - 1) return remaining;
2930
+ return Math.min(remaining, Math.min(STICKY_BUSY_WAIT_CAP_MS, retryAfterMs * 2 ** attempt) * (0.5 + rand()));
2931
+ }
2932
+
2933
+ /** R3: may a busy refusal on this cloud row WAIT and retry the same row? Only a warm
2934
+ * Claude conversation (its cache is on this row), a big prompt, nothing sent yet, and
2935
+ * inside the client's patience (the attempt cap and the live budget: nextStickyBusyWaitMs). */
2936
+ function stickyBusyWaitEligible(account, requestInfo, res, queueConfig, accountManager, waitMs) {
2937
+ if (requestInfo.isCountTokens || clientHasRealBytes(res, requestInfo)) return false;
2938
+ if ((requestInfo.bodyBytes || 0) < STICKY_BUSY_MIN_BYTES) return false;
2939
+ // Non-streaming waits only with a real deadline that still leaves time to move after.
2940
+ if (!requestInfo.stream && (requestInfo.deadlineAt == null
2941
+ || nonStreamRemainingMs(requestInfo) - waitMs < deadlineKnobs().NONSTREAM_MIN_MOVE_MS)) return false;
2942
+ if (!accountManager.holdsWarmConversation?.(account, requestInfo)) return false;
2943
+ const ceilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
2944
+ if (ceilingMs != null && Date.now() - (requestInfo.startedAt || Date.now()) + waitMs > ceilingMs) return false;
2945
+ return true;
2946
+ }
2947
+
2948
+ /** R1: classify a linked pool's 429/503 before any byte. Header cause first, then the
2949
+ * 503 body type (sent by every version), then a short retry-after = busy-class;
2950
+ * anything else is "unavailable" (never guessed busy). Reads ≤ 4 KB of the body. */
2951
+ async function readPinRefusal(upRes) {
2952
+ const num = (v) => (v == null || v === '' || !Number.isFinite(Number(v)) ? null : Number(v));
2953
+ const ra = num(upRes.headers.get('retry-after'));
2954
+ const retryAfterS = ra != null && ra > 0 ? ra : null;
2955
+ let bodyType = null;
2956
+ if (upRes.body) {
2957
+ try {
2958
+ const reader = upRes.body.getReader();
2959
+ const r = await Promise.race([reader.read(), sleep(2000).then(() => null)]);
2960
+ if (r?.value) {
2961
+ try { bodyType = JSON.parse(Buffer.from(r.value).subarray(0, 4096).toString())?.error?.type ?? null; } catch { /* not JSON */ }
2962
+ }
2963
+ reader.cancel().catch(() => {});
2964
+ } catch { /* drained */ }
2965
+ }
2966
+ const header = String(upRes.headers.get('x-maxpool-pin-refusal') || '').toLowerCase();
2967
+ const cause = ['busy', 'unavailable', 'throttled'].includes(header) ? header
2968
+ : bodyType === 'pinned_account_busy' ? 'busy'
2969
+ : bodyType === 'pinned_account_unavailable' ? 'unavailable'
2970
+ : retryAfterS != null && retryAfterS <= PIN_BUSY_MAX_RETRY_AFTER_S ? 'busy'
2971
+ : 'unavailable';
2972
+ return { cause, retryAfterS, bodyType,
2973
+ inflight: num(upRes.headers.get('x-maxpool-pin-inflight')), limit: num(upRes.headers.get('x-maxpool-pin-limit')) };
2974
+ }
2975
+
2976
+ /** Sleep `ms` unless the client leaves first. true = slept the whole time. */
2977
+ function sleepUnlessClientGone(ms, req, res) {
2978
+ if (res.destroyed || res.writableEnded) return Promise.resolve(false);
2979
+ return new Promise((resolve) => {
2980
+ let timer = null;
2981
+ const done = (stayed) => { clearTimeout(timer); req.off('aborted', gone); res.off('close', gone); resolve(stayed); };
2982
+ const gone = () => done(false);
2983
+ req.once('aborted', gone);
2984
+ res.once('close', gone);
2985
+ timer = setTimeout(() => done(!res.destroyed), ms);
2986
+ });
2987
+ }
2988
+
2765
2989
  /**
2766
2990
  * D5: one line per FAILED attempt, naming the account, the error class and what the
2767
2991
  * pool did next — so "why did this move" is answerable from the log without a forensic
@@ -3030,7 +3254,7 @@ function isContextLengthError(errorBody) {
3030
3254
  return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
3031
3255
  }
3032
3256
 
3033
- export const __serverTest = { hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
3257
+ export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
3034
3258
 
3035
3259
  async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
3036
3260
  if (!upstreamRes.body) return '';
@@ -4992,6 +5216,71 @@ function noteCapacityCache(requestInfo, usage) {
4992
5216
  requestInfo._capacityCacheCreation = Math.max(requestInfo._capacityCacheCreation || 0, usage?.cache_creation_input_tokens || 0);
4993
5217
  }
4994
5218
 
5219
+ /** Pure: the usage object one parsed SSE event carries — message_start's
5220
+ * `message.usage` or message_delta's `usage` — else null. No side effects. */
5221
+ function usageFromSseData(data) {
5222
+ if (data?.type === 'message_start' && data.message?.usage) return data.message.usage;
5223
+ if (data?.type === 'message_delta' && data.usage) return data.usage;
5224
+ return null;
5225
+ }
5226
+
5227
+ /** Running-max one event's usage into the request's capacity figures. Output only from
5228
+ * message_delta (Anthropic's interim delta usage is CUMULATIVE — M3). Providers differ
5229
+ * in WHICH event carries the cache/input counters (z.ai: all of them on message_delta,
5230
+ * message_start is zeros — captured 2026-11-06, pinned by test/capacity-seam.test.js +
5231
+ * test/fixtures/zai-sse-usage.txt), so input/cache are a running max over BOTH events. */
5232
+ function noteCapacityUsage(requestInfo, usage, withOutput) {
5233
+ requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, usage?.input_tokens || 0);
5234
+ if (withOutput) requestInfo._capacityOutput = Math.max(requestInfo._capacityOutput || 0, usage?.output_tokens || 0);
5235
+ noteCapacityCache(requestInfo, usage);
5236
+ }
5237
+
5238
+ /** Pure: the usage object at the END of a JSON message body, from its tail (the body
5239
+ * may be cut at the front). Finds the last `"usage":{…}` and parses that object. */
5240
+ function usageFromBodyTail(text) {
5241
+ const at = String(text || '').lastIndexOf('"usage"');
5242
+ if (at < 0) return null;
5243
+ const open = text.indexOf('{', at);
5244
+ if (open < 0) return null;
5245
+ let depth = 0;
5246
+ for (let i = open; i < text.length; i++) {
5247
+ if (text[i] === '{') depth++;
5248
+ else if (text[i] === '}' && --depth === 0) {
5249
+ try { return JSON.parse(text.slice(open, i + 1)); } catch { return null; }
5250
+ }
5251
+ }
5252
+ return null;
5253
+ }
5254
+
5255
+ /** LINK USAGE TAP: read the usage a linked pool relays (it is the SERVING pool and books
5256
+ * the tokens itself), into this request's capacity figures ONLY — never a ledger,
5257
+ * usage or caller booking. Streaming: SSE events split across chunks are re-joined.
5258
+ * Non-streaming: keeps the last 64 KB, parsed when called with null at stream end. */
5259
+ function tapLinkUsage(chunk, requestInfo) {
5260
+ if (requestInfo.isCountTokens) return;
5261
+ if (!requestInfo.stream) {
5262
+ if (chunk) {
5263
+ requestInfo._tapBuf = ((requestInfo._tapBuf || '') + Buffer.from(chunk).toString()).slice(-64 * 1024);
5264
+ return;
5265
+ }
5266
+ const u = usageFromBodyTail(requestInfo._tapBuf);
5267
+ requestInfo._tapBuf = '';
5268
+ if (u) noteCapacityUsage(requestInfo, u, true);
5269
+ return;
5270
+ }
5271
+ if (!chunk) return;
5272
+ const parts = ((requestInfo._tapBuf || '') + Buffer.from(chunk).toString()).split('\n\n');
5273
+ requestInfo._tapBuf = parts.pop().slice(-64 * 1024);
5274
+ for (const event of parts) {
5275
+ const dataLine = event.split('\n').find(l => l.startsWith('data: '));
5276
+ if (!dataLine) continue;
5277
+ let data;
5278
+ try { data = JSON.parse(dataLine.slice(6)); } catch { continue; }
5279
+ const u = usageFromSseData(data);
5280
+ if (u) noteCapacityUsage(requestInfo, u, data.type === 'message_delta');
5281
+ }
5282
+ }
5283
+
4995
5284
  function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
4996
5285
  const dataLine = event.split('\n').find(l => l.startsWith('data: '));
4997
5286
  if (!dataLine) return;
@@ -5002,23 +5291,13 @@ function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
5002
5291
  // M3: Anthropic interim message_delta usage is CUMULATIVE — add-semantics inflates
5003
5292
  // output on long generations. Track a per-stream RUNNING MAX, accrue once at end.
5004
5293
  // M4: count_tokens responses are prompt-size echoes, not delivered work — skip.
5005
- if (!requestInfo.isCountTokens) {
5006
- if (data.type === 'message_start' && data.message?.usage) {
5007
- accountManager.updateUsage(accountIndex, data.message.usage.input_tokens, 0);
5008
- accountManager.countCallerTokens?.(requestInfo.caller, data.message.usage.input_tokens);
5009
- requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, data.message.usage.input_tokens || 0);
5010
- noteCapacityCache(requestInfo, data.message.usage);
5011
- } else if (data.type === 'message_delta' && data.usage) {
5012
- accountManager.updateUsage(accountIndex, 0, data.usage.output_tokens);
5013
- accountManager.countCallerTokens?.(requestInfo.caller, data.usage.output_tokens);
5014
- requestInfo._capacityOutput = Math.max(requestInfo._capacityOutput || 0, data.usage.output_tokens || 0);
5015
- // Providers differ in WHICH event carries the cache/input counters (z.ai: all of
5016
- // them on message_delta, message_start is zeros — captured 2026-11-06, pinned by
5017
- // test/capacity-seam.test.js + test/fixtures/zai-sse-usage.txt), so every field is
5018
- // a running max over BOTH events.
5019
- requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, data.usage.input_tokens || 0);
5020
- noteCapacityCache(requestInfo, data.usage);
5021
- }
5294
+ const usage = requestInfo.isCountTokens ? null : usageFromSseData(data);
5295
+ if (usage) {
5296
+ const isDelta = data.type === 'message_delta';
5297
+ const n = isDelta ? usage.output_tokens : usage.input_tokens;
5298
+ accountManager.updateUsage(accountIndex, isDelta ? 0 : n, isDelta ? n : 0);
5299
+ accountManager.countCallerTokens?.(requestInfo.caller, n);
5300
+ noteCapacityUsage(requestInfo, usage, isDelta);
5022
5301
  }
5023
5302
  if (sseEventContainsThinking(data)) {
5024
5303
  accountManager.markSessionThinkingProtected?.(requestInfo.sessionKey, requestInfo.model);
@@ -5075,6 +5354,7 @@ function extractUsageFromBody(buffer, accountIndex, accountManager, requestInfo
5075
5354
  // work delivered. Claude Code calls it constantly; counting it would inflate every
5076
5355
  // cycle by whole prompt sizes.
5077
5356
  if (!requestInfo.isCountTokens) {
5357
+ noteCapacityUsage(requestInfo, json.usage, true); // efficiency figures (no booking)
5078
5358
  accountManager.accrueCapacity?.(accountIndex, {
5079
5359
  input: json.usage.input_tokens || 0,
5080
5360
  output: json.usage.output_tokens || 0,