maxpool 1.5.22 → 1.5.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.22",
+  "version": "1.5.24",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",
@@ -37,6 +37,6 @@
     "url": "https://github.com/2solarmax/maxpool/issues"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.3.0"
   }
 }

package/src/account-manager.js

@@ -2276,6 +2276,11 @@ export class AccountManager {
     // for the bounded drain; the lease holder owns all rotation.
     if (!this.writerLease) return true;
 
+    // A dead refresh token (invalid_grant) is PERMANENT until browser re-auth —
+    // never auto-retry it. Without this the prober re-POSTs the rejected token every
+    // ~60s forever (hammering Anthropic's OAuth endpoint). Cleared on re-login.
+    if (account.refreshDead) return false;
+
     if (!force && !isTokenExpiringSoon(account.expiresAt)) return true;
 
     // Coalesce concurrent refreshes
@@ -2309,6 +2314,11 @@ export class AccountManager {
             this.markTransientFailure(accountIndex, `token_refresh_${err.status || 'network'}`, { network: !err.status });
           } else {
             this.markAuthFailed(accountIndex, err.status || 401, 'token_refresh_failed');
+            // The refresh TOKEN itself was rejected (invalid_grant) — permanent until
+            // browser re-auth. Latch so ensureTokenFresh + the prober stop retrying a
+            // dead token. Set ONLY here (a rejected refresh), never in markAuthFailed
+            // (shared with provider auth failures).
+            account.refreshDead = true;
           }
           return false;
         }
@@ -2350,6 +2360,7 @@ export class AccountManager {
     account.credential = accessToken;
     if (refreshToken) account.refreshToken = refreshToken;
     account.expiresAt = expiresAt;
+    account.refreshDead = false;  // fresh tokens from re-auth revive a dead-refresh account
     if (account.status === 'error') account.status = 'active';
     console.log(`[Maxpool] Updated tokens for account "${account.name}"`);
     this._onTokenRefresh?.(accountIndex, {
@@ -2434,6 +2445,7 @@ export class AccountManager {
       account.lastError = null;
       account.lastErrorAt = null;
       account.consecutiveFailures = 0;
+      account.refreshDead = false;
     }
     return idx;
   }
@@ -2575,6 +2587,7 @@ export class AccountManager {
         priority: a.priority,
         runtime: a.runtime,
         status: a.status,
+        refreshDead: Boolean(a.refreshDead),
         inFlight: a.inFlight,
         activeWeight: a.activeWeight,
         completedRequests: a.completedRequests,

package/src/oauth.js

@@ -268,14 +268,92 @@ export function classifyZaiLimit(l, now = Date.now()) {
   return { bucket, utilization, resetAt };
 }
 
-/** Read a provider account's quota. z.ai has a pollable monitor endpoint mapped
- *  to Ses/Wk token windows; Kimi (Moonshot coding key) has NO pollable quota
- *  (web console only), so it returns a `console-only` marker instead of fake
- *  bars. Returns { ses, wk, level } | { error, status?, source? }. */
+// Kimi Coding Plan usage — the coding key (sk-kimi-…) reads its own quota at
+// <base>/v1/usages (the community `kimi-code-usage` tool's endpoint). NOTE the
+// User-Agent: the endpoint is served by the Kimi Code platform, not the Open
+// Platform (whose /users/me/balance 401s a coding key). Best-effort, non-load-
+// bearing UA — the failure path degrades to last-known + the staleness marker.
+const KIMI_USAGE_UA = 'KimiCLI/1.6';
+
+/** ms of one Kimi limit `window`, normalized from its timeUnit so the SHORTEST
+ *  window (the 5h session) is picked regardless of unit. null when unknown. */
+export function kimiWindowMs(window) {
+  const dur = Number(window?.duration);
+  if (!Number.isFinite(dur) || dur <= 0) return null;
+  const unit = String(window?.timeUnit || window?.time_unit || '').toUpperCase();
+  const mult = unit.includes('SECOND') ? 1e3
+    : unit.includes('MINUTE') ? 60e3
+    : unit.includes('HOUR') ? 3600e3
+    : unit.includes('DAY') ? 86400e3
+    : unit.includes('WEEK') ? 7 * 86400e3
+    : unit.includes('MONTH') ? 30 * 86400e3
+    : null;
+  return mult == null ? null : dur * mult;
+}
+
+/** Normalize a Kimi usage row {limit,used,remaining,resetTime} (values are STRINGS)
+ *  to { utilization, resetAt }. CRITICAL: utilization is `null` — never a computed
+ *  NaN — when the inputs are absent or limit<=0, because clamp01(NaN)=0 downstream
+ *  would render a blind account as "0% used = fully available" and route real
+ *  traffic onto quota we can't see. */
+export function parseKimiRow(d) {
+  if (!d || typeof d !== 'object') return null;
+  const limit = Number(d.limit ?? d.limit_amount);
+  let used = d.used != null ? Number(d.used ?? d.used_amount) : null;
+  const remaining = d.remaining != null ? Number(d.remaining) : null;
+  if (used == null && remaining != null && Number.isFinite(limit)) used = limit - remaining;
+  const utilization = (Number.isFinite(limit) && limit > 0 && used != null && Number.isFinite(used))
+    ? Math.max(0, Math.min(1, used / limit))
+    : null;
+  const resetRaw = d.resetTime || d.reset_at || d.reset_time;
+  const parsed = resetRaw ? Date.parse(resetRaw) : NaN;
+  return { utilization, resetAt: Number.isFinite(parsed) ? parsed : null };
+}
+
+/** Read the Kimi Coding Plan quota → { ses, wk, source:'kimi' } | { error, status? }.
+ *  Weekly = the top-level `usage` rolling window; Ses (5h) = the shortest `limits[]`
+ *  window. A transient/partial response returns {error} or null buckets so
+ *  applyProviderUsage keeps the last-known values (never a phantom 0%). */
+export async function fetchKimiUsage(account) {
+  const token = account?.credential;
+  if (!token) return { error: 'unsupported', source: null };
+  const base = String(account.upstream || 'https://api.kimi.com/coding').replace(/\/+$/, '');
+  const headers = { 'Authorization': `Bearer ${token}`, 'User-Agent': KIMI_USAGE_UA, 'Accept': 'application/json' };
+  try {
+    let res = await fetch(`${base}/v1/usages`, { headers, signal: AbortSignal.timeout(10_000) });
+    if (res.status === 404) res = await fetch(`${base}/v1/usage`, { headers, signal: AbortSignal.timeout(10_000) });
+    if (!res.ok) return { error: `HTTP ${res.status}`, status: res.status };
+    const data = await res.json();
+    // Weekly = top-level `usage` (ALWAYS present in a valid coding-plan response).
+    // If it's missing, the response is partial/malformed → return an error so
+    // applyProviderUsage keeps the last-known weekly instead of the falsy-wk branch
+    // BLANKING it (unlike z.ai, where a missing weekly genuinely means "no weekly").
+    const wkRow = parseKimiRow(data?.usage);
+    if (!wkRow || wkRow.utilization == null) return { error: 'partial_response', source: 'kimi' };
+    const wk = wkRow;
+    // Ses = the shortest limits[] window (the 5h). Fall back to first if durations unknown.
+    let ses = null, sesMs = Infinity;
+    for (const l of (Array.isArray(data?.limits) ? data.limits : [])) {
+      const detail = l && typeof l.detail === 'object' ? l.detail : l;
+      const row = parseKimiRow(detail);
+      if (!row || row.utilization == null) continue;
+      const ms = kimiWindowMs(l?.window);
+      if (ms != null && ms < sesMs) { sesMs = ms; ses = row; }
+      else if (ses == null && ms == null) ses = row;
+    }
+    return { ses, wk, source: 'kimi', level: data?.user?.membership?.level || null };
+  } catch (err) {
+    return { error: err.message || String(err), status: null };
+  }
+}
+
+/** Read a provider account's quota. z.ai and Kimi both have a pollable coding-plan
+ *  usage endpoint mapped to Ses/Wk windows. Returns { ses, wk, source } |
+ *  { error, status?, source? }. */
 export async function fetchProviderUsage(account) {
   const provider = account?.provider;
   const token = account?.credential;
-  if (provider === 'kimi') return { error: 'unsupported', source: 'console-only' };
+  if (provider === 'kimi') return fetchKimiUsage(account);
   if (provider !== 'zai' || !token) return { error: 'unsupported', source: null };
   try {
     const res = await fetch(ZAI_QUOTA_URL, {

package/src/prober.js

@@ -59,7 +59,9 @@ export class Prober {
     this._running = true;
     this._inflight = (async () => {
       try {
-        const oauth = this.am.accounts.filter(a => a.type === 'oauth' && a.credential);
+        // Skip auth-dead accounts (dead refresh token): probing them just re-POSTs
+        // the rejected token every cycle — a 400 storm. They recover only on re-auth.
+        const oauth = this.am.accounts.filter(a => a.type === 'oauth' && a.credential && !a.refreshDead);
         const providers = this.am.accounts.filter(a => a.type === 'provider' && a.credential);
         await Promise.all([
           ...oauth.map(a => this.probeOne(a)),

package/src/server.js

@@ -15,6 +15,16 @@ const DEFAULT_RETRY = {
   maxRetryBufferBytes: 10 * 1024 * 1024,
 };
 
+// Upstream-hang guards. A half-open upstream (headers then silence, never closes)
+// would block the request forever and LEAK its in-flight lease — the account climbs
+// to safetyMaxActivePerAccount and is falsely "full", breaking routing (the "106
+// phantom active" incident). Env-overridable with conservative defaults: this path
+// serves EVERY provider (Anthropic/GLM/Kimi), so the idle gap is generous enough
+// that a legitimately-slow-but-alive stream is never cut (each chunk resets it).
+const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) || 120_000);        // headers must arrive within this
+const STREAM_IDLE_MS = Math.max(30_000, Number(process.env.MAXPOOL_STREAM_IDLE_MS) || 300_000);  // max gap BETWEEN streamed chunks (reset per chunk)
+const UPSTREAM_BODY_MS = Math.max(30_000, Number(process.env.MAXPOOL_BODY_MS) || 300_000);       // non-streaming body read
+
 const DEFAULT_QUEUE = {
   enabled: true,
   maxWaitMs: 24 * 60 * 60 * 1000,
@@ -429,14 +439,31 @@ async function forwardRequest(
     }
   }
 
+  // TTFB guard: an upstream that accepts the connection but never sends response
+  // headers would hang `await fetch` forever (signal only covers client-disconnect)
+  // and leak the lease. Abort via a DEDICATED controller (so the caller can tell a
+  // TTFB timeout apart from a client-gone abort) and CLEAR it the moment headers
+  // arrive — the streaming body is then governed by the per-chunk idle guard, never
+  // this fixed clock, so a long healthy stream is never cut.
+  const ttfbController = new AbortController();
+  const ttfbTimer = setTimeout(
+    () => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: 'UPSTREAM_TTFB' })),
+    UPSTREAM_TTFB_MS,
+  );
+  ttfbTimer.unref?.();
   try {
-    const upstreamRes = await fetch(upstreamUrl, {
-      method,
-      headers,
-      body: ['GET', 'HEAD'].includes(method) ? undefined : upstreamBody,
-      redirect: 'manual',
-      signal: clientGone.signal,
-    });
+    let upstreamRes;
+    try {
+      upstreamRes = await fetch(upstreamUrl, {
+        method,
+        headers,
+        body: ['GET', 'HEAD'].includes(method) ? undefined : upstreamBody,
+        redirect: 'manual',
+        signal: AbortSignal.any([clientGone.signal, ttfbController.signal]),
+      });
+    } finally {
+      clearTimeout(ttfbTimer);
+    }
     // Response arrived — the pre-response leak window is over. Stop guarding for
     // client-disconnect via abort (streamResponse handles mid-stream disconnects).
     res.off('close', onClientClose);
@@ -840,7 +867,26 @@ async function forwardRequest(
         writeRequestLog(logDir, reqId, logSections);
       }
     } else {
-      const buf = Buffer.from(await upstreamRes.arrayBuffer());
+      // Bound the non-streaming body read so a mid-body upstream stall can't hang
+      // the request forever and leak the lease (same class as the streaming idle
+      // guard). On timeout → UPSTREAM_BODY → the caller frees the lease.
+      const bodyP = upstreamRes.arrayBuffer();
+      bodyP.catch(() => {});
+      let bodyTimer;
+      const bodyTimeout = new Promise((_, reject) => {
+        bodyTimer = setTimeout(
+          () => reject(Object.assign(new Error('upstream body timeout'), { code: 'UPSTREAM_BODY' })),
+          UPSTREAM_BODY_MS,
+        );
+        bodyTimer.unref?.();
+      });
+      let arr;
+      try {
+        arr = await Promise.race([bodyP, bodyTimeout]);
+      } finally {
+        clearTimeout(bodyTimer);
+      }
+      const buf = Buffer.from(arr);
       extractUsageFromBody(buf, account.index, accountManager);
       markThinkingFromResponse(buf, accountManager, requestInfo);
       accountManager.releaseAccount(lease, { success: upstreamRes.status < 500, status: upstreamRes.status });
@@ -896,6 +942,11 @@ async function forwardRequest(
       (err.message.includes('fetch failed') ||
         err.code === 'ECONNRESET' || err.code === 'ECONNREFUSED' ||
         err.code === 'ETIMEDOUT' || err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
+        // Upstream-hang guards: a stalled connection is network-class, not the
+        // account's fault → 5s cooldown + release + (pre-headers) retry elsewhere;
+        // once committed, isTransient falls through to sendErrorResponse which ends
+        // the client's SSE cleanly (frees the lease AND unhangs the client).
+        err.code === 'UPSTREAM_TTFB' || err.code === 'UPSTREAM_IDLE' || err.code === 'UPSTREAM_BODY' ||
         err.message.includes('terminated'));
 
     if (isTransient) {
@@ -1045,7 +1096,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
 }
 
-export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody };
+export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, streamResponse };
 
 async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
   if (!upstreamRes.body) return '';
@@ -1696,7 +1747,7 @@ function mappedModel(originalModel, account) {
 /**
  * Stream an SSE response to the client, parsing usage data along the way.
  */
-async function streamResponse(webStream, res, status, responseHeaders, accountIndex, accountManager, streamLog, requestInfo = {}) {
+async function streamResponse(webStream, res, status, responseHeaders, accountIndex, accountManager, streamLog, requestInfo = {}, idleMs = STREAM_IDLE_MS) {
   const reader = webStream.getReader();
   const decoder = new TextDecoder();
   let sseBuffer = '';
@@ -1711,9 +1762,39 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
   // pre-byte failover can still re-hold the session via queueAndRetry.
   clearQueueHeartbeat(requestInfo);
 
+  // Client left mid-stream: the caller drops its disconnect-abort once headers
+  // arrive, and the loop below only checks res.destroyed AFTER a read resolves — so
+  // a BLOCKED read() would never notice. Cancel the reader on 'close' to unblock it
+  // (resolves the pending read as done) → the loop breaks → the lease frees instead
+  // of leaking until the (maybe never) upstream close.
+  const onClose = () => { reader.cancel().catch(() => {}); };
+  res.once('close', onClose);
+
   try {
     while (true) {
-      const { done, value } = await reader.read();
+      // Idle guard: a half-open upstream (headers, then silence, never closes) would
+      // block reader.read() forever → the whole request handler hangs → neither the
+      // lease nor onRequestEnd is released (the "106 phantom active" leak). Bound each
+      // read by a per-chunk idle timeout, RESET every chunk so a healthy slow stream
+      // is never cut; on a real stall, UPSTREAM_IDLE → the caller frees the lease and
+      // ends the client SSE cleanly (isTransient → sendErrorResponse).
+      const readP = reader.read();
+      readP.catch(() => {});  // race-loser must not surface as an unhandled rejection
+      let idleTimer;
+      const idle = new Promise((_, reject) => {
+        idleTimer = setTimeout(
+          () => reject(Object.assign(new Error('upstream idle timeout'), { code: 'UPSTREAM_IDLE' })),
+          idleMs,
+        );
+        idleTimer.unref?.();
+      });
+      let result;
+      try {
+        result = await Promise.race([readP, idle]);
+      } finally {
+        clearTimeout(idleTimer);  // single live timer at a time — no per-chunk timer accumulation
+      }
+      const { done, value } = result;
       if (done) break;
 
       // Client disconnected — stop reading from upstream
@@ -1760,6 +1841,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
     readFailed = true;
     throw err;
   } finally {
+    res.off('close', onClose);
     // Cancel upstream reader to stop consuming data nobody needs
     reader.cancel().catch(() => {});
     if (!readFailed) {

package/src/tui.js

@@ -664,6 +664,10 @@ export class TUI {
         amAcct.expiresAt = creds.expiresAt;
         amAcct.accountUuid = entry.accountUuid;
         amAcct.name = name;
+        // Fresh tokens from re-auth revive a dead-refresh account. This in-place
+        // update is the actual re-login path (the index.js reconcile sees no change
+        // and never calls updateAccountTokens), so refreshDead MUST be cleared here.
+        amAcct.refreshDead = false;
         if (amAcct.status === 'error') amAcct.status = 'active';
       }
       this._addLog(`Updated account "${name}"`);
@@ -975,8 +979,13 @@ export class TUI {
     if (a.enabled !== false && a.status === 'active' && this.am._isAccountWideRejected?.(a)) {
       effectiveStatus = 'blocked';
     }
+    // A dead refresh token surfaces as "reauth" (re-login needed) rather than a
+    // generic "error", so the user knows the fix. Display-only — account.status
+    // stays 'error' so routing/eligibility still exclude it.
+    if (a.enabled !== false && a.refreshDead) effectiveStatus = 'reauth';
     switch (effectiveStatus) {
       case 'active':    status = isCur ? green('active') : 'active'; break;
+      case 'reauth':    status = yellow('reauth'); break;
       case 'blocked':   status = red('blocked'); break;
       case 'probing':   status = green('probing'); break;
       case 'waiting':   status = yellow('waiting'); break;