maxpool 1.5.16 → 1.5.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.16",
+  "version": "1.5.17",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -42,6 +42,19 @@ function emptyQuota() {
     scopedWeekly: {},
     unifiedStatus: null,   // allowed | allowed_warning | rejected
     resetsAt: null,
+    // Provider (z.ai / Kimi) quota — kept SEPARATE from unified* so a provider
+    // reading never leaks into the OAuth quota gates (_isAvailable / _weeklyRawState
+    // / _accountScarcity read unified* only). z.ai is pollable; Kimi is not.
+    providerSes: null,          // utilization 0-1 (z.ai 5h token window)
+    providerSesReset: null,     // ms
+    providerWk: null,           // utilization 0-1 (z.ai weekly), null if plan has none
+    providerWkReset: null,      // ms
+    providerQuotaSource: null,  // 'zai' (pollable) | 'console-only' (kimi) | null
+    // Freshness: last time a background usage PROBE succeeded for this account
+    // (oauth fetchUsage OR provider fetchProviderUsage). Drives the TUI staleness
+    // marker — a swallowed failing probe no longer silently freezes a stale tag.
+    // Header-driven updates do NOT stamp this (headers can't refresh scoped/provider).
+    lastProbeOkAt: null,        // ms
   };
 }
 
@@ -589,8 +602,14 @@ export class AccountManager {
     if (!fam) return null;
     const e = account.quota?.scopedWeekly?.[fam];
     if (!e || e.isActive === false) return null;
-    const exhausted = e.severity === 'critical'
-      || (e.utilization != null && e.utilization >= this.scheduler.weeklyExhaustedThreshold);
+    // Bench ONLY at genuine exhaustion (>= weeklyExhaustedThreshold). Anthropic
+    // labels a scoped weekly `severity:'critical'` well before it's actually
+    // capped (~90%), where the model still has headroom and is still served — a
+    // hard bench there strands the remainder and mislabels the account "maxed".
+    // A real scoped 429 writes utilization:1 (markRateLimited), so genuine
+    // exhaustion is still caught by the threshold. Same predicate drives the TUI
+    // `maxed` tag, so "maxed" renders iff the model is actually benched.
+    const exhausted = e.utilization != null && e.utilization >= this.scheduler.weeklyExhaustedThreshold;
     return exhausted ? { resetAt: e.resetAt || null, family: fam } : null;
   }
 
@@ -1648,6 +1667,16 @@ export class AccountManager {
     // soft de-preference of accounts burning ahead of an even pace. Never a bench.
     const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight;
 
+    // Per-model weekly de-preference: an account whose scoped weekly for THIS
+    // request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
+    // that model — shed its load toward healthier accounts BEFORE the hard bench
+    // at weeklyExhaustedThreshold, so it's chosen only as overflow (rarely
+    // re-429ing). Scoped weekly is otherwise absent from scoring. Soft, never a
+    // bench; 0 below reserve and for models with no scoped cap.
+    const scopedPace = requestInfo.model
+      ? this._scopedScarcity(account, requestInfo.model, now) * this.scheduler.paceCostWeight
+      : 0;
+
     const fleetRecentWeight = ctx?.fleetRecentWeight ?? 0;
     const recentWeight = this._loadSummary(account, this.scheduler.spreadWindowMs, now).weight;
     const share = fleetRecentWeight > 0 ? recentWeight / fleetRecentWeight : 0;
@@ -1663,7 +1692,22 @@ export class AccountManager {
     // default) learns the real number within a cycle. `probing`/requalify still
     // flags a never-seen account for learning — that path is unchanged.
 
-    return concurrency + capPenalty + paceCost + spread + ramp + failurePenalty;
+    return concurrency + capPenalty + paceCost + scopedPace + spread + ramp + failurePenalty;
+  }
+
+  /**
+   * Per-model weekly pace-overage for `model`'s family, or 0 when the account has
+   * no scoped cap for it, the cap is inactive, or it's below the reserve tier
+   * (plenty of headroom → no steering). Same pace discount as _windowScarcity, so
+   * a scoped window about to reset is cheap to spend.
+   */
+  _scopedScarcity(account, model, now = Date.now()) {
+    const fam = modelFamily(model);
+    if (!fam) return 0;
+    const e = account.quota?.scopedWeekly?.[fam];
+    if (!e || e.isActive === false || e.utilization == null) return 0;
+    if (e.utilization < this.scheduler.weeklyReserveThreshold) return 0;
+    return this._windowScarcity(e.utilization, e.resetAt, WEEK_MS, now);
   }
 
   /**
@@ -1795,8 +1839,15 @@ export class AccountManager {
     }
     // Per-model weekly sub-limits (Fable, Opus, ...). Replace wholesale with the
     // fresh probe set so a family that dropped out of the response doesn't linger
-    // stale; expiry on reset is a backstop for the between-probe window.
+    // stale; expiry on reset is a backstop for the between-probe window. EXCEPTION:
+    // a `reactive` scoped-429 bench is authoritative-high — while its resetAt is
+    // still future, a lagging probe may neither lower it nor drop it (a probe that
+    // omits the family, or reports the pre-429 level, would otherwise un-bench it
+    // and trigger an immediate re-429 flap). _clearExpiredQuotas self-clears it at
+    // resetAt even if probes die.
     if (usage.scopedWeekly && typeof usage.scopedWeekly === 'object') {
+      const now = Date.now();
+      const prev = (q.scopedWeekly && typeof q.scopedWeekly === 'object') ? q.scopedWeekly : {};
       const fresh = {};
       for (const [fam, e] of Object.entries(usage.scopedWeekly)) {
         if (!e) continue;
@@ -1807,9 +1858,21 @@ export class AccountManager {
           isActive: e.isActive !== false,
         };
       }
+      for (const [fam, pe] of Object.entries(prev)) {
+        if (!pe || !pe.reactive || pe.resetAt == null || pe.resetAt <= now) continue;
+        const f = fresh[fam];
+        // Probe absent, null, or LOWER than the reactive bench → keep the bench.
+        // Probe CONFIRMS >= the reactive level → take the fresh reading (still >=
+        // threshold, so still exhausted; no stickiness needed).
+        if (!f || f.utilization == null || f.utilization < (pe.utilization ?? 0)) {
+          fresh[fam] = { ...pe };
+        }
+      }
       q.scopedWeekly = fresh;
     }
 
+    q.lastProbeOkAt = Date.now();
+
     // If we just learned this account's weekly window while probing, re-evaluate
     // selection (same path as learning it from a live response).
     if (account.probing && q.unified7dReset != null) {
@@ -1818,6 +1881,55 @@ export class AccountManager {
     }
   }
 
+  /**
+   * Update a PROVIDER account's quota from a provider usage probe
+   * (fetchProviderUsage). z.ai maps to Ses/Wk token windows; Kimi has no pollable
+   * source and only sets a `console-only` marker. Writes ONLY the provider* fields
+   * (never the unified or scopedWeekly fields) so a provider reading can't reach
+   * the OAuth quota gates.
+   */
+  applyProviderUsage(accountIndex, usage) {
+    const account = this.accounts[accountIndex];
+    if (!account || !usage) return;
+    const q = account.quota;
+    if (usage.error) {
+      // Distinguish "no pollable quota" (Kimi) from a transient probe failure.
+      // Never clear existing values on a transient error — let them age into the
+      // staleness marker instead of blanking the bars.
+      if (usage.source === 'console-only') q.providerQuotaSource = 'console-only';
+      return;
+    }
+    q.providerQuotaSource = usage.source || 'zai';
+    if (usage.ses) {
+      if (usage.ses.utilization != null) q.providerSes = clamp01(usage.ses.utilization);
+      if (usage.ses.resetAt != null) q.providerSesReset = usage.ses.resetAt;
+    }
+    if (usage.wk) {
+      if (usage.wk.utilization != null) q.providerWk = clamp01(usage.wk.utilization);
+      if (usage.wk.resetAt != null) q.providerWkReset = usage.wk.resetAt;
+    } else {
+      // Weekly window absent from this plan/response — clear so a stale weekly
+      // reading doesn't linger after a plan/window change.
+      q.providerWk = null;
+      q.providerWkReset = null;
+    }
+    q.lastProbeOkAt = Date.now();
+  }
+
+  /**
+   * True when the background quota probe hasn't succeeded in > 2× its interval —
+   * the last-known scoped/provider values are aging with no confirmation. Returns
+   * false when the probe is off (nothing to be stale against) or has never yet
+   * succeeded (startup — shown as "no data", not "stale").
+   */
+  _quotaProbeStale(account, now = Date.now()) {
+    const interval = this.quotaProbeIntervalMs;
+    if (!interval || interval <= 0) return false;
+    const last = account?.quota?.lastProbeOkAt;
+    if (last == null) return false;
+    return (now - last) > Math.max(2 * interval, 120_000);
+  }
+
   /**
    * Update an account's quota tracking from upstream response headers.
    */
@@ -1956,6 +2068,10 @@ export class AccountManager {
         resetAt: Date.now() + (retryAfter * 1000),
         severity: 'critical',
         isActive: true,
+        // Authoritative-high: a real reject. A lagging 60s probe reporting the
+        // pre-429 level (e.g. 0.96) must NOT lower/drop this before resetAt, else
+        // the account un-benches and immediately re-429s — a per-probe flap.
+        reactive: true,
       };
       account.lastStatus = options.status || 429;
       account.lastErrorAt = Date.now();

package/src/index.js

@@ -564,6 +564,9 @@ async function serverWorkerCommand() {
   // assertions (mirrors MAXPOOL_DISABLE_SLEEP_GUARD).
   const probeSeconds = process.env.MAXPOOL_DISABLE_QUOTA_PROBE === '1' ? 0 : (config.quotaProbeSeconds || 0);
   const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000 });
+  // Tell the AM the probe cadence so the TUI can flag a scoped/provider tag whose
+  // background probe has gone stale (> 2× interval since last success).
+  accountManager.quotaProbeIntervalMs = probeSeconds * 1000;
 
   // Persist refreshed tokens back to config. Defense-in-depth: the updater reads
   // the on-disk refresh token and SKIPS the rotation if a fresher writer already

package/src/oauth.js

@@ -245,6 +245,61 @@ export async function fetchUsage(accessToken) {
   }
 }
 
+// z.ai quota monitor — a DIFFERENT host/path than the message upstream
+// (account.upstream is the Anthropic-compat endpoint). Zero-spend read.
+const ZAI_QUOTA_URL = 'https://api.z.ai/api/monitor/usage/quota/limit';
+
+/** Classify one z.ai `limits[]` entry into a Ses (5h) or Wk (weekly) TOKEN window.
+ *  Only `TOKENS_LIMIT` maps to the quota bars; `TIME_LIMIT` is a tool-call cap
+ *  (web-search/reader counts) and is intentionally ignored. `unit` is z.ai's
+ *  window enum (3 = 5-hour session, 6 = weekly); we fall back to reset-distance
+ *  when the code is unfamiliar so a new plan tier still classifies sanely. */
+export function classifyZaiLimit(l, now = Date.now()) {
+  if (!l || l.type !== 'TOKENS_LIMIT') return null;
+  const reset = Number(l.nextResetTime);
+  const resetAt = Number.isFinite(reset) && reset > 0 ? reset : null;
+  const pct = typeof l.percentage === 'number' ? l.percentage : parseFloat(l.percentage);
+  const utilization = Number.isFinite(pct) ? Math.max(0, Math.min(1, pct / 100)) : null;
+  let bucket;
+  if (l.unit === 3) bucket = 'ses';
+  else if (l.unit === 6) bucket = 'wk';
+  else if (resetAt) bucket = (resetAt - now) <= 12 * 60 * 60 * 1000 ? 'ses' : 'wk';
+  else bucket = 'ses';
+  return { bucket, utilization, resetAt };
+}
+
+/** Read a provider account's quota. z.ai has a pollable monitor endpoint mapped
+ *  to Ses/Wk token windows; Kimi (Moonshot coding key) has NO pollable quota
+ *  (web console only), so it returns a `console-only` marker instead of fake
+ *  bars. Returns { ses, wk, level } | { error, status?, source? }. */
+export async function fetchProviderUsage(account) {
+  const provider = account?.provider;
+  const token = account?.credential;
+  if (provider === 'kimi') return { error: 'unsupported', source: 'console-only' };
+  if (provider !== 'zai' || !token) return { error: 'unsupported', source: null };
+  try {
+    const res = await fetch(ZAI_QUOTA_URL, {
+      headers: { 'Authorization': `Bearer ${token}`, 'Accept': 'application/json' },
+    });
+    if (!res.ok) return { error: `HTTP ${res.status}`, status: res.status };
+    const data = await res.json();
+    if (data?.code !== 200 || !data?.data) {
+      return { error: `bad_envelope${data?.code != null ? ' code=' + data.code : ''}` };
+    }
+    const now = Date.now();
+    let ses = null, wk = null;
+    for (const l of (Array.isArray(data.data.limits) ? data.data.limits : [])) {
+      const c = classifyZaiLimit(l, now);
+      if (!c) continue;
+      if (c.bucket === 'ses') ses = { utilization: c.utilization, resetAt: c.resetAt };
+      else if (c.bucket === 'wk') wk = { utilization: c.utilization, resetAt: c.resetAt };
+    }
+    return { ses, wk, level: data.data.level || null, source: 'zai' };
+  } catch (err) {
+    return { error: err.message || String(err), status: null };
+  }
+}
+
 // OAuth config (extracted from Claude Code)
 const OAUTH_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
 const OAUTH_AUTHORIZE = 'https://claude.ai/oauth/authorize';

package/src/prober.js

@@ -7,13 +7,14 @@
 // is blind to an account it isn't actively routing to and will pile traffic onto it.
 // This is the one sanctioned active-upstream feature; the proxy is otherwise passive.
 
-import { fetchUsage } from './oauth.js';
+import { fetchUsage, fetchProviderUsage } from './oauth.js';
 
 export class Prober {
-  constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, timeoutMs = 10_000, log = console.log } = {}) {
+  constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, timeoutMs = 10_000, log = console.log } = {}) {
     this.am = accountManager;
     this.intervalMs = intervalMs;
     this.probeFn = probeFn;
+    this.providerProbeFn = providerProbeFn;
     this.timeoutMs = timeoutMs;
     this.log = log;
     this.timer = null;
@@ -49,15 +50,21 @@ export class Prober {
     if (this._inflight) { try { await this._inflight; } catch { /* swallow */ } }
   }
 
-  /** Probe every OAuth account once. Overlapping cycles are skipped. The active
-   *  cycle is tracked on `_inflight` so stop() can await it. */
+  /** Probe every OAuth account (usage endpoint) and every provider account with a
+   *  pollable/known quota source (z.ai monitor; Kimi → console-only marker) once.
+   *  Overlapping cycles are skipped. The active cycle is tracked on `_inflight` so
+   *  stop() can await it. */
   probeAll() {
     if (this._running) return this._inflight || Promise.resolve();
     this._running = true;
     this._inflight = (async () => {
       try {
-        const accounts = this.am.accounts.filter(a => a.type === 'oauth' && a.credential);
-        await Promise.all(accounts.map(a => this.probeOne(a)));
+        const oauth = this.am.accounts.filter(a => a.type === 'oauth' && a.credential);
+        const providers = this.am.accounts.filter(a => a.type === 'provider' && a.credential);
+        await Promise.all([
+          ...oauth.map(a => this.probeOne(a)),
+          ...providers.map(a => this.probeProvider(a)),
+        ]);
       } finally {
         this._running = false;
         this._inflight = null;
@@ -66,6 +73,16 @@ export class Prober {
     return this._inflight;
   }
 
+  /** Probe one PROVIDER account. Provider tokens are static API keys (no OAuth
+   *  refresh). Best-effort; never throws. */
+  async probeProvider(account) {
+    try {
+      const usage = await this._withTimeout(this.providerProbeFn(account));
+      if (!usage) return; // timed out — try again next cycle
+      this.am.applyProviderUsage(account.index, usage);
+    } catch { /* best-effort; never let a probe throw */ }
+  }
+
   async probeOne(account) {
     try {
       await this.am.ensureTokenFresh(account.index);

package/src/tui.js

@@ -984,7 +984,7 @@ export class TUI {
     status = rpad(status, 13);
 
     if (a.type === 'provider') {
-      return this._renderProviderAcct(sel, cur, name, type, status, a);
+      return this._renderProviderAcct(sel, cur, name, type, status, a, bw, showBoth);
     }
 
     // Quota ratios — prefer unified (Claude Max), fall back to standard (API key)
@@ -1013,25 +1013,53 @@ export class TUI {
     }
     const weekly = weeklyPolicyText(this.am, a);
     if (weekly) line += `  ${weekly}`;
-    // Per-model weekly caps (e.g. Fable maxed while the unified weekly still has
-    // headroom) — surfaced separately so a capped model on an otherwise-healthy
-    // account is visible, and routing away from it is explained.
+    // Per-model weekly caps (e.g. Fable, while the unified weekly still has
+    // headroom). Show the ACTUAL utilization — "Fable 90%" (yellow) while high but
+    // still usable, "Fable maxed" (red) ONLY at genuine exhaustion. This is the
+    // SAME predicate the router benches on (_scopedExhausted), so "maxed" renders
+    // iff the model is actually benched — 90%/critical is no longer mislabelled.
     const exhaustedFloor = this.am.scheduler?.weeklyExhaustedThreshold ?? 0.985;
-    const capped = Object.entries(q.scopedWeekly || {})
-      .filter(([, e]) => e && e.isActive !== false
-        && (e.severity === 'critical' || (e.utilization != null && e.utilization >= exhaustedFloor)))
-      .map(([fam]) => fam.charAt(0).toUpperCase() + fam.slice(1));
-    if (capped.length) line += `  ${red(`${capped.join(',')} maxed`)}`;
+    const reserveFloor = this.am.scheduler?.weeklyReserveThreshold ?? 0.85;
+    const scopedTags = [];
+    for (const [fam, e] of Object.entries(q.scopedWeekly || {})) {
+      if (!e || e.isActive === false || e.utilization == null) continue;
+      if (e.utilization < reserveFloor) continue;
+      const Fam = fam.charAt(0).toUpperCase() + fam.slice(1);
+      scopedTags.push(e.utilization >= exhaustedFloor
+        ? red(`${Fam} maxed`)
+        : yellow(`${Fam} ${Math.round(e.utilization * 100)}%`));
+    }
+    if (scopedTags.length) line += `  ${scopedTags.join(' ')}`;
+    // Freshness: scoped caps are refreshed ONLY by the background probe (response
+    // headers don't carry them). If the probe has gone stale (> 2× interval since
+    // last success), say so rather than imply the last-known value is current.
+    if (this.am._quotaProbeStale?.(a)) line += `  ${dim('stale')}`;
     line += `  ${dim(loadText(this._accountLoad(a)))}`;
     return line;
   }
 
-  _renderProviderAcct(sel, cur, name, type, status, a) {
+  _renderProviderAcct(sel, cur, name, type, status, a, bw = 11, showBoth = true) {
     const completed = a.completedRequests || 0;
     const failed = a.failedRequests || 0;
     const active = a.inFlight || 0;
     const last = a.lastStatus ? `${statusColor(a.lastStatus)} ${formatMs(a.lastResponseMs)}` : '-';
     const q = a.quota || {};
+
+    // Quota segment. z.ai has a pollable monitor endpoint → real Ses/Wk token bars,
+    // same rendering as OAuth accounts. Kimi (Moonshot coding key) has NO pollable
+    // quota (web console only) → an honest label, never a fake bar. Fields are the
+    // SEPARATE provider* set, so a provider reading never leaks into an OAuth bar.
+    let quotaSeg = '';
+    if (q.providerSes != null || q.providerWk != null) {
+      quotaSeg = `  Ses ${bar(q.providerSes, bw, q.providerSesReset)}`;
+      if (showBoth && q.providerWk != null) quotaSeg += `  Wk ${bar(q.providerWk, bw, q.providerWkReset)}`;
+      if (this.am._quotaProbeStale?.(a)) quotaSeg += `  ${dim('stale')}`;
+    } else if (q.providerQuotaSource === 'console-only') {
+      quotaSeg = `  ${dim('Quota console-only')}`;
+    } else if (q.providerQuotaSource === 'zai') {
+      quotaSeg = `  ${dim('Ses/Wk probing')}`;
+    }
+
     let limit = '';
     if (q.genericLimit != null && q.genericRemaining != null) {
       const used = q.genericLimit - q.genericRemaining;
@@ -1039,7 +1067,7 @@ export class TUI {
       limit = `  Lim ${used}/${q.genericLimit}${reset ? ` ${reset}` : ''}`;
     }
     const err = a.lastError ? `  Err ${String(a.lastError).slice(0, 18)}` : '';
-    return ` ${sel}${cur} ${name} ${type} ${status} Act ${String(active).padStart(2)}  OK ${String(completed).padStart(3)}  Fail ${String(failed).padStart(2)}  Last ${last}  ${dim(loadText(this._accountLoad(a)))}${limit}${err}`;
+    return ` ${sel}${cur} ${name} ${type} ${status} Act ${String(active).padStart(2)}  OK ${String(completed).padStart(3)}  Fail ${String(failed).padStart(2)}  Last ${last}${quotaSeg}  ${dim(loadText(this._accountLoad(a)))}${limit}${err}`;
   }
 
   _accountLoad(account) {