maxpool 1.24.8 → 1.24.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.24.8",
+  "version": "1.24.10",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -219,6 +219,15 @@ const DEFAULT_SCHEDULER = {
   // undiscounted, as do the hard gates (safetyMaxActivePerAccount, usage cap,
   // cooldowns) — those are the anti-dogpile backstops, not price signals.
   fastRefillDiscount: 0.6,        // 0 = off (full cost), 0.6 = discount up to 60% of the two balancing terms
+  // RESET-AWARE PREFERENCE (2026-10-08, owner): an account holding a usable, unexpired
+  // limit reset (Claude cedar_ember grant / z.ai card) has capacity its siblings lack —
+  // when it reaches its wall the prober redeems the reset and it refills (reset-policy
+  // R1). So its BALANCING costs (pace, utilization, spread) are discounted: it soaks
+  // more traffic and spends the reset instead of letting it expire. The SAFETY terms
+  // (capPenalty, reserve/critical, ramp, failure) and every hard gate stay undiscounted,
+  // so it is never dogpiled and never preferred over an unhealthy-looking pick.
+  // 0 = off. 0.35 ≈ a 1.5x share at otherwise-equal health.
+  resetBoostDiscount: 0.35,
   fastRefillFadeUtil: 0.65,       // discount reaches 0 at this session utilization (weeklySoftThreshold)
   recoveryRampWeight: 4,          // decaying penalty applied to a just-recovered account
   recoveryRampMs: 5 * 60_000,     // how long the post-recovery ramp lasts
@@ -2770,6 +2779,31 @@ export class AccountManager {
    *   - ramp:        ease a just-recovered account back in instead of slamming it
    *   - failures:    direct per-account backoff after errors
    */
+  /**
+   * Does this account hold a banked reset that can actually lift its BINDING limit?
+   * Claude grants: usable, not paused, unexpired. z.ai cards: unexpired five-hour cards
+   * always count; weekly cards count only when the account HAS a weekly limit (a legacy
+   * plan has none — z.ai support, 2026-10-03 — so a weekly card there lifts nothing).
+   * Disabled accounts never count (owner gate R0). Display state only otherwise.
+   */
+  _holdsUsableReset(account, now = Date.now()) {
+    if (!account || account.enabled === false) return false;
+    const grants = account.resetGrants?.grants || [];
+    if (grants.some(g => g && g.usableNow !== false && !g.paused && (g.endsAt == null || g.endsAt > now))) return true;
+    const cards = account.resetCards;
+    if (!cards) return false;
+    const live = c => c && !c.expired && (c.expiresAt == null || c.expiresAt > now);
+    if ((cards.fiveHour || []).some(live)) return true;
+    const hasWeeklyLimit = account.quota?.providerWk != null || account.quota?.providerWkReset != null;
+    return hasWeeklyLimit && (cards.weekly || []).some(live);
+  }
+
+  _resetMultiplier(account, now = Date.now()) {
+    const d = Number(this.scheduler.resetBoostDiscount) || 0;
+    if (d <= 0) return 1;
+    return this._holdsUsableReset(account, now) ? 1 - Math.min(d, 0.9) : 1;
+  }
+
   _scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
     const now = ctx?.now ?? Date.now();
     const reqWeight = Math.max(1, requestInfo.weight || 1);
@@ -2826,7 +2860,9 @@ export class AccountManager {
     // FAST-REFILL DISCOUNT: applied to the pace and utilization terms for an
     // account whose only cap is a fast-refilling session window — see the
     // DEFAULT_SCHEDULER block for the full rationale (refillMult computed above).
-    const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight * refillMult;
+    // RESET-AWARE: discounts the three BALANCING terms only (see resetBoostDiscount).
+    const resetMult = this._resetMultiplier(account, now);
+    const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight * refillMult * resetMult;
 
     // RAW utilization cost — direct, not pace-adjusted. The pace cost above discounts
     // by how far into the window you are, so an account at 80% with 2h left is only
@@ -2834,7 +2870,7 @@ export class AccountManager {
     // benching, but wrong for load balancing: an account at 80% should be clearly less
     // attractive than one at 10% even if both are "on pace". Measured 2026-08-10: cc at
     // 80% scored 52.30 vs glm at 10% at 52.15 — a 0.15 gap drowned by round-robin.
-    const utilizationCost = this._rawUtilization(account) * this.scheduler.utilizationWeight * refillMult;
+    const utilizationCost = this._rawUtilization(account) * this.scheduler.utilizationWeight * refillMult * resetMult;
 
     // Per-model weekly de-preference: an account whose scoped weekly for THIS
     // request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
@@ -2860,7 +2896,7 @@ export class AccountManager {
     // 2.3x early in its window, fading to parity at the same ses 0.65 the multiplier
     // already uses. A multiplier of 1 makes every discounted term byte-identical to
     // pre-2026-08-25 behaviour; reserve/critical/ramp/failure are never discounted.
-    const spread = share * this.scheduler.spreadShareWeight * refillMult;
+    const spread = share * this.scheduler.spreadShareWeight * refillMult * resetMult;
 
     const ramp = this._recoveryRamp(account, now);
     const reserveCost = this._reserveCost(account, now, weeklyState);
@@ -3344,7 +3380,13 @@ export class AccountManager {
     }
 
     q.lastProbeOkAt = Date.now();
-    // A successful probe clears any recorded failure — freshness confirmed.
+    // A successful probe clears any recorded failure — freshness confirmed — including
+    // the org-403 streak (a healthy probe proves the org still accepts OAuth).
+    q.consecutiveOrgGone = 0;
+    if (account.subscriptionGone) {
+      account.subscriptionGone = false;
+      console.log(`[Maxpool] "${account.name}" subscription check passed — un-benched (the org-403 was transient)`);
+    }
     q.lastProbeError = null;
     q.lastProbeErrorAt = null;
     q.lastProbeErrorStatus = null;
@@ -3381,6 +3423,20 @@ export class AccountManager {
     // worth having; the diagnosis that motivated it was wrong.)
     q.consecutiveProbeFailures = (q.consecutiveProbeFailures || 0) + 1;
     const n = q.consecutiveProbeFailures;
+    // ORG-403 LATCH REQUIRES ITS OWN STRIKES (2026-10-08, false "no sub" on
+    // max@gomokka.com). `n` mixes EVERY probe failure — network timeouts first among
+    // them — so after a flapping-network morning one genuine org-403 can arrive with
+    // n already >= 3 and bench a LIVE subscription (measured: latch fired at "x6"
+    // with exactly one org-403 in the whole week's log; 104 successful requests the
+    // same day). The subscription verdict must count only consecutive org-403s.
+    const isOrgGone = status === 403
+      && /not allowed for this organization|disabled.*subscription|subscription.*disabled|organization has disabled/i.test(String(message));
+    if (isOrgGone) {
+      q.consecutiveOrgGone = (q.consecutiveOrgGone || 0) + 1;
+    } else {
+      q.consecutiveOrgGone = 0;   // any OTHER outcome breaks the org-gone streak
+    }
+    const orgN = q.consecutiveOrgGone;
     // A SUSTAINED ORG-403 is the subscription being gone (canceled Max plan lapsing
     // server-side — measured 2026-09-18/20: "OAuth authentication is currently not
     // allowed for this organization" on 2solarmax@ and privacy@, both canceled Sep 16).
@@ -3388,11 +3444,10 @@ export class AccountManager {
     // pure waste. Latch subscriptionGone (3 strikes, like the 401 rule): the prober
     // skips the account, the TUI says why, and routing treats it as unavailable. A
     // successful re-auth clears it (updateAccountTokens).
-    if (status === 403
-      && /not allowed for this organization|disabled.*subscription|subscription.*disabled|organization has disabled/i.test(String(message))
-      && n >= 3 && !account.subscriptionGone) {
+    if (isOrgGone && orgN >= 3 && !account.subscriptionGone) {
       account.subscriptionGone = true;
-      console.error(`[Maxpool] "${account.name}" subscription disabled at the organization (HTTP 403 x${n}) — benching it. Re-enable after re-subscribing, or remove the account (a → d).`);
+      account._subRecheckAt = Date.now();   // first recheck one full window after latching
+      console.error(`[Maxpool] "${account.name}" subscription disabled at the organization (HTTP 403 x${orgN}) — benching it. Re-enable after re-subscribing, or remove the account (a → d).`);
     }
     // A SUSTAINED 401 is dead credentials, not a blip. Latch refreshDead so (a) the
     // prober stops re-POSTing a rejected token every 60s forever — measured 2026-08-10:
@@ -3438,7 +3493,7 @@ export class AccountManager {
   }
 
   /** Store the latest reset-card listing for an account (prober piggyback).
-   *  Cards are display/decision state only — routing never reads them. */
+   *  Read by redeem policy AND by _scoreAccount's reset-aware preference. */
   applyResetCards(accountIndex, cards) {
     const account = this.accounts[accountIndex];
     if (!account || !cards) return;
@@ -4009,7 +4064,7 @@ export class AccountManager {
     // A dead refresh token (invalid_grant) is PERMANENT until browser re-auth —
     // never auto-retry it. Without this the prober re-POSTs the rejected token every
     // ~60s forever (hammering Anthropic's OAuth endpoint). Cleared on re-login.
-    if (account.refreshDead || account.subscriptionGone) return false;
+    if (account.refreshDead || (account.subscriptionGone && !account._subRecheck)) return false;
 
     // A DISABLED account never spends its single-use refresh token. The prober still
     // READS its quota by design (you disable an exhausted account and still want to

package/src/prober.js

@@ -12,6 +12,10 @@ import { listResetCards, redeemResetCard } from './zai-reset-cards.js';
 import { listResetGrants, claimResetGrant } from './claude-reset-grants.js';
 import { decideZai, decideClaude } from './reset-policy.js';
 
+// How often a subscription-latched account is rechecked (one probe + at most one
+// token refresh per window — cheap, and bounded so a truly canceled plan isn't hammered).
+export const SUB_RECHECK_MS = 30 * 60_000;
+
 export class Prober {
   constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, grantsFn = listResetGrants, timeoutMs = 10_000, log = console.log, usageGapMs = null } = {}) {
     this.am = accountManager;
@@ -245,7 +249,19 @@ export class Prober {
     // endpoint 403s before answering anything, so probing is pure waste until the org
     // accepts OAuth again. Skipping also stops the every-60s hammer that ran 700+ times
     // on 2solarmax@ between 2026-09-18 and 09-20.
-    if (account.subscriptionGone) return { ok: false, status: 403 };
+    // SELF-CLEAR RECHECK (2026-10-08): the latch used to be terminal — the account was
+    // skipped here AND by ensureTokenFresh, so a single transient org-403 benched a live
+    // subscription until manual re-auth. Recheck it at a low cadence: a 200 clears the
+    // latch (applyUsageData), another org-403 keeps it benched.
+    if (account.subscriptionGone) {
+      // A latch set outside recordProbeError (state restore, tests, older builds) has no
+      // stamp: treat NOW as the latch moment so the first recheck waits a full window.
+      if (!account._subRecheckAt) account._subRecheckAt = Date.now();
+      const since = Date.now() - account._subRecheckAt;
+      if (since < SUB_RECHECK_MS) return { ok: false, status: 403 };
+      account._subRecheckAt = Date.now();
+      account._subRecheck = true;
+    }
     try {
       // Claude banked resets (cedar_ember) piggyback on the oauth poll — same
       // OWNER GATE as z.ai cards: a DISABLED account is never listed or claimed.
@@ -288,6 +304,8 @@ export class Prober {
     } catch (e) { // best-effort; never let a probe throw
       this.am.recordProbeError?.(account.index, e?.message || String(e), null);
       return { ok: false, status: null };
+    } finally {
+      account._subRecheck = false;
     }
   }
 

package/src/secret-resolver.js

@@ -9,6 +9,7 @@
  * broken provider degrades to "disabled" instead of crashing the proxy.
  */
 import { execFile } from 'node:child_process';
+import https from 'node:https';
 import { promisify } from 'node:util';
 import { readFileSync } from 'node:fs';
 import { homedir } from 'node:os';
@@ -57,12 +58,90 @@ function fromCache(secretName) {
   return (typeof v === 'string' && v.trim()) ? v.trim() : null;
 }
 
+// REST PATH FIRST (2026-10-08). `gcloud secrets versions access` hung on this machine
+// (>60s, 21 piled-up gcloud processes) while a plain Secret Manager REST call with the
+// same user's token answered in 0.75s. Every startup resolve AND every 5-minute
+// re-resolve went through the hung CLI, so all z.ai providers stayed
+// "secret-unresolved" until a manual restart happened to land on a good moment.
+// The token itself comes from `gcloud auth print-access-token` (fast — it reads the
+// stored credential, no Secret Manager round-trip) and is reused for ~45 min.
+let tokenMemo = { value: null, at: 0 };
+const TOKEN_TTL_MS = 45 * 60_000;
+
+async function accessToken(timeoutMs) {
+  if (tokenMemo.value && Date.now() - tokenMemo.at < TOKEN_TTL_MS) return tokenMemo.value;
+  const { stdout } = await execFileAsync('gcloud', ['auth', 'print-access-token'],
+    { timeout: Math.min(timeoutMs, 20_000), maxBuffer: 64 * 1024 });
+  const t = stdout.trim();
+  if (!t) throw new Error('empty token');
+  tokenMemo = { value: t, at: Date.now() };
+  return t;
+}
+
+// IPv4-PINNED GET (2026-10-08). On this network IPv6 to Google is black-holed:
+// secretmanager.googleapis.com advertises AAAA records, an IPv6 connect hangs until
+// timeout, and IPv4 answers fine (measured: `curl -6` 8s timeout, `curl -4` OK). gcloud's
+// own HTTP stack tries IPv6 first and HANGS — that is why every CLI resolve timed out.
+// `family: 4` makes this path immune; the CLI fallback below stays as a last resort.
+function ipv4Get(url, { headers, signal }) {
+  return new Promise((resolve, reject) => {
+    const req = https.get(url, { headers, family: 4, signal }, res => {
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => {
+        const body = Buffer.concat(chunks).toString('utf8');
+        resolve({ status: res.statusCode, ok: res.statusCode >= 200 && res.statusCode < 300, json: async () => JSON.parse(body) });
+      });
+      res.on('error', reject);
+    });
+    req.on('error', reject);
+  });
+}
+
+/** Test seam: inject a fetch + token source. */
+let restDeps = { fetch: ipv4Get, token: accessToken };
+export function __setRestDeps(d) { restDeps = { ...restDeps, ...d }; }
+export function __resetRestDeps() { restDeps = { fetch: ipv4Get, token: accessToken }; tokenMemo = { value: null, at: 0 }; }
+
+async function viaRest(secretName, project, timeoutMs) {
+  const tok = await restDeps.token(timeoutMs);
+  const url = `https://secretmanager.googleapis.com/v1/projects/${encodeURIComponent(project)}`
+    + `/secrets/${encodeURIComponent(secretName)}/versions/latest:access`;
+  const ctl = new AbortController();
+  const t = setTimeout(() => ctl.abort(), Math.min(timeoutMs, 15_000));
+  try {
+    const res = await restDeps.fetch(url, { headers: { Authorization: `Bearer ${tok}` }, signal: ctl.signal });
+    if (res.status === 401) { tokenMemo = { value: null, at: 0 }; return null; }  // stale token → CLI fallback
+    // 403/404: the server answered definitively — the CLI would get the same answer.
+    if (res.status === 403 || res.status === 404) return { value: null, definitive: true };
+    if (!res.ok) return null;
+    const j = await res.json();
+    const b64 = j?.payload?.data;
+    if (!b64) return { value: null, definitive: true };
+    const v = Buffer.from(b64, 'base64').toString('utf8').trim();
+    return { value: v || null, definitive: true };
+  } finally { clearTimeout(t); }
+}
+
 export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, timeoutMs = 45_000, useCache = true } = {}) {
   if (!secretName || typeof secretName !== 'string') return null;
   if (useCache) {
     const cached = fromCache(secretName);
     if (cached) return cached;
   }
+  // ONE time budget for the whole lookup: REST first, then the CLI with whatever is
+  // left. A definitive REST answer (the secret does not exist / no access) skips the
+  // CLI entirely — the CLI would only re-ask the same server, and on an IPv6-broken
+  // network it hangs for the full timeout anyway.
+  const deadline = Date.now() + timeoutMs;
+  try {
+    const r = await viaRest(secretName, project, timeoutMs);
+    if (r?.value) return r.value;
+    if (r?.definitive) return null;
+  } catch { /* transport failure — fall through to the CLI path */ }
+  const remaining = deadline - Date.now();
+  if (remaining < 1000) return null;
+  timeoutMs = remaining;
   try {
     const { stdout } = await execFileAsync(
       'gcloud',