@amir83nasr/10router 0.1.0

package/src/sse/services/auth.ts

@@ -0,0 +1,485 @@
+import {
+  getProviderConnections,
+  validateApiKey,
+  updateProviderConnection,
+  getSettings,
+  getProxyPools,
+} from "@/lib/localDb";
+import { resolveConnectionProxyConfig, pickProxyPoolId } from "@/lib/network/connectionProxy";
+import {
+  formatRetryAfter,
+  checkFallbackError,
+  isModelLockActive,
+  buildModelLockUpdate,
+  getEarliestModelLockUntil,
+} from "open-sse/services/accountFallback.js";
+import { MAX_RATE_LIMIT_COOLDOWN_MS } from "open-sse/config/errorConfig.js";
+import { resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers.js";
+import { getAntigravityQuotaCache } from "./antigravityQuota";
+import { getZedQuotaCache } from "./zedQuota";
+import * as log from "../utils/logger";
+
+// Mutex to prevent race conditions during account selection
+let selectionMutex: Promise<void> = Promise.resolve();
+
+const GITHUB_MONTHLY_USAGE_LIMIT = "you've reached your additional usage limit for your plan";
+
+function githubMonthlyResetMs(
+  status: string | number,
+  errorText: string | null,
+  provider: string | null,
+): number | null {
+  if (resolveProviderId(provider as string) !== "github" || Number(status) !== 402) return null;
+  if (
+    !String(errorText || "")
+      .toLowerCase()
+      .includes(GITHUB_MONTHLY_USAGE_LIMIT)
+  )
+    return null;
+  const now = new Date();
+  return Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1);
+}
+
+/**
+ * Get provider credentials from localDb
+ * Filters out unavailable accounts and returns the selected account based on strategy
+ * @param {string} provider - Provider name
+ * @param {Set<string>|string|null} excludeConnectionIds - Connection ID(s) to exclude (for retry with next account)
+ * @param {string|null} model - Model name for per-model rate limit filtering
+ */
+export async function getProviderCredentials(
+  provider: string,
+  excludeConnectionIds: Set<string> | string | null = null,
+  model: string | null = null,
+  options: { preferredConnectionId?: string } = {},
+): Promise<any> {
+  // Normalize to Set for consistent handling
+  const excludeSet =
+    excludeConnectionIds instanceof Set
+      ? excludeConnectionIds
+      : excludeConnectionIds
+        ? new Set([excludeConnectionIds])
+        : new Set();
+  const preferredConnectionId = options?.preferredConnectionId || null;
+  // Acquire mutex to prevent race conditions
+  const currentMutex = selectionMutex;
+  let resolveMutex: (() => void) | undefined;
+  selectionMutex = new Promise<void>((resolve) => {
+    resolveMutex = resolve;
+  });
+
+  try {
+    await currentMutex;
+
+    // Resolve alias to provider ID (e.g., "kc" -> "kilocode")
+    const providerId = resolveProviderId(provider);
+
+    // Inject a virtual connection for no-auth free providers (with optional proxy pool from settings)
+    if (FREE_PROVIDERS[providerId]?.noAuth) {
+      const settings = await getSettings();
+      const override = (settings.providerStrategies || {})[providerId] || {};
+      const strategy = override.rotateStrategy || "none";
+      let pickedId = override.proxyPoolId || null;
+      if (strategy !== "none") {
+        const allPools = await getProxyPools({ isActive: true });
+        const poolIds = allPools.filter((p) => p.proxyUrl).map((p: any) => p.id);
+        pickedId = pickProxyPoolId(poolIds, strategy, providerId);
+      }
+      const resolvedProxy = await resolveConnectionProxyConfig({ proxyPoolId: pickedId || "" });
+      return {
+        id: "noauth",
+        connectionName: "Public",
+        isActive: true,
+        accessToken: "public",
+        providerSpecificData: {
+          connectionProxyEnabled: resolvedProxy.connectionProxyEnabled,
+          connectionProxyUrl: resolvedProxy.connectionProxyUrl,
+          connectionNoProxy: resolvedProxy.connectionNoProxy,
+          connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
+          vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
+          strictProxy: resolvedProxy.strictProxy === true,
+        },
+      };
+    }
+
+    const connections = await getProviderConnections({ provider: providerId, isActive: true });
+    log.debug(
+      "AUTH",
+      `${provider} | total connections: ${connections.length}, excludeIds: ${excludeSet.size > 0 ? [...excludeSet].join(",") : "none"}, model: ${model || "any"}`,
+    );
+
+    if (connections.length === 0) {
+      log.warn("AUTH", `No credentials for ${provider}`);
+      return null;
+    }
+
+    // Antigravity quota cache is lazy: only populated after that account returns 409/429.
+    const isAntigravity = providerId === "antigravity";
+    const antigravityQuotaCache = isAntigravity && model ? getAntigravityQuotaCache() : null;
+    // Zed monthly-quota cache is lazy: only populated after that account returns 402/403/429.
+    const isZed = providerId === "zed";
+    const zedQuotaCache = isZed ? getZedQuotaCache() : null;
+
+    // Filter out model-locked, excluded, and Antigravity quota-exhausted connections.
+    const availableConnections = connections.filter((c: any) => {
+      if (excludeSet.has(c.id)) return false;
+      if (isModelLockActive(c, model)) return false;
+      // Antigravity: skip if live quota exhausted for this model
+      if (isAntigravity && model && antigravityQuotaCache) {
+        const quota = antigravityQuotaCache.get(c.id)?.[model];
+        if (
+          quota &&
+          quota.remainingPercentage <= 0 &&
+          quota.resetAt &&
+          new Date(quota.resetAt).getTime() > Date.now()
+        ) {
+          const account = c.id?.slice(0, 8) || "unknown";
+          log.info(
+            "AG_QUOTA",
+            `${account} | CACHE_BLOCK ${model} — skip upstream until ${quota.resetAt}`,
+          );
+          return false;
+        }
+      }
+      // Zed: skip if monthly quota exhausted for this account (account-level,
+      // not per-model — the quota rows are edit_predictions / model_requests).
+      if (isZed && zedQuotaCache) {
+        const blocked = zedQuotaCache.get(c.id);
+        if (
+          blocked &&
+          (blocked.remainingPercentage ?? 0) <= 0 &&
+          blocked.resetAt &&
+          new Date(blocked.resetAt).getTime() > Date.now()
+        ) {
+          const account = c.id?.slice(0, 8) || "unknown";
+          log.info(
+            "ZED_QUOTA",
+            `${account} | CACHE_BLOCK — monthly quota exhausted until ${blocked.resetAt}`,
+          );
+          return false;
+        }
+      }
+      return true;
+    });
+
+    log.debug(
+      "AUTH",
+      `${provider} | available: ${availableConnections.length}/${connections.length}`,
+    );
+    connections.forEach((c: any) => {
+      const excluded = excludeSet.has(c.id);
+      const locked = isModelLockActive(c, model);
+      if (excluded || locked) {
+        const lockUntil = getEarliestModelLockUntil(c);
+        log.debug(
+          "AUTH",
+          `  → ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""}`,
+        );
+      }
+    });
+
+    if (availableConnections.length === 0) {
+      // Find earliest persistent lock or lazy Antigravity quota-cache reset for retry timing.
+      const lockedConns = connections.filter((c: any) => isModelLockActive(c, model));
+      const expiries: any[] = lockedConns
+        .map((c: any) => getEarliestModelLockUntil(c))
+        .filter(Boolean);
+      if (isAntigravity && model && antigravityQuotaCache) {
+        connections.forEach((c: any) => {
+          const resetAt = antigravityQuotaCache.get(c.id)?.[model]?.resetAt;
+          if (resetAt && new Date(resetAt).getTime() > Date.now()) expiries.push(resetAt);
+        });
+      }
+      if (isZed && zedQuotaCache) {
+        connections.forEach((c: any) => {
+          const resetAt = zedQuotaCache.get(c.id)?.resetAt;
+          if (resetAt && new Date(resetAt).getTime() > Date.now()) expiries.push(resetAt);
+        });
+      }
+      const earliest = expiries.sort()[0] || null;
+      if (earliest) {
+        const earliestConn = lockedConns[0];
+        log.warn(
+          "AUTH",
+          `${provider} | all ${connections.length} accounts locked for ${model || "all"} (${formatRetryAfter(earliest)}) | lastError=${earliestConn?.lastError?.slice(0, 50)}`,
+        );
+        return {
+          allRateLimited: true,
+          retryAfter: earliest,
+          retryAfterHuman: formatRetryAfter(earliest),
+          lastError: earliestConn?.lastError || null,
+          lastErrorCode: earliestConn?.errorCode || null,
+        };
+      }
+      log.warn("AUTH", `${provider} | all ${connections.length} accounts unavailable`);
+      return null;
+    }
+
+    const settings = await getSettings();
+    // Per-provider strategy overrides global setting
+    const providerOverride = (settings.providerStrategies || {})[providerId] || {};
+    const strategy = providerOverride.fallbackStrategy || settings.fallbackStrategy || "fill-first";
+
+    let connection: any;
+    // Pin to preferred connection if specified and available
+    if (preferredConnectionId) {
+      connection = availableConnections.find((c: any) => c.id === preferredConnectionId);
+      if (connection) {
+        log.info(
+          "AUTH",
+          `${provider} | pinned to ${connection.id?.slice(0, 8)} (${connection.name || connection.email || "unnamed"})`,
+        );
+      }
+    }
+    if (connection) {
+      // skip strategy
+    } else if (strategy === "round-robin") {
+      const stickyLimit =
+        providerOverride.stickyRoundRobinLimit || settings.stickyRoundRobinLimit || 3;
+      // Bound lastUsed overlay scan to a 2-day window — stale timestamps
+      // outside it are treated as never-used so old rows can't pin ordering.
+      const twoDaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000;
+      const lastUsedMs = (c: any) => {
+        const t = c?.lastUsedAt ? new Date(c.lastUsedAt).getTime() : NaN;
+        return Number.isFinite(t) && t >= twoDaysAgo ? t : NaN;
+      };
+
+      // Sort by lastUsed (most recent first) to find current candidate
+      const byRecency = [...availableConnections].sort((a: any, b: any) => {
+        const at = lastUsedMs(a);
+        const bt = lastUsedMs(b);
+        if (Number.isNaN(at) && Number.isNaN(bt)) return (a.priority || 999) - (b.priority || 999);
+        if (Number.isNaN(at)) return 1;
+        if (Number.isNaN(bt)) return -1;
+        return bt - at;
+      });
+
+      const current = byRecency[0];
+      const currentCount = current?.consecutiveUseCount || 0;
+
+      if (current && current.lastUsedAt && currentCount < stickyLimit) {
+        // Stay with current account
+        connection = current;
+        // Update lastUsedAt and increment count (await to ensure persistence)
+        await updateProviderConnection(connection.id, {
+          lastUsedAt: new Date().toISOString(),
+          consecutiveUseCount: (connection.consecutiveUseCount || 0) + 1,
+        });
+      } else {
+        // Pick the least recently used (excluding current if possible)
+        const sortedByOldest = [...availableConnections].sort((a: any, b: any) => {
+          const at = lastUsedMs(a);
+          const bt = lastUsedMs(b);
+          if (Number.isNaN(at) && Number.isNaN(bt))
+            return (a.priority || 999) - (b.priority || 999);
+          if (Number.isNaN(at)) return -1;
+          if (Number.isNaN(bt)) return 1;
+          return at - bt;
+        });
+
+        connection = sortedByOldest[0];
+
+        // Update lastUsedAt and reset count to 1 (await to ensure persistence)
+        await updateProviderConnection(connection.id, {
+          lastUsedAt: new Date().toISOString(),
+          consecutiveUseCount: 1,
+        });
+      }
+    } else {
+      // Default: fill-first (already sorted by priority in getProviderConnections)
+      connection = availableConnections[0];
+    }
+
+    const resolvedProxy = await resolveConnectionProxyConfig(connection.providerSpecificData || {});
+
+    return {
+      authType: connection.authType,
+      apiKey: connection.apiKey,
+      accessToken: connection.accessToken,
+      refreshToken: connection.refreshToken,
+      idToken: connection.idToken,
+      expiresAt: connection.expiresAt,
+      expiresIn: connection.expiresIn,
+      lastRefreshAt: connection.lastRefreshAt,
+      projectId: connection.projectId,
+      connectionName:
+        connection.displayName || connection.name || connection.email || connection.id,
+      copilotToken: connection.providerSpecificData?.copilotToken,
+      providerSpecificData: {
+        ...(connection.providerSpecificData || {}),
+        connectionProxyEnabled: resolvedProxy.connectionProxyEnabled,
+        connectionProxyUrl: resolvedProxy.connectionProxyUrl,
+        connectionNoProxy: resolvedProxy.connectionNoProxy,
+        connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
+        vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
+        strictProxy: resolvedProxy.strictProxy === true,
+      },
+      connectionId: connection.id,
+      // Include current status for optimization check
+      testStatus: connection.testStatus,
+      lastError: connection.lastError,
+      // Pass full connection for clearAccountError to read modelLock_* keys
+      _connection: connection,
+    };
+  } finally {
+    if (resolveMutex) resolveMutex();
+  }
+}
+
+/**
+ * Mark account+model as unavailable — locks modelLock_${model} in DB.
+ * All errors (429, 401, 5xx, etc.) lock per model, not per account.
+ * @param {string} connectionId
+ * @param {number} status - HTTP status code from upstream
+ * @param {string} errorText
+ * @param {string|null} provider
+ * @param {string|null} model - The specific model that triggered the error
+ * @returns {{ shouldFallback: boolean, cooldownMs: number }}
+ */
+export async function markAccountUnavailable(
+  connectionId: string,
+  status: number,
+  errorText: string,
+  provider: string | null = null,
+  model: string | null = null,
+  resetsAtMs: number | null = null,
+): Promise<{ shouldFallback: boolean; cooldownMs: number }> {
+  if (!connectionId || connectionId === "noauth") return { shouldFallback: false, cooldownMs: 0 };
+  const connections = await getProviderConnections({ provider });
+  const conn = connections.find((c: any) => c.id === connectionId);
+  const backoffLevel = conn?.backoffLevel || 0;
+
+  // GitHub premium-request exhaustion is account-wide until the next UTC month.
+  const githubResetAtMs = githubMonthlyResetMs(status, errorText, provider);
+
+  // Provider-specific precise cooldown (e.g. codex usage_limit_reached resets_at) overrides backoff
+  let shouldFallback: boolean, cooldownMs: number, newBackoffLevel: number;
+  if (githubResetAtMs) {
+    shouldFallback = true;
+    cooldownMs = githubResetAtMs - Date.now();
+    newBackoffLevel = 0;
+  } else if (resetsAtMs && resetsAtMs > Date.now()) {
+    shouldFallback = true;
+    // Antigravity quota API provides exact per-model resetAt. Do not truncate it.
+    cooldownMs =
+      resolveProviderId(provider as string) === "antigravity"
+        ? resetsAtMs - Date.now()
+        : Math.min(resetsAtMs - Date.now(), MAX_RATE_LIMIT_COOLDOWN_MS);
+    newBackoffLevel = 0;
+  } else {
+    ({ shouldFallback, cooldownMs, newBackoffLevel } = checkFallbackError(
+      status,
+      errorText,
+      backoffLevel,
+    ));
+  }
+  if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
+
+  const reason = typeof errorText === "string" ? errorText.slice(0, 200) : "Provider error";
+  const lockUpdate = buildModelLockUpdate(githubResetAtMs ? null : model, cooldownMs);
+
+  await updateProviderConnection(connectionId, {
+    ...lockUpdate,
+    testStatus: "unavailable",
+    lastError: reason,
+    errorCode: status,
+    lastErrorAt: new Date().toISOString(),
+    backoffLevel: newBackoffLevel ?? backoffLevel,
+  });
+
+  const lockKey = Object.keys(lockUpdate)[0];
+  const connName = conn?.displayName || conn?.name || conn?.email || connectionId.slice(0, 8);
+  log.warn(
+    "AUTH",
+    `${connName} locked ${lockKey} for ${Math.round(cooldownMs / 1000)}s [${status}]`,
+  );
+
+  if (provider && status && reason) {
+    log.error(provider, `[${status}]: ${reason}`);
+  }
+
+  return { shouldFallback: true, cooldownMs };
+}
+
+/**
+ * Clear account error status on successful request.
+ * - Clears modelLock_${model} (the model that just succeeded)
+ * - Lazy-cleans any other expired modelLock_* keys
+ * - Resets error state only if no active locks remain
+ * @param {string} connectionId
+ * @param {object} currentConnection - credentials object (has _connection) or raw connection
+ * @param {string|null} model - model that succeeded
+ */
+export async function clearAccountError(
+  connectionId: string,
+  currentConnection: any,
+  model: string | null = null,
+): Promise<void> {
+  if (!connectionId || connectionId === "noauth") return;
+  const conn = currentConnection._connection || currentConnection;
+  const now = Date.now();
+  const allLockKeys = Object.keys(conn).filter((k) => k.startsWith("modelLock_"));
+
+  if (!conn.testStatus && !conn.lastError && allLockKeys.length === 0) return;
+
+  // Keys to clear: current model's lock + all expired locks
+  const keysToClear = allLockKeys.filter((k) => {
+    if (model && k === `modelLock_${model}`) return true; // succeeded model
+    if (model && k === "modelLock___all") return true; // account-level lock
+    const expiry = conn[k];
+    return expiry && new Date(expiry).getTime() <= now; // expired
+  });
+
+  if (keysToClear.length === 0 && conn.testStatus !== "unavailable" && !conn.lastError) return;
+
+  // Check if any active locks remain after clearing
+  const remainingActiveLocks = allLockKeys.filter((k) => {
+    if (keysToClear.includes(k)) return false;
+    const expiry = conn[k];
+    return expiry && new Date(expiry).getTime() > now;
+  });
+
+  const clearObj: Record<string, any> = Object.fromEntries(keysToClear.map((k) => [k, null]));
+
+  // Only reset error state if no active locks remain
+  if (remainingActiveLocks.length === 0) {
+    Object.assign(clearObj, {
+      testStatus: "active",
+      lastError: null,
+      errorCode: null,
+      lastErrorAt: null,
+      backoffLevel: 0,
+    });
+  }
+
+  await updateProviderConnection(connectionId, clearObj);
+}
+
+/**
+ * Extract API key from request headers
+ */
+export function extractApiKey(request: {
+  headers: { get(name: string): string | null };
+}): string | null {
+  // Check Authorization header first
+  const authHeader = request.headers.get("Authorization");
+  if (authHeader?.startsWith("Bearer ")) {
+    return authHeader.slice(7);
+  }
+
+  // Check Anthropic x-api-key header
+  const xApiKey = request.headers.get("x-api-key");
+  if (xApiKey) {
+    return xApiKey;
+  }
+
+  return null;
+}
+
+/**
+ * Validate API key (optional - for local use can skip)
+ */
+export async function isValidApiKey(apiKey: string): Promise<boolean> {
+  if (!apiKey) return false;
+  return await validateApiKey(apiKey);
+}

package/src/sse/services/backgroundTokenRefresh.ts

@@ -0,0 +1,190 @@
+// Background proactive OAuth token refresh — independent of inbound requests.
+// Fail-open everywhere: tick errors and per-connection failures never kill the interval.
+
+import * as log from "../utils/logger";
+import { getRefreshLeadMs } from "open-sse/services/tokenRefresh.js";
+import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager.js";
+
+/** Refresh when expiry is within 30 minutes (or the provider on-request lead, whichever larger). */
+export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000;
+const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
+const INITIAL_DELAY_MS = 10 * 1000;
+const SENSITIVE_PROVIDERS = new Set(["antigravity"]);
+
+let started = false;
+let intervalHandle: ReturnType<typeof setInterval> | null = null;
+let initialTimeoutHandle: ReturnType<typeof setTimeout> | null = null;
+let tickRunning = false;
+
+function isTruthyEnv(value: string | undefined | null): boolean {
+  if (value == null || value === "") return false;
+  const v = String(value).trim().toLowerCase();
+  return v === "1" || v === "true" || v === "yes" || v === "on";
+}
+
+function isNonServerRuntime(): boolean {
+  if (typeof window !== "undefined") return true;
+  const phase = process.env.NEXT_PHASE || "";
+  if (phase === "phase-production-build" || phase === "phase-export" || phase === "phase-static") {
+    return true;
+  }
+  // Next.js build / static generation markers
+  if (process.env.NEXT_RUNTIME === "edge") return true;
+  return false;
+}
+
+/**
+ * Pure selection: OAuth connections with a refreshToken whose access token
+ * expires within max(provider on-request lead, BACKGROUND_REFRESH_LEAD_MS).
+ *
+ * @param {Array<object>} connections
+ * @param {number} [nowMs]
+ * @returns {Array<object>}
+ */
+export function selectConnectionsNeedingRefresh<
+  T extends { authType?: any; refreshToken?: any; provider?: any },
+>(connections: T[], nowMs: number = Date.now()): T[] {
+  if (!Array.isArray(connections) || connections.length === 0) return [];
+
+  const out: T[] = [];
+  for (const conn of connections) {
+    if (!conn) continue;
+
+    const authType = String(conn.authType || "")
+      .toLowerCase()
+      .replace(/_/g, "");
+    if (authType !== "oauth") continue;
+    if (!conn.refreshToken) continue;
+
+    const expiresAtMs = getCredentialExpiryMs(conn);
+    if (expiresAtMs === null) continue;
+
+    const providerLead = getRefreshLeadMs(conn.provider);
+    const leadMs = Math.max(
+      Number.isFinite(providerLead) ? providerLead : 0,
+      BACKGROUND_REFRESH_LEAD_MS,
+    );
+
+    if (expiresAtMs - nowMs < leadMs) {
+      out.push(conn);
+    }
+  }
+  return out;
+}
+
+async function loadActiveConnections(): Promise<any[]> {
+  // Dynamic import avoids circular load with db / app graph at module eval time.
+  const { getProviderConnections } = await import("../../lib/db/repos/connectionsRepo");
+  return getProviderConnections({ isActive: true });
+}
+
+async function refreshOne(connection: any): Promise<any> {
+  const { checkAndRefreshToken } = await import("./tokenRefresh");
+  return checkAndRefreshToken(connection.provider, connection, { force: true });
+}
+
+type RefreshDeps = {
+  loadConnections?: () => Promise<any[]>;
+  refreshConnection?: (conn: any) => Promise<any>;
+  sleep?: (ms: number) => Promise<void>;
+};
+
+/**
+ * One scheduler tick. Fail-open at top level and per connection.
+ * @param {{ loadConnections?: Function, refreshConnection?: Function }} [deps]
+ */
+export async function runBackgroundTokenRefreshTick(deps: RefreshDeps = {}): Promise<void> {
+  if (tickRunning) return;
+  tickRunning = true;
+  try {
+    const load = deps.loadConnections || loadActiveConnections;
+    const refresh = deps.refreshConnection || refreshOne;
+    const sleep = deps.sleep || ((ms: number) => new Promise((res) => setTimeout(res, ms)));
+
+    const connections = await load();
+    const due = selectConnectionsNeedingRefresh(connections, Date.now());
+
+    if (due.length === 0) return;
+
+    const baseSensitiveDelay = Number(process.env.BG_REFRESH_GOOGLE_DELAY_MS) || 12_000;
+    const baseNormalDelay = Number(process.env.BG_REFRESH_DELAY_MS) || 1_500;
+
+    for (let i = 0; i < due.length; i++) {
+      const conn = due[i];
+      try {
+        await refresh(conn);
+        log.info("BG_TOKEN_REFRESH", "Connection refresh finished", {
+          id: conn.id,
+          email: conn.email || conn.name || conn.id,
+          provider: conn.provider,
+        });
+      } catch (err: any) {
+        log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
+          id: conn?.id,
+          email: conn?.email || conn?.name || conn?.id,
+          provider: conn?.provider,
+          error: err?.message ?? String(err),
+        });
+      }
+
+      // Sequential delay between accounts to prevent bursting upstream providers (especially Google Cloud)
+      if (i < due.length - 1) {
+        const isSensitive = SENSITIVE_PROVIDERS.has(conn.provider);
+        const baseDelay = isSensitive ? baseSensitiveDelay : baseNormalDelay;
+        const jitter = isSensitive ? Math.floor(Math.random() * 4000) : 200;
+        await sleep(baseDelay + jitter);
+      }
+    }
+  } catch (err: any) {
+    log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", {
+      error: err?.message ?? String(err),
+    });
+  } finally {
+    tickRunning = false;
+  }
+}
+
+/**
+ * Start the background interval. Safe to call multiple times (no-op if already started).
+ * @param {{ intervalMs?: number }} [opts]
+ * @returns {boolean} true if started this call
+ */
+export function startBackgroundTokenRefresh({ intervalMs }: { intervalMs?: number } = {}): boolean {
+  if (started) return false;
+  if (isTruthyEnv(process.env.DISABLE_BACKGROUND_TOKEN_REFRESH)) return false;
+  if (isNonServerRuntime()) return false;
+
+  started = true;
+  const period = Number.isFinite(intervalMs) && intervalMs! > 0 ? intervalMs! : DEFAULT_INTERVAL_MS;
+
+  const safeTick = () => {
+    runBackgroundTokenRefreshTick().catch((err: any) => {
+      log.warn("BG_TOKEN_REFRESH", "Unhandled tick rejection (swallowed)", {
+        error: err?.message ?? String(err),
+      });
+    });
+  };
+
+  // First pass soon after boot so idle connections don't wait a full interval.
+  initialTimeoutHandle = setTimeout(safeTick, INITIAL_DELAY_MS);
+  if (initialTimeoutHandle.unref) initialTimeoutHandle.unref();
+
+  intervalHandle = setInterval(safeTick, period);
+  if (intervalHandle.unref) intervalHandle.unref();
+
+  return true;
+}
+
+export function stopBackgroundTokenRefresh(): void {
+  if (initialTimeoutHandle) {
+    clearTimeout(initialTimeoutHandle);
+    initialTimeoutHandle = null;
+  }
+  if (intervalHandle) {
+    clearInterval(intervalHandle);
+    intervalHandle = null;
+  }
+  if (started) {
+    started = false;
+  }
+}