@bitkyc08/opencodex 2.55.0 → 2.57.0

package/src/codex/routing/cooldown-math.ts

@@ -0,0 +1,285 @@
+import {
+  CODEX_EXHAUSTED_USAGE_PERCENT,
+  CODEX_UNKNOWN_USAGE_SCORE,
+  resetAtToMs,
+} from "../quota";
+import { isThirtyDayOnlyCodexPlan } from "../plan";
+import type { CodexQuotaScope } from "./health-store";
+import type { TransientProbeGrant } from "./thread-affinity";
+
+export const CODEX_DEFAULT_QUOTA_COOLDOWN_MS = 60_000;
+export const CODEX_MAX_QUOTA_COOLDOWN_MS = 24 * 60 * 60_000;
+/**
+ * A weekly/monthly quota `resetAt` announces when the window refreshes; it is not
+ * a "come back after this" directive like Retry-After. Plan quota routinely frees
+ * up long before the advertised reset, so cap reset-derived cooldowns far below
+ * the Retry-After ceiling (#433).
+ */
+export const CODEX_MAX_RESET_DERIVED_COOLDOWN_MS = 15 * 60_000;
+/**
+ * Ceiling on quota-refusal avoidance. Generous enough to cover a full five-hour burst window,
+ * tight enough that a weekly or monthly reset four days out cannot take an account out of
+ * rotation for the {@link CODEX_MAX_QUOTA_COOLDOWN_MS} day the Retry-After ceiling allows.
+ */
+export const CODEX_MAX_QUOTA_AVOID_MS = 6 * 60 * 60_000;
+/** Minimum gap between probe leases for one cooled-down account. */
+export const CODEX_QUOTA_PROBE_INTERVAL_MS = 5 * 60_000;
+export const CODEX_FAILURE_WINDOW_MS = 5 * 60_000;
+/**
+ * How recently a 100% burst reading must have been OBSERVED to exclude an account when it
+ * carries no reset timestamp (#3425). Deliberately far tighter than the 6h disk-hydration
+ * horizon in `quota.ts`: shorter than any plausible five-hour burst window, so a persisted
+ * reading can never strand a recovered account, and long enough that a snapshot taken at
+ * admission is still fresh when selection reads it.
+ */
+export const TERMINAL_SHORT_WINDOW_FRESHNESS_MS = 5 * 60_000;
+/** How long a transient failure keeps the account out of pool selection. */
+export const CODEX_TRANSIENT_SOFT_AVOID_MS = 30_000;
+export const CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS = [
+  CODEX_TRANSIENT_SOFT_AVOID_MS,
+  2 * 60_000,
+  10 * 60_000,
+  30 * 60_000,
+] as const;
+
+export type CodexUpstreamOutcome = number | "connect_error" | "timeout" | "connect_neutral";
+export type CodexUpstreamOutcomeClass = "success" | "credential"
+  | "workspace" | "quota" | "transient" | "caller" | "neutral" | "unknown";
+export type CodexCooldownSource = "retry-after" | "reset-derived" | "default";
+
+export type CodexUpstreamOutcomeMeta = {
+  retryAfter?: string | null;
+  resetAt?: unknown | unknown[];
+  now?: number;
+  /** (provider, host) ledger key for account-neutral reachability failures (#914). */
+  hostKey?: string;
+  /**
+   * Upstream denial evidence for a 403. A workspace/entitlement denial means the CREDENTIAL
+   * is fine and the account simply cannot reach this workspace, so it must not be quarantined
+   * for reauthentication (#1789). Absent evidence keeps the historical credential handling.
+   */
+  denial?: "workspace" | "entitlement";
+  /** Stable transport code recorded alongside a neutral host failure. */
+  lastFailureCode?: string;
+  /** Native model selected for this request; used only for confirmed scoped quotas. */
+  modelId?: string;
+  /** When set, clears affinity for this thread immediately on transient failure. */
+  threadId?: string | null;
+  /**
+   * Suppress Pool rotation and quota/transient affinity mutations for an account-qualified
+   * request. Credential failures still sweep stale affinities because reauthentication is
+   * account-wide.
+   */
+  fixedAccount?: boolean;
+  /**
+   * Probe lease held by this request, when it was admitted through an active
+   * quota cooldown. Only the outcome carrying the current lease may clear the
+   * cooldown (#433).
+  */
+  probeLeaseId?: string;
+  /** Scope of `probeLeaseId` when it was granted against a model-scoped cooldown. */
+  probeQuotaScope?: CodexQuotaScope;
+  /**
+   * The half-open TRANSIENT-HOLD probe this request was granted, when it was the one request
+   * admitted to test a held account (#4701). A different lease to `probeLeaseId` above, in a
+   * different domain: that one governs a quota cooldown, this one governs a 5xx hold. The two
+   * are mutually exclusive by construction -- `isTransientOnlyAffinityBlock` refuses to
+   * recognise a transient hold on an account that carries quota health -- so a request never
+   * holds both and never pays two recovery permits for one send.
+   */
+  transientProbe?: TransientProbeGrant;
+  /**
+   * Already-chosen alternate for same-request 429 retry. When set, promotion
+   * reuses this account instead of calling {@link pickAlternateCodexAccount}
+   * again (which would advance a round-robin ring twice).
+   */
+  promoteAccountId?: string;
+  /** Generation captured when this routed account was selected. */
+  writerGeneration?: number;
+  /**
+   * Credential generation this request's bearer was read at. Distinct from
+   * `writerGeneration`, which tracks the config store.
+   *
+   * A 401 that arrives after the credential was already replaced is evidence about a
+   * token nobody is using any more, so it must not quarantine the replacement. Absent
+   * means the caller cannot supply lineage and the historical unfenced handling stands.
+   */
+  credentialGeneration?: number;
+};
+
+export function computeCodexUsageScore(quota: {
+  weeklyPercent?: number;
+  monthlyPercent?: number;
+  shortPercent?: number;
+  shortResetAt?: number;
+  shortObservedAt?: number;
+} | null, plan?: unknown, now: number = Date.now()): number {
+  if (!quota) return CODEX_UNKNOWN_USAGE_SCORE;
+  const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+  const longWindows = isThirtyDayOnlyCodexPlan(plan)
+    ? [quota.monthlyPercent]
+    : [quota.weeklyPercent, quota.monthlyPercent];
+  const knownLong = longWindows.filter(finite);
+  // The short burst window only REFINES a known long-window position; it cannot stand in for
+  // one. A snapshot carrying just `shortPercent: 0` would otherwise score a flat 0 and make an
+  // account whose weekly/monthly usage is entirely unverified look like the emptiest in the
+  // pool, so `pickLowestUsageAmong` would send every request to it. Unknown has to stay
+  // unknown until a governing window is actually observed.
+  //
+  // A FULL burst window is the exception (#3029). It is not an optimistic guess about an
+  // unobserved window — it is a direct observation that the account cannot serve a request
+  // right now, whatever its monthly position turns out to be. Unknown-means-selectable is
+  // correct for uncertainty and wrong for a measured refusal: the account stays selected,
+  // `applyQuotaAutoSwitch` never fires, and the pool wedges on an exhausted credential.
+  if (knownLong.length === 0) {
+    return isTerminalShortWindow(quota, now) ? CODEX_EXHAUSTED_USAGE_PERCENT : CODEX_UNKNOWN_USAGE_SCORE;
+  }
+  const values = finite(quota.shortPercent) ? [...knownLong, quota.shortPercent] : knownLong;
+  return Math.max(...values);
+}
+
+/**
+ * A short-only reading that proves the account is blocked NOW.
+ *
+ * Freshness is not optional. `getAccountQuota` performs no expiry check, partial updates
+ * carry a still-open short tuple forward, and disk hydration accepts a persisted reading for
+ * hours — so scoring 100 from `shortPercent` alone would keep excluding an account whose
+ * five-hour window has since reset. Merge no longer carries an elapsed shortResetAt, but an
+ * explicit incoming elapsed tuple is still stored, and a missing reset cannot be aged there.
+ * That is #3029 pointed the other way: the issue is that
+ * an exhausted account stays selected, and "a recovered account stays excluded" trades one
+ * unusable pool for another.
+ *
+ * A reading with no `shortResetAt` cannot be aged, so it stays unknown. The conservative
+ * direction here is the one that keeps an account selectable: a wrongly-selected account
+ * fails one request, while a wrongly-excluded one is invisible until someone reads the pool
+ * by hand.
+ *
+ * A missing reset can instead be aged by shortObservedAt (#3425). General updatedAt is not
+ * sufficient: credit-only updates preserve the old short tuple but advance that timestamp.
+ * Old disk snapshots without short-window provenance remain unknown.
+ */
+function isTerminalShortWindow(
+  quota: { shortPercent?: number; shortResetAt?: number; shortObservedAt?: number },
+  now: number,
+): boolean {
+  if (typeof quota.shortPercent !== "number" || !Number.isFinite(quota.shortPercent)) return false;
+  if (quota.shortPercent < CODEX_EXHAUSTED_USAGE_PERCENT) return false;
+  const resetAt = quota.shortResetAt;
+  if (typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt <= 0) {
+    const observedAt = quota.shortObservedAt;
+    if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return false;
+    const age = now - observedAt;
+    return age >= 0 && age <= TERMINAL_SHORT_WINDOW_FRESHNESS_MS;
+  }
+  // Seconds and milliseconds both reach storage, so the split lives in one place next to the
+  // merge that also ages a stored reset instant (`resetAtToMs`, src/codex/quota.ts).
+  return resetAtToMs(resetAt) > now;
+}
+
+export function classifyCodexUpstreamOutcome(
+  outcome: CodexUpstreamOutcome,
+  denial?: "workspace" | "entitlement",
+): CodexUpstreamOutcomeClass {
+  if (outcome === "connect_neutral") return "neutral";
+  if (outcome === "connect_error" || outcome === "timeout") return "transient";
+  if (!Number.isFinite(outcome)) return "unknown";
+  if (outcome >= 200 && outcome < 300) return "success";
+  // Explicit 3xx policy (#914): a redirect response is relayed as-is and is
+  // never account or host health evidence — it proves the host is reachable
+  // and says nothing about the credential. Relayed as the neutral class so a
+  // stray 3xx cannot increment an account's transient streak.
+  if (outcome >= 300 && outcome < 400) return "neutral";
+  // 401 is always a credential problem. A 403 is only a credential problem when nothing
+  // tells us otherwise: a workspace/entitlement denial (#1789) means the credential is valid
+  // and the account simply lacks access here, so quarantining it for reauth is wrong advice.
+  // Absent denial evidence the historical mapping stands, so the change fails safe.
+  if (outcome === 403 && denial !== undefined) return "workspace";
+  if (outcome === 401 || outcome === 403) return "credential";
+  // 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
+  // (same-request alternate retry records this outcome for the depleted account).
+  if (outcome === 429 || outcome === 402) return "quota";
+  if (outcome >= 400 && outcome < 500) return "caller";
+  if (outcome >= 500 && outcome < 600) return "transient";
+  return "unknown";
+}
+
+function clampCooldownMs(ms: number): number {
+  return Math.min(Math.max(ms, 1), CODEX_MAX_QUOTA_COOLDOWN_MS);
+}
+
+export function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined {
+  const text = value?.trim();
+  if (!text) return undefined;
+  if (/^\d+(?:\.\d+)?$/.test(text)) {
+    const seconds = Number(text);
+    if (Number.isFinite(seconds) && seconds > 0) return clampCooldownMs(Math.ceil(seconds * 1000));
+  }
+  const timestamp = Date.parse(text);
+  if (!Number.isFinite(timestamp)) return undefined;
+  const delay = timestamp - now;
+  return delay > 0 ? clampCooldownMs(delay) : undefined;
+}
+
+function resetTimestampMs(value: unknown): number | undefined {
+  const numeric = typeof value === "number"
+    ? value
+    : typeof value === "string" && value.trim() !== ""
+      ? Number(value)
+      : undefined;
+  if (typeof numeric !== "number" || !Number.isFinite(numeric) || numeric <= 0) return undefined;
+  return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
+}
+
+export function parseResetCooldownMs(resetAt: unknown | unknown[] | undefined, now = Date.now()): number | undefined {
+  const values = Array.isArray(resetAt) ? resetAt : [resetAt];
+  let best: number | undefined;
+  for (const value of values) {
+    const timestamp = resetTimestampMs(value);
+    if (timestamp === undefined) continue;
+    const delay = timestamp - now;
+    if (delay <= 0) continue;
+    // A far-future reset must not pin the account for the full Retry-After
+    // ceiling: quota usually frees up well before the advertised window (#433).
+    const clamped = Math.min(clampCooldownMs(delay), CODEX_MAX_RESET_DERIVED_COOLDOWN_MS);
+    if (best === undefined || clamped < best) best = clamped;
+  }
+  return best;
+}
+
+export function computeQuotaCooldown(meta: CodexUpstreamOutcomeMeta = {}): {
+  until: number;
+  source: CodexCooldownSource;
+} {
+  const now = meta.now ?? Date.now();
+  const retryAfterMs = parseRetryAfterMs(meta.retryAfter, now);
+  if (retryAfterMs !== undefined) return { until: now + retryAfterMs, source: "retry-after" };
+  const resetCooldownMs = parseResetCooldownMs(meta.resetAt, now);
+  if (resetCooldownMs !== undefined) return { until: now + resetCooldownMs, source: "reset-derived" };
+  return { until: now + CODEX_DEFAULT_QUOTA_COOLDOWN_MS, source: "default" };
+}
+
+/**
+ * When the pool should stop preferring an account after it refused on quota.
+ *
+ * The earliest window the refusal actually announced, bounded by {@link CODEX_MAX_QUOTA_AVOID_MS},
+ * and never shorter than the cooldown the same refusal produced — a Retry-After directive that
+ * outlasts every announcement still governs.
+ */
+export function quotaAvoidUntilFor(meta: CodexUpstreamOutcomeMeta, now: number, cooldownUntil: number): number {
+  const values = Array.isArray(meta.resetAt) ? meta.resetAt : [meta.resetAt];
+  let announced: number | undefined;
+  for (const value of values) {
+    const timestamp = resetTimestampMs(value);
+    if (timestamp === undefined) continue;
+    const delay = timestamp - now;
+    if (delay <= 0) continue;
+    const until = now + Math.min(delay, CODEX_MAX_QUOTA_AVOID_MS);
+    if (announced === undefined || until < announced) announced = until;
+  }
+  return Math.max(cooldownUntil, announced ?? 0);
+}
+
+export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): number {
+  return computeQuotaCooldown(meta).until;
+}

package/src/codex/routing/health-store.ts

@@ -0,0 +1,402 @@
+import { isCodexAccountGenerationLive } from "../account-store";
+import { NATIVE_RESERVE_MODEL } from "../catalog/native-models";
+import { MAIN_CODEX_ACCOUNT_ID } from "../main-account";
+import { POOL_KEY_CODEX } from "../pool-rotation";
+import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
+import type { OcxConfig } from "../../types";
+import type { CodexCooldownSource } from "./cooldown-math";
+
+export type CodexUpstreamHealth = {
+  consecutiveFailures: number;
+  /** Consecutive healthy terminals observed while recovering from escalation level 2+. */
+  consecutiveSuccesses?: number;
+  lastFailureStatus?: number;
+  lastFailureAt?: number;
+  /** Hard cooldown (quota 429). Survives a later 2xx; blocks auth + selection. */
+  cooldownUntil?: number;
+  /**
+   * How long a quota refusal keeps selection away from this account (or this native quota
+   * group), as opposed to how long it is hard-blocked.
+   *
+   * The two are deliberately different lengths. {@link CODEX_MAX_RESET_DERIVED_COOLDOWN_MS}
+   * caps the hard cooldown at 15 minutes because a reset announcement is advisory and plan
+   * quota usually frees up before it — an account must stay reachable so the pool can find
+   * that out (#433). The window the refusal announced is not 15 minutes, though, so once the
+   * cooldown lapses the account is selectable again while its burst window is still spent,
+   * and the strategy picks it straight back: this proxy reads a weekly bar a burst limit never
+   * touches, so a refused account still scores as the coolest in the pool. Every request then
+   * earns the same 429 until the process restarts, which is the only thing that drops this map.
+   *
+   * So the announcement governs avoidance and the cap still governs blocking. Avoidance is soft
+   * in the {@link softAvoidUntil} sense: it reorders the pool and releases a bound thread, and
+   * the last-resort paths still reach the account when nothing else can serve, so one pessimistic
+   * announcement cannot stall routing.
+   */
+  quotaAvoidUntil?: number;
+  /** When the current cooldown was recorded; origin of the probe interval clock. */
+  cooldownSince?: number;
+  /**
+   * What produced the cooldown. An explicit Retry-After is a literal retry
+   * directive and is never probed; a quota resetAt only announces a window
+   * refresh, so it may be probed early (#433).
+   */
+  cooldownSource?: CodexCooldownSource;
+  /**
+   * Bumped on every cooldown write. A probe lease records the generation it was
+   * issued for so a lease cannot clear a cooldown that a later 429 replaced.
+   */
+  cooldownGeneration?: number;
+  /**
+   * Identity of the in-flight probe. A cooled-down account sends no traffic, so
+   * no organic 2xx can prove recovery; only the outcome carrying this id may
+   * clear the cooldown.
+   */
+  probeLeaseId?: string;
+  /** Cooldown generation at the moment the lease was granted. */
+  probeLeaseGeneration?: number;
+  /** Last probe grant or conclusion; paces the probe interval. */
+  lastProbeAt?: number;
+  /**
+   * Soft avoid after connect_error / timeout / transient 5xx. Cleared on 2xx.
+   * Blocks pool selection + thread affinity reuse so a sticky session can leave a
+   * flaky account without throwing CodexAccountCooldownError (hard-only).
+   */
+  softAvoidUntil?: number;
+  /**
+   * Credential generation a 401/403 quarantine was derived from (#2892 gap 4).
+   *
+   * Provenance lives ON the entry rather than in a side map keyed by account id. A side map spends
+   * "whatever health is current when the old credential is found dead", which deletes a later
+   * unrelated entry: a G1 401, then a G2 save, then a genuine G2 503 would lose the 503. Only the
+   * entry that carries this field can be spent, and any later write simply replaces it.
+   */
+  credentialFailureGeneration?: number;
+};
+
+const upstreamHealth = new Map<string, CodexUpstreamHealth>();
+/**
+ * Reset-derived 429s can describe a quota owned by one native model family,
+ * rather than the whole ChatGPT account. Keep those advisory cooldowns apart
+ * from account-wide Retry-After/default throttles and transient health.
+ */
+const quotaScopedHealth = new Map<string, Map<CodexQuotaScope, CodexUpstreamHealth>>();
+/**
+ * Spend a credential-failure health entry whose credential no longer exists (#2892 gap 4).
+ *
+ * A 401/403 describes one CREDENTIAL, not an account, and a replacement can land at any point after
+ * the outcome is recorded — so re-reading the store inside `recordCodexUpstreamOutcome` narrows the
+ * window without closing it. The reader decides instead, and it may only spend an entry that
+ * actually carries credential provenance: a later transient or quota write replaces the entry and
+ * with it the tag, so this can never delete evidence that belongs to a different failure.
+ */
+export function dropSpentCredentialFailure(accountId: string): void {
+  const health = upstreamHealth.get(accountId);
+  const generation = health?.credentialFailureGeneration;
+  if (health === undefined || generation === undefined) return;
+  if (isCodexAccountGenerationLive(accountId, generation)) return;
+  upstreamHealth.delete(accountId);
+}
+let lastReconciledGeneration = 0;
+let liveHealthAccountIds = new Set<string>();
+
+/**
+ * Native Codex quota groups known to be independent upstream. Keep the mapping
+ * deliberately conservative: unlisted models share the normal native group.
+ * Add a new explicit group here only when its independent upstream quota is
+ * confirmed, so shared limits never receive cross-model bypasses.
+ */
+export type CodexQuotaScope = "shared" | "reserve";
+
+
+export const NATIVE_MODEL_QUOTA_SCOPES: Readonly<Record<string, CodexQuotaScope>> = {
+  [NATIVE_RESERVE_MODEL]: "reserve",
+};
+
+export function codexQuotaScopeForModel(modelId: string | undefined): CodexQuotaScope | undefined {
+  if (!modelId?.trim()) return undefined;
+  return NATIVE_MODEL_QUOTA_SCOPES[modelId.trim().toLowerCase()] ?? "shared";
+}
+
+/** Independent quota groups must not mutate the shared active-account cursor. */
+export function isIndependentCodexQuotaScope(quotaScope?: CodexQuotaScope): boolean {
+  return quotaScope !== undefined && quotaScope !== "shared";
+}
+
+export function codexPoolKeyForScope(quotaScope?: CodexQuotaScope): string {
+  return isIndependentCodexQuotaScope(quotaScope) ? `${POOL_KEY_CODEX}:${quotaScope}` : POOL_KEY_CODEX;
+}
+
+export function listLiveCodexAccountIds(config: OcxConfig): ReadonlySet<string> {
+  const ids = new Set((config.codexAccounts ?? []).map(account => account.id));
+  const openai = config.providers.openai;
+  if (openai && openai.disabled !== true && isCanonicalOpenAiForwardProvider(openai)) {
+    ids.add(MAIN_CODEX_ACCOUNT_ID);
+  }
+  return ids;
+}
+
+export function getCodexUpstreamHealth(
+  accountId: string,
+): CodexUpstreamHealth | null {
+  dropSpentCredentialFailure(accountId);
+  return upstreamHealth.get(accountId) ?? null;
+}
+
+export function scopedHealthFor(accountId: string, scope: CodexQuotaScope): CodexUpstreamHealth | undefined {
+  return quotaScopedHealth.get(accountId)?.get(scope);
+}
+
+export function setScopedHealth(accountId: string, scope: CodexQuotaScope, health: CodexUpstreamHealth): void {
+  let scopes = quotaScopedHealth.get(accountId);
+  if (!scopes) {
+    scopes = new Map();
+    quotaScopedHealth.set(accountId, scopes);
+  }
+  scopes.set(scope, health);
+}
+
+export function deleteScopedHealth(accountId: string, scope: CodexQuotaScope): void {
+  const scopes = quotaScopedHealth.get(accountId);
+  if (!scopes) return;
+  scopes.delete(scope);
+  if (scopes.size === 0) quotaScopedHealth.delete(accountId);
+}
+
+/** Live quota-refusal avoidance for an account, including the lane the request belongs to. */
+function codexQuotaAvoidUntil(
+  accountId: string,
+  quotaScope: CodexQuotaScope | undefined,
+  now: number,
+): number | null {
+  const live = (value: number | undefined): number | null =>
+    typeof value === "number" && Number.isFinite(value) && value > now ? value : null;
+  const account = live(upstreamHealth.get(accountId)?.quotaAvoidUntil);
+  const scoped = quotaScope === undefined
+    ? null
+    : live(scopedHealthFor(accountId, quotaScope)?.quotaAvoidUntil);
+  if (account === null) return scoped;
+  return scoped === null ? account : Math.max(account, scoped);
+}
+
+export function isCodexQuotaAvoided(
+  accountId: string,
+  quotaScope: CodexQuotaScope | undefined,
+  now: number,
+): boolean {
+  return codexQuotaAvoidUntil(accountId, quotaScope, now) !== null;
+}
+
+/**
+ * Hard-cooldown bookkeeping that ordinary success/transient transitions rebuild
+ * their health object from. Dropping these would let one late unrelated response
+ * erase a Retry-After source, a cooldown generation, or someone else's live probe.
+ */
+export function preservedCooldownFields(health: CodexUpstreamHealth | undefined): Partial<CodexUpstreamHealth> {
+  if (!health) return {};
+  // `credentialFailureGeneration` is provenance for ONE credential failure, so it must not survive
+  // into a later transient or quota entry — otherwise that entry inherits the tag and gets spent
+  // when the old credential dies, deleting evidence that was never about it (#2892 gap 4 review).
+  const {
+    consecutiveFailures: _f, consecutiveSuccesses: _s, lastFailureStatus: _st, lastFailureAt: _at,
+    softAvoidUntil: _sa, credentialFailureGeneration: _cg, ...cooldownFields
+  } = health;
+  return cooldownFields;
+}
+
+export function getCodexAccountCooldownUntil(accountId: string, now = Date.now()): number | null {
+  const cooldownUntil = upstreamHealth.get(accountId)?.cooldownUntil;
+  return typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now ? cooldownUntil : null;
+}
+
+/** Read-only cooldown snapshot for shared OAuth health projection (no write side effects). */
+export function getCodexAccountHealthSnapshot(accountId: string, now = Date.now()): {
+  cooldownUntil?: number;
+  cooldownSource?: CodexCooldownSource;
+} | null {
+  const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
+  if (cooldownUntil === null) return null;
+  const source = upstreamHealth.get(accountId)?.cooldownSource;
+  return {
+    cooldownUntil,
+    ...(source ? { cooldownSource: source } : {}),
+  };
+}
+
+/**
+ * Read the cooldown relevant to a routed native model. Account-wide cooldowns
+ * (Retry-After/default) always win; reset-derived scoped state applies only to
+ * its confirmed quota group.
+ */
+export function getCodexQuotaHealthSnapshot(
+  accountId: string,
+  quotaScope: CodexQuotaScope | undefined,
+  now = Date.now(),
+): {
+  cooldownUntil?: number;
+  cooldownSource?: CodexCooldownSource;
+  quotaScope?: CodexQuotaScope;
+} | null {
+  const account = getCodexAccountHealthSnapshot(accountId, now);
+  if (account) return account;
+  if (!quotaScope) return null;
+  const scoped = scopedHealthFor(accountId, quotaScope);
+  const cooldownUntil = scoped?.cooldownUntil;
+  if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null;
+  return {
+    cooldownUntil,
+    ...(scoped?.cooldownSource ? { cooldownSource: scoped.cooldownSource } : {}),
+    quotaScope,
+  };
+}
+
+export function isCodexAccountInCooldown(accountId: string, now = Date.now()): boolean {
+  return getCodexAccountCooldownUntil(accountId, now) !== null;
+}
+
+/**
+ * Manually lift a hard quota cooldown without touching failure history.
+ *
+ * Injected Codex routing makes this proxy the ONLY model path for Codex Desktop, so a
+ * cooldown that outlives the real upstream limit reads to the user as "the whole app is
+ * broken" with no escape but editing config.toml. This is that escape hatch.
+ *
+ * Deliberately narrow:
+ * - Failure counters and softAvoid survive. Clearing a cooldown says "the quota window
+ *   moved", not "this account is healthy"; failover must keep its knowledge.
+ * - Dropping `probeLeaseId` is what stops a stale in-flight probe from later "proving"
+ *   recovery against a NEWER cooldown: {@link ownsProbeLease} needs the id to match.
+ *   `cooldownGeneration` is preserved and bumped as redundancy only — a fresh 429 already
+ *   bumps it in {@link recordCodexUpstreamOutcome}, so the bump here is not load-bearing
+ *   today and is kept so the invariant survives a future change that retains the lease.
+ *
+ * Returns false when the account carried neither a live cooldown nor a live avoidance window.
+ * The window outlives the cooldown by design — the cooldown caps at fifteen minutes and the
+ * window runs up to six hours — so the moment an operator actually reaches for this escape
+ * hatch is usually after the cooldown lapsed and only the window is still keeping the account
+ * out of rotation. Refusing to look at the window then would leave the hatch shut in the one
+ * case it exists for.
+ */
+export function clearCodexAccountCooldown(accountId: string, now = Date.now()): boolean {
+  const clear = (health: CodexUpstreamHealth): CodexUpstreamHealth | null => {
+    const cooldownUntil = health.cooldownUntil;
+    const liveCooldown = typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now;
+    const avoidUntil = health.quotaAvoidUntil;
+    const liveAvoidance = typeof avoidUntil === "number" && Number.isFinite(avoidUntil) && avoidUntil > now;
+    if (!liveCooldown && !liveAvoidance) return null;
+    const {
+      cooldownUntil: _until,
+      cooldownSince: _since,
+      cooldownSource: _source,
+      probeLeaseId: _leaseId,
+      probeLeaseGeneration: _leaseGeneration,
+      // Same reasoning as the probe recovery above: "the quota window moved" is a statement
+      // about the whole refusal, so the avoidance it announced goes with the block it
+      // produced. Keeping it would leave this escape hatch not escaping, because selection
+      // would still pass over the account for as long as the announced window runs.
+      quotaAvoidUntil: _avoid,
+      ...rest
+    } = health;
+    return {
+      ...rest,
+      cooldownGeneration: (health.cooldownGeneration ?? 0) + 1,
+      lastProbeAt: now,
+    };
+  };
+
+  let cleared = false;
+  const accountHealth = upstreamHealth.get(accountId);
+  if (accountHealth) {
+    const next = clear(accountHealth);
+    if (next) {
+      upstreamHealth.set(accountId, next);
+      cleared = true;
+    }
+  }
+  for (const [scope, health] of quotaScopedHealth.get(accountId) ?? []) {
+    const next = clear(health);
+    if (next) {
+      setScopedHealth(accountId, scope, next);
+      cleared = true;
+    }
+  }
+  return cleared;
+}
+
+export function getCodexAccountSoftAvoidUntil(accountId: string, now = Date.now()): number | null {
+  const softAvoidUntil = upstreamHealth.get(accountId)?.softAvoidUntil;
+  return typeof softAvoidUntil === "number" && Number.isFinite(softAvoidUntil) && softAvoidUntil > now
+    ? softAvoidUntil
+    : null;
+}
+
+export function isCodexAccountSoftAvoided(accountId: string, now = Date.now()): boolean {
+  return getCodexAccountSoftAvoidUntil(accountId, now) !== null;
+}
+
+/**
+ * Closed package-internal accessors for the account-wide health maps. Selection,
+ * the probe lease, and the active cursor mutate health only through these; the
+ * Map bindings themselves never leave this module.
+ */
+export function getAccountHealth(accountId: string): CodexUpstreamHealth | undefined {
+  return upstreamHealth.get(accountId);
+}
+
+export function setAccountHealth(accountId: string, health: CodexUpstreamHealth): void {
+  upstreamHealth.set(accountId, health);
+}
+
+export function deleteAccountHealth(accountId: string): void {
+  upstreamHealth.delete(accountId);
+}
+
+export function listScopedHealthEntries(accountId: string): Array<[CodexQuotaScope, CodexUpstreamHealth]> {
+  return [...(quotaScopedHealth.get(accountId) ?? [])];
+}
+
+export function deleteAllScopedHealth(accountId: string): void {
+  quotaScopedHealth.delete(accountId);
+}
+
+export function isHealthAccountAdmissible(accountId: string, writerGeneration: number): boolean {
+  return writerGeneration >= lastReconciledGeneration || liveHealthAccountIds.has(accountId);
+}
+
+export function isHealthGenerationReconciled(generation: number): boolean {
+  return generation <= lastReconciledGeneration;
+}
+
+export function pruneHealthAccountsForContext(codexAccountIds: ReadonlySet<string>): number {
+  let removed = 0;
+  for (const accountId of upstreamHealth.keys()) {
+    if (codexAccountIds.has(accountId)) continue;
+    upstreamHealth.delete(accountId);
+    removed += 1;
+  }
+  for (const accountId of quotaScopedHealth.keys()) {
+    if (codexAccountIds.has(accountId)) continue;
+    quotaScopedHealth.delete(accountId);
+    removed += 1;
+  }
+  return removed;
+}
+
+export function commitHealthReconcile(generation: number, codexAccountIds: ReadonlySet<string>): void {
+  liveHealthAccountIds = new Set(codexAccountIds);
+  lastReconciledGeneration = generation;
+}
+
+export function clearUpstreamHealthState(): void {
+  upstreamHealth.clear();
+  quotaScopedHealth.clear();
+}
+
+export function resetHealthReconcileState(): void {
+  lastReconciledGeneration = 0;
+  liveHealthAccountIds = new Set();
+}
+
+export function deleteAllHealthForAccount(accountId: string): void {
+  upstreamHealth.delete(accountId);
+  quotaScopedHealth.delete(accountId);
+}