@bitkyc08/opencodex 2.55.0-preview.20260914 → 2.56.0

package/src/codex/routing/active-account.ts

@@ -0,0 +1,194 @@
+import { saveConfigPreservingClaudeCode } from "../../config";
+import { clearCodexAccountPin, pinnedCodexAccountId } from "../account-priority";
+import {
+  POOL_KEY_CODEX,
+  normalizeCodexAccountPoolStrategy,
+  seedPoolRotationAccount,
+} from "../pool-rotation";
+import type { OcxConfig } from "../../types";
+import { clearThreadAccountMap } from "./thread-affinity";
+import {
+  NATIVE_MODEL_QUOTA_SCOPES,
+  codexPoolKeyForScope,
+  deleteAccountHealth,
+  deleteScopedHealth,
+  getAccountHealth,
+  isIndependentCodexQuotaScope,
+  listScopedHealthEntries,
+  preservedCooldownFields,
+  setAccountHealth,
+  setScopedHealth,
+  type CodexUpstreamHealth,
+} from "./health-store";
+
+/**
+ * Process-local cursor for automatic RR/fill-first (and quota-429 when not
+ * sync-writing) picks. Keeps unrelated `saveConfig` from persisting transient
+ * rotation as the operator's `activeCodexAccountId`. Manual selection clears it
+ * so disk/`config.activeCodexAccountId` remains authoritative.
+ */
+let runtimeActiveCodexAccountId: string | undefined;
+
+/** Manual selection resets transient routing evidence without bypassing a real 429 cooldown. */
+export function resetCodexRoutingForManualSelection(accountId: string): void {
+  clearThreadAccountMap();
+  // Manual selection is the operator source of truth — drop any automatic runtime cursor.
+  runtimeActiveCodexAccountId = undefined;
+  // Record the pick as an unspent one-shot on the SHARED scope only. An independent scope
+  // gets no entry on purpose: every write site the guard protects is already skipped for
+  // independent scopes, so an entry there would be state nothing reads — and state nothing
+  // reads is what the next reader mistakes for a rule.
+  //
+  // Seeding happens ONLY here. A pool-driven promote must never create or move a preference,
+  // or the pool would manufacture an operator intent nobody expressed.
+  manualPreference.set(POOL_KEY_CODEX, accountId);
+  // Seed the RR ring so the next unbound new session honors the manually selected account
+  // under round-robin (affinity-cleared threads / null threadId). Fill-first already follows
+  // config.activeCodexAccountId, which the caller persists before invoking this.
+  seedPoolRotationAccount(POOL_KEY_CODEX, accountId);
+  for (const scope of new Set(Object.values(NATIVE_MODEL_QUOTA_SCOPES))) {
+    if (isIndependentCodexQuotaScope(scope)) {
+      seedPoolRotationAccount(codexPoolKeyForScope(scope), accountId);
+    }
+  }
+  // Quota avoidance is a preference, like the soft avoid dropped above, and an operator naming
+  // this account has overruled it. The hard cooldown is the part that survives.
+  const overrule = (health: CodexUpstreamHealth) => {
+    const { quotaAvoidUntil: _avoid, ...retained } = preservedCooldownFields(health);
+    return retained;
+  };
+  const current = getAccountHealth(accountId);
+  if (current) {
+    const retained = overrule(current);
+    if (Object.keys(retained).length === 0) deleteAccountHealth(accountId);
+    else setAccountHealth(accountId, { consecutiveFailures: 0, ...retained });
+  }
+  // A reset-derived refusal records its avoidance on the SCOPED map and returns before the
+  // account-wide entry is written, so naming the account has to reach that map too. Stopping
+  // at `upstreamHealth` — and returning early when it holds nothing — overruled nothing in
+  // the case that produces the avoidance this function exists to overrule.
+  for (const [scope, health] of [...(listScopedHealthEntries(accountId))]) {
+    const retained = overrule(health);
+    if (Object.keys(retained).length === 0) deleteScopedHealth(accountId, scope);
+    else setScopedHealth(accountId, scope, { consecutiveFailures: 0, ...retained });
+  }
+}
+
+/** Effective active: automatic runtime cursor, else operator/persisted selection. */
+/**
+ * Unspent operator selections, keyed by pool scope.
+ *
+ * Codex has no account-side equivalent of the Anthropic `selectionRevision`, so staleness
+ * cannot be detected by comparing values: a pool-driven promote legitimately moves the
+ * persisted active account, and reading that as staleness would silently spend the
+ * operator's one-shot. Invalidation is keyed to the OPERATOR path instead — another manual
+ * selection, the account leaving the pool, or a successful dispatch on it.
+ */
+const manualPreference = new Map<string, string>();
+
+/**
+ * Spend the one-shot for a pool scope once a dispatch on that account actually succeeded.
+ * This is the Codex analogue of `commitAnthropicSelectionRouting`, which Codex lacks.
+ *
+ * Wiring this BEFORE the guard below is not a style choice. Measured: with the guard in
+ * place and no consume site, the first manual selection freezes the automatic cursor
+ * permanently and 15 of 69 rotation tests fail.
+ */
+export function consumeManualPreference(accountId: string, poolKey: string): void {
+  if (manualPreference.get(poolKey) === accountId) manualPreference.delete(poolKey);
+}
+
+/**
+ * Drop an account's preference in every scope. Pause and exclusion do not route through
+ * `resetCodexRoutingForManualSelection`, so without this a preference could outlive the
+ * account it names and keep suppressing the automatic cursor.
+ */
+export function forgetManualPreference(accountId: string): void {
+  for (const [poolKey, preferred] of manualPreference) {
+    if (preferred === accountId) manualPreference.delete(poolKey);
+  }
+}
+
+/**
+ * True while an unspent operator selection for this scope names a DIFFERENT account than
+ * the automatic pick about to be recorded.
+ *
+ * Callers pass their own scope: an independent quota scope keeps its own entry and must
+ * never read the shared one. The failover promote does NOT consult this — see its call
+ * site for why.
+ */
+export function manualPreferenceBlocks(poolKey: string, accountId: string): boolean {
+  const preferred = manualPreference.get(poolKey);
+  return preferred !== undefined && preferred !== accountId;
+}
+
+export function getEffectiveActiveCodexAccountId(config: OcxConfig): string | undefined {
+  return runtimeActiveCodexAccountId ?? config.activeCodexAccountId;
+}
+
+/**
+ * Whether the account routing is currently on is there because an operator asked
+ * for it, rather than because a strategy landed on it. Surfaces read this instead
+ * of comparing the stored pin themselves, which would report a pin that a later
+ * automatic pick has already moved past.
+ */
+export function isEffectiveCodexAccountPinned(config: OcxConfig): boolean {
+  const pinned = pinnedCodexAccountId(config);
+  return pinned !== undefined && pinned === getEffectiveActiveCodexAccountId(config);
+}
+
+/**
+ * Automatic strategy / failover cursor only — never mutates `config.activeCodexAccountId`
+ * so an unrelated `saveConfig` cannot persist transient rotation as operator selection.
+ */
+export function rememberActiveCodexAccount(_config: OcxConfig, accountId: string): void {
+  runtimeActiveCodexAccountId = accountId;
+}
+
+/**
+ * End the manual pin when routing moves to a different account. Returns whether
+ * the pin changed so the caller can fold it into a write it was already making.
+ */
+function releaseCodexAccountPinFor(config: OcxConfig, accountId: string): boolean {
+  const pinned = pinnedCodexAccountId(config);
+  if (pinned === undefined || pinned === accountId) return false;
+  clearCodexAccountPin(config);
+  return true;
+}
+
+/** Persist operator (or quota-strategy) active selection to config + disk. */
+export function setActiveCodexAccount(config: OcxConfig, accountId: string): void {
+  runtimeActiveCodexAccountId = undefined;
+  const releasedPin = releaseCodexAccountPinFor(config, accountId);
+  if (config.activeCodexAccountId === accountId && !releasedPin) return;
+  config.activeCodexAccountId = accountId;
+  saveConfigPreservingClaudeCode(config);
+}
+
+/** Quota strategy persists; RR/fill-first keep a process-local cursor only. */
+export function promoteActiveCodexAccount(config: OcxConfig, accountId: string): void {
+  if (normalizeCodexAccountPoolStrategy(config.accountPoolStrategy) === "quota") {
+    setActiveCodexAccount(config, accountId);
+    return;
+  }
+  // Runtime-only, like the cursor itself: a caller that persists (pause, delete)
+  // saves this release with its own write; a transient failover does not, so the
+  // pin survives a restart that also clears the failure history behind it.
+  releaseCodexAccountPinFor(config, accountId);
+  rememberActiveCodexAccount(config, accountId);
+}
+
+export function clearAllManualPreferences(): void {
+  manualPreference.clear();
+}
+
+export function forgetRuntimeActiveCodexAccount(): void {
+  runtimeActiveCodexAccountId = undefined;
+}
+
+export function forgetRoutingPreferencesOutside(codexAccountIds: ReadonlySet<string>): void {
+  for (const [poolKey, preferred] of manualPreference) {
+    if (codexAccountIds.has(preferred)) continue;
+    manualPreference.delete(poolKey);
+  }
+}

package/src/codex/routing/cooldown-math.ts

@@ -0,0 +1,275 @@
+import {
+  CODEX_EXHAUSTED_USAGE_PERCENT,
+  CODEX_UNKNOWN_USAGE_SCORE,
+  resetAtToMs,
+} from "../quota";
+import { isThirtyDayOnlyCodexPlan } from "../plan";
+import type { CodexQuotaScope } from "./health-store";
+
+export const CODEX_DEFAULT_QUOTA_COOLDOWN_MS = 60_000;
+export const CODEX_MAX_QUOTA_COOLDOWN_MS = 24 * 60 * 60_000;
+/**
+ * A weekly/monthly quota `resetAt` announces when the window refreshes; it is not
+ * a "come back after this" directive like Retry-After. Plan quota routinely frees
+ * up long before the advertised reset, so cap reset-derived cooldowns far below
+ * the Retry-After ceiling (#433).
+ */
+export const CODEX_MAX_RESET_DERIVED_COOLDOWN_MS = 15 * 60_000;
+/**
+ * Ceiling on quota-refusal avoidance. Generous enough to cover a full five-hour burst window,
+ * tight enough that a weekly or monthly reset four days out cannot take an account out of
+ * rotation for the {@link CODEX_MAX_QUOTA_COOLDOWN_MS} day the Retry-After ceiling allows.
+ */
+export const CODEX_MAX_QUOTA_AVOID_MS = 6 * 60 * 60_000;
+/** Minimum gap between probe leases for one cooled-down account. */
+export const CODEX_QUOTA_PROBE_INTERVAL_MS = 5 * 60_000;
+export const CODEX_FAILURE_WINDOW_MS = 5 * 60_000;
+/**
+ * How recently a 100% burst reading must have been OBSERVED to exclude an account when it
+ * carries no reset timestamp (#3425). Deliberately far tighter than the 6h disk-hydration
+ * horizon in `quota.ts`: shorter than any plausible five-hour burst window, so a persisted
+ * reading can never strand a recovered account, and long enough that a snapshot taken at
+ * admission is still fresh when selection reads it.
+ */
+export const TERMINAL_SHORT_WINDOW_FRESHNESS_MS = 5 * 60_000;
+/** How long a transient failure keeps the account out of pool selection. */
+export const CODEX_TRANSIENT_SOFT_AVOID_MS = 30_000;
+export const CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS = [
+  CODEX_TRANSIENT_SOFT_AVOID_MS,
+  2 * 60_000,
+  10 * 60_000,
+  30 * 60_000,
+] as const;
+
+export type CodexUpstreamOutcome = number | "connect_error" | "timeout" | "connect_neutral";
+export type CodexUpstreamOutcomeClass = "success" | "credential"
+  | "workspace" | "quota" | "transient" | "caller" | "neutral" | "unknown";
+export type CodexCooldownSource = "retry-after" | "reset-derived" | "default";
+
+export type CodexUpstreamOutcomeMeta = {
+  retryAfter?: string | null;
+  resetAt?: unknown | unknown[];
+  now?: number;
+  /** (provider, host) ledger key for account-neutral reachability failures (#914). */
+  hostKey?: string;
+  /**
+   * Upstream denial evidence for a 403. A workspace/entitlement denial means the CREDENTIAL
+   * is fine and the account simply cannot reach this workspace, so it must not be quarantined
+   * for reauthentication (#1789). Absent evidence keeps the historical credential handling.
+   */
+  denial?: "workspace" | "entitlement";
+  /** Stable transport code recorded alongside a neutral host failure. */
+  lastFailureCode?: string;
+  /** Native model selected for this request; used only for confirmed scoped quotas. */
+  modelId?: string;
+  /** When set, clears affinity for this thread immediately on transient failure. */
+  threadId?: string | null;
+  /**
+   * Suppress Pool rotation and quota/transient affinity mutations for an account-qualified
+   * request. Credential failures still sweep stale affinities because reauthentication is
+   * account-wide.
+   */
+  fixedAccount?: boolean;
+  /**
+   * Probe lease held by this request, when it was admitted through an active
+   * quota cooldown. Only the outcome carrying the current lease may clear the
+   * cooldown (#433).
+  */
+  probeLeaseId?: string;
+  /** Scope of `probeLeaseId` when it was granted against a model-scoped cooldown. */
+  probeQuotaScope?: CodexQuotaScope;
+  /**
+   * Already-chosen alternate for same-request 429 retry. When set, promotion
+   * reuses this account instead of calling {@link pickAlternateCodexAccount}
+   * again (which would advance a round-robin ring twice).
+   */
+  promoteAccountId?: string;
+  /** Generation captured when this routed account was selected. */
+  writerGeneration?: number;
+  /**
+   * Credential generation this request's bearer was read at. Distinct from
+   * `writerGeneration`, which tracks the config store.
+   *
+   * A 401 that arrives after the credential was already replaced is evidence about a
+   * token nobody is using any more, so it must not quarantine the replacement. Absent
+   * means the caller cannot supply lineage and the historical unfenced handling stands.
+   */
+  credentialGeneration?: number;
+};
+
+export function computeCodexUsageScore(quota: {
+  weeklyPercent?: number;
+  monthlyPercent?: number;
+  shortPercent?: number;
+  shortResetAt?: number;
+  shortObservedAt?: number;
+} | null, plan?: unknown, now: number = Date.now()): number {
+  if (!quota) return CODEX_UNKNOWN_USAGE_SCORE;
+  const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+  const longWindows = isThirtyDayOnlyCodexPlan(plan)
+    ? [quota.monthlyPercent]
+    : [quota.weeklyPercent, quota.monthlyPercent];
+  const knownLong = longWindows.filter(finite);
+  // The short burst window only REFINES a known long-window position; it cannot stand in for
+  // one. A snapshot carrying just `shortPercent: 0` would otherwise score a flat 0 and make an
+  // account whose weekly/monthly usage is entirely unverified look like the emptiest in the
+  // pool, so `pickLowestUsageAmong` would send every request to it. Unknown has to stay
+  // unknown until a governing window is actually observed.
+  //
+  // A FULL burst window is the exception (#3029). It is not an optimistic guess about an
+  // unobserved window — it is a direct observation that the account cannot serve a request
+  // right now, whatever its monthly position turns out to be. Unknown-means-selectable is
+  // correct for uncertainty and wrong for a measured refusal: the account stays selected,
+  // `applyQuotaAutoSwitch` never fires, and the pool wedges on an exhausted credential.
+  if (knownLong.length === 0) {
+    return isTerminalShortWindow(quota, now) ? CODEX_EXHAUSTED_USAGE_PERCENT : CODEX_UNKNOWN_USAGE_SCORE;
+  }
+  const values = finite(quota.shortPercent) ? [...knownLong, quota.shortPercent] : knownLong;
+  return Math.max(...values);
+}
+
+/**
+ * A short-only reading that proves the account is blocked NOW.
+ *
+ * Freshness is not optional. `getAccountQuota` performs no expiry check, partial updates
+ * carry a still-open short tuple forward, and disk hydration accepts a persisted reading for
+ * hours — so scoring 100 from `shortPercent` alone would keep excluding an account whose
+ * five-hour window has since reset. Merge no longer carries an elapsed shortResetAt, but an
+ * explicit incoming elapsed tuple is still stored, and a missing reset cannot be aged there.
+ * That is #3029 pointed the other way: the issue is that
+ * an exhausted account stays selected, and "a recovered account stays excluded" trades one
+ * unusable pool for another.
+ *
+ * A reading with no `shortResetAt` cannot be aged, so it stays unknown. The conservative
+ * direction here is the one that keeps an account selectable: a wrongly-selected account
+ * fails one request, while a wrongly-excluded one is invisible until someone reads the pool
+ * by hand.
+ *
+ * A missing reset can instead be aged by shortObservedAt (#3425). General updatedAt is not
+ * sufficient: credit-only updates preserve the old short tuple but advance that timestamp.
+ * Old disk snapshots without short-window provenance remain unknown.
+ */
+function isTerminalShortWindow(
+  quota: { shortPercent?: number; shortResetAt?: number; shortObservedAt?: number },
+  now: number,
+): boolean {
+  if (typeof quota.shortPercent !== "number" || !Number.isFinite(quota.shortPercent)) return false;
+  if (quota.shortPercent < CODEX_EXHAUSTED_USAGE_PERCENT) return false;
+  const resetAt = quota.shortResetAt;
+  if (typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt <= 0) {
+    const observedAt = quota.shortObservedAt;
+    if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return false;
+    const age = now - observedAt;
+    return age >= 0 && age <= TERMINAL_SHORT_WINDOW_FRESHNESS_MS;
+  }
+  // Seconds and milliseconds both reach storage, so the split lives in one place next to the
+  // merge that also ages a stored reset instant (`resetAtToMs`, src/codex/quota.ts).
+  return resetAtToMs(resetAt) > now;
+}
+
+export function classifyCodexUpstreamOutcome(
+  outcome: CodexUpstreamOutcome,
+  denial?: "workspace" | "entitlement",
+): CodexUpstreamOutcomeClass {
+  if (outcome === "connect_neutral") return "neutral";
+  if (outcome === "connect_error" || outcome === "timeout") return "transient";
+  if (!Number.isFinite(outcome)) return "unknown";
+  if (outcome >= 200 && outcome < 300) return "success";
+  // Explicit 3xx policy (#914): a redirect response is relayed as-is and is
+  // never account or host health evidence — it proves the host is reachable
+  // and says nothing about the credential. Relayed as the neutral class so a
+  // stray 3xx cannot increment an account's transient streak.
+  if (outcome >= 300 && outcome < 400) return "neutral";
+  // 401 is always a credential problem. A 403 is only a credential problem when nothing
+  // tells us otherwise: a workspace/entitlement denial (#1789) means the credential is valid
+  // and the account simply lacks access here, so quarantining it for reauth is wrong advice.
+  // Absent denial evidence the historical mapping stands, so the change fails safe.
+  if (outcome === 403 && denial !== undefined) return "workspace";
+  if (outcome === 401 || outcome === 403) return "credential";
+  // 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
+  // (same-request alternate retry records this outcome for the depleted account).
+  if (outcome === 429 || outcome === 402) return "quota";
+  if (outcome >= 400 && outcome < 500) return "caller";
+  if (outcome >= 500 && outcome < 600) return "transient";
+  return "unknown";
+}
+
+function clampCooldownMs(ms: number): number {
+  return Math.min(Math.max(ms, 1), CODEX_MAX_QUOTA_COOLDOWN_MS);
+}
+
+export function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined {
+  const text = value?.trim();
+  if (!text) return undefined;
+  if (/^\d+(?:\.\d+)?$/.test(text)) {
+    const seconds = Number(text);
+    if (Number.isFinite(seconds) && seconds > 0) return clampCooldownMs(Math.ceil(seconds * 1000));
+  }
+  const timestamp = Date.parse(text);
+  if (!Number.isFinite(timestamp)) return undefined;
+  const delay = timestamp - now;
+  return delay > 0 ? clampCooldownMs(delay) : undefined;
+}
+
+function resetTimestampMs(value: unknown): number | undefined {
+  const numeric = typeof value === "number"
+    ? value
+    : typeof value === "string" && value.trim() !== ""
+      ? Number(value)
+      : undefined;
+  if (typeof numeric !== "number" || !Number.isFinite(numeric) || numeric <= 0) return undefined;
+  return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
+}
+
+export function parseResetCooldownMs(resetAt: unknown | unknown[] | undefined, now = Date.now()): number | undefined {
+  const values = Array.isArray(resetAt) ? resetAt : [resetAt];
+  let best: number | undefined;
+  for (const value of values) {
+    const timestamp = resetTimestampMs(value);
+    if (timestamp === undefined) continue;
+    const delay = timestamp - now;
+    if (delay <= 0) continue;
+    // A far-future reset must not pin the account for the full Retry-After
+    // ceiling: quota usually frees up well before the advertised window (#433).
+    const clamped = Math.min(clampCooldownMs(delay), CODEX_MAX_RESET_DERIVED_COOLDOWN_MS);
+    if (best === undefined || clamped < best) best = clamped;
+  }
+  return best;
+}
+
+export function computeQuotaCooldown(meta: CodexUpstreamOutcomeMeta = {}): {
+  until: number;
+  source: CodexCooldownSource;
+} {
+  const now = meta.now ?? Date.now();
+  const retryAfterMs = parseRetryAfterMs(meta.retryAfter, now);
+  if (retryAfterMs !== undefined) return { until: now + retryAfterMs, source: "retry-after" };
+  const resetCooldownMs = parseResetCooldownMs(meta.resetAt, now);
+  if (resetCooldownMs !== undefined) return { until: now + resetCooldownMs, source: "reset-derived" };
+  return { until: now + CODEX_DEFAULT_QUOTA_COOLDOWN_MS, source: "default" };
+}
+
+/**
+ * When the pool should stop preferring an account after it refused on quota.
+ *
+ * The earliest window the refusal actually announced, bounded by {@link CODEX_MAX_QUOTA_AVOID_MS},
+ * and never shorter than the cooldown the same refusal produced — a Retry-After directive that
+ * outlasts every announcement still governs.
+ */
+export function quotaAvoidUntilFor(meta: CodexUpstreamOutcomeMeta, now: number, cooldownUntil: number): number {
+  const values = Array.isArray(meta.resetAt) ? meta.resetAt : [meta.resetAt];
+  let announced: number | undefined;
+  for (const value of values) {
+    const timestamp = resetTimestampMs(value);
+    if (timestamp === undefined) continue;
+    const delay = timestamp - now;
+    if (delay <= 0) continue;
+    const until = now + Math.min(delay, CODEX_MAX_QUOTA_AVOID_MS);
+    if (announced === undefined || until < announced) announced = until;
+  }
+  return Math.max(cooldownUntil, announced ?? 0);
+}
+
+export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): number {
+  return computeQuotaCooldown(meta).until;
+}