@bitkyc08/opencodex 2.22.0 → 2.23.0

package/src/codex/management-convergence.ts

@@ -3,6 +3,7 @@ import { captureCatalogAdmissionSnapshot } from "./catalog-admission";
 import { convergeCodexCatalog } from "./convergence";
 import type {
   CatalogDisposition,
+  CatalogFailureCause,
   CatalogOnlyOutcome,
   CodexHistoryState,
   CodexObservedState,
@@ -49,6 +50,58 @@ function notEvaluatedObserved(history: CodexHistoryState): CodexObservedState {
   };
 }
 
+/** Recognized errno/code tokens. Anything else is dropped rather than echoed. */
+const RECOGNIZED_FAILURE_CODES: ReadonlySet<string> = new Set([
+  "ENOSPC", "EACCES", "EPERM", "EROFS", "ENOENT", "SQLITE_BUSY",
+]);
+
+/**
+ * Reduce a caught error to an allowlisted cause (#1784).
+ *
+ * Nothing from the error text reaches the caller: `kind` is chosen from a closed set and
+ * `code` is only emitted when it is a recognized token. An `Error.message` routinely carries
+ * paths, home directories and account ids, and `redactSecretString` masks token shapes but
+ * none of those, so the message is never a safe thing to forward from here.
+ */
+function catalogFailureCause(error: unknown): CatalogFailureCause {
+  const raw = (error as { code?: unknown } | null)?.code;
+  const code = typeof raw === "string" && RECOGNIZED_FAILURE_CODES.has(raw)
+    ? raw as CatalogFailureCause["code"]
+    : undefined;
+  if (error instanceof TypeError || error instanceof RangeError || error instanceof SyntaxError) {
+    return { kind: "invalid-request", ...(code ? { code } : {}) };
+  }
+  if (code === "SQLITE_BUSY") return { kind: "lock-busy", code };
+  if (code !== undefined) return { kind: "io", code };
+  return { kind: "unknown" };
+}
+
+/**
+ * Classify a failure that is NOT a filesystem problem.
+ *
+ * `disk` used to be the catch-all, so an operator saw "non-retryable disk failure" for a
+ * malformed request. Reserve `disk` for real IO and route the rest to honest reasons.
+ */
+function classifiedCatalogFailure(error: unknown, commitBegan: boolean): CatalogDisposition {
+  const cause = catalogFailureCause(error);
+  const reason = cause.kind === "invalid-request"
+    ? "request-invalid" as const
+    : cause.kind === "lock-busy"
+      ? "admission" as const
+      : cause.kind === "io"
+        ? "disk" as const
+        : "internal" as const;
+  return {
+    status: "failed",
+    reason,
+    phase: commitBegan ? "commit" : "gather",
+    // Contention is the one class worth retrying unchanged.
+    retryable: reason === "admission",
+    partialWrite: commitBegan,
+    cause,
+  };
+}
+
 function unexpectedCatalogFailure(commitBegan: boolean): CatalogDisposition {
   return {
     status: "failed",
@@ -64,7 +117,7 @@ function admissionFailure(error: unknown): CatalogDisposition {
   if (message.includes("config generation is busy") || message.includes("config generation is database")) {
     return { status: "skipped", reason: "busy", retryable: true };
   }
-  return unexpectedCatalogFailure(false);
+  return classifiedCatalogFailure(error, false);
 }
 
 /** Project catalog work into the shared no-change/not-evaluated outcome shape. */
@@ -107,7 +160,7 @@ export function createManagementConvergeCodex(
     } catch (error) {
       return projectCatalogOnlyOutcome({
         changed: false,
-        catalogRefresh: commitBegan ? unexpectedCatalogFailure(true) : admissionFailure(error),
+        catalogRefresh: commitBegan ? classifiedCatalogFailure(error, true) : admissionFailure(error),
       });
     }
   };

package/src/codex/quota-rejection.ts

@@ -23,6 +23,63 @@ export interface CodexPreStreamRejection {
   alternateRetryEligible: boolean;
   resetCreditEligible: boolean;
   semanticCode?: CodexResetEligibleExhaustionCode;
+  /**
+   * Structured denial evidence for a 403. Present only when the upstream body names a
+   * workspace/entitlement denial, which proves the CREDENTIAL is valid and the account
+   * simply lacks access here (#1789). Status alone can never set this.
+   */
+  denial?: "workspace" | "entitlement";
+}
+
+/**
+ * Upstream codes that identify a WORKSPACE denial rather than a bad credential.
+ *
+ * #1789: a K12 account whose credential validates and whose WHAM usage returns 200 still
+ * gets 403 `codex_workspace_access_denied` on a routed prompt. Treating that as a credential
+ * failure tells the user to re-authenticate a credential that is already valid, and the loop
+ * repeats forever.
+ */
+const WORKSPACE_DENIAL_CODES: ReadonlySet<string> = new Set([
+  "codex_workspace_access_denied",
+  "workspace_access_denied",
+]);
+
+const ENTITLEMENT_DENIAL_CODES: ReadonlySet<string> = new Set([
+  "codex_entitlement_missing",
+  "entitlement_missing",
+]);
+
+/** Read a structured denial code out of a 403 body. Fails closed to undefined. */
+async function denialFromResponse(
+  response: Response,
+  signal?: AbortSignal,
+): Promise<"workspace" | "entitlement" | undefined> {
+  try {
+    const body = await readBoundedResponseBody(response.clone(), { signal, fatalUtf8: true });
+    if (!body.displaySafe || body.truncated || !body.text.trim()) return undefined;
+    if (isUnsafeJsonDocument(body.text)) return undefined;
+    const payload = JSON.parse(body.text) as unknown;
+    const code = structuredDenialCode(payload);
+    if (code === undefined) return undefined;
+    if (WORKSPACE_DENIAL_CODES.has(code)) return "workspace";
+    if (ENTITLEMENT_DENIAL_CODES.has(code)) return "entitlement";
+    return undefined;
+  } catch {
+    // Same fail-closed rule as the exhaustion classifier: an unreadable body must not
+    // downgrade a credential failure into a workspace one.
+    return undefined;
+  }
+}
+
+/** Own-property `code` lookup at the top level or under `error`. No coercion, no accessors. */
+function structuredDenialCode(payload: unknown): string | undefined {
+  if (payload === null || typeof payload !== "object" || Array.isArray(payload)) return undefined;
+  const direct = (payload as Record<string, unknown>).code;
+  if (typeof direct === "string") return direct;
+  const error = (payload as Record<string, unknown>).error;
+  if (error === null || typeof error !== "object" || Array.isArray(error)) return undefined;
+  const nested = (error as Record<string, unknown>).code;
+  return typeof nested === "string" ? nested : undefined;
 }
 
 const RESET_ELIGIBLE_CODES: ReadonlySet<string> = new Set(RESET_ELIGIBLE_CODE_VALUES);
@@ -205,7 +262,10 @@ export async function classifyCodexPreStreamRejection(
 ): Promise<CodexPreStreamRejection> {
   const status = response.status;
   if (status === 401) return rejection(status, "authentication-error");
-  if (status === 403) return rejection(status, "permission-error");
+  if (status === 403) {
+    const denial = await denialFromResponse(response, options.signal);
+    return { ...rejection(status, "permission-error"), ...(denial ? { denial } : {}) };
+  }
   if (TRANSIENT_SERVER_STATUSES.has(status)) return rejection(status, "transient-server-error");
   if (status !== 429 && status !== 402) return rejection(status, "other");
 

package/src/codex/quota.ts

@@ -9,6 +9,20 @@ export type StoredAccountQuota = {
   monthlyPercent?: number;
   weeklyResetAt?: number;
   monthlyResetAt?: number;
+  /**
+   * A sub-day burst window, when upstream declares one (#1791).
+   *
+   * K12 and similar plans enforce a rolling 5-hour limit ALONGSIDE the weekly one.
+   * Not folding it into `weeklyPercent` stopped the mislabeling, but dropping it
+   * entirely hides a limit that genuinely blocks the account: a 429 at 100% here is
+   * real even while the weekly quota is untouched.
+   *
+   * `shortWindowSeconds` is retained because the duration is the only thing that makes
+   * this window self-describing; the slot it arrived in is not stable across plans.
+   */
+  shortPercent?: number;
+  shortResetAt?: number;
+  shortWindowSeconds?: number;
   resetCredits?: number;
   /**
    * True when `monthlyPercent` came from an explicitly-monthly PRIMARY window —
@@ -56,6 +70,19 @@ type WhamUsageWindow = {
 };
 
 const MONTHLY_WINDOW_MIN_SECONDS = 28 * 24 * 60 * 60;
+/**
+ * Shortest window still plausibly the WEEKLY quota (#1791).
+ *
+ * K12 and similar plans send a 5-hour primary window plus a 7-day secondary. Folding the
+ * primary into `weeklyPercent` reported the 5-hour bar as the weekly one and discarded the
+ * real weekly reading entirely, so the dashboard showed a window that reset every few hours
+ * and routing never saw the limit that actually gates the account.
+ *
+ * 24h is the discriminator: anything shorter is a burst window, not a weekly one. A window
+ * with no declared duration is unchanged, because older payloads omit `limit_window_seconds`
+ * and guessing there would break every legacy account.
+ */
+const WEEKLY_WINDOW_MIN_SECONDS = 24 * 60 * 60;
 const MONTHLY_WINDOW_MIN_MINUTES = MONTHLY_WINDOW_MIN_SECONDS / 60;
 
 const accountQuota = new Map<string, StoredAccountQuota>();
@@ -72,13 +99,16 @@ export const CODEX_UNKNOWN_USAGE_SCORE = 101;
 export const CODEX_EXHAUSTED_USAGE_PERCENT = 100;
 
 export function isCodexQuotaExhausted(
-  quota: Pick<StoredAccountQuota, "weeklyPercent" | "monthlyPercent"> | null,
+  quota: Pick<StoredAccountQuota, "weeklyPercent" | "monthlyPercent" | "shortPercent"> | null,
   plan?: unknown,
 ): boolean {
   if (!quota) return false;
+  // The burst window counts on EVERY plan. It is upstream-enforced independently, so an
+  // account at 100% there is blocked regardless of which longer window governs its plan;
+  // omitting it would route traffic straight into a 429 (#1791).
   const values = codexQuotaWindowForPlan(plan) === "monthly"
-    ? [quota.monthlyPercent]
-    : [quota.weeklyPercent, quota.monthlyPercent];
+    ? [quota.monthlyPercent, quota.shortPercent]
+    : [quota.weeklyPercent, quota.monthlyPercent, quota.shortPercent];
   return values.some(value => typeof value === "number"
     && Number.isFinite(value)
     && value >= CODEX_EXHAUSTED_USAGE_PERCENT);
@@ -104,7 +134,7 @@ export function codexQuotaWindowForPlan(plan?: unknown): "monthly" | "weekly" {
 }
 
 export function isCompleteCodexQuotaRecoverySnapshot(
-  quota: Pick<StoredAccountQuota, "weeklyPercent" | "monthlyPercent" | "monthlyIsPrimaryWindow"> | null,
+  quota: Pick<StoredAccountQuota, "weeklyPercent" | "monthlyPercent" | "monthlyIsPrimaryWindow" | "shortPercent"> | null,
   plan?: unknown,
 ): boolean {
   if (!quota || isCodexQuotaExhausted(quota, plan)) return false;
@@ -160,6 +190,15 @@ function hasKnownQuotaValue(quota: Omit<StoredAccountQuota, "updatedAt">): boole
     .some(value => typeof value === "number" && Number.isFinite(value));
 }
 
+/** True only for a window that DECLARES a duration shorter than a day. */
+function isExplicitShortWindow(window: WhamUsageWindow | null | undefined): boolean {
+  const seconds = window?.limit_window_seconds;
+  return typeof seconds === "number"
+    && Number.isFinite(seconds)
+    && seconds > 0
+    && seconds < WEEKLY_WINDOW_MIN_SECONDS;
+}
+
 function isExplicitMonthlyWindow(window: WhamUsageWindow | null | undefined): boolean {
   const seconds = window?.limit_window_seconds;
   return typeof seconds === "number"
@@ -465,10 +504,25 @@ export function parseUsageQuota(data: WhamUsageResponse): Omit<StoredAccountQuot
   // - 선택한 방식: only an explicit primary duration of at least 28 days changes it to monthly.
   // - 다른 대안 대신 이 방식을 선택한 이유: it accepts calendar-month variance and preserves legacy payloads.
   // - 장점, 단점 및 영향: Team monthly quotas classify correctly; unknown durations remain weekly by design.
-  const weeklyPercent = primaryIsMonthly ? secondaryPercent : primaryPercent ?? secondaryPercent;
+  // #1791: a primary window that declares a sub-day duration is a burst window, not the
+  // weekly one. Skip it so the secondary (the real 7-day window) is what lands in
+  // `weeklyPercent`; without this the 5-hour bar was reported as weekly and the actual
+  // weekly reading was dropped on the floor.
+  const primaryIsShort = isExplicitShortWindow(primaryWindow);
+  const weeklyCandidatePercent = primaryIsShort ? undefined : primaryPercent;
+  const weeklyCandidateResetAt = primaryIsShort ? undefined : primaryResetAt;
+  // Keep the burst reading instead of dropping it on the floor: it is a real limit, and
+  // the account is blocked when it fills even though the weekly window is fine (#1791).
+  if (primaryIsShort && primaryPercent !== undefined) {
+    quota.shortPercent = primaryPercent;
+    if (primaryResetAt !== undefined) quota.shortResetAt = primaryResetAt;
+    const seconds = primaryWindow?.limit_window_seconds;
+    if (typeof seconds === "number" && Number.isFinite(seconds)) quota.shortWindowSeconds = seconds;
+  }
+  const weeklyPercent = primaryIsMonthly ? secondaryPercent : weeklyCandidatePercent ?? secondaryPercent;
   const weeklyResetAt = primaryIsMonthly
     ? secondaryResetAt
-    : primaryPercent !== undefined ? primaryResetAt : secondaryResetAt;
+    : weeklyCandidatePercent !== undefined ? weeklyCandidateResetAt : secondaryResetAt;
   const monthlyPercent = primaryIsMonthly ? primaryPercent ?? tertiaryPercent : tertiaryPercent;
   const monthlyResetAt = primaryIsMonthly && primaryPercent !== undefined ? primaryResetAt : tertiaryResetAt;
   if (thirtyDayOnly) {

package/src/codex/routing.ts

@@ -127,7 +127,8 @@ let lastReconciledGeneration = 0;
 let liveHealthAccountIds = new Set<string>();
 
 export type CodexUpstreamOutcome = number | "connect_error" | "timeout" | "connect_neutral";
-export type CodexUpstreamOutcomeClass = "success" | "credential" | "quota" | "transient" | "caller" | "neutral" | "unknown";
+export type CodexUpstreamOutcomeClass = "success" | "credential"
+  | "workspace" | "quota" | "transient" | "caller" | "neutral" | "unknown";
 export type CodexCooldownSource = "retry-after" | "reset-derived" | "default";
 /**
  * Native Codex quota groups known to be independent upstream. Keep the mapping
@@ -193,6 +194,12 @@ export type CodexUpstreamOutcomeMeta = {
   now?: number;
   /** (provider, host) ledger key for account-neutral reachability failures (#914). */
   hostKey?: string;
+  /**
+   * Upstream denial evidence for a 403. A workspace/entitlement denial means the CREDENTIAL
+   * is fine and the account simply cannot reach this workspace, so it must not be quarantined
+   * for reauthentication (#1789). Absent evidence keeps the historical credential handling.
+   */
+  denial?: "workspace" | "entitlement";
   /** Stable transport code recorded alongside a neutral host failure. */
   lastFailureCode?: string;
   /** Native model selected for this request; used only for confirmed scoped quotas. */
@@ -327,7 +334,10 @@ export function computeCodexUsageScore(quota: {
   return values.length > 0 ? Math.max(...values) : CODEX_UNKNOWN_USAGE_SCORE;
 }
 
-export function classifyCodexUpstreamOutcome(outcome: CodexUpstreamOutcome): CodexUpstreamOutcomeClass {
+export function classifyCodexUpstreamOutcome(
+  outcome: CodexUpstreamOutcome,
+  denial?: "workspace" | "entitlement",
+): CodexUpstreamOutcomeClass {
   if (outcome === "connect_neutral") return "neutral";
   if (outcome === "connect_error" || outcome === "timeout") return "transient";
   if (!Number.isFinite(outcome)) return "unknown";
@@ -337,6 +347,11 @@ export function classifyCodexUpstreamOutcome(outcome: CodexUpstreamOutcome): Cod
   // and says nothing about the credential. Relayed as the neutral class so a
   // stray 3xx cannot increment an account's transient streak.
   if (outcome >= 300 && outcome < 400) return "neutral";
+  // 401 is always a credential problem. A 403 is only a credential problem when nothing
+  // tells us otherwise: a workspace/entitlement denial (#1789) means the credential is valid
+  // and the account simply lacks access here, so quarantining it for reauth is wrong advice.
+  // Absent denial evidence the historical mapping stands, so the change fails safe.
+  if (outcome === 403 && denial !== undefined) return "workspace";
   if (outcome === 401 || outcome === 403) return "credential";
   // 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
   // (same-request alternate retry records this outcome for the depleted account).
@@ -1596,7 +1611,7 @@ export function recordCodexUpstreamOutcome(
   const writerGeneration = meta.writerGeneration ?? captureConfigGeneration();
   if (writerGeneration < lastReconciledGeneration && !liveHealthAccountIds.has(accountId)) return;
   const now = meta.now ?? Date.now();
-  const outcomeClass = classifyCodexUpstreamOutcome(outcome);
+  const outcomeClass = classifyCodexUpstreamOutcome(outcome, meta.denial);
   const quotaScope = codexQuotaScopeForModel(meta.modelId);
   if (outcomeClass === "success") {
     const scopedProbe = meta.probeQuotaScope
@@ -1676,6 +1691,18 @@ export function recordCodexUpstreamOutcome(
   }
 
   const lastFailureStatus = typeof outcome === "number" ? outcome : 0;
+  if (outcomeClass === "workspace") {
+    // The credential is valid; this account just cannot reach this workspace (#1789).
+    // Record the failure so routing stops preferring it, but do not mark it for
+    // reauthentication and do not sweep its thread affinities: telling the user to
+    // re-login is wrong advice that cannot fix a workspace grant.
+    upstreamHealth.set(accountId, {
+      consecutiveFailures: (upstreamHealth.get(accountId)?.consecutiveFailures ?? 0) + 1,
+      lastFailureStatus,
+      lastFailureAt: now,
+    });
+    return;
+  }
   if (outcomeClass === "credential") {
     // 401/403 quarantines the account for reauth. That supersedes quota state
     // entirely: a cooldown (and any probe lease) on an unusable account is moot.

package/src/combos/failover.ts

@@ -121,6 +121,26 @@ export function comboFailureDecision(
   if (isCyberPolicyCode(options?.code)) return "stop";
   const error = classifyError(status, "upstream_error", message);
   if (isCyberPolicyCode(error.code)) return "stop";
+  // A local input-admission refusal (#1524) says "this candidate cannot fit the request",
+  // not "the request is impossible": the next candidate may have a larger context window.
+  //
+  // This MUST be tested before the generic stop list below. Our own refusal message says
+  // "context window" -- that is what it refuses on -- and the classifier remaps that phrase,
+  // so checking the stop list first swallowed the signal and ended the chain. An UPSTREAM
+  // `context_length_exceeded` carries no admission code and still falls through to stop.
+  //
+  // Matched on the STRUCTURED code only, which classifyError now preserves for our own
+  // refusal. A raw substring test would additionally let any upstream override a terminal
+  // verdict by echoing the token in prose we do not control.
+  //
+  // Precise about what this is NOT: an upstream can still SET this code deliberately, since
+  // both extractors read the upstream error object. That is bounded rather than dangerous --
+  // an upstream already controls other hop signals (429, 5xx), and traversal is finite: policy
+  // tries each candidate once via `tried`, and combo excludes each attempted target. So this is
+  // structured-code-only, not provably local.
+  if (options?.code === "input_admission_refused" || error.code === "input_admission_refused") {
+    return "hop";
+  }
   if (["origin_rejected", "context_length_exceeded", "invalid_request_error"].includes(error.code ?? "")) {
     return "stop";
   }