@bitkyc08/opencodex 2.88.0 → 2.90.0

package/src/server/responses/opengateway-completion-guard.ts

@@ -0,0 +1,232 @@
+import { guardResponseBodyInactivity } from "../../lib/response-body-inactivity";
+import { isNonReplayableResponse } from "../../lib/upstream-retry";
+import { isPlainObject } from "../responses-snapshot-codec";
+import { BoundedSseFrameBuffer, joinSseFrameBytes } from "../sse-frame-buffer";
+import { sseDataPayload } from "../sse-payload-rewrite";
+import { EMPTY_COMPLETION_MAX_BUFFERED_BYTES, EMPTY_COMPLETION_MAX_BUFFERED_EVENTS, EMPTY_COMPLETION_RETRY_ENV } from "./empty-completion-guard";
+import { selfContainedResponsesBody } from "./reset-replay";
+import { carryResponseToolOwnership } from "./response-tool-ownership";
+
+const MAX_BYTES = EMPTY_COMPLETION_MAX_BUFFERED_BYTES;
+const MAX_EVENTS = EMPTY_COMPLETION_MAX_BUFFERED_EVENTS;
+const MAX_HOLD_MS = 30_000;
+const decoder = new TextDecoder();
+const reasoningEvents = new Set([
+  "response.reasoning_summary_part.added", "response.reasoning_summary_part.done",
+  "response.reasoning_summary_text.delta", "response.reasoning_summary_text.done",
+  "response.reasoning_text.delta", "response.reasoning_text.done",
+]);
+type RecordValue = Record<string, unknown>;
+
+function partCommits(part: unknown): boolean {
+  return !isPlainObject(part) || part.type !== "output_text" || part.text !== ""
+    || (part.annotations !== undefined && (!Array.isArray(part.annotations) || part.annotations.length > 0));
+}
+function outputCommits(output: unknown): boolean {
+  if (output === undefined) return false;
+  return !Array.isArray(output) || output.some(item => !isPlainObject(item)
+    || (item.type !== "reasoning" && (item.type !== "message" || !Array.isArray(item.content) || item.content.some(partCommits))));
+}
+function wellFormedEmpty(snapshot: unknown): snapshot is RecordValue {
+  return isPlainObject(snapshot) && typeof snapshot.id === "string" && snapshot.id.trim().length > 0
+    && snapshot.object === "response" && snapshot.status === "completed" && Array.isArray(snapshot.output)
+    && snapshot.error == null && snapshot.incomplete_details == null
+    && snapshot.output.every(item => isPlainObject(item) && typeof item.id === "string" && item.id.length > 0
+      && (item.type === "reasoning" || (item.type === "message" && item.role === "assistant"
+        && item.status === "completed" && Array.isArray(item.content) && !item.content.some(partCommits))));
+}
+function eventCommits(event: RecordValue): boolean {
+  if (event.replayUnsafe === true) return true;
+  switch (event.type) {
+    case "response.created": case "response.in_progress": case "response.queued": case "response.heartbeat":
+      return isPlainObject(event.response) && outputCommits(event.response.output);
+    case "response.output_item.added": case "response.output_item.done": return outputCommits([event.item]);
+    case "response.content_part.added": case "response.content_part.done": return partCommits(event.part);
+    case "response.output_text.delta": return event.delta !== "";
+    case "response.output_text.done": return event.text !== "";
+    default: return !reasoningEvents.has(String(event.type));
+  }
+}
+
+export interface OpenGatewayCompletionGuardOptions {
+  readonly url: string;
+  readonly body: unknown;
+  readonly signal: AbortSignal;
+  readonly retry: () => Promise<Response>;
+  readonly recordUsage: (usage: unknown) => void;
+  readonly retryEnabled?: boolean;
+  readonly stallTimeoutMs?: number;
+  /** Expiry releases original bytes; it never aborts a healthy upstream. */
+  readonly holdTimeoutMs?: number;
+}
+
+type Inspection = { kind: "pass"; response: Response }
+  | { kind: "empty" | "invalid"; snapshot: RecordValue; streaming: boolean };
+
+async function inspect(response: Response, options: OpenGatewayCompletionGuardOptions): Promise<Inspection> {
+  const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+  const streaming = contentType.includes("text/event-stream")
+    || (!contentType && isPlainObject(options.body) && options.body.stream === true);
+  if (!response.body || (!streaming && !contentType.includes("application/json"))) return { kind: "pass", response };
+  const guarded = guardResponseBodyInactivity(response, options.signal, streaming ? options.stallTimeoutMs ?? 0 : 0);
+  const reader = guarded.response.body?.getReader();
+  if (!reader) return { kind: "pass", response };
+  const framer = new BoundedSseFrameBuffer(MAX_BYTES);
+  const held: Array<Uint8Array | undefined> = [];
+  let bytes = 0, events = 0, transferred = false, cleaned = false;
+  let pending: ReturnType<typeof reader.read> | undefined;
+  const holdMs = options.holdTimeoutMs ?? MAX_HOLD_MS;
+  const deadline = performance.now() + holdMs;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const expired = new Promise<"expired">(resolve => { timer = setTimeout(() => resolve("expired"), holdMs); });
+  const dropHeld = (): void => {
+    for (let index = 0; index < held.length; index += 1) held[index] = undefined;
+  };
+  const finishOwned = (): void => {
+    if (cleaned) return;
+    cleaned = true;
+    guarded.dispose();
+    try { reader.releaseLock(); }
+    catch (error) { if (!(error instanceof TypeError)) throw error; }
+  };
+  const cancelOwned = (reason?: unknown): void => {
+    if (cleaned) return;
+    cleaned = true;
+    dropHeld();
+    try { void reader.cancel(reason).catch(() => {}); }
+    catch (error) { if (!(error instanceof TypeError)) throw error; }
+    guarded.dispose();
+    try { reader.releaseLock(); }
+    catch (error) { if (!(error instanceof TypeError)) throw error; }
+  };
+  const pass = (): Inspection => {
+    transferred = true;
+    let index = 0;
+    const body = new ReadableStream<Uint8Array>({
+      async pull(controller) {
+        if (index < held.length) {
+          const chunk = held[index];
+          held[index] = undefined;
+          index += 1;
+          if (chunk) controller.enqueue(chunk);
+          return;
+        }
+        try {
+          const next = await (pending ?? reader.read());
+          pending = undefined;
+          if (next.done) { finishOwned(); controller.close(); }
+          else controller.enqueue(next.value);
+        } catch (error) { cancelOwned(error); controller.error(error); }
+      },
+      cancel(reason) { dropHeld(); cancelOwned(reason); },
+    }, { highWaterMark: 0 });
+    return { kind: "pass", response: carryResponseToolOwnership(response, new Response(body, {
+      status: response.status, statusText: response.statusText, headers: response.headers,
+    })) };
+  };
+  const terminal = (snapshot: unknown): Inspection => ({
+    kind: wellFormedEmpty(snapshot) ? "empty" : "invalid",
+    snapshot: isPlainObject(snapshot) ? snapshot : {}, streaming,
+  });
+  try {
+    for (;;) {
+      if (performance.now() >= deadline) return pass();
+      pending = reader.read();
+      let next: Awaited<typeof pending> | "expired";
+      try { next = await Promise.race([pending, expired]); }
+      catch { return pass(); }
+      if (next === "expired") return pass();
+      pending = undefined;
+      if (!next.done) {
+        if (bytes + next.value.byteLength > MAX_BYTES || held.length >= MAX_EVENTS) {
+          held.push(next.value);
+          return pass();
+        }
+        held.push(next.value.slice());
+        bytes += next.value.byteLength;
+      }
+      if (!streaming) {
+        if (!next.done) continue;
+        let snapshot: unknown;
+        try { snapshot = JSON.parse(decoder.decode(joinSseFrameBytes(held.flatMap(chunk => chunk ? [chunk] : [])))); }
+        catch { return terminal(undefined); }
+        if (isPlainObject(snapshot) && (outputCommits(snapshot.output)
+          || snapshot.error != null || ["failed", "incomplete", "queued", "in_progress"].includes(String(snapshot.status)))) return pass();
+        return terminal(snapshot);
+      }
+      let frames;
+      try { frames = next.done ? [{ block: framer.finish() }] : framer.feed(next.value); }
+      catch { return pass(); }
+      for (const raw of frames) {
+        if (raw.block.byteLength === 0) continue;
+        if (++events >= MAX_EVENTS) return pass();
+        const block = decoder.decode(raw.block), data = sseDataPayload(block);
+        if (data === null) { if (/^event:/m.test(block)) return pass(); continue; }
+        let event: unknown;
+        try { event = JSON.parse(data); } catch { return pass(); }
+        if (!isPlainObject(event)) return pass();
+        if (event.replayUnsafe === true) return pass();
+        if (event.type === "response.completed") {
+          if (isPlainObject(event.response) && outputCommits(event.response.output)) return pass();
+          return terminal(event.response);
+        }
+        if (eventCommits(event)) return pass();
+      }
+      if (next.done) return pass();
+    }
+  } finally {
+    clearTimeout(timer); framer.dispose();
+    if (!transferred) cancelOwned();
+  }
+}
+
+function terminalResponse(source: Response, inspection: Exclude<Inspection, { kind: "pass" }>, reason: string): Response {
+  const incomplete = reason === "max_output_tokens";
+  const snapshot: RecordValue = { ...inspection.snapshot, status: incomplete ? "incomplete" : "failed" };
+  if (incomplete) {
+    snapshot.incomplete_details = { reason };
+    delete snapshot.error;
+  } else {
+    snapshot.error = {
+      type: "upstream_error", code: reason, message: "OpenGateway returned no usable completion before output.",
+    };
+    delete snapshot.incomplete_details;
+  }
+  const headers = new Headers(source.headers);
+  headers.delete("content-length"); headers.delete("content-encoding");
+  if (inspection.streaming) headers.set("content-type", "text/event-stream");
+  const text = inspection.streaming
+    ? `event: response.${snapshot.status}\ndata: ${JSON.stringify({ type: `response.${snapshot.status}`, response: snapshot })}\n\n`
+    : JSON.stringify(snapshot);
+  return carryResponseToolOwnership(source, new Response(text, {
+    status: inspection.streaming || incomplete ? 200 : 502, headers,
+  }));
+}
+
+export async function guardOpenGatewayCompletion(response: Response, options: OpenGatewayCompletionGuardOptions): Promise<Response> {
+  if (options.url !== "https://apis.opengateway.ai/v1/responses" || response.status !== 200) return response;
+  let body: unknown;
+  try { body = typeof options.body === "string" ? JSON.parse(options.body) : options.body; }
+  catch { return response; }
+  if (!isPlainObject(body)) return response;
+  const canReplay = options.retryEnabled !== false && process.env[EMPTY_COMPLETION_RETRY_ENV] !== "0"
+    && selfContainedResponsesBody(body) && !isNonReplayableResponse(response);
+  let selected = response;
+  for (let attempt = 0; ; attempt++) {
+    const inspected = await inspect(selected, options);
+    if (inspected.kind === "pass") return inspected.response;
+    const usage = inspected.snapshot.usage;
+    options.recordUsage(usage);
+    if (inspected.kind === "invalid") return terminalResponse(selected, inspected, "invalid_completion");
+    if (typeof body.max_output_tokens === "number" && body.max_output_tokens > 0
+      && isPlainObject(usage) && typeof usage.output_tokens === "number" && usage.output_tokens >= body.max_output_tokens) {
+      return terminalResponse(selected, inspected, "max_output_tokens");
+    }
+    if (attempt > 0 || !canReplay || options.signal.aborted) {
+      return terminalResponse(selected, inspected, attempt > 0
+        ? "empty_completion_retry_failed" : "empty_completion");
+    }
+    selected = await options.retry();
+    if (selected.status !== 200) return selected;
+  }
+}

package/src/server/responses/passthrough-completion.ts

@@ -0,0 +1,41 @@
+import type { AdapterRequest } from "../../adapters/base";
+import { formatErrorResponse } from "../../bridge";
+import { resolveStallTimeoutMs } from "../../stall-timeout";
+import type { OcxConfig } from "../../types";
+import { recordKeyWireAttemptUsage, usageFromResponsesPayload, type RequestLogContext } from "../request-log";
+import { isPlainObject } from "../responses-snapshot-codec";
+import { guardOpenGatewayCompletion } from "./opengateway-completion-guard";
+
+export function parseOutboundRequestBody(bodyText: string): Record<string, unknown> | undefined {
+  try {
+    const body: unknown = JSON.parse(bodyText);
+    return isPlainObject(body) ? body : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function guardPassthroughCompletion(
+  response: Response,
+  request: AdapterRequest,
+  config: Pick<OcxConfig, "emptyCompletionRetry" | "stallTimeoutSec">,
+  signal: AbortSignal,
+  logCtx: RequestLogContext,
+  rebuildAndRefetch: (kind: "empty-completion") => Promise<Response | { failed: Response | { kind: "adapter-handoff" } }>,
+): Promise<Response> {
+  if (request.url !== "https://apis.opengateway.ai/v1/responses") return Promise.resolve(response);
+  return guardOpenGatewayCompletion(response, {
+    url: request.url,
+    body: parseOutboundRequestBody(request.body),
+    signal,
+    retryEnabled: config.emptyCompletionRetry,
+    stallTimeoutMs: resolveStallTimeoutMs(config.stallTimeoutSec),
+    recordUsage: usage => { recordKeyWireAttemptUsage(logCtx, usageFromResponsesPayload(usage)); },
+    retry: async () => {
+      const result = await rebuildAndRefetch("empty-completion");
+      if (!("failed" in result)) return result;
+      return result.failed instanceof Response ? result.failed
+        : formatErrorResponse(502, "upstream_error", "Native recovery changed the provider wire");
+    },
+  });
+}

package/src/server/responses/passthrough-delivery.ts

@@ -1,3 +1,4 @@
+import { wasPermanentAccountRefusal } from "../../oauth/account-auto-pause";
 import { isNativeControlResponse } from "./native-response-control";
 import { createOutboundCredentialMask, createTerminalErrorRedactionBlockRewrite } from "./terminal-error-redaction";
 import type { ResponsesRequestContext, ResponsesAdmissionState } from "./core-options";
@@ -36,6 +37,7 @@ import type { ResponsesTerminalStatus } from "../../bridge";
 import { isCodexWsQuotaObservedResponse, isCodexWsUpstreamResponse, isCodexWsPreludeProjection } from "./ws-upstream";
 import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-model-fallback";
 import { recordCodexUpstreamOutcome } from "../../codex/routing";
+import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window";
 import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context";
 import { consumeComboFailure } from "./core-combo-failure";
 import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors";
@@ -527,13 +529,14 @@ export async function deliverPassthroughResponse(
       } else if (!shouldDeferCodexResetDerivedCooldown(
         upstreamResponse,
         options.deferCodexResetDerivedCooldown,
-      ) && !isReplayRefusalResponse(upstreamResponse)) {
+      ) && !isReplayRefusalResponse(upstreamResponse) && !wasPermanentAccountRefusal(upstreamResponse)) {
         // A refusal this proxy made is not evidence about the account. Recording it would
         // classify the synthetic 429 as quota exhaustion and write a default cooldown against
         // a credential the request may never have reached, and that false signal outlives the
         // request. The sibling recorders on this path already decline: the terminal recorder
         // needs an ok streaming body, and the quota-header snapshot finds no quota headers.
         recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, upstreamResponse.status, {
+          ...codexFailureWindowOutcomeMeta(admissionState.authCtx),
           ...quotaMeta,
           threadId: admissionState.authCtx.affinityKey,
           fixedAccount: admissionState.authCtx.fixedAccount,
@@ -572,7 +575,7 @@ export async function deliverPassthroughResponse(
     // through sanitizePassthroughHeaders) so a redirect to a dead host can never
     // masquerade as a pre-connection failure after the credential was seen.
     // The numeric outcome above already classified it neutral — no streak.
-    if (isNonReplayableResponse(upstreamResponse) && !isReplayRefusalResponse(upstreamResponse)) {
+    if (isNonReplayableResponse(upstreamResponse) && !isReplayRefusalResponse(upstreamResponse) && !wasPermanentAccountRefusal(upstreamResponse)) {
       return await sanitizeNonReplayableUpstreamError(upstreamResponse, upstream.signal);
     }
     if (upstreamResponse.status >= 300 && upstreamResponse.status < 400) {

package/src/server/responses/passthrough-dispatch.ts

@@ -1,3 +1,5 @@
+import { recoverAutoPausedAccount, rotatePassthroughKeyOn429 } from "./passthrough-key-rotation";
+import { adoptCodexWsStage, capturePassthroughAffinity } from "./passthrough-observations";
 import { spendLedgerStorageErrorResponse } from "./spend-storage-error";
 import { requestPacingOverloadResponse } from "./pacing-overload";
 import { recordDistributedResponseFailure } from "../../providers/api-key-response-ownership";
@@ -51,6 +53,7 @@ import {
 import { restoreRoutedCustomCalls, RoutedCustomToolCompatError } from "../../responses/custom-tool-compat";
 import { XaiToolSchemaCompatibilityError } from "../../adapters/xai-tool-schema";
 import { MuseToolChoiceCompatibilityError } from "../../adapters/openai-responses/muse-tool-choice";
+import { OpenGatewayInputCompatibilityError } from "../../adapters/openai-responses/opengateway-input";
 import { formatErrorResponse } from "../../bridge";
 import { redactSecretString } from "../../lib/redact";
 import {
@@ -99,7 +102,6 @@ import {
   upstreamHostCircuitOpenResponse,
   usesCodexForwardPoolAuth,
   codexWsQuotaObserver,
-  isFixedCodexAccount,
   codexPoolAccountModel400Denial,
   shouldRetryCodexPoolAccountQuota,
   shouldRetryCodexPoolAccountTransient,
@@ -111,7 +113,7 @@ import {
   clearCodexModelDenialEvidence,
   recordCodexModelDenialEvidence,
 } from "../../codex/model-entitlements";
-import { codexWsSocketDeathStage, isCodexWsUpstreamResponse, readCodexWsStage } from "./codex-ws-wire";
+import { codexWsSocketDeathStage, isCodexWsUpstreamResponse } from "./codex-ws-wire";
 import { linkAbortSignal } from "./core-lifetime";
 import type { CodexAuthContext } from "../../codex/auth-context";
 import { checkOutboundBodySize, describeOutboundBodyRefusal } from "./outbound-body-guard";
@@ -130,7 +132,8 @@ import {
 } from "../../lib/upstream-retry";
 import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error";
 import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability";
-import { recordCodexUpstreamOutcome } from "../../codex/routing";
+import { captureAccountAutoPauseAttempt, wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause";
+import { codexFailureWindowOutcomeMeta, recordCodexUpstreamOutcome } from "../../codex/routing";
 import { describeUpstreamConnectFailure } from "./upstream-error";
 import type { OpaqueBlobRecoveryGuard } from "./core-opaque-recovery";
 import {
@@ -152,7 +155,6 @@ import {
   rotateGenericOAuthAccountOn429,
   failoverAccountSnapshot,
 } from "../../oauth/generic-account-failover";
-import { captureCodexAffinityDiagnostic } from "../../codex/affinity-debug";
 import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "../../codex/catalog/native-models";
 import {
   attemptOpaqueBlobRecovery,
@@ -171,6 +173,7 @@ import { upstreamErrorMessageFromPayload, ENCRYPTED_FUNCTION_OUTPUT_REJECTION }
 import { isTransientConsoleGoUploadRejection } from "../../providers/opencode-zen-rate-limit";
 import { planReasoningEffortDowngrade } from "../../providers/reasoning-metadata";
 import { unboundPoolSpendRefusalResponse } from "../workflow-refusal";
+import { guardPassthroughCompletion, parseOutboundRequestBody } from "./passthrough-completion";
 
 /** Transfer an uncommitted Copilot exchange to the existing translated pipeline without new admission. */
 export interface PassthroughAdapterHandoff { kind: "adapter-handoff" }
@@ -201,6 +204,7 @@ export async function preparePassthroughExchange(
     | "genericFailoverAccountId"
     | "passiveQuotaWriterGeneration"
     | "oauthDispatch"
+    | "recoverAutoPausedSelection"
     | "resolveSelectionAdapter"
     | "resolveCopilotSelection"
     | "copilotRefusalResponse"
@@ -384,6 +388,7 @@ export async function preparePassthroughExchange(
         || error instanceof XaiToolSchemaCompatibilityError
         || error instanceof RoutedCustomToolCompatError
         || error instanceof MuseToolChoiceCompatibilityError
+        || error instanceof OpenGatewayInputCompatibilityError
       ) {
         return formatErrorResponse(400, "invalid_request_error", redactSecretString(error.message));
       }
@@ -432,16 +437,6 @@ export async function preparePassthroughExchange(
     // `{model, input, stream}` with no tools and Copilot answers with a `custom_tool_call` for
     // `apply_patch`. Policing an absent catalog truncates that turn. An unreadable body lands there
     // too because the proxy cannot establish the caller's declared authorization boundary.
-    const parseOutboundRequestBody = (bodyText: string): Record<string, unknown> | undefined => {
-      try {
-        const body = JSON.parse(bodyText) as unknown;
-        return body && typeof body === "object" && !Array.isArray(body)
-          ? body as Record<string, unknown>
-          : undefined;
-      } catch {
-        return undefined;
-      }
-    };
     let outboundRequestBody: Record<string, unknown> | undefined;
     const declaredWireToolNames = new Set<string>();
     const declaredBareWireToolNames = new Set<string>();
@@ -733,22 +728,11 @@ export async function preparePassthroughExchange(
       }
       nativeHostState.lease = null;
     };
-    /**
-     * #4191: a Codex WS exchange pins its content-free stage record on the
-     * Response it resolves (markCodexWsStage). Adopting the record here, at
-     * the single funnel every physical upstream response passes through,
-     * binds it to the attempt that actually served it — including the 502/504
-     * pre-response JSON settles that never reach the SSE relay.
-     */
-    const adoptCodexWsStage = (response: Response): void => {
-      const stage = readCodexWsStage(response);
-      if (stage && logCtx.activeAttempt) logCtx.activeAttempt.codexWsStage = stage;
-    };
     let upstreamResponse: Response, lastDistributedResponse: Response | undefined;
     const adoptObservedResponse = <T extends Response>(response: T): T => {
       if (!isNonReplayableResponse(response)) lastDistributedResponse = response.status === 401 || response.status === 429 ? response : undefined;
       settleObservedHostResponse();
-      adoptCodexWsStage(response);
+      adoptCodexWsStage(response, logCtx);
       return response;
     };
     let passthroughEstimate = typeof request.usageLog?.inputTokens === "number"
@@ -999,6 +983,7 @@ export async function preparePassthroughExchange(
       }
       if (usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)) {
         recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, outcome, {
+          ...codexFailureWindowOutcomeMeta(admissionState.authCtx),
           threadId: admissionState.authCtx.affinityKey,
           fixedAccount: admissionState.authCtx.fixedAccount,
           modelId: route.modelId,
@@ -1050,6 +1035,7 @@ export async function preparePassthroughExchange(
               providerName: route.providerName,
               modelId: route.modelId,
               onPhysicalDispatch: receiptMode ? sendBudgetState.noteInitialDispatch : undefined,
+              accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers),
               onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId),
               beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider)
                 ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined,
@@ -1165,6 +1151,7 @@ export async function preparePassthroughExchange(
         `${route.providerName}|${route.modelId}|${recovery}`,
         { allowFinalRecoveryReserve: transientSendPolicy() === null },
       );
+      if (recovery === "empty-completion") allowance.attempts = Math.min(1, allowance.attempts);
       try {
         return await fetchWithTransientRetry(
           innerRecovery => {
@@ -1187,6 +1174,7 @@ export async function preparePassthroughExchange(
                 dispatchOverride: oauthDispatch(request, parsed, "passthrough"),
                 providerName: route.providerName,
                 modelId: route.modelId,
+                accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers),
                 onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId),
                 beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider)
                   ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined,
@@ -1195,7 +1183,8 @@ export async function preparePassthroughExchange(
               .then(adoptObservedResponse);
           },
           { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts,
-            onSendsConsumed: transientSendReporter(allowance.permit), claimAmbiguousResend: claimPreHeaderResend },
+            onSendsConsumed: transientSendReporter(allowance.permit),
+            claimAmbiguousResend: recovery === "empty-completion" ? undefined : claimPreHeaderResend },
         );
       } catch (err) {
         return { failed: transportFailureResponse(err) };
@@ -1209,8 +1198,18 @@ export async function preparePassthroughExchange(
 
     // Keep recovery kinds in sync with the generic `recovery:` loop below.
     passthroughRecovery: for (;;) {
+      if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") {
+        const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop,
+          remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed),
+          refresh: () => transportState.recoverAutoPausedSelection(parsed), refetch: () => rebuildAndRefetch("oauth-account-429"), setPermit: permit => { sendBudgetState.pendingHopPermit = permit; } });
+        if (!recovered) break;
+        if ("failed" in recovered) return recovered.failed;
+        upstreamResponse = recovered; continue passthroughRecovery;
+      }
+
+
     if (
-      upstreamResponse.status === 401
+      upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
       && (admissionState.authCtx.kind === "main-pool" || admissionState.authCtx.kind === "pool")
       && usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)
       && codex401ReplayKind === null
@@ -1293,6 +1292,7 @@ export async function preparePassthroughExchange(
             dispatchOverride: oauthDispatch(request, parsed, "passthrough"),
             providerName: route.providerName,
             modelId: route.modelId,
+            accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers),
             onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId),
             beforeDispatch: headers => {
               replayGuard?.(headers);
@@ -1336,14 +1336,14 @@ export async function preparePassthroughExchange(
       continue passthroughRecovery;
     }
 
-    if (codex401ReplayKind !== null && upstreamResponse.status === 401) break;
+    if (codex401ReplayKind !== null && upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)) break;
 
     // Native Responses providers return before the generic adapter recovery loop below. Keep
     // their OAuth contract identical: one pre-stream 401 forces a credential refresh and one
     // rebuilt replay. xAI's current subscription models use this branch now that their official
     // Grok CLI catalog declares the Responses backend.
     if (
-      upstreamResponse.status === 401
+      upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
       && isOAuth401ReplayProvider
       && transportState.sentOAuthSnapshot
       && !oauth401ReplayAttempted
@@ -1440,6 +1440,7 @@ export async function preparePassthroughExchange(
                 dispatchOverride: oauthDispatch(request, parsed, "passthrough"),
                 providerName: route.providerName,
                 modelId: route.modelId,
+                accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers),
                 onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId),
                 beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider)
                   ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined,
@@ -1586,6 +1587,7 @@ export async function preparePassthroughExchange(
                 dispatchOverride: oauthDispatch(request, parsed, "passthrough"),
                 providerName: route.providerName,
                 modelId: route.modelId,
+                accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers),
                 onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId),
                 beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider)
                   ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined,
@@ -1600,6 +1602,7 @@ export async function preparePassthroughExchange(
         return transportFailureResponse(err);
       }
     }
+    const keyRotation = await rotatePassthroughKeyOn429({ config, route, parsed, response: upstreamResponse, adapterName: transportState.adapter.name, budgetSpent: () => sendBudgetExhausted(transientSendAttempts()), refetch: () => rebuildAndRefetch("key-429") }); if (keyRotation) { if ("failed" in keyRotation) return keyRotation.failed; upstreamResponse = keyRotation; continue passthroughRecovery; }
     const captureAffinityResponse = (
       response: Response,
       captureAuthCtx: CodexAuthContext = admissionState.authCtx,
@@ -1608,18 +1611,7 @@ export async function preparePassthroughExchange(
         || captureAuthCtx.kind === "pool"
         || captureAuthCtx.kind === "main-pool",
     ): void => {
-      if (!isCanonicalOpenAiForwardProvider(route.provider)) return;
-      captureCodexAffinityDiagnostic({
-        inboundHeaders: req.headers,
-        outboundHeaders: captureRequest.headers,
-        authKind: captureAuthCtx.kind,
-        accountMode: route.codexAccountMode,
-        fixedAccount: isFixedCodexAccount(captureAuthCtx),
-        credentialSubstituted,
-        accountGatedModel: ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId),
-        wireModelNormalized: parsed.modelId !== route.modelId,
-        status: response.status,
-      });
+      capturePassthroughAffinity(response, req.headers, captureRequest.headers, captureAuthCtx, route, parsed.modelId, credentialSubstituted);
     };
     recordDistributedResponseFailure(lastDistributedResponse ?? upstreamResponse, config, route.providerName, options.abortSignal);
     captureAffinityResponse(upstreamResponse);
@@ -1646,7 +1638,9 @@ export async function preparePassthroughExchange(
         options.abortSignal,
         parsed.modelId,
       );
-      if (model400Denial !== undefined) {
+      if (wasAccountAutoPaused(upstreamResponse)) {
+        poolRetryOutcome = upstreamResponse.status;
+      } else if (model400Denial !== undefined) {
         // Spend this refusal on more than one retry. It is the account's own authenticated
         // answer about this model, and the roster cache that selection otherwise reads expires
         // five minutes after a catalog sync fills it -- so without remembering this, the next
@@ -1679,7 +1673,7 @@ export async function preparePassthroughExchange(
         // ladder does have one — retrying the account the refreshed roster still grants — and
         // keeps it. An earlier revision also broke here on a non-400 outcome, which no test could
         // justify because this flag already produced the identical result.
-        const storedReplaySpent = codex401ReplayKind === "stored";
+        const storedReplaySpent = codex401ReplayKind === "stored" && !wasAccountAutoPaused(upstreamResponse);
         const retry = await retryCodexPoolOnAlternateAccount({
           callerAuthHeaders,
           config,
@@ -1698,7 +1692,7 @@ export async function preparePassthroughExchange(
           httpOnly: canonicalBufferedJson,
           preOutputRefusalHold: preOutputRefusalHold(),
           onResponse: (response, retryAuthCtx, retryRequest) => {
-            adoptCodexWsStage(response);
+            adoptCodexWsStage(response, logCtx);
             captureAffinityResponse(
               response,
               retryAuthCtx,
@@ -1945,6 +1939,9 @@ export async function preparePassthroughExchange(
     break;
     }
 
+  upstreamResponse = await guardPassthroughCompletion(
+    upstreamResponse, request, config, upstream.signal, logCtx, rebuildAndRefetch,
+  );
   // Where this relay dials upstream. Local infrastructure (loopback / private / `.local` / `.lan`)
   // is operator-trusted and its silent phases are normal, so an unset stall budget resolves to
   // disabled for it. Provider rotation above keeps the same endpoint origin, so the routed

package/src/server/responses/passthrough-key-rotation.ts

@@ -0,0 +1,40 @@
+import { isNonReplayableResponse } from "../../lib/upstream-retry";
+import { hasKeyPoolFailover, rotateProviderTransportOn429 } from "../../providers/key-failover";
+import type { RouteResult } from "../../router";
+import type { OcxConfig, OcxParsedRequest } from "../../types";
+import { bindRouteReasoningReplayScope } from "./core-replay";
+
+// Passthrough's credential moves share one import site so passthrough-dispatch.ts stays under its line cap.
+export { recoverAutoPausedAccount } from "./account-auto-pause-recovery";
+
+/**
+ * Key-pool failover for a native passthrough 429 before any output. This wire returns before the
+ * adapter loop's key-429 arm, so without it a pooled key's 429 reached the client while the same
+ * refusal rotates on the adapter and chat-native wires. Distributed (round-robin) selection already
+ * cools the key on the send and deliberately answers without a replay. A synthesized replay refusal
+ * is not a provider rate limit and must not cool a key. A spent send budget is checked before any
+ * key is cooled, so it keeps the real 429 body. Returns undefined when nothing rotated.
+ */
+export async function rotatePassthroughKeyOn429<T>(options: {
+  config: OcxConfig;
+  route: Pick<RouteResult, "provider" | "providerName">;
+  parsed: OcxParsedRequest;
+  response: Response;
+  adapterName: string;
+  budgetSpent: () => boolean;
+  refetch: () => Promise<T>;
+}): Promise<T | undefined> {
+  const { config, route, parsed, response, adapterName, refetch } = options;
+  if (response.status !== 429 || isNonReplayableResponse(response) || !hasKeyPoolFailover(route.provider)
+    || route.provider._apiKeyAttempt?.distributed || options.budgetSpent()) return;
+  const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, {
+    retryAfter: response.headers.get("retry-after"),
+    attemptedKey: route.provider.apiKey,
+    promptCacheKey: parsed.options.promptCacheKey,
+  });
+  if (!rotated) return;
+  try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
+  route.provider = rotated;
+  bindRouteReasoningReplayScope({ parsed, providerName: route.providerName, provider: rotated, adapterName });
+  return refetch();
+}