@bitkyc08/opencodex 2.56.0 → 2.58.0

package/src/server/responses/combo-stream-preflight.ts

@@ -1,4 +1,6 @@
 import type { ResponsesTerminalStatus } from "../../bridge";
+import { comboFailureDecision } from "../../combos";
+import { httpStatusFromTerminalError } from "../../lib/errors";
 import type { RequestLogContext } from "../request-log";
 import { createSseInspector } from "../relay";
 import { MAX_CLIENT_SSE_FRAME_BYTES } from "../sse-frame-buffer";
@@ -30,12 +32,67 @@ const RETRYABLE_ZERO_OUTPUT_INCOMPLETE_REASONS = new Set([
   "upstream_stall_timeout",
 ]);
 
+function bareErrorStatus(payload: unknown): number | undefined {
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined;
+  const event = payload as Record<string, unknown>;
+  if (event.type !== "error") return undefined;
+  const nested = event.error;
+  const error = nested && typeof nested === "object" && !Array.isArray(nested)
+    ? nested as Record<string, unknown>
+    : event;
+  const explicitStatus = [
+    event.status,
+    event.status_code,
+    event.http_status,
+    error.status,
+    error.status_code,
+    error.http_status,
+  ]
+    .map(value => typeof value === "number" && Number.isInteger(value)
+      ? value
+      : typeof value === "string" && /^\d{3}$/.test(value.trim())
+        ? Number(value)
+        : undefined)
+    .find(value => value !== undefined && value >= 400 && value <= 599);
+  const code = typeof error.code === "string"
+    ? error.code
+    : typeof event.code === "string" ? event.code : null;
+  if (explicitStatus === undefined && code === "invalid_request_error") return 400;
+  return explicitStatus ?? httpStatusFromTerminalError({
+    type: typeof error.type === "string" && error.type !== "error" ? error.type : undefined,
+    code,
+    message: typeof error.message === "string"
+      ? error.message
+      : typeof event.message === "string" ? event.message : undefined,
+  });
+}
+
+function bareErrorIsRetryable(payload: unknown): boolean {
+  const status = bareErrorStatus(payload);
+  if (status === undefined || !payload || typeof payload !== "object" || Array.isArray(payload)) {
+    return false;
+  }
+  const event = payload as Record<string, unknown>;
+  const nested = event.error;
+  const error = nested && typeof nested === "object" && !Array.isArray(nested)
+    ? nested as Record<string, unknown>
+    : event;
+  const code = typeof error.code === "string"
+    ? error.code
+    : typeof event.code === "string" ? event.code : null;
+  const message = typeof error.message === "string"
+    ? error.message
+    : typeof event.message === "string" ? event.message : "";
+  return comboFailureDecision(status, message, { code }) === "hop";
+}
+
 function retryableZeroOutputTerminal(payload: unknown): boolean {
   if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false;
   const event = payload as {
     type?: unknown;
     response?: { incomplete_details?: { reason?: unknown } };
   };
+  if (bareErrorIsRetryable(event)) return true;
   if (event.type === "response.failed") return true;
   if (event.type !== "response.incomplete") return false;
   const reason = event.response?.incomplete_details?.reason;
@@ -95,12 +152,17 @@ function failedTerminalResponse(
     ? nested as Record<string, unknown>
     : {};
   const nestedError = terminalResponse.error;
+  const topLevelError = terminalPayload.error;
   const error = nestedError && typeof nestedError === "object" && !Array.isArray(nestedError)
     ? nestedError as Record<string, unknown>
+    : topLevelError && typeof topLevelError === "object" && !Array.isArray(topLevelError)
+      ? topLevelError as Record<string, unknown>
     : {
       type: "upstream_error",
       code: "upstream_server_error",
-      message: logCtx.upstreamError ?? "Provider stream failed before producing output",
+      message: typeof terminalPayload.message === "string"
+        ? terminalPayload.message
+        : logCtx.upstreamError ?? "Provider stream failed before producing output",
     };
   const headers = new Headers(response.headers);
   headers.set("content-type", "application/json");
@@ -117,7 +179,7 @@ function failedTerminalResponse(
       ...(usage && typeof usage === "object" && !Array.isArray(usage) ? { usage } : {}),
     },
   }), {
-    status: logCtx.terminalHttpStatus ?? 502,
+    status: logCtx.terminalHttpStatus ?? bareErrorStatus(terminalPayload) ?? 502,
     headers,
   });
 }
@@ -154,11 +216,12 @@ export async function preflightComboStreamResponse(
   const inspector = createSseInspector({
     logCtx,
     onParsedPayload: payload => {
+      if (terminalStatus !== undefined || outputCommitted || retryableTerminalPayload) return;
       const retryable = retryableTerminal(payload);
       const matchedBareError = retryable && payload !== null && typeof payload === "object"
         && !Array.isArray(payload) && (payload as { type?: unknown }).type === "error";
-      // Only an explicit caller predicate may opt a known bare error into replay.
-      // Default combo classification still commits unknown/error events.
+      // A zero-output bare error is terminal evidence. Explicit client errors stay
+      // committed; unknown and retryable upstream failures may advance the combo.
       if (comboStreamPayloadCommitsOutput(payload) && !matchedBareError) outputCommitted = true;
       if (!payload || typeof payload !== "object" || Array.isArray(payload)) return;
       if (retryable) retryableTerminalPayload = payload as Record<string, unknown>;
@@ -197,7 +260,7 @@ export async function preflightComboStreamResponse(
       }
 
       // A bare error event is not a protocol terminal (terminalStatus stays undefined),
-      // so its exact-message retryable match doubles as the terminal evidence.
+      // so its retryable classification doubles as the terminal evidence.
       if ((terminalStatus === "failed" || terminalStatus === "incomplete"
         || retryableTerminalPayload?.type === "error")
         && !outputCommitted && retryableTerminalPayload) {

package/src/server/responses/compact.ts

@@ -53,6 +53,7 @@ import {
   resolveCodexAuthContext,
   codexPoolAffinityKey,
   codexProbeLeaseId,
+  codexTransientProbeGrant,
   codexProbeQuotaScope,
   releaseCodexAuthContextProbeLease,
   stripCodexRuntimeProviderFields,
@@ -71,7 +72,9 @@ import {
 import {
   applyAccountChangeConversationStateScrub,
   conversationStateBindingFromAuth,
+  accountChangeFileReferenceRefusal,
   rememberServingConversationStateIssuer,
+  conversationCarriesUploadedFiles,
 } from "./account-change-state";
 import {
   TokenRefreshError,
@@ -82,6 +85,7 @@ import {
   fetchWithResetRetry,
   fetchWithTransientRetry,
   applyUpstreamRecoveryInit,
+  isNonReplayableResponse,
   SendBudgetExhaustedError,
   TRANSIENT_RETRY_MAX_ATTEMPTS,
   type UpstreamSendRecovery,
@@ -140,7 +144,6 @@ import {
   catalogModelSupportsServiceTier,
   finishRequestAttempt,
   inspectResponseLogJson,
-  noteAttemptSend,
   readConfiguredCodexServiceTier,
   requestLogSpeedLabel,
   sealRequestAttemptIdentity,
@@ -735,6 +738,14 @@ export async function handleResponsesCompact(
           beginCodexAccountSelection: codexAccountSelectionForTurn(turnAdmissionLease),
           signal: req.signal,
           nativeMainRefreshDependencies: options.nativeMainRefreshDependencies,
+          // #4778: the same retention the regular Responses path passes at its final auth. The
+          // post-429 guard below only declines to move AFTER this resolution has already bound
+          // an account, so without the bit here a quota-driven rebind could have carried the
+          // conversation off its issuing account before that guard is ever consulted -- and an
+          // uploaded file is readable only by the account that received it. Answered from `raw`,
+          // the same object and the same predicate the guard below uses, so the two can never
+          // disagree about which conversations are in scope.
+          retainAccountForUploadedFiles: conversationCarriesUploadedFiles(raw),
         });
         logCtx.accountLogLabel = codexAuthContextLogLabel(authCtx, config);
         const selected = await materializeCodexUpstreamAuthAsync(req.headers, authCtx, {
@@ -789,6 +800,14 @@ export async function handleResponsesCompact(
     {
       const binding = conversationStateBindingFromAuth(authCtx, codexPoolAffinityKey(req.headers));
       if (binding) {
+        // Refused rather than scrubbed: an uploaded file is content the caller attached, not
+        // continuation state the turn can do without.
+        const refusal = accountChangeFileReferenceRefusal({
+          body: raw,
+          bindingKey: binding.bindingKey,
+          servingAccountId: binding.accountId,
+        });
+        if (refusal) return refusal;
         applyAccountChangeConversationStateScrub({
           body: raw,
           bindingKey: binding.bindingKey,
@@ -869,6 +888,7 @@ export async function handleResponsesCompact(
         // replacement (#2887). Also covers the replay's own second 401.
         ...(ctx.kind === "pool" ? { credentialGeneration: ctx.generation } : {}),
         probeQuotaScope: codexProbeQuotaScope(ctx),
+        transientProbe: codexTransientProbeGrant(ctx),
         writerGeneration: ctx.kind === "pool" || ctx.kind === "main-pool"
           ? ctx.writerGeneration
           : undefined,
@@ -1068,6 +1088,10 @@ export async function handleResponsesCompact(
     // — reporting exhausted retries while another pool account sat idle (#913).
     if (
       (upstream.status === 429 || upstream.status === 402)
+      // A replay refusal this proxy synthesized carries 429 for the client's benefit only.
+      // It is not pool quota evidence, and the alternate account below is another send of a
+      // compact turn that may already have been processed.
+      && !isNonReplayableResponse(upstream)
       && !storedPool401ReplayAttempted
       && usesCodexForwardPoolAuth(authCtx, route.provider)
       && !authCtx.fixedAccount
@@ -1082,15 +1106,22 @@ export async function handleResponsesCompact(
       ].filter(Boolean);
       // Build the alternate COMPLETELY before cancelling the first body: if construction
       // throws, the first rejection is still intact and can be returned to the client.
-      const alternate = await resolveAlternateCompactContext({
-        req,
-        admission,
-        config,
-        route,
-        selectedModelId,
-        excludeAccountId: authCtx.accountId,
-        turnAdmissionLease,
-      });
+      // The same reasoning refuses an uploaded-file move here: no alternate can read a file the
+      // issuing account received, so which one is chosen is irrelevant and asking before the
+      // resolution costs nothing. It also reuses the guarantee the comment above depends on --
+      // the first body is still uncancelled -- so the client gets the original rejection rather
+      // than an inaccessible-file error from account B (#4710).
+      const alternate = conversationCarriesUploadedFiles(raw)
+        ? undefined
+        : await resolveAlternateCompactContext({
+          req,
+          admission,
+          config,
+          route,
+          selectedModelId,
+          excludeAccountId: authCtx.accountId,
+          turnAdmissionLease,
+        });
       // Resolution can await a credential refresh, so the client may have gone away
       // while we were choosing B. Re-check before spending anything: recording A,
       // cancelling its body, and sending B are all observable side effects, and B's
@@ -1193,8 +1224,14 @@ export async function handleResponsesCompact(
     const bufferedErrorText = buffered.ok
       ? ""
       : await buffered.clone().text().catch(() => "");
-    const explicitQuotaStatus = buffered.status === 429 || buffered.status === 402;
-    const bodyInferredQuota = !buffered.ok
+    // The client-facing 429 of a synthesized replay refusal says nothing about this
+    // account's quota. Pool accounting keeps reading it as the transport failure it is,
+    // which is also what it recorded before the status was corrected for the client.
+    const replayRefused = isNonReplayableResponse(upstream);
+    const explicitQuotaStatus = !replayRefused
+      && (buffered.status === 429 || buffered.status === 402);
+    const bodyInferredQuota = !replayRefused
+      && !buffered.ok
       && !explicitQuotaStatus
       && isRateLimitOrQuotaFailureMessage(bufferedErrorText);
     const quotaFailure = explicitQuotaStatus || bodyInferredQuota;
@@ -1207,7 +1244,11 @@ export async function handleResponsesCompact(
     // A body-confirmed quota failure can arrive behind a generic 5xx. Record it as
     // quota evidence; otherwise preserve the real upstream status so a local buffering
     // failure after a 200 cannot soft-avoid a healthy account or rotate a thread.
-    recordCompactPoolOutcome(outcomeCtx, bodyInferredQuota ? 429 : upstream.status, { retryAfter, resetAt });
+    recordCompactPoolOutcome(
+      outcomeCtx,
+      bodyInferredQuota ? 429 : replayRefused ? 502 : upstream.status,
+      { retryAfter, resetAt },
+    );
     // Lift usage and response metadata from the buffered upstream JSON into the
     // request log; the routed branch gets the same through handleResponses. The
     // synthetic buffer errors are not upstream bodies and stay uninspected.

package/src/server/responses/core-auth.ts

@@ -138,6 +138,7 @@ export async function resolveResponsesCodexAuth(
   route: RouteResult,
   options: HandleResponsesOptions,
   credentialDomainWasRewritten = false,
+  retainAccountForUploadedFiles = false,
 ): Promise<ResponsesAuthResolution> {
   try {
     let authInputHeaders = codexRouteCredentialDomainHeaders(
@@ -232,6 +233,7 @@ export async function resolveResponsesCodexAuth(
         resolveCodexModelEntitlements: options.resolveCodexModelEntitlements,
         signal: options.abortSignal,
         nativeMainRefreshDependencies: options.nativeMainRefreshDependencies,
+        retainAccountForUploadedFiles,
       });
       options.onCodexAuthContextResolved?.(authCtx);
     } else {

package/src/server/responses/core-codex-account.ts

@@ -17,6 +17,7 @@ import {
   resetUpstreamHostHealth,
 } from "../../codex/upstream-host-health";
 import { safeOriginLabel, fetchWithHeaderTimeout, providerFetch } from "./fetch-helpers";
+import { classifyPoolRecoveryDispatch } from "../../routing/probe-lease";
 import { formatErrorResponse } from "../../bridge";
 import { readBoundedResponseBody } from "../../lib/bounded-body";
 import { upstreamErrorMessageFromPayload, isRateLimitOrQuotaFailureMessage } from "../../lib/errors";
@@ -40,6 +41,7 @@ import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account";
 import { slugsEquivalent } from "../../providers/slug-codec";
 import {
   codexProbeLeaseId,
+  codexTransientProbeGrant,
   codexProbeQuotaScope,
   releaseCodexAuthContextProbeLease,
   resolveCodexAuthContext,
@@ -60,13 +62,14 @@ import { bindRouteReasoningReplayScope } from "./core-replay";
 import {
   conversationStateBindingFromAuth,
   applyAccountChangeConversationStateScrub,
+  conversationCarriesUploadedFiles,
 } from "./account-change-state";
 import {
   recordAdapterReasoning,
   recordAdapterTier,
   sealRequestAttemptIdentity,
   recordAttemptCredentialSource,
-  noteAttemptSend,
+  noteProviderAttemptSend,
 } from "../request-log";
 import { codexAuthContextLogLabel } from "../../codex/account-label";
 import { chargeWorkflowSends } from "../../lib/workflow-budget";
@@ -221,7 +224,17 @@ export async function shouldRetryCodexPoolAccountQuota(
   // A post-send WebSocket gateway status must not become a second account's send; the
   // body carries no quota evidence either, but the marker is the contract, not the prose.
   if (isNonReplayableResponse(response)) return false;
-  if (response.status === 402 || response.status === 429) return true;
+  if (response.status === 402 || response.status === 429) {
+    // Status alone used to authorize the move, which is right for a limit the ACCOUNT owns and
+    // wrong for one it merely belongs to. An organization- or project-scoped exhaustion refuses
+    // every credential inside that organization, so the second account meets the same counter
+    // and the only thing the rotation buys is a second cold prompt prefix (#4546). Positive
+    // evidence is required to withhold it: the helper fails closed, so an unreadable or
+    // ambiguous body keeps the broad #584 behaviour unchanged, and `rate_limit_exceeded`,
+    // `slow_down` and plan-level exhaustion still rotate exactly as before.
+    const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection");
+    return await codexScopedExhaustionCode(response, { signal }) === undefined;
+  }
   if (response.status < 500 || response.status >= 600) return false;
   try {
     // Reject malformed UTF-8 instead of matching quota words around replacement characters.
@@ -459,6 +472,7 @@ export async function retryCodexPoolOnAlternateAccount(
       modelId: route.modelId,
       probeLeaseId: codexProbeLeaseId(firstAuthCtx),
       probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
+      transientProbe: codexTransientProbeGrant(firstAuthCtx),
       writerGeneration: firstAuthCtx.writerGeneration,
     });
   };
@@ -489,6 +503,17 @@ export async function retryCodexPoolOnAlternateAccount(
     recordUnmovedTransientOutcome();
     return { kind: "no-alternate" };
   }
+  // An uploaded file is readable only by the account it was sent to, so NO alternate can serve
+  // this body. Which account would be chosen does not change that, which is why this asks before
+  // the resolution rather than after it: refusing here reserves no send, cancels no response, and
+  // leaves the caller holding the first account's rejection to return unchanged (#4710). The
+  // initial-dispatch sites answer with a 400 instead, because there is no earlier response there
+  // to fall back to. A same-account replay -- the gated-model 400 ladder above -- is unaffected,
+  // since it never leaves the issuing account.
+  if (!retryAuthCtx && conversationCarriesUploadedFiles(parsed._rawBody)) {
+    recordUnmovedTransientOutcome();
+    return { kind: "no-alternate" };
+  }
   // An account move is the guarded profile's fourth send and draws the single shared
   // final-recovery reserve. Nothing bounded it per request before: `excludeAccountId` excludes
   // only the account that just failed, and the caller's recovery loop can return here after the
@@ -557,6 +582,7 @@ export async function retryCodexPoolOnAlternateAccount(
         modelId: route.modelId,
         probeLeaseId: codexProbeLeaseId(firstAuthCtx),
         probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
+        transientProbe: codexTransientProbeGrant(firstAuthCtx),
         writerGeneration: firstAuthCtx.writerGeneration,
       });
     }
@@ -588,6 +614,7 @@ export async function retryCodexPoolOnAlternateAccount(
       modelId: route.modelId,
       probeLeaseId: codexProbeLeaseId(firstAuthCtx),
       probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
+      transientProbe: codexTransientProbeGrant(firstAuthCtx),
       writerGeneration: firstAuthCtx.writerGeneration,
       // Retry already advanced the RR ring via excludeAccountId — reuse for promotion.
       ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}),
@@ -693,16 +720,35 @@ export async function retryCodexPoolOnAlternateAccount(
       // The same-account gated-model 400 ladder below keeps its own `maxRetrySends` bound and
       // does not take the reserve again; only the move itself does.
       if (accountMovePermit) {
+        // The pool-wide recovery window is consulted BEFORE the request-local permit is used.
+        // `reserveDispatch` charges at reservation time and `release()` is the only way back, so
+        // using the permit first and refusing afterwards would spend a send the request never
+        // made. An account move is recovery traffic like any other: one request's own budget
+        // cannot see that a thousand other requests are moving at the same moment, which is
+        // precisely the amplification this window exists to bound (#4701).
+        //
+        // A refusal here is not a new failure mode: "no alternate was available" is already the
+        // outcome when the pool has nowhere to move this request to, and it is handled.
+        if (!classifyPoolRecoveryDispatch("retry").admitted) {
+          accountMovePermit.release();
+          accountMovePermit = undefined;
+          // The alternate context was resolved and will not send. Hand back whatever recovery
+          // lease it is holding rather than leaving that account unprobeable.
+          releaseCodexAuthContextProbeLease(retryAuthCtx);
+          recordUnmovedTransientOutcome();
+          return { kind: "no-alternate" };
+        }
         const charged = accountMovePermit.use();
         accountMovePermit = undefined;
         if (!charged) {
+          releaseCodexAuthContextProbeLease(retryAuthCtx);
           recordUnmovedTransientOutcome();
           return { kind: "no-alternate" };
         }
         // The move is a physical send like any other, so the root workflow is charged too.
         chargeWorkflowSends(args.options.workflowRootId, 1);
       }
-      noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
+      noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate);
       try {
         upstreamResponse = await fetchWithHeaderTimeout(
           request.url,
@@ -827,6 +873,7 @@ export function codexForwardTerminalOutcomeRecorder(
         modelId,
         probeLeaseId: codexProbeLeaseId(authCtx),
         probeQuotaScope: codexProbeQuotaScope(authCtx),
+        transientProbe: codexTransientProbeGrant(authCtx),
         writerGeneration: authCtx.writerGeneration,
       });
       return;
@@ -849,6 +896,7 @@ export function codexForwardTerminalOutcomeRecorder(
       modelId,
       probeLeaseId: codexProbeLeaseId(authCtx),
       probeQuotaScope: codexProbeQuotaScope(authCtx),
+      transientProbe: codexTransientProbeGrant(authCtx),
       writerGeneration: authCtx.writerGeneration,
       // A mid-stream terminal can carry a semantic 401 long after the credential was
       // replaced. It is never replayed — the client already saw output — but it must