@bitkyc08/opencodex 2.54.0 → 2.55.0

package/src/server/responses/core.ts

@@ -165,7 +165,7 @@ import {
   shouldResolveOpenAiPassthroughWebSearchBridge,
 } from "../../web-search/passthrough-bridge";
 import { buildImageTool, buildVideoTool, planImageBridge, planVideoBridge, runWithImageBridge, clampImageMaxRounds, IMAGE_GEN_TOOL_NAME, VIDEO_GEN_TOOL_NAME } from "../../images";
-import { describeImagesInPlace, isModelTextOnly, planVisionSidecar, resolveOpenAiVisionModel, shouldResolveOpenAiVisionSidecar, stripImagesInPlace } from "../../vision";
+import { describeImagesInPlace, planVisionSidecar, requiresVisionPreprocessing, resolveOpenAiVisionModel, shouldResolveOpenAiVisionSidecar, stripImagesInPlace } from "../../vision";
 import { createAdapterEventQueue, preflightAdapterEvents, type AdapterEventQueue } from "../../adapters/run-turn-queue";
 import {
   applyCodexAuthContextToProvider,
@@ -223,7 +223,20 @@ import {
   isTransientUpstreamStatus,
   prepareSameTarget429Wait,
   sleepWithAbort,
+  TRANSIENT_RETRY_MAX_ATTEMPTS,
+  SendBudgetExhaustedError,
+  type TransientSendBudget,
 } from "../../lib/upstream-retry";
+import {
+  createRequestExecutionBudget,
+  isRequestExecutionBudget,
+  type SendClass,
+  type SingleUseDispatchPermit,
+} from "../../lib/request-execution-budget";
+import {
+  chargeWorkflowSends,
+  workflowSendCeilingReached,
+} from "../../lib/workflow-budget";
 import {
   ForwardAdmissionCredentialError,
   hasForwardableCodexBearer,
@@ -781,6 +794,18 @@ function isEncryptedFunctionOutputRejection(bodyText: string): boolean {
   }
 }
 
+/**
+ * #4469: reasoning encrypted_content is minted per caller identity, so replaying it under a
+ * different caller is rejected with "reasoning `encrypted_content` was not issued to this
+ * caller". Substring checks tolerate the optional backticks and a leading or trailing
+ * sentence, while the "was not issued to this caller" anchor plus an encrypted-content or
+ * reasoning subject keep unrelated invalid_request_error prose from gaining a hidden resend.
+ */
+function isReasoningBlobCallerMismatchMessage(message: string): boolean {
+  if (!message.includes("was not issued to this caller")) return false;
+  return message.includes("encrypted_content") || message.includes("reasoning");
+}
+
 function isSelfIdentifiedOpaqueBlobRejection(bodyText: string): boolean {
   if (isEncryptedFunctionOutputRejection(bodyText)) return true;
   try {
@@ -793,7 +818,7 @@ function isSelfIdentifiedOpaqueBlobRejection(bodyText: string): boolean {
   try {
     const payload = JSON.parse(bodyText) as unknown;
     if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false;
-    const record = payload as { code?: unknown; error?: unknown };
+    const record = payload as { code?: unknown; type?: unknown; message?: unknown; error?: unknown };
 
     if (record.error && typeof record.error === "object" && !Array.isArray(record.error)) {
       const error = record.error as { type?: unknown; code?: unknown; message?: unknown };
@@ -807,9 +832,23 @@ function isSelfIdentifiedOpaqueBlobRejection(bodyText: string): boolean {
             " could not be verified. Reason: Encrypted content could not be decrypted or parsed.",
           )
         ) return true;
+        // #4469: the caller-mismatch wording arrives without a dedicated code, so the
+        // message itself is the identity. It is not gated on code being null — the upstream
+        // may attach a generic code — because the anchored phrase is already specific.
+        if (typeof error.message === "string" && isReasoningBlobCallerMismatchMessage(error.message)) {
+          return true;
+        }
       }
     }
 
+    // The flat stream-error envelope carries type/message at the top level rather than under
+    // an error object; the same anchored identity applies there.
+    if (
+      record.type === "invalid_request_error"
+      && typeof record.message === "string"
+      && isReasoningBlobCallerMismatchMessage(record.message)
+    ) return true;
+
     if (record.code !== "invalid-argument" || typeof record.error !== "string") return false;
     return record.error.startsWith("Could not decode the compaction blob")
       || record.error.startsWith("Could not decrypt the provided encrypted_content");
@@ -824,8 +863,9 @@ function isSelfIdentifiedOpaqueBlobRejection(bodyText: string): boolean {
  * The outbound-body check is intentional: the inbound transcript may contain a proxy envelope or
  * compaction blob that the adapter already lowered, in which case a replay would be byte-identical.
  * OpenAI usually exposes a dedicated nested code; ChatGPT also emits one exact code-less
- * unverifiable-ciphertext message. xAI's code is generic, so its two concrete decoder error
- * identities are also required. Unrelated error prose must never gain a hidden resend.
+ * unverifiable-ciphertext message, and #4469 added the anchored caller-mismatch wording for
+ * reasoning blobs minted under a different caller. xAI's code is generic, so its two concrete
+ * decoder error identities are also required. Unrelated error prose must never gain a hidden resend.
  */
 export function shouldAttemptOpaqueBlobRecovery(args: {
   status: number;
@@ -1259,6 +1299,10 @@ interface CodexPoolAccountRetryArgs {
     translatorBudget: TranslatorBudget;
     turnAdmissionLease?: AdmissionLease;
     resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
+    /** The logical request's execution budget: the account move is its fourth send. */
+    sendBudget?: TransientSendBudget;
+    /** Root workflow this turn belongs to, so the move is charged there as well. */
+    workflowRootId?: string;
   };
   firstAuthCtx: Extract<CodexAuthContext, { kind: "pool" | "main-pool" }>;
   firstResponse: Response;
@@ -1453,6 +1497,27 @@ async function retryCodexPoolOnAlternateAccount(
     recordUnmovedTransientOutcome();
     return { kind: "no-alternate" };
   }
+  // An account move is the guarded profile's fourth send and draws the single shared
+  // final-recovery reserve. Nothing bounded it per request before: `excludeAccountId` excludes
+  // only the account that just failed, and the caller's recovery loop can return here after the
+  // alternate fails too, so one request could walk the pool an account at a time. The permit is
+  // consumed immediately before the physical send, so a resolution that finds no alternate
+  // costs nothing.
+  const executionBudget = isRequestExecutionBudget(args.options.sendBudget)
+    ? args.options.sendBudget
+    : undefined;
+  let accountMovePermit: SingleUseDispatchPermit | undefined;
+  if (!retryAuthCtx && executionBudget) {
+    const decision = executionBudget.reserveDispatch({
+      sendClass: "account-failover",
+      targetKey: `${route.providerName}|${route.modelId}|alternate-account`,
+    });
+    if (!decision.allowed) {
+      recordUnmovedTransientOutcome();
+      return { kind: "no-alternate" };
+    }
+    accountMovePermit = decision.permit;
+  }
   try {
     retryAuthCtx ??= await resolveCodexAuthContext(
         callerAuthHeaders,
@@ -1591,6 +1656,18 @@ async function retryCodexPoolOnAlternateAccount(
   let upstreamResponse: Response;
   try {
     while (true) {
+      // The same-account gated-model 400 ladder below keeps its own `maxRetrySends` bound and
+      // does not take the reserve again; only the move itself does.
+      if (accountMovePermit) {
+        const charged = accountMovePermit.use();
+        accountMovePermit = undefined;
+        if (!charged) {
+          recordUnmovedTransientOutcome();
+          return { kind: "no-alternate" };
+        }
+        // The move is a physical send like any other, so the root workflow is charged too.
+        chargeWorkflowSends(args.options.workflowRootId, 1);
+      }
       noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
       try {
         upstreamResponse = await fetchWithHeaderTimeout(
@@ -1874,6 +1951,12 @@ export interface HandleResponsesOptions {
   onStoredPool401ReplayDispatched?: () => void;
   /** Caller-owned for Chat/Claude replay; omitted only at genuine Responses ingress. */
   translatorBudget?: TranslatorBudget;
+  /**
+   * Transient sends already spent by this logical request. Combo children inherit the parent's
+   * holder through the options spread, so a fan-out shares one allowance instead of taking a
+   * fresh one per target (#4546).
+   */
+  sendBudget?: TransientSendBudget;
   /**
    * Terminal vision-describe marker (roadmap 180): true when the inbound
    * request IS the vision sidecar's own loopback describe call. The plan site
@@ -3420,6 +3503,9 @@ export async function handleResponses(
       visionDescribeTerminal: options.visionDescribeTerminal === true
         || req.headers.get("x-opencodex-vision-describe") === "1",
       translatorBudget,
+      // Created once at genuine ingress; a combo child arrives with the parent's holder already
+      // in options and must not start a fresh allowance.
+      sendBudget: options.sendBudget ?? createRequestExecutionBudget(),
     });
     return ownsBudget ? finalizeOwnedTranslatorBudget(response, translatorBudget) : response;
   } catch (error) {
@@ -4187,6 +4273,13 @@ async function handleResponsesInner(
     ? `${route.providerName}-${route.codexAccountNamespace}`
     : formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config);
   logCtx.accountLogLabel = codexAuthContextLogLabel(authCtx, config);
+  // A move is the expensive event: it discards the prefix warmed on the previous account. Record
+  // it as an event with its cause, so the operator reads it off one line instead of inferring it
+  // from account labels across many (#4546).
+  if (authCtx.kind === "pool" && authCtx.affinityDecision) {
+    logCtx.affinity = authCtx.affinityDecision.move;
+    logCtx.affinityReason = authCtx.affinityDecision.reason;
+  }
   // Seed an account-derived scope before final adapter binding. Cursor never treats it as
   // authoritative: bindRouteReasoningReplayScope replaces it with the exact route owner or a
   // per-request fail-closed sentinel after the final provider and credential are known.
@@ -4769,7 +4862,7 @@ async function handleResponsesInner(
   const routedCompaction = parsed._compactionRequest === true
     && !isCanonicalOpenAiForwardProvider(route.provider);
   const needsOpenAiVision = !visionDescribeTerminal
-    && shouldResolveOpenAiVisionSidecar(config, route.provider, route.modelId, parsed);
+    && shouldResolveOpenAiVisionSidecar(config, route.provider, route.modelId, parsed, route.providerName);
   const needsOpenAiSearch = !routedCompaction && !adapter.runTurn
     && (shouldResolveOpenAiWebSearchSidecar(config, parsed, isPassthrough)
       || shouldResolveOpenAiPassthroughWebSearchBridge(route.provider, parsed, isPassthrough));
@@ -4839,7 +4932,7 @@ async function handleResponsesInner(
   const visionPlan = visionDescribeTerminal
     ? undefined
     : planVisionSidecar(config, route.provider, route.modelId, parsed, openAiSidecar, {
-      admission: options.admission, codexAuthPolicy: options.codexAuthPolicy,
+      admission: options.admission, codexAuthPolicy: options.codexAuthPolicy, providerName: route.providerName,
     });
   const recordSidecarOutcome = openAiSidecar?.recordOutcome;
   if (visionPlan) {
@@ -4851,9 +4944,9 @@ async function handleResponsesInner(
       recordSidecarOutcome,
       translatorBudget,
     );
-  } else if (isModelTextOnly(route.provider, route.modelId)) {
-    // Sidecar-covered model but NO plan (no forward provider / missing forwarded auth / sidecar
-    // disabled): fail closed — never forward raw images to a text-only upstream.
+  } else if (requiresVisionPreprocessing(config, route.provider, route.modelId, route.providerName)) {
+    // Image capability is not positively proven but no sidecar plan is dispatchable: fail closed.
+    // Never forward raw image bytes to an unverified upstream.
     stripImagesInPlace(parsed, translatorBudget);
   }
 
@@ -4937,6 +5030,73 @@ async function handleResponsesInner(
     routedMuseToolNameAliases = builtRequest.convertedMuseToolNameAliases ?? new Map();
   };
 
+  // One transient-retry budget for the whole LOGICAL request, read ABOVE the passthrough branch
+  // so that branch shares it too. It used to be a local declared below, which put it in the
+  // temporal dead zone for the passthrough sends and left each recovery leg taking the helper's
+  // fresh default of 3. It is now a holder carried on options, so a combo child inherits the
+  // parent's spend instead of starting over per target -- both halves of the measured
+  // amplification in #4546.
+  const sendBudget = options.sendBudget ?? createRequestExecutionBudget();
+  // The root workflow is the user-visible task. A per-request cap cannot bound a fan-out that
+  // sends once per child seven hundred times, so every send charged to the request is charged
+  // to the root as well (#4546).
+  const workflowRootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined;
+  const noteTransientSends = (used: number): void => {
+    const charged = Math.max(0, used);
+    sendBudget.used += charged;
+    chargeWorkflowSends(workflowRootId, charged);
+  };
+  // Refused before any dispatch, and deliberately not by evicting the root's ledger entry:
+  // dropping the record to make room would hand the fan-out a fresh allowance, which is the
+  // laundering this ceiling exists to stop. The client is told the task needs a new grant
+  // rather than being given a synthetic upstream error.
+  if (workflowSendCeilingReached(workflowRootId)) {
+    return formatErrorResponse(
+      429,
+      "workflow_budget_exhausted",
+      "This task has used its whole send budget, so no further upstream request was made. Requests already in flight settle as they finish.",
+    );
+  }
+  // No floor. Math.max(1, ...) meant an exhausted request still funded one send on every
+  // recovery leg, so a bounded per-leg allowance never became a bounded per-request one.
+  const remainingTransientSendBudget = (budget: number): number =>
+    isRequestExecutionBudget(sendBudget)
+      ? sendBudget.remainingBaseSends(budget)
+      : Math.max(0, budget - sendBudget.used);
+  // The adapter contract needs the full budget, not just the counter. options.sendBudget is
+  // typed as the narrow holder so a caller that predates this can still pass one, so narrow it
+  // once here rather than asserting at each adapter call site.
+  const adapterSendBudget = isRequestExecutionBudget(sendBudget) ? sendBudget : undefined;
+  const sendBudgetExhausted = (): boolean =>
+    remainingTransientSendBudget(TRANSIENT_RETRY_MAX_ATTEMPTS) === 0;
+  /**
+   * How many sends a recovery leg may make, and the permit that authorises the last one.
+   *
+   * The base allowance is spent first. Once it is gone a recovery class may still draw the
+   * single shared final-recovery reserve -- which is what keeps the validated sanitized rebuild
+   * after a 5xx streak alive at four total sends -- but an account move and a rebuild cannot
+   * each take one. `countedExternally` is set because these legs run through the retry helper,
+   * which reports the same send again through `onSendsConsumed`.
+   */
+  const recoverySendAllowance = (
+    cap: number,
+    sendClass: SendClass,
+    targetKey: string,
+  ): { attempts: number; permit?: SingleUseDispatchPermit } => {
+    const base = remainingTransientSendBudget(cap);
+    if (base > 0) return { attempts: base };
+    if (!isRequestExecutionBudget(sendBudget)) return { attempts: 0 };
+    const decision = sendBudget.reserveDispatch({ sendClass, targetKey, countedExternally: true });
+    return decision.allowed ? { attempts: 1, permit: decision.permit } : { attempts: 0 };
+  };
+  /**
+   * Both classes share the one reserve, so this only changes what the decision is called --
+   * but a recovery event that says "repair" when a credential refresh drove it is the kind of
+   * mislabelled evidence #4592 existed to stop.
+   */
+  const recoveryClassFor = (recovery: AttemptRecoveryKind): SendClass =>
+    /401|429|oauth|rate-limit|key/.test(recovery) ? "auth-recovery" : "repair";
+
   if ("passthrough" in adapter && adapter.passthrough && !routedCompaction) {
     let hostAdmissionLease = pendingHostAdmissionLease;
     pendingHostAdmissionLease = null;
@@ -5409,6 +5569,15 @@ async function handleResponsesInner(
         releaseCodexAuthContextProbeLease(authCtx);
         return clientCancelledResponse();
       }
+      // A budget refusal is a proxy decision, not an upstream fault. Reporting it as
+      // 502 upstream_error would blame the provider for a limit this process applied, and
+      // would record a fake reachability failure against the account's health.
+      if (err instanceof SendBudgetExhaustedError) {
+        releaseUpstreamHostAdmission(hostAdmissionLease);
+        hostAdmissionLease = null;
+        releaseCodexAuthContextProbeLease(authCtx);
+        return formatErrorResponse(429, "request_send_budget_exhausted", err.message);
+      }
       const localRefusal = mapCodexAuthContextErrorToResponse(unwrapUpstreamRetryEvidenceError(err), {
         now: Date.now(), accountSelector: route.codexAccountNamespace,
       });
@@ -5479,7 +5648,7 @@ async function handleResponsesInner(
             // retry wrapper replaces — proves the host was reached (#914 review).
             .then(adoptObservedResponse);
         },
-        { abortSignal: upstream.signal, label: safeHostLabel(request.url) },
+        { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(TRANSIENT_RETRY_MAX_ATTEMPTS), onSendsConsumed: noteTransientSends },
       );
     } catch (err) {
       return transportFailureResponse(err);
@@ -5540,8 +5709,21 @@ async function handleResponsesInner(
       const rebuiltBodyRefusal = refuseOversizedOutboundBody(request);
       if (rebuiltBodyRefusal) return { failed: rebuiltBodyRefusal };
       try {
+        // The base allowance is spent first; once it is gone this leg may still draw the one
+        // shared final-recovery reserve, which is what keeps a validated sanitized rebuild
+        // after a 5xx streak alive at four total sends instead of dying at three.
+        const allowance = recoverySendAllowance(
+          TRANSIENT_RETRY_MAX_ATTEMPTS,
+          recoveryClassFor(recovery),
+          `${route.providerName}|${route.modelId}|${recovery}`,
+        );
         return await fetchWithTransientRetry(
           innerRecovery => {
+            // Gated on the return, not fire-and-forget: a consumed permit means this leg
+            // already sent once, and letting the second call through would be a free send.
+            if (allowance.permit && !allowance.permit.use()) {
+              throw new SendBudgetExhaustedError(safeHostLabel(request.url));
+            }
             noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, innerRecovery ?? recovery);
             return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({
               method: request.method,
@@ -5559,7 +5741,7 @@ async function handleResponsesInner(
               route.provider.authMode === "forward")
               .then(adoptObservedResponse);
           },
-          { abortSignal: upstream.signal, label: safeHostLabel(request.url) },
+          { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts, onSendsConsumed: noteTransientSends },
         );
       } catch (err) {
         return { failed: transportFailureResponse(err) };
@@ -5682,6 +5864,10 @@ async function handleResponsesInner(
       && isOAuth401ReplayProvider
       && sentOAuthSnapshot
       && !oauth401ReplayAttempted
+      // Refused here, before the 401 body is cancelled: once it is gone the request can only
+      // answer with a synthetic 502, which would report a proxy budget decision as an upstream
+      // fault and throw away the credential evidence the client needs.
+      && !sendBudgetExhausted()
     ) {
       oauth401ReplayAttempted = true;
       try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
@@ -5779,7 +5965,7 @@ async function handleResponsesInner(
               route.provider.authMode === "forward")
               .then(adoptObservedResponse);
           },
-          { abortSignal: upstream.signal, label: safeHostLabel(request.url) },
+          { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(TRANSIENT_RETRY_MAX_ATTEMPTS), onSendsConsumed: noteTransientSends },
         );
       } catch (err) {
         return transportFailureResponse(err);
@@ -5832,6 +6018,10 @@ async function handleResponsesInner(
       upstreamResponse.status === 429
       && rateLimitPolicy !== null
       && rateLimitRetries < rateLimitPolicy.attempts
+      // Checked here rather than inside the helper: prepareSameTarget429Wait releases the 429
+      // body, so a refusal discovered after the wait can no longer return the real rate-limit
+      // answer and would surface a synthetic 502 instead.
+      && !sendBudgetExhausted()
     ) {
       rateLimitRetries += 1;
       // Release unread body + deliberate wait via the shared same-target helper.
@@ -5876,7 +6066,7 @@ async function handleResponsesInner(
               route.provider.authMode === "forward")
               .then(adoptObservedResponse);
           },
-          { abortSignal: upstream.signal, label: safeHostLabel(request.url) },
+          { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(TRANSIENT_RETRY_MAX_ATTEMPTS), onSendsConsumed: noteTransientSends },
         );
       } catch (err) {
         return transportFailureResponse(err);
@@ -5943,7 +6133,7 @@ async function handleResponsesInner(
           route,
           parsed,
           logCtx,
-          options,
+          options: { ...options, workflowRootId },
           firstAuthCtx: authCtx,
           firstResponse: upstreamResponse,
           outcomeStatus: poolRetryOutcome,
@@ -6251,6 +6441,7 @@ async function handleResponsesInner(
         openAiSidecar,
       );
       const webSearchBridgePlan = planPassthroughWebSearchBridge(parsed, route.provider, {
+        providerName: route.providerName,
         isPassthrough: true,
         stream: parsed.stream === true,
         auth: webSearchBridgeAuth,
@@ -6291,7 +6482,7 @@ async function handleResponsesInner(
             providerApiKey: route.provider.apiKey ?? "",
             auth: webSearchBridgeAuth,
             hostedTool: parsed._webSearch,
-            describeImages: isModelTextOnly(route.provider, route.modelId),
+            describeImages: requiresVisionPreprocessing(config, route.provider, route.modelId, route.providerName),
             sidecar: config.webSearchSidecar,
           }),
           // Appending a search result can push the continuation past the ceiling the first leg
@@ -6817,7 +7008,7 @@ async function handleResponsesInner(
   //     can proceed for web-search-only turns
   const wsPlan = !routedCompaction
     ? planWebSearch(config, parsed, false, route.provider, route.modelId, openAiSidecar, {
-      admission: options.admission, codexAuthPolicy: options.codexAuthPolicy,
+      admission: options.admission, codexAuthPolicy: options.codexAuthPolicy, providerName: route.providerName,
     })
     : undefined;
   const imgPlan = !routedCompaction ? await planImageBridge(config, parsed, route.provider) : undefined;
@@ -7516,13 +7707,6 @@ async function handleResponsesInner(
     notifyResponseComplete(json);
     return new Response(JSON.stringify(json), { headers: { "Content-Type": "application/json" } });
   }
-  // One request-scoped transient-retry budget owner, declared here so BOTH the initial send
-  // and the later recovery refetches (429, key/account rotation, OAuth replay) share it. A
-  // per-leg budget would let a request that recovers several times multiply upstream load.
-  let transientSendsUsed = 0;
-  const noteTransientSends = (used: number): void => { transientSendsUsed += Math.max(0, used); };
-  const remainingTransientSendBudget = (budget: number): number =>
-    Math.max(1, budget - transientSendsUsed);
   try {
     initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget });
     refreshRequestToolAliases(initialRequest);
@@ -7565,6 +7749,7 @@ async function handleResponsesInner(
       upstreamResponse = await activeAdapter.fetchResponse(builtInitialRequest, {
         abortSignal: upstream.signal,
         timeoutMs: connectMs,
+        sendBudget: adapterSendBudget,
         stream: parsed.stream,
         executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
               dispatchOverride: oauthDispatch(builtInitialRequest),
@@ -7602,7 +7787,13 @@ async function handleResponsesInner(
           abortSignal: upstream.signal,
           label: safeHostLabel(builtInitialRequest.url),
           ...(transientPolicy
-            ? { attempts: transientPolicy.attempts, onSendsConsumed: noteTransientSends }
+            // Draws the remainder, not the raw policy. A combo child inherits the parent's
+            // holder but used to take a fresh full allowance on its own first send, so the
+            // shared counter was inherited without ever being read as a limit.
+            ? {
+              attempts: remainingTransientSendBudget(transientPolicy.attempts),
+              onSendsConsumed: noteTransientSends,
+            }
             : {}),
         },
       );
@@ -7693,6 +7884,7 @@ async function handleResponsesInner(
             return await activeAdapter.fetchResponse(retryRequest, {
               abortSignal: upstream.signal,
               timeoutMs: connectMs,
+            sendBudget: adapterSendBudget,
               stream: parsed.stream,
               executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
               dispatchOverride: oauthDispatch(retryRequest),
@@ -7711,8 +7903,22 @@ async function handleResponsesInner(
           const refetchWithPolicy = (route.provider.adapter === "google" || refetchTransientPolicy)
             ? fetchWithTransientRetry
             : fetchWithResetRetry;
+          // Same rule as the passthrough rebuild: spend the base allowance first, then the one
+          // shared final-recovery reserve, so a recovery that follows a spent streak still gets
+          // its single send instead of dying at three.
+          const refetchAllowance = refetchTransientPolicy
+            ? recoverySendAllowance(
+              refetchTransientPolicy.attempts,
+              recoveryClassFor(recovery),
+              `${route.providerName}|${route.modelId}|${recovery}`,
+            )
+            : undefined;
           return await refetchWithPolicy(
-            recoveryKind => fetchWithHeaderTimeout(retryRequest.url,
+            recoveryKind => {
+              if (refetchAllowance?.permit && !refetchAllowance.permit.use()) {
+                throw new SendBudgetExhaustedError(safeHostLabel(retryRequest.url));
+              }
+              return fetchWithHeaderTimeout(retryRequest.url,
               applyUpstreamRecoveryInit({
                 method: retryRequest.method, headers: retryRequest.headers, body: retryRequest.body,
               }, recoveryKind), upstream.signal, connectMs, parsed.stream,
@@ -7720,13 +7926,14 @@ async function handleResponsesInner(
               dispatchOverride: oauthDispatch(retryRequest),
                 providerName: route.providerName,
                 modelId: route.modelId,
-              })),
+              }));
+            },
             {
               abortSignal: upstream.signal,
               label: safeHostLabel(retryRequest.url),
-              ...(refetchTransientPolicy
+              ...(refetchAllowance
                 ? {
-                  attempts: remainingTransientSendBudget(refetchTransientPolicy.attempts),
+                  attempts: refetchAllowance.attempts,
                   onSendsConsumed: noteTransientSends,
                 }
                 : {}),
@@ -7752,6 +7959,7 @@ async function handleResponsesInner(
         && isOAuth401ReplayProvider
         && sentOAuthSnapshot
         && !oauth401ReplayAttempted
+        && !sendBudgetExhausted()
       ) {
         oauth401ReplayAttempted = true;
         try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
@@ -7846,6 +8054,7 @@ async function handleResponsesInner(
         upstreamResponse.status === 429
         && rateLimitPolicy !== null
         && rateLimitRetries < rateLimitPolicy.attempts
+        && !sendBudgetExhausted()
       ) {
         rateLimitRetries += 1;
         // Release unread body + deliberate wait via the shared same-target helper.
@@ -8233,6 +8442,7 @@ async function handleResponsesInner(
           return await activeAdapter.fetchResponse(builtContinuationRequest, {
             abortSignal: upstream.signal,
             timeoutMs: connectMs,
+              sendBudget: adapterSendBudget,
             stream: nextParsed.stream,
             executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
               dispatchOverride: oauthDispatch(builtContinuationRequest, nextParsed),

package/src/server/search.ts

@@ -4,9 +4,11 @@
  * codex-rs's built-in search client executes CLIENT-SIDE: it POSTs `alpha/search` against the
  * configured base_url with the same ChatGPT bearer auth used for model requests. Under Design B
  * injection base_url is this proxy, so the request otherwise dies on the /v1/* JSON-404 guard.
- * The endpoint is private to the ChatGPT Codex backend, so routed providers and OpenAI API-key
- * providers cannot serve it. Relay the JSON request and response verbatim through the configured
- * ChatGPT forward provider.
+ * The endpoint is private to the ChatGPT Codex backend, so the honest answer while a forward
+ * provider is configured is to copy bytes. When none is, a configured web-search sidecar
+ * (anthropic / xai / gemini / exa) can still answer — see src/web-search/alpha-search.ts.
+ * That fallback never runs while a forward candidate exists, and never borrows a different
+ * paid backend than the one the operator named.
  */
 import { formatErrorResponse } from "../bridge";
 import {
@@ -34,6 +36,7 @@ import {
   type ExactOpenAiSidecarAccount,
 } from "../providers/openai-sidecar";
 import { routeModel } from "../router";
+import { handleAlphaSearchSidecarFallback } from "../web-search/alpha-search";
 import { readJsonRequestBody, resolveInboundBodyLimitBytes } from "./request-decompress";
 import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "./auth-cors";
 import type { RequestLogContext } from "./request-log";
@@ -105,12 +108,7 @@ export async function handleSearch(
   }
   const candidates = listOpenAiForwardSidecarCandidates(config);
   if (candidates.length === 0) {
-    return formatErrorResponse(
-      400,
-      "invalid_request_error",
-      "Built-in web search needs a ChatGPT forward provider, but none is configured in opencodex. "
-      + "Routed and OpenAI API-key providers cannot serve /v1/alpha/search.",
-    );
+    return handleAlphaSearchSidecarFallback(body, config, req.signal, logCtx);
   }
 
   let upstream: Awaited<ReturnType<typeof resolveFirstUsableOpenAiSidecar>>;

package/src/types/config.ts

@@ -668,6 +668,16 @@ export interface OcxConfig {
    * so absence is the only default state this feature has.
    */
   quotaResetNotify?: OcxQuotaResetNotifyConfig;
+  /**
+   * Periodic provider model-catalog refresh (issue #3630). Absent means off: no timer, no
+   * refresh pass, no outcome record.
+   *
+   * Off by default for the same reason every optional subsystem here is: a refresh spends a
+   * live /models call against every enabled provider, and this repository's rule is that a
+   * default install runs no detection code and starts no live timer work. Not in
+   * `getDefaultConfig()` — absence is the only default state this feature has.
+   */
+  catalogAutoRefresh?: OcxCatalogAutoRefreshConfig;
   /** Active provider context limits; native long windows remain within their supported ceilings. */
   providerContextCaps?: Record<string, number>;
   /** Last selected provider caps; retained while a cap is switched off. Not an active limit. */
@@ -850,14 +860,26 @@ export interface OcxConfig {
   pool?: {
     kernel?: boolean;
     /**
-     * Opt-in cache-affinity ordering, off by default.
+     * Cache-affinity ordering for bound Codex threads. **On unless set to `false`.**
      *
-     * With it on, a bound Codex thread keeps its account until that account genuinely cannot
-     * serve, instead of moving the moment usage crosses `autoSwitchThreshold`. Moving a live
+     * A bound Codex thread keeps its account until that account genuinely cannot serve,
+     * instead of moving the moment usage crosses `autoSwitchThreshold`. Moving a live
      * conversation throws away the prompt cache warmed on that account, and a threshold
-     * crossing is a hint rather than evidence the account is spent. Separate from `kernel`
-     * on purpose: that one governs the generic OAuth strategy consumer, and one switch
-     * carrying two unrelated meanings cannot be turned on alone.
+     * crossing is a hint rather than evidence the account is spent.
+     *
+     * This shipped as an opt-in (#4292) and then #4546 measured what the opt-in default
+     * costs: a pool whose accounts all sit in the 80-99% band hands a conversation from
+     * account to account, re-sending the whole prefix every turn, and the install that gets
+     * hurt is precisely the one that never heard of this setting. `false` restores
+     * capacity-first routing for operators who want it.
+     *
+     * Separate from `kernel` on purpose: that one governs the generic OAuth strategy
+     * consumer, and one switch carrying two unrelated meanings cannot be turned on alone.
+     *
+     * Note what this does NOT govern. Unbound placement still follows
+     * `autoSwitchThreshold` and the configured strategy. A bound thread's destination must
+     * have real headroom under either setting, and a transient failure streak holds the
+     * binding under either setting -- neither is a cache-affinity preference.
      */
     cacheAffinity?: boolean;
   };
@@ -1303,3 +1325,26 @@ export interface OcxQuotaResetNotifyConfig {
    */
   command?: string[];
 }
+
+/**
+ * Periodic model-catalog auto-refresh settings (issue #3630).
+ *
+ * Every field is optional and the whole section defaults to off. Each tick converges the
+ * served catalog the same way `ocx sync` does, which costs a live /models call against
+ * every enabled provider — so an install that never asked for this must run no refresh
+ * code and start no timer, matching the optional-subsystem rule the rest of this file
+ * follows.
+ */
+export interface OcxCatalogAutoRefreshConfig {
+  /** Master switch. Default false — no scheduler, no tick, no upstream calls. */
+  enabled?: boolean;
+  /**
+   * Minutes between refresh ticks. Default 60, floor 15, and 0 keeps the timer dormant
+   * while leaving the section configured.
+   *
+   * The floor exists for the same reason src/quota/reset-poller.ts has MIN_INTERVAL_MS:
+   * provider catalogs are cached upstream for minutes, so a faster cadence buys no
+   * freshness and only risks a rate limit against every enabled provider at once.
+   */
+  intervalMinutes?: number;
+}