@bitkyc08/opencodex 2.58.0 → 2.60.0

package/src/web-search/executor.ts

@@ -3,7 +3,14 @@ import { FORWARD_HEADERS } from "../adapters/openai-responses";
 import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
 import { redactSecretString } from "../lib/redact";
 import { sidecarEnter } from "../lib/sidecar-tracker";
-import { applyUpstreamRecoveryInit, fetchWithResetRetry } from "../lib/upstream-retry";
+import {
+  applyUpstreamRecoveryInit,
+  fetchWithResetRetry,
+  releaseResponseBodyBestEffort,
+  retryBackoffDelayMs,
+  sleepWithAbort,
+  RETRY_AFTER_CEILING_MS,
+} from "../lib/upstream-retry";
 import { withUpstreamHttpVersion } from "../lib/upstream-http-version";
 import { parseSidecarSSE, type WebSearchResult } from "./parse";
 import type { CodexUpstreamOutcome } from "../codex/routing";
@@ -35,6 +42,22 @@ export const IMAGE_INSTRUCTION =
 
 /** A search result, or an `error` string when the search couldn't run (surfaced as a tool result). */
 export type SidecarOutcome = WebSearchResult & { error?: string };
+
+/**
+ * Bounded same-search 429 replays for the sidecar POST.
+ *
+ * The forward backend throttles burst sidecar traffic, and without a replay the 429 becomes a
+ * failed tool result that poisons the query for the whole turn (see failedQueries in loop.ts).
+ * 1 initial send + 2 replays; Retry-After is honored as a lower bound and capped by
+ * RETRY_AFTER_CEILING_MS (an instruction past the ceiling ends with the 429 instead of
+ * parking the search). Each wait releases the unread 429 body first so sockets do not
+ * accumulate under a rate-limit storm. Abort or timeout ends the wait through the existing
+ * catch, exactly like an abort during the SSE parse.
+ */
+const SIDECAR_429_MAX_ATTEMPTS = 3;
+const SIDECAR_429_BASE_DELAY_MS = 1_000;
+const SIDECAR_429_MAX_DELAY_MS = 10_000;
+
 export type SidecarOutcomeRecorder = (outcome: CodexUpstreamOutcome) => void;
 
 /**
@@ -79,7 +102,7 @@ export async function runWebSearch(
   const sidecarExit = sidecarEnter("web-search");
   const t0 = Date.now();
   try {
-    const res = await fetchWithResetRetry(
+    const sendOnce = () => fetchWithResetRetry(
       // Recovery nests INSIDE the version helper: applyUpstreamRecoveryInit then always receives a
       // defined init, and withUpstreamHttpVersion spreads the result, so `protocol` and the
       // recovery fields (`connection: close` + Bun's transport-level `keepalive: false`) survive
@@ -96,6 +119,22 @@ export async function runWebSearch(
       }, recovery), forwardProvider)),
       { replaySafe: true, abortSignal: linkedSignal.signal, label: "web-search-sidecar" },
     );
+    let res = await sendOnce();
+    for (let attempt = 0; res.status === 429 && attempt + 1 < SIDECAR_429_MAX_ATTEMPTS; attempt++) {
+      const delay = retryBackoffDelayMs(attempt, {
+        baseDelayMs: SIDECAR_429_BASE_DELAY_MS,
+        maxDelayMs: SIDECAR_429_MAX_DELAY_MS,
+        headers: res.headers,
+        retryAfterIsLowerBound: true,
+      });
+      // A deadline, not a clamp: an instruction past the ceiling ends the search with the
+      // 429 instead of parking it at a provider that already said it would refuse.
+      if (delay > RETRY_AFTER_CEILING_MS) break;
+      console.warn(`[web-search] sidecar HTTP 429 — retrying (${attempt + 2}/${SIDECAR_429_MAX_ATTEMPTS}) after ${delay}ms`);
+      await releaseResponseBodyBestEffort(res.body, linkedSignal.signal);
+      await sleepWithAbort(delay, linkedSignal.signal);
+      res = await sendOnce();
+    }
     // Attach the body guard before ANY branch reads it. The success path guarded itself below,
     // but the failure branch's `res.text()` runs first, so a cancel landing between fetch
     // resolution and reader attach orphaned the internal rejection (found investigating #1419).

package/src/web-search/loop.ts

@@ -333,10 +333,15 @@ export interface WebSearchLoopDeps {
    * 429 states the window's reset epoch even when it omits Retry-After -- and a rotation that
    * cannot see it cools the drained account for the short default instead of until the window
    * actually reopens. Optional so existing callers keep compiling.
+   *
+   * `retryParsed` is the exact iteration-local request the retry will be built from. The loop
+   * sends a shallow copy of the outer parsed request, so a rotation that rebinds only the outer
+   * object never reaches the wire. Optional so existing callers keep compiling.
    */
   on429?: (
     retryAfterHeader: string | null,
     responseHeaders?: Headers,
+    retryParsed?: OcxParsedRequest,
   ) => ProviderAdapter | null | Promise<ProviderAdapter | null>;
   /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */
   retryOn429Policy?: Required<RateLimitRetryPolicy> | null;
@@ -558,7 +563,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
       // 429 key-failover parity with the normal routed path: rotate pool keys until one responds
       // or the pool is exhausted (deps.on429 returns null — cooldown map guarantees termination).
       while (prepared.response.status === 429 && deps.on429) {
-        const rotated = await deps.on429(prepared.response.headers.get("retry-after"), prepared.response.headers);
+        const rotated = await deps.on429(prepared.response.headers.get("retry-after"), prepared.response.headers, iterParsed);
         if (!rotated) break;
         // Never let a broken body's cancel promise outlive the cumulative header deadline. Observe
         // it, but proceed immediately to the rotated fetch under the SAME deadline signal.

package/src/web-search/passthrough-bridge.ts

@@ -75,6 +75,7 @@ import {
 } from "./sidecar-providers";
 import { providerDestinationConfigError } from "../lib/destination-policy";
 import { redactSecretString } from "../lib/redact";
+import { rememberBridgeSearchReplay } from "../responses/bridge-search-replay-cache";
 
 /** Canonical Ollama Cloud origin. The only origin the "ollama" backend derives on its own. */
 export const OLLAMA_CLOUD_ORIGIN = "https://ollama.com";
@@ -377,11 +378,18 @@ export interface PassthroughWebSearchBridgeStreamOptions {
   /** Sends one continuation leg and resolves with its response. */
   send: (body: string) => Promise<Response>;
   execute: PassthroughWebSearchBridgeExecutor;
+  /**
+   * Destination identity for the executed-search memo (#4587). When absent nothing is recorded,
+   * and the next turn replays the hosted cell exactly as it does today.
+   */
+  destinationScope?: string;
   /**
    * Re-applies the caller's outbound body ceiling to a continuation body. Returns a refusal
    * message when the extended body may not be sent, or undefined when it is admitted.
    */
   checkOutboundBody?: (body: string) => string | undefined;
+  /** Releases request-scoped resources when the stream completes, fails, or is cancelled. */
+  onFinalize?: () => void;
   signal?: AbortSignal;
 }
 
@@ -1094,11 +1102,22 @@ async function* bridgeStreamBlocks(
         outcome = await options.execute(queries, options.signal);
       }
       yield* emit(state.searchEndFrames(call, queries, outcome));
-      turns.push({
-        call,
-        // The model needs a readable result either way; an executor error is reported as the
-        // tool result rather than as a turn failure, so it can still answer without the search.
-        output: outcome.error ? "Web search failed: " + outcome.error : outcome.text,
+      // The model needs a readable result either way; an executor error is reported as the
+      // tool result rather than as a turn failure, so it can still answer without the search.
+      const output = outcome.error ? "Web search failed: " + outcome.error : outcome.text;
+      turns.push({ call, output });
+      // Record what a continuation leg WOULD put on the wire, whether or not this leg sends one
+      // (#4587). The caller keeps the hosted cell and replays it next turn; the pre-dispatch
+      // rewrite in the Responses adapter uses this to hand the destination back its own call and
+      // result instead of an item type it never produced. Recording the same text that
+      // appendBridgeSearchTurn would append is what keeps a replayed turn and a continued turn
+      // showing the destination one consistent conversation.
+      rememberBridgeSearchReplay(options.destinationScope, call.cellItemId, {
+        callId: call.callId,
+        sourceItemId: call.sourceItemId,
+        name: WEB_SEARCH_TOOL_NAME,
+        argumentsText: call.argumentsText,
+        output,
       });
     }
 
@@ -1170,21 +1189,36 @@ export function createPassthroughWebSearchBridgeStream(
   const aborted = (): boolean => cancelled || options.signal?.aborted === true;
   const iterator = bridgeStreamBlocks(options, aborted)[Symbol.asyncIterator]();
   const encoder = new TextEncoder();
+  let finalized = false;
+  const finalize = (): void => {
+    if (finalized) return;
+    finalized = true;
+    options.onFinalize?.();
+  };
   return new ReadableStream<Uint8Array>({
     async pull(controller) {
       try {
         const next = await iterator.next();
         if (next.done) {
+          finalize();
           controller.close();
           return;
         }
         controller.enqueue(encoder.encode(next.value));
       } catch (error) {
+        finalize();
         controller.error(error);
       }
     },
     cancel(reason) {
       cancelled = true;
+      // Release request-scoped authority immediately. An async generator cannot
+      // process a queued return() while its active next() is blocked on an
+      // upstream read, so deferring finalize until that settles would hold the
+      // sidecar probe lease for as long as the abandoned upstream leg does.
+      // Optional chaining would also skip finalize entirely for an iterator
+      // with no return method.
+      finalize();
       void iterator.return?.(reason);
     },
   });