@iislee/opencodex 2.11.0

package/src/web-search/parse.ts

@@ -0,0 +1,220 @@
+/** A single web source backing the sidecar's answer. */
+export interface WebSearchSource {
+  url: string;
+  title?: string;
+}
+
+/** The sidecar's synthesized answer plus its sources (empty `sources` is fine). */
+export interface WebSearchResult {
+  text: string;
+  sources: WebSearchSource[];
+  /** Set only when the stream surfaced an error AND produced no usable answer text. */
+  error?: string;
+}
+
+interface AnnotationLike {
+  type?: string;
+  url?: string;
+  title?: string;
+}
+interface OutputTextBlock {
+  type?: string;
+  text?: string;
+  annotations?: AnnotationLike[];
+}
+interface OutputItem {
+  type?: string;
+  content?: OutputTextBlock[];
+}
+
+/** Push a `url_citation` annotation as a source, de-duplicated by URL. */
+function collectAnnotation(ann: AnnotationLike | undefined, sources: WebSearchSource[], seen: Set<string>): void {
+  if (!ann || ann.type !== "url_citation" || typeof ann.url !== "string" || seen.has(ann.url)) return;
+  seen.add(ann.url);
+  sources.push({ url: ann.url, ...(ann.title ? { title: ann.title } : {}) });
+}
+
+/**
+ * Hosted web_search (gpt-mini) rarely emits structured `url_citation` annotations; instead it ends
+ * its answer with a markdown `Sources:` section. Extract sources from that TRAILING section only
+ * (a whole-body URL scan would false-positive on URLs the model merely mentions), and return the
+ * answer text with that section stripped so the tool_result renderer doesn't double-print sources.
+ *
+ * Handles the per-line forms seen from the backend: `- title: url`, `- title (url)`,
+ * `- [title](url)`, `- <url>`, `- url`, numbered `1. ...` variants, a markdown-prefixed header
+ * (`### Sources:`, `**Sources**`), a title line whose URL sits on the FOLLOWING line, and trailing
+ * URL punctuation (`;`, `,`, `)`, `]`, `.`). Prose that follows the source list is preserved.
+ */
+const URL_RE = /https?:\/\/[^\s<>()\[\]]+/;
+// A "Sources:" / "Source:" header, allowing markdown prefixes (#, *, -, >) and bold/italic wrappers.
+const SOURCES_HEADER_RE = /^\s*(?:#{1,6}\s*)?[-*>\s]*\**\s*sources?\s*\**\s*:?\s*\**\s*$/i;
+
+/** Trim wrapping/trailing noise from a captured URL: angle brackets, then trailing punctuation. */
+function cleanUrl(url: string): string {
+  return url.replace(/^<+/, "").replace(/[)>\].,;:]+$/, "");
+}
+
+/** Derive a human title from the list-item text preceding the URL (strip markers, md link, seps). */
+function cleanTitle(prefix: string): string {
+  let title = prefix.replace(/^[-*>\d.)\s]+/, "").trim();
+  // `[title](` from a markdown link, or a leading `[`.
+  title = title.replace(/^\[/, "").replace(/\]\(?$/, "").replace(/[:\-—(<]\s*$/, "").trim();
+  return title;
+}
+
+function extractTrailingSources(text: string): { text: string; sources: WebSearchSource[] } {
+  const lines = text.split("\n");
+  // Find the LAST line that is a "Sources:" header (markdown prefixes allowed).
+  let headerIdx = -1;
+  for (let i = lines.length - 1; i >= 0; i--) {
+    if (SOURCES_HEADER_RE.test(lines[i])) { headerIdx = i; break; }
+  }
+  if (headerIdx === -1) return { text, sources: [] };
+  const sources: WebSearchSource[] = [];
+  const seen = new Set<string>();
+  // Track the last line index actually consumed as part of the source list so trailing prose after
+  // the list survives (we strip the header through the last consumed source line, not to EOF).
+  let lastConsumed = headerIdx;
+  // A title line whose URL is expected on a following line (multiline entry).
+  let pendingTitle: string | null = null;
+  for (let i = headerIdx + 1; i < lines.length; i++) {
+    const raw = lines[i].trim();
+    if (raw === "") {
+      // Blank line between header and first entry is fine; a blank AFTER entries ends the list.
+      if (sources.length > 0 || pendingTitle !== null) break;
+      continue;
+    }
+    const m = raw.match(URL_RE);
+    if (!m) {
+      // A list-ish line with no URL may be a title whose URL is on the next line. Only treat it as a
+      // pending title when it looks like a list item; otherwise it's prose → stop.
+      if (/^[-*>\d.)]/.test(raw) || pendingTitle === null) {
+        if (/^[-*>\d.)]/.test(raw)) { pendingTitle = raw; lastConsumed = i; continue; }
+      }
+      break;
+    }
+    const url = cleanUrl(m[0]);
+    if (!url) { break; }
+    lastConsumed = i;
+    // Title: text before the URL on this line, else a buffered title from a preceding line.
+    const inlinePrefix = raw.slice(0, m.index);
+    const title = cleanTitle(inlinePrefix) || (pendingTitle ? cleanTitle(pendingTitle) : "");
+    pendingTitle = null;
+    if (seen.has(url)) continue;
+    seen.add(url);
+    sources.push(title ? { url, title } : { url });
+  }
+  if (sources.length === 0) return { text, sources: [] };
+  // Keep text before the header AND any prose after the consumed source lines.
+  const before = lines.slice(0, headerIdx).join("\n").replace(/\s+$/, "");
+  const after = lines.slice(lastConsumed + 1).join("\n").replace(/^\s+/, "");
+  const body = after ? (before ? `${before}\n\n${after}` : after) : before;
+  return { text: body, sources };
+}
+
+/** Pull final text + url_citation sources from a completed Responses `output[]` array. */
+function fromOutputArray(output: OutputItem[], seen: Set<string>): WebSearchResult {
+  let text = "";
+  const sources: WebSearchSource[] = [];
+  for (const item of output) {
+    if (item.type !== "message" || !Array.isArray(item.content)) continue;
+    for (const block of item.content) {
+      if (block.type === "output_text" && typeof block.text === "string") {
+        text += block.text;
+        for (const ann of block.annotations ?? []) collectAnnotation(ann, sources, seen);
+      }
+    }
+  }
+  return { text, sources };
+}
+
+/**
+ * Parse the sidecar's streamed Responses SSE into a final answer + sources. Tolerant of the full set of
+ * Responses streaming events: prefers the authoritative `response.completed` output[], then the
+ * `response.output_text.done` text; falls back to accumulated `response.output_text.delta`. Sources are
+ * collected from EVERY shape they arrive in — `response.output_text.annotation.added` events (the
+ * streaming path, which earlier testing missed → empty citations), `done`-block `annotations[]`, and
+ * the final output[]. `response.failed`/`error` events surface as `error` when no answer text was produced.
+ */
+export async function parseSidecarSSE(response: Response): Promise<WebSearchResult> {
+  if (!response.body) return { text: "", sources: [] };
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  const seen = new Set<string>();
+  // Holder object — fields are mutated inside the closure, so they can't live as narrowed locals.
+  const acc: {
+    deltaText: string;
+    doneText: string;
+    final: WebSearchResult | null;
+    streamSources: WebSearchSource[];
+    error: string | null;
+  } = { deltaText: "", doneText: "", final: null, streamSources: [], error: null };
+
+  const handle = (payload: string): void => {
+    if (!payload || payload === "[DONE]") return;
+    let data: Record<string, unknown>;
+    try { data = JSON.parse(payload) as Record<string, unknown>; } catch {
+      console.warn(`[web-search-parse] malformed SSE JSON (${payload.length} chars): ${payload.slice(0, 120)}`);
+      return;
+    }
+    const type = data.type as string | undefined;
+    if (type === "response.output_text.delta" && typeof data.delta === "string") {
+      acc.deltaText += data.delta;
+    } else if (type === "response.output_text.done" && typeof data.text === "string") {
+      // The `done` event carries the full, authoritative text for one content part.
+      acc.doneText += data.text;
+    } else if (type === "response.completed" || type === "response.done") {
+      const resp = data.response as { output?: OutputItem[] } | undefined;
+      if (resp?.output) acc.final = fromOutputArray(resp.output, seen);
+    } else if (type === "response.failed" || type === "response.incomplete" || type === "error") {
+      const resp = data.response as { error?: { message?: string } } | undefined;
+      const msg = resp?.error?.message
+        ?? (data.error as { message?: string } | undefined)?.message
+        ?? (typeof data.message === "string" ? data.message : undefined);
+      if (msg) acc.error = msg;
+    }
+    // Citations stream as a dedicated `response.output_text.annotation.added` event (singular
+    // `annotation`); capture it regardless of the exact event name so they aren't lost.
+    if (data.annotation) collectAnnotation(data.annotation as AnnotationLike, acc.streamSources, seen);
+  };
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        if (line.startsWith("data: ")) handle(line.slice(6).trim());
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  // Prefer the authoritative completed output[], then the done text, then accumulated deltas.
+  const text = (acc.final?.text.trim() ? acc.final.text : "")
+    || acc.doneText.trim() && acc.doneText
+    || acc.deltaText;
+  // Merge sources from the final output[] and the streaming annotation events.
+  const sources = [...(acc.final?.sources ?? [])];
+  const seenMerge = new Set(sources.map(s => s.url));
+  for (const s of acc.streamSources) {
+    if (!seenMerge.has(s.url)) { seenMerge.add(s.url); sources.push(s); }
+  }
+  // Hosted web_search usually omits url_citation annotations and lists sources in a trailing
+  // `Sources:` markdown block instead. Pull those out (and strip the block from the answer so the
+  // tool_result renderer doesn't print sources twice). Annotation titles win; text-block titles
+  // only fill a gap. URL-deduped against annotation sources.
+  const { text: body, sources: textSources } = extractTrailingSources(typeof text === "string" ? text : "");
+  for (const s of textSources) {
+    if (seenMerge.has(s.url)) continue;
+    seenMerge.add(s.url);
+    sources.push(s);
+  }
+  const finalText = textSources.length > 0 ? body : (typeof text === "string" ? text : "");
+  if (!finalText.trim() && acc.error) return { text: "", sources, error: acc.error };
+  return { text: finalText, sources };
+}

package/src/web-search/progress-stream.ts

@@ -0,0 +1,342 @@
+import type { ProviderAdapter } from "../adapters/base";
+import type { AdapterEvent } from "../types";
+import {
+  isTranslatorBudgetExceededError,
+  TRANSLATOR_MAX_TURN_BYTES,
+  TranslatorBudgetExceededError,
+  type TranslatorBudget,
+} from "../lib/translator-budget";
+
+const DEFAULT_POST_TERMINAL_DRAIN_TIMEOUT_MS = 5_000;
+
+export class RoutedModelInactivityError extends Error {
+  readonly timeoutMs: number;
+
+  constructor(timeoutMs: number) {
+    super(`Routed model generation timeout after ${timeoutMs}ms without response bytes during web-search`);
+    this.name = "RoutedModelInactivityError";
+    this.timeoutMs = timeoutMs;
+  }
+}
+
+export class WebSearchStreamProtocolError extends Error {
+  constructor(message: string) {
+    super(`Web-search adapter stream protocol error: ${message}`);
+    this.name = "WebSearchStreamProtocolError";
+  }
+}
+
+export interface ParseStreamWithProgressOptions {
+  inactivityTimeoutMs: number;
+  translatorBudget: TranslatorBudget;
+  signal?: AbortSignal;
+  /** Kept configurable for focused tests; production callers should use the 5 second default. */
+  postTerminalDrainTimeoutMs?: number;
+}
+
+type ParseStream = ProviderAdapter["parseStream"];
+type Resolver<T> = (value: T | PromiseLike<T>) => void;
+
+interface PendingDelivery {
+  event: AdapterEvent;
+  resolve: () => void;
+  reject: (reason: unknown) => void;
+}
+
+/**
+ * Capacity-one handoff. Semantic events are lossless and acknowledged when received; raw-byte
+ * progress is only a coalesced liveness bit and never displaces a semantic event.
+ */
+class ProgressHandoff {
+  private semantic: PendingDelivery | undefined;
+  private progress = false;
+  private receiver: { resolve: Resolver<IteratorResult<AdapterEvent>>; reject: (reason: unknown) => void } | undefined;
+  private ended = false;
+  private failed = false;
+  private failure: unknown | undefined;
+
+  async deliver(event: AdapterEvent): Promise<void> {
+    if (this.failed) throw this.failure;
+    if (this.ended) throw new WebSearchStreamProtocolError("event delivered after collector closed");
+
+    if (this.receiver) {
+      const receiver = this.receiver;
+      this.receiver = undefined;
+      receiver.resolve({ done: false, value: event });
+      return;
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      this.semantic = { event, resolve, reject };
+    });
+  }
+
+  offerProgress(): void {
+    if (this.ended || this.failed) return;
+    if (this.receiver && !this.semantic) {
+      const receiver = this.receiver;
+      this.receiver = undefined;
+      receiver.resolve({ done: false, value: { type: "heartbeat" } });
+      return;
+    }
+    this.progress = true;
+  }
+
+  receive(): Promise<IteratorResult<AdapterEvent>> {
+    if (this.failed) return Promise.reject(this.failure);
+    if (this.semantic) {
+      const delivery = this.semantic;
+      this.semantic = undefined;
+      delivery.resolve();
+      return Promise.resolve({ done: false, value: delivery.event });
+    }
+    if (this.progress) {
+      this.progress = false;
+      return Promise.resolve({ done: false, value: { type: "heartbeat" } });
+    }
+    if (this.ended) return Promise.resolve({ done: true, value: undefined });
+
+    return new Promise<IteratorResult<AdapterEvent>>((resolve, reject) => {
+      this.receiver = { resolve, reject };
+    });
+  }
+
+  fail(reason: unknown): void {
+    if (this.ended || this.failed) return;
+    this.failed = true;
+    this.failure = reason;
+    this.progress = false;
+    this.semantic?.reject(reason);
+    this.semantic = undefined;
+    this.receiver?.reject(reason);
+    this.receiver = undefined;
+  }
+
+  close(): void {
+    if (this.ended || this.failed) return;
+    this.ended = true;
+    this.progress = false;
+    this.semantic?.reject(new WebSearchStreamProtocolError("collector closed with a pending event"));
+    this.semantic = undefined;
+    this.receiver?.resolve({ done: true, value: undefined });
+    this.receiver = undefined;
+  }
+}
+
+function normalizeTimeout(value: number, label: string): number {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new RangeError(`${label} must be a positive finite number`);
+  }
+  return value;
+}
+
+/**
+ * Parse an adapter stream while exposing invisible raw-byte liveness heartbeats.
+ *
+ * The original body is never cloned or tee'd. This function owns its sole reader and supplies the
+ * adapter with a demand-driven, highWaterMark-zero response body.
+ */
+export async function* parseStreamWithProgress(
+  response: Response,
+  parseStream: ParseStream,
+  options: ParseStreamWithProgressOptions,
+): AsyncGenerator<AdapterEvent> {
+  const inactivityTimeoutMs = normalizeTimeout(options.inactivityTimeoutMs, "inactivityTimeoutMs");
+  const postTerminalDrainTimeoutMs = normalizeTimeout(
+    options.postTerminalDrainTimeoutMs ?? DEFAULT_POST_TERMINAL_DRAIN_TIMEOUT_MS,
+    "postTerminalDrainTimeoutMs",
+  );
+  const handoff = new ProgressHandoff();
+  const reader = response.body?.getReader();
+  if (!reader) throw new WebSearchStreamProtocolError("upstream response has no body");
+
+  let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
+  let tappedController: ReadableStreamDefaultController<Uint8Array> | undefined;
+  let settled = false;
+  let iterator: AsyncGenerator<AdapterEvent> | undefined;
+  let iteratorReturnStarted = false;
+  let detachAbort = (): void => {};
+  let stopSignalled = false;
+  let signalStop!: (reason: unknown) => void;
+  const stopped = new Promise<unknown>(resolve => { signalStop = resolve; });
+
+  const clearInactivity = (): void => {
+    if (inactivityTimer !== undefined) clearTimeout(inactivityTimer);
+    inactivityTimer = undefined;
+  };
+
+  const cancelReader = (reason: unknown): void => {
+    try {
+      void reader.cancel(reason).catch(() => {});
+    } catch { /* reader cancellation is best-effort and must never replace the foreground outcome */ }
+  };
+
+  const closeIterator = (): void => {
+    if (!iterator || iteratorReturnStarted) return;
+    iteratorReturnStarted = true;
+    try {
+      const returned = iterator.return(undefined);
+      void returned.catch(() => {});
+    } catch { /* iterator return is best-effort and must never delay foreground failure */ }
+  };
+
+  const stop = (reason: unknown): void => {
+    if (stopSignalled) return;
+    stopSignalled = true;
+    signalStop(reason);
+  };
+
+  const fail = (reason: unknown): void => {
+    // A second failure may arrive from the parser after reader cancellation. Iterator cleanup is
+    // still required even when the foreground failure has already won the settlement race.
+    closeIterator();
+    if (settled) return;
+    settled = true;
+    clearInactivity();
+    detachAbort();
+    stop(reason);
+    handoff.fail(reason);
+    try { tappedController?.error(reason); } catch { /* already closed/cancelled */ }
+    cancelReader(reason);
+  };
+
+  const resetInactivity = (): void => {
+    if (settled) return;
+    clearInactivity();
+    inactivityTimer = setTimeout(() => fail(new RoutedModelInactivityError(inactivityTimeoutMs)), inactivityTimeoutMs);
+  };
+
+  const onAbort = (): void => fail(options.signal?.reason);
+  detachAbort = (): void => options.signal?.removeEventListener("abort", onAbort);
+  if (options.signal?.aborted) fail(options.signal.reason);
+  else options.signal?.addEventListener("abort", onAbort, { once: true });
+
+  const tappedBody = new ReadableStream<Uint8Array>({
+    start(controller) {
+      tappedController = controller;
+    },
+    async pull(controller) {
+      try {
+        while (!settled) {
+          const result = await reader.read();
+          if (result.done) {
+            controller.close();
+            return;
+          }
+          if (result.value.byteLength === 0) continue;
+          resetInactivity();
+          handoff.offerProgress();
+          controller.enqueue(result.value);
+          return;
+        }
+      } catch (error) {
+        fail(error);
+      }
+    },
+    cancel(reason) {
+      cancelReader(reason);
+      closeIterator();
+    },
+  }, { highWaterMark: 0 });
+
+  const tappedResponse = new Response(tappedBody, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: response.headers,
+  });
+
+  resetInactivity();
+
+  const parserPump = (async (): Promise<void> => {
+    let heldTerminal: Extract<AdapterEvent, { type: "done" | "incomplete" }> | undefined;
+    try {
+      iterator = parseStream(tappedResponse, options.translatorBudget);
+      if (settled) {
+        closeIterator();
+        return;
+      }
+      while (true) {
+        let result: IteratorResult<AdapterEvent>;
+        if (heldTerminal) {
+          let drainTimer: ReturnType<typeof setTimeout> | undefined;
+          const drainTimeout = new Promise<never>((_, reject) => {
+            drainTimer = setTimeout(() => reject(new WebSearchStreamProtocolError(
+              `adapter did not return within ${postTerminalDrainTimeoutMs}ms after ${heldTerminal?.type ?? "terminal event"}`,
+            )), postTerminalDrainTimeoutMs);
+          });
+          const stoppedDuringDrain = stopped.then(reason => { throw reason; });
+          try {
+            result = await Promise.race([iterator.next(), drainTimeout, stoppedDuringDrain]);
+          } finally {
+            if (drainTimer !== undefined) clearTimeout(drainTimer);
+          }
+        } else {
+          result = await iterator.next();
+        }
+
+        if (result.done) {
+          if (!heldTerminal) throw new WebSearchStreamProtocolError("adapter returned without a terminal event");
+          await handoff.deliver(heldTerminal);
+          if (!settled) {
+            settled = true;
+            clearInactivity();
+            detachAbort();
+            handoff.close();
+            cancelReader(undefined);
+          }
+          return;
+        }
+
+        const event = result.value;
+        if (heldTerminal) {
+          throw new WebSearchStreamProtocolError(
+            event.type === "done" || event.type === "incomplete"
+              ? "adapter yielded more than one terminal event"
+              : "adapter yielded an event after its terminal event",
+          );
+        }
+        if (event.type === "error") {
+          fail(event.code === "translation_buffer_limit"
+            ? new TranslatorBudgetExceededError("live_transient", TRANSLATOR_MAX_TURN_BYTES)
+            : new Error(event.message));
+          return;
+        }
+        if (event.type === "done" || event.type === "incomplete") {
+          heldTerminal = event;
+          continue;
+        }
+        await handoff.deliver(event);
+      }
+    } catch (error) {
+      fail(isTranslatorBudgetExceededError(error)
+        ? error
+        : error instanceof WebSearchStreamProtocolError
+        ? error
+        : new WebSearchStreamProtocolError(
+          `adapter threw${heldTerminal ? " after its terminal event" : ""}: ${error instanceof Error ? error.message : String(error)}`,
+        ));
+    }
+  })();
+  // The foreground intentionally never awaits this task during cancellation; keep every rejection observed.
+  void parserPump.catch(() => {});
+
+  try {
+    while (true) {
+      const result = await handoff.receive();
+      if (result.done) return;
+      yield result.value;
+    }
+  } finally {
+    detachAbort();
+    if (!settled) {
+      const reason = new Error("Web-search stream consumer stopped");
+      settled = true;
+      clearInactivity();
+      stop(reason);
+      handoff.close();
+      cancelReader(reason);
+    }
+    closeIterator();
+    try { reader.releaseLock(); } catch { /* an outstanding read may still own the lock */ }
+  }
+}

package/src/web-search/synthetic-tool.ts

@@ -0,0 +1,47 @@
+import type { OcxTool } from "../types";
+
+/** The function name the chat model sees + the name the loop intercepts. */
+export const WEB_SEARCH_TOOL_NAME = "web_search";
+
+/**
+ * Find the hosted `{type:"web_search", ...}` entry in a Responses request's `tools[]` and return it
+ * verbatim (so its config — external_web_access/filters/user_location/search_context_size — can be
+ * replayed into the sidecar's REAL web_search tool). Returns undefined when web search isn't enabled.
+ */
+export function extractHostedWebSearch(tools: unknown[] | undefined): Record<string, unknown> | undefined {
+  if (!Array.isArray(tools)) return undefined;
+  for (const t of tools) {
+    if (t && typeof t === "object" && (t as { type?: string }).type === "web_search") {
+      return t as Record<string, unknown>;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * The synthetic function tool exposed to a chat/anthropic model in place of the dropped hosted
+ * web_search. The model calls it like any function; the proxy intercepts the call and runs the real
+ * search via the sidecar (the call is never relayed to Codex). `webSearch:true` flags it for the loop.
+ */
+export function buildWebSearchTool(): OcxTool {
+  return {
+    name: WEB_SEARCH_TOOL_NAME,
+    description:
+      "Search the web for current, real-world, or post-training-cutoff information. " +
+      "Returns a concise answer synthesized from live results, with sources. " +
+      "Use it whenever the user asks about recent events, versions, prices, docs, or anything you are unsure is current.",
+    parameters: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "A single search query — a focused natural-language question or keywords." },
+        queries: {
+          type: "array",
+          items: { type: "string" },
+          description: "Optional: run several related queries together in one call. Use instead of `query` to batch independent searches.",
+        },
+      },
+      // Either `query` or `queries` is accepted; the proxy normalizes them.
+    },
+    webSearch: true,
+  };
+}