@bitkyc08/opencodex 2.7.9-preview.20260712.2 → 2.7.9

package/src/types.ts

@@ -320,6 +320,10 @@ export interface OcxClaudeCodeConfig {
    * free. Only ocx-*.md files are owned/pruned. Default: enabled.
    */
   injectAgents?: boolean;
+  /** Claude-originated web-search override. Unset fields inherit the global sidecar settings. */
+  webSearchSidecar?: { backend?: "openai" | "anthropic"; model?: string };
+  /** Claude-originated vision override. Unset fields inherit the global sidecar settings. */
+  visionSidecar?: { backend?: "openai" | "anthropic"; model?: string };
 }
 
 export interface OcxConfig {
@@ -485,10 +489,14 @@ export interface OcxSearchConfig {
 }
 
 export interface OcxVisionSidecarConfig {
-  /** Master switch. Default: enabled when a forward (ChatGPT) provider exists and the caller is logged in. */
+  /** Master switch. Default: enabled when the selected backend has a usable credential. */
   enabled?: boolean;
-  /** Vision model that describes images (must be a native ChatGPT model with image input). */
+  /** Description backend. Unset prefers a usable stored Anthropic OAuth credential, else OpenAI. */
+  backend?: "openai" | "anthropic";
+  /** Vision model that describes images. */
   model?: string;
+  /** Max description cache misses admitted in one main-model turn. Zero disables description calls. */
+  maxDescriptionsPerTurn?: number;
   /** Sidecar fetch timeout (ms). */
   timeoutMs?: number;
 }
@@ -496,6 +504,13 @@ export interface OcxVisionSidecarConfig {
 export interface OcxWebSearchSidecarConfig {
   /** Master switch. Default: enabled when a forward (ChatGPT) provider exists and the caller is logged in. */
   enabled?: boolean;
+  /**
+   * Which backend actually runs the server-side search. "openai" replays the hosted web_search via
+   * the ChatGPT forward provider (gpt-mini sidecar); "anthropic" runs web_search_20250305 on a Claude
+   * model authenticated by the STORED anthropic OAuth credential. Unset resolves to "anthropic" when a
+   * usable anthropic OAuth credential exists, else "openai".
+   */
+  backend?: "openai" | "anthropic";
   /** Sidecar model that runs the real server-side web_search (must be a native ChatGPT model). */
   model?: string;
   /** Reasoning effort for the sidecar — "minimal" (non-thinking) keeps it fast/cheap. */

package/src/vision/anthropic-describe.ts

@@ -0,0 +1,185 @@
+import type { OcxProviderConfig } from "../types";
+import { CLAUDE_CODE_HEADERS, claudeCodeSessionId } from "../adapters/client-fingerprint";
+import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
+import { sidecarEnter } from "../lib/sidecar-tracker";
+import { fetchWithResetRetry } from "../lib/upstream-retry";
+import { getValidAccessToken } from "../oauth";
+import { ANTHROPIC_OAUTH_BETA, CLAUDE_CODE_SYSTEM_INSTRUCTION } from "../oauth/anthropic";
+import type { DescribeOutcome, VisionSettings } from "./describe";
+
+const ANTHROPIC_VISION_MAX_TOKENS = 1024;
+const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]);
+const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
+const DESCRIBE_INSTRUCTION =
+  "You are a vision describer for a text-only model that cannot see the image. Describe the image " +
+  "thoroughly and factually so that model can fully reason about it: transcribe any visible text " +
+  "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on " +
+  "what's relevant to the user's request. Output only the description.";
+
+type AnthropicImageBlock =
+  | { type: "image"; source: { type: "base64"; media_type: string; data: string } }
+  | { type: "image"; source: { type: "url"; url: string } };
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function buildImageBlock(imageUrl: string): { block?: AnthropicImageBlock; error?: string } {
+  if (imageUrl.startsWith("data:")) {
+    // Anthropic's base64 image source requires actual base64 bytes, so a non-base64 data URL
+    // (e.g. `data:image/png,raw`) is rejected here. This is intentionally stricter than the OpenAI
+    // vision executor, which forwards the raw data URL to `image_url` (review F3, documented delta).
+    const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(imageUrl);
+    if (!match || !match[2]) return { error: "malformed data URL" };
+    const mime = match[1].toLowerCase();
+    if (!ALLOWED_IMAGE_MIME.has(mime)) return { error: `unsupported image type "${mime}"` };
+    const bytes = Math.floor((match[3].length * 3) / 4);
+    if (bytes > MAX_IMAGE_BYTES) return { error: `image too large (~${Math.round(bytes / 1024 / 1024)}MB)` };
+    return { block: { type: "image", source: { type: "base64", media_type: mime, data: match[3] } } };
+  }
+  if (imageUrl.startsWith("https://")) {
+    return { block: { type: "image", source: { type: "url", url: imageUrl } } };
+  }
+  return { error: "unsupported image URL scheme (expected data: or https:)" };
+}
+
+/** Fold Anthropic Messages text deltas into one description. Malformed frames are ignored. */
+export async function parseAnthropicVisionSSE(res: Response): Promise<DescribeOutcome> {
+  if (!res.body) return { text: "", error: "anthropic vision sidecar returned no response body" };
+
+  let text = "";
+  let terminalError = "";
+  const decoder = new TextDecoder();
+  const reader = res.body.getReader();
+  let buffer = "";
+
+  const processFrame = (rawFrame: string): void => {
+    let dataLine = "";
+    for (const line of rawFrame.split("\n")) {
+      if (line.startsWith("data:")) dataLine += line.slice(line.startsWith("data: ") ? 6 : 5);
+    }
+    if (!dataLine || dataLine === "[DONE]") return;
+    let data: unknown;
+    try { data = JSON.parse(dataLine); } catch { return; }
+    if (!isRecord(data)) return;
+
+    if (data.type === "content_block_delta") {
+      const delta = isRecord(data.delta) ? data.delta : {};
+      if (delta.type === "text_delta" && typeof delta.text === "string") text += delta.text;
+    } else if (data.type === "error") {
+      const error = isRecord(data.error) ? data.error : {};
+      terminalError = typeof error.message === "string" ? error.message : "anthropic vision sidecar stream error";
+    }
+  };
+
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer = (buffer + decoder.decode(value, { stream: true })).replace(/\r\n/g, "\n");
+      let separator: number;
+      while ((separator = buffer.indexOf("\n\n")) !== -1) {
+        processFrame(buffer.slice(0, separator));
+        buffer = buffer.slice(separator + 2);
+      }
+    }
+    buffer = (buffer + decoder.decode()).replace(/\r\n/g, "\n");
+    if (buffer.trim()) processFrame(buffer);
+  } catch {
+    // A mid-stream read/decode failure after partial text is NOT a usable description. Mark it
+    // terminal so the caller returns an error and never caches an incomplete result (review F1).
+    if (!terminalError) terminalError = "anthropic vision sidecar stream ended abnormally";
+  }
+
+  const trimmed = text.trim();
+  // A terminal error (an in-stream `error` frame OR an abnormal body failure) invalidates any partial
+  // text: return an error outcome so vision/index.ts never caches an incomplete description (review F1).
+  if (terminalError) return { text: "", error: terminalError };
+  if (!trimmed) return { text: "", error: "anthropic vision sidecar produced no description" };
+  return { text: trimmed };
+}
+
+/** Describe one image through a stored Anthropic OAuth credential. Never throws. */
+export async function describeImageAnthropic(
+  imageUrl: string,
+  detail: string | undefined,
+  contextText: string,
+  providerName: string,
+  provider: OcxProviderConfig,
+  settings: VisionSettings,
+  abortSignal?: AbortSignal,
+): Promise<DescribeOutcome> {
+  const image = buildImageBlock(imageUrl);
+  if (!image.block) return { text: "", error: image.error ?? "invalid image" };
+
+  let token: string;
+  try {
+    token = await getValidAccessToken(providerName);
+  } catch (error) {
+    return { text: "", error: `anthropic vision sidecar auth failed: ${error instanceof Error ? error.message : String(error)}` };
+  }
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    "anthropic-version": "2023-06-01",
+    "Accept": "text/event-stream",
+    "User-Agent": "@anthropic-ai/sdk/0.74.0",
+    "Authorization": `Bearer ${token}`,
+    "anthropic-beta": ANTHROPIC_OAUTH_BETA,
+    ...CLAUDE_CODE_HEADERS,
+    "X-Claude-Code-Session-Id": claudeCodeSessionId(token),
+    "x-client-request-id": crypto.randomUUID(),
+  };
+  if (provider.headers) Object.assign(headers, provider.headers);
+
+  const content: unknown[] = [];
+  if (contextText) content.push({ type: "text", text: `The user's request about this image: ${contextText}` });
+  content.push(image.block);
+  const body = {
+    model: settings.model,
+    max_tokens: ANTHROPIC_VISION_MAX_TOKENS,
+    thinking: { type: "disabled" },
+    system: [
+      { type: "text", text: CLAUDE_CODE_SYSTEM_INSTRUCTION },
+      { type: "text", text: DESCRIBE_INSTRUCTION },
+    ],
+    messages: [{ role: "user", content }],
+    stream: true,
+  };
+
+  // Anthropic image blocks have no detail field, but detail remains part of the cache identity.
+  void detail;
+  const base = provider.baseUrl.replace(/\/v1\/?$/, "");
+  const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
+  const sidecarExit = sidecarEnter("vision");
+  const startedAt = Date.now();
+  try {
+    const res = await fetchWithResetRetry(
+      () => fetch(`${base}/v1/messages`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(body),
+        signal: linkedSignal.signal,
+      }),
+      { abortSignal: linkedSignal.signal, label: "vision-sidecar-anthropic" },
+    );
+    if (!res.ok) {
+      const responseText = await res.text().catch(() => "");
+      console.warn(`[vision] anthropic sidecar HTTP ${res.status} (${Date.now() - startedAt}ms)`);
+      return { text: "", error: `anthropic vision sidecar HTTP ${res.status}: ${responseText.slice(0, 200)}` };
+    }
+    const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
+    try {
+      return await parseAnthropicVisionSSE(res);
+    } finally {
+      detachBodyGuard();
+    }
+  } catch (error) {
+    const kind = error instanceof Error && error.name === "TimeoutError" ? "timeout" : "connect_error";
+    console.warn(`[vision] anthropic sidecar ${kind} (${Date.now() - startedAt}ms)`);
+    return { text: "", error: error instanceof Error ? error.message : String(error) };
+  } finally {
+    sidecarExit();
+    linkedSignal.cleanup();
+  }
+}

package/src/vision/index.ts

@@ -1,13 +1,20 @@
+import { createHash } from "node:crypto";
 import type { OcxConfig, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTextContent } from "../types";
 import { modelInList } from "../types";
-import { describeImage, type VisionSettings } from "./describe";
+import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe";
+import { describeImageAnthropic } from "./anthropic-describe";
 import type { CodexAuthContext } from "../codex/auth-context";
+import { getAccountSet } from "../oauth/store";
 import type { SidecarOutcomeRecorder } from "../web-search/executor";
 
 export { describeImage } from "./describe";
+export { describeImageAnthropic, parseAnthropicVisionSSE } from "./anthropic-describe";
 
 const DEFAULT_VISION_MODEL = "gpt-5.4-mini";
+const DEFAULT_ANTHROPIC_VISION_MODEL = "claude-sonnet-5";
 const DEFAULT_TIMEOUT_MS = 45_000;
+const DEFAULT_MAX_DESCRIPTIONS_PER_TURN = 8;
+const DESCRIPTION_CACHE_MAX_ENTRIES = 256;
 /** Max images described in parallel — keeps first-token latency bounded without flooding the backend. */
 const VISION_CONCURRENCY = 3;
 /** Per-image description hard cap (chars) so multi-image turns can't blow the main model's context. */
@@ -15,6 +22,59 @@ const DESC_MAX_CHARS = 2000;
 /** User-text context passed to the describer, capped. */
 const CONTEXT_MAX_CHARS = 800;
 
+export interface VisionDescriptionCache {
+  get(key: string): string | undefined;
+  set(key: string, value: string): void;
+  clear(): void;
+}
+
+class BoundedLruDescriptionCache implements VisionDescriptionCache {
+  private readonly entries = new Map<string, string>();
+
+  constructor(private readonly maxEntries: number) {}
+
+  get(key: string): string | undefined {
+    const value = this.entries.get(key);
+    if (value === undefined) return undefined;
+    this.entries.delete(key);
+    this.entries.set(key, value);
+    return value;
+  }
+
+  set(key: string, value: string): void {
+    this.entries.delete(key);
+    this.entries.set(key, value);
+    while (this.entries.size > this.maxEntries) {
+      const oldest = this.entries.keys().next().value;
+      if (oldest === undefined) break;
+      this.entries.delete(oldest);
+    }
+  }
+
+  clear(): void {
+    this.entries.clear();
+  }
+}
+
+let descriptionCache: VisionDescriptionCache = new BoundedLruDescriptionCache(DESCRIPTION_CACHE_MAX_ENTRIES);
+
+/** Replace the process cache (primarily for deterministic tests). Passing undefined restores the default LRU. */
+export function setVisionDescriptionCache(cache?: VisionDescriptionCache): void {
+  descriptionCache = cache ?? new BoundedLruDescriptionCache(DESCRIPTION_CACHE_MAX_ENTRIES);
+}
+
+export function resetVisionDescriptionCache(): void {
+  descriptionCache.clear();
+}
+
+/** Runtime config is permissive: zero is intentional; malformed values fall back to the bounded default. */
+export function resolveMaxDescriptionsPerTurn(value: unknown): number {
+  if (value === 0) return 0;
+  return typeof value === "number" && Number.isInteger(value) && value > 0
+    ? value
+    : DEFAULT_MAX_DESCRIPTIONS_PER_TURN;
+}
+
 /** Run `worker` over `items` with bounded concurrency, preserving input order in the result array. */
 async function runBounded<T, R>(items: T[], limit: number, worker: (item: T) => Promise<R>): Promise<R[]> {
   const results = new Array<R>(items.length);
@@ -33,7 +93,7 @@ function clamp(s: string, max: number): string {
   return s.length <= max ? s : `${s.slice(0, max)}\n…[description truncated]`;
 }
 
-/** First configured forward (ChatGPT passthrough) provider — the path with native image input. */
+/** First configured forward (ChatGPT passthrough) provider — the OpenAI path with native image input. */
 function findForwardProvider(config: OcxConfig): OcxProviderConfig | undefined {
   for (const prov of Object.values(config.providers)) {
     if (prov.disabled === true) continue;
@@ -42,6 +102,30 @@ function findForwardProvider(config: OcxConfig): OcxProviderConfig | undefined {
   return undefined;
 }
 
+export interface AnthropicVisionProvider {
+  providerName: string;
+  provider: OcxProviderConfig;
+}
+
+/** First enabled Anthropic OAuth provider whose active stored account is not marked for reauth. */
+export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionProvider | undefined {
+  for (const [providerName, provider] of Object.entries(config.providers)) {
+    if (provider.disabled === true || provider.adapter !== "anthropic" || provider.authMode !== "oauth") continue;
+    const accountSet = getAccountSet(providerName);
+    const active = accountSet?.accounts.find(account => account.id === accountSet.activeAccountId);
+    if (active && active.needsReauth !== true) return { providerName, provider };
+  }
+  return undefined;
+}
+
+export function resolveVisionBackend(
+  explicit: "openai" | "anthropic" | undefined,
+  anthropicSidecar: AnthropicVisionProvider | undefined,
+): "openai" | "anthropic" {
+  if (explicit === "openai" || explicit === "anthropic") return explicit;
+  return anthropicSidecar ? "anthropic" : "openai";
+}
+
 /** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
 function carriesImages(role: string): boolean {
   return role === "user" || role === "developer" || role === "toolResult";
@@ -53,15 +137,18 @@ function messagesHaveImage(parsed: OcxParsedRequest): boolean {
 }
 
 export interface VisionPlan {
-  forwardProvider: OcxProviderConfig;
+  backend: "openai" | "anthropic";
+  forwardProvider?: OcxProviderConfig;
+  anthropicSidecar?: AnthropicVisionProvider;
   settings: VisionSettings;
+  maxDescriptionsPerTurn: number;
 }
 
 /**
  * Decide whether the vision sidecar should pre-describe images for this request, returning the plan
  * if so. Active when: the routed model is in `provider.noVisionModels`, the request actually carries
- * an image, a forward provider exists, the sidecar isn't disabled, and the caller forwarded ChatGPT
- * auth. Returns undefined otherwise (the request takes the normal path — images sent natively).
+ * an image, the sidecar isn't disabled, and the selected backend has usable auth. Returns undefined
+ * otherwise (the caller strips images before sending to a text-only model).
  */
 export function planVisionSidecar(
   config: OcxConfig,
@@ -75,12 +162,28 @@ export function planVisionSidecar(
   if (!messagesHaveImage(parsed)) return undefined;
   const cfg = config.visionSidecar ?? {};
   if (cfg.enabled === false) return undefined;
+  const anthropicSidecar = findAnthropicVisionProvider(config);
+  const backend = resolveVisionBackend(cfg.backend, anthropicSidecar);
+  const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn);
+
+  if (backend === "anthropic") {
+    if (!anthropicSidecar) return undefined;
+    return {
+      backend,
+      anthropicSidecar,
+      settings: { model: cfg.model ?? DEFAULT_ANTHROPIC_VISION_MODEL, timeoutMs: cfg.timeoutMs ?? DEFAULT_TIMEOUT_MS },
+      maxDescriptionsPerTurn,
+    };
+  }
+
   if (authContext.kind === "main" && !incomingHeaders.get("authorization")) return undefined;
   const forwardProvider = findForwardProvider(config);
   if (!forwardProvider) return undefined;
   return {
+    backend,
     forwardProvider,
     settings: { model: cfg.model ?? DEFAULT_VISION_MODEL, timeoutMs: cfg.timeoutMs ?? DEFAULT_TIMEOUT_MS },
+    maxDescriptionsPerTurn,
   };
 }
 
@@ -100,6 +203,69 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten
   };
 }
 
+function sha256(value: string | Uint8Array): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function normalizedContext(contextText: string): string {
+  return contextText.trim().replace(/\s+/g, " ");
+}
+
+function descriptionIdentity(job: ImageJob, plan: VisionPlan): { key: string; persistent: boolean } {
+  let imageHash: string;
+  let persistent = false;
+  const data = /^data:[^;,]+;base64,(.*)$/s.exec(job.imageUrl);
+  if (data) {
+    imageHash = sha256(Buffer.from(data[1], "base64"));
+    persistent = true;
+  } else {
+    imageHash = sha256(job.imageUrl);
+  }
+  return {
+    key: JSON.stringify([
+      plan.backend,
+      plan.settings.model,
+      job.detail ?? "high",
+      imageHash,
+      sha256(normalizedContext(job.contextText)),
+    ]),
+    persistent,
+  };
+}
+
+async function executeDescription(
+  job: ImageJob,
+  plan: VisionPlan,
+  selectedForwardHeaders: Headers,
+  abortSignal?: AbortSignal,
+  recordSidecarOutcome?: SidecarOutcomeRecorder,
+): Promise<DescribeOutcome> {
+  if (plan.backend === "anthropic") {
+    const sidecar = plan.anthropicSidecar;
+    if (!sidecar) return { text: "", error: "anthropic vision sidecar is unavailable" };
+    return describeImageAnthropic(
+      job.imageUrl,
+      job.detail,
+      job.contextText,
+      sidecar.providerName,
+      sidecar.provider,
+      plan.settings,
+      abortSignal,
+    );
+  }
+  if (!plan.forwardProvider) return { text: "", error: "OpenAI vision sidecar is unavailable" };
+  return describeImage(
+    job.imageUrl,
+    job.detail,
+    job.contextText,
+    plan.forwardProvider,
+    selectedForwardHeaders,
+    plan.settings,
+    abortSignal,
+    recordSidecarOutcome,
+  );
+}
+
 /**
  * Replace every image part in the request with a gpt-described text part, so a text-only model can
  * reason about it. Mutates `parsed.context.messages` in place; uses the message's own text as the
@@ -108,9 +274,8 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten
  */
 export async function describeImagesInPlace(
   parsed: OcxParsedRequest,
-  forwardProvider: OcxProviderConfig,
+  plan: VisionPlan,
   selectedForwardHeaders: Headers,
-  settings: VisionSettings,
   abortSignal?: AbortSignal,
   recordSidecarOutcome?: SidecarOutcomeRecorder,
 ): Promise<void> {
@@ -133,9 +298,53 @@ export async function describeImagesInPlace(
   }
   if (jobs.length === 0) return;
 
-  // 2. Describe all images with bounded concurrency (order preserved).
-  const outcomes = await runBounded(jobs, VISION_CONCURRENCY, j =>
-    describeImage(j.imageUrl, j.detail, j.contextText, forwardProvider, selectedForwardHeaders, settings, abortSignal, recordSidecarOutcome));
+  // 2. Admit misses in source order. Cache hits and same-turn waiters do not consume the cap.
+  const inFlight = new Map<string, Promise<DescribeOutcome>>();
+  const executions: Array<() => Promise<void>> = [];
+  const outcomePromises: Array<Promise<DescribeOutcome>> = [];
+  let misses = 0;
+
+  for (const job of jobs) {
+    const identity = descriptionIdentity(job, plan);
+    const cached = identity.persistent ? descriptionCache.get(identity.key) : undefined;
+    if (cached !== undefined) {
+      outcomePromises.push(Promise.resolve({ text: cached }));
+      continue;
+    }
+
+    const existing = inFlight.get(identity.key);
+    if (existing) {
+      outcomePromises.push(existing);
+      continue;
+    }
+
+    if (misses >= plan.maxDescriptionsPerTurn) {
+      const capped = Promise.resolve<DescribeOutcome>({ text: "", error: "description cap reached for this turn" });
+      inFlight.set(identity.key, capped);
+      outcomePromises.push(capped);
+      continue;
+    }
+
+    misses += 1;
+    let resolveOutcome!: (outcome: DescribeOutcome) => void;
+    const pending = new Promise<DescribeOutcome>(resolve => { resolveOutcome = resolve; });
+    inFlight.set(identity.key, pending);
+    outcomePromises.push(pending);
+    executions.push(async () => {
+      let outcome: DescribeOutcome;
+      try {
+        outcome = await executeDescription(job, plan, selectedForwardHeaders, abortSignal, recordSidecarOutcome);
+      } catch (error) {
+        outcome = { text: "", error: error instanceof Error ? error.message : String(error) };
+      }
+      const successfulText = outcome.error ? "" : outcome.text.trim();
+      if (identity.persistent && successfulText) descriptionCache.set(identity.key, successfulText);
+      resolveOutcome(outcome);
+    });
+  }
+
+  await runBounded(executions, VISION_CONCURRENCY, execute => execute());
+  const outcomes = await Promise.all(outcomePromises);
 
   // 3. Rebuild each message, replacing image parts with their descriptions in order.
   let oi = 0;