@iislee/opencodex 2.11.0 → 2.35.0

package/src/vision/backends.ts

@@ -0,0 +1,97 @@
+/**
+ * Which backends may DESCRIBE images for the vision sidecar, and which
+ * candidate rows each can describe through (#2188 vision rules; roadmap 170
+ * REVISED: the "routed" backend).
+ *
+ * A SIBLING of WEB_SEARCH_BACKENDS, not a shared table: vision has no
+ * per-model probe (rule 2 is "− provably text-only", enforced by
+ * modelAcceptsImageInput, not here), carries per-side baseline models, and
+ * excludes non-LLM backends like exa.
+ *
+ * Three backends, not one per provider: "openai" and "anthropic" carry auth
+ * semantics loopback routing cannot replicate (forwarded ChatGPT headers,
+ * OAuth beta fences) and their defaults must not drift. Every OTHER
+ * picker-visible provider row reaches the describer through "routed" — a
+ * loopback self-fetch of the proxy's own /v1/chat/completions, where the
+ * router and adapters already speak each provider's wire. That is what makes
+ * this table closed under provider growth: a new provider needs no new
+ * describe executor.
+ */
+import type { OcxConfig } from "../types";
+import type { SidecarAuthState } from "../sidecar/auth";
+import { listOpenAiForwardSidecarCandidates } from "../providers/openai-sidecar";
+import type { VisionCandidateModel, VisionSidecarBackend } from "./eligibility";
+
+export interface VisionBackendDescriptor {
+  backend: VisionSidecarBackend;
+  /** Liveness signal for this backend. */
+  isActive(auth: SidecarAuthState, config: OcxConfig): boolean;
+  /** Which candidate rows this backend's describe executor can actually run. */
+  candidateMatch(candidate: VisionCandidateModel, auth: SidecarAuthState): boolean;
+  /**
+   * Default entry for this side: cheap, image-capable, present in every
+   * deployment. Only the two universal sides carry one — "routed" has no
+   * universal model to name.
+   */
+  baseline?: string;
+  /** Stable option ordering (baselines first within a side). */
+  rank: number;
+}
+
+export const VISION_BACKENDS: readonly VisionBackendDescriptor[] = [
+  {
+    backend: "openai",
+    // The OpenAI describer needs a CANONICAL ChatGPT forward provider, not
+    // merely a provider keyed "openai" — same predicate the runtime sidecar
+    // resolver uses. Deliberately NOT auth.isCodexAuth: tightening to a live
+    // credential here would change which options a fresh install sees, and
+    // the options list is a suggestion surface, not the write gate.
+    isActive: (_auth, config) => listOpenAiForwardSidecarCandidates(config).length > 0,
+    candidateMatch: candidate => candidate.native === true || candidate.provider === "openai",
+    baseline: "gpt-5.6-luna",
+    rank: 0,
+  },
+  {
+    backend: "anthropic",
+    isActive: auth => auth.isAnthropicAuth,
+    // The runtime dispatches through exactly ONE Anthropic provider — the
+    // resolved OAuth row. Same-adapter keyed rows are unreachable (see
+    // visionBackendForCandidate's original stance).
+    candidateMatch: (candidate, auth) => candidate.provider === auth.anthropicProviderName,
+    baseline: "claude-haiku-4-5",
+    rank: 1,
+  },
+  {
+    backend: "routed",
+    // Always offered: options only materialize when a matching picker row
+    // exists, and the row's own provider config is the liveness signal — the
+    // loopback request fails closed through ordinary routing errors.
+    isActive: () => true,
+    // Any row the other two executors do NOT own. Auth-slot rows are
+    // entitlements of the openai/anthropic sides and never route here.
+    candidateMatch: (candidate, auth) =>
+      candidate.native !== true
+      && candidate.provider !== "openai"
+      && candidate.provider !== auth.anthropicProviderName,
+    rank: 2,
+  },
+];
+
+export function visionBackendDescriptor(backend: VisionSidecarBackend): VisionBackendDescriptor {
+  const descriptor = VISION_BACKENDS.find(entry => entry.backend === backend);
+  if (!descriptor) throw new Error(`unknown vision backend "${backend}"`);
+  return descriptor;
+}
+
+/**
+ * The active backend set for option generation. Falls back to the two
+ * UNIVERSAL sides when neither is active (fresh install: picker stays
+ * populated, permissive-unknown rule); "routed" is active by construction.
+ */
+export function activeVisionBackends(auth: SidecarAuthState, config: OcxConfig): VisionSidecarBackend[] {
+  const active = VISION_BACKENDS.filter(entry => entry.isActive(auth, config)).map(entry => entry.backend);
+  return active.includes("openai") || active.includes("anthropic")
+    ? active
+    : ["openai", "anthropic", ...active.filter(backend => backend === "routed")];
+}
+

package/src/vision/describe.ts

@@ -1,4 +1,5 @@
 import type { OcxProviderConfig } from "../types";
+import type { VisionReasoningEffort } from "../reasoning-effort";
 import { FORWARD_HEADERS } from "../adapters/openai-responses";
 import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
 import { redactSecretString } from "../lib/redact";
@@ -9,6 +10,7 @@ import type { SidecarOutcomeRecorder } from "../web-search/executor";
 
 export interface VisionSettings {
   model: string;
+  reasoning: VisionReasoningEffort;
   timeoutMs: number;
 }
 
@@ -77,9 +79,9 @@ export async function describeImage(
       "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on " +
       "what's relevant to the user's request. Output only the description.",
     input: [{ type: "message", role: "user", content }],
-    reasoning: { effort: "low" },
-    // The ChatGPT (codex) backend rejects `max_output_tokens` ("Unsupported parameter"); the
-    // description is clamped downstream (DESC_MAX_CHARS) instead.
+    reasoning: { effort: settings.reasoning },
+    // The ChatGPT (codex) backend rejects `max_output_tokens` ("Unsupported parameter"); the shared
+    // SSE parser bounds raw response bytes before DESC_MAX_CHARS applies its display clamp.
     store: false,
     stream: true,
   };
@@ -93,29 +95,38 @@ export async function describeImage(
         headers,
         body: JSON.stringify(body),
         signal: linkedSignal.signal,
+        // Credential-bearing: do not follow a cross-origin 3xx. Bun strips `Authorization`
+        // across origins but forwards nonstandard headers such as `chatgpt-account-id`,
+        // `session_id`, and `x-codex-turn-metadata` to the redirect target.
+        redirect: "manual",
       }),
       { abortSignal: linkedSignal.signal, label: "vision-sidecar" },
     );
-    recordOutcome?.(res.status);
-    if (!res.ok) {
-      const t = await res.text().catch(() => "");
-      console.warn(`[vision] sidecar HTTP ${res.status} (${Date.now() - t0}ms)`);
-      return { text: "", error: `vision sidecar HTTP ${res.status}: ${redactSecretString(t.slice(0, 200))}` };
-    }
     const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
-    let parsed;
     try {
-      parsed = await parseSidecarSSE(res);
+      if (!res.ok) {
+        recordOutcome?.(res.status);
+        const t = await res.text().catch(() => "");
+        console.warn(`[vision] sidecar HTTP ${res.status} (${Date.now() - t0}ms)`);
+        return { text: "", error: `vision sidecar HTTP ${res.status}: ${redactSecretString(t.slice(0, 200))}` };
+      }
+      const parsed = await parseSidecarSSE(res);
+      if (linkedSignal.signal.aborted) throw linkedSignal.signal.reason;
+      recordOutcome?.(res.status);
+      // The backend can return HTTP 200 then stream a `response.failed`/`error` event with no text;
+      // surface that as a describe error instead of an empty (silently-blank) description.
+      if (!parsed.text.trim() && parsed.error) return { text: "", error: parsed.error };
+      return { text: parsed.text };
     } finally {
       detachBodyGuard();
     }
-    // The backend can return HTTP 200 then stream a `response.failed`/`error` event with no text;
-    // surface that as a describe error instead of an empty (silently-blank) description.
-    if (!parsed.text.trim() && parsed.error) return { text: "", error: parsed.error };
-    return { text: parsed.text };
   } catch (e) {
-    recordOutcome?.(e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error");
     const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
+    const callerAborted = abortSignal?.aborted === true
+      && linkedSignal.signal.aborted
+      && linkedSignal.signal.reason === abortSignal.reason
+      && e === linkedSignal.signal.reason;
+    recordOutcome?.(callerAborted ? "connect_neutral" : kind);
     console.warn(`[vision] sidecar ${kind} (${Date.now() - t0}ms)`);
     return { text: "", error: e instanceof Error ? e.message : String(e) };
   } finally {

package/src/vision/eligibility.ts

@@ -0,0 +1,250 @@
+/**
+ * Which models may serve AS the vision sidecar (the describer), as opposed to the
+ * models the sidecar describes FOR.
+ *
+ * Three rules make this non-obvious, and all are load-bearing:
+ *
+ * 1. `provider.noVisionModels` marks models the proxy describes images for, and
+ *    `applyProviderConfigHints` deliberately ADDS "image" to their advertised
+ *    input modalities so the Codex app does not block the attachment client-side.
+ *    A blind model therefore advertises image input. Membership in that list is a
+ *    hard disqualifier here, checked BEFORE the modality list it rewrote.
+ * 2. The catalog enriches configured providers from the registry before it
+ *    applies that rule. Eligibility must use the same effective provider policy,
+ *    including a `noVisionModels` list learned after the config was persisted.
+ * 3. Catalog rows frequently omit `inputModalities` entirely (the live
+ *    `/api/models` response carries none for either openai or anthropic rows).
+ *    Unknown is not zero: when no source can speak, the model stays eligible
+ *    rather than silently vanishing from the picker.
+ *    Native OpenAI metadata is likewise authoritative only for a native row or
+ *    the canonical OpenAI provider; a routed row with the same id owns its own
+ *    explicit modalities.
+ */
+import { modelInList, type OcxConfig, type OcxProviderConfig } from "../types";
+import { getModelMetadataCaseInsensitive, resolveMetadataProvider } from "../generated/model-metadata";
+import { nativeInputModalities } from "../codex/catalog/metadata";
+import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models";
+import { enrichProviderFromRegistry } from "../providers/derive";
+
+/**
+ * The wire protocols `planVisionSidecar` can dispatch to (#2188 roadmap 170
+ * REVISED). "routed" describes through the proxy's OWN router via loopback —
+ * one executor for every non-forward, non-OAuth-Anthropic provider row.
+ */
+export type VisionSidecarBackend = "openai" | "anthropic" | "routed";
+
+/** The two sides every deployment has; also the empty-auth fallback set. */
+export type UniversalVisionBackend = "openai" | "anthropic";
+
+/**
+ * Default entry per backend: cheap, image-capable, and present in every deployment. Offered
+ * whenever its side is enabled, and withheld only when that provider explicitly lists it as a
+ * model the sidecar describes FOR — never merely because a metadata table stayed silent.
+ *
+ * Keyed by the UNIVERSAL subset on purpose (roadmap 170, audit blocker A):
+ * xai/gemini are auth-gated sides whose catalogs are present whenever the side
+ * is, so they carry no baseline, and a narrow-key total record documents that
+ * without sprinkling non-null assertions at the consumers.
+ */
+export const BASELINE_VISION_MODELS: Record<UniversalVisionBackend, string> = {
+  openai: "gpt-5.6-luna",
+  anthropic: "claude-haiku-4-5",
+};
+
+export interface VisionCandidateModel {
+  provider: string;
+  id: string;
+  inputModalities?: string[];
+  native?: boolean;
+}
+
+export interface VisionModelOption {
+  value: string;
+  label: string;
+  backend: VisionSidecarBackend;
+  /** True when the row is a guaranteed baseline rather than a catalog discovery. */
+  baseline?: boolean;
+}
+
+type EnrichedProviderCache = Map<string, OcxProviderConfig>;
+
+function advertisesImageInput(modalities: readonly string[] | undefined): boolean | undefined {
+  if (!modalities || modalities.length === 0) return undefined;
+  return modalities.includes("image");
+}
+
+/** Vendor-table modalities for a routed row, or undefined when the table has no opinion. */
+function metadataImageInput(provider: string, modelId: string): boolean | undefined {
+  const resolved = resolveMetadataProvider(provider) ?? provider;
+  const meta = getModelMetadataCaseInsensitive(resolved, modelId);
+  return advertisesImageInput(meta?.input);
+}
+
+/**
+ * Mirrors the catalog's registry enrichment without mutating saved configuration. The picker
+ * checks many rows from one provider, so its caller shares this small call-local cache.
+ */
+function enrichedProviderForVision(
+  config: Pick<OcxConfig, "providers">,
+  providerName: string,
+  cache: EnrichedProviderCache,
+): OcxProviderConfig | undefined {
+  const cached = cache.get(providerName);
+  if (cached) return cached;
+  const configured = config.providers?.[providerName];
+  if (!configured) return undefined;
+  const enriched = structuredClone(configured);
+  enrichProviderFromRegistry(providerName, enriched);
+  cache.set(providerName, enriched);
+  return enriched;
+}
+
+function isVisionSidecarConsumerWithCache(
+  config: Pick<OcxConfig, "providers">,
+  providerName: string,
+  modelId: string,
+  cache: EnrichedProviderCache,
+): boolean {
+  return modelInList(enrichedProviderForVision(config, providerName, cache)?.noVisionModels, modelId);
+}
+
+/**
+ * Is this model listed as one the sidecar describes FOR? Such a model cannot be
+ * the describer, and its advertised modalities are untrustworthy.
+ */
+export function isVisionSidecarConsumer(config: Pick<OcxConfig, "providers">, providerName: string, modelId: string): boolean {
+  return isVisionSidecarConsumerWithCache(config, providerName, modelId, new Map());
+}
+
+/**
+ * Can this model accept an image on the wire? Sources are consulted in descending
+ * trustworthiness; the first that speaks wins. `undefined` means nothing knows,
+ * which callers treat as eligible.
+ */
+export function modelAcceptsImageInput(
+  config: Pick<OcxConfig, "providers">,
+  candidate: VisionCandidateModel,
+): boolean | undefined {
+  return modelAcceptsImageInputWithCache(config, candidate, new Map());
+}
+
+function modelAcceptsImageInputWithCache(
+  config: Pick<OcxConfig, "providers">,
+  candidate: VisionCandidateModel,
+  cache: EnrichedProviderCache,
+): boolean | undefined {
+  if (isVisionSidecarConsumerWithCache(config, candidate.provider, candidate.id, cache)) return false;
+  if (candidate.native === true || (candidate.provider === "openai" && SUPPORTED_NATIVE_OPENAI_SLUGS.has(candidate.id))) {
+    return advertisesImageInput(nativeInputModalities(candidate.id)) ?? true;
+  }
+  const fromRow = advertisesImageInput(candidate.inputModalities);
+  if (fromRow !== undefined) return fromRow;
+  return metadataImageInput(candidate.provider, candidate.id);
+}
+
+/** Eligible = not a sidecar consumer, and not positively known to be text-only. */
+export function isVisionEligibleModel(
+  config: Pick<OcxConfig, "providers">,
+  candidate: VisionCandidateModel,
+): boolean {
+  return isVisionEligibleModelWithCache(config, candidate, new Map());
+}
+
+function isVisionEligibleModelWithCache(
+  config: Pick<OcxConfig, "providers">,
+  candidate: VisionCandidateModel,
+  cache: EnrichedProviderCache,
+): boolean {
+  return modelAcceptsImageInputWithCache(config, candidate, cache) !== false;
+}
+
+/** Which executor can describe through this row. */
+export function visionBackendForCandidate(
+  config: Pick<OcxConfig, "providers">,
+  candidate: VisionCandidateModel,
+  anthropicProviderName?: string,
+): VisionSidecarBackend | undefined {
+  if (candidate.native || candidate.provider === "openai") return "openai";
+  // The runtime dispatches through exactly ONE Anthropic provider — the enabled OAuth row
+  // with a healthy active account that `findAnthropicVisionProvider` picks. Speaking the
+  // Messages wire is not enough: a key-auth row of the same adapter is unreachable, and an
+  // option that cannot be dispatched is worse than a missing one, because selecting it fails
+  // at describe time rather than at pick time.
+  if (anthropicProviderName !== undefined && candidate.provider === anthropicProviderName) {
+    return "anthropic";
+  }
+  // EVERY other provider row describes through the proxy's own router
+  // (roadmap 170 revised): the loopback executor covers all provider wires,
+  // so no row is left without an executor.
+  return "routed";
+}
+
+function baselineCandidate(
+  backend: UniversalVisionBackend,
+  anthropicProviderName: string | undefined,
+): VisionCandidateModel {
+  return {
+    provider: backend === "openai" ? "openai" : anthropicProviderName ?? "anthropic",
+    id: BASELINE_VISION_MODELS[backend],
+    // Both baselines are known image-capable. This makes their only exclusion path the
+    // provider's explicit consumer list, never a silent or stale metadata table.
+    inputModalities: ["text", "image"],
+  };
+}
+
+/**
+ * The picker's option list: every eligible row reachable by an enabled
+ * executor, plus each enabled universal side's baseline unless that baseline
+ * is explicitly excluded, de-duplicated and stably ordered (side rank order,
+ * baselines first within a side). Anthropic rows must belong to the OAuth
+ * provider that would actually execute them, so `anthropicProviderName` is
+ * what makes that side's catalog rows eligible at all; xai/gemini rows map by
+ * provider identity and appear only when the caller enabled those backends.
+ *
+ * This is the SUGGESTION list (narrow): it emits only rows an executor can reach
+ * and some source has heard of. It is deliberately NOT the same set as the write
+ * gate. `modelAcceptsImageInput` answers "can we prove this model cannot see?"
+ * and is the only input to rejection. Absence from this list must never imply
+ * rejection — an unknown id stays eligible via the undefined → eligible fallback.
+ *
+ * De-duplication is by BARE model id, first eligible row wins. Two providers of
+ * the same adapter family can expose the same id; they resolve to the same
+ * backend, and only `value` reaches the client, so first-wins costs nothing.
+ */
+export function visionEligibleModelOptions(
+  config: Pick<OcxConfig, "providers">,
+  candidates: readonly VisionCandidateModel[],
+  enabledBackends: readonly VisionSidecarBackend[],
+  anthropicProviderName?: string,
+): VisionModelOption[] {
+  const enabled = new Set(enabledBackends);
+  const byValue = new Map<string, VisionModelOption>();
+  const enrichedProviders: EnrichedProviderCache = new Map();
+
+  for (const backend of Object.keys(BASELINE_VISION_MODELS) as UniversalVisionBackend[]) {
+    if (!enabled.has(backend)) continue;
+    const candidate = baselineCandidate(backend, anthropicProviderName);
+    if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
+    const id = candidate.id;
+    byValue.set(id, { value: id, label: id, backend, baseline: true });
+  }
+  for (const candidate of candidates) {
+    const backend = visionBackendForCandidate(config, candidate, anthropicProviderName);
+    if (!backend || !enabled.has(backend)) continue;
+    if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
+    // Routed rows carry NAMESPACED values ("provider/model") so the loopback
+    // dispatch is unambiguous under routeModel; the legacy sides keep bare ids
+    // (GUI current-value compatibility, and the forward/OAuth executors POST
+    // the string verbatim). De-dup stays keyed by the emitted value.
+    const value = backend === "routed" ? `${candidate.provider}/${candidate.id}` : candidate.id;
+    if (byValue.has(value)) continue;
+    byValue.set(value, { value, label: value, backend });
+  }
+
+  // Two slots per side (baseline first), ranked openai < anthropic < routed
+  // so widening the union appends rather than interleaves (roadmap 170).
+  const sideRank: Record<VisionSidecarBackend, number> = { openai: 0, anthropic: 2, routed: 4 };
+  const order = (option: VisionModelOption) =>
+    sideRank[option.backend] + (option.baseline ? 0 : 1);
+  return [...byValue.values()].sort((a, b) => order(a) - order(b) || a.value.localeCompare(b.value));
+}