@bitkyc08/opencodex 2.29.0 → 2.31.0-preview.20260822

package/src/vision/eligibility.ts

@@ -26,15 +26,27 @@ import { nativeInputModalities } from "../codex/catalog/metadata";
 import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models";
 import { enrichProviderFromRegistry } from "../providers/derive";
 
-/** The two wire protocols `planVisionSidecar` can actually dispatch to. */
-export type VisionSidecarBackend = "openai" | "anthropic";
+/**
+ * The wire protocols `planVisionSidecar` can dispatch to (#2188 roadmap 170
+ * REVISED). "routed" describes through the proxy's OWN router via loopback —
+ * one executor for every non-forward, non-OAuth-Anthropic provider row.
+ */
+export type VisionSidecarBackend = "openai" | "anthropic" | "routed";
+
+/** The two sides every deployment has; also the empty-auth fallback set. */
+export type UniversalVisionBackend = "openai" | "anthropic";
 
 /**
  * Default entry per backend: cheap, image-capable, and present in every deployment. Offered
  * whenever its side is enabled, and withheld only when that provider explicitly lists it as a
  * model the sidecar describes FOR — never merely because a metadata table stayed silent.
+ *
+ * Keyed by the UNIVERSAL subset on purpose (roadmap 170, audit blocker A):
+ * xai/gemini are auth-gated sides whose catalogs are present whenever the side
+ * is, so they carry no baseline, and a narrow-key total record documents that
+ * without sprinkling non-null assertions at the consumers.
  */
-export const BASELINE_VISION_MODELS: Record<VisionSidecarBackend, string> = {
+export const BASELINE_VISION_MODELS: Record<UniversalVisionBackend, string> = {
   openai: "gpt-5.6-luna",
   anthropic: "claude-haiku-4-5",
 };
@@ -146,7 +158,7 @@ function isVisionEligibleModelWithCache(
   return modelAcceptsImageInputWithCache(config, candidate, cache) !== false;
 }
 
-/** Which executor can describe through this row, or undefined when neither can. */
+/** Which executor can describe through this row. */
 export function visionBackendForCandidate(
   config: Pick<OcxConfig, "providers">,
   candidate: VisionCandidateModel,
@@ -158,17 +170,17 @@ export function visionBackendForCandidate(
   // Messages wire is not enough: a key-auth row of the same adapter is unreachable, and an
   // option that cannot be dispatched is worse than a missing one, because selecting it fails
   // at describe time rather than at pick time.
-  //
-  // So the executor's name is REQUIRED for an Anthropic suggestion. When the caller has no
-  // executor to name, no catalog row qualifies; the side's baseline is added separately and
-  // keeps the picker populated. Narrowing here never widens the write gate, which is a
-  // different predicate (`modelAcceptsImageInput`) and still treats unknown as allowed.
-  if (anthropicProviderName === undefined) return undefined;
-  return candidate.provider === anthropicProviderName ? "anthropic" : undefined;
+  if (anthropicProviderName !== undefined && candidate.provider === anthropicProviderName) {
+    return "anthropic";
+  }
+  // EVERY other provider row describes through the proxy's own router
+  // (roadmap 170 revised): the loopback executor covers all provider wires,
+  // so no row is left without an executor.
+  return "routed";
 }
 
 function baselineCandidate(
-  backend: VisionSidecarBackend,
+  backend: UniversalVisionBackend,
   anthropicProviderName: string | undefined,
 ): VisionCandidateModel {
   return {
@@ -181,12 +193,13 @@ function baselineCandidate(
 }
 
 /**
- * The picker's option list: every eligible row reachable by one of the two
- * executors, plus each enabled side's baseline unless that baseline is explicitly
- * excluded, de-duplicated and stably ordered (openai side first, baselines first
- * within a side). Anthropic rows must belong to the OAuth provider that would
- * actually execute them, so `anthropicProviderName` is what makes that side's
- * catalog rows eligible at all.
+ * The picker's option list: every eligible row reachable by an enabled
+ * executor, plus each enabled universal side's baseline unless that baseline
+ * is explicitly excluded, de-duplicated and stably ordered (side rank order,
+ * baselines first within a side). Anthropic rows must belong to the OAuth
+ * provider that would actually execute them, so `anthropicProviderName` is
+ * what makes that side's catalog rows eligible at all; xai/gemini rows map by
+ * provider identity and appear only when the caller enabled those backends.
  *
  * This is the SUGGESTION list (narrow): it emits only rows an executor can reach
  * and some source has heard of. It is deliberately NOT the same set as the write
@@ -208,7 +221,7 @@ export function visionEligibleModelOptions(
   const byValue = new Map<string, VisionModelOption>();
   const enrichedProviders: EnrichedProviderCache = new Map();
 
-  for (const backend of ["openai", "anthropic"] as const) {
+  for (const backend of Object.keys(BASELINE_VISION_MODELS) as UniversalVisionBackend[]) {
     if (!enabled.has(backend)) continue;
     const candidate = baselineCandidate(backend, anthropicProviderName);
     if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
@@ -219,11 +232,19 @@ export function visionEligibleModelOptions(
     const backend = visionBackendForCandidate(config, candidate, anthropicProviderName);
     if (!backend || !enabled.has(backend)) continue;
     if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
-    if (byValue.has(candidate.id)) continue;
-    byValue.set(candidate.id, { value: candidate.id, label: candidate.id, backend });
+    // Routed rows carry NAMESPACED values ("provider/model") so the loopback
+    // dispatch is unambiguous under routeModel; the legacy sides keep bare ids
+    // (GUI current-value compatibility, and the forward/OAuth executors POST
+    // the string verbatim). De-dup stays keyed by the emitted value.
+    const value = backend === "routed" ? `${candidate.provider}/${candidate.id}` : candidate.id;
+    if (byValue.has(value)) continue;
+    byValue.set(value, { value, label: value, backend });
   }
 
+  // Two slots per side (baseline first), ranked openai < anthropic < routed
+  // so widening the union appends rather than interleaves (roadmap 170).
+  const sideRank: Record<VisionSidecarBackend, number> = { openai: 0, anthropic: 2, routed: 4 };
   const order = (option: VisionModelOption) =>
-    (option.backend === "openai" ? 0 : 2) + (option.baseline ? 0 : 1);
+    sideRank[option.backend] + (option.baseline ? 0 : 1);
   return [...byValue.values()].sort((a, b) => order(a) - order(b) || a.value.localeCompare(b.value));
 }

package/src/vision/index.ts

@@ -5,6 +5,8 @@ import { modelRecordValue } from "../reasoning-effort";
 import type { VisionReasoningEffort } from "../reasoning-effort";
 import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe";
 import { describeImageAnthropic } from "./anthropic-describe";
+import { describeImageRouted } from "./routed-describe";
+import { modelAcceptsImageInput } from "./eligibility";
 import { normalizeVisionReasoningForModel } from "./reasoning";
 import type { CodexAuthContext } from "../codex/auth-context";
 import { resolveSidecarAuth } from "../sidecar/auth";
@@ -226,16 +228,23 @@ export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionP
 }
 
 export function resolveVisionBackend(
-  explicit: "openai" | "anthropic" | undefined,
+  explicit: "openai" | "anthropic" | "routed" | undefined,
   anthropicSidecar: AnthropicVisionProvider | undefined,
 ): "openai" | "anthropic" {
   if (explicit === "openai" || explicit === "anthropic") return explicit;
+  // "routed" collapses to the legacy default order until its describe executor
+  // lands (roadmap 170 → 180 revised): a persisted routed backend without a
+  // dispatchable arm degrades exactly like unset rather than crashing. wp3
+  // replaces this collapse with the real routed arm in planVisionSidecar.
   return anthropicSidecar ? "anthropic" : "openai";
 }
 
 /** Native model used by the OpenAI vision helper, including its bounded default. */
 export function resolveOpenAiVisionModel(config: Pick<OcxConfig, "visionSidecar">): string {
-  return config.visionSidecar?.model || DEFAULT_VISION_MODEL;
+  const configured = config.visionSidecar?.model;
+  // Namespaced routed ids never reach the forward executor (see
+  // resolveEffectiveVisionModel).
+  return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL;
 }
 
 /** Effective describer model for the backend `planVisionSidecar` selected. */
@@ -243,9 +252,15 @@ export function resolveEffectiveVisionModel(
   config: Pick<OcxConfig, "visionSidecar">,
   backend: "openai" | "anthropic",
 ): string {
+  const configured = config.visionSidecar?.model;
+  // A namespaced "provider/model" id belongs to the routed backend only; the
+  // forward/OAuth executors POST the model string verbatim, so it falls back
+  // to the side's default here (PUT coherence rejects new writes of this
+  // shape, but a legacy or hand-edited config must not break the executor).
+  const usable = configured && !configured.includes("/") ? configured : undefined;
   return backend === "anthropic"
-    ? config.visionSidecar?.model || DEFAULT_ANTHROPIC_VISION_MODEL
-    : resolveOpenAiVisionModel(config);
+    ? usable || DEFAULT_ANTHROPIC_VISION_MODEL
+    : usable || DEFAULT_VISION_MODEL;
 }
 
 /** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
@@ -271,9 +286,13 @@ export function shouldResolveOpenAiVisionSidecar(
 }
 
 export interface VisionPlan {
-  backend: "openai" | "anthropic";
+  backend: "openai" | "anthropic" | "routed";
   forwardSidecar?: ResolvedOpenAiForwardSidecar;
   anthropicSidecar?: AnthropicVisionProvider;
+  /** Namespaced "provider/model" describer for the routed backend (roadmap 180). */
+  routedModel?: string;
+  /** Loopback dispatch inputs for the routed backend. */
+  routedConfig?: Pick<OcxConfig, "port" | "apiKeys">;
   settings: VisionSettings;
   maxDescriptionsPerTurn: number;
 }
@@ -295,8 +314,45 @@ export function planVisionSidecar(
   if (!messagesHaveImage(parsed)) return undefined;
   const cfg = config.visionSidecar ?? {};
   if (cfg.enabled === false) return undefined;
+
+  // Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit
+  // model only — never inferred from credential availability. Plan-time
+  // fence: the target must not be provably blind, and must not itself be a
+  // model this planner would re-enter for (belt; the terminal marker on the
+  // loopback request is the braces).
+  if (cfg.backend === "routed") {
+    const routedModel = cfg.model;
+    const sep = routedModel ? routedModel.indexOf("/") : -1;
+    if (routedModel && sep > 0) {
+      const targetProvider = routedModel.slice(0, sep);
+      const targetId = routedModel.slice(sep + 1);
+      const targetProviderConfig = config.providers?.[targetProvider];
+      const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false
+        && !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId));
+      if (targetVisible) {
+        return {
+          backend: "routed",
+          routedModel,
+          routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) },
+          settings: {
+            model: routedModel,
+            reasoning: DEFAULT_REASONING,
+            timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
+          },
+          maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn),
+        };
+      }
+    }
+    // Misconfigured routed backend (bare id, unknown provider, or provably
+    // blind target): fall through to the legacy default order below rather
+    // than dispatching a describe that cannot work.
+  }
+
   const anthropicSidecar = findAnthropicVisionProvider(config);
   const backend = resolveVisionBackend(cfg.backend, anthropicSidecar);
+  // A namespaced routed model must never reach the forward/OAuth executors
+  // (they POST the string verbatim); the effective-model resolver falls back
+  // to each side's default in that case.
   const model = resolveEffectiveVisionModel(config, backend);
   const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn);
 
@@ -447,6 +503,18 @@ async function executeDescription(
   abortSignal?: AbortSignal,
   recordSidecarOutcome?: SidecarOutcomeRecorder,
 ): Promise<DescribeOutcome> {
+  if (plan.backend === "routed") {
+    if (!plan.routedModel || !plan.routedConfig) return { text: "", error: "routed vision sidecar is unavailable" };
+    return describeImageRouted(
+      job.imageUrl,
+      job.detail,
+      job.contextText,
+      plan.routedModel,
+      plan.routedConfig,
+      plan.settings,
+      abortSignal,
+    );
+  }
   if (plan.backend === "anthropic") {
     const sidecar = plan.anthropicSidecar;
     if (!sidecar) return { text: "", error: "anthropic vision sidecar is unavailable" };

package/src/vision/routed-describe.ts

@@ -0,0 +1,175 @@
+/**
+ * Describe ONE image via a ROUTED model through the proxy's own
+ * /v1/chat/completions on loopback (#2188 roadmap 180 revised).
+ *
+ * One executor for every provider the router can reach: the chat inbound
+ * translates image_url parts and each adapter compiles its own wire
+ * (Anthropic blocks, Antigravity inlineData, xai Responses input_image, plain
+ * openai-chat), so provider coverage is the router's job, not this file's.
+ *
+ * Recursion fence: the request carries `x-opencodex-vision-describe: 1`.
+ * The Chat surface detects the raw header before its bridge rebuilds headers
+ * and carries it into handleResponses as `visionDescribeTerminal`; a marked
+ * request STRIPS images instead of planning another describe (depth cap 1,
+ * holds under predicate drift and combo re-resolution — audit rounds 2-4).
+ *
+ * Admission ladder (audit round 3): configuredApiAuthToken() (env token) ||
+ * service token file || first config.apiKeys entry, sent as
+ * `x-opencodex-api-key` — never Authorization (gateway-cache.ts rule: an
+ * admission secret in a forwardable header is a forwarding hazard). Loopback
+ * binds require no token at all (resolveApiAuth admits loopback).
+ *
+ * Known limitation (recorded in roadmap 170): a bindHost where 127.0.0.1
+ * does not answer cannot reach its own loopback — same latent limitation
+ * gateway-cache has.
+ */
+import type { OcxConfig } from "../types";
+import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
+import { redactSecretString } from "../lib/redact";
+import { sidecarEnter } from "../lib/sidecar-tracker";
+import { configuredApiAuthToken, configuredPort } from "../server/auth-cors";
+import { loadServiceTokenFromFile } from "../lib/service-secrets";
+import type { DescribeOutcome, VisionSettings } from "./describe";
+
+export const VISION_DESCRIBE_TERMINAL_HEADER = "x-opencodex-vision-describe";
+
+const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]);
+const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
+/** Bound the loopback JSON response; descriptions are clamped to ~2k chars by the caller anyway. */
+const MAX_ROUTED_RESPONSE_BYTES = 4 * 1024 * 1024;
+
+const DESCRIBE_INSTRUCTION =
+  "You are a vision describer for a text-only model that cannot see the image. Describe the image "
+  + "thoroughly and factually so that model can fully reason about it: transcribe any visible text "
+  + "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on "
+  + "what's relevant to the user's request. Output only the description.";
+
+function validateImageUrl(url: string): string | null {
+  if (url.startsWith("data:")) {
+    const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url);
+    if (!match) return "malformed data URL";
+    const mime = match[1].toLowerCase();
+    if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`;
+    if (match[2]) {
+      const bytes = Math.floor((match[3].length * 3) / 4);
+      if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`;
+    }
+    return null;
+  }
+  if (url.startsWith("https://")) return null;
+  return "unsupported image URL scheme (expected data: or https:)";
+}
+
+/** The admission ladder: env token, service token file, first configured API key. */
+export function routedDescribeAdmissionToken(config: Pick<OcxConfig, "apiKeys">): string | undefined {
+  const envToken = configuredApiAuthToken();
+  if (envToken) return envToken;
+  const fileToken = loadServiceTokenFromFile(process.env);
+  if (fileToken) return fileToken;
+  const first = config.apiKeys?.[0]?.key?.trim();
+  return first || undefined;
+}
+
+/** Base URL seam for tests; production always self-fetches loopback. */
+export function routedDescribeBaseUrl(config: Pick<OcxConfig, "port">): string {
+  // config.port can be 0 (ephemeral bind, tests) or stale after a live port
+  // override; the server records its ACTUAL bound port via setCorsOrigin at
+  // startup, so prefer that when config carries no positive port.
+  const port = config.port && config.port > 0 ? String(config.port) : configuredPort();
+  return `http://127.0.0.1:${port}`;
+}
+
+export async function describeImageRouted(
+  imageUrl: string,
+  _detail: string | undefined,
+  contextText: string,
+  routedModel: string,
+  config: Pick<OcxConfig, "port" | "apiKeys">,
+  settings: VisionSettings,
+  abortSignal?: AbortSignal,
+  baseUrlOverride?: string,
+): Promise<DescribeOutcome> {
+  const invalid = validateImageUrl(imageUrl);
+  if (invalid) return { text: "", error: invalid };
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    [VISION_DESCRIBE_TERMINAL_HEADER]: "1",
+  };
+  const admission = routedDescribeAdmissionToken(config);
+  if (admission) headers["x-opencodex-api-key"] = admission;
+
+  const requestBody = {
+    model: routedModel,
+    stream: false,
+    messages: [
+      { role: "system", content: DESCRIBE_INSTRUCTION },
+      {
+        role: "user",
+        content: [
+          ...(contextText ? [{ type: "text", text: `User's request context: ${contextText}` }] : []),
+          { type: "image_url", image_url: { url: imageUrl } },
+        ],
+      },
+    ],
+  };
+
+  const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
+  const sidecarExit = sidecarEnter("vision");
+  const t0 = Date.now();
+  try {
+    const res = await fetch(`${baseUrlOverride ?? routedDescribeBaseUrl(config)}/v1/chat/completions`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(requestBody),
+      signal: linkedSignal.signal,
+      redirect: "manual",
+    });
+    const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
+    try {
+      const raw = await res.text();
+      if (raw.length > MAX_ROUTED_RESPONSE_BYTES) {
+        return { text: "", error: "routed describe response exceeded byte bound" };
+      }
+      if (!res.ok) {
+        return { text: "", error: `routed describe HTTP ${res.status}: ${redactSecretString(raw.slice(0, 200))}` };
+      }
+      let payload: unknown;
+      try { payload = JSON.parse(raw); } catch {
+        return { text: "", error: "routed describe returned non-JSON" };
+      }
+      const content = extractChatContent(payload);
+      if (!content) return { text: "", error: "routed describe returned no text" };
+      return { text: content };
+    } finally {
+      detachBodyGuard();
+    }
+  } catch (e) {
+    const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
+    console.warn(`[vision] routed describe ${kind} (${Date.now() - t0}ms)`);
+    return { text: "", error: redactSecretString(e instanceof Error ? e.message : String(e)) };
+  } finally {
+    sidecarExit();
+    linkedSignal.cleanup();
+  }
+}
+
+function extractChatContent(payload: unknown): string | undefined {
+  if (!payload || typeof payload !== "object") return undefined;
+  const choices = (payload as { choices?: unknown }).choices;
+  if (!Array.isArray(choices) || choices.length === 0) return undefined;
+  const message = (choices[0] as { message?: unknown })?.message;
+  if (!message || typeof message !== "object") return undefined;
+  const content = (message as { content?: unknown }).content;
+  if (typeof content === "string" && content.trim().length > 0) return content;
+  // Some adapters emit content parts; join text parts.
+  if (Array.isArray(content)) {
+    const joined = content
+      .map(part => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
+        ? (part as { text: string }).text
+        : ""))
+      .join("");
+    if (joined.trim().length > 0) return joined;
+  }
+  return undefined;
+}