@bitkyc08/opencodex 2.42.0 → 2.43.0

package/src/vision/describe.ts

@@ -7,11 +7,14 @@ import { sidecarEnter } from "../lib/sidecar-tracker";
 import { applyUpstreamRecoveryInit, fetchWithResetRetry } from "../lib/upstream-retry";
 import { parseSidecarSSE } from "../web-search/parse";
 import type { SidecarOutcomeRecorder } from "../web-search/executor";
+import { NATIVE_RESERVE_MODEL } from "../codex/catalog/native-models";
 
 export interface VisionSettings {
   model: string;
   reasoning: VisionReasoningEffort;
   timeoutMs: number;
+  /** Effective Desktop authless compatibility does not grant auxiliary model use. */
+  reserveCompatibility?: boolean;
 }
 
 /** A description, or an `error` string when it couldn't run (caller injects a graceful marker). */
@@ -58,6 +61,9 @@ export async function describeImage(
   abortSignal?: AbortSignal,
   recordOutcome?: SidecarOutcomeRecorder,
 ): Promise<DescribeOutcome> {
+  if (settings.reserveCompatibility && settings.model === NATIVE_RESERVE_MODEL) {
+    return { text: "", error: "Luna Reserve compatibility is only available as a conversation model, not a vision helper. Choose another vision helper model." };
+  }
   const invalid = validateImageUrl(imageUrl);
   if (invalid) return { text: "", error: invalid };
 

package/src/vision/image-rewrite.ts

@@ -0,0 +1,108 @@
+import type { OcxContentPart, OcxParsedRequest, OcxTextContent } from "../types";
+import type { TranslatorBudget } from "../lib/translator-budget";
+
+export const descriptionEncoder = new TextEncoder();
+
+/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
+export function carriesImages(role: string): boolean {
+  return role === "user" || role === "developer" || role === "toolResult";
+}
+
+
+const IMAGE_OMITTED_TEXT = "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]";
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Keep the native Responses passthrough body aligned with image replacements made in the parsed
+ * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize
+ * `context.messages`; updating only the latter would send the original pixels to a text-only
+ * Responses upstream even after the vision sidecar produced a caption.
+ *
+ * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses
+ * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally
+ * untouched.
+ */
+export function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void {
+  const rawBody = parsed._rawBody;
+  if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return;
+
+  let nextDescription = 0;
+  const rewriteImages = (value: unknown): unknown => {
+    if (Array.isArray(value)) {
+      let changed = false;
+      const rewritten = value.map(entry => {
+        const next = rewriteImages(entry);
+        if (next !== entry) changed = true;
+        return next;
+      });
+      return changed ? rewritten : value;
+    }
+    if (!isPlainRecord(value)) return value;
+    if (value.type === "input_image" && typeof value.image_url === "string") {
+      // Both message and tool-output parsers exclude empty URLs from caption jobs.
+      if (value.image_url.length === 0) {
+        const fileId = typeof value.file_id === "string" && value.file_id.length > 0 ? value.file_id : undefined;
+        return { type: "input_text", text: fileId ? `[image: ${fileId}]` : IMAGE_OMITTED_TEXT };
+      }
+      const description = descriptions[nextDescription++];
+      return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT };
+    }
+    return value;
+  };
+
+  let changed = false;
+  const input = rawBody.input.map(item => {
+    if (!isPlainRecord(item)) return item;
+    const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : "");
+    const role = typeof item.role === "string" ? item.role : "";
+    const isMessageContent = (
+      (type === "message" && (role === "user" || role === "developer"))
+      || type === "agent_message"
+    );
+    const field = isMessageContent
+      ? "content"
+      : (type === "function_call_output" || type === "custom_tool_call_output")
+        ? "output"
+        : undefined;
+    if (!field) return item;
+    const rewritten = rewriteImages(item[field]);
+    if (rewritten === item[field]) return item;
+    changed = true;
+    return { ...item, [field]: rewritten };
+  });
+
+  if (changed) rawBody.input = input;
+}
+
+/**
+ * Fail-closed image strip for sidecar-covered models when NO sidecar plan exists (no forward
+ * provider / missing forwarded auth / sidecar disabled): the upstream is text-only, so forwarding
+ * raw images would 400 or silently confuse it. Replace each image with an explicit marker so the
+ * model (and the user, via its reply) knows the image was dropped rather than ignored.
+ */
+export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean {
+  let stripped = false;
+  const descriptions: string[] = [];
+  for (const msg of parsed.context.messages) {
+    if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue;
+    const parts = msg.content as OcxContentPart[];
+    if (!parts.some(p => p.type === "image")) continue;
+    msg.content = parts.map(p => {
+      if (p.type !== "image") return p;
+      const replacement = { type: "text", text: IMAGE_OMITTED_TEXT } as OcxContentPart;
+      descriptions.push((replacement as OcxTextContent).text);
+      const reservation = translatorBudget?.reserveTransient(
+        descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength,
+        { kind: "request_copies" },
+      );
+      reservation?.commitRetained();
+      return replacement;
+    });
+    stripped = true;
+  }
+  syncRawBodyImageDescriptions(parsed, descriptions);
+  return stripped;
+}

package/src/vision/index.ts

@@ -1,26 +1,16 @@
 import { createHash } from "node:crypto";
-import type { OcxConfig, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTextContent } from "../types";
-import type { VisionReasoningEffort } from "../reasoning-effort";
-import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe";
+import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxTextContent } from "../types";
+import { describeImage, type DescribeOutcome } from "./describe";
 import { describeImageAnthropic } from "./anthropic-describe";
 import { describeImageRouted } from "./routed-describe";
-import { isModelVisionSidecarConsumer as isModelTextOnly, modelAcceptsImageInput } from "./eligibility";
-import { normalizeVisionReasoningForModel } from "./reasoning";
 import type { CodexAuthContext } from "../codex/auth-context";
-import { resolveSidecarAuth } from "../sidecar/auth";
-import type { ResolvedOpenAiForwardSidecar } from "../providers/openai-sidecar";
 import type { SidecarOutcomeRecorder } from "../web-search/executor";
 import { enforceAppOwnedMemoryBudget } from "../lib/app-owned-memory";
 import type { TranslatorBudget } from "../lib/translator-budget";
-import {
-  DEFAULT_VISION_TIMEOUT_MS,
-  MAX_VISION_TIMEOUT_MS,
-  MIN_VISION_TIMEOUT_MS,
-} from "./timeout-bounds";
+import type { VisionPlan } from "./plan";
+import { carriesImages, descriptionEncoder, syncRawBodyImageDescriptions } from "./image-rewrite";
 
 export { describeImage } from "./describe";
-
-/** Backward-compatible request-time name for the shared vision-sidecar consumer predicate. */
 export { isModelVisionSidecarConsumer as isModelTextOnly } from "./eligibility";
 export { describeImageAnthropic, parseAnthropicVisionSSE } from "./anthropic-describe";
 export {
@@ -33,19 +23,25 @@ export {
   visionEligibleModelOptions,
 } from "./eligibility";
 export type { VisionCandidateModel, VisionModelOption, VisionSidecarBackend } from "./eligibility";
+export { DEFAULT_VISION_TIMEOUT_MS, MAX_VISION_TIMEOUT_MS, MIN_VISION_TIMEOUT_MS } from "./timeout-bounds";
 export {
-  DEFAULT_VISION_TIMEOUT_MS,
-  MAX_VISION_TIMEOUT_MS,
-  MIN_VISION_TIMEOUT_MS,
-};
+  DEFAULT_MAX_DESCRIPTIONS_PER_TURN,
+  resolveMaxDescriptionsPerTurn,
+  isValidVisionTimeoutMs,
+  resolveVisionTimeoutMs,
+  findAnthropicVisionProvider,
+  resolveVisionBackend,
+  resolveOpenAiVisionModel,
+  resolveEffectiveVisionModel,
+  shouldResolveOpenAiVisionSidecar,
+  planVisionSidecar,
+} from "./plan";
+export type { AnthropicVisionProvider, VisionPlan } from "./plan";
+export { stripImagesInPlace } from "./image-rewrite";
+
 
-const DEFAULT_VISION_MODEL = "gpt-5.4-mini";
-const DEFAULT_ANTHROPIC_VISION_MODEL = "claude-sonnet-5";
-const DEFAULT_REASONING: VisionReasoningEffort = "low";
-export const DEFAULT_MAX_DESCRIPTIONS_PER_TURN = 8;
 const DESCRIPTION_CACHE_MAX_ENTRIES = 256;
 export const VISION_DESCRIPTION_CACHE_MAX_BYTES = 1024 * 1024;
-const descriptionEncoder = new TextEncoder();
 /** Max images described in parallel — keeps first-token latency bounded without flooding the backend. */
 const VISION_CONCURRENCY = 3;
 /** Per-image description hard cap (chars) so multi-image turns can't blow the main model's context. */
@@ -160,25 +156,6 @@ export function evictOldestVisionDescriptionForBudget(): number {
   return descriptionCache.evictOldest?.() ?? 0;
 }
 
-/** Runtime config is permissive: zero is intentional; malformed values fall back to the bounded default. */
-export function resolveMaxDescriptionsPerTurn(value: unknown): number {
-  if (value === 0) return 0;
-  return typeof value === "number" && Number.isInteger(value) && value > 0
-    ? value
-    : DEFAULT_MAX_DESCRIPTIONS_PER_TURN;
-}
-
-export function isValidVisionTimeoutMs(value: unknown): value is number {
-  return typeof value === "number"
-    && Number.isInteger(value)
-    && value >= MIN_VISION_TIMEOUT_MS
-    && value <= MAX_VISION_TIMEOUT_MS;
-}
-
-/** Runtime config is permissive: malformed or out-of-range values fall back to the default. */
-export function resolveVisionTimeoutMs(value: unknown): number {
-  return isValidVisionTimeoutMs(value) ? value : DEFAULT_VISION_TIMEOUT_MS;
-}
 
 /** Run `worker` over `items` with bounded concurrency, preserving input order in the result array. */
 async function runBounded<T, R>(items: T[], limit: number, worker: (item: T) => Promise<R>): Promise<R[]> {
@@ -198,176 +175,7 @@ function clamp(s: string, max: number): string {
   return s.length <= max ? s : `${s.slice(0, max)}\n…[description truncated]`;
 }
 
-export interface AnthropicVisionProvider {
-  providerName: string;
-  provider: OcxProviderConfig;
-}
-
-/**
- * First enabled Anthropic OAuth provider whose active stored account is not marked for reauth.
- * Delegates to the shared sidecar auth module (#2188) — same predicate as web-search.
- */
-export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionProvider | undefined {
-  const auth = resolveSidecarAuth(config);
-  if (!auth.isAnthropicAuth || !auth.anthropicProviderName || !auth.anthropicProvider) return undefined;
-  return { providerName: auth.anthropicProviderName, provider: auth.anthropicProvider };
-}
-
-export function resolveVisionBackend(
-  explicit: "openai" | "anthropic" | "routed" | undefined,
-  anthropicSidecar: AnthropicVisionProvider | undefined,
-): "openai" | "anthropic" {
-  if (explicit === "openai" || explicit === "anthropic") return explicit;
-  // "routed" collapses to the legacy default order until its describe executor
-  // lands (roadmap 170 → 180 revised): a persisted routed backend without a
-  // dispatchable arm degrades exactly like unset rather than crashing. wp3
-  // replaces this collapse with the real routed arm in planVisionSidecar.
-  return anthropicSidecar ? "anthropic" : "openai";
-}
-
-/** Native model used by the OpenAI vision helper, including its bounded default. */
-export function resolveOpenAiVisionModel(config: Pick<OcxConfig, "visionSidecar">): string {
-  const configured = config.visionSidecar?.model;
-  // Namespaced routed ids never reach the forward executor (see
-  // resolveEffectiveVisionModel).
-  return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL;
-}
-
-/** Effective describer model for the backend `planVisionSidecar` selected. */
-export function resolveEffectiveVisionModel(
-  config: Pick<OcxConfig, "visionSidecar">,
-  backend: "openai" | "anthropic",
-): string {
-  const configured = config.visionSidecar?.model;
-  // A namespaced "provider/model" id belongs to the routed backend only; the
-  // forward/OAuth executors POST the model string verbatim, so it falls back
-  // to the side's default here (PUT coherence rejects new writes of this
-  // shape, but a legacy or hand-edited config must not break the executor).
-  const usable = configured && !configured.includes("/") ? configured : undefined;
-  return backend === "anthropic"
-    ? usable || DEFAULT_ANTHROPIC_VISION_MODEL
-    : usable || DEFAULT_VISION_MODEL;
-}
-
-/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
-function carriesImages(role: string): boolean {
-  return role === "user" || role === "developer" || role === "toolResult";
-}
-
-function messagesHaveImage(parsed: OcxParsedRequest): boolean {
-  return parsed.context.messages.some(m =>
-    carriesImages(m.role) && Array.isArray(m.content) && (m.content as OcxContentPart[]).some(p => p.type === "image"));
-}
-
-export function shouldResolveOpenAiVisionSidecar(
-  config: OcxConfig,
-  provider: OcxProviderConfig,
- modelId: string,
- parsed: OcxParsedRequest,
-): boolean {
-  if (!isModelTextOnly(provider, modelId) || !messagesHaveImage(parsed)) return false;
-  const cfg = config.visionSidecar ?? {};
-  if (cfg.enabled === false) return false;
-  return resolveVisionBackend(cfg.backend, findAnthropicVisionProvider(config)) === "openai";
-}
-
-export interface VisionPlan {
-  backend: "openai" | "anthropic" | "routed";
-  forwardSidecar?: ResolvedOpenAiForwardSidecar;
-  anthropicSidecar?: AnthropicVisionProvider;
-  /** Namespaced "provider/model" describer for the routed backend (roadmap 180). */
-  routedModel?: string;
-  /** Loopback dispatch inputs for the routed backend. */
-  routedConfig?: Pick<OcxConfig, "port" | "apiKeys">;
-  settings: VisionSettings;
-  maxDescriptionsPerTurn: number;
-}
-
-/**
- * Decide whether the vision sidecar should pre-describe images for this request, returning the plan
- * if so. Active when: the routed model is in `provider.noVisionModels`, the request actually carries
- * an image, the sidecar isn't disabled, and the selected backend has usable auth. Returns undefined
- * otherwise (the caller strips images before sending to a text-only model).
- */
-export function planVisionSidecar(
-  config: OcxConfig,
-  provider: OcxProviderConfig,
-  modelId: string,
-  parsed: OcxParsedRequest,
-  openAiSidecar?: ResolvedOpenAiForwardSidecar,
-): VisionPlan | undefined {
-  if (!isModelTextOnly(provider, modelId)) return undefined;
-  if (!messagesHaveImage(parsed)) return undefined;
-  const cfg = config.visionSidecar ?? {};
-  if (cfg.enabled === false) return undefined;
-
-  // Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit
-  // model only — never inferred from credential availability. Plan-time
-  // fence: the target must not be provably blind, and must not itself be a
-  // model this planner would re-enter for (belt; the terminal marker on the
-  // loopback request is the braces).
-  if (cfg.backend === "routed") {
-    const routedModel = cfg.model;
-    const sep = routedModel ? routedModel.indexOf("/") : -1;
-    if (routedModel && sep > 0) {
-      const targetProvider = routedModel.slice(0, sep);
-      const targetId = routedModel.slice(sep + 1);
-      const targetProviderConfig = config.providers?.[targetProvider];
-      const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false
-        && !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId));
-      if (targetVisible) {
-        return {
-          backend: "routed",
-          routedModel,
-          routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) },
-          settings: {
-            model: routedModel,
-            reasoning: DEFAULT_REASONING,
-            timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
-          },
-          maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn),
-        };
-      }
-    }
-    // Misconfigured routed backend (bare id, unknown provider, or provably
-    // blind target): fall through to the legacy default order below rather
-    // than dispatching a describe that cannot work.
-  }
-
-  const anthropicSidecar = findAnthropicVisionProvider(config);
-  const backend = resolveVisionBackend(cfg.backend, anthropicSidecar);
-  // A namespaced routed model must never reach the forward/OAuth executors
-  // (they POST the string verbatim); the effective-model resolver falls back
-  // to each side's default in that case.
-  const model = resolveEffectiveVisionModel(config, backend);
-  const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn);
 
-  if (backend === "anthropic") {
-    if (!anthropicSidecar) return undefined;
-    return {
-      backend,
-      anthropicSidecar,
-      settings: {
-        model,
-        reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING,
-        timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
-      },
-      maxDescriptionsPerTurn,
-    };
-  }
-
-  if (!openAiSidecar) return undefined;
-  return {
-    backend,
-    forwardSidecar: openAiSidecar,
-    settings: {
-      model,
-      reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING,
-        timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
-    },
-    maxDescriptionsPerTurn,
-  };
-}
 
 interface ImageJob {
   imageUrl: string;
@@ -385,71 +193,6 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten
   };
 }
 
-const IMAGE_OMITTED_TEXT = "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]";
-
-function isPlainRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-/**
- * Keep the native Responses passthrough body aligned with image replacements made in the parsed
- * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize
- * `context.messages`; updating only the latter would send the original pixels to a text-only
- * Responses upstream even after the vision sidecar produced a caption.
- *
- * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses
- * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally
- * untouched.
- */
-function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void {
-  const rawBody = parsed._rawBody;
-  if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return;
-
-  let nextDescription = 0;
-  const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => {
-    if (Array.isArray(value)) {
-      let changed = false;
-      const rewritten = value.map(entry => {
-        const next = rewriteImages(entry, nonEmptyImageUrlsOnly);
-        if (next !== entry) changed = true;
-        return next;
-      });
-      return changed ? rewritten : value;
-    }
-    if (!isPlainRecord(value)) return value;
-    if (value.type === "input_image" && typeof value.image_url === "string") {
-      if (nonEmptyImageUrlsOnly && value.image_url.length === 0) {
-        return { type: "input_text", text: IMAGE_OMITTED_TEXT };
-      }
-      const description = descriptions[nextDescription++];
-      return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT };
-    }
-    return value;
-  };
-
-  let changed = false;
-  const input = rawBody.input.map(item => {
-    if (!isPlainRecord(item)) return item;
-    const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : "");
-    const role = typeof item.role === "string" ? item.role : "";
-    const isMessageContent = (
-      (type === "message" && (role === "user" || role === "developer"))
-      || type === "agent_message"
-    );
-    const field = isMessageContent
-      ? "content"
-      : (type === "function_call_output" || type === "custom_tool_call_output")
-        ? "output"
-        : undefined;
-    if (!field) return item;
-    const rewritten = rewriteImages(item[field], isMessageContent);
-    if (rewritten === item[field]) return item;
-    changed = true;
-    return { ...item, [field]: rewritten };
-  });
-
-  if (changed) rawBody.input = input;
-}
 
 function sha256(value: string | Uint8Array): string {
   return createHash("sha256").update(value).digest("hex");
@@ -635,33 +378,3 @@ export async function describeImagesInPlace(
   }
   syncRawBodyImageDescriptions(parsed, descriptions);
 }
-
-/**
- * Fail-closed image strip for sidecar-covered models when NO sidecar plan exists (no forward
- * provider / missing forwarded auth / sidecar disabled): the upstream is text-only, so forwarding
- * raw images would 400 or silently confuse it. Replace each image with an explicit marker so the
- * model (and the user, via its reply) knows the image was dropped rather than ignored.
- */
-export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean {
-  let stripped = false;
-  const descriptions: string[] = [];
-  for (const msg of parsed.context.messages) {
-    if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue;
-    const parts = msg.content as OcxContentPart[];
-    if (!parts.some(p => p.type === "image")) continue;
-    msg.content = parts.map(p => {
-      if (p.type !== "image") return p;
-      const replacement = { type: "text", text: IMAGE_OMITTED_TEXT } as OcxContentPart;
-      descriptions.push((replacement as OcxTextContent).text);
-      const reservation = translatorBudget?.reserveTransient(
-        descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength,
-        { kind: "request_copies" },
-      );
-      reservation?.commitRetained();
-      return replacement;
-    });
-    stripped = true;
-  }
-  syncRawBodyImageDescriptions(parsed, descriptions);
-  return stripped;
-}