@iislee/opencodex 2.11.0 → 2.35.0

package/src/server/responses/input-admission.ts

@@ -0,0 +1,187 @@
+/**
+ * Pre-dispatch input admission (#1412).
+ *
+ * Refuses a turn whose estimated input cannot plausibly fit the model context window,
+ * BEFORE auth resolution, circuit admission, or any upstream I/O. #1412 reported ~127k of
+ * real context compounding to 1.3M-1.6M tokens and crashing the proxy; the provider would
+ * reject such a turn anyway, so paying for the round trip buys nothing.
+ *
+ * Deliberately narrow. This is not a context manager and not a compaction trigger: it
+ * catches the pathological case and stays out of the way otherwise. Every uncertainty
+ * resolves toward admitting.
+ */
+import { nativeOpenAiContextWindow, nativeOpenAiMaxInputTokens, type NativeContextLimitsInput } from "../../codex/catalog/metadata";
+import { estimateTokens } from "../../lib/token-estimate";
+import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers";
+import { modelRecordValue } from "../../reasoning-effort";
+import type { OcxContentPart, OcxParsedRequest, OcxProviderConfig } from "../../types";
+
+/**
+ * Multiplier applied to the ceiling before refusing.
+ *
+ * 2.5, not something tighter, because `estimateTokens` can overshoot by 1.6x on its own.
+ * `cjkRatio` samples every `stride`-th character, so a payload of fixed-width records whose
+ * length aligns with the stride samples as 100% CJK while being ~1.6% CJK, firing the
+ * 2.5-chars/token clamp instead of the 4.0 default. Measured on Bun 1.3.14: 126,046 chars,
+ * true CJK ratio 0.0161, sampled ratio 1.0, estimate inflated 1.6x. Since 4.0 / 2.5 = 1.6
+ * is that branch maximum divergence, a threshold at or under 1.6 would convert the
+ * estimator error bar into false 413s.
+ *
+ * 2.5 sits above it with room for the ~10% model-family ratio spread, and still refuses the
+ * #1412 shape (10x) four times over.
+ */
+export const ADMISSION_TOLERANCE = 2.5;
+
+/**
+ * Token cost charged for a remote image URL. The bytes are not in this request — the
+ * provider fetches them — so the URL own length is not the cost. A small flat charge
+ * acknowledges the tiles the image will occupy without pretending to know its dimensions.
+ */
+const REMOTE_IMAGE_TOKENS = 850;
+
+/** Decoded image bytes per token. Coarse tile-count proxy, not a provider formula. */
+const IMAGE_BYTES_PER_TOKEN = 750;
+
+export interface InputAdmissionResult {
+  admitted: boolean;
+  estimatedTokens: number;
+  /** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */
+  ceiling: number | null;
+}
+
+function positive(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
+}
+
+/**
+ * Charge a `data:` URL by its DECODED size rather than its character length: base64 inflates
+ * by 4/3, so charging the string would overcount by a third. A remote URL is charged flat.
+ */
+function imageTokens(imageUrl: string): number {
+  if (!imageUrl.startsWith("data:")) return REMOTE_IMAGE_TOKENS;
+  const comma = imageUrl.indexOf(",");
+  if (comma < 0) return REMOTE_IMAGE_TOKENS;
+  const payload = imageUrl.length - comma - 1;
+  if (payload <= 0) return 0;
+  const decoded = Math.floor((payload * 3) / 4);
+  return Math.max(1, Math.ceil(decoded / IMAGE_BYTES_PER_TOKEN));
+}
+
+function contentPartTokens(part: OcxContentPart, modelId: string): number {
+  if (part.type === "image") return imageTokens(part.imageUrl);
+  if (part.type === "video") return imageTokens(part.videoUrl);
+  return estimateTokens(part.text, modelId);
+}
+
+function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number {
+  if (typeof content === "string") return estimateTokens(content, modelId);
+  let total = 0;
+  for (const part of content) total += contentPartTokens(part, modelId);
+  return total;
+}
+
+/**
+ * Estimate the input tokens of a parsed request.
+ *
+ * Walks the whole `OcxMessage` union rather than user text alone. Assistant turns carry
+ * their content as `OcxAssistantContentPart[]` — text, thinking blocks, and tool calls whose
+ * JSON arguments are frequently the largest single item in an agent conversation. A walk
+ * that counted only `{type:"text"}` would undercount exactly the turns that trigger this
+ * gate.
+ */
+export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string): number {
+  const { context } = parsed;
+  let total = 0;
+
+  for (const prompt of context.systemPrompt ?? []) total += estimateTokens(prompt, modelId);
+
+  for (const message of context.messages) {
+    if (message.role === "assistant") {
+      for (const part of message.content) {
+        if (part.type === "text") total += estimateTokens(part.text, modelId);
+        else if (part.type === "thinking") total += estimateTokens(part.thinking, modelId);
+        else total += estimateTokens(part.name, modelId) + estimateTokens(JSON.stringify(part.arguments), modelId);
+      }
+      // Opaque provider blob replayed verbatim upstream, so it costs real input tokens.
+      if (message.kiroRedactedReasoning) total += estimateTokens(message.kiroRedactedReasoning, modelId);
+      continue;
+    }
+    total += contentTokens(message.content, modelId);
+  }
+
+  // Tool schemas ride every turn: name, description, and the JSON parameter schema all
+  // reach the upstream, and a large MCP catalog can dominate a short conversation.
+  for (const tool of context.tools ?? []) {
+    total += estimateTokens(tool.name, modelId)
+      + estimateTokens(tool.description, modelId)
+      + estimateTokens(JSON.stringify(tool.parameters), modelId);
+  }
+
+  return total;
+}
+
+/**
+ * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan.
+ *
+ * `provider` must be the ROUTED config (`route.provider`), which `routedProviderConfig`
+ * has already transport-guarded and merged. Re-deriving from `config.providers[name]` would
+ * reject a user-defined provider that merely shares a built-in name using limits that
+ * belong to a different service.
+ */
+export function resolveInputCeiling(
+  provider: OcxProviderConfig,
+  providerName: string,
+  modelId: string,
+  // Operator cap for the canonical native provider. Passed in rather than read from a
+  // config here so this stays pure: no filesystem, no catalog, no registry scan.
+  nativeContextCap?: NativeContextLimitsInput,
+): number | null {
+  // `modelRecordValue`, not a bare lookup: the catalog resolves these same two maps that
+  // way, so a `gpt-oss` entry covers `gpt-oss:120b`. Reading raw here made the gate fall
+  // back to the provider-wide window and refuse turns the model can plainly hold.
+  const configured = positive(modelRecordValue(provider.modelContextWindows, modelId))
+    ?? positive(provider.contextWindow);
+
+  // The canonical `openai` registry entry declares no context fields, so without this the
+  // gate would be inert on the default Codex route. All three clauses are load-bearing: a
+  // transport-mismatched custom provider named "openai" is preserved verbatim by routing
+  // and must not inherit built-in native limits, and a routed `provider/model` id is not a
+  // native slug. Static maps only — no catalog read.
+  const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID
+    && isCanonicalOpenAiForwardProvider(provider)
+    && !modelId.includes("/");
+  const nativeLimits = canonicalNativeBare && configured !== null
+    ? {
+        ...(typeof nativeContextCap === "number" ? { cap: nativeContextCap } : (nativeContextCap ?? {})),
+        modelWindows: { [modelId]: configured },
+      }
+    : nativeContextCap;
+  const native = canonicalNativeBare
+    ? positive(nativeOpenAiContextWindow(modelId, nativeLimits))
+    : null;
+  const nativeMaxInput = canonicalNativeBare ? positive(nativeOpenAiMaxInputTokens(modelId, nativeLimits)) : null;
+
+  const window = canonicalNativeBare ? native : configured;
+  // modelMaxInputTokens is an input-only cap, so it can only tighten the window.
+  const configuredMaxInput = positive(modelRecordValue(provider.modelMaxInputTokens, modelId));
+  const limits = [window, configuredMaxInput, nativeMaxInput].filter((v): v is number => v !== null);
+  return limits.length === 0 ? null : Math.min(...limits);
+}
+
+/**
+ * Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`.
+ *
+ * The caller is responsible for skipping compaction turns — see the call site in core.ts.
+ */
+export function checkInputAdmission(
+  parsed: OcxParsedRequest,
+  provider: OcxProviderConfig,
+  providerName: string,
+  modelId: string,
+  nativeContextCap?: NativeContextLimitsInput,
+): InputAdmissionResult {
+  const ceiling = resolveInputCeiling(provider, providerName, modelId, nativeContextCap);
+  if (ceiling === null) return { admitted: true, estimatedTokens: 0, ceiling: null };
+  const estimatedTokens = estimateInputTokens(parsed, modelId);
+  return { admitted: estimatedTokens <= ceiling * ADMISSION_TOLERANCE, estimatedTokens, ceiling };
+}

package/src/server/responses/pacing-overload.ts

@@ -0,0 +1,13 @@
+import { formatErrorResponse } from "../../bridge";
+import { RequestPacingQueueOverloadError } from "../../providers/request-pacing";
+
+/** Convert local request-pacing admission failures into a retryable HTTP response. */
+export function requestPacingOverloadResponse(error: unknown): Response | undefined {
+  if (!(error instanceof RequestPacingQueueOverloadError)) return undefined;
+  return formatErrorResponse(
+    429,
+    "rate_limit_error",
+    error.message,
+    { retryAfter: String(error.retryAfterSeconds) },
+  );
+}

package/src/server/responses/passthrough-error.ts

@@ -1,9 +1,29 @@
 import { formatErrorResponse } from "../../bridge";
+import { isCyberPolicyCode, isCyberPolicyMessage } from "../../lib/errors";
 import {
   resolveClientRetryAfter,
   validateClientRetryAfterHeader,
 } from "../../lib/retry-after";
 
+function isCyberPolicyBody(body: string): boolean {
+  if (isCyberPolicyMessage(body)) return true;
+  try {
+    const parsed = JSON.parse(body) as Record<string, unknown>;
+    const response = parsed.response && typeof parsed.response === "object" && !Array.isArray(parsed.response)
+      ? parsed.response as Record<string, unknown>
+      : undefined;
+    for (const candidate of [parsed.error, response?.error, response?.last_error, parsed.last_error, parsed]) {
+      if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue;
+      const record = candidate as Record<string, unknown>;
+      if (isCyberPolicyCode(typeof record.code === "string" ? record.code : undefined)) return true;
+      if (typeof record.message === "string" && isCyberPolicyMessage(record.message)) return true;
+    }
+  } catch {
+    /* non-JSON body — message detection above is the only safe fallback */
+  }
+  return false;
+}
+
 /**
  * Passthrough adapters historically relayed upstream non-2xx bodies verbatim.
  * Codex maps an *empty* body to the literal client string "Unknown error"
@@ -32,18 +52,22 @@ export function formatPassthroughUpstreamError(
   const now = options?.now ?? Date.now();
   const upstreamRetryAfter = options?.headers?.get("retry-after")?.trim() || undefined;
   const originalValid = validateClientRetryAfterHeader(upstreamRetryAfter, now);
-  const resolved = resolveClientRetryAfter({
-    status,
-    message: trimmed || `Provider error ${status}: (empty body)`,
-    upstreamRetryAfter,
-    now,
-  });
+  const cyberPolicyFailure = isCyberPolicyBody(trimmed);
+  const resolved = cyberPolicyFailure
+    ? undefined
+    : resolveClientRetryAfter({
+      status,
+      message: trimmed || `Provider error ${status}: (empty body)`,
+      upstreamRetryAfter,
+      now,
+    });
 
   if (trimmed) {
     const needsSet = resolved !== undefined && upstreamRetryAfter !== resolved;
-    const needsDelete = resolved === undefined
-      && upstreamRetryAfter !== undefined
-      && originalValid === undefined;
+    const needsDelete = (cyberPolicyFailure && upstreamRetryAfter !== undefined)
+      || (resolved === undefined
+        && upstreamRetryAfter !== undefined
+        && originalValid === undefined);
 
     if (!needsSet && !needsDelete) {
       return new Response(bodyText, {

package/src/server/responses/policy-fallback.ts

@@ -0,0 +1,178 @@
+import { comboFailureDecision } from "../../combos/failover";
+import { readBoundedResponseBody } from "../../lib/bounded-body";
+import { readJsonRequestBody } from "../request-decompress";
+import { finishRequestAttempt, type RequestLogContext } from "../request-log";
+import type { OcxConfig } from "../../types";
+import type { RouteCandidateTrace, RouteDecisionTraceV1 } from "../../routing/trace";
+import { handleResponses as handleResponsesCore } from "./core";
+import { requestPacingOverloadResponse } from "./pacing-overload";
+
+type CoreHandler = typeof handleResponsesCore;
+type CoreOptions = Parameters<CoreHandler>[3];
+
+export interface PolicyFallbackDeps {
+  runCore?: CoreHandler;
+}
+
+function candidateKey(candidate: Pick<RouteCandidateTrace, "provider" | "model">): string {
+  return `${candidate.provider}\u0000${candidate.model}`;
+}
+
+/**
+ * Rank the remaining candidates from the ORIGINAL policy trace. The initial
+ * decision stays immutable; fallback execution belongs in attempts[], not in a
+ * rewritten decision trace.
+ */
+export function rankPolicyFallbackCandidates(
+  trace: RouteDecisionTraceV1,
+  tried: ReadonlySet<string>,
+): RouteCandidateTrace[] {
+  return trace.candidates
+    .map((candidate, index) => ({ candidate, index }))
+    .filter(({ candidate }) =>
+      candidate.eligible
+      && candidate.exclusions.length === 0
+      && !tried.has(candidateKey(candidate)))
+    .sort((left, right) => {
+      const scoreDelta = (right.candidate.score?.total ?? Number.NEGATIVE_INFINITY)
+        - (left.candidate.score?.total ?? Number.NEGATIVE_INFINITY);
+      return scoreDelta || left.index - right.index;
+    })
+    .map(({ candidate }) => candidate);
+}
+
+function requestWithCandidate(
+  req: Request,
+  rawBody: Record<string, unknown>,
+  candidate: Pick<RouteCandidateTrace, "provider" | "model">,
+): Request {
+  const headers = new Headers(req.headers);
+  headers.delete("content-encoding");
+  headers.delete("content-length");
+  headers.set("content-type", "application/json");
+  return new Request(req.url, {
+    method: req.method,
+    headers,
+    body: JSON.stringify({ ...rawBody, model: `${candidate.provider}/${candidate.model}` }),
+    signal: req.signal,
+  });
+}
+
+function errorCodeFromText(text: string): string | undefined {
+  if (!text) return undefined;
+  try {
+    const payload = JSON.parse(text) as { error?: { code?: unknown; type?: unknown }; code?: unknown };
+    const candidate = payload.error?.code ?? payload.error?.type ?? payload.code;
+    return typeof candidate === "string" ? candidate : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+async function shouldHopPolicyCandidate(response: Response, signal?: AbortSignal): Promise<boolean> {
+  if (response.status < 400 || signal?.aborted) return false;
+  try {
+    const inspected = await readBoundedResponseBody(response.clone(), { signal });
+    const text = inspected.displaySafe ? inspected.text : "";
+    return comboFailureDecision(response.status, text, { code: errorCodeFromText(text) }) === "hop";
+  } catch {
+    return false;
+  }
+}
+
+function isPolicyDecision(trace: RouteDecisionTraceV1 | undefined): trace is RouteDecisionTraceV1 {
+  return trace?.routeKind === "policy" && !!trace.profile;
+}
+
+/** Finalize the failed physical attempt so the retry receives a fresh attempt row. */
+function finishFailedPolicyAttempt(logCtx: RequestLogContext, status: number): void {
+  const attempt = logCtx.activeAttempt;
+  if (attempt) {
+    const startedAt = logCtx.activeAttemptStartedAt ?? Date.now();
+    finishRequestAttempt(attempt, status, Math.max(0, Date.now() - startedAt), attempt.usage ?? logCtx.usage);
+  }
+  delete logCtx.activeAttempt;
+  delete logCtx.activeAttemptStartedAt;
+  delete logCtx.usage;
+  delete logCtx.usageFromBridge;
+  delete logCtx.upstreamError;
+  delete logCtx.terminalHttpStatus;
+  delete logCtx.terminalErrorCode;
+  delete logCtx.terminalIncompleteReason;
+}
+
+/**
+ * Run a Responses request and, only for an explicitly selected policy profile,
+ * hop to the next eligible policy candidate after a retryable pre-success
+ * failure. The initial policy trace remains the canonical selection evidence;
+ * physical retries continue to accumulate in the existing request attempts.
+ */
+export async function handleResponsesWithPolicyFallback(
+  req: Request,
+  config: OcxConfig,
+  logCtx: RequestLogContext,
+  options: CoreOptions = {},
+  deps: PolicyFallbackDeps = {},
+): Promise<Response> {
+  const runCore = deps.runCore ?? handleResponsesCore;
+  let requestBodyReadNotified = false;
+  const coreOptions: CoreOptions = options.onRequestBodyRead
+    ? {
+      ...options,
+      onRequestBodyRead: () => {
+        if (requestBodyReadNotified) return;
+        requestBodyReadNotified = true;
+        options.onRequestBodyRead?.();
+      },
+    }
+    : options;
+  let rawBody: Record<string, unknown> | null = null;
+  try {
+    const parsed = await readJsonRequestBody(req.clone());
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) rawBody = parsed as Record<string, unknown>;
+  } catch {
+    // Core owns the client-facing parse/decompression error.
+  }
+
+  let response: Response;
+  try {
+    response = await runCore(req, config, logCtx, coreOptions);
+  } catch (error) {
+    const overload = requestPacingOverloadResponse(error);
+    if (overload) return overload;
+    throw error;
+  }
+  const initialTrace = logCtx.routeDecision;
+  const initialRequestedModel = logCtx.requestedModel;
+  if (!rawBody || !isPolicyDecision(initialTrace)) return response;
+
+  const tried = new Set<string>([
+    candidateKey({ provider: initialTrace.selected.provider, model: initialTrace.selected.model }),
+  ]);
+
+  while (await shouldHopPolicyCandidate(response, req.signal)) {
+    if (req.signal.aborted) return response;
+    const next = rankPolicyFallbackCandidates(initialTrace, tried)[0];
+    if (!next) return response;
+    tried.add(candidateKey(next));
+
+    finishFailedPolicyAttempt(logCtx, response.status);
+    const retryRequest = requestWithCandidate(req, rawBody, next);
+    try {
+      try {
+        response = await runCore(retryRequest, config, logCtx, coreOptions);
+      } catch (error) {
+        const overload = requestPacingOverloadResponse(error);
+        if (overload) return overload;
+        throw error;
+      }
+    } finally {
+      logCtx.requestedModel = initialRequestedModel;
+      logCtx.routeDecision = initialTrace;
+    }
+  }
+
+  return response;
+}
+
+export const handleResponses = handleResponsesWithPolicyFallback;