@bitkyc08/opencodex 2.81.0 → 2.83.0

package/src/server/request-log-terminal-status.ts

@@ -0,0 +1,186 @@
+import {
+  CYBER_POLICY_ERROR_CODE,
+  httpStatusFromTerminalError,
+  isCyberPolicyCode,
+  isCyberPolicyMessage,
+  isRateLimitOrQuotaFailureMessage,
+  recognizedUpstreamError,
+} from "../lib/errors";
+import { redactSecretString } from "../lib/redact";
+
+/** First bounded upstream diagnostics shared by request contexts and final rows. */
+export interface UpstreamErrorDiagnostics {
+  upstreamErrorCode?: string;
+  upstreamErrorType?: string;
+  upstreamRequestId?: string;
+}
+
+interface TerminalStatusContext extends UpstreamErrorDiagnostics {
+  terminalHttpStatus?: number;
+  terminalErrorCode?: typeof CYBER_POLICY_ERROR_CODE;
+}
+
+export interface UpstreamErrorPayload {
+  type?: unknown;
+  code?: unknown;
+  message?: unknown;
+  error?: { type?: unknown; code?: unknown; message?: unknown };
+  last_error?: { type?: unknown; code?: unknown; message?: unknown };
+  response?: {
+    error?: { type?: unknown; code?: unknown; message?: unknown };
+    incomplete_details?: { code?: unknown; message?: unknown; reason?: unknown };
+  };
+}
+
+// Bare error evidence is provisional: only a genuine terminal owns account-health status.
+const provisionalStatuses = new WeakMap<TerminalStatusContext, number>();
+const genuineTerminals = new WeakSet<TerminalStatusContext>();
+
+const UPSTREAM_DIAGNOSTIC_TOKEN = /^[A-Za-z0-9_.:-]{1,128}$/;
+
+/**
+ * Upstream diagnostics are provider-controlled. The token alphabet alone also admits
+ * credential formats (an echoed `sk-...` key fits it), so a value the shared redactor
+ * would change is dropped rather than recorded, warned, or projected into log rows.
+ */
+function isSafeDiagnosticToken(value: unknown): value is string {
+  return typeof value === "string" && UPSTREAM_DIAGNOSTIC_TOKEN.test(value) && redactSecretString(value) === value;
+}
+
+/**
+ * Error classes published by the supported provider APIs (OpenAI/Responses, Anthropic, Gemini).
+ * The type is diagnostic only, so the vocabulary is closed: a gateway can echo anything in
+ * `error.type`, including a configured credential whose format no redactor recognizes, and a
+ * value outside this set is simply not recorded, which is what 2.82.0 did for every type.
+ */
+const KNOWN_UPSTREAM_ERROR_TYPES: ReadonlySet<string> = new Set([
+  "api_error", "authentication_error", "billing_error", "insufficient_quota", "invalid_request_error",
+  "not_found_error", "overloaded_error", "permission_error", "rate_limit_error", "request_too_large",
+  "requests", "server_error", "timeout_error", "tokens",
+  "INVALID_ARGUMENT", "FAILED_PRECONDITION", "PERMISSION_DENIED", "UNAUTHENTICATED", "NOT_FOUND",
+  "RESOURCE_EXHAUSTED", "INTERNAL", "UNAVAILABLE", "DEADLINE_EXCEEDED",
+]);
+
+/** Bare top-level error codes worth keeping: the classes recognizedUpstreamError maps, plus server_error. */
+const KNOWN_BARE_ERROR_CODES: ReadonlySet<string> = new Set([
+  "rate_limit_exceeded", "rate_limit_error", "server_is_overloaded", "overloaded_error", "server_error",
+]);
+
+function noteBoundedDiagnostic(
+  logCtx: TerminalStatusContext,
+  field: "upstreamErrorCode" | "upstreamErrorType" | "upstreamRequestId",
+  value: unknown,
+): void {
+  if (logCtx[field] !== undefined) return;
+  if (!isSafeDiagnosticToken(value)) return;
+  if (field === "upstreamErrorType" && !KNOWN_UPSTREAM_ERROR_TYPES.has(value)) return;
+  logCtx[field] = value;
+  const status = logCtx.terminalHttpStatus;
+  if (status === undefined || status >= 500) console.warn(`[opencodex] upstream failure${status ? ` status=${status}` : ""}${logCtx.upstreamErrorCode ? ` code=${logCtx.upstreamErrorCode}` : ""}${logCtx.upstreamRequestId ? ` request_id=${logCtx.upstreamRequestId}` : ""}`);
+}
+
+export function noteUpstreamRequestId(logCtx: TerminalStatusContext, headers: Headers): void {
+  noteBoundedDiagnostic(logCtx, "upstreamRequestId", headers.get("openai-request-id") ?? headers.get("x-request-id"));
+}
+
+export function captureUpstreamTerminalDiagnostics(logCtx: TerminalStatusContext, json: UpstreamErrorPayload): void {
+  captureTerminalHttpStatus(logCtx, json);
+  noteBoundedDiagnostic(logCtx, "upstreamErrorCode",
+    json.error?.code ?? json.last_error?.code ?? json.response?.error?.code
+      // A bare top-level code (new in 2.83.0) has no envelope contract, so only a known class is kept.
+      ?? (typeof json.code === "string" && KNOWN_BARE_ERROR_CODES.has(json.code) ? json.code : undefined));
+  noteBoundedDiagnostic(logCtx, "upstreamErrorType",
+    json.error?.type ?? json.last_error?.type ?? json.response?.error?.type);
+}
+
+export function upstreamDiagnosticLogFields(logCtx: UpstreamErrorDiagnostics): UpstreamErrorDiagnostics {
+  return {
+    ...(logCtx.upstreamErrorCode ? { upstreamErrorCode: logCtx.upstreamErrorCode } : {}),
+    ...(logCtx.upstreamErrorType ? { upstreamErrorType: logCtx.upstreamErrorType } : {}),
+    ...(logCtx.upstreamRequestId ? { upstreamRequestId: logCtx.upstreamRequestId } : {}),
+  };
+}
+
+function captureTerminalHttpStatus(
+  logCtx: TerminalStatusContext,
+  json: UpstreamErrorPayload,
+): void {
+  const type = json.type;
+  const terminal = type === "response.failed" || type === "response.completed" || type === "response.incomplete";
+  if (!terminal && type !== "error") return;
+  if (terminal && provisionalStatuses.has(logCtx)) {
+    const provisional = provisionalStatuses.get(logCtx);
+    provisionalStatuses.delete(logCtx);
+    if (logCtx.terminalHttpStatus === provisional) {
+      delete logCtx.terminalHttpStatus;
+      delete logCtx.terminalErrorCode;
+    }
+  }
+  if (logCtx.terminalHttpStatus !== undefined || (type === "error" && genuineTerminals.has(logCtx))) return;
+  if (terminal) genuineTerminals.add(logCtx);
+  const noteStatus = (status: number): void => {
+    logCtx.terminalHttpStatus = status;
+    if (type === "error") provisionalStatuses.set(logCtx, status);
+  };
+  // A completed terminal only retires provisional bare-error evidence; it records no status.
+  if (type === "response.completed") return;
+  const responseError = json.response?.error;
+  const responseDetails = json.response?.incomplete_details;
+  const candidates: Array<{ type?: unknown; code?: unknown; message?: unknown } | undefined> = [
+    json.error, json.last_error, responseError, responseDetails, json,
+  ];
+  const policy = candidates.some(candidate => (
+    candidate?.code === null || typeof candidate?.code === "string"
+  ) && isCyberPolicyCode(candidate.code as string | null | undefined))
+    || candidates.some(candidate => (
+      typeof candidate?.message === "string"
+      && candidate.message.trim().length > 0
+      && isCyberPolicyMessage(candidate.message)
+    ));
+  if (policy) {
+    logCtx.terminalErrorCode = CYBER_POLICY_ERROR_CODE;
+    noteStatus(400);
+    return;
+  }
+  // A quota terminal can carry only a structured reason, without an error message.
+  // Keep this separate from normal output limits and from the policy precedence above.
+  const quotaTag = (value: unknown): boolean => value === "usage_limit_reached"
+    || value === "rate_limit_exceeded" || value === "insufficient_quota";
+  const structuredRefusal = candidates.some(candidate => [400, 401, 403, 499].includes(
+    httpStatusFromTerminalError({
+      type: typeof candidate?.type === "string" ? candidate.type : undefined,
+      code: typeof candidate?.code === "string" ? candidate.code : undefined,
+    }),
+  ));
+  const ordinaryIncompleteReason = typeof responseDetails?.reason === "string"
+    && ["max_output_tokens", "content_filter", "steered", "upstream_stall_timeout", "adapter_eof"].includes(responseDetails.reason);
+  if (type === "response.incomplete" && !structuredRefusal && (quotaTag(responseDetails?.reason) || candidates.some(candidate =>
+    quotaTag(candidate?.code)
+    || quotaTag(candidate?.type) || candidate?.type === "rate_limit_error"
+    || (!ordinaryIncompleteReason && typeof candidate?.message === "string" && isRateLimitOrQuotaFailureMessage(candidate.message))
+  ))) {
+    // The shared quota classifier also accepts a numeric HTTP status as its message.
+    // Preserve explicit payment-required evidence rather than relabeling it as 429.
+    noteStatus(candidates.some(candidate => typeof candidate?.message === "string"
+      && Number(candidate.message.trim()) === 402) ? 402 : 429);
+    return;
+  }
+  if (type === "error") {
+    const structured = recognizedUpstreamError({
+      code: candidates.find(candidate => typeof candidate?.code === "string")?.code,
+      type: candidates.slice(0, -1).find(candidate => typeof candidate?.type === "string")?.type,
+    });
+    // Unrecognized bare errors leave the status to explicit event fields (combo preflight).
+    if (structured) noteStatus(structured.httpStatus);
+    return;
+  }
+  if (type !== "response.failed" || !responseError || typeof responseError !== "object") return;
+  const responseCode = responseError.code === null || typeof responseError.code === "string"
+    ? responseError.code
+    : undefined;
+  noteStatus(httpStatusFromTerminalError({
+    type: typeof responseError.type === "string" ? responseError.type : undefined,
+    code: responseCode,
+    message: typeof responseError.message === "string" ? responseError.message : undefined,
+  }));
+}

package/src/server/request-log.ts

@@ -4,6 +4,8 @@ import { stampApiKeyAccountLabel, usesApiKeyAccount } from "../providers/label";
 import { KEY_ACCOUNT_LOG_LABEL_RE } from "../codex/account-label";
 import { attemptAccountChanged, sealRequestAttemptIdentity } from "./request-log-account-rotation";
 export { sealRequestAttemptIdentity };
+import { captureUpstreamTerminalDiagnostics, upstreamDiagnosticLogFields, type UpstreamErrorDiagnostics, type UpstreamErrorPayload } from "./request-log-terminal-status";
+export { noteUpstreamRequestId } from "./request-log-terminal-status";
 import { readBoundedResponseBody } from "../lib/bounded-body";
 import type { ResponsesTerminalStatus } from "../bridge";
 import {
@@ -12,8 +14,6 @@ import {
   httpStatusFromTerminalError as httpStatusFromClassifiedTerminalError,
   isClientClosedMessage,
   isCyberPolicyCode,
-  isCyberPolicyMessage,
-  isRateLimitOrQuotaFailureMessage,
   isUpstreamResetReplayRefusedMessage,
   upstreamErrorMessageFromPayload,
 } from "../lib/errors";
@@ -104,7 +104,7 @@ function cacheDiagnosticHooks(): CacheDiagnosticHooks | undefined {
   return (globalThis as Record<symbol, CacheDiagnosticHooks | undefined>)[CACHE_DIAGNOSTIC_HOOK];
 }
 
-export interface RequestLogContext {
+export interface RequestLogContext extends UpstreamErrorDiagnostics {
   model: string;
   provider: string;
   /** Optional process-lifetime aggregate sink, injected by the server composition owner. */
@@ -254,6 +254,8 @@ export interface RequestLogContext {
   conversationStateScrub?: "account-change";
   transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse";
   terminalSource?: "upstream" | "synthetic";
+  /** Closed failure evidence supplied by the transport, never inferred from provider text. */
+  causeHint?: RequestFailureCause;
   /** Bounded route-decision trace (RI-01); never contains secrets. */
   routeDecision?: RouteDecisionTraceV1;
   /** Full eligible provider/model membership from a policy route; never logged. */
@@ -282,7 +284,7 @@ export function rebindCacheDiagnosticBody(
   cacheDiagnosticHooks()?.rebind(body, draft);
 }
 
-export interface RequestLogEntry {
+export interface RequestLogEntry extends UpstreamErrorDiagnostics {
   requestId: string;
   /** The logical request this row belongs to (#4546); absent on rows written without a budget. */
   logicalRequestId?: string;
@@ -840,6 +842,7 @@ export function requestLogErrorCode(
   status: number,
   upstreamError?: string,
   terminalErrorCode?: string,
+  replayRefusal = false,
 ): string | undefined {
   if (status >= 200 && status < 400) return undefined;
   // A structured terminal code is authoritative even when the provider message is localized,
@@ -868,10 +871,8 @@ export function requestLogErrorCode(
     return "permission_denied";
   }
   if (status === 429) {
-    // A refused ambiguous reset answers 429 by design (it must not invite a client
-    // retry that could duplicate inference); classify it by its message so the log
-    // distinguishes a proxy refusal from provider throttling.
-    if (upstreamError?.trim() && isUpstreamResetReplayRefusedMessage(upstreamError)) {
+    // Prefer transport provenance; retain the legacy message fallback for older callers.
+    if (replayRefusal || (upstreamError?.trim() && isUpstreamResetReplayRefusedMessage(upstreamError))) {
       return UPSTREAM_RESET_REPLAY_REFUSED_CODE;
     }
     return "rate_limit_exceeded";
@@ -1112,16 +1113,8 @@ function captureUpstreamErrorParsed(
   parsed: unknown | undefined,
 ): void {
   if (parsed !== undefined && parsed !== null) {
-    const json = parsed as {
-      type?: unknown;
-      error?: { message?: unknown };
-      last_error?: { message?: unknown };
-      response?: {
-        error?: { type?: unknown; code?: unknown; message?: unknown };
-        incomplete_details?: { reason?: unknown; message?: unknown };
-      };
-    };
-    captureTerminalHttpStatus(logCtx, json);
+    const json = parsed as UpstreamErrorPayload;
+    captureUpstreamTerminalDiagnostics(logCtx, json);
     const reason = json?.response?.incomplete_details?.reason;
     if (json.type === "response.incomplete"
       && logCtx.terminalIncompleteReason === undefined
@@ -1165,75 +1158,6 @@ function incompleteReasonLabel(reason: string): string {
   }
 }
 
-function captureTerminalHttpStatus(
-  logCtx: RequestLogContext,
-  json: {
-    type?: unknown;
-    code?: unknown;
-    message?: unknown;
-    error?: { type?: unknown; code?: unknown; message?: unknown };
-    last_error?: { type?: unknown; code?: unknown; message?: unknown };
-    response?: {
-      error?: { type?: unknown; code?: unknown; message?: unknown };
-      incomplete_details?: { code?: unknown; message?: unknown; reason?: unknown };
-    };
-  },
-): void {
-  if (logCtx.terminalHttpStatus !== undefined) return;
-  const type = json.type;
-  if (type !== "response.failed" && type !== "response.incomplete" && type !== "error") return;
-  const responseError = json.response?.error;
-  const responseDetails = json.response?.incomplete_details;
-  const candidates: Array<{ type?: unknown; code?: unknown; message?: unknown } | undefined> = [
-    json.error, json.last_error, responseError, responseDetails, json,
-  ];
-  const policy = candidates.some(candidate => (
-    candidate?.code === null || typeof candidate?.code === "string"
-  ) && isCyberPolicyCode(candidate.code as string | null | undefined))
-    || candidates.some(candidate => (
-      typeof candidate?.message === "string"
-      && candidate.message.trim().length > 0
-      && isCyberPolicyMessage(candidate.message)
-    ));
-  if (policy) {
-    logCtx.terminalErrorCode = CYBER_POLICY_ERROR_CODE;
-    logCtx.terminalHttpStatus = 400;
-    return;
-  }
-  // A quota terminal can carry only a structured reason, without an error message.
-  // Keep this separate from normal output limits and from the policy precedence above.
-  const quotaTag = (value: unknown): boolean => value === "usage_limit_reached"
-    || value === "rate_limit_exceeded" || value === "insufficient_quota";
-  const structuredRefusal = candidates.some(candidate => [400, 401, 403, 499].includes(
-    httpStatusFromTerminalError({
-      type: typeof candidate?.type === "string" ? candidate.type : undefined,
-      code: typeof candidate?.code === "string" ? candidate.code : undefined,
-    }),
-  ));
-  const ordinaryIncompleteReason = typeof responseDetails?.reason === "string"
-    && ["max_output_tokens", "content_filter", "steered", "upstream_stall_timeout", "adapter_eof"].includes(responseDetails.reason);
-  if (type === "response.incomplete" && !structuredRefusal && (quotaTag(responseDetails?.reason) || candidates.some(candidate =>
-    quotaTag(candidate?.code)
-    || quotaTag(candidate?.type) || candidate?.type === "rate_limit_error"
-    || (!ordinaryIncompleteReason && typeof candidate?.message === "string" && isRateLimitOrQuotaFailureMessage(candidate.message))
-  ))) {
-    // The shared quota classifier also accepts a numeric HTTP status as its message.
-    // Preserve explicit payment-required evidence rather than relabeling it as 429.
-    logCtx.terminalHttpStatus = candidates.some(candidate => typeof candidate?.message === "string"
-      && Number(candidate.message.trim()) === 402) ? 402 : 429;
-    return;
-  }
-  if (type !== "response.failed" || !responseError || typeof responseError !== "object") return;
-  const responseCode = responseError.code === null || typeof responseError.code === "string"
-    ? responseError.code
-    : undefined;
-  logCtx.terminalHttpStatus = httpStatusFromTerminalError({
-    type: typeof responseError.type === "string" ? responseError.type : undefined,
-    code: responseCode,
-    message: typeof responseError.message === "string" ? responseError.message : undefined,
-  });
-}
-
 /** Map a terminal Responses error object to the HTTP status we record in /api/logs. */
 export function httpStatusFromTerminalError(error: {
   type?: string;
@@ -1440,12 +1364,12 @@ export function addFinalRequestLog(
   const effectiveStatus = status >= 500 && logCtx.upstreamError && isClientClosedMessage(logCtx.upstreamError)
     ? 499
     : status;
-  // A locally assigned code wins: it names a refusal this proxy made itself, which no
-  // status-plus-upstream-message classification can reconstruct.
+  // Locally assigned codes name proxy refusals that status and upstream text cannot reconstruct.
   const errorCode = logCtx.errorCode ?? requestLogErrorCode(
     effectiveStatus,
     logCtx.upstreamError,
     logCtx.terminalErrorCode,
+    logCtx.causeHint === "transport-ambiguous" && logCtx.terminalSource === "synthetic",
   );
   // A response.failed whose classified status is 499 is still a client cancel, not an upstream
   // terminal failure — keep /api/logs closeReason aligned with that.
@@ -1475,6 +1399,7 @@ export function addFinalRequestLog(
     ...(logCtx.transportPhase ? { transportPhase: logCtx.transportPhase } : {}),
     ...(logCtx.terminalSource ? { terminalSource: logCtx.terminalSource } : {}),
     outputObserved: logCtx.firstOutputMs !== undefined,
+    ...(logCtx.causeHint ? { causeHint: logCtx.causeHint } : {}),
     locallyAnswered: logCtx.localTerminalReason !== undefined,
     ...(logCtx.activeAttempt ? { attempt: logCtx.activeAttempt } : {}),
   });
@@ -1593,6 +1518,7 @@ export function addFinalRequestLog(
     ...(meta?.terminalStatus ? { terminalStatus: meta.terminalStatus } : {}),
     ...(closeReason ? { closeReason } : {}),
     ...(logCtx.upstreamError ? { upstreamError: logCtx.upstreamError } : {}),
+    ...upstreamDiagnosticLogFields(logCtx),
     usageStatus,
     ...(loggedUsage ? { usage: loggedUsage } : {}),
     ...(totalTokens !== undefined ? { totalTokens } : {}),

package/src/server/responses/adapter-continuation.ts

@@ -83,6 +83,7 @@ export function createAdapterContinuations(
     | "oauthDispatch"
     | "invalidateSameTargetRequest"
     | "resolveSelectionAdapter"
+    | "resolveCopilotSelection"
     | "anthropicInstance"
     | "currentAnthropicRouteDecision"
     | "anthropicRouteDecision"
@@ -156,7 +157,8 @@ export function createAdapterContinuations(
   /**
    * One bounded internal re-ask for Anthropic end_turn-without-tool-call turns. Replays the
    * continuation on a 429 with the same-key retry budget (hoisted per request), then falls
-   * back to key/account failover; a failure becomes an in-stream adapter error so the client
+   * back to key/account failover, rebuilding Copilot model, wire, and credential negotiation.
+   * A failure becomes an in-stream adapter error so the client
    * never sees a second hidden HTTP response or an unbounded retry loop.
    */
   const fetchTerminalGuardContinuation = async function* (
@@ -377,6 +379,12 @@ export function createAdapterContinuations(
         if (rotated) {
           try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
           route.provider = rotated;
+          try { await transportState.resolveCopilotSelection(nextParsed); }
+          catch (error) {
+            yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}`,
+              ...(options.abortSignal?.aborted || upstream.signal.aborted ? { status: 499 } : {}) };
+            return;
+          }
           invalidateSameTargetRequest();
           transportState.activeAdapter = resolveSelectionAdapter(
             resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
@@ -401,7 +409,7 @@ export function createAdapterContinuations(
         }
       }
      if (
-       (response.status === 429 || response.status === 403)
+       (response.status === 429 || response.status === 403 || response.status === 401)
        && anthropicInstance
        && transportState.anthropicPoolAccountId
         && !isNonReplayableResponse(response)
@@ -428,7 +436,7 @@ export function createAdapterContinuations(
             );
             sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
             recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
-            nextContinuationRecoveryKind = "anthropic-oauth-429";
+            nextContinuationRecoveryKind = response.status === 401 ? "oauth-401" : "anthropic-oauth-429";
             continue;
           } catch {
             // fall through to emit continuation error below