codex-copilot-dx 0.7.7 → 0.7.9

package/src/response-failures.mjs

@@ -0,0 +1,100 @@
+import { terminalCell } from "./cli-table.mjs";
+
+const DEFAULT_RECENT_FAILURES = 10;
+
+function safeIdentifier(value, fallback = "unknown") {
+  const text = terminalCell(value, { fallback });
+  return text.slice(0, 200);
+}
+
+function safeFailureMessage(value) {
+  let text = terminalCell(value, { fallback: "Response failed without an error message" });
+  text = text
+    .replace(/(encrypted(?: function output)? content\s+)[A-Za-z0-9+/_=-]{24,}/gi, "$1[redacted]")
+    .replace(/\b[A-Za-z0-9+/_=-]{96,}\b/g, "[redacted]");
+  return text.slice(0, 500);
+}
+
+function upstreamRequestId(headers) {
+  if (!headers?.get) return null;
+  for (const name of ["x-request-id", "x-github-request-id", "x-ms-request-id", "request-id", "x-correlation-id"]) {
+    const value = headers.get(name);
+    if (value) return safeIdentifier(value, "");
+  }
+  return null;
+}
+
+export function responseFailureDetails(event, eventType = event?.type, {
+  model,
+  headers,
+  retried = false,
+  retryPolicy = null,
+  retrySkipped = null,
+} = {}) {
+  const response = event?.response && typeof event.response === "object" ? event.response : event;
+  const error = response?.error && typeof response.error === "object"
+    ? response.error
+    : event?.error && typeof event.error === "object"
+      ? event.error
+      : event;
+  return {
+    at: new Date().toISOString(),
+    event_type: safeIdentifier(eventType || "response.failed"),
+    model: safeIdentifier(response?.model || model),
+    code: safeIdentifier(error?.code || "unknown_error"),
+    message: safeFailureMessage(error?.message),
+    response_id: response?.id ? safeIdentifier(response.id, "") : null,
+    upstream_request_id: upstreamRequestId(headers),
+    retried: Boolean(retried),
+    retry_policy: retryPolicy ? safeIdentifier(retryPolicy, "") : null,
+    retry_skipped: retrySkipped ? safeIdentifier(retrySkipped, "") : null,
+  };
+}
+
+export function responseFailureText(event, eventType = event?.type) {
+  const response = event?.response && typeof event.response === "object" ? event.response : event;
+  const error = response?.error && typeof response.error === "object"
+    ? response.error
+    : event?.error && typeof event.error === "object"
+      ? event.error
+      : event;
+  return JSON.stringify({ error: { code: error?.code, message: error?.message } });
+}
+
+export function formatResponseFailureLog(details) {
+  const fields = [
+    `Responses ${details.event_type}`,
+    `model=${details.model}`,
+    `code=${details.code}`,
+    details.response_id ? `response_id=${details.response_id}` : null,
+    details.upstream_request_id ? `upstream_request_id=${details.upstream_request_id}` : null,
+    details.retried ? `retry=${details.retry_policy || "yes"}` : null,
+    details.retry_skipped ? `retry_skipped=${details.retry_skipped}` : null,
+    `message=${details.message}`,
+  ].filter(Boolean);
+  return fields.join(" ");
+}
+
+export function createResponseFailureDiagnostics({
+  maxEntries = DEFAULT_RECENT_FAILURES,
+  now = () => new Date().toISOString(),
+} = {}) {
+  const recent = [];
+  let total = 0;
+  let retried = 0;
+  return {
+    record(details) {
+      total += 1;
+      if (details.retried) retried += 1;
+      recent.push({ ...details, at: now() });
+      if (recent.length > maxEntries) recent.splice(0, recent.length - maxEntries);
+    },
+    snapshot() {
+      return {
+        total,
+        retried,
+        recent: recent.map((entry) => ({ ...entry })),
+      };
+    },
+  };
+}

package/src/responses-handler.mjs

@@ -109,6 +109,7 @@ export function createResponsesHandler(options) {
     requestBodyTimeoutMs,
     responsesPayloadOptions,
     responsesFn,
+    responseFailures,
     streamHandshakeTimeoutMs,
     streamIdleTimeoutMs,
     upstreamTimeoutMs,
@@ -255,6 +256,7 @@ export function createResponsesHandler(options) {
           abort,
           onUpstreamStart: startUpstreamTimeout,
           releaseRequest: releaseUpstreamPayload,
+          responseFailures,
           streamIdleTimeoutMs,
         });
         if (result?.successful) imagePressure?.markSuccess?.(responseHistoryPressureRootId(prepared));

package/src/responses-proxy.mjs

@@ -8,8 +8,18 @@ import {
   sendUpstreamError,
   writeOrDrain,
 } from "./http-transport.mjs";
-import { openCopilotResponse } from "./copilot-responses-compat.mjs";
-import { rememberResponseHistory } from "./responses-request.mjs";
+import { openCopilotResponse, selectCopilotResponseRetry } from "./copilot-responses-compat.mjs";
+import {
+  dropMaterializedResponseHistory,
+  hasEncryptedResponseState,
+  rememberResponseHistory,
+  restoreMaterializedResponseHistoryForRetry,
+} from "./responses-request.mjs";
+import {
+  formatResponseFailureLog,
+  responseFailureDetails,
+  responseFailureText,
+} from "./response-failures.mjs";
 import {
   compactionInputWithoutTrigger,
   parseResponsesCompactionResult,
@@ -30,9 +40,11 @@ import {
 } from "./stream-performance.mjs";
 import { safeUpstreamResponseHeaders } from "./upstream-headers.mjs";
 import { recordResponsesUsage } from "./usage.mjs";
+import { status } from "./status.mjs";
 
 const MAX_SSE_BUFFER_BYTES = loadRuntimeConfig().maxSseBufferBytes;
 const SSE_WRITE_BATCH_BYTES = 64 * 1024;
+const STREAM_RETRY_PRELUDE_MAX_BYTES = 256 * 1024;
 const RESPONSES_TERMINAL_EVENT_TYPES = new Set([
   "response.completed",
   "response.incomplete",
@@ -124,7 +136,9 @@ function inspectResponseSseEvent(state, eventName, data) {
     );
   }
   state.sawTerminal = true;
-  if (eventType === "response.failed" || eventType === "error") markStreamFailure();
+  if (eventType === "response.failed" || eventType === "error") {
+    state.failure = { event, eventType };
+  }
   if (eventType === "response.completed") {
     const response = event.response;
     state.completed = { response, event };
@@ -292,68 +306,182 @@ function storeCompletedResponse(reqContext, completed) {
   });
 }
 
-export async function proxyCopilotResponses(reqContext, req, res, upstream = copilotResponses, options = {}) {
-  let opened;
+function streamHeaders(resp) {
+  return {
+    ...safeUpstreamResponseHeaders(resp.headers, { defaultContentType: "text/event-stream" }),
+    "Cache-Control": "no-cache",
+    Connection: "keep-alive",
+  };
+}
+
+function selectStreamFailureRetry(reqContext, failure, usedRetryPolicies) {
+  if (!failure) return null;
   try {
-    opened = await openCopilotResponse(reqContext, upstream, options);
-  } finally {
-    options.releaseRequest?.(opened?.reqContext);
-  }
-  const { resp, errorText } = opened;
-  reqContext = opened.reqContext;
-  if (errorText !== undefined) {
-    sendUpstreamError(res, resp, errorText);
-    return { successful: false, compacted: false, upstreamStatus: resp.status };
+    const restored = restoreMaterializedResponseHistoryForRetry(reqContext);
+    return {
+      retry: selectCopilotResponseRetry(
+        restored,
+        400,
+        responseFailureText(failure.event, failure.eventType),
+        usedRetryPolicies,
+      ),
+      retrySkipped: null,
+    };
+  } catch {
+    return { retry: null, retrySkipped: "history_unavailable" };
   }
+}
 
-  if (reqContext.body.stream) {
+function recordResponseFailure(options, reqContext, resp, failure, {
+  retry,
+  retrySkipped,
+} = {}) {
+  if (!failure) return;
+  const details = responseFailureDetails(failure.event, failure.eventType, {
+    model: reqContext.body?.model,
+    headers: resp.headers,
+    retried: Boolean(retry),
+    retryPolicy: retry?.policyId,
+    retrySkipped,
+  });
+  options.responseFailures?.record?.(details);
+  console.warn(status("warn", formatResponseFailureLog(details)));
+}
+
+async function proxyStreamingResponses(opened, res, upstream, options) {
+  while (true) {
+    let {
+      resp,
+      errorText,
+      reqContext,
+      streamRetryEligible = false,
+      usedRetryPolicies = new Set(),
+    } = opened;
+    if (errorText !== undefined) {
+      sendUpstreamError(res, resp, errorText);
+      return { successful: false, compacted: false, upstreamStatus: resp.status };
+    }
     await requireUpstreamEventStream(resp);
     options.abort?.setTimeout(options.streamIdleTimeoutMs, "stream_idle_timeout");
-    res.writeHead(resp.status, {
-      ...safeUpstreamResponseHeaders(resp.headers, { defaultContentType: "text/event-stream" }),
-      "Cache-Control": "no-cache",
-      Connection: "keep-alive",
-    });
+    const headers = streamHeaders(resp);
+    let headersSent = false;
+    const writeHeaders = () => {
+      if (headersSent || res.headersSent) return;
+      headersSent = true;
+      res.writeHead(resp.status, headers);
+    };
     const reader = resp.body.getReader();
     const streamState = {
       completed: null,
+      failure: null,
       sawTerminal: false,
       sawOutput: false,
       toolArgumentGuard: new ToolArgumentDeltaGuard(),
       messageIds: new Map(),
     };
     const transformer = createResponseSseTransformer(streamState);
+    let holdPrelude = streamRetryEligible;
+    let retryDisabledReason = holdPrelude ? null : "request_not_eligible";
+    const prelude = [];
+    let preludeBytes = 0;
+    const flushPrelude = async () => {
+      if (!preludeBytes) return true;
+      writeHeaders();
+      const body = prelude.length === 1 ? prelude[0] : Buffer.concat(prelude, preludeBytes);
+      prelude.length = 0;
+      preludeBytes = 0;
+      return writeOrDrain(res, body);
+    };
+    const forwardFrame = async (frame) => {
+      if (!frame) return true;
+      if (!holdPrelude) {
+        writeHeaders();
+        return writeOrDrain(res, frame);
+      }
+      prelude.push(frame);
+      preludeBytes += frame.byteLength;
+      if (!streamState.sawOutput && preludeBytes <= STREAM_RETRY_PRELUDE_MAX_BYTES) return true;
+      retryDisabledReason = streamState.sawOutput ? "output_started" : "prelude_limit";
+      holdPrelude = false;
+      return flushPrelude();
+    };
+
     try {
       while (true) {
         const { done, value } = await reader.read();
         if (done) {
           if (!streamState.sawTerminal) throw incompleteUpstreamStream("a terminal Responses event");
-          storeCompletedResponse(reqContext, streamState.completed);
-          res.end();
-          return { successful: Boolean(streamState.completed), compacted: false };
+          break;
         }
         options.abort?.setTimeout(options.streamIdleTimeoutMs, "stream_idle_timeout");
-        // Batch within this read, not across reads, and await drain before processing more.
         for (let offset = 0; offset < value.byteLength; offset += SSE_WRITE_BATCH_BYTES) {
           const frame = transformer.push(value.subarray(offset, offset + SSE_WRITE_BATCH_BYTES));
-          if (frame && !await writeOrDrain(res, frame)) return;
+          if (!await forwardFrame(frame)) return { successful: false, compacted: false };
           if (streamState.sawTerminal) break;
         }
-        if (streamState.sawTerminal) {
-          storeCompletedResponse(reqContext, streamState.completed);
-          res.end();
-          return { successful: Boolean(streamState.completed), compacted: false };
-        }
+        if (streamState.sawTerminal) break;
+      }
+
+      let retry = null;
+      if (streamState.failure && holdPrelude && !streamState.sawOutput) {
+        const selected = selectStreamFailureRetry(reqContext, streamState.failure, usedRetryPolicies);
+        retry = selected?.retry || null;
+        if (!retry) retryDisabledReason = selected?.retrySkipped || "not_applicable";
+      }
+      if (streamState.failure) {
+        recordResponseFailure(options, reqContext, resp, streamState.failure, {
+          retry,
+          retrySkipped: retry ? null : retryDisabledReason,
+        });
+      }
+      if (retry) {
+        console.warn(status("warn", retry.warning));
+        opened = await openCopilotResponse(retry.reqContext, upstream, {
+          ...options,
+          usedRetryPolicies: retry.usedPolicies,
+        });
+        opened.streamRetryEligible = hasEncryptedResponseState(opened.reqContext.body?.input);
+        dropMaterializedResponseHistory(opened.reqContext);
+        continue;
       }
-    } catch (e) {
-      logRequestFailure("Responses", e, options.abort);
-      await endStreamWithError(res, e, options.abort);
+      if (streamState.failure) markStreamFailure();
+      if (holdPrelude && !await flushPrelude()) return { successful: false, compacted: false };
+      storeCompletedResponse(reqContext, streamState.completed);
+      if (!res.writableEnded) res.end();
+      return { successful: Boolean(streamState.completed), compacted: false };
+    } catch (error) {
+      logRequestFailure("Responses", error, options.abort);
+      writeHeaders();
+      await endStreamWithError(res, error, options.abort);
       return { successful: false, compacted: false };
     } finally {
       await reader.cancel().catch(() => {});
       reader.releaseLock();
     }
-  } else {
+  }
+}
+
+export async function proxyCopilotResponses(reqContext, req, res, upstream = copilotResponses, options = {}) {
+  let opened;
+  try {
+    opened = await openCopilotResponse(reqContext, upstream, options);
+    opened.streamRetryEligible = opened.reqContext.body.stream === true
+      && hasEncryptedResponseState(opened.reqContext.body?.input);
+  } finally {
+    options.releaseRequest?.(opened?.reqContext);
+  }
+  reqContext = opened.reqContext;
+  if (reqContext.body.stream) {
+    return proxyStreamingResponses(opened, res, upstream, options);
+  }
+
+  while (true) {
+    const { resp, errorText, usedRetryPolicies = new Set() } = opened;
+    reqContext = opened.reqContext;
+    if (errorText !== undefined) {
+      sendUpstreamError(res, resp, errorText);
+      return { successful: false, compacted: false, upstreamStatus: resp.status };
+    }
     const data = resp.ok
       ? await readBoundedResponseText(resp, {
         maxBytes: MAX_UPSTREAM_RESPONSES_SUCCESS_BODY_BYTES,
@@ -385,6 +513,24 @@ export async function proxyCopilotResponses(reqContext, req, res, upstream = cop
     }
     if (resp.ok) {
       const response = parseSuccessfulResponsesResult(data);
+      if (response.status === "failed") {
+        const failure = { event: { type: "response.failed", response }, eventType: "response.failed" };
+        const selected = selectStreamFailureRetry(reqContext, failure, usedRetryPolicies);
+        const retry = selected?.retry || null;
+        recordResponseFailure(options, reqContext, resp, failure, {
+          retry,
+          retrySkipped: retry ? null : selected?.retrySkipped || "not_applicable",
+        });
+        if (retry) {
+          console.warn(status("warn", retry.warning));
+          opened = await openCopilotResponse(retry.reqContext, upstream, {
+            ...options,
+            usedRetryPolicies: retry.usedPolicies,
+          });
+          dropMaterializedResponseHistory(opened.reqContext);
+          continue;
+        }
+      }
       rememberResponseHistory(reqContext, response);
       recordResponsesUsage({
         surface: reqContext.surface,
@@ -397,7 +543,7 @@ export async function proxyCopilotResponses(reqContext, req, res, upstream = cop
         contentType: "application/json",
       }));
       res.end(data);
-      return { successful: true, compacted: false };
+      return { successful: response.status !== "failed" && response.status !== "incomplete", compacted: false };
     }
     res.writeHead(resp.status, safeUpstreamResponseHeaders(resp.headers, {
       contentType: "application/json",

package/src/responses-request.mjs

@@ -84,6 +84,7 @@ function isEncryptedContentPart(value) {
 const OMIT_ENCRYPTED_CONTENT = Symbol("omit-encrypted-content");
 const ENCRYPTED_HISTORY_REBASE = Symbol("encrypted-history-rebase");
 const RELEASE_HISTORY_INPUT = Symbol("release-history-input");
+const RELEASED_MATERIALIZED_HISTORY = Symbol("released-materialized-history");
 const HISTORY_PRESSURE_ROOT = Symbol("history-pressure-root");
 const ENCRYPTED_TOOL_OUTPUT_MARKER = "[CCDX: encrypted tool output omitted because upstream could not decrypt it.]";
 
@@ -187,6 +188,10 @@ function responseValueHasOpaqueState(value) {
   return Object.values(value).some(responseValueHasOpaqueState);
 }
 
+export function hasEncryptedResponseState(value) {
+  return responseValueHasOpaqueState(value);
+}
+
 export function sanitizeEncryptedReasoningRequest(reqContext, { historicalOnly = false } = {}) {
   const state = { changed: false };
   let body = cloneJson(reqContext.body);
@@ -379,10 +384,28 @@ export function dropMaterializedResponseHistory(reqContext) {
   reqContext.body = { ...reqContext.body, input: releaseInput };
   reqContext.inputItems = releaseInput;
   delete reqContext[RELEASE_HISTORY_INPUT];
+  reqContext[RELEASED_MATERIALIZED_HISTORY] = true;
   reqContext.currentInputStart = 0;
   return true;
 }
 
+export function restoreMaterializedResponseHistoryForRetry(reqContext, { assertActive } = {}) {
+  if (!reqContext?.[RELEASED_MATERIALIZED_HISTORY] || !reqContext.historyParentId) return reqContext;
+  const body = cloneJson(reqContext.body);
+  body.previous_response_id = reqContext.historyParentId;
+  body.input = cloneJson(reqContext.historyInputItems || reqContext.body.input);
+  let restored = prepareResponsesRequest(body, {
+    assertActive,
+    copilotBoundary: false,
+    mutate: true,
+  });
+  restored.surface = reqContext.surface;
+  if (reqContext.routePlan) {
+    restored = applyResponseHistoryRoutePlan(restored, reqContext.routePlan);
+  }
+  return restored;
+}
+
 export function rememberResponseHistory(reqContext, responseJson) {
   if (!responseJson?.id || !Array.isArray(reqContext?.historyInputItems || reqContext?.inputItems)) return;
   const compacted = isSuccessfulCompactionResponse(responseJson);