codex-copilot-dx 0.4.43 → 0.4.44

package/src/responses-proxy.mjs

@@ -6,33 +6,82 @@ import {
   writeOrDrain,
 } from "./http-transport.mjs";
 import { openCopilotResponse, rememberResponseHistory } from "./responses-request.mjs";
+import {
+  compactionInputWithoutTrigger,
+  parseResponsesCompactionResult,
+} from "./responses-compaction.mjs";
 import { loadRuntimeConfig } from "./runtime-config.mjs";
 import { endStreamWithError } from "./stream-errors.mjs";
+import {
+  incompleteUpstreamStream,
+  invalidUpstreamStream,
+  requireUpstreamEventStream,
+  ToolArgumentDeltaGuard,
+} from "./stream-contract.mjs";
+import {
+  isResponsesOutputEvent,
+  markFirstOutput,
+  markOutputTokens,
+  markStreamFailure,
+} from "./stream-performance.mjs";
+import { safeUpstreamResponseHeaders } from "./upstream-headers.mjs";
 import { recordResponsesUsage } from "./usage.mjs";
 
 const MAX_SSE_BUFFER_BYTES = loadRuntimeConfig().maxSseBufferBytes;
+const RESPONSES_TERMINAL_EVENT_TYPES = new Set([
+  "response.completed",
+  "response.incomplete",
+  "response.failed",
+  "error",
+]);
 
-function storeCompletedResponseFromSse(reqContext, data) {
-  if (!data || data === "[DONE]") return;
+function inspectResponseSseEvent(state, eventName, data) {
+  if (!data || state.sawTerminal) return;
+  if (data === "[DONE]") {
+    throw invalidUpstreamStream(
+      "Upstream sent [DONE] before a terminal Responses event",
+      "upstream_stream_terminal_missing",
+    );
+  }
+  let event;
   try {
-    const event = JSON.parse(data);
-    const response = event.response;
-    if (response?.id && response.status === "completed") {
-      rememberResponseHistory(reqContext, response);
-      recordResponsesUsage({
-        surface: reqContext.surface,
-        mode: "stream",
-        model: reqContext.body?.model,
-        response,
-        event,
-      });
-    }
+    event = JSON.parse(data);
   } catch {
-    // Ignore non-JSON stream fragments.
+    throw invalidUpstreamStream(
+      "Upstream Responses SSE event contained invalid JSON",
+      "upstream_stream_json_invalid",
+    );
+  }
+  const eventType = event.type || eventName;
+  const outputTokens = event.response?.usage?.output_tokens ?? event.usage?.output_tokens;
+  if (outputTokens !== undefined) markOutputTokens(outputTokens);
+  if (!state.sawOutput && isResponsesOutputEvent(event, eventType)) {
+    state.sawOutput = true;
+    markFirstOutput();
+  }
+  if (["response.function_call_arguments.delta", "response.custom_tool_call_input.delta"].includes(eventType)) {
+    state.toolArgumentGuard.observe(event.output_index ?? event.item_id ?? event.call_id, event.delta);
   }
+  if (!RESPONSES_TERMINAL_EVENT_TYPES.has(eventType)) return;
+  const expectedStatus = {
+    "response.completed": "completed",
+    "response.incomplete": "incomplete",
+    "response.failed": "failed",
+  }[eventType];
+  if (expectedStatus && (!event.response?.id || event.response.status !== expectedStatus)) {
+    throw invalidUpstreamStream(
+      `Upstream ${eventType} event contained an invalid response resource`,
+      "upstream_stream_terminal_invalid",
+    );
+  }
+  state.sawTerminal = true;
+  if (eventType === "response.failed" || eventType === "error") markStreamFailure();
+  if (eventType !== "response.completed") return;
+  const response = event.response;
+  state.completed = { response, event };
 }
 
-function readSseEvents(buffer, onData) {
+function readSseEvents(buffer, onEvent) {
   while (true) {
     const match = buffer.match(/\r?\n\r?\n/);
     if (!match) return buffer;
@@ -41,15 +90,31 @@ function readSseEvents(buffer, onData) {
       throw httpError(`Upstream SSE buffer exceeds ${MAX_SSE_BUFFER_BYTES} bytes`, 502);
     }
     buffer = buffer.slice(match.index + match[0].length);
-    const data = chunk
-      .split(/\r?\n/)
+    const lines = chunk.split(/\r?\n/);
+    const eventName = lines
+      .find((line) => line.startsWith("event:"))
+      ?.slice(6)
+      .trim();
+    const data = lines
       .filter((line) => line.startsWith("data:"))
       .map((line) => line.slice(5).trimStart())
       .join("\n");
-    if (data) onData(data);
+    if (data) onEvent(eventName, data);
   }
 }
 
+function storeCompletedResponse(reqContext, completed) {
+  if (!completed) return;
+  rememberResponseHistory(reqContext, completed.response);
+  recordResponsesUsage({
+    surface: reqContext.surface,
+    mode: "stream",
+    model: reqContext.body?.model,
+    response: completed.response,
+    event: completed.event,
+  });
+}
+
 export async function proxyCopilotResponses(reqContext, req, res, upstream = copilotResponses, options = {}) {
   let opened;
   try {
@@ -65,15 +130,22 @@ export async function proxyCopilotResponses(reqContext, req, res, upstream = cop
   }
 
   if (reqContext.body.stream) {
+    await requireUpstreamEventStream(resp);
     options.abort?.setTimeout(options.streamIdleTimeoutMs, "stream_idle_timeout");
     res.writeHead(resp.status, {
-      "Content-Type": resp.headers.get("content-type") || "text/event-stream",
+      ...safeUpstreamResponseHeaders(resp.headers, { defaultContentType: "text/event-stream" }),
       "Cache-Control": "no-cache",
       Connection: "keep-alive",
     });
     const reader = resp.body.getReader();
     const decoder = new TextDecoder();
     let buffer = "";
+    const streamState = {
+      completed: null,
+      sawTerminal: false,
+      sawOutput: false,
+      toolArgumentGuard: new ToolArgumentDeltaGuard(),
+    };
     try {
       while (true) {
         const { done, value } = await reader.read();
@@ -82,17 +154,24 @@ export async function proxyCopilotResponses(reqContext, req, res, upstream = cop
           if (Buffer.byteLength(buffer) > MAX_SSE_BUFFER_BYTES) {
             throw httpError(`Upstream SSE buffer exceeds ${MAX_SSE_BUFFER_BYTES} bytes`, 502);
           }
-          readSseEvents(buffer, (data) => storeCompletedResponseFromSse(reqContext, data));
+          readSseEvents(buffer, (eventName, data) => inspectResponseSseEvent(streamState, eventName, data));
+          if (!streamState.sawTerminal) throw incompleteUpstreamStream("a terminal Responses event");
+          storeCompletedResponse(reqContext, streamState.completed);
           res.end();
           return;
         }
         options.abort?.setTimeout(options.streamIdleTimeoutMs, "stream_idle_timeout");
         buffer += decoder.decode(value, { stream: true });
-        buffer = readSseEvents(buffer, (data) => storeCompletedResponseFromSse(reqContext, data));
+        buffer = readSseEvents(buffer, (eventName, data) => inspectResponseSseEvent(streamState, eventName, data));
         if (Buffer.byteLength(buffer) > MAX_SSE_BUFFER_BYTES) {
           throw httpError(`Upstream SSE buffer exceeds ${MAX_SSE_BUFFER_BYTES} bytes`, 502);
         }
         if (!await writeOrDrain(res, value)) return;
+        if (streamState.sawTerminal) {
+          storeCompletedResponse(reqContext, streamState.completed);
+          res.end();
+          return;
+        }
       }
     } catch (e) {
       logRequestFailure("Responses", e, options.abort);
@@ -103,7 +182,28 @@ export async function proxyCopilotResponses(reqContext, req, res, upstream = cop
     }
   } else {
     const data = await resp.text();
-    res.writeHead(resp.status, { "Content-Type": "application/json" });
+    if (resp.ok && reqContext.surface === "responses_compact") {
+      const response = parseResponsesCompactionResult(
+        compactionInputWithoutTrigger(reqContext.body.input),
+        data,
+      );
+      rememberResponseHistory(reqContext, response);
+      recordResponsesUsage({
+        surface: reqContext.surface,
+        mode: "json",
+        model: reqContext.body?.model,
+        response,
+        event: response,
+      });
+      res.writeHead(resp.status, safeUpstreamResponseHeaders(resp.headers, {
+        contentType: "application/json",
+      }));
+      res.end(JSON.stringify(response));
+      return;
+    }
+    res.writeHead(resp.status, safeUpstreamResponseHeaders(resp.headers, {
+      contentType: "application/json",
+    }));
     res.end(data);
     if (resp.ok) {
       try {

package/src/stream-contract.mjs

@@ -0,0 +1,102 @@
+const DEFAULT_BLANK_TOOL_ARGUMENT_LIMIT = 20;
+
+function streamProtocolError(message, code) {
+  const error = new Error(message);
+  error.statusCode = 502;
+  error.code = code;
+  error.jsonBody = {
+    error: {
+      message,
+      type: "upstream_protocol_error",
+      code,
+    },
+  };
+  return error;
+}
+
+export function invalidUpstreamStream(message, code = "upstream_stream_invalid") {
+  return streamProtocolError(message, code);
+}
+
+export function withChatStreamUsage(request) {
+  const streamOptions = request?.stream_options && typeof request.stream_options === "object"
+    ? request.stream_options
+    : {};
+  if (request?.stream === true && streamOptions.include_usage === true) return request;
+  return {
+    ...request,
+    stream: true,
+    stream_options: { ...streamOptions, include_usage: true },
+  };
+}
+
+export async function requireUpstreamEventStream(response) {
+  let error = null;
+  if (!response?.body) {
+    error = streamProtocolError(
+      "Upstream returned a successful streaming response without a body",
+      "upstream_stream_body_missing",
+    );
+  } else {
+    const contentType = response.headers?.get?.("content-type") || "";
+    const mediaType = contentType.split(";", 1)[0].trim().toLowerCase();
+    if (mediaType !== "text/event-stream") {
+      error = streamProtocolError(
+        "Upstream returned a successful streaming response with a non-SSE Content-Type",
+        "upstream_stream_content_type_invalid",
+      );
+    }
+  }
+
+  if (!error) return;
+  await response?.body?.cancel?.().catch(() => {});
+  throw error;
+}
+
+export function incompleteUpstreamStream(expectedTerminator) {
+  return streamProtocolError(
+    `Upstream stream ended before ${expectedTerminator}`,
+    "upstream_stream_incomplete",
+  );
+}
+
+export function upstreamChatStreamError(event) {
+  const payload = event?.error;
+  if (!payload || typeof payload !== "object") return null;
+  const type = typeof payload.type === "string" && payload.type ? `${payload.type}: ` : "";
+  const message = typeof payload.message === "string" && payload.message
+    ? payload.message.slice(0, 500)
+    : "Upstream stream error";
+  return streamProtocolError(
+    `Upstream Chat Completions SSE error: ${type}${message}`,
+    "upstream_chat_stream_error",
+  );
+}
+
+export class ToolArgumentDeltaGuard {
+  constructor(limit = DEFAULT_BLANK_TOOL_ARGUMENT_LIMIT) {
+    this.limit = limit;
+    this.whitespaceRuns = new Map();
+  }
+
+  observe(callIndex, delta) {
+    if (typeof delta !== "string") return;
+    const key = callIndex ?? "unknown";
+    let count = this.whitespaceRuns.get(key) || 0;
+    for (const character of delta) {
+      if (character === "\r" || character === "\n" || character === "\t") {
+        count += 1;
+        if (count > this.limit) {
+          this.whitespaceRuns.set(key, count);
+          throw streamProtocolError(
+            `Upstream sent more than ${this.limit} consecutive line-break or tab characters in tool arguments`,
+            "upstream_tool_arguments_stalled",
+          );
+        }
+      } else if (character !== " ") {
+        count = 0;
+      }
+    }
+    this.whitespaceRuns.set(key, count);
+  }
+}

package/src/stream-errors.mjs

@@ -1,7 +1,8 @@
 import { writeOrDrain } from "./http-transport.mjs";
+import { markStreamFailure } from "./stream-performance.mjs";
 
 function streamErrorData(protocol, error, abort) {
-  const code = abort?.reason || "upstream_stream_error";
+  const code = abort?.reason || error?.code || "upstream_stream_error";
   const message = `${code}: ${error?.message || "Upstream stream failed"}`;
   if (protocol === "anthropic") {
     return { type: "error", error: { type: "api_error", message } };
@@ -10,6 +11,7 @@ function streamErrorData(protocol, error, abort) {
 }
 
 export async function endStreamWithError(res, protocol, error, abort) {
+  markStreamFailure();
   if (res.destroyed || res.writableEnded) return;
   const data = streamErrorData(protocol, error, abort);
   await writeOrDrain(res, `event: error\ndata: ${JSON.stringify(data)}\n\n`).catch(() => false);

package/src/stream-performance.mjs

@@ -0,0 +1,177 @@
+import { performance } from "node:perf_hooks";
+import { currentRequestContext } from "./request-context.mjs";
+
+const PERFORMANCE_ROUTES = Object.freeze(["responses", "responses_compact", "messages"]);
+const TTFT_EDGES_MS = Object.freeze([
+  100, 200, 300, 500, 700, 1_000, 1_400, 2_000, 2_800, 4_000,
+  5_500, 8_000, 12_000, 16_000, 24_000, 36_000, 60_000, 120_000, 300_000,
+]);
+const TPOT_EDGES_US = Object.freeze([
+  500, 1_000, 2_000, 3_333, 5_000, 6_667, 8_333, 10_000, 12_500, 14_286,
+  16_667, 20_000, 25_000, 33_333, 40_000, 50_000, 66_667, 100_000,
+  150_000, 250_000, 500_000, 1_000_000, 2_500_000, 10_000_000,
+]);
+
+const RESPONSES_OUTPUT_EVENT_TYPES = new Set([
+  "response.output_text.delta",
+  "response.function_call_arguments.delta",
+  "response.custom_tool_call_input.delta",
+  "response.refusal.delta",
+  "response.reasoning_text.delta",
+  "response.reasoning_summary_text.delta",
+]);
+
+function nonEmptyString(value) {
+  return typeof value === "string" && value.length > 0;
+}
+
+export function isResponsesOutputEvent(event, eventType = event?.type) {
+  return RESPONSES_OUTPUT_EVENT_TYPES.has(eventType) && nonEmptyString(event?.delta);
+}
+
+export function isChatOutputDelta(delta) {
+  if (!delta || typeof delta !== "object") return false;
+  return nonEmptyString(delta.content)
+    || (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0)
+    || nonEmptyString(delta.refusal)
+    || nonEmptyString(delta.reasoning)
+    || nonEmptyString(delta.reasoning_content)
+    || nonEmptyString(delta.reasoning_text);
+}
+
+function createHistogram(edges) {
+  return {
+    edges,
+    counts: new Uint32Array(edges.length + 1),
+    samples: 0,
+    total: 0,
+    max: 0,
+  };
+}
+
+function observe(histogram, value) {
+  if (!Number.isFinite(value) || value < 0) return;
+  let index = 0;
+  while (index < histogram.edges.length && value > histogram.edges[index]) index += 1;
+  histogram.counts[index] += 1;
+  histogram.samples += 1;
+  histogram.total += value;
+  histogram.max = Math.max(histogram.max, value);
+}
+
+function histogramSnapshot(histogram, unit) {
+  let lower = 0;
+  const buckets = histogram.edges.map((upper, index) => {
+    const bucket = { lower, upper, count: histogram.counts[index] };
+    lower = upper;
+    return bucket;
+  });
+  buckets.push({ lower, upper: null, count: histogram.counts.at(-1) });
+  return {
+    unit,
+    samples: histogram.samples,
+    avg: histogram.samples > 0 ? Number((histogram.total / histogram.samples).toFixed(1)) : 0,
+    max: histogram.max,
+    buckets,
+  };
+}
+
+function createRoutePerformance() {
+  return {
+    ttft: createHistogram(TTFT_EDGES_MS),
+    tpot: createHistogram(TPOT_EDGES_US),
+    success_with_output: 0,
+    errors_with_output: 0,
+    zero_output_errors: 0,
+    neutral: 0,
+  };
+}
+
+function routeSnapshot(route) {
+  return {
+    success_with_output: route.success_with_output,
+    errors_with_output: route.errors_with_output,
+    zero_output_errors: route.zero_output_errors,
+    neutral: route.neutral,
+    ttft_ms: histogramSnapshot(route.ttft, "ms"),
+    tpot_us: histogramSnapshot(route.tpot, "us"),
+  };
+}
+
+export function createStreamPerformanceMetrics({ now = () => performance.now() } = {}) {
+  const routes = Object.fromEntries(PERFORMANCE_ROUTES.map((name) => [name, createRoutePerformance()]));
+
+  return {
+    begin(routeName) {
+      const route = routes[routeName];
+      if (!route) return null;
+      let upstreamStartedAt = null;
+      let firstOutputAt = null;
+      let outputTokens = null;
+      let failed = false;
+      let finished = false;
+
+      return {
+        upstreamStarted() {
+          if (upstreamStartedAt === null) upstreamStartedAt = now();
+        },
+        firstOutput() {
+          if (firstOutputAt === null) firstOutputAt = now();
+        },
+        setOutputTokens(value) {
+          const tokens = Number(value);
+          if (Number.isFinite(tokens) && tokens >= 0) outputTokens = tokens;
+        },
+        fail() {
+          failed = true;
+        },
+        finish({ failed: finishFailed = false } = {}) {
+          if (finished) return;
+          finished = true;
+          failed ||= finishFailed;
+          const finishedAt = now();
+          if (upstreamStartedAt === null) {
+            route.neutral += 1;
+            return;
+          }
+          if (firstOutputAt === null) {
+            if (failed) route.zero_output_errors += 1;
+            else route.neutral += 1;
+            return;
+          }
+          observe(route.ttft, Math.max(0, firstOutputAt - upstreamStartedAt));
+          if (outputTokens !== null && outputTokens >= 2) {
+            observe(route.tpot, Math.max(0, ((finishedAt - firstOutputAt) * 1_000) / (outputTokens - 1)));
+          }
+          if (failed) route.errors_with_output += 1;
+          else route.success_with_output += 1;
+        },
+      };
+    },
+    snapshot() {
+      return {
+        by_route: Object.fromEntries(PERFORMANCE_ROUTES.map((name) => [name, routeSnapshot(routes[name])])),
+      };
+    },
+  };
+}
+
+function tracker() {
+  return currentRequestContext()?.streamPerformance || null;
+}
+
+export function markUpstreamStarted() {
+  tracker()?.upstreamStarted();
+}
+
+export function markFirstOutput() {
+  tracker()?.firstOutput();
+}
+
+export function markOutputTokens(value) {
+  tracker()?.setOutputTokens(value);
+}
+
+export function markStreamFailure() {
+  tracker()?.fail();
+}

package/src/upstream-headers.mjs

@@ -0,0 +1,67 @@
+const REQUEST_ID_HEADERS = [
+  "x-request-id",
+  "x-github-request-id",
+  "x-ms-request-id",
+  "request-id",
+  "x-correlation-id",
+];
+
+const SAFE_EXACT_HEADERS = new Set([
+  "retry-after",
+  "ratelimit-limit",
+  "ratelimit-remaining",
+  "ratelimit-reset",
+  "ratelimit-policy",
+  "openai-model",
+  "openai-processing-ms",
+  "traceparent",
+]);
+
+const SAFE_HEADER_PREFIXES = [
+  "x-ratelimit-",
+  "x-ms-ratelimit-",
+];
+
+function headerValue(headers, name) {
+  const value = headers?.get?.(name);
+  return typeof value === "string" && value.trim() ? value : null;
+}
+
+function canonicalHeaderName(name) {
+  return String(name).split("-").map((part) => (
+    part ? `${part[0].toUpperCase()}${part.slice(1).toLowerCase()}` : part
+  )).join("-");
+}
+
+function isSafeMetadataHeader(name) {
+  return SAFE_EXACT_HEADERS.has(name)
+    || SAFE_HEADER_PREFIXES.some((prefix) => name.startsWith(prefix));
+}
+
+export function safeUpstreamResponseHeaders(headers, {
+  contentType,
+  defaultContentType,
+} = {}) {
+  const out = {};
+  const resolvedContentType = contentType
+    ?? headerValue(headers, "content-type")
+    ?? defaultContentType;
+  if (resolvedContentType) out["Content-Type"] = resolvedContentType;
+
+  if (headers && typeof headers[Symbol.iterator] === "function") {
+    for (const [rawName, value] of headers) {
+      const name = String(rawName).toLowerCase();
+      if (!isSafeMetadataHeader(name)) continue;
+      out[canonicalHeaderName(name)] = value;
+    }
+  }
+
+  for (const name of REQUEST_ID_HEADERS) {
+    const value = headerValue(headers, name);
+    if (value) {
+      out["X-Upstream-Request-Id"] = value;
+      break;
+    }
+  }
+  return out;
+}