codex-copilot-dx 0.1.4 → 0.1.6

package/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## 0.1.6
+
+### Fixed
+- **413 Payload Too Large** on `/v1/responses` for long computer-use sessions. The Codex Desktop computer-use tool embeds screenshots as `input_image` parts inside `function_call_output.output` (sometimes as a stringified JSON array). After dozens of tool turns these accumulate to >5MB and hit GitHub Copilot's request size limit. The adapter now walks tool outputs, decodes embedded base64 images, and re-encodes them as WebP at long-edge ≤ 2048px before forwarding to the upstream.
+
+### Added
+- Automatic image optimization with `sharp`. Tunable via env vars:
+  - `CCDX_IMG_MAX_DIM` (default `2048`) – max long edge in pixels
+  - `CCDX_IMG_QUALITY` (default `85`) – WebP quality
+  - `CCDX_IMG_MIN_BYTES` (default `100000`) – skip images smaller than this
+  - `CCDX_DISABLE_IMG_OPT=1` – disable optimization entirely
+  - `CCDX_DEBUG=1` – verbose diagnostics
+- Per-request log line with byte counts before/after optimization, image count, and biggest item size.
+- Upstream error responses (status, headers, body) are now logged when non-2xx, making 413/auth issues much easier to diagnose.
+
+### Internal
+- Request headers to Copilot now set explicit `Content-Length`, `Content-Type: application/json; charset=utf-8`, and `Accept` to avoid undici picking chunked transfer for large bodies.

package/README.md

@@ -40,6 +40,17 @@ Environment variables:
 |----------|---------|-------------|
 | `COPILOT_API_PORT` | `4141` | Port for copilot-api |
 | `ADAPTER_PORT` | `4142` | Port for the adapter |
+| `CCDX_IMG_MAX_DIM` | `2048` | Max long-edge (px) for image downscaling |
+| `CCDX_IMG_QUALITY` | `85` | WebP quality used when re-encoding images |
+| `CCDX_IMG_MIN_BYTES` | `100000` | Images smaller than this are left untouched |
+| `CCDX_DISABLE_IMG_OPT` | unset | Set to `1` to disable image optimization |
+| `CCDX_DEBUG` | unset | Set to `1` for verbose request diagnostics |
+
+### Image optimization
+
+Long computer-use sessions can accumulate dozens of screenshots inside the conversation history. Each screenshot is shipped on every subsequent turn, and GitHub Copilot's `/v1/responses` endpoint rejects requests larger than ~5 MB with `413 Payload Too Large`.
+
+To avoid that, the adapter automatically downsamples embedded screenshots to long-edge ≤ 2048 px and re-encodes them as WebP before forwarding the request. Typical UI screenshots shrink to 10–15% of their original size with minimal perceptual loss. Disable with `CCDX_DISABLE_IMG_OPT=1` if you need pixel-perfect originals.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-copilot-dx",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Use Codex Desktop app with your GitHub Copilot subscription",
   "type": "module",
   "bin": {
@@ -9,7 +9,8 @@
   "files": [
     "bin/",
     "src/",
-    "README.md"
+    "README.md",
+    "CHANGELOG.md"
   ],
   "keywords": [
     "codex",
@@ -21,6 +22,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "copilot-api": "^0.7.0"
+    "copilot-api": "^0.7.0",
+    "sharp": "^0.33.5"
   }
 }

package/src/adapter.mjs

@@ -4,11 +4,140 @@ import fs from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { randomUUID, createHash } from "node:crypto";
+import sharp from "sharp";
 
 const COPILOT_API_PORT = parseInt(process.env.COPILOT_API_PORT || "4141");
 const COPILOT_API = "https://api.githubcopilot.com";
 const GITHUB_API = "https://api.github.com";
 
+// Image optimization knobs
+const IMG_MAX_DIM = parseInt(process.env.CCDX_IMG_MAX_DIM || "2048");
+const IMG_QUALITY = parseInt(process.env.CCDX_IMG_QUALITY || "85");
+const IMG_MIN_BYTES = parseInt(process.env.CCDX_IMG_MIN_BYTES || "100000");
+const IMG_OPT_DISABLED = process.env.CCDX_DISABLE_IMG_OPT === "1";
+const CCDX_DEBUG = process.env.CCDX_DEBUG === "1";
+
+async function optimizeImageDataUrl(dataUrl) {
+  if (IMG_OPT_DISABLED) return dataUrl;
+  if (typeof dataUrl !== "string" || !dataUrl.startsWith("data:")) return dataUrl;
+  const m = /^data:(image\/[a-z+.-]+);base64,(.+)$/i.exec(dataUrl);
+  if (!m) return dataUrl;
+  const mime = m[1].toLowerCase();
+  const b64 = m[2];
+  const raw = Buffer.from(b64, "base64");
+  if (raw.length < IMG_MIN_BYTES) return dataUrl;
+  // Skip animated formats — sharp will pick first frame and lose animation; better to leave alone
+  if (mime.includes("gif")) return dataUrl;
+  try {
+    const out = await sharp(raw, { failOn: "none" })
+      .rotate() // honor EXIF orientation
+      .resize(IMG_MAX_DIM, IMG_MAX_DIM, { fit: "inside", withoutEnlargement: true })
+      .webp({ quality: IMG_QUALITY, effort: 4 })
+      .toBuffer();
+    const ratio = ((out.length / raw.length) * 100).toFixed(1);
+    console.log(`[codex-copilot-dx] image: ${(raw.length/1024).toFixed(0)}KB ${mime} → ${(out.length/1024).toFixed(0)}KB webp (${ratio}%)`);
+    return `data:image/webp;base64,${out.toString("base64")}`;
+  } catch (e) {
+    console.warn(`[codex-copilot-dx] image optimize failed (${mime}, ${raw.length}b): ${e.message}`);
+    return dataUrl;
+  }
+}
+
+async function optimizeImagesInBody(reqBody) {
+  if (IMG_OPT_DISABLED) return reqBody;
+  if (!Array.isArray(reqBody.input)) return reqBody;
+  const tasks = [];
+
+  const visitParts = (parts) => {
+    if (!Array.isArray(parts)) return;
+    for (const part of parts) {
+      if (!part) continue;
+      // Responses API image part shape: { type: "input_image", image_url: "data:..." }
+      if (part.type === "input_image" && typeof part.image_url === "string") {
+        tasks.push((async () => { part.image_url = await optimizeImageDataUrl(part.image_url); })());
+      } else if (part.type === "image" && part.source?.type === "base64" && part.source?.data) {
+        // Anthropic style
+        tasks.push((async () => {
+          const dataUrl = `data:${part.source.media_type || "image/png"};base64,${part.source.data}`;
+          const opt = await optimizeImageDataUrl(dataUrl);
+          const m = /^data:([^;]+);base64,(.+)$/.exec(opt);
+          if (m) { part.source.media_type = m[1]; part.source.data = m[2]; }
+        })());
+      }
+    }
+  };
+
+  for (const item of reqBody.input) {
+    if (!item) continue;
+    // Plain user/assistant message
+    if (item.type === "message" && Array.isArray(item.content)) {
+      visitParts(item.content);
+    }
+    // computer-use / tool results: function_call_output.output can be a
+    // structured array of parts (input_text, input_image, ...) carrying screenshots.
+    if (item.type === "function_call_output") {
+      if (Array.isArray(item.output)) {
+        visitParts(item.output);
+      } else if (typeof item.output === "string") {
+        // Some tools stringify a JSON array of parts. Parse, optimize images
+        // inside, then re-serialize AFTER all image tasks for this array finish.
+        const trimmed = item.output.trim();
+        if (trimmed.startsWith("[")) {
+          try {
+            const parsed = JSON.parse(trimmed);
+            if (Array.isArray(parsed)) {
+              const before = tasks.length;
+              visitParts(parsed);
+              const localTasks = tasks.slice(before); // tasks added by visitParts
+              if (localTasks.length) {
+                tasks.push(
+                  Promise.all(localTasks).then(() => {
+                    item.output = JSON.stringify(parsed);
+                  })
+                );
+              }
+            }
+          } catch { /* not JSON, leave it */ }
+        }
+      }
+    }
+  }
+  if (tasks.length) await Promise.all(tasks);
+  return reqBody;
+}
+
+function summarizeReqBody(reqBody) {
+  try {
+    const inp = reqBody.input;
+    if (!Array.isArray(inp)) return { items: 0, images: 0, biggest: 0, biggestKind: "n/a" };
+    let biggest = 0, biggestKind = "", biggestItem = null, images = 0;
+    const countImages = (parts) => {
+      if (!Array.isArray(parts)) return;
+      for (const p of parts) {
+        if (p?.type === "input_image" || p?.type === "image") images++;
+      }
+    };
+    for (const it of inp) {
+      const s = JSON.stringify(it);
+      if (s.length > biggest) { biggest = s.length; biggestKind = it.type || "?"; biggestItem = it; }
+      if (it?.type === "message") countImages(it.content);
+      if (it?.type === "function_call_output") {
+        if (Array.isArray(it.output)) countImages(it.output);
+        else if (typeof it.output === "string" && it.output.trim().startsWith("[")) {
+          try { const p = JSON.parse(it.output); if (Array.isArray(p)) countImages(p); } catch {}
+        }
+      }
+    }
+    // Diagnostic: peek at the biggest function_call_output to see if it embeds images
+    let biggestPeek = "";
+    if (CCDX_DEBUG && biggestItem?.type === "function_call_output") {
+      const out = typeof biggestItem.output === "string" ? biggestItem.output : JSON.stringify(biggestItem.output);
+      biggestPeek = out.slice(0, 200);
+    }
+    return { items: inp.length, images, biggest, biggestKind, biggestPeek };
+  } catch { return { items: -1, images: -1, biggest: -1, biggestKind: "err", biggestPeek: "" }; }
+}
+
 // Models that only support Responses API (not chat/completions)
 const RESPONSES_ONLY = new Set([
   "gpt-5.5",
@@ -46,18 +175,38 @@ async function getCopilotToken() {
 // ── Direct Copilot Responses API proxy ──
 
 async function proxyCopilotResponses(reqBody, req, res) {
+  const beforeBytes = Buffer.byteLength(JSON.stringify(reqBody));
+  await optimizeImagesInBody(reqBody);
   const token = await getCopilotToken();
+  const bodyStr = JSON.stringify(reqBody);
+  const bodyBuf = Buffer.from(bodyStr, "utf-8");
+  const summary = summarizeReqBody(reqBody);
+  console.log(`[codex-copilot-dx] → responses model=${reqBody.model} stream=${!!reqBody.stream} bytes=${bodyBuf.length} (was ${beforeBytes}) input_items=${summary.items} images=${summary.images} biggest_item=${summary.biggest}b (${summary.biggestKind})`);
+  if (summary.biggestPeek) console.log(`[codex-copilot-dx]   peek: ${JSON.stringify(summary.biggestPeek)}`);
+
   const resp = await fetch(`${COPILOT_API}/v1/responses`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json",
+      "Content-Type": "application/json; charset=utf-8",
+      "Content-Length": String(bodyBuf.length),
       "Editor-Version": "vscode/1.90.0",
       "Copilot-Integration-Id": "vscode-chat",
+      "Accept": reqBody.stream ? "text/event-stream" : "application/json",
     },
-    body: JSON.stringify(reqBody),
+    body: bodyBuf,
   });
 
+  if (!resp.ok) {
+    const errText = await resp.text();
+    console.error(`[codex-copilot-dx] ✗ upstream ${resp.status} ${resp.statusText}`);
+    console.error(`[codex-copilot-dx]   headers:`, Object.fromEntries(resp.headers));
+    console.error(`[codex-copilot-dx]   body:`, errText.slice(0, 2000));
+    res.writeHead(resp.status, { "Content-Type": resp.headers.get("content-type") || "application/json" });
+    res.end(errText);
+    return;
+  }
+
   if (reqBody.stream) {
     res.writeHead(resp.status, {
       "Content-Type": resp.headers.get("content-type") || "text/event-stream",
@@ -79,19 +228,32 @@ async function proxyCopilotResponses(reqBody, req, res) {
 
 async function proxyCopilotResponsesWS(reqBody, wsSend, onDone) {
   const token = await getCopilotToken();
+  const payload = { ...reqBody, stream: true };
+  const beforeBytes = Buffer.byteLength(JSON.stringify(payload));
+  await optimizeImagesInBody(payload);
+  const bodyStr = JSON.stringify(payload);
+  const bodyBuf = Buffer.from(bodyStr, "utf-8");
+  const summary = summarizeReqBody(payload);
+  console.log(`[codex-copilot-dx] → responses (ws) model=${payload.model} bytes=${bodyBuf.length} (was ${beforeBytes}) input_items=${summary.items} images=${summary.images} biggest_item=${summary.biggest}b (${summary.biggestKind})`);
+
   const resp = await fetch(`${COPILOT_API}/v1/responses`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json",
+      "Content-Type": "application/json; charset=utf-8",
+      "Content-Length": String(bodyBuf.length),
       "Editor-Version": "vscode/1.90.0",
       "Copilot-Integration-Id": "vscode-chat",
+      "Accept": "text/event-stream",
     },
-    body: JSON.stringify({ ...reqBody, stream: true }),
+    body: bodyBuf,
   });
 
   if (!resp.ok) {
     const err = await resp.text();
+    console.error(`[codex-copilot-dx] ✗ upstream(ws) ${resp.status} ${resp.statusText}`);
+    console.error(`[codex-copilot-dx]   headers:`, Object.fromEntries(resp.headers));
+    console.error(`[codex-copilot-dx]   body:`, err.slice(0, 2000));
     wsSend(JSON.stringify({ type: "error", error: { message: err, code: resp.status } }));
     onDone();
     return;
@@ -100,18 +262,42 @@ async function proxyCopilotResponsesWS(reqBody, wsSend, onDone) {
   const reader = resp.body.getReader();
   const decoder = new TextDecoder();
   let buf = "";
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    buf += decoder.decode(value, { stream: true });
-    const lines = buf.split("\n");
-    buf = lines.pop();
-    for (const line of lines) {
-      if (!line.startsWith("data: ")) continue;
-      const data = line.slice(6).trim();
-      if (data === "[DONE]") continue;
-      try { wsSend(JSON.stringify(JSON.parse(data))); } catch {}
+  let currentEvent = null;
+  let sawCompleted = false;
+  let lastResponseId = `resp_${uid()}`;
+  let lastModel = reqBody.model || "unknown";
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buf += decoder.decode(value, { stream: true });
+      const lines = buf.split("\n");
+      buf = lines.pop();
+      for (const line of lines) {
+        if (line.startsWith("event: ")) { currentEvent = line.slice(7).trim(); continue; }
+        if (!line.startsWith("data: ")) { if (line === "") currentEvent = null; continue; }
+        const data = line.slice(6).trim();
+        if (data === "[DONE]") continue;
+        let parsed;
+        try { parsed = JSON.parse(data); } catch { continue; }
+        if (!parsed.type && currentEvent) parsed.type = currentEvent;
+        if (!parsed.type) continue;
+        if (parsed.response?.id) lastResponseId = parsed.response.id;
+        if (parsed.response?.model) lastModel = parsed.response.model;
+        if (parsed.type === "response.completed") sawCompleted = true;
+        wsSend(JSON.stringify(parsed));
+      }
     }
+  } catch (e) {
+    wsSend(JSON.stringify({ type: "error", error: { message: e.message || String(e), code: 500 } }));
+    onDone();
+    return;
+  }
+  if (!sawCompleted) {
+    wsSend(JSON.stringify({
+      type: "response.completed",
+      response: { id: lastResponseId, object: "response", status: "completed", model: lastModel, output: [], usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } },
+    }));
   }
   onDone();
 }
@@ -179,7 +365,23 @@ function processStream(upstreamRes, emitEvent, onDone) {
   emitEvent("response.output_item.added", { output_index: 0, item: { type: "message", id: itemId, role: "assistant", status: "in_progress", content: [] } });
   emitEvent("response.content_part.added", { output_index: 0, content_index: 0, part: { type: "output_text", text: "" } });
 
-  let buf = "", fullText = "", toolCalls = {}, hasToolCalls = false;
+  let buf = "", fullText = "", toolCalls = {}, hasToolCalls = false, finished = false;
+
+  const finish = (errMsg) => {
+    if (finished) return;
+    finished = true;
+    if (errMsg) {
+      emitEvent("error", { error: { message: errMsg, code: 500 } });
+    } else {
+      emitEvent("response.output_text.done", { output_index: 0, content_index: 0, text: fullText });
+      emitEvent("response.output_item.done", { output_index: 0, item: { type: "message", id: itemId, role: "assistant", status: "completed", content: [{ type: "output_text", text: fullText }] } });
+      const output = hasToolCalls
+        ? Object.entries(toolCalls).map(([id, tc]) => ({ type: "function_call", id, call_id: id, name: tc.name, arguments: tc.arguments, status: "completed" }))
+        : [{ type: "message", id: itemId, role: "assistant", status: "completed", content: [{ type: "output_text", text: fullText }] }];
+      emitEvent("response.completed", { response: { id: respId, object: "response", status: "completed", model: actualModel, output, usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } } });
+    }
+    onDone();
+  };
 
   upstreamRes.on("data", (chunk) => {
     buf += chunk.toString();
@@ -197,6 +399,7 @@ function processStream(upstreamRes, emitEvent, onDone) {
           ? Object.entries(toolCalls).map(([id, tc]) => ({ type: "function_call", id, call_id: id, name: tc.name, arguments: tc.arguments, status: "completed" }))
           : [{ type: "message", id: itemId, role: "assistant", status: "completed", content: [{ type: "output_text", text: fullText }] }];
         emitEvent("response.completed", { response: { id: respId, object: "response", status: "completed", model: actualModel, output, usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } } });
+        finished = true;
         onDone();
         return;
       }
@@ -217,8 +420,8 @@ function processStream(upstreamRes, emitEvent, onDone) {
       }
     }
   });
-  upstreamRes.on("end", () => onDone());
-  upstreamRes.on("error", () => onDone());
+  upstreamRes.on("end", () => finish());
+  upstreamRes.on("error", (e) => finish(e?.message || "upstream stream error"));
 }
 
 function forwardToChat(chatReq, emitEvent, onDone, onError) {
@@ -274,7 +477,16 @@ function wsSend(socket, data, opcode = 0x1) {
 }
 
 async function handleWebSocket(req, socket) {
-  const key = req.headers["sec-websocket-key"];
+  let rawKey = req.headers["sec-websocket-key"];
+  if (Array.isArray(rawKey)) rawKey = rawKey[0];
+  const key = (rawKey || "").trim();
+  const upgrade = String(req.headers["upgrade"] || "").toLowerCase();
+  const connection = String(req.headers["connection"] || "").toLowerCase();
+  if (!key || upgrade !== "websocket" || !connection.includes("upgrade")) {
+    socket.write("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+    socket.destroy();
+    return;
+  }
   const accept = createHash("sha1").update(key + "258EAFA5-E914-47DA-95CA-5AB9FC6AB199").digest("base64");
   socket.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + accept + "\r\n\r\n");
 
@@ -357,9 +569,13 @@ export function startAdapter(port = 4142) {
   });
 
   server.on("upgrade", (req, socket) => {
-    if (req.url?.startsWith("/v1/responses")) {
-      handleWebSocket(req, socket).catch((e) => { console.error("[codex-copilot-dx] WS error:", e.message); socket.destroy(); });
-    } else socket.destroy();
+    // Codex Desktop 0.130+ negotiates a "responses_websockets" server-push
+    // protocol on WS upgrade. We don't implement that protocol; accepting the
+    // upgrade and waiting for a client request just hangs and triggers a
+    // 5-attempt reconnect storm. Refuse the upgrade so Codex falls back to
+    // plain HTTP SSE, which this adapter handles correctly.
+    socket.write("HTTP/1.1 426 Upgrade Required\r\nConnection: close\r\nContent-Length: 0\r\n\r\n");
+    socket.destroy();
   });
 
   return new Promise((resolve) => {