codex-copilot-dx 0.1.5 → 0.1.6

package/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## 0.1.6
+
+### Fixed
+- **413 Payload Too Large** on `/v1/responses` for long computer-use sessions. The Codex Desktop computer-use tool embeds screenshots as `input_image` parts inside `function_call_output.output` (sometimes as a stringified JSON array). After dozens of tool turns these accumulate to >5MB and hit GitHub Copilot's request size limit. The adapter now walks tool outputs, decodes embedded base64 images, and re-encodes them as WebP at long-edge ≤ 2048px before forwarding to the upstream.
+
+### Added
+- Automatic image optimization with `sharp`. Tunable via env vars:
+  - `CCDX_IMG_MAX_DIM` (default `2048`) – max long edge in pixels
+  - `CCDX_IMG_QUALITY` (default `85`) – WebP quality
+  - `CCDX_IMG_MIN_BYTES` (default `100000`) – skip images smaller than this
+  - `CCDX_DISABLE_IMG_OPT=1` – disable optimization entirely
+  - `CCDX_DEBUG=1` – verbose diagnostics
+- Per-request log line with byte counts before/after optimization, image count, and biggest item size.
+- Upstream error responses (status, headers, body) are now logged when non-2xx, making 413/auth issues much easier to diagnose.
+
+### Internal
+- Request headers to Copilot now set explicit `Content-Length`, `Content-Type: application/json; charset=utf-8`, and `Accept` to avoid undici picking chunked transfer for large bodies.

package/README.md

@@ -40,6 +40,17 @@ Environment variables:
 |----------|---------|-------------|
 | `COPILOT_API_PORT` | `4141` | Port for copilot-api |
 | `ADAPTER_PORT` | `4142` | Port for the adapter |
+| `CCDX_IMG_MAX_DIM` | `2048` | Max long-edge (px) for image downscaling |
+| `CCDX_IMG_QUALITY` | `85` | WebP quality used when re-encoding images |
+| `CCDX_IMG_MIN_BYTES` | `100000` | Images smaller than this are left untouched |
+| `CCDX_DISABLE_IMG_OPT` | unset | Set to `1` to disable image optimization |
+| `CCDX_DEBUG` | unset | Set to `1` for verbose request diagnostics |
+
+### Image optimization
+
+Long computer-use sessions can accumulate dozens of screenshots inside the conversation history. Each screenshot is shipped on every subsequent turn, and GitHub Copilot's `/v1/responses` endpoint rejects requests larger than ~5 MB with `413 Payload Too Large`.
+
+To avoid that, the adapter automatically downsamples embedded screenshots to long-edge ≤ 2048 px and re-encodes them as WebP before forwarding the request. Typical UI screenshots shrink to 10–15% of their original size with minimal perceptual loss. Disable with `CCDX_DISABLE_IMG_OPT=1` if you need pixel-perfect originals.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-copilot-dx",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Use Codex Desktop app with your GitHub Copilot subscription",
   "type": "module",
   "bin": {
@@ -9,7 +9,8 @@
   "files": [
     "bin/",
     "src/",
-    "README.md"
+    "README.md",
+    "CHANGELOG.md"
   ],
   "keywords": [
     "codex",
@@ -21,6 +22,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "copilot-api": "^0.7.0"
+    "copilot-api": "^0.7.0",
+    "sharp": "^0.33.5"
   }
 }

package/src/adapter.mjs

@@ -4,11 +4,140 @@ import fs from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { randomUUID, createHash } from "node:crypto";
+import sharp from "sharp";
 
 const COPILOT_API_PORT = parseInt(process.env.COPILOT_API_PORT || "4141");
 const COPILOT_API = "https://api.githubcopilot.com";
 const GITHUB_API = "https://api.github.com";
 
+// Image optimization knobs
+const IMG_MAX_DIM = parseInt(process.env.CCDX_IMG_MAX_DIM || "2048");
+const IMG_QUALITY = parseInt(process.env.CCDX_IMG_QUALITY || "85");
+const IMG_MIN_BYTES = parseInt(process.env.CCDX_IMG_MIN_BYTES || "100000");
+const IMG_OPT_DISABLED = process.env.CCDX_DISABLE_IMG_OPT === "1";
+const CCDX_DEBUG = process.env.CCDX_DEBUG === "1";
+
+async function optimizeImageDataUrl(dataUrl) {
+  if (IMG_OPT_DISABLED) return dataUrl;
+  if (typeof dataUrl !== "string" || !dataUrl.startsWith("data:")) return dataUrl;
+  const m = /^data:(image\/[a-z+.-]+);base64,(.+)$/i.exec(dataUrl);
+  if (!m) return dataUrl;
+  const mime = m[1].toLowerCase();
+  const b64 = m[2];
+  const raw = Buffer.from(b64, "base64");
+  if (raw.length < IMG_MIN_BYTES) return dataUrl;
+  // Skip animated formats — sharp will pick first frame and lose animation; better to leave alone
+  if (mime.includes("gif")) return dataUrl;
+  try {
+    const out = await sharp(raw, { failOn: "none" })
+      .rotate() // honor EXIF orientation
+      .resize(IMG_MAX_DIM, IMG_MAX_DIM, { fit: "inside", withoutEnlargement: true })
+      .webp({ quality: IMG_QUALITY, effort: 4 })
+      .toBuffer();
+    const ratio = ((out.length / raw.length) * 100).toFixed(1);
+    console.log(`[codex-copilot-dx] image: ${(raw.length/1024).toFixed(0)}KB ${mime} → ${(out.length/1024).toFixed(0)}KB webp (${ratio}%)`);
+    return `data:image/webp;base64,${out.toString("base64")}`;
+  } catch (e) {
+    console.warn(`[codex-copilot-dx] image optimize failed (${mime}, ${raw.length}b): ${e.message}`);
+    return dataUrl;
+  }
+}
+
+async function optimizeImagesInBody(reqBody) {
+  if (IMG_OPT_DISABLED) return reqBody;
+  if (!Array.isArray(reqBody.input)) return reqBody;
+  const tasks = [];
+
+  const visitParts = (parts) => {
+    if (!Array.isArray(parts)) return;
+    for (const part of parts) {
+      if (!part) continue;
+      // Responses API image part shape: { type: "input_image", image_url: "data:..." }
+      if (part.type === "input_image" && typeof part.image_url === "string") {
+        tasks.push((async () => { part.image_url = await optimizeImageDataUrl(part.image_url); })());
+      } else if (part.type === "image" && part.source?.type === "base64" && part.source?.data) {
+        // Anthropic style
+        tasks.push((async () => {
+          const dataUrl = `data:${part.source.media_type || "image/png"};base64,${part.source.data}`;
+          const opt = await optimizeImageDataUrl(dataUrl);
+          const m = /^data:([^;]+);base64,(.+)$/.exec(opt);
+          if (m) { part.source.media_type = m[1]; part.source.data = m[2]; }
+        })());
+      }
+    }
+  };
+
+  for (const item of reqBody.input) {
+    if (!item) continue;
+    // Plain user/assistant message
+    if (item.type === "message" && Array.isArray(item.content)) {
+      visitParts(item.content);
+    }
+    // computer-use / tool results: function_call_output.output can be a
+    // structured array of parts (input_text, input_image, ...) carrying screenshots.
+    if (item.type === "function_call_output") {
+      if (Array.isArray(item.output)) {
+        visitParts(item.output);
+      } else if (typeof item.output === "string") {
+        // Some tools stringify a JSON array of parts. Parse, optimize images
+        // inside, then re-serialize AFTER all image tasks for this array finish.
+        const trimmed = item.output.trim();
+        if (trimmed.startsWith("[")) {
+          try {
+            const parsed = JSON.parse(trimmed);
+            if (Array.isArray(parsed)) {
+              const before = tasks.length;
+              visitParts(parsed);
+              const localTasks = tasks.slice(before); // tasks added by visitParts
+              if (localTasks.length) {
+                tasks.push(
+                  Promise.all(localTasks).then(() => {
+                    item.output = JSON.stringify(parsed);
+                  })
+                );
+              }
+            }
+          } catch { /* not JSON, leave it */ }
+        }
+      }
+    }
+  }
+  if (tasks.length) await Promise.all(tasks);
+  return reqBody;
+}
+
+function summarizeReqBody(reqBody) {
+  try {
+    const inp = reqBody.input;
+    if (!Array.isArray(inp)) return { items: 0, images: 0, biggest: 0, biggestKind: "n/a" };
+    let biggest = 0, biggestKind = "", biggestItem = null, images = 0;
+    const countImages = (parts) => {
+      if (!Array.isArray(parts)) return;
+      for (const p of parts) {
+        if (p?.type === "input_image" || p?.type === "image") images++;
+      }
+    };
+    for (const it of inp) {
+      const s = JSON.stringify(it);
+      if (s.length > biggest) { biggest = s.length; biggestKind = it.type || "?"; biggestItem = it; }
+      if (it?.type === "message") countImages(it.content);
+      if (it?.type === "function_call_output") {
+        if (Array.isArray(it.output)) countImages(it.output);
+        else if (typeof it.output === "string" && it.output.trim().startsWith("[")) {
+          try { const p = JSON.parse(it.output); if (Array.isArray(p)) countImages(p); } catch {}
+        }
+      }
+    }
+    // Diagnostic: peek at the biggest function_call_output to see if it embeds images
+    let biggestPeek = "";
+    if (CCDX_DEBUG && biggestItem?.type === "function_call_output") {
+      const out = typeof biggestItem.output === "string" ? biggestItem.output : JSON.stringify(biggestItem.output);
+      biggestPeek = out.slice(0, 200);
+    }
+    return { items: inp.length, images, biggest, biggestKind, biggestPeek };
+  } catch { return { items: -1, images: -1, biggest: -1, biggestKind: "err", biggestPeek: "" }; }
+}
+
 // Models that only support Responses API (not chat/completions)
 const RESPONSES_ONLY = new Set([
   "gpt-5.5",
@@ -46,18 +175,38 @@ async function getCopilotToken() {
 // ── Direct Copilot Responses API proxy ──
 
 async function proxyCopilotResponses(reqBody, req, res) {
+  const beforeBytes = Buffer.byteLength(JSON.stringify(reqBody));
+  await optimizeImagesInBody(reqBody);
   const token = await getCopilotToken();
+  const bodyStr = JSON.stringify(reqBody);
+  const bodyBuf = Buffer.from(bodyStr, "utf-8");
+  const summary = summarizeReqBody(reqBody);
+  console.log(`[codex-copilot-dx] → responses model=${reqBody.model} stream=${!!reqBody.stream} bytes=${bodyBuf.length} (was ${beforeBytes}) input_items=${summary.items} images=${summary.images} biggest_item=${summary.biggest}b (${summary.biggestKind})`);
+  if (summary.biggestPeek) console.log(`[codex-copilot-dx]   peek: ${JSON.stringify(summary.biggestPeek)}`);
+
   const resp = await fetch(`${COPILOT_API}/v1/responses`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json",
+      "Content-Type": "application/json; charset=utf-8",
+      "Content-Length": String(bodyBuf.length),
       "Editor-Version": "vscode/1.90.0",
       "Copilot-Integration-Id": "vscode-chat",
+      "Accept": reqBody.stream ? "text/event-stream" : "application/json",
     },
-    body: JSON.stringify(reqBody),
+    body: bodyBuf,
   });
 
+  if (!resp.ok) {
+    const errText = await resp.text();
+    console.error(`[codex-copilot-dx] ✗ upstream ${resp.status} ${resp.statusText}`);
+    console.error(`[codex-copilot-dx]   headers:`, Object.fromEntries(resp.headers));
+    console.error(`[codex-copilot-dx]   body:`, errText.slice(0, 2000));
+    res.writeHead(resp.status, { "Content-Type": resp.headers.get("content-type") || "application/json" });
+    res.end(errText);
+    return;
+  }
+
   if (reqBody.stream) {
     res.writeHead(resp.status, {
       "Content-Type": resp.headers.get("content-type") || "text/event-stream",
@@ -79,19 +228,32 @@ async function proxyCopilotResponses(reqBody, req, res) {
 
 async function proxyCopilotResponsesWS(reqBody, wsSend, onDone) {
   const token = await getCopilotToken();
+  const payload = { ...reqBody, stream: true };
+  const beforeBytes = Buffer.byteLength(JSON.stringify(payload));
+  await optimizeImagesInBody(payload);
+  const bodyStr = JSON.stringify(payload);
+  const bodyBuf = Buffer.from(bodyStr, "utf-8");
+  const summary = summarizeReqBody(payload);
+  console.log(`[codex-copilot-dx] → responses (ws) model=${payload.model} bytes=${bodyBuf.length} (was ${beforeBytes}) input_items=${summary.items} images=${summary.images} biggest_item=${summary.biggest}b (${summary.biggestKind})`);
+
   const resp = await fetch(`${COPILOT_API}/v1/responses`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
-      "Content-Type": "application/json",
+      "Content-Type": "application/json; charset=utf-8",
+      "Content-Length": String(bodyBuf.length),
       "Editor-Version": "vscode/1.90.0",
       "Copilot-Integration-Id": "vscode-chat",
+      "Accept": "text/event-stream",
     },
-    body: JSON.stringify({ ...reqBody, stream: true }),
+    body: bodyBuf,
   });
 
   if (!resp.ok) {
     const err = await resp.text();
+    console.error(`[codex-copilot-dx] ✗ upstream(ws) ${resp.status} ${resp.statusText}`);
+    console.error(`[codex-copilot-dx]   headers:`, Object.fromEntries(resp.headers));
+    console.error(`[codex-copilot-dx]   body:`, err.slice(0, 2000));
     wsSend(JSON.stringify({ type: "error", error: { message: err, code: resp.status } }));
     onDone();
     return;