jeo-code 0.12.6 → 0.12.7

package/CHANGELOG.md

@@ -9,6 +9,17 @@ The README mirrors the latest 5 entries — regenerate with `bun run changelog:s
 
 ## [Unreleased]
 
+## [0.12.7] - 2026-10-18
+_Repair desktop screenshot vision feedback, macOS scrolling, complete drag gestures, and keyboard normalization._
+
+### Fixed
+- **Screenshot vision feedback** — carry PNG attachments through tool execution, batches, and native Anthropic/OpenAI Responses replay instead of spilling base64 as text. Preserve CLI screenshot base64 stdout.
+- **Desktop input** — use Quartz scroll events on macOS (cliclick `w:` waits rather than scrolling), release the mouse after dragging on macOS/Linux, normalize macOS key aliases/modifier combinations, and terminate Linux text options before user text.
+- **Execution reliability** — drain subprocess pipes concurrently with exit, use collision-resistant screenshot paths, reject non-finite numeric inputs and out-of-range waits/scrolls, and explicitly reject unsupported platforms. Existing enablement/supervisor gates remain unchanged.
+
+### Documentation
+- Added [desktop setup and troubleshooting](docs/computer-use.md), including platform requirements, OS permissions, coordinate caveats, and supported action semantics.
+
 ## [0.12.6] - 2026-10-17
 _Preserve OpenAI subscription authentication behind configured proxies, isolate account catalogs, and give provider-specific recovery instructions while retaining Antigravity OAuth calls._
 

package/README.ja.md

@@ -416,10 +416,10 @@ CI は `.github/workflows/npm-publish.yml` で公開します — GitHub リリ
 
 <!-- CHANGELOG:START (auto-generated from CHANGELOG.md — run `bun run changelog:sync`) -->
 - **[Unreleased]**
+- **[0.12.7]** (2026-10-18) — Repair desktop screenshot vision feedback, macOS scrolling, complete drag gestures, and keyboard normalization.
 - **[0.12.6]** (2026-10-17) — Preserve OpenAI subscription authentication behind configured proxies, isolate account catalogs, and give provider-specific recovery instructions while retaining Antigravity OAuth calls.
 - **[0.12.5]** (2026-10-17) — Antigravity and Kimi provider support in interactive setup, Kimi live model cache rehydration, immediate discovery caching on provider probing.
 - **[0.12.4]** (2026-10-17) — Refresh Antigravity provider (`antigravity/*`) account catalogs with `antigravity/hub/2.23.1` so entitled Gemini 4 Argon and Gemini 3.8 Flash tiers appear directly in `/model provider antigravity`.
-- **[0.12.3]** (2026-10-17) — Anthropic static model catalog trimmed to claude-5.x with live API discovery, and `/route off` strictly fixes the active model with no failover.
 
 See [CHANGELOG.md](CHANGELOG.md) for the full history.
 <!-- CHANGELOG:END -->

package/README.ko.md

@@ -417,10 +417,10 @@ CI는 `.github/workflows/npm-publish.yml`로 배포합니다 — GitHub 릴리
 
 <!-- CHANGELOG:START (auto-generated from CHANGELOG.md — run `bun run changelog:sync`) -->
 - **[Unreleased]**
+- **[0.12.7]** (2026-10-18) — Repair desktop screenshot vision feedback, macOS scrolling, complete drag gestures, and keyboard normalization.
 - **[0.12.6]** (2026-10-17) — Preserve OpenAI subscription authentication behind configured proxies, isolate account catalogs, and give provider-specific recovery instructions while retaining Antigravity OAuth calls.
 - **[0.12.5]** (2026-10-17) — Antigravity and Kimi provider support in interactive setup, Kimi live model cache rehydration, immediate discovery caching on provider probing.
 - **[0.12.4]** (2026-10-17) — Refresh Antigravity provider (`antigravity/*`) account catalogs with `antigravity/hub/2.23.1` so entitled Gemini 4 Argon and Gemini 3.8 Flash tiers appear directly in `/model provider antigravity`.
-- **[0.12.3]** (2026-10-17) — Anthropic static model catalog trimmed to claude-5.x with live API discovery, and `/route off` strictly fixes the active model with no failover.
 
 See [CHANGELOG.md](CHANGELOG.md) for the full history.
 <!-- CHANGELOG:END -->

package/README.md

@@ -409,10 +409,10 @@ Huge thanks to [gajae-code](https://github.com/Yeachan-Heo/gajae-code) for the i
 
 <!-- CHANGELOG:START (auto-generated from CHANGELOG.md — run `bun run changelog:sync`) -->
 - **[Unreleased]**
+- **[0.12.7]** (2026-10-18) — Repair desktop screenshot vision feedback, macOS scrolling, complete drag gestures, and keyboard normalization.
 - **[0.12.6]** (2026-10-17) — Preserve OpenAI subscription authentication behind configured proxies, isolate account catalogs, and give provider-specific recovery instructions while retaining Antigravity OAuth calls.
 - **[0.12.5]** (2026-10-17) — Antigravity and Kimi provider support in interactive setup, Kimi live model cache rehydration, immediate discovery caching on provider probing.
 - **[0.12.4]** (2026-10-17) — Refresh Antigravity provider (`antigravity/*`) account catalogs with `antigravity/hub/2.23.1` so entitled Gemini 4 Argon and Gemini 3.8 Flash tiers appear directly in `/model provider antigravity`.
-- **[0.12.3]** (2026-10-17) — Anthropic static model catalog trimmed to claude-5.x with live API discovery, and `/route off` strictly fixes the active model with no failover.
 
 See [CHANGELOG.md](CHANGELOG.md) for the full history.
 <!-- CHANGELOG:END -->

package/README.zh.md

@@ -414,10 +414,10 @@ CI 通过 `.github/workflows/npm-publish.yml` 发布 — GitHub 发布 release
 
 <!-- CHANGELOG:START (auto-generated from CHANGELOG.md — run `bun run changelog:sync`) -->
 - **[Unreleased]**
+- **[0.12.7]** (2026-10-18) — Repair desktop screenshot vision feedback, macOS scrolling, complete drag gestures, and keyboard normalization.
 - **[0.12.6]** (2026-10-17) — Preserve OpenAI subscription authentication behind configured proxies, isolate account catalogs, and give provider-specific recovery instructions while retaining Antigravity OAuth calls.
 - **[0.12.5]** (2026-10-17) — Antigravity and Kimi provider support in interactive setup, Kimi live model cache rehydration, immediate discovery caching on provider probing.
 - **[0.12.4]** (2026-10-17) — Refresh Antigravity provider (`antigravity/*`) account catalogs with `antigravity/hub/2.23.1` so entitled Gemini 4 Argon and Gemini 3.8 Flash tiers appear directly in `/model provider antigravity`.
-- **[0.12.3]** (2026-10-17) — Anthropic static model catalog trimmed to claude-5.x with live API discovery, and `/route off` strictly fixes the active model with no failover.
 
 See [CHANGELOG.md](CHANGELOG.md) for the full history.
 <!-- CHANGELOG:END -->

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jeo-code",
-  "version": "0.12.6",
+  "version": "0.12.7",
   "description": "Clean, highly optimized AI coding agent using spec-first loop",
   "type": "module",
   "main": "src/cli.ts",

package/src/agent/engine.ts

@@ -1215,6 +1215,7 @@ export async function runAgentLoop(history: Message[], opts: AgentLoopOptions):
       const { tool, arguments: args } = call;
       let success: boolean;
       let output: string;
+      let images: import("../ai/types").ImageAttachment[] | undefined;
 
       if (tool === "done") {
         success = false;
@@ -1242,6 +1243,7 @@ export async function runAgentLoop(history: Message[], opts: AgentLoopOptions):
               const onProgress = ev.onToolProgress ? (partial: string) => ev.onToolProgress!(tool, partial) : undefined;
               const res = await handler(args ?? {}, cwd, onProgress);
               success = res.success;
+              images = res.images;
               output = res.success ? res.output : (res.error ? (res.output ? `${res.error}\n${res.output}` : res.error) : res.output);
             } catch (err: any) {
               success = false;
@@ -1250,7 +1252,7 @@ export async function runAgentLoop(history: Message[], opts: AgentLoopOptions):
           }
         }
       }
-      return { success, output };
+      return { success, output, images };
     };
 
     const WRITE_TOOLS = new Set(["write", "edit"]);
@@ -1293,7 +1295,7 @@ export async function runAgentLoop(history: Message[], opts: AgentLoopOptions):
       }
     }
 
-    const results: { success: boolean; output: string; executed: boolean }[] = Array.from(
+    const results: { success: boolean; output: string; executed: boolean; images?: import("../ai/types").ImageAttachment[] }[] = Array.from(
       { length: toolCalls.length },
       () => ({ success: false, output: "", executed: false })
     );
@@ -1385,6 +1387,8 @@ export async function runAgentLoop(history: Message[], opts: AgentLoopOptions):
       }));
       pushAssistantTurn(history, responseText, reasonBuf, artifactBuf, toolUse);
       const resultMsg: Message = { role: "user", content: resultBlocks.join("\n\n"), toolResults };
+      const images = indices.flatMap(idx => results[idx].images ?? []);
+      if (images.length) resultMsg.images = images;
       if (hookExtra) resultMsg.toolResultExtra = hookExtra;
       history.push(resultMsg);
     };

package/src/agent/tools.ts

@@ -22,6 +22,7 @@ async function readMutationLock(cwd: string): Promise<WorkflowState | null> {
 }
 
 export interface ToolResult {
+  images?: import("../ai/types").ImageAttachment[];
   success: boolean;
   output: string;
   error?: string;

package/src/ai/providers/anthropic.ts

@@ -240,6 +240,7 @@ export function buildAnthropicMessages(messages: Message[], model: string, think
       const blocks: AnthropicContentBlock[] = m.toolResults.map(tr => ({
         type: "tool_result", tool_use_id: tr.id, content: tr.output, is_error: tr.isError,
       }));
+      for (const img of m.images ?? []) blocks.push({ type: "image", source: { type: "base64", media_type: img.mediaType, data: img.data } });
       if (m.toolResultExtra) blocks.push({ type: "text", text: m.toolResultExtra });
       return { role: "user", content: blocks };
     }

package/src/ai/providers/openai-responses.ts

@@ -87,6 +87,7 @@ export function buildResponsesInput(messages: Message[], modelKey: string, strip
         && nonSystem[i - 1].role === "assistant" && responsesNativizable(nonSystem[i - 1], modelKey)) {
       for (const tr of m.toolResults) items.push({ type: "function_call_output", call_id: tr.id, output: tr.output });
       if (m.toolResultExtra) items.push({ role: "user", content: [{ type: "input_text", text: m.toolResultExtra }] });
+      if (m.images?.length) items.push({ role: "user", content: m.images.map(img => ({ type: "input_image", image_url: `data:${img.mediaType};base64,${img.data}` })) });
       return;
     }
     items.push(plain(m));

package/src/commands/computer.ts

@@ -2,6 +2,7 @@ import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import * as os from "node:os";
 import { computerSupervisor } from "../agent/computer-supervisor";
+import type { ImageAttachment } from "../ai/types";
 import { readGlobalConfig } from "../agent/state";
 
 export interface ComputerAction {
@@ -36,9 +37,11 @@ async function logAudit(action: ComputerAction, success: boolean, error?: string
 async function runCommand(cmd: string[]): Promise<{ success: boolean; stdout: string; stderr: string }> {
   try {
     const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" });
-    await proc.exited;
-    const stdout = await new Response(proc.stdout).text();
-    const stderr = await new Response(proc.stderr).text();
+    const [stdout, stderr] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ]);
     return {
       success: proc.exitCode === 0,
       stdout,
@@ -53,7 +56,7 @@ async function runCommand(cmd: string[]): Promise<{ success: boolean; stdout: st
   }
 }
 
-export async function executeComputerAction(action: ComputerAction, opts?: { enabledOverride?: boolean }): Promise<{ success: boolean; output: string; error?: string }> {
+export async function executeComputerAction(action: ComputerAction, opts?: { enabledOverride?: boolean }): Promise<{ success: boolean; output: string; error?: string; images?: ImageAttachment[] }> {
   let enabled = opts?.enabledOverride;
   if (enabled === undefined) {
     const config = await readGlobalConfig().catch(() => null);
@@ -68,6 +71,18 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
   }
 
 
+  if (!action || typeof action !== "object") return { success: false, output: "", error: "Expected a computer action object." };
+  for (const field of ["x", "y", "deltaX", "deltaY", "duration"] as const) {
+    if (action[field] !== undefined && (typeof action[field] !== "number" || !Number.isFinite(action[field]))) {
+      return { success: false, output: "", error: `Invalid finite number for ${field}.` };
+    }
+  }
+  if (action.duration !== undefined && (action.duration < 0 || action.duration > 60)) {
+    return { success: false, output: "", error: "duration must be between 0 and 60 seconds." };
+  }
+  if (os.platform() !== "darwin" && os.platform() !== "linux") {
+    return { success: false, output: "", error: "Computer use supports macOS and Linux/X11 only." };
+  }
   // Read-only actions don't need supervisor inputAllowed check
   const isReadOnly = action.action === "screenshot" || action.action === "wait";
   if (!isReadOnly) {
@@ -83,7 +98,7 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
   switch (action.action) {
     case "screenshot": {
       const tmpDir = os.tmpdir();
-      const screenPath = path.join(tmpDir, `jeo-screen-${Date.now()}.png`);
+      const screenPath = path.join(tmpDir, `jeo-screen-${crypto.randomUUID()}.png`);
       let cmd: string[];
       if (isMac) {
         cmd = ["screencapture", "-x", screenPath];
@@ -100,7 +115,7 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
         const base64 = bytes.toString("base64");
         await fs.unlink(screenPath).catch(() => {});
         await logAudit(action, true);
-        return { success: true, output: base64 };
+        return { success: true, output: "Screenshot captured.", images: [{ mediaType: "image/png", data: base64 }] };
       } catch (err: any) {
         await logAudit(action, false, err.message);
         return { success: false, output: "", error: `Failed to read screenshot file: ${err.message}` };
@@ -164,9 +179,9 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
       }
       let cmd: string[];
       if (isMac) {
-        cmd = ["cliclick", `dd:${action.x},${action.y}`];
+        cmd = ["cliclick", "dd:.", `dm:${action.x},${action.y}`, `du:${action.x},${action.y}`];
       } else {
-        cmd = ["xdotool", "mousemove", String(action.x), String(action.y), "mousedown", "1"];
+        cmd = ["xdotool", "mousedown", "1", "mousemove", String(action.x), String(action.y), "mouseup", "1"];
       }
       const res = await runCommand(cmd);
       await logAudit(action, res.success, res.success ? undefined : res.stderr);
@@ -179,9 +194,14 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
       let cmd: string[];
       const dx = action.deltaX ?? 0;
       const dy = action.deltaY ?? 0;
+      if (!Number.isInteger(dx) || !Number.isInteger(dy) || Math.abs(dx) > 1000 || Math.abs(dy) > 1000) {
+        return { success: false, output: "", error: "Scroll deltas must be integers between -1000 and 1000." };
+      }
+      if (dx === 0 && dy === 0) return { success: true, output: "Scroll skipped (zero deltas)." };
       if (isMac) {
-        // cliclick scroll deltaY deltaX
-        cmd = ["cliclick", `w:${dy},${dx}`];
+        // cliclick's w: command WAITS; it cannot scroll. Use Quartz line events.
+        cmd = ["/usr/bin/osascript", "-l", "JavaScript", "-e",
+          `ObjC.import('Cocoa'); var event = $.CGEventCreateScrollWheelEvent(null, 1, 2, ${dy}, ${dx}); $.CGEventPost(0, event);`];
       } else {
         // xdotool click 4 (scroll up), 5 (scroll down), 6 (scroll left), 7 (scroll right)
         const actions: string[] = [];
@@ -212,7 +232,7 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
       if (isMac) {
         cmd = ["cliclick", `t:${action.text}`];
       } else {
-        cmd = ["xdotool", "type", action.text];
+        cmd = ["xdotool", "type", "--clearmodifiers", "--", action.text];
       }
       const res = await runCommand(cmd);
       await logAudit(action, res.success, res.success ? undefined : res.stderr);
@@ -227,7 +247,16 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
       }
       let cmd: string[];
       if (isMac) {
-        cmd = ["cliclick", `kp:${action.key}`];
+        const aliases: Record<string, string> = { return: "return", enter: "return", escape: "esc", backspace: "delete", arrowup: "arrow-up", arrowdown: "arrow-down", arrowleft: "arrow-left", arrowright: "arrow-right", pageup: "page-up", pagedown: "page-down" };
+        const parts = action.key.toLowerCase().split("+");
+        const key = parts.pop()!;
+        const modifiers = parts.map(p => ({ command: "cmd", meta: "cmd", control: "ctrl", option: "alt" }[p] ?? p));
+        if (modifiers.some(p => !["cmd", "ctrl", "alt", "shift", "fn"].includes(p))) {
+          return { success: false, output: "", error: "Unknown key modifier." };
+        }
+        const press = key.length === 1 ? `t:${key}` : `kp:${aliases[key] ?? key}`;
+        cmd = ["cliclick", ...(modifiers.length ? [`kd:${modifiers.join(",")}`] : []), press,
+          ...(modifiers.length ? [`ku:${modifiers.join(",")}`] : [])];
       } else {
         cmd = ["xdotool", "key", action.key];
       }
@@ -250,6 +279,7 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
         return { success: false, output: "", error: "Missing actions array for batch action." };
       }
       const results: string[] = [];
+      const images: ImageAttachment[] = [];
       for (let i = 0; i < action.actions.length; i++) {
         const subAction = action.actions[i];
         const res = await executeComputerAction(subAction, opts);
@@ -257,12 +287,13 @@ export async function executeComputerAction(action: ComputerAction, opts?: { ena
         if (!res.success) {
           const err = `Batch failed at step ${i}: ${res.error}`;
           await logAudit(action, false, err);
-          return { success: false, output: results.join("\n"), error: err };
+          return { success: false, output: results.join("\n"), error: err, images };
         }
         results.push(`Step ${i}: ${res.output}`);
+        images.push(...(res.images ?? []));
       }
       await logAudit(action, true);
-      return { success: true, output: results.join("\n") };
+      return { success: true, output: results.join("\n"), images };
     }
 
     default:
@@ -329,7 +360,7 @@ export async function runComputerCommand(args: string[]): Promise<void> {
   const res = await executeComputerAction(action);
   if (res.success) {
     if (actionName === "screenshot") {
-      console.log(res.output); // base64 output
+      console.log(res.images?.[0]?.data ?? res.output); // preserve CLI base64 output
     } else {
       console.log(res.output);
     }