junis 0.4.2 → 0.4.4

package/dist/cli/index.js

@@ -190,13 +190,16 @@ async function uploadLargeFile(relay, base64Data, filename, contentType) {
 function isLargeBase64(base64) {
   return base64.length * 0.75 > LARGE_FILE_THRESHOLD;
 }
-function detectContentType(base64) {
-  const header = base64.slice(0, 16);
-  if (header.startsWith("/9j/")) return "image/jpeg";
-  if (header.startsWith("iVBOR")) return "image/png";
-  if (header.startsWith("R0lGO")) return "image/gif";
-  if (header.startsWith("UklGR")) return "image/webp";
-  if (header.startsWith("JVBER")) return "application/pdf";
+function detectContentType(buffer) {
+  if (buffer[0] === 137 && buffer[1] === 80 && buffer[2] === 78 && buffer[3] === 71) return "image/png";
+  if (buffer[0] === 255 && buffer[1] === 216 && buffer[2] === 255) return "image/jpeg";
+  if (buffer[0] === 71 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 56) return "image/gif";
+  if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) return "application/pdf";
+  if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70 && buffer[8] === 87 && buffer[9] === 69 && buffer[10] === 66 && buffer[11] === 80) return "image/webp";
+  if (buffer[4] === 102 && buffer[5] === 116 && buffer[6] === 121 && buffer[7] === 112) return "video/mp4";
+  if (buffer[0] === 82 && buffer[1] === 73 && buffer[2] === 70 && buffer[3] === 70 && buffer[8] === 87 && buffer[9] === 65 && buffer[10] === 86 && buffer[11] === 69) return "audio/wav";
+  if (buffer[0] === 73 && buffer[1] === 68 && buffer[2] === 51) return "audio/mpeg";
+  if (buffer[0] === 255 && (buffer[1] === 251 || buffer[1] === 243 || buffer[1] === 242)) return "audio/mpeg";
   return "application/octet-stream";
 }
 
@@ -222,6 +225,14 @@ var RelayClient = class {
   heartbeatTimer = null;
   destroyed = false;
   lastPongTime = 0;
+  _currentRequestId = null;
+  _currentSessionId = null;
+  get currentRequestId() {
+    return this._currentRequestId;
+  }
+  get currentSessionId() {
+    return this._currentSessionId;
+  }
   // upload_url_response 대기용 pending 맵
   pendingUploadRequests = /* @__PURE__ */ new Map();
   // signed_url_response 대기용 pending 맵
@@ -274,11 +285,17 @@ var RelayClient = class {
           return;
         }
         if (msg.type === "mcp_request") {
+          this._currentRequestId = msg.id;
+          this._currentSessionId = msg.session_id || null;
           try {
             let result = await this.onMCPRequest(msg.id, msg.payload);
+            this._currentRequestId = null;
+            this._currentSessionId = null;
             result = await this.processLargeFiles(result);
             this.send({ type: "mcp_response", id: msg.id, payload: result });
           } catch (err) {
+            this._currentRequestId = null;
+            this._currentSessionId = null;
             this.send({
               type: "mcp_response",
               id: msg.id,
@@ -331,6 +348,14 @@ var RelayClient = class {
       this.ws.send(JSON.stringify(data));
     }
   }
+  sendProgress(requestId, line) {
+    this.send({
+      type: "mcp_stdout",
+      request_id: requestId,
+      session_id: this._currentSessionId || "",
+      line
+    });
+  }
   /**
    * 서버에 presigned PUT URL 요청.
    * WebSocket으로 upload_url_request 전송 → upload_url_response 대기.
@@ -433,7 +458,8 @@ var RelayClient = class {
         }
       } else if (item.type === "text" && typeof item.text === "string" && isLargeBase64(item.text) && /^[A-Za-z0-9+/\n\r]+=*$/.test(item.text.trim())) {
         try {
-          const contentType = detectContentType(item.text);
+          const buffer = Buffer.from(item.text, "base64");
+          const contentType = detectContentType(buffer);
           if (contentType === "application/octet-stream") {
             content[i] = { type: "text", text: "[Binary file detected. Use the share_file tool to provide a download link to the user. Do not display this message.]" };
             continue;
@@ -443,7 +469,8 @@ var RelayClient = class {
           content[i] = { type: "text", text: url };
         } catch (err) {
           console.error("Failed to upload large text base64:", err);
-          const contentType = detectContentType(item.text);
+          const buffer = Buffer.from(item.text, "base64");
+          const contentType = detectContentType(buffer);
           const ext = contentType.split("/")[1] || "bin";
           item.text = `[\uD30C\uC77C \uC5C5\uB85C\uB4DC \uC2E4\uD328: ${String(err)}. \uD30C\uC77C\uBA85: file.${ext}]`;
         }
@@ -488,7 +515,7 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
 import { createServer } from "http";
 
 // src/tools/filesystem.ts
-import { exec, execFile } from "child_process";
+import { exec, execFile, spawn } from "child_process";
 import { promisify } from "util";
 import fs2 from "fs/promises";
 import path2 from "path";
@@ -559,7 +586,9 @@ function checkPermission(toolName) {
 var execAsync = promisify(exec);
 var execFileAsync = promisify(execFile);
 var FilesystemTools = class {
-  register(server) {
+  relayClient;
+  register(server, relayClient) {
+    this.relayClient = relayClient;
     server.tool(
       "execute_command",
       [
@@ -570,9 +599,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
-        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size for click/move/drag.",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
         "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
-        "  This is the most reliable way to get pixel-accurate coordinates on macOS.",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",
@@ -595,26 +624,84 @@ var FilesystemTools = class {
           exec(command);
           return { content: [{ type: "text", text: "Background execution started" }] };
         }
-        try {
-          const { stdout, stderr } = await execAsync(command, {
+        const requestId = this.relayClient?.currentRequestId ?? null;
+        if (requestId) {
+          this.relayClient?.sendProgress(requestId, `$ ${command}`);
+        }
+        return new Promise((resolve) => {
+          const child = spawn("sh", ["-c", command], {
             timeout: timeout_ms
           });
-          return {
-            content: [{ type: "text", text: stdout || stderr || "(no output)" }]
-          };
-        } catch (err) {
-          const error = err;
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error (exit ${error.code ?? "?"}): ${error.message}
-${error.stderr ?? ""}`
+          let stdoutBuf = "";
+          let stderrBuf = "";
+          let stdoutLineBuffer = "";
+          let stderrLineBuffer = "";
+          const flushLines = (buffer, newChunk) => {
+            const combined = buffer + newChunk;
+            const lines = combined.split("\n");
+            const incomplete = lines.pop() ?? "";
+            for (const line of lines) {
+              if (requestId) {
+                this.relayClient?.sendProgress(requestId, line);
               }
-            ],
-            isError: true
+            }
+            return incomplete;
           };
-        }
+          child.stdout.on("data", (chunk) => {
+            const text = chunk.toString();
+            stdoutBuf += text;
+            stdoutLineBuffer = flushLines(stdoutLineBuffer, text);
+          });
+          child.stderr.on("data", (chunk) => {
+            const text = chunk.toString();
+            stderrBuf += text;
+            stderrLineBuffer = flushLines(stderrLineBuffer, text);
+          });
+          child.on("close", (code, signal) => {
+            if (stdoutLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stdoutLineBuffer);
+            }
+            if (stderrLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stderrLineBuffer);
+            }
+            if (signal) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Killed by signal ${signal} (timeout: ${timeout_ms}ms)`
+                  }
+                ],
+                isError: true
+              });
+            } else if (code !== 0 && code !== null) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Error (exit ${code}): ${stderrBuf || stdoutBuf || "(no output)"}`
+                  }
+                ],
+                isError: true
+              });
+            } else {
+              resolve({
+                content: [{ type: "text", text: stdoutBuf || stderrBuf || "(no output)" }]
+              });
+            }
+          });
+          child.on("error", (err) => {
+            resolve({
+              content: [
+                {
+                  type: "text",
+                  text: `Error: ${err.message}`
+                }
+              ],
+              isError: true
+            });
+          });
+        });
       }
     );
     server.tool(
@@ -1782,14 +1869,14 @@ Cause: ${e.message}${hint}` }],
       },
       async ({ text }) => {
         const p = platform();
-        const { spawn: spawn2 } = await import("child_process");
+        const { spawn: spawn3 } = await import("child_process");
         const cmd = {
           mac: { bin: "pbcopy", args: [] },
           win: { bin: "powershell", args: ["-Command", "$input | Set-Clipboard"] },
           linux: { bin: "xclip", args: ["-selection", "clipboard"] }
         }[p];
         await new Promise((resolve, reject) => {
-          const proc = spawn2(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
+          const proc = spawn3(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
           proc.on("error", reject);
           proc.on("close", (code) => code === 0 ? resolve() : reject(new Error(`${cmd.bin} exited ${code}`)));
           proc.stdin.end(text);
@@ -1821,9 +1908,9 @@ Cause: ${e.message}${hint}` }],
             return { content: [{ type: "text", text: "Already recording." }] };
           }
           const tmpPath = output_path ?? `/tmp/junis_record_${Date.now()}.mp4`;
-          const { spawn: spawn2 } = await import("child_process");
+          const { spawn: spawn3 } = await import("child_process");
           const cmd = p === "mac" ? ["screencapture", ["-v", tmpPath]] : ["ffmpeg", ["-f", p === "win" ? "gdigrab" : "x11grab", "-i", p === "win" ? "desktop" : ":0.0", tmpPath]];
-          const child = spawn2(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
+          const child = spawn3(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
           child.unref();
           screenRecordPid = child.pid ?? null;
           return { content: [{ type: "text", text: `Recording started. Output path: ${tmpPath} (PID: ${screenRecordPid})` }] };
@@ -2310,9 +2397,10 @@ var DesktopTools = class {
     server.tool(
       "desktop_screenshot",
       [
-        "Take a screenshot. Returns base64 image at logical resolution (1:1 with click coordinate system).",
-        "Use for visual context, verify UI state, or locate elements when query-based methods fail.",
-        "For clicking, prefer desktop_click(query, app) which auto-resolves coords. If that fails, use osascript or visual estimation from this image as fallback."
+        "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
+        "Use for visual context or to verify UI state ONLY.",
+        "WARNING: NEVER estimate click coordinates from screenshot images \u2014 visual estimation causes misclicks. Use desktop_click(query, app) for auto-resolution, or execute_command + osascript for exact coords.",
+        "Do not use visual coordinates from this screenshot directly. Use execute_command + osascript to verify exact element position first."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -2352,7 +2440,7 @@ var DesktopTools = class {
         "Click a UI element by text label (query), AX role (role), coordinates (coords), or element ID (on).",
         "BEST: Use query with app \u2014 auto-resolves coords via accessibility API (L1-L3 depth), then falls back to full tree search for deeply nested elements. No desktop_see needed.",
         "ROLE: Use role (e.g. 'AXTextArea', 'AXTextField') to find elements without visible text labels (input fields, text areas). Searches L1-L3 depth.",
-        "GOOD: Use coords 'x,y' \u2014 best from osascript center calculation (position + size/2), or from screenshot if osascript is unavailable.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript center calculation (position + size/2). NEVER estimate from screenshots.",
         "CAUTION: 'on' (element ID) has known offset bug \u2014 use query, role, or coords instead.",
         "If not found: try desktop_menu for menu items, desktop_hotkey for shortcuts.",
         "NOTE: Web page elements (inside browser) are invisible \u2014 use browser_* tools instead."
@@ -2362,7 +2450,7 @@ var DesktopTools = class {
         role: z5.string().optional().describe("AX role to find (e.g. 'AXTextArea', 'AXTextField', 'AXButton'). Use when element has no text label."),
         on: z5.string().optional().describe("Element ID from desktop_see (e.g. 'B1', 'T2')"),
         coords: z5.string().optional().describe("Screen coordinates 'x,y' (e.g. '500,300')"),
-        app: z5.string().optional().describe("App name (always specify for faster resolution)"),
+        app: z5.string().optional().describe("App name"),
         snapshot: z5.string().optional().describe("Snapshot ID from desktop_see"),
         doubleClick: z5.boolean().optional().default(false).describe("Double-click"),
         rightClick: z5.boolean().optional().default(false).describe("Right-click (context menu)"),
@@ -2513,7 +2601,7 @@ var DesktopTools = class {
       "desktop_move",
       [
         "Move mouse cursor without clicking. Use before scroll or to hover.",
-        "For coords, prefer osascript (position + size/2) or use 'to' with text label for auto-resolution."
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         coords: z5.string().optional().describe("Screen coordinates 'x,y'"),
@@ -2550,7 +2638,7 @@ var DesktopTools = class {
       [
         "Drag and drop between elements or coordinates. Supports cross-app drag (e.g. file to Trash).",
         "Prefer fromCoords/toCoords for accuracy. Element IDs (from/to) have known offset bug.",
-        "For coords, prefer osascript (position + size/2). If unavailable, screenshot-based estimation is acceptable."
+        "Before using fromCoords/toCoords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         from: z5.string().optional().describe("Source element ID from desktop_see"),
@@ -2760,13 +2848,17 @@ var DesktopTools = class {
 var mcpPort = 3e3;
 var globalBrowserTools = null;
 var desktopToolsEnabled = false;
+var globalRelayClient = null;
+function setRelayClient(client) {
+  globalRelayClient = client;
+}
 function createMcpServer() {
   const server = new McpServer({
     name: "junis",
     version: "0.1.0"
   });
   const fsTools = new FilesystemTools();
-  fsTools.register(server);
+  fsTools.register(server, globalRelayClient ?? void 0);
   if (globalBrowserTools) {
     globalBrowserTools.register(server);
   }
@@ -3089,7 +3181,7 @@ import { createRequire } from "module";
 import fs6 from "fs";
 import path3 from "path";
 import os2 from "os";
-import { execSync, spawn } from "child_process";
+import { execSync, spawn as spawn2 } from "child_process";
 var CONFIG_DIR2 = path3.join(os2.homedir(), ".junis");
 var PID_FILE = path3.join(CONFIG_DIR2, "junis.pid");
 var LOG_DIR = path3.join(CONFIG_DIR2, "logs");
@@ -3127,7 +3219,7 @@ function startDaemon(port) {
   const scriptPath = process.argv[1];
   const out = fs6.openSync(LOG_FILE, "a");
   const err = fs6.openSync(LOG_FILE, "a");
-  const child = spawn(nodePath, [scriptPath, "start", "--daemon", "--port", String(port)], {
+  const child = spawn2(nodePath, [scriptPath, "start", "--daemon", "--port", String(port)], {
     detached: true,
     stdio: ["ignore", out, err],
     env: { ...process.env }
@@ -3366,6 +3458,7 @@ async function runForeground(config, port) {
       process.exit(1);
     }
   });
+  setRelayClient(relay);
   await relay.connect();
   const webUrl = process.env.JUNIS_WEB_URL ?? "https://junis.ai";
   console.log(" \u25C9 Relay connected");
@@ -3584,6 +3677,7 @@ program.command("start", { isDefault: true }).description("Start Junis agent con
         process.exit(1);
       }
     });
+    setRelayClient(relay);
     await relay.connect();
     console.log("[junis daemon] relay connected");
     return;

package/dist/server/mcp.js

@@ -4,7 +4,7 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
 import { createServer } from "http";
 
 // src/tools/filesystem.ts
-import { exec, execFile } from "child_process";
+import { exec, execFile, spawn } from "child_process";
 import { promisify } from "util";
 import fs from "fs/promises";
 import path from "path";
@@ -75,7 +75,9 @@ function checkPermission(toolName) {
 var execAsync = promisify(exec);
 var execFileAsync = promisify(execFile);
 var FilesystemTools = class {
-  register(server) {
+  relayClient;
+  register(server, relayClient) {
+    this.relayClient = relayClient;
     server.tool(
       "execute_command",
       [
@@ -86,9 +88,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
-        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size for click/move/drag.",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
         "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
-        "  This is the most reliable way to get pixel-accurate coordinates on macOS.",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",
@@ -111,26 +113,84 @@ var FilesystemTools = class {
           exec(command);
           return { content: [{ type: "text", text: "Background execution started" }] };
         }
-        try {
-          const { stdout, stderr } = await execAsync(command, {
+        const requestId = this.relayClient?.currentRequestId ?? null;
+        if (requestId) {
+          this.relayClient?.sendProgress(requestId, `$ ${command}`);
+        }
+        return new Promise((resolve) => {
+          const child = spawn("sh", ["-c", command], {
             timeout: timeout_ms
           });
-          return {
-            content: [{ type: "text", text: stdout || stderr || "(no output)" }]
-          };
-        } catch (err) {
-          const error = err;
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error (exit ${error.code ?? "?"}): ${error.message}
-${error.stderr ?? ""}`
+          let stdoutBuf = "";
+          let stderrBuf = "";
+          let stdoutLineBuffer = "";
+          let stderrLineBuffer = "";
+          const flushLines = (buffer, newChunk) => {
+            const combined = buffer + newChunk;
+            const lines = combined.split("\n");
+            const incomplete = lines.pop() ?? "";
+            for (const line of lines) {
+              if (requestId) {
+                this.relayClient?.sendProgress(requestId, line);
               }
-            ],
-            isError: true
+            }
+            return incomplete;
           };
-        }
+          child.stdout.on("data", (chunk) => {
+            const text = chunk.toString();
+            stdoutBuf += text;
+            stdoutLineBuffer = flushLines(stdoutLineBuffer, text);
+          });
+          child.stderr.on("data", (chunk) => {
+            const text = chunk.toString();
+            stderrBuf += text;
+            stderrLineBuffer = flushLines(stderrLineBuffer, text);
+          });
+          child.on("close", (code, signal) => {
+            if (stdoutLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stdoutLineBuffer);
+            }
+            if (stderrLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stderrLineBuffer);
+            }
+            if (signal) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Killed by signal ${signal} (timeout: ${timeout_ms}ms)`
+                  }
+                ],
+                isError: true
+              });
+            } else if (code !== 0 && code !== null) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Error (exit ${code}): ${stderrBuf || stdoutBuf || "(no output)"}`
+                  }
+                ],
+                isError: true
+              });
+            } else {
+              resolve({
+                content: [{ type: "text", text: stdoutBuf || stderrBuf || "(no output)" }]
+              });
+            }
+          });
+          child.on("error", (err) => {
+            resolve({
+              content: [
+                {
+                  type: "text",
+                  text: `Error: ${err.message}`
+                }
+              ],
+              isError: true
+            });
+          });
+        });
       }
     );
     server.tool(
@@ -1298,14 +1358,14 @@ Cause: ${e.message}${hint}` }],
       },
       async ({ text }) => {
         const p = platform();
-        const { spawn } = await import("child_process");
+        const { spawn: spawn2 } = await import("child_process");
         const cmd = {
           mac: { bin: "pbcopy", args: [] },
           win: { bin: "powershell", args: ["-Command", "$input | Set-Clipboard"] },
           linux: { bin: "xclip", args: ["-selection", "clipboard"] }
         }[p];
         await new Promise((resolve, reject) => {
-          const proc = spawn(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
+          const proc = spawn2(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
           proc.on("error", reject);
           proc.on("close", (code) => code === 0 ? resolve() : reject(new Error(`${cmd.bin} exited ${code}`)));
           proc.stdin.end(text);
@@ -1337,9 +1397,9 @@ Cause: ${e.message}${hint}` }],
             return { content: [{ type: "text", text: "Already recording." }] };
           }
           const tmpPath = output_path ?? `/tmp/junis_record_${Date.now()}.mp4`;
-          const { spawn } = await import("child_process");
+          const { spawn: spawn2 } = await import("child_process");
           const cmd = p === "mac" ? ["screencapture", ["-v", tmpPath]] : ["ffmpeg", ["-f", p === "win" ? "gdigrab" : "x11grab", "-i", p === "win" ? "desktop" : ":0.0", tmpPath]];
-          const child = spawn(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
+          const child = spawn2(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
           child.unref();
           screenRecordPid = child.pid ?? null;
           return { content: [{ type: "text", text: `Recording started. Output path: ${tmpPath} (PID: ${screenRecordPid})` }] };
@@ -1826,9 +1886,10 @@ var DesktopTools = class {
     server.tool(
       "desktop_screenshot",
       [
-        "Take a screenshot. Returns base64 image at logical resolution (1:1 with click coordinate system).",
-        "Use for visual context, verify UI state, or locate elements when query-based methods fail.",
-        "For clicking, prefer desktop_click(query, app) which auto-resolves coords. If that fails, use osascript or visual estimation from this image as fallback."
+        "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
+        "Use for visual context or to verify UI state ONLY.",
+        "WARNING: NEVER estimate click coordinates from screenshot images \u2014 visual estimation causes misclicks. Use desktop_click(query, app) for auto-resolution, or execute_command + osascript for exact coords.",
+        "Do not use visual coordinates from this screenshot directly. Use execute_command + osascript to verify exact element position first."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -1868,7 +1929,7 @@ var DesktopTools = class {
         "Click a UI element by text label (query), AX role (role), coordinates (coords), or element ID (on).",
         "BEST: Use query with app \u2014 auto-resolves coords via accessibility API (L1-L3 depth), then falls back to full tree search for deeply nested elements. No desktop_see needed.",
         "ROLE: Use role (e.g. 'AXTextArea', 'AXTextField') to find elements without visible text labels (input fields, text areas). Searches L1-L3 depth.",
-        "GOOD: Use coords 'x,y' \u2014 best from osascript center calculation (position + size/2), or from screenshot if osascript is unavailable.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript center calculation (position + size/2). NEVER estimate from screenshots.",
         "CAUTION: 'on' (element ID) has known offset bug \u2014 use query, role, or coords instead.",
         "If not found: try desktop_menu for menu items, desktop_hotkey for shortcuts.",
         "NOTE: Web page elements (inside browser) are invisible \u2014 use browser_* tools instead."
@@ -1878,7 +1939,7 @@ var DesktopTools = class {
         role: z5.string().optional().describe("AX role to find (e.g. 'AXTextArea', 'AXTextField', 'AXButton'). Use when element has no text label."),
         on: z5.string().optional().describe("Element ID from desktop_see (e.g. 'B1', 'T2')"),
         coords: z5.string().optional().describe("Screen coordinates 'x,y' (e.g. '500,300')"),
-        app: z5.string().optional().describe("App name (always specify for faster resolution)"),
+        app: z5.string().optional().describe("App name"),
         snapshot: z5.string().optional().describe("Snapshot ID from desktop_see"),
         doubleClick: z5.boolean().optional().default(false).describe("Double-click"),
         rightClick: z5.boolean().optional().default(false).describe("Right-click (context menu)"),
@@ -2029,7 +2090,7 @@ var DesktopTools = class {
       "desktop_move",
       [
         "Move mouse cursor without clicking. Use before scroll or to hover.",
-        "For coords, prefer osascript (position + size/2) or use 'to' with text label for auto-resolution."
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         coords: z5.string().optional().describe("Screen coordinates 'x,y'"),
@@ -2066,7 +2127,7 @@ var DesktopTools = class {
       [
         "Drag and drop between elements or coordinates. Supports cross-app drag (e.g. file to Trash).",
         "Prefer fromCoords/toCoords for accuracy. Element IDs (from/to) have known offset bug.",
-        "For coords, prefer osascript (position + size/2). If unavailable, screenshot-based estimation is acceptable."
+        "Before using fromCoords/toCoords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         from: z5.string().optional().describe("Source element ID from desktop_see"),
@@ -2276,13 +2337,17 @@ var DesktopTools = class {
 var mcpPort = 3e3;
 var globalBrowserTools = null;
 var desktopToolsEnabled = false;
+var globalRelayClient = null;
+function setRelayClient(client) {
+  globalRelayClient = client;
+}
 function createMcpServer() {
   const server = new McpServer({
     name: "junis",
     version: "0.1.0"
   });
   const fsTools = new FilesystemTools();
-  fsTools.register(server);
+  fsTools.register(server, globalRelayClient ?? void 0);
   if (globalBrowserTools) {
     globalBrowserTools.register(server);
   }
@@ -2573,6 +2638,7 @@ async function handleMCPRequest(id, payload) {
 export {
   checkPermission,
   handleMCPRequest,
+  setRelayClient,
   startMCPServer,
   toolPermissions
 };

package/dist/server/stdio.js

@@ -5,7 +5,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 
 // src/tools/filesystem.ts
-import { exec, execFile } from "child_process";
+import { exec, execFile, spawn } from "child_process";
 import { promisify } from "util";
 import fs from "fs/promises";
 import path from "path";
@@ -76,7 +76,9 @@ function checkPermission(toolName) {
 var execAsync = promisify(exec);
 var execFileAsync = promisify(execFile);
 var FilesystemTools = class {
-  register(server) {
+  relayClient;
+  register(server, relayClient) {
+    this.relayClient = relayClient;
     server.tool(
       "execute_command",
       [
@@ -87,9 +89,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
-        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size for click/move/drag.",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
         "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
-        "  This is the most reliable way to get pixel-accurate coordinates on macOS.",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",
@@ -112,26 +114,84 @@ var FilesystemTools = class {
           exec(command);
           return { content: [{ type: "text", text: "Background execution started" }] };
         }
-        try {
-          const { stdout, stderr } = await execAsync(command, {
+        const requestId = this.relayClient?.currentRequestId ?? null;
+        if (requestId) {
+          this.relayClient?.sendProgress(requestId, `$ ${command}`);
+        }
+        return new Promise((resolve) => {
+          const child = spawn("sh", ["-c", command], {
             timeout: timeout_ms
           });
-          return {
-            content: [{ type: "text", text: stdout || stderr || "(no output)" }]
-          };
-        } catch (err) {
-          const error = err;
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error (exit ${error.code ?? "?"}): ${error.message}
-${error.stderr ?? ""}`
+          let stdoutBuf = "";
+          let stderrBuf = "";
+          let stdoutLineBuffer = "";
+          let stderrLineBuffer = "";
+          const flushLines = (buffer, newChunk) => {
+            const combined = buffer + newChunk;
+            const lines = combined.split("\n");
+            const incomplete = lines.pop() ?? "";
+            for (const line of lines) {
+              if (requestId) {
+                this.relayClient?.sendProgress(requestId, line);
               }
-            ],
-            isError: true
+            }
+            return incomplete;
           };
-        }
+          child.stdout.on("data", (chunk) => {
+            const text = chunk.toString();
+            stdoutBuf += text;
+            stdoutLineBuffer = flushLines(stdoutLineBuffer, text);
+          });
+          child.stderr.on("data", (chunk) => {
+            const text = chunk.toString();
+            stderrBuf += text;
+            stderrLineBuffer = flushLines(stderrLineBuffer, text);
+          });
+          child.on("close", (code, signal) => {
+            if (stdoutLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stdoutLineBuffer);
+            }
+            if (stderrLineBuffer && requestId) {
+              this.relayClient?.sendProgress(requestId, stderrLineBuffer);
+            }
+            if (signal) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Killed by signal ${signal} (timeout: ${timeout_ms}ms)`
+                  }
+                ],
+                isError: true
+              });
+            } else if (code !== 0 && code !== null) {
+              resolve({
+                content: [
+                  {
+                    type: "text",
+                    text: `Error (exit ${code}): ${stderrBuf || stdoutBuf || "(no output)"}`
+                  }
+                ],
+                isError: true
+              });
+            } else {
+              resolve({
+                content: [{ type: "text", text: stdoutBuf || stderrBuf || "(no output)" }]
+              });
+            }
+          });
+          child.on("error", (err) => {
+            resolve({
+              content: [
+                {
+                  type: "text",
+                  text: `Error: ${err.message}`
+                }
+              ],
+              isError: true
+            });
+          });
+        });
       }
     );
     server.tool(
@@ -1299,14 +1359,14 @@ Cause: ${e.message}${hint}` }],
       },
       async ({ text }) => {
         const p = platform();
-        const { spawn } = await import("child_process");
+        const { spawn: spawn2 } = await import("child_process");
         const cmd = {
           mac: { bin: "pbcopy", args: [] },
           win: { bin: "powershell", args: ["-Command", "$input | Set-Clipboard"] },
           linux: { bin: "xclip", args: ["-selection", "clipboard"] }
         }[p];
         await new Promise((resolve, reject) => {
-          const proc = spawn(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
+          const proc = spawn2(cmd.bin, cmd.args, { stdio: ["pipe", "ignore", "ignore"] });
           proc.on("error", reject);
           proc.on("close", (code) => code === 0 ? resolve() : reject(new Error(`${cmd.bin} exited ${code}`)));
           proc.stdin.end(text);
@@ -1338,9 +1398,9 @@ Cause: ${e.message}${hint}` }],
             return { content: [{ type: "text", text: "Already recording." }] };
           }
           const tmpPath = output_path ?? `/tmp/junis_record_${Date.now()}.mp4`;
-          const { spawn } = await import("child_process");
+          const { spawn: spawn2 } = await import("child_process");
           const cmd = p === "mac" ? ["screencapture", ["-v", tmpPath]] : ["ffmpeg", ["-f", p === "win" ? "gdigrab" : "x11grab", "-i", p === "win" ? "desktop" : ":0.0", tmpPath]];
-          const child = spawn(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
+          const child = spawn2(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
           child.unref();
           screenRecordPid = child.pid ?? null;
           return { content: [{ type: "text", text: `Recording started. Output path: ${tmpPath} (PID: ${screenRecordPid})` }] };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "junis",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "description": "One-line device control for AI agents",
   "type": "module",
   "bin": {