junis 0.3.17 → 0.4.0

package/dist/cli/index.js

@@ -570,6 +570,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
+        "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",
@@ -2101,19 +2104,8 @@ function checkBlacklist(app) {
 function json(data) {
   return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
 }
-async function resolveElementCoords(query, app) {
-  if (!app) {
-    try {
-      const { stdout } = await execa("osascript", [
-        "-e",
-        'tell application "System Events" to get name of first application process whose frontmost is true'
-      ]);
-      app = stdout.trim();
-    } catch {
-      return null;
-    }
-  }
-  const safeApp = app.replace(/[\\"]/g, "\\$&");
+async function searchProcessElements(processName, query) {
+  const safeApp = processName.replace(/[\\"]/g, "\\$&");
   const safeQuery = query.replace(/[\\"]/g, "\\$&");
   const script = `
 tell application "System Events"
@@ -2158,6 +2150,26 @@ return "NOT_FOUND"
     return null;
   }
 }
+async function resolveElementCoords(query, app) {
+  let targetApp = app;
+  if (!targetApp) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      targetApp = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const result = await searchProcessElements(targetApp, query);
+  if (result) return result;
+  if (targetApp !== "Dock") {
+    return await searchProcessElements("Dock", query);
+  }
+  return null;
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
@@ -2197,8 +2209,9 @@ var DesktopTools = class {
       "desktop_screenshot",
       [
         "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
-        "Use for visual context or to verify UI state. Screenshot pixel coordinates map directly to desktop_click(coords).",
-        "For clicking, prefer desktop_click(query, app) which auto-resolves coordinates. Use screenshot coords as fallback."
+        "Use for visual context or to verify UI state ONLY.",
+        "WARNING: NEVER estimate click coordinates from screenshot images \u2014 visual estimation causes misclicks. Use desktop_click(query, app) for auto-resolution, or execute_command + osascript for exact coords.",
+        "Do not use visual coordinates from this screenshot directly. Use execute_command + osascript to verify exact element position first."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -2237,9 +2250,10 @@ var DesktopTools = class {
       [
         "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
         "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
-        "GOOD: Use coords 'x,y' from desktop_screenshot for pixel-perfect accuracy.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript position+size center calculation, NEVER from visual screenshot estimation.",
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots.",
         "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
-        "If not found, try: desktop_screenshot to find coords, desktop_menu for menu items, or desktop_hotkey for shortcuts.",
+        "If query not found: use execute_command + osascript to get element position+size, then center = (x + w/2, y + h/2). Or try desktop_menu, desktop_hotkey.",
         "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
       ].join("\n"),
       {
@@ -2305,6 +2319,7 @@ var DesktopTools = class {
       "desktop_paste",
       [
         "Paste via clipboard (Cmd+V). Atomic: saves clipboard \u2192 sets content \u2192 pastes \u2192 restores.",
+        "IMPORTANT: Focus the target field first (click it with desktop_click) before pasting.",
         "Supports all Unicode (Korean, Japanese, Chinese, emoji). Use instead of desktop_type for non-ASCII.",
         "Can also paste file contents via filePath."
       ].join("\n"),
@@ -2387,7 +2402,10 @@ var DesktopTools = class {
     );
     server.tool(
       "desktop_move",
-      "Move mouse cursor without clicking. Use before scroll or to hover.",
+      [
+        "Move mouse cursor without clicking. Use before scroll or to hover.",
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
+      ].join("\n"),
       {
         coords: z5.string().optional().describe("Screen coordinates 'x,y'"),
         to: z5.string().optional().describe("Element text/label to move to"),
@@ -2422,7 +2440,8 @@ var DesktopTools = class {
       "desktop_drag",
       [
         "Drag and drop between elements or coordinates. Supports cross-app drag (e.g. file to Trash).",
-        "Use element IDs from desktop_see or raw coordinates."
+        "Prefer fromCoords/toCoords for accuracy. Element IDs (from/to) have known offset bug.",
+        "Before using fromCoords/toCoords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         from: z5.string().optional().describe("Source element ID from desktop_see"),
@@ -2619,6 +2638,7 @@ var DesktopTools = class {
         app: z5.string().optional().describe("App to open with")
       },
       async ({ url, app }) => {
+        checkBlacklist(app);
         const args = ["open", url];
         if (app) args.push("--app", app);
         return json(await peekaboo(args));

package/dist/server/mcp.js

@@ -86,6 +86,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
+        "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",
@@ -1617,19 +1620,8 @@ function checkBlacklist(app) {
 function json(data) {
   return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
 }
-async function resolveElementCoords(query, app) {
-  if (!app) {
-    try {
-      const { stdout } = await execa("osascript", [
-        "-e",
-        'tell application "System Events" to get name of first application process whose frontmost is true'
-      ]);
-      app = stdout.trim();
-    } catch {
-      return null;
-    }
-  }
-  const safeApp = app.replace(/[\\"]/g, "\\$&");
+async function searchProcessElements(processName, query) {
+  const safeApp = processName.replace(/[\\"]/g, "\\$&");
   const safeQuery = query.replace(/[\\"]/g, "\\$&");
   const script = `
 tell application "System Events"
@@ -1674,6 +1666,26 @@ return "NOT_FOUND"
     return null;
   }
 }
+async function resolveElementCoords(query, app) {
+  let targetApp = app;
+  if (!targetApp) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      targetApp = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const result = await searchProcessElements(targetApp, query);
+  if (result) return result;
+  if (targetApp !== "Dock") {
+    return await searchProcessElements("Dock", query);
+  }
+  return null;
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
@@ -1713,8 +1725,9 @@ var DesktopTools = class {
       "desktop_screenshot",
       [
         "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
-        "Use for visual context or to verify UI state. Screenshot pixel coordinates map directly to desktop_click(coords).",
-        "For clicking, prefer desktop_click(query, app) which auto-resolves coordinates. Use screenshot coords as fallback."
+        "Use for visual context or to verify UI state ONLY.",
+        "WARNING: NEVER estimate click coordinates from screenshot images \u2014 visual estimation causes misclicks. Use desktop_click(query, app) for auto-resolution, or execute_command + osascript for exact coords.",
+        "Do not use visual coordinates from this screenshot directly. Use execute_command + osascript to verify exact element position first."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -1753,9 +1766,10 @@ var DesktopTools = class {
       [
         "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
         "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
-        "GOOD: Use coords 'x,y' from desktop_screenshot for pixel-perfect accuracy.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript position+size center calculation, NEVER from visual screenshot estimation.",
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots.",
         "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
-        "If not found, try: desktop_screenshot to find coords, desktop_menu for menu items, or desktop_hotkey for shortcuts.",
+        "If query not found: use execute_command + osascript to get element position+size, then center = (x + w/2, y + h/2). Or try desktop_menu, desktop_hotkey.",
         "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
       ].join("\n"),
       {
@@ -1821,6 +1835,7 @@ var DesktopTools = class {
       "desktop_paste",
       [
         "Paste via clipboard (Cmd+V). Atomic: saves clipboard \u2192 sets content \u2192 pastes \u2192 restores.",
+        "IMPORTANT: Focus the target field first (click it with desktop_click) before pasting.",
         "Supports all Unicode (Korean, Japanese, Chinese, emoji). Use instead of desktop_type for non-ASCII.",
         "Can also paste file contents via filePath."
       ].join("\n"),
@@ -1903,7 +1918,10 @@ var DesktopTools = class {
     );
     server.tool(
       "desktop_move",
-      "Move mouse cursor without clicking. Use before scroll or to hover.",
+      [
+        "Move mouse cursor without clicking. Use before scroll or to hover.",
+        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
+      ].join("\n"),
       {
         coords: z5.string().optional().describe("Screen coordinates 'x,y'"),
         to: z5.string().optional().describe("Element text/label to move to"),
@@ -1938,7 +1956,8 @@ var DesktopTools = class {
       "desktop_drag",
       [
         "Drag and drop between elements or coordinates. Supports cross-app drag (e.g. file to Trash).",
-        "Use element IDs from desktop_see or raw coordinates."
+        "Prefer fromCoords/toCoords for accuracy. Element IDs (from/to) have known offset bug.",
+        "Before using fromCoords/toCoords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots."
       ].join("\n"),
       {
         from: z5.string().optional().describe("Source element ID from desktop_see"),
@@ -2135,6 +2154,7 @@ var DesktopTools = class {
         app: z5.string().optional().describe("App to open with")
       },
       async ({ url, app }) => {
+        checkBlacklist(app);
         const args = ["open", url];
         if (app) args.push("--app", app);
         return json(await peekaboo(args));

package/dist/server/stdio.js

@@ -87,6 +87,9 @@ var FilesystemTools = class {
         "- For reading files prefer read_file, for editing prefer edit_block, for searching prefer search_code.",
         "- NOT for macOS app GUI interaction. Use desktop_* tools instead: desktop_open_app, desktop_see, desktop_click, desktop_type, desktop_paste, desktop_hotkey, desktop_scroll, desktop_move, desktop_menu, desktop_screenshot.",
         "- Exception: permission fix commands (swift -e, peekaboo permissions, open 'x-apple.systempreferences:...').",
+        "- Exception: osascript coordinate queries \u2014 Use osascript via this tool to query exact UI element position and size before any click/move/drag operation.",
+        "  Pattern: osascript + System Events \u2192 position + size \u2192 center = (x + w/2, y + h/2) \u2192 desktop_click(coords).",
+        "  This is the ONLY reliable way to get pixel-accurate coordinates on macOS. NEVER estimate coords from screenshots.",
         "",
         "BEHAVIOR:",
         "- Execute commands directly when the user requests them. Do not ask for confirmation \u2014 the user has already decided.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "junis",
-  "version": "0.3.17",
+  "version": "0.4.0",
   "description": "One-line device control for AI agents",
   "type": "module",
   "bin": {