junis 0.3.16 → 0.3.17

package/dist/cli/index.js

@@ -525,13 +525,19 @@ var toolPermissions = {
   desktop_click: "confirm",
   desktop_type: "confirm",
   desktop_hotkey: "confirm",
+  desktop_press: "confirm",
   desktop_scroll: "confirm",
   desktop_move: "confirm",
+  desktop_drag: "confirm",
   desktop_menu: "confirm",
   desktop_paste: "confirm",
+  desktop_clipboard: "confirm",
+  desktop_dialog: "confirm",
   desktop_screenshot: "confirm",
   desktop_open_app: "auto",
   desktop_open_url: "auto",
+  desktop_app_quit: "confirm",
+  desktop_window: "confirm",
   cron_create: "confirm",
   cron_delete: "confirm",
   edit_block: "confirm",
@@ -2095,15 +2101,72 @@ function checkBlacklist(app) {
 function json(data) {
   return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
 }
+async function resolveElementCoords(query, app) {
+  if (!app) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      app = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const safeApp = app.replace(/[\\"]/g, "\\$&");
+  const safeQuery = query.replace(/[\\"]/g, "\\$&");
+  const script = `
+tell application "System Events"
+  tell process "${safeApp}"
+    set topElems to UI elements
+    repeat with elem in topElems
+      try
+        if (name of elem contains "${safeQuery}") or (description of elem contains "${safeQuery}") then
+          set pos to position of elem
+          set sz to size of elem
+          set cx to (item 1 of pos) + (item 1 of sz) / 2
+          set cy to (item 2 of pos) + (item 2 of sz) / 2
+          return ((cx as integer) as text) & "," & ((cy as integer) as text)
+        end if
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with elem in UI elements of parent
+          try
+            if (name of elem contains "${safeQuery}") or (description of elem contains "${safeQuery}") then
+              set pos to position of elem
+              set sz to size of elem
+              set cx to (item 1 of pos) + (item 1 of sz) / 2
+              set cy to (item 2 of pos) + (item 2 of sz) / 2
+              return ((cx as integer) as text) & "," & ((cy as integer) as text)
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+  end tell
+end tell
+return "NOT_FOUND"
+`;
+  try {
+    const { stdout } = await execa("osascript", ["-e", script], { timeout: 1e4 });
+    const result = stdout.trim();
+    if (result === "NOT_FOUND" || !result.includes(",")) return null;
+    return result;
+  } catch {
+    return null;
+  }
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
       "desktop_see",
       [
-        "Capture UI element tree of an app. Returns snapshot ID + element IDs (B1 for buttons, T1 for text fields\u2026) with absolute screen coordinates.",
-        "ALWAYS call this before clicking or typing to get fresh element IDs. Snapshots are ephemeral \u2014 re-capture when stale.",
-        "If timeout on complex apps, use desktop_screenshot + desktop_click(coords) as fallback.",
-        "For CJK/emoji text input, use desktop_paste (not desktop_type)."
+        "Capture native UI element tree. Returns snapshot ID + elements with id/role/label/description.",
+        "Useful for simple/moderate apps. May timeout on complex apps (100+ elements) \u2014 use desktop_click(query, app) which auto-resolves coordinates without needing desktop_see.",
+        "IMPORTANT: Only sees native macOS UI. Web page content inside browsers is invisible \u2014 use browser_* tools.",
+        "If timeout, use desktop_screenshot for visual context + desktop_click(query, app) or desktop_click(coords) to interact."
       ].join("\n"),
       {
         app: z5.string().optional().describe("App name, 'frontmost', or 'menubar'. Omit for frontmost."),
@@ -2125,7 +2188,7 @@ var DesktopTools = class {
           id: e.id,
           role: e.role,
           label: e.label,
-          bounds: e.bounds
+          description: e.description
         })) ?? [];
         return json({ snapshotId, elements });
       }
@@ -2133,9 +2196,9 @@ var DesktopTools = class {
     server.tool(
       "desktop_screenshot",
       [
-        "Take a screenshot. Returns base64 image.",
-        "Use when you need visual context or as fallback when desktop_see times out.",
-        "For automation, prefer desktop_see which returns actionable element IDs."
+        "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
+        "Use for visual context or to verify UI state. Screenshot pixel coordinates map directly to desktop_click(coords).",
+        "For clicking, prefer desktop_click(query, app) which auto-resolves coordinates. Use screenshot coords as fallback."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -2172,9 +2235,12 @@ var DesktopTools = class {
     server.tool(
       "desktop_click",
       [
-        "Click a UI element. Provide one of: query (text search), on (element ID from desktop_see), or coords ('x,y').",
-        "Prefer element IDs from desktop_see for reliability. Clicks the center of the element.",
-        "If click fails or element not found, re-capture with desktop_see and try again. Alternatively try desktop_menu or desktop_hotkey."
+        "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
+        "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
+        "GOOD: Use coords 'x,y' from desktop_screenshot for pixel-perfect accuracy.",
+        "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
+        "If not found, try: desktop_screenshot to find coords, desktop_menu for menu items, or desktop_hotkey for shortcuts.",
+        "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
       ].join("\n"),
       {
         query: z5.string().optional().describe("Text/label to click (case-insensitive)"),
@@ -2190,9 +2256,18 @@ var DesktopTools = class {
         checkBlacklist(app);
         if (!query && !on && !coords) throw new Error("Provide query, on, or coords.");
         const args = ["click"];
-        if (coords) args.push("--coords", coords);
-        else if (on) args.push("--on", on);
-        else if (query) args.push(query);
+        if (coords) {
+          args.push("--coords", coords);
+        } else if (query) {
+          const resolved = await resolveElementCoords(query, app);
+          if (resolved) {
+            args.push("--coords", resolved);
+          } else {
+            args.push(query);
+          }
+        } else if (on) {
+          args.push("--on", on);
+        }
         if (app) args.push("--app", app);
         if (snapshot) args.push("--snapshot", snapshot);
         if (doubleClick) args.push("--double");
@@ -2325,9 +2400,18 @@ var DesktopTools = class {
         checkBlacklist(app);
         if (!coords && !to && !id) throw new Error("Provide coords, to, or id.");
         const args = ["move"];
-        if (coords) args.push(coords);
-        else if (id) args.push("--id", id);
-        else if (to) args.push("--to", to);
+        if (coords) {
+          args.push(coords);
+        } else if (to) {
+          const resolved = await resolveElementCoords(to, app);
+          if (resolved) {
+            args.push(resolved);
+          } else {
+            args.push("--to", to);
+          }
+        } else if (id) {
+          args.push("--id", id);
+        }
         if (app) args.push("--app", app);
         if (snapshot) args.push("--snapshot", snapshot);
         if (smooth) args.push("--smooth");
@@ -2368,7 +2452,10 @@ var DesktopTools = class {
     );
     server.tool(
       "desktop_open_app",
-      "Launch or activate a macOS app. Already running apps are brought to front. After launch, call desktop_see to confirm UI is ready before automation. Terminal/iTerm/Finder blocked.",
+      [
+        "Launch or activate a macOS app. Already running apps are brought to front. Terminal/iTerm/Finder blocked.",
+        "After launch, wait briefly then use desktop_click(query, app) to interact. desktop_see may timeout on complex apps \u2014 use desktop_screenshot as visual fallback."
+      ].join("\n"),
       {
         app: z5.string().describe("App name (e.g. 'Safari', 'KakaoTalk', 'Slack')")
       },

package/dist/server/mcp.js

@@ -41,13 +41,19 @@ var toolPermissions = {
   desktop_click: "confirm",
   desktop_type: "confirm",
   desktop_hotkey: "confirm",
+  desktop_press: "confirm",
   desktop_scroll: "confirm",
   desktop_move: "confirm",
+  desktop_drag: "confirm",
   desktop_menu: "confirm",
   desktop_paste: "confirm",
+  desktop_clipboard: "confirm",
+  desktop_dialog: "confirm",
   desktop_screenshot: "confirm",
   desktop_open_app: "auto",
   desktop_open_url: "auto",
+  desktop_app_quit: "confirm",
+  desktop_window: "confirm",
   cron_create: "confirm",
   cron_delete: "confirm",
   edit_block: "confirm",
@@ -1611,15 +1617,72 @@ function checkBlacklist(app) {
 function json(data) {
   return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
 }
+async function resolveElementCoords(query, app) {
+  if (!app) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      app = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const safeApp = app.replace(/[\\"]/g, "\\$&");
+  const safeQuery = query.replace(/[\\"]/g, "\\$&");
+  const script = `
+tell application "System Events"
+  tell process "${safeApp}"
+    set topElems to UI elements
+    repeat with elem in topElems
+      try
+        if (name of elem contains "${safeQuery}") or (description of elem contains "${safeQuery}") then
+          set pos to position of elem
+          set sz to size of elem
+          set cx to (item 1 of pos) + (item 1 of sz) / 2
+          set cy to (item 2 of pos) + (item 2 of sz) / 2
+          return ((cx as integer) as text) & "," & ((cy as integer) as text)
+        end if
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with elem in UI elements of parent
+          try
+            if (name of elem contains "${safeQuery}") or (description of elem contains "${safeQuery}") then
+              set pos to position of elem
+              set sz to size of elem
+              set cx to (item 1 of pos) + (item 1 of sz) / 2
+              set cy to (item 2 of pos) + (item 2 of sz) / 2
+              return ((cx as integer) as text) & "," & ((cy as integer) as text)
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+  end tell
+end tell
+return "NOT_FOUND"
+`;
+  try {
+    const { stdout } = await execa("osascript", ["-e", script], { timeout: 1e4 });
+    const result = stdout.trim();
+    if (result === "NOT_FOUND" || !result.includes(",")) return null;
+    return result;
+  } catch {
+    return null;
+  }
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
       "desktop_see",
       [
-        "Capture UI element tree of an app. Returns snapshot ID + element IDs (B1 for buttons, T1 for text fields\u2026) with absolute screen coordinates.",
-        "ALWAYS call this before clicking or typing to get fresh element IDs. Snapshots are ephemeral \u2014 re-capture when stale.",
-        "If timeout on complex apps, use desktop_screenshot + desktop_click(coords) as fallback.",
-        "For CJK/emoji text input, use desktop_paste (not desktop_type)."
+        "Capture native UI element tree. Returns snapshot ID + elements with id/role/label/description.",
+        "Useful for simple/moderate apps. May timeout on complex apps (100+ elements) \u2014 use desktop_click(query, app) which auto-resolves coordinates without needing desktop_see.",
+        "IMPORTANT: Only sees native macOS UI. Web page content inside browsers is invisible \u2014 use browser_* tools.",
+        "If timeout, use desktop_screenshot for visual context + desktop_click(query, app) or desktop_click(coords) to interact."
       ].join("\n"),
       {
         app: z5.string().optional().describe("App name, 'frontmost', or 'menubar'. Omit for frontmost."),
@@ -1641,7 +1704,7 @@ var DesktopTools = class {
           id: e.id,
           role: e.role,
           label: e.label,
-          bounds: e.bounds
+          description: e.description
         })) ?? [];
         return json({ snapshotId, elements });
       }
@@ -1649,9 +1712,9 @@ var DesktopTools = class {
     server.tool(
       "desktop_screenshot",
       [
-        "Take a screenshot. Returns base64 image.",
-        "Use when you need visual context or as fallback when desktop_see times out.",
-        "For automation, prefer desktop_see which returns actionable element IDs."
+        "Take a screenshot. Returns base64 image at logical resolution (matches click coordinate system 1:1).",
+        "Use for visual context or to verify UI state. Screenshot pixel coordinates map directly to desktop_click(coords).",
+        "For clicking, prefer desktop_click(query, app) which auto-resolves coordinates. Use screenshot coords as fallback."
       ].join("\n"),
       {
         app: z5.string().optional().describe("Capture specific app window"),
@@ -1688,9 +1751,12 @@ var DesktopTools = class {
     server.tool(
       "desktop_click",
       [
-        "Click a UI element. Provide one of: query (text search), on (element ID from desktop_see), or coords ('x,y').",
-        "Prefer element IDs from desktop_see for reliability. Clicks the center of the element.",
-        "If click fails or element not found, re-capture with desktop_see and try again. Alternatively try desktop_menu or desktop_hotkey."
+        "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
+        "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
+        "GOOD: Use coords 'x,y' from desktop_screenshot for pixel-perfect accuracy.",
+        "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
+        "If not found, try: desktop_screenshot to find coords, desktop_menu for menu items, or desktop_hotkey for shortcuts.",
+        "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
       ].join("\n"),
       {
         query: z5.string().optional().describe("Text/label to click (case-insensitive)"),
@@ -1706,9 +1772,18 @@ var DesktopTools = class {
         checkBlacklist(app);
         if (!query && !on && !coords) throw new Error("Provide query, on, or coords.");
         const args = ["click"];
-        if (coords) args.push("--coords", coords);
-        else if (on) args.push("--on", on);
-        else if (query) args.push(query);
+        if (coords) {
+          args.push("--coords", coords);
+        } else if (query) {
+          const resolved = await resolveElementCoords(query, app);
+          if (resolved) {
+            args.push("--coords", resolved);
+          } else {
+            args.push(query);
+          }
+        } else if (on) {
+          args.push("--on", on);
+        }
         if (app) args.push("--app", app);
         if (snapshot) args.push("--snapshot", snapshot);
         if (doubleClick) args.push("--double");
@@ -1841,9 +1916,18 @@ var DesktopTools = class {
         checkBlacklist(app);
         if (!coords && !to && !id) throw new Error("Provide coords, to, or id.");
         const args = ["move"];
-        if (coords) args.push(coords);
-        else if (id) args.push("--id", id);
-        else if (to) args.push("--to", to);
+        if (coords) {
+          args.push(coords);
+        } else if (to) {
+          const resolved = await resolveElementCoords(to, app);
+          if (resolved) {
+            args.push(resolved);
+          } else {
+            args.push("--to", to);
+          }
+        } else if (id) {
+          args.push("--id", id);
+        }
         if (app) args.push("--app", app);
         if (snapshot) args.push("--snapshot", snapshot);
         if (smooth) args.push("--smooth");
@@ -1884,7 +1968,10 @@ var DesktopTools = class {
     );
     server.tool(
       "desktop_open_app",
-      "Launch or activate a macOS app. Already running apps are brought to front. After launch, call desktop_see to confirm UI is ready before automation. Terminal/iTerm/Finder blocked.",
+      [
+        "Launch or activate a macOS app. Already running apps are brought to front. Terminal/iTerm/Finder blocked.",
+        "After launch, wait briefly then use desktop_click(query, app) to interact. desktop_see may timeout on complex apps \u2014 use desktop_screenshot as visual fallback."
+      ].join("\n"),
       {
         app: z5.string().describe("App name (e.g. 'Safari', 'KakaoTalk', 'Slack')")
       },

package/dist/server/stdio.js

@@ -42,13 +42,19 @@ var toolPermissions = {
   desktop_click: "confirm",
   desktop_type: "confirm",
   desktop_hotkey: "confirm",
+  desktop_press: "confirm",
   desktop_scroll: "confirm",
   desktop_move: "confirm",
+  desktop_drag: "confirm",
   desktop_menu: "confirm",
   desktop_paste: "confirm",
+  desktop_clipboard: "confirm",
+  desktop_dialog: "confirm",
   desktop_screenshot: "confirm",
   desktop_open_app: "auto",
   desktop_open_url: "auto",
+  desktop_app_quit: "confirm",
+  desktop_window: "confirm",
   cron_create: "confirm",
   cron_delete: "confirm",
   edit_block: "confirm",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "junis",
-  "version": "0.3.16",
+  "version": "0.3.17",
   "description": "One-line device control for AI agents",
   "type": "module",
   "bin": {