junis 0.4.0 → 0.4.1

package/dist/cli/index.js

@@ -2137,6 +2137,28 @@ tell application "System Events"
         end repeat
       end try
     end repeat
+    repeat with parent in topElems
+      try
+        repeat with child in UI elements of parent
+          try
+            set childRole to role of child
+            if childRole is not "AXTable" and childRole is not "AXOutline" and childRole is not "AXList" then
+              repeat with gc in UI elements of child
+                try
+                  if (name of gc contains "${safeQuery}") or (description of gc contains "${safeQuery}") then
+                    set pos to position of gc
+                    set sz to size of gc
+                    set cx to (item 1 of pos) + (item 1 of sz) / 2
+                    set cy to (item 2 of pos) + (item 2 of sz) / 2
+                    return ((cx as integer) as text) & "," & ((cy as integer) as text)
+                  end if
+                end try
+              end repeat
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
   end tell
 end tell
 return "NOT_FOUND"
@@ -2170,6 +2192,86 @@ async function resolveElementCoords(query, app) {
   }
   return null;
 }
+async function findFirstByRole(role, app) {
+  let targetApp = app;
+  if (!targetApp) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      targetApp = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const safeApp = targetApp.replace(/[\\"]/g, "\\$&");
+  const safeRole = role.replace(/[\\"]/g, "\\$&");
+  const script = `
+tell application "System Events"
+  tell process "${safeApp}"
+    set topElems to UI elements
+    repeat with elem in topElems
+      try
+        if role of elem is "${safeRole}" then
+          set pos to position of elem
+          set sz to size of elem
+          set cx to (item 1 of pos) + (item 1 of sz) / 2
+          set cy to (item 2 of pos) + (item 2 of sz) / 2
+          return ((cx as integer) as text) & "," & ((cy as integer) as text)
+        end if
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with elem in UI elements of parent
+          try
+            if role of elem is "${safeRole}" then
+              set pos to position of elem
+              set sz to size of elem
+              set cx to (item 1 of pos) + (item 1 of sz) / 2
+              set cy to (item 2 of pos) + (item 2 of sz) / 2
+              return ((cx as integer) as text) & "," & ((cy as integer) as text)
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with child in UI elements of parent
+          try
+            set childRole to role of child
+            if childRole is not "AXTable" and childRole is not "AXOutline" and childRole is not "AXList" then
+              repeat with gc in UI elements of child
+                try
+                  if role of gc is "${safeRole}" then
+                    set pos to position of gc
+                    set sz to size of gc
+                    set cx to (item 1 of pos) + (item 1 of sz) / 2
+                    set cy to (item 2 of pos) + (item 2 of sz) / 2
+                    return ((cx as integer) as text) & "," & ((cy as integer) as text)
+                  end if
+                end try
+              end repeat
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+  end tell
+end tell
+return "NOT_FOUND"
+`;
+  try {
+    const { stdout } = await execa("osascript", ["-e", script], { timeout: 1e4 });
+    const result = stdout.trim();
+    if (result === "NOT_FOUND" || !result.includes(",")) return null;
+    return result;
+  } catch {
+    return null;
+  }
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
@@ -2248,16 +2350,17 @@ var DesktopTools = class {
     server.tool(
       "desktop_click",
       [
-        "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
-        "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
-        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript position+size center calculation, NEVER from visual screenshot estimation.",
-        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots.",
-        "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
-        "If query not found: use execute_command + osascript to get element position+size, then center = (x + w/2, y + h/2). Or try desktop_menu, desktop_hotkey.",
-        "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
+        "Click a UI element by text label (query), AX role (role), coordinates (coords), or element ID (on).",
+        "BEST: Use query with app \u2014 auto-resolves coords via accessibility API (L1-L3 depth), then falls back to full tree search for deeply nested elements. No desktop_see needed.",
+        "ROLE: Use role (e.g. 'AXTextArea', 'AXTextField') to find elements without visible text labels (input fields, text areas). Searches L1-L3 depth.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript center calculation (position + size/2). NEVER estimate from screenshots.",
+        "CAUTION: 'on' (element ID) has known offset bug \u2014 use query, role, or coords instead.",
+        "If not found: try desktop_menu for menu items, desktop_hotkey for shortcuts.",
+        "NOTE: Web page elements (inside browser) are invisible \u2014 use browser_* tools instead."
       ].join("\n"),
       {
-        query: z5.string().optional().describe("Text/label to click (case-insensitive)"),
+        query: z5.string().optional().describe("Text/label to click (case-insensitive). Works for deeply nested elements."),
+        role: z5.string().optional().describe("AX role to find (e.g. 'AXTextArea', 'AXTextField', 'AXButton'). Use when element has no text label."),
         on: z5.string().optional().describe("Element ID from desktop_see (e.g. 'B1', 'T2')"),
         coords: z5.string().optional().describe("Screen coordinates 'x,y' (e.g. '500,300')"),
         app: z5.string().optional().describe("App name"),
@@ -2266,12 +2369,19 @@ var DesktopTools = class {
         rightClick: z5.boolean().optional().default(false).describe("Right-click (context menu)"),
         waitFor: z5.number().optional().describe("Max ms to wait for element to appear (default 5000)")
       },
-      async ({ query, on, coords, app, snapshot, doubleClick, rightClick, waitFor }) => {
+      async ({ query, role, on, coords, app, snapshot, doubleClick, rightClick, waitFor }) => {
         checkBlacklist(app);
-        if (!query && !on && !coords) throw new Error("Provide query, on, or coords.");
+        if (!query && !role && !on && !coords) throw new Error("Provide query, role, on, or coords.");
         const args = ["click"];
         if (coords) {
           args.push("--coords", coords);
+        } else if (role) {
+          const resolved = await findFirstByRole(role, app);
+          if (resolved) {
+            args.push("--coords", resolved);
+          } else {
+            throw new Error(`No element with role '${role}' found in ${app ?? "frontmost app"}.`);
+          }
         } else if (query) {
           const resolved = await resolveElementCoords(query, app);
           if (resolved) {
@@ -2294,7 +2404,7 @@ var DesktopTools = class {
       "desktop_type",
       [
         "Type text via keyboard. Supports \\n (return), \\t (tab) escape sequences.",
-        "IMPORTANT: Focus the target field first (click it with desktop_click) before typing. Types at current keyboard focus.",
+        "IMPORTANT: Focus the target field first \u2014 use desktop_click(query, app) for labeled fields, or desktop_click(role:'AXTextArea', app) for unlabeled input areas.",
         "For Korean/Japanese/Chinese/emoji, use desktop_paste instead (keyboard sim is ASCII only).",
         "Use clear=true to replace existing text (Cmd+A \u2192 Delete before typing)."
       ].join("\n"),
@@ -2319,7 +2429,7 @@ var DesktopTools = class {
       "desktop_paste",
       [
         "Paste via clipboard (Cmd+V). Atomic: saves clipboard \u2192 sets content \u2192 pastes \u2192 restores.",
-        "IMPORTANT: Focus the target field first (click it with desktop_click) before pasting.",
+        "IMPORTANT: Focus the target field first \u2014 use desktop_click(query, app) for labeled fields, or desktop_click(role:'AXTextArea', app) for unlabeled input areas.",
         "Supports all Unicode (Korean, Japanese, Chinese, emoji). Use instead of desktop_type for non-ASCII.",
         "Can also paste file contents via filePath."
       ].join("\n"),

package/dist/server/mcp.js

@@ -1653,6 +1653,28 @@ tell application "System Events"
         end repeat
       end try
     end repeat
+    repeat with parent in topElems
+      try
+        repeat with child in UI elements of parent
+          try
+            set childRole to role of child
+            if childRole is not "AXTable" and childRole is not "AXOutline" and childRole is not "AXList" then
+              repeat with gc in UI elements of child
+                try
+                  if (name of gc contains "${safeQuery}") or (description of gc contains "${safeQuery}") then
+                    set pos to position of gc
+                    set sz to size of gc
+                    set cx to (item 1 of pos) + (item 1 of sz) / 2
+                    set cy to (item 2 of pos) + (item 2 of sz) / 2
+                    return ((cx as integer) as text) & "," & ((cy as integer) as text)
+                  end if
+                end try
+              end repeat
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
   end tell
 end tell
 return "NOT_FOUND"
@@ -1686,6 +1708,86 @@ async function resolveElementCoords(query, app) {
   }
   return null;
 }
+async function findFirstByRole(role, app) {
+  let targetApp = app;
+  if (!targetApp) {
+    try {
+      const { stdout } = await execa("osascript", [
+        "-e",
+        'tell application "System Events" to get name of first application process whose frontmost is true'
+      ]);
+      targetApp = stdout.trim();
+    } catch {
+      return null;
+    }
+  }
+  const safeApp = targetApp.replace(/[\\"]/g, "\\$&");
+  const safeRole = role.replace(/[\\"]/g, "\\$&");
+  const script = `
+tell application "System Events"
+  tell process "${safeApp}"
+    set topElems to UI elements
+    repeat with elem in topElems
+      try
+        if role of elem is "${safeRole}" then
+          set pos to position of elem
+          set sz to size of elem
+          set cx to (item 1 of pos) + (item 1 of sz) / 2
+          set cy to (item 2 of pos) + (item 2 of sz) / 2
+          return ((cx as integer) as text) & "," & ((cy as integer) as text)
+        end if
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with elem in UI elements of parent
+          try
+            if role of elem is "${safeRole}" then
+              set pos to position of elem
+              set sz to size of elem
+              set cx to (item 1 of pos) + (item 1 of sz) / 2
+              set cy to (item 2 of pos) + (item 2 of sz) / 2
+              return ((cx as integer) as text) & "," & ((cy as integer) as text)
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+    repeat with parent in topElems
+      try
+        repeat with child in UI elements of parent
+          try
+            set childRole to role of child
+            if childRole is not "AXTable" and childRole is not "AXOutline" and childRole is not "AXList" then
+              repeat with gc in UI elements of child
+                try
+                  if role of gc is "${safeRole}" then
+                    set pos to position of gc
+                    set sz to size of gc
+                    set cx to (item 1 of pos) + (item 1 of sz) / 2
+                    set cy to (item 2 of pos) + (item 2 of sz) / 2
+                    return ((cx as integer) as text) & "," & ((cy as integer) as text)
+                  end if
+                end try
+              end repeat
+            end if
+          end try
+        end repeat
+      end try
+    end repeat
+  end tell
+end tell
+return "NOT_FOUND"
+`;
+  try {
+    const { stdout } = await execa("osascript", ["-e", script], { timeout: 1e4 });
+    const result = stdout.trim();
+    if (result === "NOT_FOUND" || !result.includes(",")) return null;
+    return result;
+  } catch {
+    return null;
+  }
+}
 var DesktopTools = class {
   register(server) {
     server.tool(
@@ -1764,16 +1866,17 @@ var DesktopTools = class {
     server.tool(
       "desktop_click",
       [
-        "Click a UI element by text label (query), coordinates (coords), or element ID (on).",
-        "BEST: Use query with app name \u2014 auto-resolves to exact screen coords via accessibility API. No desktop_see needed.",
-        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript position+size center calculation, NEVER from visual screenshot estimation.",
-        "Before using coords, always query exact position via execute_command + osascript. Calculate center = position + size/2. Never estimate from screenshots.",
-        "CAUTION: 'on' (element ID) has known offset bug \u2014 use coords or query instead when possible.",
-        "If query not found: use execute_command + osascript to get element position+size, then center = (x + w/2, y + h/2). Or try desktop_menu, desktop_hotkey.",
-        "NOTE: Web page elements (inside browser) are invisible to desktop tools. Use browser_* tools instead."
+        "Click a UI element by text label (query), AX role (role), coordinates (coords), or element ID (on).",
+        "BEST: Use query with app \u2014 auto-resolves coords via accessibility API (L1-L3 depth), then falls back to full tree search for deeply nested elements. No desktop_see needed.",
+        "ROLE: Use role (e.g. 'AXTextArea', 'AXTextField') to find elements without visible text labels (input fields, text areas). Searches L1-L3 depth.",
+        "GOOD: Use coords 'x,y' \u2014 MUST be from osascript center calculation (position + size/2). NEVER estimate from screenshots.",
+        "CAUTION: 'on' (element ID) has known offset bug \u2014 use query, role, or coords instead.",
+        "If not found: try desktop_menu for menu items, desktop_hotkey for shortcuts.",
+        "NOTE: Web page elements (inside browser) are invisible \u2014 use browser_* tools instead."
       ].join("\n"),
       {
-        query: z5.string().optional().describe("Text/label to click (case-insensitive)"),
+        query: z5.string().optional().describe("Text/label to click (case-insensitive). Works for deeply nested elements."),
+        role: z5.string().optional().describe("AX role to find (e.g. 'AXTextArea', 'AXTextField', 'AXButton'). Use when element has no text label."),
         on: z5.string().optional().describe("Element ID from desktop_see (e.g. 'B1', 'T2')"),
         coords: z5.string().optional().describe("Screen coordinates 'x,y' (e.g. '500,300')"),
         app: z5.string().optional().describe("App name"),
@@ -1782,12 +1885,19 @@ var DesktopTools = class {
         rightClick: z5.boolean().optional().default(false).describe("Right-click (context menu)"),
         waitFor: z5.number().optional().describe("Max ms to wait for element to appear (default 5000)")
       },
-      async ({ query, on, coords, app, snapshot, doubleClick, rightClick, waitFor }) => {
+      async ({ query, role, on, coords, app, snapshot, doubleClick, rightClick, waitFor }) => {
         checkBlacklist(app);
-        if (!query && !on && !coords) throw new Error("Provide query, on, or coords.");
+        if (!query && !role && !on && !coords) throw new Error("Provide query, role, on, or coords.");
         const args = ["click"];
         if (coords) {
           args.push("--coords", coords);
+        } else if (role) {
+          const resolved = await findFirstByRole(role, app);
+          if (resolved) {
+            args.push("--coords", resolved);
+          } else {
+            throw new Error(`No element with role '${role}' found in ${app ?? "frontmost app"}.`);
+          }
         } else if (query) {
           const resolved = await resolveElementCoords(query, app);
           if (resolved) {
@@ -1810,7 +1920,7 @@ var DesktopTools = class {
       "desktop_type",
       [
         "Type text via keyboard. Supports \\n (return), \\t (tab) escape sequences.",
-        "IMPORTANT: Focus the target field first (click it with desktop_click) before typing. Types at current keyboard focus.",
+        "IMPORTANT: Focus the target field first \u2014 use desktop_click(query, app) for labeled fields, or desktop_click(role:'AXTextArea', app) for unlabeled input areas.",
         "For Korean/Japanese/Chinese/emoji, use desktop_paste instead (keyboard sim is ASCII only).",
         "Use clear=true to replace existing text (Cmd+A \u2192 Delete before typing)."
       ].join("\n"),
@@ -1835,7 +1945,7 @@ var DesktopTools = class {
       "desktop_paste",
       [
         "Paste via clipboard (Cmd+V). Atomic: saves clipboard \u2192 sets content \u2192 pastes \u2192 restores.",
-        "IMPORTANT: Focus the target field first (click it with desktop_click) before pasting.",
+        "IMPORTANT: Focus the target field first \u2014 use desktop_click(query, app) for labeled fields, or desktop_click(role:'AXTextArea', app) for unlabeled input areas.",
         "Supports all Unicode (Korean, Japanese, Chinese, emoji). Use instead of desktop_type for non-ASCII.",
         "Can also paste file contents via filePath."
       ].join("\n"),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "junis",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "One-line device control for AI agents",
   "type": "module",
   "bin": {