arena-bridge 0.6.3 → 0.6.4

package/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project are documented here.
 The format follows [Keep a Changelog](https://keepachangelog.com/) and the project uses
 [Semantic Versioning](https://semver.org/).
 
+## 0.6.4
+
+### Fixed
+- The accessibility element map is read from Peekaboo's real payload (`data.ui_elements`), so
+  `mac_see` and `mac_element` now return the controls of a window instead of an empty list.
+  Elements carry their label, description, identifier and value-settable flag.
+- Commands follow the Peekaboo v4 grammar: `set-value "<value>" --on <id>`, `menu click --item|--path`
+  with an explicit app, `space switch --to <n> --foreground`, and the tree-only read
+  `see --tree --no-screenshot --json` (no screenshot artifact for pure lookups).
+
 ## 0.6.3
 
 ### Added

package/README.md

@@ -150,7 +150,7 @@ cookies и localStorage нет.
 {"tool": "mac_set", "args": {"on": "B2", "value": "hello"}}      // вписать значение напрямую
 {"tool": "mac_type", "args": {"text": "hello", "submit": true}}
 {"tool": "mac_key", "args": {"key": "cmd+s"}}
-{"tool": "mac_menu_click", "args": {"item": "Save", "app": "Safari"}}  // пункт меню по названию
+{"tool": "mac_menu_click", "args": {"app": "Safari", "item": "Save"}}  // пункт меню по названию
 {"tool": "mac_wait", "args": {"name": "Saved", "timeout": 20}}        // дождаться результата
 {"tool": "mac_apps", "args": {}}                                  // что запущено
 {"tool": "mac_permissions", "args": {}}                           // чего не хватает в macOS
@@ -172,6 +172,7 @@ macOS отдаёт приложениям два разных слоя: карт
 | как вписать текст | свайп, фокус, клавиатура | `mac_set {"on":"<id>","value":"..."}` — без фокуса и без клавиш |
 | как открыть «Файл → Экспорт» | перемещать курсор по меню | `mac_menu_click {"item":"Экспорт"}` — по названию |
 | как убедиться, что получилось | смотреть скриншот | `mac_wait {"name":"Готово"}` — ждать появления |
+| как заполнить поле | свайп + клавиатура + фокус | `mac_set {"on":"<id>","value":"..."}` — значение пишется в элемент |
 | чужой ввод и фокус | мешает вам | действия уходят в фоновое окно |
 
 Практический вывод: с выданным Accessibility агент работает заметно точнее и тише. Разрешение

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arena-bridge",
-  "version": "0.6.3",
+  "version": "0.6.4",
   "description": "Give a cloud AI agent (Arena.ai Agent Mode, Claude, Cursor, ...) controlled access to your own machine over HTTP/MCP: files, shell, Chrome and the macOS desktop, with an approval gate, trust windows and an audit log.",
   "type": "module",
   "bin": {

package/src/direct.mjs

@@ -29,7 +29,7 @@ export function createDirectServer(cfg, engine) {
           ok: true,
           mode: 'direct',
           device: cfg.device,
-          version: '0.6.3',
+          version: '0.6.4',
           uptime_sec: Math.round(process.uptime()),
         });
       }

package/src/hub.mjs

@@ -90,7 +90,7 @@ export function createHubServer(cfg) {
         return sendJson(res, 200, {
           ok: true,
           mode: 'hub',
-          version: '0.6.3',
+          version: '0.6.4',
           devices: [...devices.keys()],
         });
       }

package/src/mac.mjs

@@ -100,21 +100,31 @@ function parseJson(text) {
 }
 
 function collectElements(node, out = [], depth = 0) {
-  if (!node || depth > 6) return out;
+  if (!node || depth > 8) return out;
   if (Array.isArray(node)) {
     for (const n of node) collectElements(n, out, depth + 1);
     return out;
   }
   if (typeof node !== 'object') return out;
-  if (node.elements && Array.isArray(node.elements)) {
-    for (const e of node.elements) {
+  for (const key of ['ui_elements', 'elements']) {
+    const list = node[key];
+    if (!Array.isArray(list) || !list.length) continue;
+    for (const e of list) {
+      if (!e || typeof e !== 'object') continue;
+      const value = e.value;
       out.push({
-        id: e.id ?? e.elementId ?? e.element_id ?? null,
-        role: e.role ?? e.elementRole ?? e.type ?? null,
+        id: e.id ?? e.element_id ?? e.elementId ?? null,
+        role: e.role ?? e.ax_role ?? e.elementRole ?? e.type ?? null,
         title: e.title ?? e.label ?? e.name ?? e.placeholder ?? null,
-        value: typeof e.value === 'string' ? e.value.slice(0, 200) : null,
-        bounds: e.frame ?? e.bounds ?? e.rect ?? null,
-        enabled: e.enabled ?? null,
+        label: e.label ?? null,
+        description: e.description ?? null,
+        identifier: e.identifier ?? null,
+        value: typeof value === 'string' ? value.slice(0, 200) : (value ?? null),
+        bounds: e.bounds ?? e.frame ?? e.rect ?? null,
+        enabled: e.is_enabled ?? e.enabled ?? null,
+        actionable: e.is_actionable ?? null,
+        settable: e.is_value_settable ?? null,
+        shortcut: e.keyboard_shortcut ?? null,
       });
     }
     return out;
@@ -178,10 +188,16 @@ export function macTools(cfg = {}) {
   const seeVariants = (args, outPath) => {
     const mode = args.mode === 'window' ? 'window' : 'screen';
     const base = ['see', '--mode', mode, ...targetFlags(args)];
+    if (!outPath) {
+      return [
+        ['see', '--tree', '--no-screenshot', '--json', ...targetFlags(args)],
+        ['see', '--tree', '--json', ...targetFlags(args)],
+        ['see', '--json', ...targetFlags(args)],
+      ];
+    }
     return [
-      [...base, '--json', '--path', outPath, '--snapshot'],
-      [...base, '--json', '--path', outPath, '--tree'],
       [...base, '--json', '--path', outPath],
+      [...base, '--path', outPath, '--json'],
       ['image', '--mode', mode, ...targetFlags(args), '--json', '--path', outPath],
     ];
   };
@@ -202,7 +218,10 @@ export function macTools(cfg = {}) {
 
   const matchElement = (e, args) => {
     const q = (v) => String(v ?? '').toLowerCase();
-    if (args.name && !q(e.title).includes(q(args.name)) && !q(e.id).includes(q(args.name)) && !q(e.value).includes(q(args.name))) return false;
+    const text = [e.title, e.label, e.description, e.identifier, e.value, e.id]
+      .map(q)
+      .join('\n');
+    if (args.name && !text.includes(q(args.name))) return false;
     if (args.role && !q(e.role).includes(q(args.role))) return false;
     if (args.value && !q(e.value).includes(q(args.value))) return false;
     return true;
@@ -210,11 +229,8 @@ export function macTools(cfg = {}) {
 
   const searchPath = () => path.join(os.tmpdir(), 'arena-bridge-see', 'see.png');
 
-  const searchSee = async (args) => {
-    const out = searchPath();
-    ensureDir(path.dirname(out));
-    return collectSee({ app: args.app, window_title: args.window_title, mode: args.mode || 'screen' }, out);
-  };
+  const searchSee = (args) =>
+    collectSee({ app: args.app, window_title: args.window_title, mode: args.mode || 'screen' }, null);
 
   return [
     {
@@ -607,9 +623,10 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
+            ['set-value', String(args.value), '--on', String(args.on), ...flags],
+            ['set-value', String(args.value), '--on', String(args.on)],
             ['set-value', '--on', String(args.on), String(args.value), ...flags],
             ['set-value', '--on', String(args.on), '--value', String(args.value), ...flags],
-            ['set-value', String(args.on), String(args.value), ...flags],
           ],
           { timeout: 30 },
         );
@@ -645,22 +662,42 @@ export function macTools(cfg = {}) {
     },
     {
       name: 'mac_menu_click',
-      description: 'Click a menu item by its visible name (e.g. "Save", "Copy Link") — no pointer, no coordinates.',
+      description:
+        'Trigger a menu item by name (e.g. "Save", "Copy Link") or a nested path ("File > Export") — no pointer, no coordinates.',
       approval: true,
-      schema: S({ item: { type: 'string' }, app: { type: 'string' } }, ['item']),
-      required: ['item'],
+      schema: S(
+        {
+          item: { type: 'string', description: 'name of the menu item' },
+          path: { type: 'string', description: 'nested path, e.g. "File > Export"' },
+          app: { type: 'string', description: 'required: the background menu click needs a target app' },
+        },
+        [],
+      ),
+      required: [],
       handler: async (args) => {
         const p = checkPeek(peek());
         if (!p.ok) return p;
-        const flags = args.app ? ['--app', String(args.app)] : [];
-        const res = await runVariants(
-          p.bin,
-          [
-            ['menu', 'click', '--item', String(args.item), ...flags],
-            ['menu', 'click', String(args.item), ...flags],
-          ],
-          { timeout: 30 },
-        );
+        if (!args.item && !args.path) {
+          return { ok: false, error: 'missing_item', hint: 'Pass item ("Save") or path ("File > Export").' };
+        }
+        if (!args.app) {
+          return {
+            ok: false,
+            error: 'missing_app',
+            hint: 'A background menu click needs an explicit app: mac_menu_click {"app":"Safari","item":"Save"}.',
+          };
+        }
+        const flags = ['--app', String(args.app)];
+        const variants = args.path
+          ? [
+              ['menu', 'click', '--path', String(args.path), ...flags],
+              ['menu', 'click', String(args.path), ...flags],
+            ]
+          : [
+              ['menu', 'click', '--item', String(args.item), ...flags],
+              ['menu', 'click', String(args.item), ...flags],
+            ];
+        const res = await runVariants(p.bin, variants, { timeout: 30 });
         return finish(res, 'menu-click');
       },
     },
@@ -703,8 +740,8 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
-            ['space', 'switch', '--to', String(args.to)],
-            ['space', 'switch', String(args.to)],
+            ['space', 'switch', '--to', String(args.to), '--foreground'],
+            ['space', 'switch', String(args.to), '--foreground'],
           ],
           { timeout: 30 },
         );

package/src/mcp.mjs

@@ -48,7 +48,7 @@ export async function handleMcp(req, res, ctx) {
           mcpResponse(id, {
             protocolVersion: PROTOCOL_VERSION,
             capabilities: { tools: { listChanged: false } },
-            serverInfo: { name: 'arena-bridge', version: '0.6.3' },
+            serverInfo: { name: 'arena-bridge', version: '0.6.4' },
             instructions:
               'Tools operate the user\'s own machine. Dangerous operations may return ' +
               'PROTECTED and require a local human approval.',

package/src/prompt.mjs

@@ -92,7 +92,7 @@ ${tools.join('\n')}
 6. Stay out of the user's way: do not pass \`foreground\`, do not focus windows and do not move the
    pointer unless the user asks for it. Background actions are delivered to the target window only.
 7. \`mac_set\` writes a value straight into a control (no keystrokes, no focus change);
-   \`mac_menu_click\` triggers a menu item by its name — both are more reliable than typing.
+   \`mac_menu_click\` triggers a menu item by its name (it needs the \`app\` name) — both are more reliable than typing.
 8. \`mac_wait\` waits for a control to appear or disappear instead of sleeping and guessing.
 9. \`mac_spaces\` lists the macOS virtual desktops; \`mac_space_switch\` moves the agent to another
    Space so the user keeps the screen they are looking at.