arena-bridge 0.6.3 → 0.6.5

package/CHANGELOG.md

@@ -4,6 +4,25 @@ All notable changes to this project are documented here.
 The format follows [Keep a Changelog](https://keepachangelog.com/) and the project uses
 [Semantic Versioning](https://semver.org/).
 
+## 0.6.5
+
+### Fixed
+- Peekaboo 4.9 compatibility: `peekaboo list` was removed in v4, so `mac_apps` now uses
+  `app list` and `mac_windows` uses `window list` (enumerating the running apps when no app is
+  given). `mac_menu` / `mac_menu_click` accept a process id, run against the local host
+  (`--no-remote`) and read the `menu_structure` payload; `mac_spaces` / `mac_space_switch` also
+  run locally, because a remote host cannot read or switch Spaces.
+
+## 0.6.4
+
+### Fixed
+- The accessibility element map is read from Peekaboo's real payload (`data.ui_elements`), so
+  `mac_see` and `mac_element` now return the controls of a window instead of an empty list.
+  Elements carry their label, description, identifier and value-settable flag.
+- Commands follow the Peekaboo v4 grammar: `set-value "<value>" --on <id>`, `menu click --item|--path`
+  with an explicit app, `space switch --to <n> --foreground`, and the tree-only read
+  `see --tree --no-screenshot --json` (no screenshot artifact for pure lookups).
+
 ## 0.6.3
 
 ### Added

package/README.md

@@ -12,7 +12,7 @@
   <img src="https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square" alt="license">
   <img src="https://img.shields.io/badge/node-%3E%3D18-green.svg?style=flat-square" alt="node">
   <img src="https://img.shields.io/badge/dependencies-0-brightgreen.svg?style=flat-square" alt="dependencies">
-  <img src="https://img.shields.io/badge/tests-110%20passed-brightgreen.svg?style=flat-square" alt="tests">
+  <img src="https://img.shields.io/badge/tests-112%20passed-brightgreen.svg?style=flat-square" alt="tests">
 </p>
 
 <p align="center">
@@ -150,13 +150,14 @@ cookies и localStorage нет.
 {"tool": "mac_set", "args": {"on": "B2", "value": "hello"}}      // вписать значение напрямую
 {"tool": "mac_type", "args": {"text": "hello", "submit": true}}
 {"tool": "mac_key", "args": {"key": "cmd+s"}}
-{"tool": "mac_menu_click", "args": {"item": "Save", "app": "Safari"}}  // пункт меню по названию
+{"tool": "mac_menu_click", "args": {"app": "Safari", "item": "Save"}}  // пункт меню по названию
 {"tool": "mac_wait", "args": {"name": "Saved", "timeout": 20}}        // дождаться результата
 {"tool": "mac_apps", "args": {}}                                  // что запущено
 {"tool": "mac_permissions", "args": {}}                           // чего не хватает в macOS
 ```
 
 Установить Peekaboo: `brew install peekaboo` (или бинарник в `/usr/local/bin/peekaboo`).
+Проверено с Peekaboo 4.9 (4.3 тоже работает, но синтаксис команд обновлён под 4.9).
 macOS спросит разрешения **Screen Recording** и **Accessibility** для терминала, из которого
 запущен мост — без них скриншоты и клики не работают; `mac_permissions` покажет, что не так.
 
@@ -172,6 +173,7 @@ macOS отдаёт приложениям два разных слоя: карт
 | как вписать текст | свайп, фокус, клавиатура | `mac_set {"on":"<id>","value":"..."}` — без фокуса и без клавиш |
 | как открыть «Файл → Экспорт» | перемещать курсор по меню | `mac_menu_click {"item":"Экспорт"}` — по названию |
 | как убедиться, что получилось | смотреть скриншот | `mac_wait {"name":"Готово"}` — ждать появления |
+| как заполнить поле | свайп + клавиатура + фокус | `mac_set {"on":"<id>","value":"..."}` — значение пишется в элемент |
 | чужой ввод и фокус | мешает вам | действия уходят в фоновое окно |
 
 Практический вывод: с выданным Accessibility агент работает заметно точнее и тише. Разрешение

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arena-bridge",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "description": "Give a cloud AI agent (Arena.ai Agent Mode, Claude, Cursor, ...) controlled access to your own machine over HTTP/MCP: files, shell, Chrome and the macOS desktop, with an approval gate, trust windows and an audit log.",
   "type": "module",
   "bin": {

package/src/direct.mjs

@@ -29,7 +29,7 @@ export function createDirectServer(cfg, engine) {
           ok: true,
           mode: 'direct',
           device: cfg.device,
-          version: '0.6.3',
+          version: '0.6.5',
           uptime_sec: Math.round(process.uptime()),
         });
       }

package/src/hub.mjs

@@ -90,7 +90,7 @@ export function createHubServer(cfg) {
         return sendJson(res, 200, {
           ok: true,
           mode: 'hub',
-          version: '0.6.3',
+          version: '0.6.5',
           devices: [...devices.keys()],
         });
       }

package/src/mac.mjs

@@ -100,21 +100,31 @@ function parseJson(text) {
 }
 
 function collectElements(node, out = [], depth = 0) {
-  if (!node || depth > 6) return out;
+  if (!node || depth > 8) return out;
   if (Array.isArray(node)) {
     for (const n of node) collectElements(n, out, depth + 1);
     return out;
   }
   if (typeof node !== 'object') return out;
-  if (node.elements && Array.isArray(node.elements)) {
-    for (const e of node.elements) {
+  for (const key of ['ui_elements', 'elements']) {
+    const list = node[key];
+    if (!Array.isArray(list) || !list.length) continue;
+    for (const e of list) {
+      if (!e || typeof e !== 'object') continue;
+      const value = e.value;
       out.push({
-        id: e.id ?? e.elementId ?? e.element_id ?? null,
-        role: e.role ?? e.elementRole ?? e.type ?? null,
+        id: e.id ?? e.element_id ?? e.elementId ?? null,
+        role: e.role ?? e.ax_role ?? e.elementRole ?? e.type ?? null,
         title: e.title ?? e.label ?? e.name ?? e.placeholder ?? null,
-        value: typeof e.value === 'string' ? e.value.slice(0, 200) : null,
-        bounds: e.frame ?? e.bounds ?? e.rect ?? null,
-        enabled: e.enabled ?? null,
+        label: e.label ?? null,
+        description: e.description ?? null,
+        identifier: e.identifier ?? null,
+        value: typeof value === 'string' ? value.slice(0, 200) : (value ?? null),
+        bounds: e.bounds ?? e.frame ?? e.rect ?? null,
+        enabled: e.is_enabled ?? e.enabled ?? null,
+        actionable: e.is_actionable ?? null,
+        settable: e.is_value_settable ?? null,
+        shortcut: e.keyboard_shortcut ?? null,
       });
     }
     return out;
@@ -178,10 +188,18 @@ export function macTools(cfg = {}) {
   const seeVariants = (args, outPath) => {
     const mode = args.mode === 'window' ? 'window' : 'screen';
     const base = ['see', '--mode', mode, ...targetFlags(args)];
+    if (!outPath) {
+      return [
+        ['see', '--tree', '--no-screenshot', '--json', ...targetFlags(args)],
+        ['see', '--tree', '--no-screenshot', '--no-remote', '--json', ...targetFlags(args)],
+        ['see', '--tree', '--json', ...targetFlags(args)],
+        ['see', '--json', ...targetFlags(args)],
+      ];
+    }
     return [
-      [...base, '--json', '--path', outPath, '--snapshot'],
-      [...base, '--json', '--path', outPath, '--tree'],
       [...base, '--json', '--path', outPath],
+      [...base, '--path', outPath, '--json'],
+      [...base, '--no-remote', '--json', '--path', outPath],
       ['image', '--mode', mode, ...targetFlags(args), '--json', '--path', outPath],
     ];
   };
@@ -202,7 +220,10 @@ export function macTools(cfg = {}) {
 
   const matchElement = (e, args) => {
     const q = (v) => String(v ?? '').toLowerCase();
-    if (args.name && !q(e.title).includes(q(args.name)) && !q(e.id).includes(q(args.name)) && !q(e.value).includes(q(args.name))) return false;
+    const text = [e.title, e.label, e.description, e.identifier, e.value, e.id]
+      .map(q)
+      .join('\n');
+    if (args.name && !text.includes(q(args.name))) return false;
     if (args.role && !q(e.role).includes(q(args.role))) return false;
     if (args.value && !q(e.value).includes(q(args.value))) return false;
     return true;
@@ -210,11 +231,8 @@ export function macTools(cfg = {}) {
 
   const searchPath = () => path.join(os.tmpdir(), 'arena-bridge-see', 'see.png');
 
-  const searchSee = async (args) => {
-    const out = searchPath();
-    ensureDir(path.dirname(out));
-    return collectSee({ app: args.app, window_title: args.window_title, mode: args.mode || 'screen' }, out);
-  };
+  const searchSee = (args) =>
+    collectSee({ app: args.app, window_title: args.window_title, mode: args.mode || 'screen' }, null);
 
   return [
     {
@@ -451,9 +469,9 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
-            ['list', 'apps', '--json'],
             ['app', 'list', '--json'],
-            ['list', 'apps'],
+            ['list', 'apps', '--json'],
+            ['app', 'list'],
           ],
           { timeout: 30 },
         );
@@ -476,12 +494,16 @@ export function macTools(cfg = {}) {
         const p = checkPeek(peek());
         if (!p.ok) return p;
         const flags = args.app ? ['--app', String(args.app)] : [];
+        if (!flags.length) {
+          const windows = await windowsOfAllApps(p.bin);
+          return { ok: true, windows, source: 'per-app', hint: 'Pass app or pid to narrow the list down.' };
+        }
         const res = await runVariants(
           p.bin,
           [
-            ['list', 'windows', ...flags, '--json'],
             ['window', 'list', ...flags, '--json'],
-            ['list', 'windows', ...flags],
+            ['list', 'windows', ...flags, '--json'],
+            ['window', 'list', ...flags],
           ],
           { timeout: 30 },
         );
@@ -607,9 +629,10 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
+            ['set-value', String(args.value), '--on', String(args.on), ...flags],
+            ['set-value', String(args.value), '--on', String(args.on)],
             ['set-value', '--on', String(args.on), String(args.value), ...flags],
             ['set-value', '--on', String(args.on), '--value', String(args.value), ...flags],
-            ['set-value', String(args.on), String(args.value), ...flags],
           ],
           { timeout: 30 },
         );
@@ -625,10 +648,11 @@ export function macTools(cfg = {}) {
       handler: async (args) => {
         const p = checkPeek(peek());
         if (!p.ok) return p;
-        const flags = args.app ? ['--app', String(args.app)] : [];
+        const flags = menuFlags(args);
         const res = await runVariants(
           p.bin,
           [
+            ['menu', 'list', ...flags, '--no-remote', '--json'],
             ['menu', 'list', ...flags, '--json'],
             ['menu', 'list', ...flags],
           ],
@@ -637,30 +661,52 @@ export function macTools(cfg = {}) {
         const json = parseJson(res.stdout);
         return {
           ok: res.exit_code === 0,
-          menus: json ? (json.menus || json.data?.menus || json) : undefined,
+          menus: json ? (json.menus || json.data?.menus || json.data?.menu_structure || json) : undefined,
           output: json ? undefined : res.stdout.slice(0, 4000),
           error: res.exit_code === 0 ? undefined : (res.stderr || '').trim().slice(0, 600),
+          hint: res.exit_code === 0 ? undefined : menuHint(res),
         };
       },
     },
     {
       name: 'mac_menu_click',
-      description: 'Click a menu item by its visible name (e.g. "Save", "Copy Link") — no pointer, no coordinates.',
+      description:
+        'Trigger a menu item by name (e.g. "Save", "Copy Link") or a nested path ("File > Export") — no pointer, no coordinates.',
       approval: true,
-      schema: S({ item: { type: 'string' }, app: { type: 'string' } }, ['item']),
-      required: ['item'],
+      schema: S(
+        {
+          item: { type: 'string', description: 'name of the menu item' },
+          path: { type: 'string', description: 'nested path, e.g. "File > Export"' },
+          app: { type: 'string', description: 'required: the background menu click needs a target app' },
+          pid: { type: 'number', description: 'process id, when the app name is ambiguous' },
+        },
+        [],
+      ),
+      required: [],
       handler: async (args) => {
         const p = checkPeek(peek());
         if (!p.ok) return p;
-        const flags = args.app ? ['--app', String(args.app)] : [];
-        const res = await runVariants(
-          p.bin,
-          [
-            ['menu', 'click', '--item', String(args.item), ...flags],
-            ['menu', 'click', String(args.item), ...flags],
-          ],
-          { timeout: 30 },
-        );
+        if (!args.item && !args.path) {
+          return { ok: false, error: 'missing_item', hint: 'Pass item ("Save") or path ("File > Export").' };
+        }
+        if (!args.app && !args.pid) {
+          return {
+            ok: false,
+            error: 'missing_app',
+            hint: 'A background menu click needs an explicit app: mac_menu_click {"app":"Safari","item":"Save"}.',
+          };
+        }
+        const flags = menuFlags(args);
+        const variants = args.path
+          ? [
+              ['menu', 'click', '--path', String(args.path), ...flags, '--no-remote'],
+              ['menu', 'click', '--path', String(args.path), ...flags],
+            ]
+          : [
+              ['menu', 'click', '--item', String(args.item), ...flags, '--no-remote'],
+              ['menu', 'click', '--item', String(args.item), ...flags],
+            ];
+        const res = await runVariants(p.bin, variants, { timeout: 30 });
         return finish(res, 'menu-click');
       },
     },
@@ -676,6 +722,7 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
+            ['space', 'list', '--no-remote', '--json'],
             ['space', 'list', '--json'],
             ['space', 'list'],
           ],
@@ -703,8 +750,8 @@ export function macTools(cfg = {}) {
         const res = await runVariants(
           p.bin,
           [
-            ['space', 'switch', '--to', String(args.to)],
-            ['space', 'switch', String(args.to)],
+            ['space', 'switch', '--to', String(args.to), '--foreground', '--no-remote'],
+            ['space', 'switch', '--to', String(args.to), '--foreground'],
           ],
           { timeout: 30 },
         );
@@ -784,6 +831,43 @@ export function macTools(cfg = {}) {
   ];
 }
 
+function menuFlags(args) {
+  const f = [];
+  if (args.app) f.push('--app', String(args.app));
+  if (args.pid) f.push('--pid', String(args.pid));
+  return f;
+}
+
+function menuHint(res) {
+  const text = (res.stderr || res.stdout || '').trim();
+  if (/multiple apps match/i.test(text)) {
+    return 'Several apps share that name — pass the process id from mac_apps: mac_menu {"pid":12345}.';
+  }
+  return text.slice(0, 600);
+}
+
+async function windowsOfAllApps(bin) {
+  const res = await runVariants(
+    bin,
+    [
+      ['app', 'list', '--json'],
+      ['list', 'apps', '--json'],
+    ],
+    { timeout: 30 },
+  );
+  const json = parseJson(res.stdout);
+  const apps = json ? (json.apps || json.data?.apps || []) : [];
+  const out = [];
+  for (const app of Array.isArray(apps) ? apps.slice(0, 6) : []) {
+    if (!app || !app.pid) continue;
+    const w = await run(bin, ['window', 'list', '--pid', String(app.pid), '--json'], { timeout: 20 });
+    const data = parseJson(w.stdout);
+    const list = data ? (data.windows || data.data?.windows || []) : [];
+    for (const win of Array.isArray(list) ? list : []) out.push({ ...win, app: app.name, pid: app.pid });
+  }
+  return out;
+}
+
 function checkPeek(p) {
   if (!p) return { ok: false, error: 'peekaboo_not_found', hint: NOT_FOUND_HINT };
   return { ok: true, bin: p };

package/src/mcp.mjs

@@ -48,7 +48,7 @@ export async function handleMcp(req, res, ctx) {
           mcpResponse(id, {
             protocolVersion: PROTOCOL_VERSION,
             capabilities: { tools: { listChanged: false } },
-            serverInfo: { name: 'arena-bridge', version: '0.6.3' },
+            serverInfo: { name: 'arena-bridge', version: '0.6.5' },
             instructions:
               'Tools operate the user\'s own machine. Dangerous operations may return ' +
               'PROTECTED and require a local human approval.',

package/src/prompt.mjs

@@ -92,7 +92,7 @@ ${tools.join('\n')}
 6. Stay out of the user's way: do not pass \`foreground\`, do not focus windows and do not move the
    pointer unless the user asks for it. Background actions are delivered to the target window only.
 7. \`mac_set\` writes a value straight into a control (no keystrokes, no focus change);
-   \`mac_menu_click\` triggers a menu item by its name — both are more reliable than typing.
+   \`mac_menu_click\` triggers a menu item by its name (it needs the \`app\` name) — both are more reliable than typing.
 8. \`mac_wait\` waits for a control to appear or disappear instead of sleeping and guessing.
 9. \`mac_spaces\` lists the macOS virtual desktops; \`mac_space_switch\` moves the agent to another
    Space so the user keeps the screen they are looking at.