@bivy/bivy 0.17.3-staging.30 → 0.17.3-staging.31

package/bin/bivy.mjs

@@ -2440,6 +2440,7 @@ async function cmdApp(args = []) {
        bivy app scroll <x> <y> [--up|--down|--left|--right] [--steps <n>] [--app <app>]
        bivy app drag <x> <y> <to-x> <to-y> [--app <app>]
        bivy app move <x> <y> [--app <app>]
+       bivy app menu ["File > Save"] [--app <app>]   macOS: list the menu bar, or choose an item
 
 Manifest: {"version":1,"name":"My app","views":[
   {"kind":"web","name":"Website","source":{"kind":"service","port":3000}},
@@ -2469,6 +2470,8 @@ app (default: the one opened last). Keys: a character or enter, tab, escape,
 backspace, delete, arrows (up…), home, end, pageup, pagedown, f1–f12, with
 modifiers shift, ctrl, alt/option, cmd. On a Mac, the pointer and keyboard
 focus move to the app while it's used.
+"menu" (macOS) lists the app's menu bar with shortcuts; "menu 'File > Save'"
+chooses an item — more reliable than clicking, and prints a screenshot after.
 "present" says a change is ready for the user to look at: they get a card in
 the chat with the app at phone width (and before/after), and it opens the live
 preview. Use it when you finish a visible change, or when the user asks to see
@@ -2483,6 +2486,7 @@ Web previews require operator setup; see docs/apps.md.`);
     return;
   }
   if (INPUT_ACTIONS[action]) return appInput(action, rest);
+  if (action === "menu") return appMenu(rest);
   if (!["publish", "list", "remove", "shot", "run", "present", "share"].includes(action)) throw new Error("Unknown app command. Run bivy app --help.");
   // `run -- <command> [args…]`: publish a one-window desktop app without a manifest file.
   let runManifest;
@@ -2546,7 +2550,39 @@ Web previews require operator setup; see docs/apps.md.`);
   await appRequest(action === "run" ? "publish" : action, body);
 }
 
-async function appRequest(action, body) {
+async function appMenu(rest) {
+  const options = {};
+  const positional = [];
+  for (let i = 0; i < rest.length; i++) {
+    if (rest[i] === "--app" || rest[i] === "--session") {
+      const value = rest[++i];
+      if (value === undefined || value.startsWith("--")) throw new Error(`${rest[i - 1]} requires a value.`);
+      options[rest[i - 1]] = value;
+    } else if (rest[i].startsWith("--")) throw new Error(`Unknown option ${rest[i]}. Run bivy app --help.`);
+    else positional.push(rest[i]);
+  }
+  if (positional.length > 1) throw new Error('Usage: bivy app menu ["File > Save"] [--app <app>]');
+  const sessionId = resolveAttachSessionId({ sessionFlag: options["--session"], env: process.env });
+  if (!sessionId) throw new Error("Set --session <id> or run inside a Bivy agent session.");
+  const body = { sessionId, ...(options["--app"] ? { target: options["--app"] } : {}) };
+  if (positional[0]) body.path = positional[0].split(">").map((part) => part.trim());
+  const result = await appRequest("menu", body, { print: false });
+  if (!result.menus) { console.log(JSON.stringify(result, null, 2)); return; }
+  // A tree an agent can read: one item per line, indented by level.
+  const lines = [];
+  const walk = (items, depth) => {
+    for (const item of items) {
+      if (item.separator) continue;
+      const note = [item.shortcut, item.checked && "checked", item.enabled === false && "disabled"].filter(Boolean).join(", ");
+      lines.push(`${"  ".repeat(depth)}${item.title}${note ? `  (${note})` : ""}`);
+      if (item.items) walk(item.items, depth + 1);
+    }
+  };
+  for (const menu of result.menus) { lines.push(menu.title); walk(menu.items, 1); }
+  console.log(lines.length ? `${lines.join("\n")}\n\nChoose one with: bivy app menu "<Menu> > <Item>"` : "This app has no menus.");
+}
+
+async function appRequest(action, body, { print = true } = {}) {
   const config = loadConfig();
   if (!(await ensureNodeRunning(config))) throw new Error("Could not reach the Bivy node.");
   const token = await localDeviceToken(config);
@@ -2561,7 +2597,8 @@ async function appRequest(action, body) {
     console.error("Anyone with this link can use the app, including its live backend, until it expires or you revoke it (Apps → Revoke access).");
     return;
   }
-  console.log(JSON.stringify(result, null, 2));
+  if (print) console.log(JSON.stringify(result, null, 2));
+  return result;
 }
 
 // Computer use in desktop apps: each command's positional arguments, and the

package/dist/apps/display-viewer.js

@@ -2,6 +2,7 @@
 // Copyright (c) 2026 Petter André Sjulstad
 export const DISPLAY_SOCKET_PATH = "/__bivy/display";
 export const DISPLAY_STATS_PATH = "/__bivy/display-stats";
+export const DISPLAY_MENU_PATH = "/__bivy/display-menu";
 export const NOVNC_PATH = "/__bivy/novnc/";
 /** The modifier apps paste with, as a keysym and noVNC key code: ⌘ on a Mac. */
 const PASTE_MODIFIER = { darwin: [0xffeb, "MetaLeft"] };
@@ -9,7 +10,8 @@ const PASTE_MODIFIER = { darwin: [0xffeb, "MetaLeft"] };
  * the app's display. It asks the display to match its size in device pixels
  * (`scale` is what the display was started at), scales down when the display
  * is larger, reconnects while the app starts, carries the clipboard both ways
- * on a tap, gives phones a keyboard, and reports stream latency to Bivy. */
+ * on a tap, gives phones a keyboard, offers the app's menu bar where the
+ * display can read it (macOS), and reports stream latency to Bivy. */
 export function displayViewer(nonce, name, scale, platform = process.platform) {
     const [modifier, modifierCode] = PASTE_MODIFIER[platform] ?? [0xffe3, "ControlLeft"];
     const title = name.replace(/[<>&"]/g, (c) => `&#${c.charCodeAt(0)};`);
@@ -20,11 +22,24 @@ export function displayViewer(nonce, name, scale, platform = process.platform) {
 button,#paste{font:inherit;font-size:14px;min-height:44px;padding:0 14px;border-radius:22px;border:1px solid GrayText;background:Canvas;color:CanvasText}
 #keys{display:none;min-width:44px;padding:0}@media (pointer:coarse){#keys{display:block}}
 #paste{display:flex;gap:8px;align-items:center;padding:4px 4px 4px 14px}#paste input{font:inherit;min-height:36px;border:0;background:transparent;color:inherit;width:12em}
-#typing{position:fixed;left:0;top:0;width:1px;height:1px;opacity:0;border:0;padding:0}</style>
+#typing{position:fixed;left:0;top:0;width:1px;height:1px;opacity:0;border:0;padding:0}
+#menu{position:fixed;left:12px;bottom:72px;width:min(22rem,calc(100vw - 24px));max-height:min(70vh,36rem);display:flex;flex-direction:column;background:Canvas;color:CanvasText;border:1px solid GrayText;border-radius:16px;box-shadow:0 8px 24px color-mix(in srgb,CanvasText 25%,transparent);overflow:hidden}
+#menu header{display:flex;align-items:center;gap:4px;padding:4px;border-bottom:1px solid GrayText}
+#menu header button{border:0;min-width:44px;padding:0;font-size:18px}
+#menu h2{flex:1;margin:0;padding:0 8px;font-size:15px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+#menu-status{margin:0;padding:12px 16px;color:GrayText;font-size:14px}#menu-status:empty{display:none}
+#menu ul{list-style:none;margin:0;padding:4px 0;overflow:auto;overscroll-behavior:contain}
+#menu li[role=separator]{border-top:1px solid GrayText;margin:4px 16px;opacity:.5}
+#menu li button{display:flex;align-items:center;gap:8px;width:100%;border:0;border-radius:0;padding:0 16px;text-align:start}
+#menu li button:hover:not(:disabled),#menu li button:focus-visible{background:Highlight;color:HighlightText;outline:none}
+#menu li button:disabled{color:GrayText}
+#menu .mark{width:1em;flex:none}#menu .label{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}#menu .keys{flex:none;opacity:.7;font-variant-numeric:tabular-nums}</style>
 <div id="screen"></div><p id="status" role="status">Starting the app…</p>
-<div id="tools"><button id="copy" type="button" hidden>Copy from app</button>
+<div id="tools"><button id="menu-open" type="button" aria-haspopup="menu" aria-controls="menu" aria-expanded="false" hidden>Menu</button><button id="copy" type="button" hidden>Copy from app</button>
 <form id="paste" hidden><input id="paste-text" aria-label="Text to paste into the app" placeholder="Paste here"><button type="submit">Send</button></form>
 <button id="paste-open" type="button">Paste</button><button id="keys" type="button" aria-label="Show keyboard">⌨</button></div>
+<div id="menu" role="dialog" aria-labelledby="menu-title" hidden><header><button id="menu-back" type="button" aria-label="Back">‹</button><h2 id="menu-title">Menu</h2><button id="menu-close" type="button" aria-label="Close menu">✕</button></header>
+<p id="menu-status" role="status"></p><ul id="menu-items" role="menu" aria-labelledby="menu-title"></ul></div>
 <input id="typing" aria-label="Type into the app" autocapitalize="off" autocomplete="off" autocorrect="off" spellcheck="false">
 <script type="module" nonce="${nonce}">import RFB from "${NOVNC_PATH}core/rfb.js";
 const $=id=>document.getElementById(id),screen=$("screen"),status=$("status"),typing=$("typing"),SCALE=${scale === 2 ? 2 : 1};
@@ -57,7 +72,7 @@ function connect(){
   rfb._screenSize=()=>{const s=measure();return{w:Math.round(s.w*ask),h:Math.round(s.h*ask)};};
   rfb._requestRemoteResize=()=>{ask=SCALE;try{request();}finally{ask=1;}};
   rfb.resizeSession=true;rfb.scaleViewport=true;rfb.focusOnClick=true;
-  rfb.addEventListener("connect",()=>{wait=500;say("");});
+  rfb.addEventListener("connect",()=>{wait=500;say("");hasMenu();});
   rfb.addEventListener("disconnect",()=>{say("Starting the app…");setTimeout(connect,wait);wait=Math.min(wait*2,5000);});
   // The app copied text: offer it; the browser only lets a tap write the clipboard.
   rfb.addEventListener("clipboard",e=>{remote=e.detail.text;$("copy").hidden=!remote;$("copy").textContent="Copy from app";});
@@ -75,6 +90,61 @@ $("paste-open").onclick=async()=>{
 $("paste").onsubmit=e=>{e.preventDefault();paste($("paste-text").value);$("paste-text").value="";$("paste").hidden=true;};
 $("paste-text").addEventListener("keydown",e=>{if(e.key==="Escape"){$("paste").hidden=true;rfb.focus();}});
 
+// The app's menu bar, read when opened so enabled and checked states are
+// current. Drill down through submenus; choosing an item closes the menu.
+// Titles come from the app: shown as text only.
+const MENU="${DISPLAY_MENU_PATH}",menu=$("menu"),items=$("menu-items"),menuStatus=$("menu-status");
+let stack=[];
+const hasMenu=()=>fetch(MENU,{method:"HEAD"}).then(r=>{$("menu-open").hidden=!r.ok;}).catch(()=>{});
+const setOpen=open=>{menu.hidden=!open;$("tools").hidden=open;$("menu-open").setAttribute("aria-expanded",String(open));};
+const closeMenu=()=>{setOpen(false);stack=[];rfb.focus();};
+const cell=(cls,text)=>{const span=document.createElement("span");span.className=cls;span.textContent=text;return span;};
+function show(){
+  const level=stack[stack.length-1];
+  $("menu-title").textContent=level.title;$("menu-back").hidden=stack.length<2;
+  items.replaceChildren(...level.items.map(item=>{
+    const li=document.createElement("li");
+    if(item.separator){li.setAttribute("role","separator");return li;}
+    li.setAttribute("role","none");
+    const button=document.createElement("button");
+    button.type="button";button.setAttribute("role",item.checked?"menuitemcheckbox":"menuitem");
+    if(item.checked)button.setAttribute("aria-checked","true");
+    button.disabled=item.enabled===false;
+    button.append(cell("mark",item.checked?"✓":""),cell("label",item.title));
+    if(item.items){button.setAttribute("aria-haspopup","menu");button.append(cell("keys","›"));button.onclick=()=>{stack.push({title:item.title,items:item.items,path:[...level.path,item.title]});show();};}
+    else{if(item.shortcut){button.append(cell("keys",item.shortcut));button.setAttribute("aria-keyshortcuts",item.shortcut);}button.onclick=()=>choose([...level.path,item.title]);}
+    li.append(button);return li;
+  }));
+  items.querySelector("button:not(:disabled)")?.focus();
+}
+async function choose(path){
+  menuStatus.textContent="Choosing "+path.join(" › ")+"…";
+  try{
+    const r=await fetch(MENU,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({path})});
+    if(!r.ok)throw new Error((await r.json().catch(()=>({}))).error||"The app didn’t take it.");
+    menuStatus.textContent="";closeMenu();
+  }catch(e){menuStatus.textContent=e.message;}
+}
+$("menu-open").onclick=async()=>{
+  stack=[];items.replaceChildren();setOpen(true);$("menu-back").hidden=true;$("menu-title").textContent="Menu";
+  menuStatus.textContent="Reading the app’s menus…";$("menu-close").focus();
+  try{
+    const r=await fetch(MENU);const body=await r.json();
+    if(!r.ok)throw new Error(body.error||"Couldn’t read the app’s menus.");
+    menuStatus.textContent=body.menus.length?"":"This app has no menus.";
+    stack=[{title:"Menu",items:body.menus.map(m=>({title:m.title,enabled:true,items:m.items})),path:[]}];show();
+  }catch(e){menuStatus.textContent=e.message;}
+};
+$("menu-close").onclick=closeMenu;
+$("menu-back").onclick=()=>{stack.pop();show();};
+menu.addEventListener("keydown",e=>{
+  if(e.key==="Escape"){e.preventDefault();if(stack.length>1){stack.pop();show();}else closeMenu();return;}
+  if(e.key!=="ArrowDown"&&e.key!=="ArrowUp")return;
+  const all=[...items.querySelectorAll("button:not(:disabled)")];if(!all.length)return;
+  e.preventDefault();const at=all.indexOf(document.activeElement);
+  all[(at+(e.key==="ArrowDown"?1:-1)+all.length)%all.length].focus();
+});
+
 // Phones: a soft keyboard needs a text field; forward what's typed as keys.
 const SPECIAL={Backspace:0xff08,Enter:0xff0d,Tab:0xff09,Escape:0xff1b,ArrowLeft:0xff51,ArrowUp:0xff52,ArrowRight:0xff53,ArrowDown:0xff54};
 $("keys").onclick=()=>typing.focus();

package/dist/apps/gateway.js

@@ -9,7 +9,8 @@ import { readFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { WebSocketServer } from "ws";
 import { previewShell } from "./preview-shell.js";
-import { displayViewer, DISPLAY_SOCKET_PATH, DISPLAY_STATS_PATH, NOVNC_PATH } from "./display-viewer.js";
+import { displayViewer, DISPLAY_MENU_PATH, DISPLAY_SOCKET_PATH, DISPLAY_STATS_PATH, NOVNC_PATH } from "./display-viewer.js";
+import { pressMenu, readMenuPath, readMenus } from "./menu.js";
 import { inspectorScript } from "./inspector.js";
 const MIME = {
     ".html": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8", ".mjs": "text/javascript; charset=utf-8",
@@ -601,6 +602,10 @@ fetch('${REDEEM_PATH}',{method:'POST',headers:{'Content-Type':'text/plain'},body
             res.end();
             return;
         }
+        if (req.url === DISPLAY_MENU_PATH) {
+            await this.displayMenu(req, res, entry);
+            return;
+        }
         if (!["GET", "HEAD"].includes(req.method ?? "")) {
             res.writeHead(405, { Allow: "GET, HEAD" });
             res.end();
@@ -629,6 +634,55 @@ fetch('${REDEEM_PATH}',{method:'POST',headers:{'Content-Type':'text/plain'},body
         res.writeHead(200, { ...headers, "content-type": "text/html; charset=utf-8", "content-length": body.length });
         res.end(req.method === "HEAD" ? undefined : body);
     }
+    /** The app's menu bar for the viewer: HEAD says whether it has one, GET
+     * reads it, POST {path} chooses an item (same-origin, checked above). */
+    async displayMenu(req, res, entry) {
+        const control = entry.displayControl;
+        const json = (status, body) => { res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); res.end(JSON.stringify(body)); };
+        if (req.method === "HEAD") {
+            res.writeHead(control ? 204 : 404);
+            res.end();
+            return;
+        }
+        if (!control) {
+            json(404, { error: "This app's menus aren't available." });
+            return;
+        }
+        try {
+            if (req.method === "GET") {
+                json(200, { menus: await readMenus(control) });
+                return;
+            }
+            if (req.method !== "POST") {
+                res.writeHead(405, { Allow: "GET, HEAD, POST" });
+                res.end();
+                return;
+            }
+            let body = "";
+            for await (const chunk of req) {
+                body += chunk.toString();
+                if (body.length > 4096) {
+                    res.writeHead(413);
+                    res.end();
+                    return;
+                }
+            }
+            let path;
+            try {
+                path = readMenuPath(JSON.parse(body).path);
+            }
+            catch (error) {
+                json(400, { error: error.message });
+                return;
+            }
+            await pressMenu(control, path);
+            res.writeHead(204);
+            res.end();
+        }
+        catch (error) {
+            json(409, { error: error.message });
+        }
+    }
     /** Relays a viewer's WebSocket to the display's private VNC socket. */
     displaySocket(req, socket, head, entry) {
         this.displays.handleUpgrade(req, socket, head, (ws) => {

package/dist/apps/macos-display.js

@@ -119,8 +119,9 @@ export class MacDisplayHost {
             throw new Error(missing);
         }
         const socket = path.join(state.dir, "vnc.sock");
+        const control = path.join(state.dir, "control.sock");
         const token = randomBytes(16).toString("hex");
-        const child = spawn(binary, ["serve", socket, token, String(scale)], { stdio: ["ignore", "pipe", "pipe"] });
+        const child = spawn(binary, ["serve", socket, control, token, String(scale)], { stdio: ["ignore", "pipe", "pipe"] });
         state.child = child;
         const wm = { count: 0 };
         let errors = "";
@@ -141,7 +142,7 @@ export class MacDisplayHost {
         });
         if (!up)
             throw new Error(`Couldn't start a display for this app.${errors ? ` ${errors.trim().split("\n").at(-1)}` : ""}`);
-        return { socket, wm, scale, env: { BIVY_MAC_DISPLAY: token }, launch: [binary, "run", "--"] };
+        return { socket, control, wm, scale, env: { BIVY_MAC_DISPLAY: token }, launch: [binary, "run", "--"] };
     }
     stop(id) {
         const state = this.displays.get(id);

package/dist/apps/macos-display.swift

@@ -7,7 +7,7 @@
 // screenshots work unchanged.
 //
 //   macos-display check [--prompt]              permissions, as JSON
-//   macos-display serve <socket> <token> <scale>
+//   macos-display serve <socket> <control> <token> <scale>
 //   macos-display run -- <command> [args…]      starts the app (see below)
 //
 // The app is found by its environment: Bivy starts it through `run` with
@@ -18,6 +18,11 @@
 // ScreenCaptureKit (Screen Recording permission); clicks, keys and window
 // sizing go through Accessibility. Keys are sent to the app only; clicks land
 // only when the app's window is the one under the pointer.
+//
+// The control socket answers one JSON line per connection: {"menu":true}
+// returns the app's menu bar, read through Accessibility; {"press":["File",
+// "Save"]} presses the item at that path. The app's own full screen is
+// switched off: the preview already sizes it to the viewer.
 import AppKit
 import ApplicationServices
 import CoreMedia
@@ -142,6 +147,103 @@ func axWindow(_ win: Win) -> AXUIElement? {
   } ?? axValue(app, kAXMainWindowAttribute)
 }
 
+/// A listening Unix socket only this user can connect to.
+func listening(_ path: String) -> Int32 {
+  unlink(path)
+  let fd = Darwin.socket(AF_UNIX, SOCK_STREAM, 0)
+  var addr = sockaddr_un()
+  addr.sun_family = sa_family_t(AF_UNIX)
+  withUnsafeMutableBytes(of: &addr.sun_path) { buf in _ = path.withCString { strncpy(buf.baseAddress!.assumingMemoryBound(to: CChar.self), $0, buf.count - 1) } }
+  let old = umask(0o177)
+  let bound = withUnsafePointer(to: &addr) { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { Darwin.bind(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) } }
+  umask(old)
+  guard bound == 0, chmod(path, 0o600) == 0, Darwin.listen(fd, 16) == 0 else { warn("Couldn't listen on \(path)"); exit(1) }
+  return fd
+}
+
+// MARK: - Menus
+
+/// Menus deeper than this, or past this many items, are left out.
+let MENU_DEPTH = 4, MENU_ITEMS = 3000
+/// How a menu item's shortcut modifiers are written (AXMenuItemCmdModifiers
+/// bits: 1 Shift, 2 Option, 4 Control, 8 no Command).
+let SHORTCUT_MODIFIERS: [(bit: Int, symbol: String)] = [(4, "⌃"), (2, "⌥"), (1, "⇧")]
+
+func shortcut(_ char: String?, _ modifiers: Int?) -> String? {
+  guard let char, !char.isEmpty, let modifiers else { return nil }
+  let symbols = SHORTCUT_MODIFIERS.filter { modifiers & $0.bit != 0 }.map(\.symbol).joined()
+  return symbols + (modifiers & 8 == 0 ? "⌘" : "") + char.uppercased()
+}
+
+/// The app's menu bar as data: titles, enabled, checked, shortcut, submenus.
+/// The Apple menu is left out; the app menu (Settings, Quit) stays.
+func readMenus(_ pid: pid_t) -> [[String: Any]] {
+  let app = AXUIElementCreateApplication(pid)
+  AXUIElementSetMessagingTimeout(app, 1)
+  guard let bar: AXUIElement = axValue(app, kAXMenuBarAttribute) else { return [] }
+  var budget = MENU_ITEMS
+  let keys = [kAXTitleAttribute, kAXEnabledAttribute, "AXMenuItemCmdChar", "AXMenuItemCmdModifiers", "AXMenuItemMarkChar", kAXChildrenAttribute] as CFArray
+  func items(_ menu: AXUIElement, depth: Int) -> [[String: Any]] {
+    var out: [[String: Any]] = []
+    for item in (axValue(menu, kAXChildrenAttribute) as [AXUIElement]?) ?? [] {
+      guard budget > 0 else { break }
+      budget -= 1
+      var values: CFArray?
+      guard AXUIElementCopyMultipleAttributeValues(item, keys, AXCopyMultipleAttributeOptions(), &values) == .success, let v = values as? [Any] else { continue }
+      let title = (v[0] as? String) ?? ""
+      if title.isEmpty {
+        if let last = out.last, last["separator"] == nil { out.append(["separator": true]) }
+        continue
+      }
+      var entry: [String: Any] = ["title": title, "enabled": (v[1] as? Bool) ?? false]
+      if let keys = shortcut(v[2] as? String, v[3] as? Int) { entry["shortcut"] = keys }
+      if let mark = v[4] as? String, !mark.isEmpty { entry["checked"] = true }
+      if depth < MENU_DEPTH, let submenu = (v[5] as? [AXUIElement])?.first {
+        let children = items(submenu, depth: depth + 1)
+        if !children.isEmpty { entry["items"] = children }
+      }
+      out.append(entry)
+    }
+    if out.last?["separator"] != nil { out.removeLast() }
+    return out
+  }
+  let tops: [AXUIElement] = axValue(bar, kAXChildrenAttribute) ?? []
+  return tops.dropFirst().compactMap { top in
+    guard let title: String = axValue(top, kAXTitleAttribute), !title.isEmpty, let menu = (axValue(top, kAXChildrenAttribute) as [AXUIElement]?)?.first else { return nil }
+    return ["title": title, "items": items(menu, depth: 1)]
+  }
+}
+
+/// Presses the menu item at a path of titles, like choosing it from the menu bar.
+func pressMenu(_ pid: pid_t, _ path: [String]) -> String? {
+  let app = AXUIElementCreateApplication(pid)
+  AXUIElementSetMessagingTimeout(app, 1)
+  guard var node: AXUIElement = axValue(app, kAXMenuBarAttribute) else { return "The app has no menu bar." }
+  for (depth, title) in path.enumerated() {
+    // Below the bar, each level's items sit in a menu element.
+    let parent = depth == 0 ? node : ((axValue(node, kAXChildrenAttribute) as [AXUIElement]?)?.first ?? node)
+    let children: [AXUIElement] = axValue(parent, kAXChildrenAttribute) ?? []
+    guard let next = children.dropFirst(depth == 0 ? 1 : 0).first(where: { (axValue($0, kAXTitleAttribute) as String?) == title }) else {
+      return "No menu item \"\(path[0...depth].joined(separator: " > "))\"."
+    }
+    node = next
+  }
+  if (axValue(node, kAXEnabledAttribute) as Bool?) == false { return "\"\(path.joined(separator: " > "))\" is disabled right now." }
+  AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
+  return AXUIElementPerformAction(node, kAXPressAction as CFString) == .success ? nil : "The app didn't accept the menu item."
+}
+
+/// Switches the app's own full screen off, window by window. True if any was.
+func leaveFullScreen(_ pid: pid_t) -> Bool {
+  let app = AXUIElementCreateApplication(pid)
+  AXUIElementSetMessagingTimeout(app, 1)
+  var left = false
+  for window in (axValue(app, kAXWindowsAttribute) as [AXUIElement]?) ?? [] where (axValue(window, "AXFullScreen") as Bool?) == true {
+    left = AXUIElementSetAttributeValue(window, "AXFullScreen" as CFString, kCFBooleanFalse) == .success || left
+  }
+  return left
+}
+
 // MARK: - Keys
 
 /// US-layout key codes for the characters a keysym can name.
@@ -275,6 +377,13 @@ final class Host: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendabl
   var wanted: (ids: [CGWindowID], region: CGRect)?
   var captured: (ids: [CGWindowID], region: CGRect)?
   var acceptor: DispatchSourceRead?
+  /// Menus and full screen are Accessibility round trips to the app: off the
+  /// frame queue, one at a time.
+  let ax = DispatchQueue(label: "sh.bivy.macos-display.ax")
+  var control: DispatchSourceRead?
+  /// The app's processes, including ones whose windows aren't on screen now
+  /// (a full-screen window in another Space).
+  var apps = Set<pid_t>()
   /// The last capture error, and when: retried after a pause, reported once.
   var failure = (message: "", at: 0.0)
 
@@ -300,6 +409,7 @@ final class Host: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendabl
     let all = onScreenWindows()
     known = known.filter { entry in all.contains { $0.pid == entry.key } }
     ours = all.filter { !SYSTEM_LAYERS.contains($0.layer) && owns($0.pid) }
+    apps.formUnion(ours.map(\.pid))
     let normal = ours.filter { $0.layer == 0 }
     if normal.count != count { count = normal.count; say("windows \(count)") }
     main = normal.max { $0.bounds.width * $0.bounds.height < $1.bounds.width * $1.bounds.height }
@@ -400,15 +510,7 @@ final class Host: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendabl
   // MARK: Serving viewers
 
   func listen(path: String) {
-    unlink(path)
-    let fd = Darwin.socket(AF_UNIX, SOCK_STREAM, 0)
-    var addr = sockaddr_un()
-    addr.sun_family = sa_family_t(AF_UNIX)
-    withUnsafeMutableBytes(of: &addr.sun_path) { buf in _ = path.withCString { strncpy(buf.baseAddress!.assumingMemoryBound(to: CChar.self), $0, buf.count - 1) } }
-    let old = umask(0o177)
-    let bound = withUnsafePointer(to: &addr) { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { Darwin.bind(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) } }
-    umask(old)
-    guard bound == 0, chmod(path, 0o600) == 0, Darwin.listen(fd, 16) == 0 else { warn("Couldn't listen on \(path)"); exit(1) }
+    let fd = listening(path)
     let accept = DispatchSource.makeReadSource(fileDescriptor: fd, queue: q)
     accept.setEventHandler { [self] in
       let peer = Darwin.accept(fd, nil, nil)
@@ -429,6 +531,50 @@ final class Host: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendabl
     acceptor = accept
   }
 
+  /// The control socket: one request line, one JSON answer, then closed.
+  func serveControl(path: String) {
+    let fd = listening(path)
+    let accept = DispatchSource.makeReadSource(fileDescriptor: fd, queue: ax)
+    accept.setEventHandler { [self] in
+      let peer = Darwin.accept(fd, nil, nil)
+      guard peer >= 0 else { return }
+      defer { Darwin.close(peer) }
+      var wait = timeval(tv_sec: 5, tv_usec: 0), on: Int32 = 1
+      setsockopt(peer, SOL_SOCKET, SO_RCVTIMEO, &wait, socklen_t(MemoryLayout<timeval>.size))
+      setsockopt(peer, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout<Int32>.size))
+      var line: [UInt8] = []
+      var byte: UInt8 = 0
+      while line.count < 16384, Darwin.read(peer, &byte, 1) == 1, byte != 10 { line.append(byte) }
+      let answer = respond(to: (try? JSONSerialization.jsonObject(with: Data(line))) as? [String: Any] ?? [:])
+      let data = ((try? JSONSerialization.data(withJSONObject: answer)) ?? Data()) + Data([10])
+      _ = data.withUnsafeBytes { Darwin.write(peer, $0.baseAddress, $0.count) }
+    }
+    accept.resume()
+    control = accept
+    // Full screen is checked here too, so it never holds up frames.
+    let timer = DispatchSource.makeTimerSource(queue: ax)
+    timer.schedule(deadline: .now() + 1, repeating: .seconds(1))
+    timer.setEventHandler { [self] in
+      let pids = q.sync { apps = apps.filter { kill($0, 0) == 0 }; return apps }
+      guard pids.contains(where: leaveFullScreen) else { return }
+      // Once it has animated back, fit it to the viewer again.
+      q.asyncAfter(deadline: .now() + 1) { self.fitted = nil }
+    }
+    timer.resume()
+    fullScreenTimer = timer
+  }
+  var fullScreenTimer: DispatchSourceTimer?
+
+  func respond(to request: [String: Any]) -> [String: Any] {
+    guard let pid = q.sync(execute: { main?.pid }) else { return ["error": "The app has no window yet."] }
+    if request["menu"] as? Bool == true { return ["menus": readMenus(pid)] }
+    if let path = request["press"] as? [String], (1...8).contains(path.count) {
+      if let error = pressMenu(pid, path) { return ["error": error] }
+      return ["ok": true]
+    }
+    return ["error": "Unknown request."]
+  }
+
   func drop(_ client: Client) {
     guard let index = clients.firstIndex(where: { $0 === client }) else { return }
     clients.remove(at: index)
@@ -711,9 +857,10 @@ case "run" where args.count > 3 && args[2] == "--":
   run(Array(args[3...]))
 case "check":
   say(permissions(prompt: args.contains("--prompt")))
-case "serve" where args.count == 5:
-  let host = Host(token: args[3], scale: CGFloat(Double(args[4]) == 2 ? 2 : 1))
+case "serve" where args.count == 6:
+  let host = Host(token: args[4], scale: CGFloat(Double(args[5]) == 2 ? 2 : 1))
   host.q.sync { host.listen(path: args[2]) }
+  host.serveControl(path: args[3])
   let timer = DispatchSource.makeTimerSource(queue: host.q)
   timer.schedule(deadline: .now(), repeating: .milliseconds(200))
   timer.setEventHandler { host.tick() }
@@ -721,6 +868,6 @@ case "serve" where args.count == 5:
   say("ready")
   RunLoop.main.run()
 default:
-  warn("Usage: macos-display check [--prompt] | serve <socket> <token> <scale> | run -- <command> [args…]")
+  warn("Usage: macos-display check [--prompt] | serve <socket> <control> <token> <scale> | run -- <command> [args…]")
   exit(2)
 }

package/dist/apps/menu.js

@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Petter André Sjulstad
+import net from "node:net";
+const MAX_ITEMS = 3000;
+const MAX_DEPTH = 4;
+const TIMEOUT_MS = 5_000;
+/** A path of titles, top menu first: "File > Export…" or ["File", "Export…"]. */
+export function readMenuPath(input) {
+    const path = typeof input === "string" ? input.split(">").map((part) => part.trim()) : input;
+    if (!Array.isArray(path) || !path.length || path.length > 8 || path.some((part) => typeof part !== "string" || !part || part.length > 200)) {
+        throw new Error("A menu item is a path of titles, like \"File > Save\".");
+    }
+    return path;
+}
+/** Keeps what the display says to known fields and bounded sizes: the menu
+ * comes from the app, so it is data to show, never markup. */
+function clean(items, depth, budget) {
+    if (!Array.isArray(items))
+        return [];
+    const out = [];
+    for (const raw of items) {
+        if (budget.left-- <= 0)
+            break;
+        const item = raw;
+        if (item?.separator === true) {
+            out.push({ separator: true });
+            continue;
+        }
+        if (typeof item?.title !== "string" || !item.title)
+            continue;
+        const entry = { title: item.title.slice(0, 200), enabled: item.enabled === true };
+        if (typeof item.shortcut === "string" && item.shortcut.length <= 20)
+            entry.shortcut = item.shortcut;
+        if (item.checked === true)
+            entry.checked = true;
+        const children = depth < MAX_DEPTH ? clean(item.items, depth + 1, budget) : [];
+        if (children.length)
+            entry.items = children;
+        out.push(entry);
+    }
+    return out;
+}
+/** One request to a display's control socket, one JSON answer. */
+function ask(control, request) {
+    return new Promise((resolve, reject) => {
+        const socket = net.connect(control);
+        let data = "";
+        socket.setTimeout(TIMEOUT_MS, () => socket.destroy(new Error("The app didn't answer in time.")));
+        socket.on("data", (chunk) => { data += chunk; if (data.length > 4 * 1024 * 1024)
+            socket.destroy(new Error("The app's menu is too large.")); });
+        socket.on("error", (error) => reject(error.code ? new Error("The app isn't running.") : error));
+        socket.on("end", () => {
+            try {
+                const answer = JSON.parse(data);
+                if (typeof answer.error === "string")
+                    reject(new Error(answer.error.slice(0, 300)));
+                else
+                    resolve(answer);
+            }
+            catch {
+                reject(new Error("The app's display gave an unreadable answer."));
+            }
+        });
+        socket.end(`${JSON.stringify(request)}\n`);
+    });
+}
+/** The app's menu bar, read now, so enabled and checked states are current. */
+export async function readMenus(control) {
+    const answer = await ask(control, { menu: true });
+    const budget = { left: MAX_ITEMS };
+    return (Array.isArray(answer.menus) ? answer.menus : []).flatMap((menu) => typeof menu?.title === "string" && menu.title ? [{ title: menu.title.slice(0, 200), items: clean(menu.items, 1, budget) }] : []);
+}
+/** Chooses the menu item at `path`, as if picked from the menu bar. */
+export async function pressMenu(control, path) {
+    await ask(control, { press: path });
+}

package/dist/apps/service.js

@@ -8,6 +8,7 @@ import { takeShots } from "./screenshot.js";
 import { approximate, composite, readStrokes } from "./annotate.js";
 import { captureFrame, encodePng, sendInput } from "./rfb.js";
 import { inputEvents, readAction } from "./input.js";
+import { pressMenu, readMenuPath, readMenus } from "./menu.js";
 import { pngSize } from "./review.js";
 const NO_REVIEWS = { publish: (review) => review, expire: () => { } };
 const NO_DISPLAYS = { unavailable: () => "Desktop app views aren't available on this machine.", ensure: () => Promise.reject(new Error("Desktop app views aren't available on this machine.")), stop: () => { } };
@@ -373,12 +374,31 @@ export class AppService {
         const { width, height } = await sendInput(entry.display, inputEvents(action));
         if (!this.screenshots.enabled())
             return { width, height, message: "Done. Agent screenshots are off on this machine, so there's no picture of the result: bivy config set sessions.appScreenshots true" };
+        return { width, height, shot: await this.after(entry, sessionId), message: "Done. The shot is the app now." };
+    }
+    /** A desktop app's menu bar: listed, or one item chosen by its path of
+     * titles ("File > Save"). Choosing returns the app afterwards, like `act`. */
+    async menu(sessionId, target, path) {
+        const item = path === undefined ? undefined : readMenuPath(path);
+        const entry = this.pickView(sessionId, target, true);
+        if (!entry.display)
+            await this.windowShown(entry);
+        if (!entry.displayControl)
+            throw new Error("This machine's desktop apps don't expose their menus (macOS only). Use keyboard shortcuts: bivy app key.");
+        if (!item)
+            return { menus: await readMenus(entry.displayControl) };
+        await pressMenu(entry.displayControl, item);
+        if (!this.screenshots.enabled())
+            return { message: "Chosen. Agent screenshots are off on this machine, so there's no picture of the result." };
+        return { shot: await this.after(entry, sessionId), message: "Chosen. The shot is the app now." };
+    }
+    /** The app a moment after an action, as a screenshot. */
+    async after(entry, sessionId) {
         await new Promise((r) => setTimeout(r, AFTER_INPUT_MS));
         const outDir = path.join(os.tmpdir(), "bivy-shots", sessionId.replace(/[^A-Za-z0-9_-]/g, "_"), String(Date.now()));
         const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [], themes: [], path: "/" }, outDir));
         this.shooting = run;
-        const [shot] = await run;
-        return { width, height, shot, message: "Done. The shot is the app now." };
+        return (await run)[0];
     }
     /** Draw on the preview: the user's marks on a picture of what they saw.
      * The picture is the Compare screenshot they drew on, a desktop app's
@@ -462,6 +482,7 @@ export class AppService {
             throw new Error("This view has no program.");
         return this.displays.ensure(entry.view.id, entry.app.name, entry.displayScale).then((display) => {
             entry.display = display.socket;
+            entry.displayControl = display.control;
             const [command, ...args] = [...display.launch ?? [], target.command, ...target.args];
             return { command: command, args, workspace: target.workspace, env: display.env, name: `${entry.app.name} · ${entry.view.name}` };
         });
@@ -518,6 +539,7 @@ export class AppService {
         const entry = this.registry.getView(id);
         if (entry) {
             entry.display = undefined;
+            entry.displayControl = undefined;
             entry.displayScale = undefined;
         }
         const server = this.servers.get(id);

package/dist/controllers/app-commands.js

@@ -23,6 +23,7 @@ export function createAppCommands(service, workspaceFor, published = () => { })
         "apps.open": (msg) => service.open(String(msg.sessionId), String(msg.appId), String(msg.viewId), typeof msg.returnTo === "string" ? msg.returnTo : undefined, msg.direct === true, typeof msg.scale === "number" ? msg.scale : undefined, typeof msg.path === "string" ? msg.path : undefined),
         "apps.shot": (msg) => service.shot(String(msg.sessionId), typeof msg.appId === "string" ? msg.appId : undefined, { widths: msg.widths, themes: msg.themes, path: typeof msg.path === "string" ? msg.path : undefined }),
         "apps.input": (msg) => service.act(String(msg.sessionId), typeof msg.target === "string" ? msg.target : undefined, msg.action),
+        "apps.menu": (msg) => service.menu(String(msg.sessionId), typeof msg.target === "string" ? msg.target : undefined, msg.path),
         "apps.present": (msg) => service.present(String(msg.sessionId), { target: typeof msg.target === "string" ? msg.target : undefined, path: typeof msg.path === "string" ? msg.path : undefined, note: typeof msg.note === "string" ? msg.note : undefined }),
         "apps.showMe": (msg) => service.present(String(msg.sessionId), { target: typeof msg.appId === "string" ? msg.appId : undefined, trigger: "asked" }),
         "apps.mute": (msg) => service.mute(String(msg.sessionId)),

package/dist/protocol/client-command-routes.js

@@ -3,7 +3,7 @@
 /** Direct-mode adapters for operations whose canonical implementation is the
  * client command registry. Relay and HTTP now differ only in framing. */
 export const CLIENT_COMMAND_ROUTES = [
-    ...["list", "publish", "offers", "adopt", "open", "logs", "shot", "input", "present", "showMe", "mute", "reviewMode", "annotate", "clearNotes", "share", "revoke", "remove"].map((action) => ({ method: "post", path: `/api/apps/${action}`, kind: `apps.${action}` })),
+    ...["list", "publish", "offers", "adopt", "open", "logs", "shot", "input", "menu", "present", "showMe", "mute", "reviewMode", "annotate", "clearNotes", "share", "revoke", "remove"].map((action) => ({ method: "post", path: `/api/apps/${action}`, kind: `apps.${action}` })),
     { method: "post", path: "/api/artifacts/list", kind: "artifacts.list" },
     { method: "post", path: "/api/auth/credentials/native-preview", kind: "credentials.native.preview" },
     { method: "post", path: "/api/auth/credentials/native-import", kind: "credentials.native.import" },

package/dist/protocol/client-command-schemas.js

@@ -16,6 +16,7 @@ export const CLIENT_COMMAND_SCHEMAS = {
     "apps.open": Type.Object({ ...request, ...session, appId: Type.String(), viewId: Type.String(), returnTo: Type.Optional(Type.String({ maxLength: 2048 })), direct: Type.Optional(Type.Boolean()), scale: Type.Optional(Type.Integer({ minimum: 1, maximum: 2 })), path: Type.Optional(Type.String({ maxLength: 2048 })) }),
     "apps.shot": Type.Object({ ...request, ...session, appId: Type.Optional(Type.String()), widths: Type.Optional(Type.Array(Type.Integer(), { maxItems: 4 })), themes: Type.Optional(Type.Array(Type.Union([Type.Literal("light"), Type.Literal("dark")]), { maxItems: 2 })), path: Type.Optional(Type.String({ maxLength: 2048 })) }),
     "apps.input": Type.Object({ ...request, ...session, target: Type.Optional(Type.String({ maxLength: 200 })), action: Type.Object({ kind: Type.String({ maxLength: 20 }) }) }),
+    "apps.menu": Type.Object({ ...request, ...session, target: Type.Optional(Type.String({ maxLength: 200 })), path: Type.Optional(Type.Array(Type.String({ minLength: 1, maxLength: 200 }), { minItems: 1, maxItems: 8 })) }),
     "apps.present": Type.Object({ ...request, ...session, target: Type.Optional(Type.String({ maxLength: 200 })), path: Type.Optional(Type.String({ maxLength: 2048 })), note: Type.Optional(Type.String({ maxLength: 2000 })) }),
     "apps.showMe": Type.Object({ ...request, ...session, appId: Type.Optional(Type.String()) }),
     "apps.mute": Type.Object({ ...request, ...session }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bivy/bivy",
-  "version": "0.17.3-staging.30",
+  "version": "0.17.3-staging.31",
   "type": "module",
   "license": "AGPL-3.0-only",
   "description": "Run coding agents on machines you own. Open-source, self-hostable agent workspace.",