agent-device 0.11.0 → 0.11.2

package/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift

@@ -1,11 +1,15 @@
 import AppKit
 import ApplicationServices
+import CoreGraphics
 import Foundation
 
 private enum SnapshotTraversalLimits {
   static let maxDesktopApps = 24
   static let maxNodes = 1500
   static let maxDepth = 12
+  static let maxMenuBarBandY = 64.0
+  static let maxMenuBarBandHeight = 64.0
+  static let maxMenuBarExtraWidth = 256.0
 }
 
 struct RectResponse: Encodable {
@@ -62,7 +66,17 @@ private struct SnapshotTraversalState {
   var truncated = false
 }
 
-func captureSnapshotResponse(surface: String) throws -> SnapshotResponse {
+private struct MenuBarWindowFallbackCandidate {
+  let windowNumber: Int
+  let rect: RectResponse
+  let layer: Int
+
+  var area: Double {
+    rect.width * rect.height
+  }
+}
+
+func captureSnapshotResponse(surface: String, bundleId: String? = nil) throws -> SnapshotResponse {
   let result: SnapshotBuildResult
   switch surface {
   case "frontmost-app":
@@ -70,7 +84,7 @@ func captureSnapshotResponse(surface: String) throws -> SnapshotResponse {
   case "desktop":
     result = snapshotDesktop()
   case "menubar":
-    result = snapshotMenuBar()
+    result = try snapshotMenuBar(bundleId: bundleId)
   default:
     throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
   }
@@ -200,8 +214,9 @@ private func appendApplicationSnapshot(
   return true
 }
 
-private func snapshotMenuBar() -> SnapshotBuildResult {
+private func snapshotMenuBar(bundleId: String?) throws -> SnapshotBuildResult {
   var state = SnapshotTraversalState()
+  let screenRect = mainScreenRectResponse()
   guard
     let rootIndex = appendSyntheticSnapshotNode(
       into: &state,
@@ -209,29 +224,54 @@ private func snapshotMenuBar() -> SnapshotBuildResult {
       label: "Menu Bar",
       depth: 0,
       parentIndex: nil,
-      surface: "menubar"
+      surface: "menubar",
+      rect: screenRect
     )
   else {
     return SnapshotBuildResult(nodes: state.nodes, truncated: true)
   }
 
-  if let frontmost = NSWorkspace.shared.frontmostApplication {
-    let frontmostElement = AXUIElementCreateApplication(frontmost.processIdentifier)
-    if let menuBar = elementAttribute(frontmostElement, attribute: kAXMenuBarAttribute as String) {
-      _ = appendElementSnapshot(
-        menuBar,
+  if let bundleId {
+    let targetApp = try resolveTargetApplication(bundleId: bundleId, surface: nil)
+    let appendedExtras = appendMenuBarSnapshot(
+      targetApp,
+      attribute: kAXExtrasMenuBarAttribute as String,
+      depth: 1,
+      parentIndex: rootIndex,
+      surface: "menubar",
+      state: &state
+    )
+    if !appendedExtras {
+      let appendedMenuBar = appendMenuBarSnapshot(
+        targetApp,
+        attribute: kAXMenuBarAttribute as String,
         depth: 1,
         parentIndex: rootIndex,
-        context: SnapshotContext(
-          surface: "menubar",
-          pid: Int32(frontmost.processIdentifier),
-          bundleId: frontmost.bundleIdentifier,
-          appName: frontmost.localizedName,
-          windowTitle: frontmost.localizedName
-        ),
+        surface: "menubar",
         state: &state
       )
+      if !appendedMenuBar {
+        _ = appendMenuBarWindowFallback(
+          targetApp,
+          depth: 1,
+          parentIndex: rootIndex,
+          surface: "menubar",
+          state: &state
+        )
+      }
     }
+    return SnapshotBuildResult(nodes: state.nodes, truncated: state.truncated)
+  }
+
+  if let frontmost = NSWorkspace.shared.frontmostApplication {
+    _ = appendMenuBarSnapshot(
+      frontmost,
+      attribute: kAXMenuBarAttribute as String,
+      depth: 1,
+      parentIndex: rootIndex,
+      surface: "menubar",
+      state: &state
+    )
   }
 
   if !state.truncated,
@@ -239,20 +279,24 @@ private func snapshotMenuBar() -> SnapshotBuildResult {
        withBundleIdentifier: "com.apple.systemuiserver"
      ).first
   {
-    let systemUiElement = AXUIElementCreateApplication(systemUiServer.processIdentifier)
-    if let menuExtras = elementAttribute(systemUiElement, attribute: kAXMenuBarAttribute as String) {
-      _ = appendElementSnapshot(
-        menuExtras,
+    let appendedExtras = appendMenuBarSnapshot(
+      systemUiServer,
+      attribute: kAXExtrasMenuBarAttribute as String,
+      depth: 1,
+      parentIndex: rootIndex,
+      surface: "menubar",
+      state: &state,
+      windowTitle: "System Menu Extras"
+    )
+    if !appendedExtras {
+      _ = appendMenuBarSnapshot(
+        systemUiServer,
+        attribute: kAXMenuBarAttribute as String,
         depth: 1,
         parentIndex: rootIndex,
-        context: SnapshotContext(
-          surface: "menubar",
-          pid: Int32(systemUiServer.processIdentifier),
-          bundleId: systemUiServer.bundleIdentifier,
-          appName: systemUiServer.localizedName,
-          windowTitle: "System Menu Extras"
-        ),
-        state: &state
+        surface: "menubar",
+        state: &state,
+        windowTitle: "System Menu Extras"
       )
     }
   }
@@ -260,6 +304,67 @@ private func snapshotMenuBar() -> SnapshotBuildResult {
   return SnapshotBuildResult(nodes: state.nodes, truncated: state.truncated)
 }
 
+@discardableResult
+private func appendMenuBarSnapshot(
+  _ app: NSRunningApplication,
+  attribute: String,
+  depth: Int,
+  parentIndex: Int,
+  surface: String,
+  state: inout SnapshotTraversalState,
+  windowTitle: String? = nil
+) -> Bool {
+  let appElement = AXUIElementCreateApplication(app.processIdentifier)
+  guard let menuBar = elementAttribute(appElement, attribute: attribute) else {
+    return false
+  }
+
+  let nodeCountBefore = state.nodes.count
+  _ = appendElementSnapshot(
+    menuBar,
+    depth: depth,
+    parentIndex: parentIndex,
+    context: SnapshotContext(
+      surface: surface,
+      pid: Int32(app.processIdentifier),
+      bundleId: app.bundleIdentifier,
+      appName: app.localizedName,
+      windowTitle: windowTitle ?? app.localizedName
+    ),
+    state: &state
+  )
+  return state.nodes.count > nodeCountBefore
+}
+
+@discardableResult
+private func appendMenuBarWindowFallback(
+  _ app: NSRunningApplication,
+  depth: Int,
+  parentIndex: Int,
+  surface: String,
+  state: inout SnapshotTraversalState
+) -> Bool {
+  guard let candidate = menuBarWindowFallbackCandidate(for: app) else {
+    return false
+  }
+
+  return appendSyntheticSnapshotNode(
+    into: &state,
+    type: "MenuBarItem",
+    label: app.localizedName ?? app.bundleIdentifier ?? "Menu Bar Item",
+    depth: depth,
+    parentIndex: parentIndex,
+    surface: surface,
+    identifier: "cgwindow:\(candidate.windowNumber)",
+    pid: Int32(app.processIdentifier),
+    bundleId: app.bundleIdentifier,
+    appName: app.localizedName,
+    windowTitle: app.localizedName,
+    rect: candidate.rect,
+    hittable: true
+  ) != nil
+}
+
 @discardableResult
 private func appendSyntheticSnapshotNode(
   into state: inout SnapshotTraversalState,
@@ -272,7 +377,9 @@ private func appendSyntheticSnapshotNode(
   pid: Int32? = nil,
   bundleId: String? = nil,
   appName: String? = nil,
-  windowTitle: String? = nil
+  windowTitle: String? = nil,
+  rect: RectResponse? = nil,
+  hittable: Bool = false
 ) -> Int? {
   guard reserveSnapshotNodeCapacity(&state) else {
     return nil
@@ -288,10 +395,10 @@ private func appendSyntheticSnapshotNode(
       label: label,
       value: nil,
       identifier: identifier ?? "surface:\(surface):\(type.lowercased())",
-      rect: nil,
+      rect: rect,
       enabled: true,
       selected: nil,
-      hittable: false,
+      hittable: hittable && rect != nil,
       depth: depth,
       parentIndex: parentIndex,
       pid: pid,
@@ -304,6 +411,83 @@ private func appendSyntheticSnapshotNode(
   return index
 }
 
+private func mainScreenRectResponse() -> RectResponse? {
+  guard let screenFrame = NSScreen.main?.frame, screenFrame.width > 0, screenFrame.height > 0 else {
+    return nil
+  }
+  return RectResponse(
+    x: Double(screenFrame.origin.x),
+    y: Double(screenFrame.origin.y),
+    width: Double(screenFrame.width),
+    height: Double(screenFrame.height)
+  )
+}
+
+private func menuBarWindowFallbackCandidate(
+  for app: NSRunningApplication
+) -> MenuBarWindowFallbackCandidate? {
+  guard
+    let windowInfoList = CGWindowListCopyWindowInfo([.optionAll], kCGNullWindowID)
+      as? [[String: Any]]
+  else {
+    return nil
+  }
+
+  let pid = Int(app.processIdentifier)
+  let allCandidates = windowInfoList.compactMap { info -> MenuBarWindowFallbackCandidate? in
+    let ownerPid = (info[kCGWindowOwnerPID as String] as? NSNumber)?.intValue
+    guard ownerPid == pid else {
+      return nil
+    }
+    guard let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary,
+          let bounds = CGRect(dictionaryRepresentation: boundsDictionary)
+    else {
+      return nil
+    }
+    guard bounds.width > 0, bounds.height > 0 else {
+      return nil
+    }
+    let alpha = (info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1
+    guard alpha > 0 else {
+      return nil
+    }
+    let rect = RectResponse(
+      x: Double(bounds.origin.x),
+      y: Double(bounds.origin.y),
+      width: Double(bounds.width),
+      height: Double(bounds.height)
+    )
+    let windowNumber = (info[kCGWindowNumber as String] as? NSNumber)?.intValue ?? 0
+    let layer = (info[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
+    return MenuBarWindowFallbackCandidate(
+      windowNumber: windowNumber,
+      rect: rect,
+      layer: layer
+    )
+  }
+
+  // CGWindowList can surface multiple app-owned utility windows. Prefer the small
+  // top-band window that matches typical menu bar extra geometry before ranking by area.
+  let menuBarBandCandidates = allCandidates.filter { candidate in
+    candidate.rect.y <= SnapshotTraversalLimits.maxMenuBarBandY
+      && candidate.rect.height <= SnapshotTraversalLimits.maxMenuBarBandHeight
+  }
+  let narrowCandidates = menuBarBandCandidates.filter { candidate in
+    candidate.rect.width <= SnapshotTraversalLimits.maxMenuBarExtraWidth
+  }
+  let rankedCandidates = (narrowCandidates.isEmpty ? menuBarBandCandidates : narrowCandidates)
+    .sorted { left, right in
+      if left.area != right.area {
+        return left.area < right.area
+      }
+      if left.layer != right.layer {
+        return left.layer < right.layer
+      }
+      return left.windowNumber < right.windowNumber
+    }
+  return rankedCandidates.first
+}
+
 @discardableResult
 private func appendElementSnapshot(
   _ element: AXUIElement,
@@ -361,7 +545,7 @@ private func appendElementSnapshot(
     return index
   }
 
-  for child in children(of: element) {
+  for child in snapshotChildren(of: element, role: role) {
     if state.truncated {
       break
     }
@@ -522,9 +706,9 @@ private func accessibilityAxValue(_ value: CFTypeRef?) -> AXValue? {
   return (value as! AXValue)
 }
 
-func children(of element: AXUIElement) -> [AXUIElement] {
+private func elementArrayAttribute(_ element: AXUIElement, attribute: String) -> [AXUIElement] {
   var value: CFTypeRef?
-  guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) == .success,
+  guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
         let children = value as? [AXUIElement]
   else {
     return []
@@ -532,6 +716,23 @@ func children(of element: AXUIElement) -> [AXUIElement] {
   return children
 }
 
+func children(of element: AXUIElement) -> [AXUIElement] {
+  return elementArrayAttribute(element, attribute: kAXChildrenAttribute as String)
+}
+
+private func snapshotChildren(of element: AXUIElement, role: String?) -> [AXUIElement] {
+  let directChildren = children(of: element)
+  if !directChildren.isEmpty {
+    return directChildren
+  }
+  switch role {
+  case "AXMenuBar", "AXMenuBarItem", "AXMenu":
+    return elementArrayAttribute(element, attribute: kAXVisibleChildrenAttribute as String)
+  default:
+    return []
+  }
+}
+
 func windows(of appElement: AXUIElement) -> [AXUIElement] {
   var value: CFTypeRef?
   guard AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &value) == .success,

package/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -2,6 +2,9 @@ import AppKit
 import ApplicationServices
 import CoreGraphics
 import Foundation
+import ImageIO
+import ScreenCaptureKit
+import UniformTypeIdentifiers
 
 enum HelperError: Error {
   case invalidArgs(String)
@@ -57,6 +60,19 @@ struct ReadResponse: Encodable {
   let text: String
 }
 
+struct PressResponse: Encodable {
+  let x: Double
+  let y: Double
+  let bundleId: String?
+  let surface: String?
+}
+
+struct ScreenshotResponse: Encodable {
+  let path: String
+  let surface: String?
+  let fullscreen: Bool
+}
+
 struct AgentDeviceMacOSHelper {
   static func main() {
     do {
@@ -100,6 +116,10 @@ struct AgentDeviceMacOSHelper {
       return try handleSnapshot(arguments: Array(arguments.dropFirst()))
     case "read":
       return try handleRead(arguments: Array(arguments.dropFirst()))
+    case "press":
+      return try handlePress(arguments: Array(arguments.dropFirst()))
+    case "screenshot":
+      return try handleScreenshot(arguments: Array(arguments.dropFirst()))
     default:
       throw HelperError.invalidArgs("unknown command: \(command)")
     }
@@ -315,11 +335,13 @@ struct AgentDeviceMacOSHelper {
       throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
     }
 
+    let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+
     switch surface {
     case "frontmost-app":
-      return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface))
+      return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId))
     case "desktop", "menubar":
-      return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface))
+      return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId))
     default:
       throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
     }
@@ -339,6 +361,35 @@ struct AgentDeviceMacOSHelper {
     let text = try readTextAtPosition(bundleId: bundleId, surface: surface, x: x, y: y)
     return SuccessEnvelope(data: ReadResponse(text: text))
   }
+
+  static func handlePress(arguments: [String]) throws -> any Encodable {
+    guard let rawX = optionValue(arguments: arguments, name: "--x"),
+          let rawY = optionValue(arguments: arguments, name: "--y"),
+          let x = Double(rawX),
+          let y = Double(rawY)
+    else {
+      throw HelperError.invalidArgs("press requires --x <number> --y <number>")
+    }
+
+    let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+    let surface = optionValue(arguments: arguments, name: "--surface")
+    try pressAtPosition(bundleId: bundleId, surface: surface, x: x, y: y)
+    return SuccessEnvelope(data: PressResponse(x: x, y: y, bundleId: bundleId, surface: surface))
+  }
+
+  static func handleScreenshot(arguments: [String]) throws -> any Encodable {
+    guard let outPath = optionValue(arguments: arguments, name: "--out")?
+      .trimmingCharacters(in: .whitespacesAndNewlines),
+      !outPath.isEmpty
+    else {
+      throw HelperError.invalidArgs("screenshot requires --out <path>")
+    }
+
+    let surface = optionValue(arguments: arguments, name: "--surface")
+    let fullscreen = arguments.contains("--fullscreen")
+    try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen)
+    return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen))
+  }
 }
 
 private func optionValue(arguments: [String], name: String) -> String? {
@@ -395,6 +446,74 @@ private func readTextAtPosition(bundleId: String?, surface: String?, x: Double,
   throw HelperError.commandFailed("read did not resolve text")
 }
 
+private func pressAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws {
+  _ = bundleId
+  _ = surface
+  let point = CGPoint(x: x, y: y)
+  guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left),
+        let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
+        let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)
+  else {
+    throw HelperError.commandFailed("press action failed", details: ["reason": "event_creation_failed"])
+  }
+  move.post(tap: .cghidEventTap)
+  down.post(tap: .cghidEventTap)
+  up.post(tap: .cghidEventTap)
+}
+
+private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws {
+  _ = fullscreen
+  guard #available(macOS 15.2, *) else {
+    throw HelperError.commandFailed(
+      "screenshot on macOS desktop and menubar surfaces requires macOS 15.2 or newer"
+    )
+  }
+  guard let screenFrame = NSScreen.main?.frame, screenFrame.width > 0, screenFrame.height > 0 else {
+    throw HelperError.commandFailed("screenshot could not resolve main screen bounds")
+  }
+
+  let rect = CGRect(origin: screenFrame.origin, size: screenFrame.size)
+  let semaphore = DispatchSemaphore(value: 0)
+  var capturedImage: CGImage?
+  var capturedError: Error?
+  SCScreenshotManager.captureImage(in: rect) { image, error in
+    capturedImage = image
+    capturedError = error
+    semaphore.signal()
+  }
+  semaphore.wait()
+
+  if let error = capturedError as NSError? {
+    if error.domain == "com.apple.ScreenCaptureKit.SCStreamErrorDomain", error.code == -3801 {
+      throw HelperError.commandFailed(
+        "screenshot requires Screen Recording permission on macOS desktop and menubar surfaces",
+        details: ["surface": surface ?? "", "permission": "screen-recording"]
+      )
+    }
+    throw HelperError.commandFailed("screenshot failed", details: ["error": error.localizedDescription])
+  }
+  guard let capturedImage else {
+    throw HelperError.commandFailed("screenshot failed")
+  }
+
+  let outputURL = URL(fileURLWithPath: outPath)
+  if let parent = outputURL.deletingLastPathComponent().path.removingPercentEncoding, !parent.isEmpty {
+    try FileManager.default.createDirectory(atPath: parent, withIntermediateDirectories: true)
+  }
+  guard let destination = CGImageDestinationCreateWithURL(
+    outputURL as CFURL,
+    UTType.png.identifier as CFString,
+    1,
+    nil
+  ) else {
+    throw HelperError.commandFailed("screenshot could not create PNG destination")
+  }
+  CGImageDestinationAddImage(destination, capturedImage, nil)
+  guard CGImageDestinationFinalize(destination) else {
+    throw HelperError.commandFailed("screenshot could not write PNG file")
+  }
+}
+
 private func readableText(for element: AXUIElement) -> String? {
   return
     stringAttribute(element, attribute: kAXValueAttribute as String)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "Unified control plane for physical and virtual devices via an agent-driven CLI.",
   "license": "MIT",
   "author": "Callstack",
@@ -24,13 +24,18 @@
     "clean:daemon": "rm -f ~/.agent-device/daemon.json && rm -f ~/.agent-device/daemon.lock",
     "build:node": "pnpm build && pnpm clean:daemon",
     "build:xcuitest": "pnpm build:xcuitest:ios && pnpm build:xcuitest:macos",
-    "build:xcuitest:ios": "rm -rf ~/.agent-device/ios-runner/derived/device && xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination \"generic/platform=iOS Simulator\" -derivedDataPath ~/.agent-device/ios-runner/derived",
-    "build:xcuitest:macos": "rm -rf ~/.agent-device/ios-runner/derived/macos && xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination \"platform=macOS,arch=$(uname -m)\" -derivedDataPath ~/.agent-device/ios-runner/derived/macos CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY=\"\" DEVELOPMENT_TEAM=\"\" COMPILER_INDEX_STORE_ENABLE=NO ENABLE_CODE_COVERAGE=NO",
-    "build:xcuitest:tvos": "rm -rf ~/.agent-device/ios-runner/derived/tvos && xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination \"generic/platform=tvOS Simulator\" -derivedDataPath ~/.agent-device/ios-runner/derived/tvos",
+    "build:xcuitest:ios": "AGENT_DEVICE_XCUITEST_PLATFORM=ios AGENT_DEVICE_IOS_CLEAN_DERIVED=1 sh ./scripts/build-xcuitest-apple.sh",
+    "build:xcuitest:macos": "AGENT_DEVICE_XCUITEST_PLATFORM=macos sh ./scripts/build-xcuitest-apple.sh",
+    "build:xcuitest:tvos": "AGENT_DEVICE_XCUITEST_PLATFORM=tvos AGENT_DEVICE_IOS_CLEAN_DERIVED=1 sh ./scripts/build-xcuitest-apple.sh",
     "build:macos-helper": "swift build -c release --package-path macos-helper",
     "build:all": "pnpm build:node && pnpm build:xcuitest",
     "ad": "node bin/agent-device.mjs",
-    "format": "prettier --write src test skills",
+    "lint": "oxlint . --deny-warnings",
+    "format": "oxfmt --write src test skills package.json tsconfig.json .oxlintrc.json .oxfmtrc.json",
+    "check:quick": "pnpm lint && pnpm typecheck",
+    "check:tooling": "pnpm lint && pnpm typecheck && pnpm build",
+    "check:unit": "pnpm test:unit && pnpm test:smoke",
+    "check": "pnpm check:tooling && pnpm check:unit",
     "prepack": "pnpm build:all",
     "typecheck": "tsc -p tsconfig.json",
     "test": "vitest run",
@@ -66,22 +71,17 @@
     "ios",
     "android"
   ],
-  "prettier": {
-    "singleQuote": true,
-    "semi": true,
-    "trailingComma": "all",
-    "printWidth": 100
-  },
   "dependencies": {
     "pngjs": "^7.0.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",
-    "@rslib/core": "0.19.4",
+    "@rslib/core": "0.20.1",
     "@types/node": "^22.0.0",
     "@types/pngjs": "^6.0.5",
-    "prettier": "^3.3.3",
-    "typescript": "^5.9.3",
+    "oxfmt": "^0.42.0",
+    "oxlint": "^1.57.0",
+    "typescript": "^6.0.2",
     "vitest": "^4.1.2"
   }
 }

package/skills/agent-device/references/exploration.md

@@ -112,6 +112,19 @@ agent-device press 'id="camera_row" || label="Camera" role=button'
 agent-device is visible 'id="camera_settings_anchor"'
 ```
 
+## Interaction fallbacks
+
+When `press @ref` fails:
+
+1. Re-snapshot if the UI may have changed.
+2. Retry `press @ref` or a selector-based `press`.
+3. If `screenshot --overlay-refs --json` returned a reliable `overlayRefs[].center`, use `agent-device press <x> <y>`.
+4. Use an external vision-based tap tool only after semantic and coordinate targeting fail.
+
+- Prefer `@ref` over coordinates.
+- Do not guess coordinates from the image when structured `center` is available.
+- `agent-device` does not provide a built-in vision-tap flag.
+
 ## Text entry rules
 
 - Use `fill` to replace text in an editable field.

package/skills/agent-device/references/macos-desktop.md

@@ -30,7 +30,8 @@ agent-device close
 - `app`: default surface and the normal choice for `click`, `fill`, `press`, `scroll`, `screenshot`, and `record`.
 - `frontmost-app`: inspect the currently focused app without naming it first.
 - `desktop`: inspect visible desktop windows across apps.
-- `menubar`: inspect the active app menu bar and system menu extras.
+- `menubar`: inspect the active app menu bar and system menu extras. Use `open <app> --platform macos --surface menubar` when you need one menu bar app's extras, such as a status-item app.
+- Menu bar apps can expose a sparse or empty default `app` tree. Prefer the `menubar` surface first when the app lives entirely in the top bar.
 
 Use inspect-first surfaces to understand desktop-global UI, then switch back to `app` when you need to act in one app.
 
@@ -81,6 +82,6 @@ Troubleshooting:
 
 - If visible content is missing from `snapshot -i`, re-snapshot after the UI settles.
 - If `desktop` is too broad, retry with `frontmost-app`.
-- If `menubar` is missing the expected menu, make the app frontmost first and retry.
+- If `menubar` is missing the expected menu, retry with `open <app> --platform macos --surface menubar` for menu bar apps, or make the app frontmost first and retry the generic menubar surface.
 - If the wrong menu opened, retry secondary-clicking the row or cell wrapper rather than the nested text node.
 - If the app has multiple windows, make the correct window frontmost before relying on refs.

package/skills/agent-device/references/verification.md

@@ -22,7 +22,7 @@ Do not use verification tools as the first exploration step. First get the app i
 agent-device open Settings --platform ios
 # after using exploration to reach the state you want to verify
 agent-device snapshot
-agent-device screenshot /tmp/settings-proof.png
+agent-device screenshot /tmp/settings-proof.png --overlay-refs
 agent-device close
 ```
 
@@ -45,6 +45,8 @@ agent-device diff snapshot -i
 
 Use `screenshot` when the proof needs a rendered image instead of a structural tree.
 
+- Add `--overlay-refs` when you want the saved PNG to show fresh `@eN` refs burned into the screenshot.
+
 ## Session recording
 
 Use `record` for debugging, documentation, or shareable verification artifacts.