agent-device 0.1.1 → 0.1.3

package/ios-runner/AXSnapshot/Sources/AXSnapshot/main.swift

@@ -19,17 +19,13 @@ struct AXNode: Codable {
     let children: [AXNode]
 }
 
-struct AXSnapshot: Codable {
-    let windowFrame: AXNode.Frame?
-    let root: AXNode
-}
-
 struct AXSnapshotError: Error, CustomStringConvertible {
     let message: String
     var description: String { message }
 }
 
 let simulatorBundleId = "com.apple.iphonesimulator"
+let defaultMaxDepth = 40
 
 func hasAccessibilityPermission() -> Bool {
     AXIsProcessTrusted()
@@ -51,15 +47,19 @@ func getAttribute<T>(_ element: AXUIElement, _ attribute: CFString) -> T? {
 }
 
 func getChildren(_ element: AXUIElement) -> [AXUIElement] {
-    getAttribute(element, kAXChildrenAttribute as CFString) ?? []
-}
-
-func getRole(_ element: AXUIElement) -> String? {
-    getAttribute(element, kAXRoleAttribute as CFString)
-}
-
-func getSubrole(_ element: AXUIElement) -> String? {
-    getAttribute(element, kAXSubroleAttribute as CFString)
+    if let children: [AXUIElement] = getAttribute(element, kAXChildrenAttribute as CFString),
+       !children.isEmpty {
+        return children
+    }
+    if let children: [AXUIElement] = getAttribute(element, kAXVisibleChildrenAttribute as CFString),
+       !children.isEmpty {
+        return children
+    }
+    if let children: [AXUIElement] = getAttribute(element, kAXContentsAttribute as CFString),
+       !children.isEmpty {
+        return children
+    }
+    return []
 }
 
 func getLabel(_ element: AXUIElement) -> String? {
@@ -72,6 +72,10 @@ func getLabel(_ element: AXUIElement) -> String? {
     return nil
 }
 
+func getDescription(_ element: AXUIElement) -> String? {
+    getAttribute(element, kAXDescriptionAttribute as CFString)
+}
+
 func getValue(_ element: AXUIElement) -> String? {
     if let value: String = getAttribute(element, kAXValueAttribute as CFString) {
         return value
@@ -111,11 +115,13 @@ func getFrame(_ element: AXUIElement) -> AXNode.Frame? {
     )
 }
 
-func buildTree(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 40) -> AXNode {
-    let children = depth < maxDepth ? getChildren(element).map { buildTree($0, depth: depth + 1, maxDepth: maxDepth) } : []
+func buildTree(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = defaultMaxDepth) -> AXNode {
+    let children = depth < maxDepth
+        ? getChildren(element).map { buildTree($0, depth: depth + 1, maxDepth: maxDepth) }
+        : []
     return AXNode(
-        role: getRole(element),
-        subrole: getSubrole(element),
+        role: getAttribute(element, kAXRoleAttribute as CFString),
+        subrole: getAttribute(element, kAXSubroleAttribute as CFString),
         label: getLabel(element),
         value: getValue(element),
         identifier: getIdentifier(element),
@@ -124,17 +130,250 @@ func buildTree(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 40) -> AX
     )
 }
 
-func findIOSAppRoot(in simulator: NSRunningApplication) -> (AXUIElement, AXNode.Frame?)? {
+func findIOSAppSnapshot(in simulator: NSRunningApplication) -> (AXUIElement, AXNode.Frame?, AXUIElement, [AXUIElement], [AXUIElement])? {
     let appElement = axElement(for: simulator)
-    let windows = getChildren(appElement).filter { getRole($0) == "AXWindow" }
+    let windows = getChildren(appElement).filter {
+        (getAttribute($0, kAXRoleAttribute as CFString) as String?) == (kAXWindowRole as String)
+    }
+    if windows.isEmpty { return nil }
+
+    if let focused: AXUIElement = getAttribute(appElement, kAXFocusedWindowAttribute as CFString),
+       let root = chooseRoot(in: focused) {
+        let extras = dedupeElements(findToolbarExtras(in: focused, root: root) + findTabBarExtras(in: focused, root: root))
+        let modalRoots = findAdditionalWindowRoots(windows: windows, excluding: focused, windowFrame: getFrame(focused))
+        return (root, getFrame(focused), focused, extras, modalRoots)
+    }
+
+    let sorted = windows.sorted { lhs, rhs in
+        let l = getFrame(lhs)
+        let r = getFrame(rhs)
+        let la = (l?.width ?? 0) * (l?.height ?? 0)
+        let ra = (r?.width ?? 0) * (r?.height ?? 0)
+        return la > ra
+    }
+    for window in sorted {
+        if let root = chooseRoot(in: window) {
+            let extras = dedupeElements(findToolbarExtras(in: window, root: root) + findTabBarExtras(in: window, root: root))
+            let modalRoots = findAdditionalWindowRoots(windows: windows, excluding: window, windowFrame: getFrame(window))
+            return (root, getFrame(window), window, extras, modalRoots)
+        }
+    }
+    return nil
+}
+
+private func findAdditionalWindowRoots(
+    windows: [AXUIElement],
+    excluding mainWindow: AXUIElement,
+    windowFrame: AXNode.Frame?
+) -> [AXUIElement] {
+    var roots: [AXUIElement] = []
     for window in windows {
-        for child in getChildren(window) {
-            if getRole(child) == "AXGroup" {
-                return (child, getFrame(window))
+        if CFEqual(window, mainWindow) { continue }
+        let frame = getFrame(window)
+        if let windowFrame = windowFrame, !frameIntersects(frame, windowFrame) {
+            continue
+        }
+        if let root = chooseRoot(in: window) {
+            roots.append(root)
+        }
+    }
+    return dedupeElements(roots)
+}
+
+private func dedupeElements(_ elements: [AXUIElement]) -> [AXUIElement] {
+    var seen: Set<AXWrapper> = []
+    var result: [AXUIElement] = []
+    for element in elements {
+        let wrapper = AXWrapper(element)
+        if seen.contains(wrapper) { continue }
+        seen.insert(wrapper)
+        result.append(element)
+    }
+    return result
+}
+
+func chooseRoot(in window: AXUIElement) -> AXUIElement? {
+    let windowFrame = getFrame(window)
+    let candidates = findGroupCandidates(in: window, windowFrame: windowFrame)
+    if let best = candidates.first?.element { return best }
+    return findLargestChildCandidate(in: window, windowFrame: windowFrame)
+}
+
+private func findLargestChildCandidate(in window: AXUIElement, windowFrame: AXNode.Frame?) -> AXUIElement? {
+    var best: (element: AXUIElement, area: Double)? = nil
+    for child in getChildren(window) {
+        let children = getChildren(child)
+        if children.isEmpty { continue }
+        let area = frameArea(getFrame(child), windowFrame: windowFrame)
+        if area <= 0 { continue }
+        if best == nil || area > best!.area {
+            best = (child, area)
+        }
+    }
+    return best?.element
+}
+
+private func frameIntersects(_ frame: AXNode.Frame?, _ target: AXNode.Frame?) -> Bool {
+    guard let frame = frame, let target = target else { return false }
+    let fx1 = frame.x
+    let fy1 = frame.y
+    let fx2 = frame.x + frame.width
+    let fy2 = frame.y + frame.height
+    let tx1 = target.x
+    let ty1 = target.y
+    let tx2 = target.x + target.width
+    let ty2 = target.y + target.height
+    return fx1 < tx2 && fx2 > tx1 && fy1 < ty2 && fy2 > ty1
+}
+
+private func isToolbarLike(_ element: AXUIElement) -> Bool {
+    let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
+    let subrole = (getAttribute(element, kAXSubroleAttribute as CFString) as String?) ?? ""
+    if role == (kAXToolbarRole as String) ||
+        role == (kAXTabGroupRole as String) ||
+        role == "AXTabBar" {
+        return true
+    }
+    if subrole == "AXTabBar" {
+        return true
+    }
+    return false
+}
+
+private func isTabBarLike(_ element: AXUIElement) -> Bool {
+    let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
+    let subrole = (getAttribute(element, kAXSubroleAttribute as CFString) as String?) ?? ""
+    if role == (kAXTabGroupRole as String) || role == "AXTabBar" { return true }
+    if subrole == "AXTabBar" { return true }
+    let desc = (getDescription(element) ?? "").lowercased()
+    if desc.contains("tab bar") { return true }
+    let label = (getLabel(element) ?? "").lowercased()
+    if label.contains("tab bar") { return true }
+    return false
+}
+
+private func findToolbarExtras(in window: AXUIElement, root: AXUIElement) -> [AXUIElement] {
+    let rootFrame = getFrame(root)
+    let rootIds = collectDescendantWrappers(from: root)
+    var extras: [AXUIElement] = []
+    var stack = getChildren(window)
+    while !stack.isEmpty {
+        let current = stack.removeLast()
+        if isToolbarLike(current) && !rootIds.contains(AXWrapper(current)) {
+            let frame = getFrame(current)
+            if frameIntersects(frame, rootFrame) {
+                extras.append(current)
             }
         }
+        stack.append(contentsOf: getChildren(current))
     }
-    return nil
+    return extras
+}
+
+private func findTabBarExtras(in window: AXUIElement, root: AXUIElement) -> [AXUIElement] {
+    let rootFrame = getFrame(root)
+    let rootIds = collectDescendantWrappers(from: root)
+    var extras: [AXUIElement] = []
+    var stack = getChildren(window)
+    while !stack.isEmpty {
+        let current = stack.removeLast()
+        if isTabBarLike(current) && !rootIds.contains(AXWrapper(current)) {
+            let frame = getFrame(current)
+            if frameIntersects(frame, rootFrame) {
+                extras.append(current)
+            }
+        }
+        stack.append(contentsOf: getChildren(current))
+    }
+    return extras
+}
+
+private struct GroupCandidate {
+    let element: AXUIElement
+    let area: Double
+    let childCount: Int
+}
+
+private func findGroupCandidates(in root: AXUIElement, windowFrame: AXNode.Frame?) -> [GroupCandidate] {
+    var candidates: [GroupCandidate] = []
+    var visited: Set<AXWrapper> = []
+    func walk(_ element: AXUIElement) {
+        let wrapper = AXWrapper(element)
+        if visited.contains(wrapper) { return }
+        visited.insert(wrapper)
+        let children = getChildren(element)
+        let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
+        let isContainer = role == (kAXGroupRole as String) ||
+            role == (kAXScrollAreaRole as String) ||
+            role == (kAXUnknownRole as String)
+        if isContainer {
+            let hasNonToolbarChild = children.contains {
+                ((getAttribute($0, kAXRoleAttribute as CFString) as String?) ?? "") != (kAXToolbarRole as String)
+            }
+            if hasNonToolbarChild {
+                let frame = getFrame(element)
+                let area = frameArea(frame, windowFrame: windowFrame)
+                if area > 0 {
+                    let childCount = children.count
+                    candidates.append(
+                        GroupCandidate(
+                            element: element,
+                            area: area,
+                            childCount: childCount
+                        )
+                    )
+                }
+            }
+        }
+        for child in children {
+            walk(child)
+        }
+    }
+    walk(root)
+    candidates.sort { lhs, rhs in
+        if lhs.area == rhs.area { return lhs.childCount > rhs.childCount }
+        return lhs.area > rhs.area
+    }
+    return candidates
+}
+
+private func frameArea(_ frame: AXNode.Frame?, windowFrame: AXNode.Frame?) -> Double {
+    guard let frame = frame else { return 0 }
+    if let windowFrame = windowFrame {
+        let windowArea = max(1.0, windowFrame.width * windowFrame.height)
+        let area = frame.width * frame.height
+        if area > windowArea { return 0 }
+        return area
+    }
+    return frame.width * frame.height
+}
+
+private final class AXWrapper: Hashable {
+    let element: AXUIElement
+    init(_ element: AXUIElement) { self.element = element }
+    func hash(into hasher: inout Hasher) { hasher.combine(CFHash(element)) }
+    static func == (lhs: AXWrapper, rhs: AXWrapper) -> Bool {
+        return CFEqual(lhs.element, rhs.element)
+    }
+}
+
+private func collectDescendantWrappers(from root: AXUIElement) -> Set<AXWrapper> {
+    var seen: Set<AXWrapper> = []
+    var stack = [root]
+    while !stack.isEmpty {
+        let current = stack.removeLast()
+        let wrapper = AXWrapper(current)
+        if seen.contains(wrapper) { continue }
+        seen.insert(wrapper)
+        stack.append(contentsOf: getChildren(current))
+    }
+    return seen
+}
+
+
+private struct SnapshotPayload: Codable {
+    let windowFrame: AXNode.Frame?
+    let root: AXNode
 }
 
 func main() throws {
@@ -144,14 +383,52 @@ func main() throws {
     guard let simulator = findSimulatorApp() else {
         throw AXSnapshotError(message: "iOS Simulator is not running.")
     }
-    guard let (root, windowFrame) = findIOSAppRoot(in: simulator) else {
+    let maxAttempts = 5
+    var snapshot: (AXUIElement, AXNode.Frame?, AXUIElement, [AXUIElement], [AXUIElement])? = nil
+    for attempt in 0..<maxAttempts {
+        if let candidate = findIOSAppSnapshot(in: simulator) {
+            let (root, _, _, _, modalRoots) = candidate
+            if !getChildren(root).isEmpty || !modalRoots.isEmpty {
+                snapshot = candidate
+                break
+            }
+        }
+        if attempt < maxAttempts - 1 {
+            usleep(300_000)
+        }
+    }
+    guard let (root, windowFrame, _, extras, modalRoots) = snapshot else {
         throw AXSnapshotError(message: "Could not find iOS app content in Simulator.")
     }
-    let tree = buildTree(root)
-    let snapshot = AXSnapshot(windowFrame: windowFrame, root: tree)
+    var tree = buildTree(root)
+    if !extras.isEmpty {
+        let extraNodes = extras.map { buildTree($0) }
+        tree = AXNode(
+            role: tree.role,
+            subrole: tree.subrole,
+            label: tree.label,
+            value: tree.value,
+            identifier: tree.identifier,
+            frame: tree.frame,
+            children: tree.children + extraNodes
+        )
+    }
+    if !modalRoots.isEmpty {
+        let modalNodes = modalRoots.map { buildTree($0) }
+        tree = AXNode(
+            role: tree.role,
+            subrole: tree.subrole,
+            label: tree.label,
+            value: tree.value,
+            identifier: tree.identifier,
+            frame: tree.frame,
+            children: tree.children + modalNodes
+        )
+    }
+    let payload = SnapshotPayload(windowFrame: windowFrame, root: tree)
     let encoder = JSONEncoder()
     encoder.outputFormatting = [.sortedKeys]
-    let data = try encoder.encode(snapshot)
+    let data = try encoder.encode(payload)
     if let json = String(data: data, encoding: .utf8) {
         print(json)
     } else {

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -249,21 +249,6 @@ final class RunnerTests: XCTestCase {
       }
       let found = findElement(app: activeApp, text: text) != nil
       return Response(ok: true, data: DataPayload(found: found))
-    case .rect:
-      guard let text = command.text else {
-        return Response(ok: false, error: ErrorPayload(message: "rect requires text"))
-      }
-      guard let element = findElement(app: activeApp, text: text) else {
-        return Response(ok: false, error: ErrorPayload(message: "element not found"))
-      }
-      let frame = element.frame
-      let rect = SnapshotRect(
-        x: Double(frame.origin.x),
-        y: Double(frame.origin.y),
-        width: Double(frame.size.width),
-        height: Double(frame.size.height)
-      )
-      return Response(ok: true, data: DataPayload(rect: rect))
     case .listTappables:
       let elements = activeApp.descendants(matching: .any).allElementsBoundByIndex
       let labels = elements.compactMap { element -> String? in
@@ -287,9 +272,62 @@ final class RunnerTests: XCTestCase {
         return Response(ok: true, data: snapshotRaw(app: activeApp, options: options))
       }
       return Response(ok: true, data: snapshotFast(app: activeApp, options: options))
+    case .back:
+      if tapNavigationBack(app: activeApp) {
+        return Response(ok: true, data: DataPayload(message: "back"))
+      }
+      performBackGesture(app: activeApp)
+      return Response(ok: true, data: DataPayload(message: "back"))
+    case .home:
+      XCUIDevice.shared.press(.home)
+      return Response(ok: true, data: DataPayload(message: "home"))
+    case .appSwitcher:
+      performAppSwitcherGesture(app: activeApp)
+      return Response(ok: true, data: DataPayload(message: "appSwitcher"))
+    case .alert:
+      let action = (command.action ?? "get").lowercased()
+      let alert = activeApp.alerts.firstMatch
+      if !alert.exists {
+        return Response(ok: false, error: ErrorPayload(message: "alert not found"))
+      }
+      if action == "accept" {
+        let button = alert.buttons.allElementsBoundByIndex.first
+        button?.tap()
+        return Response(ok: true, data: DataPayload(message: "accepted"))
+      }
+      if action == "dismiss" {
+        let button = alert.buttons.allElementsBoundByIndex.last
+        button?.tap()
+        return Response(ok: true, data: DataPayload(message: "dismissed"))
+      }
+      let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
+      return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
     }
   }
 
+  private func tapNavigationBack(app: XCUIApplication) -> Bool {
+    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
+    if let back = buttons.first(where: { $0.isHittable }) {
+      back.tap()
+      return true
+    }
+    return false
+  }
+
+  private func performBackGesture(app: XCUIApplication) {
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
+    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
+    start.press(forDuration: 0.05, thenDragTo: end)
+  }
+
+  private func performAppSwitcherGesture(app: XCUIApplication) {
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
+    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
+    start.press(forDuration: 0.6, thenDragTo: end)
+  }
+
   private func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
     let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
     let element = app.descendants(matching: .any).matching(predicate).firstMatch
@@ -602,7 +640,10 @@ enum CommandType: String, Codable {
   case findText
   case listTappables
   case snapshot
-  case rect
+  case back
+  case home
+  case appSwitcher
+  case alert
   case shutdown
 }
 
@@ -617,6 +658,7 @@ struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
   let text: String?
+  let action: String?
   let x: Double?
   let y: Double?
   let direction: SwipeDirection?
@@ -645,22 +687,19 @@ struct DataPayload: Codable {
   let items: [String]?
   let nodes: [SnapshotNode]?
   let truncated: Bool?
-  let rect: SnapshotRect?
 
   init(
     message: String? = nil,
     found: Bool? = nil,
     items: [String]? = nil,
     nodes: [SnapshotNode]? = nil,
-    truncated: Bool? = nil,
-    rect: SnapshotRect? = nil
+    truncated: Bool? = nil
   ) {
     self.message = message
     self.found = found
     self.items = items
     self.nodes = nodes
     self.truncated = truncated
-    self.rect = rect
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Unified control plane for physical and virtual devices via an agent-driven CLI.",
   "license": "MIT",
   "author": "Callstack",
@@ -16,8 +16,10 @@
     "build": "rslib build",
     "build:node": "pnpm build && rm -f ~/.agent-device/daemon.json",
     "build:swift": "swift build -c release --package-path ios-runner/AXSnapshot",
-    "build:axsnapshot": "pnpm build:swift && mkdir -p bin && cp -f ios-runner/AXSnapshot/.build/release/axsnapshot bin/axsnapshot && chmod +x bin/axsnapshot",
+    "build:axsnapshot": "pnpm build:swift && mkdir -p dist/bin && cp -f ios-runner/AXSnapshot/.build/release/axsnapshot dist/bin/axsnapshot && chmod +x dist/bin/axsnapshot",
+    "build:xcuitest": "AGENT_DEVICE_IOS_CLEAN_DERIVED=1 xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination \"generic/platform=iOS Simulator\" -derivedDataPath ~/.agent-device/ios-runner/derived",
     "build:clis": "pnpm build:node && pnpm build:axsnapshot",
+    "build:all": "pnpm build:node && pnpm build:axsnapshot && pnpm build:xcuitest",
     "format": "prettier --write .",
     "prepublishOnly": "pnpm build:node && pnpm build:axsnapshot",
     "prepack": "pnpm build:node && pnpm build:axsnapshot",
@@ -29,7 +31,6 @@
   "files": [
     "bin",
     "dist",
-    "bin/axsnapshot",
     "ios-runner",
     "!ios-runner/**/.build",
     "!ios-runner/**/.swiftpm",

package/src/cli.ts

@@ -84,6 +84,28 @@ export async function runCli(argv: string[]): Promise<void> {
         if (logTailStopper) logTailStopper();
         return;
       }
+      if (response.data && typeof response.data === 'object') {
+        const data = response.data as Record<string, unknown>;
+        if (command === 'devices') {
+          const devices = Array.isArray((data as any).devices) ? (data as any).devices : [];
+          const lines = devices.map((d: any) => {
+            const name = d?.name ?? d?.id ?? 'unknown';
+            const platform = d?.platform ?? 'unknown';
+            const kind = d?.kind ? ` ${d.kind}` : '';
+            const booted = typeof d?.booted === 'boolean' ? ` booted=${d.booted}` : '';
+            return `${name} (${platform}${kind})${booted}`;
+          });
+          process.stdout.write(`${lines.join('\n')}\n`);
+          if (logTailStopper) logTailStopper();
+          return;
+        }
+        if (command === 'apps') {
+          const apps = Array.isArray((data as any).apps) ? (data as any).apps : [];
+          process.stdout.write(`${apps.join('\n')}\n`);
+          if (logTailStopper) logTailStopper();
+          return;
+        }
+      }
       if (logTailStopper) logTailStopper();
       return;
     }