agent-device 0.4.2 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "Unified control plane for physical and virtual devices via an agent-driven CLI.",
   "license": "MIT",
   "author": "Callstack",
@@ -14,16 +14,15 @@
   "scripts": {
     "lint": "node --eval \"console.log('no lint')\"",
     "build": "rslib build",
-    "build:node": "pnpm build && rm -f ~/.agent-device/daemon.json",
-    "build:swift": "swift build -c release --package-path ios-runner/AXSnapshot",
-    "build:axsnapshot": "pnpm build:swift && mkdir -p dist/bin && cp -f ios-runner/AXSnapshot/.build/release/axsnapshot dist/bin/axsnapshot && chmod +x dist/bin/axsnapshot",
+    "clean:daemon": "rm -f ~/.agent-device/daemon.json && rm -f ~/.agent-device/daemon.lock",
+    "build:node": "pnpm build && pnpm clean:daemon",
     "build:xcuitest": "AGENT_DEVICE_IOS_CLEAN_DERIVED=1 xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination \"generic/platform=iOS Simulator\" -derivedDataPath ~/.agent-device/ios-runner/derived",
-    "build:clis": "pnpm build:node && pnpm build:axsnapshot",
-    "build:all": "pnpm build:node && pnpm build:axsnapshot && pnpm build:xcuitest",
+    "build:clis": "pnpm build:node",
+    "build:all": "pnpm build:node && pnpm build:xcuitest",
     "ad": "node bin/agent-device.mjs",
     "format": "prettier --write .",
-    "prepublishOnly": "pnpm build:node && pnpm build:axsnapshot",
-    "prepack": "pnpm build:node && pnpm build:axsnapshot",
+    "prepublishOnly": "pnpm build:node",
+    "prepack": "pnpm build:node",
     "typecheck": "tsc -p tsconfig.json",
     "test": "node --test",
     "test:unit": "node --test src/__tests__/*.test.ts src/core/__tests__/*.test.ts src/daemon/__tests__/*.test.ts src/daemon/handlers/__tests__/*.test.ts src/platforms/**/__tests__/*.test.ts src/utils/**/__tests__/*.test.ts",
@@ -39,7 +38,6 @@
     "!ios-runner/**/xcuserdata",
     "!ios-runner/**/*.xcuserstate",
     "skills",
-    "src",
     "README.md",
     "LICENSE"
   ],

package/skills/agent-device/SKILL.md

@@ -64,11 +64,9 @@ agent-device snapshot -c               # Compact output
 agent-device snapshot -d 3             # Limit depth
 agent-device snapshot -s "Camera"      # Scope to label/identifier
 agent-device snapshot --raw            # Raw node output
-agent-device snapshot --backend xctest # default: XCTest snapshot (fast, complete, no permissions)
-agent-device snapshot --backend ax     # macOS Accessibility tree (manual diagnostics only; no automatic fallback)
 ```
 
-XCTest is the default: fast and complete and does not require permissions. Use AX only for manual diagnostics, and prefer XCTest for normal automation flows. agent-device does not automatically fall back to AX.
+XCTest is the iOS snapshot engine: fast, complete, and no Accessibility permission required.
 
 ### Find (semantic)
 
@@ -157,7 +155,7 @@ agent-device replay -u ./session.ad   # Update selector drift and rewrite .ad sc
 `--save-script` path is a file path; parent directories are created automatically.
 For ambiguous bare values, use `--save-script=workflow.ad` or `./workflow.ad`.
 
-### Trace logs (AX/XCTest)
+### Trace logs (XCTest)
 
 ```bash
 agent-device trace start               # Start trace capture
@@ -187,12 +185,11 @@ agent-device apps --platform android --user-installed
 - Snapshot refs are the core mechanism for interactive agent flows.
 - Use selectors for deterministic replay artifacts and assertions (e.g. in e2e test workflows).
 - Prefer `snapshot -i` to reduce output size.
-- On iOS, `xctest` is the default and does not require Accessibility permission.
+- On iOS, snapshots use XCTest and do not require Accessibility permission.
 - If XCTest returns 0 nodes (foreground app changed), treat it as an explicit failure and retry the flow/app state.
 - `open <app|url> [url]` can be used within an existing session to switch apps or open deep links.
 - `open <app>` updates session app bundle context; `open <app> <url>` opens a deep link on iOS.
 - Use `open <app> --relaunch` during React Native/Fast Refresh debugging when you need a fresh app process without ending the session.
-- If AX returns the Simulator window or empty tree, restart Simulator or use `--backend xctest`.
 - Use `--session <name>` for parallel sessions; avoid device contention.
 - Use `--activity <component>` on Android to launch a specific activity (e.g. TV apps with LEANBACK); do not combine with URL opens.
 - On iOS devices, `http(s)://` URLs fall back to Safari automatically; custom scheme URLs require an active app in the session.

package/skills/agent-device/references/permissions.md

@@ -1,19 +1,8 @@
 # Permissions and Setup
 
-## iOS AX snapshot
+## iOS snapshots
 
-AX snapshot is available for manual diagnostics when needed; it is not used as an automatic fallback. It uses macOS Accessibility APIs and requires permission:
-
-System Settings > Privacy & Security > Accessibility
-
-If permission is missing, use XCTest backend:
-
-```bash
-agent-device snapshot --backend xctest --platform ios
-```
-
-Hybrid/AX is fast; XCTest is equally fast but does not require permissions.
-AX backend is simulator-only.
+iOS snapshots use XCTest and do not require macOS Accessibility permissions.
 
 ## iOS physical device runner
 
@@ -35,5 +24,4 @@ If daemon startup fails with stale metadata hints, clean stale files and retry:
 
 ## Simulator troubleshooting
 
-- If AX shows the Simulator chrome instead of app, restart Simulator.
-- If AX returns empty, restart Simulator and re-open app.
+- If snapshots return 0 nodes, restart Simulator and re-open the app.

package/skills/agent-device/references/snapshot-refs.md

@@ -53,10 +53,7 @@ agent-device snapshot -i -s @e3
 ## Troubleshooting
 
 - Ref not found: re-snapshot.
-- AX returns Simulator window: restart Simulator and re-run.
-- AX empty: verify Accessibility permission or use `--backend xctest` (XCTest is more complete).
-- AX backend is simulator-only; use `--backend xctest` on iOS devices.
-- agent-device does not automatically fall back to AX when XCTest fails.
+- If XCTest returns 0 nodes, foreground app state may have changed. Re-open the app or retry after state is stable.
 
 ## Replay note
 

package/ios-runner/AXSnapshot/Package.swift

@@ -1,18 +0,0 @@
-// swift-tools-version: 5.9
-import PackageDescription
-
-let package = Package(
-    name: "axsnapshot",
-    platforms: [
-        .macOS(.v13)
-    ],
-    products: [
-        .executable(name: "axsnapshot", targets: ["AXSnapshot"])
-    ],
-    targets: [
-        .executableTarget(
-            name: "AXSnapshot",
-            path: "Sources/AXSnapshot"
-        )
-    ]
-)

package/ios-runner/AXSnapshot/Sources/AXSnapshot/main.swift

@@ -1,444 +0,0 @@
-import Foundation
-import ApplicationServices
-import Cocoa
-
-struct AXNode: Codable {
-    struct Frame: Codable {
-        let x: Double
-        let y: Double
-        let width: Double
-        let height: Double
-    }
-
-    let role: String?
-    let subrole: String?
-    let label: String?
-    let value: String?
-    let identifier: String?
-    let frame: Frame?
-    let children: [AXNode]
-}
-
-struct AXSnapshotError: Error, CustomStringConvertible {
-    let message: String
-    var description: String { message }
-}
-
-let simulatorBundleId = "com.apple.iphonesimulator"
-let defaultMaxDepth = 40
-
-func hasAccessibilityPermission() -> Bool {
-    AXIsProcessTrusted()
-}
-
-func findSimulatorApp() -> NSRunningApplication? {
-    NSWorkspace.shared.runningApplications.first { $0.bundleIdentifier == simulatorBundleId }
-}
-
-func axElement(for app: NSRunningApplication) -> AXUIElement {
-    AXUIElementCreateApplication(app.processIdentifier)
-}
-
-func getAttribute<T>(_ element: AXUIElement, _ attribute: CFString) -> T? {
-    var value: AnyObject?
-    let result = AXUIElementCopyAttributeValue(element, attribute, &value)
-    guard result == .success else { return nil }
-    return value as? T
-}
-
-func getChildren(_ element: AXUIElement) -> [AXUIElement] {
-    if let children: [AXUIElement] = getAttribute(element, kAXChildrenAttribute as CFString),
-       !children.isEmpty {
-        return children
-    }
-    if let children: [AXUIElement] = getAttribute(element, kAXVisibleChildrenAttribute as CFString),
-       !children.isEmpty {
-        return children
-    }
-    if let children: [AXUIElement] = getAttribute(element, kAXContentsAttribute as CFString),
-       !children.isEmpty {
-        return children
-    }
-    return []
-}
-
-func getLabel(_ element: AXUIElement) -> String? {
-    if let label: String = getAttribute(element, "AXLabel" as CFString) {
-        return label
-    }
-    if let desc: String = getAttribute(element, kAXDescriptionAttribute as CFString) {
-        return desc
-    }
-    return nil
-}
-
-func getDescription(_ element: AXUIElement) -> String? {
-    getAttribute(element, kAXDescriptionAttribute as CFString)
-}
-
-func getValue(_ element: AXUIElement) -> String? {
-    if let value: String = getAttribute(element, kAXValueAttribute as CFString) {
-        return value
-    }
-    if let number: NSNumber = getAttribute(element, kAXValueAttribute as CFString) {
-        return number.stringValue
-    }
-    return nil
-}
-
-func getIdentifier(_ element: AXUIElement) -> String? {
-    getAttribute(element, kAXIdentifierAttribute as CFString)
-}
-
-func getFrame(_ element: AXUIElement) -> AXNode.Frame? {
-    var positionRef: CFTypeRef?
-    var sizeRef: CFTypeRef?
-    AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionRef)
-    AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeRef)
-    guard let posValue = positionRef, let sizeValue = sizeRef else {
-        return nil
-    }
-    if CFGetTypeID(posValue) != AXValueGetTypeID() || CFGetTypeID(sizeValue) != AXValueGetTypeID() {
-        return nil
-    }
-    let posAx = posValue as! AXValue
-    let sizeAx = sizeValue as! AXValue
-    var point = CGPoint.zero
-    var size = CGSize.zero
-    AXValueGetValue(posAx, .cgPoint, &point)
-    AXValueGetValue(sizeAx, .cgSize, &size)
-    return AXNode.Frame(
-        x: Double(point.x),
-        y: Double(point.y),
-        width: Double(size.width),
-        height: Double(size.height)
-    )
-}
-
-func buildTree(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = defaultMaxDepth) -> AXNode {
-    let children = depth < maxDepth
-        ? getChildren(element).map { buildTree($0, depth: depth + 1, maxDepth: maxDepth) }
-        : []
-    return AXNode(
-        role: getAttribute(element, kAXRoleAttribute as CFString),
-        subrole: getAttribute(element, kAXSubroleAttribute as CFString),
-        label: getLabel(element),
-        value: getValue(element),
-        identifier: getIdentifier(element),
-        frame: getFrame(element),
-        children: children
-    )
-}
-
-func findIOSAppSnapshot(in simulator: NSRunningApplication) -> (AXUIElement, AXNode.Frame?, AXUIElement, [AXUIElement], [AXUIElement])? {
-    let appElement = axElement(for: simulator)
-    let windows = getChildren(appElement).filter {
-        (getAttribute($0, kAXRoleAttribute as CFString) as String?) == (kAXWindowRole as String)
-    }
-    if windows.isEmpty { return nil }
-
-    if let focused: AXUIElement = getAttribute(appElement, kAXFocusedWindowAttribute as CFString),
-       let root = chooseRoot(in: focused) {
-        let extras = dedupeElements(findToolbarExtras(in: focused, root: root) + findTabBarExtras(in: focused, root: root))
-        let modalRoots = findAdditionalWindowRoots(windows: windows, excluding: focused, windowFrame: getFrame(focused))
-        return (root, getFrame(focused), focused, extras, modalRoots)
-    }
-
-    let sorted = windows.sorted { lhs, rhs in
-        let l = getFrame(lhs)
-        let r = getFrame(rhs)
-        let la = (l?.width ?? 0) * (l?.height ?? 0)
-        let ra = (r?.width ?? 0) * (r?.height ?? 0)
-        return la > ra
-    }
-    for window in sorted {
-        if let root = chooseRoot(in: window) {
-            let extras = dedupeElements(findToolbarExtras(in: window, root: root) + findTabBarExtras(in: window, root: root))
-            let modalRoots = findAdditionalWindowRoots(windows: windows, excluding: window, windowFrame: getFrame(window))
-            return (root, getFrame(window), window, extras, modalRoots)
-        }
-    }
-    return nil
-}
-
-private func findAdditionalWindowRoots(
-    windows: [AXUIElement],
-    excluding mainWindow: AXUIElement,
-    windowFrame: AXNode.Frame?
-) -> [AXUIElement] {
-    var roots: [AXUIElement] = []
-    for window in windows {
-        if CFEqual(window, mainWindow) { continue }
-        let frame = getFrame(window)
-        if let windowFrame = windowFrame, !frameIntersects(frame, windowFrame) {
-            continue
-        }
-        if let root = chooseRoot(in: window) {
-            roots.append(root)
-        }
-    }
-    return dedupeElements(roots)
-}
-
-private func dedupeElements(_ elements: [AXUIElement]) -> [AXUIElement] {
-    var seen: Set<AXWrapper> = []
-    var result: [AXUIElement] = []
-    for element in elements {
-        let wrapper = AXWrapper(element)
-        if seen.contains(wrapper) { continue }
-        seen.insert(wrapper)
-        result.append(element)
-    }
-    return result
-}
-
-func chooseRoot(in window: AXUIElement) -> AXUIElement? {
-    let windowFrame = getFrame(window)
-    let candidates = findGroupCandidates(in: window, windowFrame: windowFrame)
-    if let best = candidates.first?.element { return best }
-    return findLargestChildCandidate(in: window, windowFrame: windowFrame)
-}
-
-private func findLargestChildCandidate(in window: AXUIElement, windowFrame: AXNode.Frame?) -> AXUIElement? {
-    var best: (element: AXUIElement, area: Double)? = nil
-    for child in getChildren(window) {
-        let children = getChildren(child)
-        if children.isEmpty { continue }
-        let area = frameArea(getFrame(child), windowFrame: windowFrame)
-        if area <= 0 { continue }
-        if best == nil || area > best!.area {
-            best = (child, area)
-        }
-    }
-    return best?.element
-}
-
-private func frameIntersects(_ frame: AXNode.Frame?, _ target: AXNode.Frame?) -> Bool {
-    guard let frame = frame, let target = target else { return false }
-    let fx1 = frame.x
-    let fy1 = frame.y
-    let fx2 = frame.x + frame.width
-    let fy2 = frame.y + frame.height
-    let tx1 = target.x
-    let ty1 = target.y
-    let tx2 = target.x + target.width
-    let ty2 = target.y + target.height
-    return fx1 < tx2 && fx2 > tx1 && fy1 < ty2 && fy2 > ty1
-}
-
-private func isToolbarLike(_ element: AXUIElement) -> Bool {
-    let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
-    let subrole = (getAttribute(element, kAXSubroleAttribute as CFString) as String?) ?? ""
-    if role == (kAXToolbarRole as String) ||
-        role == (kAXTabGroupRole as String) ||
-        role == "AXTabBar" {
-        return true
-    }
-    if subrole == "AXTabBar" {
-        return true
-    }
-    return false
-}
-
-private func isTabBarLike(_ element: AXUIElement) -> Bool {
-    let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
-    let subrole = (getAttribute(element, kAXSubroleAttribute as CFString) as String?) ?? ""
-    if role == (kAXTabGroupRole as String) || role == "AXTabBar" { return true }
-    if subrole == "AXTabBar" { return true }
-    let desc = (getDescription(element) ?? "").lowercased()
-    if desc.contains("tab bar") { return true }
-    let label = (getLabel(element) ?? "").lowercased()
-    if label.contains("tab bar") { return true }
-    return false
-}
-
-private func findToolbarExtras(in window: AXUIElement, root: AXUIElement) -> [AXUIElement] {
-    let rootFrame = getFrame(root)
-    let rootIds = collectDescendantWrappers(from: root)
-    var extras: [AXUIElement] = []
-    var stack = getChildren(window)
-    while !stack.isEmpty {
-        let current = stack.removeLast()
-        if isToolbarLike(current) && !rootIds.contains(AXWrapper(current)) {
-            let frame = getFrame(current)
-            if frameIntersects(frame, rootFrame) {
-                extras.append(current)
-            }
-        }
-        stack.append(contentsOf: getChildren(current))
-    }
-    return extras
-}
-
-private func findTabBarExtras(in window: AXUIElement, root: AXUIElement) -> [AXUIElement] {
-    let rootFrame = getFrame(root)
-    let rootIds = collectDescendantWrappers(from: root)
-    var extras: [AXUIElement] = []
-    var stack = getChildren(window)
-    while !stack.isEmpty {
-        let current = stack.removeLast()
-        if isTabBarLike(current) && !rootIds.contains(AXWrapper(current)) {
-            let frame = getFrame(current)
-            if frameIntersects(frame, rootFrame) {
-                extras.append(current)
-            }
-        }
-        stack.append(contentsOf: getChildren(current))
-    }
-    return extras
-}
-
-private struct GroupCandidate {
-    let element: AXUIElement
-    let area: Double
-    let childCount: Int
-}
-
-private func findGroupCandidates(in root: AXUIElement, windowFrame: AXNode.Frame?) -> [GroupCandidate] {
-    var candidates: [GroupCandidate] = []
-    var visited: Set<AXWrapper> = []
-    func walk(_ element: AXUIElement) {
-        let wrapper = AXWrapper(element)
-        if visited.contains(wrapper) { return }
-        visited.insert(wrapper)
-        let children = getChildren(element)
-        let role = (getAttribute(element, kAXRoleAttribute as CFString) as String?) ?? ""
-        let isContainer = role == (kAXGroupRole as String) ||
-            role == (kAXScrollAreaRole as String) ||
-            role == (kAXUnknownRole as String)
-        if isContainer {
-            let hasNonToolbarChild = children.contains {
-                ((getAttribute($0, kAXRoleAttribute as CFString) as String?) ?? "") != (kAXToolbarRole as String)
-            }
-            if hasNonToolbarChild {
-                let frame = getFrame(element)
-                let area = frameArea(frame, windowFrame: windowFrame)
-                if area > 0 {
-                    let childCount = children.count
-                    candidates.append(
-                        GroupCandidate(
-                            element: element,
-                            area: area,
-                            childCount: childCount
-                        )
-                    )
-                }
-            }
-        }
-        for child in children {
-            walk(child)
-        }
-    }
-    walk(root)
-    candidates.sort { lhs, rhs in
-        if lhs.area == rhs.area { return lhs.childCount > rhs.childCount }
-        return lhs.area > rhs.area
-    }
-    return candidates
-}
-
-private func frameArea(_ frame: AXNode.Frame?, windowFrame: AXNode.Frame?) -> Double {
-    guard let frame = frame else { return 0 }
-    if let windowFrame = windowFrame {
-        let windowArea = max(1.0, windowFrame.width * windowFrame.height)
-        let area = frame.width * frame.height
-        if area > windowArea { return 0 }
-        return area
-    }
-    return frame.width * frame.height
-}
-
-private final class AXWrapper: Hashable {
-    let element: AXUIElement
-    init(_ element: AXUIElement) { self.element = element }
-    func hash(into hasher: inout Hasher) { hasher.combine(CFHash(element)) }
-    static func == (lhs: AXWrapper, rhs: AXWrapper) -> Bool {
-        return CFEqual(lhs.element, rhs.element)
-    }
-}
-
-private func collectDescendantWrappers(from root: AXUIElement) -> Set<AXWrapper> {
-    var seen: Set<AXWrapper> = []
-    var stack = [root]
-    while !stack.isEmpty {
-        let current = stack.removeLast()
-        let wrapper = AXWrapper(current)
-        if seen.contains(wrapper) { continue }
-        seen.insert(wrapper)
-        stack.append(contentsOf: getChildren(current))
-    }
-    return seen
-}
-
-
-private struct SnapshotPayload: Codable {
-    let windowFrame: AXNode.Frame?
-    let root: AXNode
-}
-
-func main() throws {
-    guard hasAccessibilityPermission() else {
-        throw AXSnapshotError(message: "Accessibility permission not granted. Enable it in System Settings > Privacy & Security > Accessibility.")
-    }
-    guard let simulator = findSimulatorApp() else {
-        throw AXSnapshotError(message: "iOS Simulator is not running.")
-    }
-    let maxAttempts = 5
-    var snapshot: (AXUIElement, AXNode.Frame?, AXUIElement, [AXUIElement], [AXUIElement])? = nil
-    for attempt in 0..<maxAttempts {
-        if let candidate = findIOSAppSnapshot(in: simulator) {
-            let (root, _, _, _, modalRoots) = candidate
-            if !getChildren(root).isEmpty || !modalRoots.isEmpty {
-                snapshot = candidate
-                break
-            }
-        }
-        if attempt < maxAttempts - 1 {
-            usleep(300_000)
-        }
-    }
-    guard let (root, windowFrame, _, extras, modalRoots) = snapshot else {
-        throw AXSnapshotError(message: "Could not find iOS app content in Simulator.")
-    }
-    var tree = buildTree(root)
-    if !extras.isEmpty {
-        let extraNodes = extras.map { buildTree($0) }
-        tree = AXNode(
-            role: tree.role,
-            subrole: tree.subrole,
-            label: tree.label,
-            value: tree.value,
-            identifier: tree.identifier,
-            frame: tree.frame,
-            children: tree.children + extraNodes
-        )
-    }
-    if !modalRoots.isEmpty {
-        let modalNodes = modalRoots.map { buildTree($0) }
-        tree = AXNode(
-            role: tree.role,
-            subrole: tree.subrole,
-            label: tree.label,
-            value: tree.value,
-            identifier: tree.identifier,
-            frame: tree.frame,
-            children: tree.children + modalNodes
-        )
-    }
-    let payload = SnapshotPayload(windowFrame: windowFrame, root: tree)
-    let encoder = JSONEncoder()
-    encoder.outputFormatting = [.sortedKeys]
-    let data = try encoder.encode(payload)
-    if let json = String(data: data, encoding: .utf8) {
-        print(json)
-    } else {
-        throw AXSnapshotError(message: "Failed to encode AX snapshot JSON.")
-    }
-}
-
-do {
-    try main()
-} catch {
-    fputs("axsnapshot error: \(error)\n", stderr)
-    exit(1)
-}