@swmansion/argent 0.23.0 → 0.23.1-next.0

package/dist/ios-device-runner/ArgentRunner/ArgentRunnerUITests/ArgentRunnerSession+Snapshot.swift

@@ -0,0 +1,327 @@
+import XCTest
+
+extension ArgentRunnerSession {
+    /// Element types an agent can act on directly. These are always included.
+    static let interactiveTypes: Set<XCUIElement.ElementType> = [
+        .button, .cell, .checkBox, .collectionView, .datePicker, .link,
+        .menuItem,
+        .picker, .pickerWheel, .searchField, .segmentedControl, .slider,
+        .stepper,
+        .switch, .tabBar, .textField, .secureTextField, .textView, .toggle,
+        .webView,
+    ]
+
+    /// Containers whose content scrolls. Included even when unlabeled so the
+    /// tree shows where scrolling is possible.
+    static let scrollContainerTypes: Set<XCUIElement.ElementType> = [
+        .scrollView, .table, .collectionView, .webView,
+    ]
+
+    /// Hard ceiling on emitted nodes. Bounds the reply size for very deep trees.
+    static let snapshotNodeBudget = 1500
+
+    /// Guard against cyclic or extremely deep raw trees.
+    private static let rawDepthLimit = 100
+
+    /// Ceiling on emitted node depth.
+    private static let emittedDepthLimit = 60
+
+    /// Captures the app's accessibility tree and flattens it into a snapshot
+    /// payload. The capture is retried once after a short pause: this is the
+    /// one runner-side retry, kept here because the capture can read its own
+    /// failure, a thrown error or an accessibility exception from a tree
+    /// that moved mid-capture, and a moment is usually enough for the tree
+    /// to settle. performOnMain only guards; the host retries read-only sends.
+    func captureSnapshot(of app: XCUIApplication) -> Envelope {
+        var root: XCUIElementSnapshot?
+        var lastError = ""
+
+        for attempt in 0..<2 {
+            if attempt > 0 {
+                NSLog("ARGENT_RUNNER_RETRY command=snapshot reason=%@", lastError)
+                Thread.sleep(forTimeInterval: 0.4)
+            }
+
+            // One XPC round trip captures the whole tree. Flattening it in-process
+            // avoids per-element AX queries and their stalls. The capture can throw
+            // a Swift error or raise an accessibility NSException; the guard turns
+            // the exception into a description instead of a crash.
+            let exceptionDescription = ArgentExceptionGuard.runCatching {
+                do {
+                    root = try app.snapshot()
+                } catch {
+                    lastError = String(describing: error)
+                }
+            }
+
+            if let exceptionDescription {
+                lastError = exceptionDescription
+            }
+
+            if root != nil { break }
+        }
+
+        guard let root else {
+            return .failure(
+                .snapshotFailed,
+                "XCTest could not capture the accessibility tree: \(lastError)",
+                hint:
+                    "Retry after the UI settles, or use screenshot as visual truth."
+            )
+        }
+
+        let nodes = Self.flatten(root)
+        let capped = nodes.count >= Self.snapshotNodeBudget
+
+        return .success(
+            SnapshotPayload(
+                nodes: nodes,
+                quality: SnapshotQualityPayload(
+                    state: capped ? "degraded" : "healthy",
+                    backend: "xctest",
+                    reason: capped
+                        ? "node budget reached; deeper content was dropped"
+                        : nil,
+                    reasonCode: capped ? "node_cap" : nil
+                )
+            )
+        )
+    }
+
+    /// Flattens the snapshot tree into an ordered node list. Keeps elements an
+    /// agent can name or act on, assigns indices and parent links in emission
+    /// order, and dedupes mirror elements.
+    static func flatten(_ root: XCUIElementSnapshot) -> [SnapshotNodePayload] {
+        struct WorkItem {
+            let snapshot: XCUIElementSnapshot
+            let rawDepth: Int
+            let emittedDepth: Int
+            let parentIndex: Int
+        }
+
+        let viewport = root.frame
+        var nodes = [makeNode(root, index: 0, depth: 0, parentIndex: nil)]
+        var seen: Set<String> = [identity(root)]
+
+        var stack = root.children.reversed().map {
+            WorkItem(snapshot: $0, rawDepth: 1, emittedDepth: 1, parentIndex: 0)
+        }
+
+        while let item = stack.popLast() {
+            if nodes.count >= snapshotNodeBudget { break }
+            if item.rawDepth > rawDepthLimit { continue }
+
+            let snapshot = item.snapshot
+            let frame = snapshot.frame
+            let visible =
+                viewport.isEmpty
+                || (!frame.isEmpty && viewport.intersects(frame))
+            let include =
+                visible && item.emittedDepth <= emittedDepthLimit
+                && shouldInclude(snapshot)
+            let key = identity(snapshot)
+            let duplicate = seen.contains(key)
+
+            var childParentIndex = item.parentIndex
+            var childEmittedDepth = item.emittedDepth
+
+            if include && !duplicate {
+                seen.insert(key)
+
+                let index = nodes.count
+
+                nodes.append(
+                    makeNode(
+                        snapshot,
+                        index: index,
+                        depth: item.emittedDepth,
+                        parentIndex: item.parentIndex
+                    )
+                )
+
+                childParentIndex = index
+                childEmittedDepth += 1
+            }
+
+            for child in snapshot.children.reversed() {
+                stack.append(
+                    WorkItem(
+                        snapshot: child,
+                        rawDepth: item.rawDepth + 1,
+                        emittedDepth: childEmittedDepth,
+                        parentIndex: childParentIndex
+                    )
+                )
+            }
+        }
+
+        return nodes
+    }
+
+    /// Whether an element earns a node: interactive, a scroll container, or
+    /// carrying any text.
+    private static func shouldInclude(_ snapshot: XCUIElementSnapshot) -> Bool {
+        if interactiveTypes.contains(snapshot.elementType) { return true }
+        if scrollContainerTypes.contains(snapshot.elementType) { return true }
+
+        return !snapshot.label.isEmpty || !snapshot.identifier.isEmpty
+            || valueText(snapshot.value) != nil
+    }
+
+    /// Converts one element snapshot into a wire node payload.
+    private static func makeNode(
+        _ snapshot: XCUIElementSnapshot,
+        index: Int,
+        depth: Int,
+        parentIndex: Int?
+    ) -> SnapshotNodePayload {
+        let frame = snapshot.frame
+
+        return SnapshotNodePayload(
+            index: index,
+            type: elementTypeName(snapshot.elementType),
+            label: snapshot.label.isEmpty ? nil : snapshot.label,
+            identifier: snapshot.identifier.isEmpty ? nil : snapshot.identifier,
+            value: valueText(snapshot.value),
+            // A geometry-less element reports CGRect.null with an infinite origin,
+            // and JSONEncoder rejects non-finite doubles. Coordinates are forced
+            // finite so one such node cannot fail the whole encode.
+            rect: SnapshotRect(
+                x: finite(frame.minX),
+                y: finite(frame.minY),
+                width: finite(frame.width),
+                height: finite(frame.height)
+            ),
+            enabled: snapshot.isEnabled,
+            focused: snapshot.hasFocus ? true : nil,
+            selected: snapshot.isSelected ? true : nil,
+            depth: depth,
+            parentIndex: parentIndex
+        )
+    }
+
+    /// Dedup key for one node. Mirror elements surfaced twice by the AX tree
+    /// share type, texts, and geometry and collapse to the same key.
+    private static func identity(_ snapshot: XCUIElementSnapshot) -> String {
+        let frame = snapshot.frame
+
+        return
+            "\(snapshot.elementType.rawValue)|\(snapshot.label)|\(snapshot.identifier)|"
+            + "\(keyCoordinate(frame.minX)),\(keyCoordinate(frame.minY)),"
+            + "\(keyCoordinate(frame.width)),\(keyCoordinate(frame.height))"
+    }
+
+    /// Stable dedup-key text for one frame coordinate. The key needs to be
+    /// stable, not exact.
+    private static func keyCoordinate(_ v: CGFloat) -> String {
+        // Int(_: Double) traps on non-finite or out-of-range input, and a trap
+        // kills the runner process. Geometry-less elements do reach here with
+        // CGRect.null frames, so every conversion must be total.
+        guard v.isFinite else { return String(describing: v) }
+
+        return String(Int(min(max(v.rounded(), -1e15), 1e15)))
+    }
+
+    /// Collapses a non-finite coordinate to 0 for the wire payload.
+    private static func finite(_ v: CGFloat) -> Double {
+        v.isFinite ? Double(v) : 0
+    }
+
+    /// The element's value as wire text, nil when absent or empty.
+    private static func valueText(_ value: Any?) -> String? {
+        guard let value, !(value is NSNull) else { return nil }
+        let text = String(describing: value)
+        return text.isEmpty ? nil : text
+    }
+
+    /// Stable type names for the wire payload. The TypeScript describe adapter
+    /// maps them onto accessibility roles.
+    static func elementTypeName(_ type: XCUIElement.ElementType) -> String {
+        switch type {
+        case .any: return "Any"
+        case .other: return "Other"
+        case .application: return "Application"
+        case .group: return "Group"
+        case .window: return "Window"
+        case .sheet: return "Sheet"
+        case .drawer: return "Drawer"
+        case .alert: return "Alert"
+        case .dialog: return "Dialog"
+        case .button: return "Button"
+        case .radioButton: return "RadioButton"
+        case .radioGroup: return "RadioGroup"
+        case .checkBox: return "CheckBox"
+        case .disclosureTriangle: return "DisclosureTriangle"
+        case .popUpButton: return "PopUpButton"
+        case .comboBox: return "ComboBox"
+        case .menuButton: return "MenuButton"
+        case .toolbarButton: return "ToolbarButton"
+        case .popover: return "Popover"
+        case .keyboard: return "Keyboard"
+        case .key: return "Key"
+        case .navigationBar: return "NavigationBar"
+        case .tabBar: return "TabBar"
+        case .tabGroup: return "TabGroup"
+        case .toolbar: return "Toolbar"
+        case .statusBar: return "StatusBar"
+        case .table: return "Table"
+        case .tableRow: return "TableRow"
+        case .tableColumn: return "TableColumn"
+        case .outline: return "Outline"
+        case .outlineRow: return "OutlineRow"
+        case .browser: return "Browser"
+        case .collectionView: return "CollectionView"
+        case .slider: return "Slider"
+        case .pageIndicator: return "PageIndicator"
+        case .progressIndicator: return "ProgressIndicator"
+        case .activityIndicator: return "ActivityIndicator"
+        case .segmentedControl: return "SegmentedControl"
+        case .picker: return "Picker"
+        case .pickerWheel: return "PickerWheel"
+        case .switch: return "Switch"
+        case .toggle: return "Toggle"
+        case .link: return "Link"
+        case .image: return "Image"
+        case .icon: return "Icon"
+        case .searchField: return "SearchField"
+        case .scrollView: return "ScrollView"
+        case .scrollBar: return "ScrollBar"
+        case .staticText: return "StaticText"
+        case .textField: return "TextField"
+        case .secureTextField: return "SecureTextField"
+        case .datePicker: return "DatePicker"
+        case .textView: return "TextView"
+        case .menu: return "Menu"
+        case .menuItem: return "MenuItem"
+        case .menuBar: return "MenuBar"
+        case .menuBarItem: return "MenuBarItem"
+        case .map: return "Map"
+        case .webView: return "WebView"
+        case .incrementArrow: return "IncrementArrow"
+        case .decrementArrow: return "DecrementArrow"
+        case .timeline: return "Timeline"
+        case .ratingIndicator: return "RatingIndicator"
+        case .valueIndicator: return "ValueIndicator"
+        case .splitGroup: return "SplitGroup"
+        case .splitter: return "Splitter"
+        case .relevanceIndicator: return "RelevanceIndicator"
+        case .colorWell: return "ColorWell"
+        case .helpTag: return "HelpTag"
+        case .matte: return "Matte"
+        case .dockItem: return "DockItem"
+        case .ruler: return "Ruler"
+        case .rulerMarker: return "RulerMarker"
+        case .grid: return "Grid"
+        case .levelIndicator: return "LevelIndicator"
+        case .cell: return "Cell"
+        case .layoutArea: return "LayoutArea"
+        case .layoutItem: return "LayoutItem"
+        case .handle: return "Handle"
+        case .stepper: return "Stepper"
+        case .tab: return "Tab"
+        case .touchBar: return "TouchBar"
+        case .statusItem: return "StatusItem"
+        @unknown default: return "Other"
+        }
+    }
+}

package/dist/ios-device-runner/ArgentRunner/ArgentRunnerUITests/ArgentRunnerSession+TextEntry.swift

@@ -0,0 +1,157 @@
+import XCTest
+
+extension ArgentRunnerSession {
+    /// The dedicated answer for typing with no first responder.
+    private static func textInputNotFocusedEnvelope() -> Envelope {
+        .failure(
+            .textInputNotFocused,
+            "no text input has keyboard focus",
+            hint: "Tap the target input first, then retry."
+        )
+    }
+
+    /// Whether any element in the target app currently has keyboard focus.
+    /// One scoped element query answers it. nil when the probe itself failed
+    /// (a degraded tree must not block typing; the recorded-failure audit in
+    /// performOnMain still covers the attempt).
+    private func hasKeyboardFocus(in app: XCUIApplication) -> Bool? {
+        var focused: Bool?
+        let exceptionDescription = ArgentExceptionGuard.runCatching {
+            focused =
+                app.descendants(matching: .any)
+                .matching(NSPredicate(format: "hasKeyboardFocus == true"))
+                .firstMatch.exists
+        }
+
+        return exceptionDescription == nil ? focused : nil
+    }
+
+    /// Types text into whatever element currently has keyboard focus. The tool
+    /// layer focuses an input with a tap before sending a type command.
+    func performType(_ request: CommandRequest, on app: XCUIApplication)
+        -> Envelope
+    {
+        guard let text = request.text, !text.isEmpty else {
+            return .failure(.invalidRequest, "type requires non-empty text")
+        }
+
+        // Wait for the keyboard's presentation animation to finish.
+        _ = app.keyboards.firstMatch.waitForExistence(timeout: 3)
+
+        return typeGuarded(
+            text,
+            in: app,
+            success: "typed \(text.count) characters",
+            failure: "unable to type"
+        )
+    }
+
+    /// Matches the keyboard's return key by label across the UIReturnKeyType
+    /// variants an app can configure. Case-insensitive, so the lowercase "go"
+    /// and "search" some keyboards show need no entry of their own. An OR of
+    /// `==[c]` terms rather than `IN[c]`: Foundation ignores the [c] option
+    /// on IN (checked on macOS, where "Return" fails `label IN[c] {"return"}`).
+    private static let submitKeyPredicate = NSCompoundPredicate(
+        orPredicateWithSubpredicates: [
+            "return", "enter", "go", "search", "next", "done", "send", "join",
+            "continue",
+        ].map { NSPredicate(format: "label ==[c] %@", $0) }
+    )
+
+    /// Presses the keyboard's return or submit key.
+    func performKeyboardReturn(on app: XCUIApplication) -> Envelope {
+        // Prefer tapping the visible submit key. Its label carries the action the
+        // app configured, such as Search or Go, and tapping works even when
+        // typeText rejects the focus state.
+        let keyboard = app.keyboards.firstMatch
+
+        // Query only when a keyboard is visibly up, and then with one predicate
+        // query per key collection. A label-by-label scan cost an exists and an
+        // isHittable probe per candidate, each a live AX query, and on a heavy
+        // screen the scan alone approached the 30s watchdog.
+        if keyboard.exists && !keyboard.frame.isEmpty {
+            let submitKey = Self.submitKeyPredicate
+
+            for candidate in [
+                app.keyboards.buttons.matching(submitKey).firstMatch,
+                app.keyboards.keys.matching(submitKey).firstMatch,
+            ] where candidate.exists && candidate.isHittable {
+                let label = candidate.label
+                candidate.tap()
+
+                return .success(
+                    MessagePayload(message: "pressed keyboard \(label)")
+                )
+            }
+        }
+
+        // No visible keyboard key matched, so the return character goes
+        // through typeText, behind the same focus probe as `type`.
+        return typeGuarded(
+            XCUIKeyboardKey.return.rawValue,
+            in: app,
+            success: "typed return",
+            failure: "unable to press the keyboard return key"
+        )
+    }
+
+    /// Presses the keyboard's delete key once. Unlike return there is no
+    /// labeled key to prefer, so the delete character always goes through
+    /// typeText.
+    func performKeyboardDelete(on app: XCUIApplication) -> Envelope {
+        // Wait for the keyboard's presentation animation to finish.
+        _ = app.keyboards.firstMatch.waitForExistence(timeout: 3)
+
+        return typeGuarded(
+            XCUIKeyboardKey.delete.rawValue,
+            in: app,
+            success: "typed delete",
+            failure: "unable to press the keyboard delete key"
+        )
+    }
+
+    /// Types `text` into the first responder behind the focus probe and the
+    /// exception guard, and maps the outcome onto the wire. On hardware,
+    /// typeText with no first responder RECORDS an XCTest failure instead of
+    /// throwing, so the exception branch alone never fires and the reply
+    /// would demote to the generic XCTEST_RECORDED_FAILURE (audited on an
+    /// iPhone 15). The probe answers the common case with the dedicated code;
+    /// only a probe that positively finds no focus refuses to type. typeText
+    /// itself targets the first responder directly, with no element
+    /// resolution, so typing works on screens whose accessibility trees
+    /// degrade.
+    private func typeGuarded(
+        _ text: String,
+        in app: XCUIApplication,
+        success message: String,
+        failure context: String
+    ) -> Envelope {
+        if hasKeyboardFocus(in: app) == false {
+            return Self.textInputNotFocusedEnvelope()
+        }
+
+        let exceptionDescription = ArgentExceptionGuard.runCatching {
+            app.typeText(text)
+        }
+
+        if let exceptionDescription {
+            // Backstop for the runtimes where a missing first responder does
+            // throw instead of recording.
+            if exceptionDescription.contains("keyboard focus") {
+                return Self.textInputNotFocusedEnvelope()
+            }
+
+            // XCTest throwing mid-command is a failed command, not hardware
+            // the device lacks, so it answers COMMAND_FAILED like every other
+            // command's exception does.
+            return .failure(
+                .commandFailed,
+                "\(context): \(exceptionDescription)",
+                hint:
+                    "Re-observe the screen, tap the target input if it lost focus, then retry."
+            )
+        }
+
+        return .success(MessagePayload(message: message))
+    }
+}