@ulpi/browse 1.4.4 → 2.3.1

package/browse-ios-runner/BrowseRunnerUITests/ActionHandler.swift

@@ -0,0 +1,277 @@
+import XCTest
+
+// MARK: - Action Handler
+
+/// Executes actions on XCUIElements resolved by tree path.
+///
+/// Supports tap, double-tap, long-press, swipe, and value setting.
+/// All actions resolve the target element using `TreeBuilder.resolveElement`.
+enum ActionHandler {
+
+    // MARK: - Perform Action
+
+    /// Perform a named action on the element at the given tree path.
+    ///
+    /// Supported actions:
+    ///   - `tap` / `press` — single tap
+    ///   - `doubleTap` — double tap
+    ///   - `longPress` — long press (1 second)
+    ///   - `swipeUp`, `swipeDown`, `swipeLeft`, `swipeRight` — swipe gestures
+    ///   - `twoFingerTap` — two-finger tap
+    ///
+    /// - Parameters:
+    ///   - app: The XCUIApplication to search in.
+    ///   - path: Tree path (array of child indices) to the target element.
+    ///   - actionName: Name of the action to perform.
+    /// - Returns: A dictionary with `success` and optional `error` keys.
+    static func performAction(
+        app: XCUIApplication,
+        path: [Int],
+        actionName: String
+    ) -> [String: Any] {
+        guard let element = TreeBuilder.resolveElement(in: app, path: path) else {
+            return [
+                "success": false,
+                "error": "Element not found at path \(path). Run /tree to refresh the element hierarchy.",
+            ]
+        }
+
+        guard element.exists else {
+            return [
+                "success": false,
+                "error": "Element at path \(path) no longer exists. The UI may have changed.",
+            ]
+        }
+
+        switch actionName.lowercased() {
+        case "tap", "press", "axpress":
+            element.tap()
+
+        case "doubletap":
+            element.doubleTap()
+
+        case "longpress":
+            element.press(forDuration: 1.0)
+
+        case "swipeup":
+            let upTarget = swipeTarget(for: element, in: app)
+            coordinateSwipe(upTarget, direction: "up")
+
+        case "swipedown":
+            let downTarget = swipeTarget(for: element, in: app)
+            coordinateSwipe(downTarget, direction: "down")
+
+        case "swipeleft":
+            let leftTarget = swipeTarget(for: element, in: app)
+            coordinateSwipe(leftTarget, direction: "left")
+
+        case "swiperight":
+            let rightTarget = swipeTarget(for: element, in: app)
+            coordinateSwipe(rightTarget, direction: "right")
+
+        case "twofingertap":
+            element.twoFingerTap()
+
+        case "forcetap":
+            // Use coordinate-based tap for elements that may not be directly tappable
+            let coordinate = element.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
+            coordinate.tap()
+
+        default:
+            return [
+                "success": false,
+                "error": "Unknown action '\(actionName)'. Supported: tap, doubleTap, longPress, swipeUp, swipeDown, swipeLeft, swipeRight, twoFingerTap, forceTap",
+            ]
+        }
+        return ["success": true]
+    }
+
+    // MARK: - Swipe Target Resolution
+
+    /// When swiping on the app root (XCUIApplication), find the first scrollable
+    /// descendant so the swipe actually scrolls content (e.g. WebView, ScrollView, Table).
+    /// Falls back to the original element if no scrollable descendant is found.
+    private static func swipeTarget(for element: XCUIElement, in app: XCUIApplication) -> XCUIElement {
+        // Only apply smart resolution when the target is the root app element
+        guard element.elementType == .application else { return element }
+
+        // Prioritized list of scrollable element types
+        let scrollableTypes: [XCUIElement.ElementType] = [.webView, .scrollView, .table, .collectionView]
+        for type in scrollableTypes {
+            let match = app.descendants(matching: type).firstMatch
+            if match.exists {
+                return match
+            }
+        }
+        return element
+    }
+
+    /// Perform a coordinate-based swipe on the given element.
+    /// Uses XCUICoordinate press-and-drag which reliably scrolls WebViews
+    /// and other content that doesn't respond to element.swipeUp().
+    private static func coordinateSwipe(_ element: XCUIElement, direction: String) {
+        // Swipe across the middle 60% of the element for a reliable scroll
+        let start: CGVector
+        let end: CGVector
+        switch direction {
+        case "up":    start = CGVector(dx: 0.5, dy: 0.75); end = CGVector(dx: 0.5, dy: 0.25)
+        case "down":  start = CGVector(dx: 0.5, dy: 0.25); end = CGVector(dx: 0.5, dy: 0.75)
+        case "left":  start = CGVector(dx: 0.75, dy: 0.5); end = CGVector(dx: 0.25, dy: 0.5)
+        case "right": start = CGVector(dx: 0.25, dy: 0.5); end = CGVector(dx: 0.75, dy: 0.5)
+        default: element.swipeUp(); return
+        }
+        let startCoord = element.coordinate(withNormalizedOffset: start)
+        let endCoord = element.coordinate(withNormalizedOffset: end)
+        startCoord.press(forDuration: 0.05, thenDragTo: endCoord)
+    }
+
+    // MARK: - Set Value
+
+    /// Set the text value of an editable element at the given tree path.
+    ///
+    /// Clears the existing value first, then types the new value.
+    /// The element must be a text field, text view, or search field.
+    ///
+    /// - Parameters:
+    ///   - app: The XCUIApplication to search in.
+    ///   - path: Tree path to the target element.
+    ///   - value: The text to set.
+    /// - Returns: A dictionary with `success` and optional `error` keys.
+    static func setValue(
+        app: XCUIApplication,
+        path: [Int],
+        value: String
+    ) -> [String: Any] {
+        guard let element = TreeBuilder.resolveElement(in: app, path: path) else {
+            return [
+                "success": false,
+                "error": "Element not found at path \(path). Run /tree to refresh the element hierarchy.",
+            ]
+        }
+
+        guard element.exists else {
+            return [
+                "success": false,
+                "error": "Element at path \(path) no longer exists.",
+            ]
+        }
+
+        // Tap to focus the element first
+        element.tap()
+
+        // Clear existing text: select all + delete
+        // Use keyboard shortcut on iOS: triple-tap to select all, then type replacement
+        let currentValue = element.value as? String ?? ""
+        if !currentValue.isEmpty {
+            // Select all text by tapping with enough taps to select all
+            element.press(forDuration: 1.0)
+            // Wait for the selection menu
+            let selectAll = app.menuItems["Select All"]
+            if selectAll.waitForExistence(timeout: 2.0) {
+                selectAll.tap()
+            }
+        }
+
+        // Type the new value
+        element.typeText(value)
+
+        return ["success": true]
+    }
+
+    // MARK: - Type Text
+
+    /// Type text using the focused element or the application.
+    ///
+    /// The text is typed character by character through the keyboard.
+    /// An element should already have focus before calling this.
+    ///
+    /// - Parameters:
+    ///   - app: The XCUIApplication.
+    ///   - text: The text to type.
+    /// - Returns: A dictionary with `success` and optional `error` keys.
+    static func typeText(
+        app: XCUIApplication,
+        text: String
+    ) -> [String: Any] {
+        app.typeText(text)
+        return ["success": true]
+    }
+
+    // MARK: - Press Key
+
+    /// Press a named key.
+    ///
+    /// Supported keys:
+    ///   - `return` / `enter` — Return key
+    ///   - `delete` / `backspace` — Delete backward
+    ///   - `tab` — Tab key
+    ///   - `escape` — Escape key
+    ///   - `space` — Space bar
+    ///   - `home` — Home button (via XCUIDevice)
+    ///   - `volumeup`, `volumedown` — Volume buttons
+    ///
+    /// - Parameters:
+    ///   - app: The XCUIApplication.
+    ///   - key: The key name to press.
+    /// - Returns: A dictionary with `success` and optional `error` keys.
+    static func pressKey(
+        app: XCUIApplication,
+        key: String
+    ) -> [String: Any] {
+        switch key.lowercased() {
+        case "return", "enter":
+            app.typeText("\n")
+
+        case "delete", "backspace":
+            app.typeText(XCUIKeyboardKey.delete.rawValue)
+
+        case "tab":
+            app.typeText("\t")
+
+        case "escape":
+            // No direct escape key on iOS; try pressing the keyboard dismiss if visible
+            if app.keyboards.count > 0 {
+                app.typeText(XCUIKeyboardKey.escape.rawValue)
+            }
+
+        case "space":
+            app.typeText(" ")
+
+        case "home":
+            XCUIDevice.shared.press(.home)
+
+        case "volumeup":
+            #if !targetEnvironment(simulator)
+            XCUIDevice.shared.press(.volumeUp)
+            #else
+            return [
+                "success": false,
+                "error": "volumeUp is not available in the iOS Simulator.",
+            ]
+            #endif
+
+        case "volumedown":
+            #if !targetEnvironment(simulator)
+            XCUIDevice.shared.press(.volumeDown)
+            #else
+            return [
+                "success": false,
+                "error": "volumeDown is not available in the iOS Simulator.",
+            ]
+            #endif
+
+        default:
+            // Try to type it as a single character
+            if key.count == 1 {
+                app.typeText(key)
+            } else {
+                return [
+                    "success": false,
+                    "error": "Unknown key '\(key)'. Supported: return, delete, tab, escape, space, home, volumeUp, volumeDown, or single characters.",
+                ]
+            }
+        }
+
+        return ["success": true]
+    }
+}

package/browse-ios-runner/BrowseRunnerUITests/BrowseRunnerUITests.swift

@@ -0,0 +1,60 @@
+import XCTest
+
+final class BrowseRunnerUITests: XCTestCase {
+
+    override func setUpWithError() throws {
+        continueAfterFailure = true
+        executionTimeAllowance = 86400
+    }
+
+    /// Synchronous test that keeps the main thread alive via RunLoop.
+    /// The FlyingFox server runs in a detached Task on a background thread.
+    /// XCUITest API calls hop to the main thread via DispatchQueue.main +
+    /// withCheckedContinuation (the `onMain` helper in RunnerServer.swift).
+    /// RunLoop.current.run(until:) drains the main dispatch queue each iteration.
+    func testRunServer() throws {
+        let portString = ProcessInfo.processInfo.environment["BROWSE_RUNNER_PORT"] ?? "9820"
+        let port = UInt16(portString) ?? 9820
+
+        let envBundleId = ProcessInfo.processInfo.environment["BROWSE_TARGET_BUNDLE_ID"]
+        let targetBundleId = envBundleId ?? "io.ulpi.browse-ios-runner"
+
+        // Setup on main thread (synchronous — no actor issues)
+        let hostApp = XCUIApplication()
+        hostApp.launch()
+
+        let targetApp: XCUIApplication
+        if targetBundleId != "io.ulpi.browse-ios-runner" {
+            targetApp = XCUIApplication(bundleIdentifier: targetBundleId)
+            targetApp.activate()
+        } else {
+            targetApp = hostApp
+        }
+
+        let context = RunnerContext(targetApp: targetApp, targetBundleId: targetBundleId)
+        let server = RunnerServer(port: port)
+
+        server.route("/health", handler: HealthHandler())
+        server.route("/configure", handler: ConfigureHandler(context: context))
+        server.route("/tree", handler: TreeHandler(context: context))
+        server.route("/action", handler: ActionRouteHandler(context: context))
+        server.route("/set-value", handler: SetValueRouteHandler(context: context))
+        server.route("/type", handler: TypeRouteHandler(context: context))
+        server.route("/press", handler: PressRouteHandler(context: context))
+        server.route("/screenshot", handler: ScreenshotRouteHandler(context: context))
+        server.route("/state", handler: StateRouteHandler(context: context))
+
+        NSLog("[BrowseRunner] Starting server on port %u, target: %@", port, targetBundleId)
+
+        // Start FlyingFox on a background thread
+        Task.detached {
+            try await server.start()
+        }
+
+        // Pump the main RunLoop forever — this is what allows
+        // DispatchQueue.main.async blocks (from onMain) to execute.
+        while true {
+            RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.05))
+        }
+    }
+}

package/browse-ios-runner/BrowseRunnerUITests/Models.swift

@@ -0,0 +1,78 @@
+import Foundation
+
+// MARK: - RawIOSNode
+
+/// Matches the `RawIOSNode` type in `src/app/ios/protocol.ts`.
+/// Represents a single node in the iOS accessibility tree.
+struct RawIOSNode: Encodable {
+    let elementType: String
+    let identifier: String
+    let label: String
+    let value: String
+    let placeholderValue: String
+    let frame: NodeFrame
+    let isEnabled: Bool
+    let isSelected: Bool
+    let hasFocus: Bool
+    let traits: [String]
+    let children: [RawIOSNode]
+}
+
+/// Bounding frame in screen coordinates.
+struct NodeFrame: Encodable {
+    let x: Double
+    let y: Double
+    let width: Double
+    let height: Double
+}
+
+// MARK: - IOSState
+
+/// Matches the `IOSState` type in `src/app/ios/protocol.ts`.
+/// Lightweight state snapshot from the runner.
+struct IOSState: Encodable {
+    let bundleId: String
+    let screenTitle: String
+    let elementCount: Int
+    let alertPresent: Bool
+    let keyboardVisible: Bool
+    let orientation: String
+    let statusBarTime: String
+}
+
+// MARK: - Action Request
+
+/// JSON body for `POST /action`.
+struct ActionRequest: Decodable {
+    let path: [Int]
+    let actionName: String
+}
+
+// MARK: - SetValue Request
+
+/// JSON body for `POST /set-value`.
+struct SetValueRequest: Decodable {
+    let path: [Int]
+    let value: String
+}
+
+// MARK: - Type Request
+
+/// JSON body for `POST /type`.
+struct TypeRequest: Decodable {
+    let text: String
+}
+
+// MARK: - Press Request
+
+/// JSON body for `POST /press`.
+struct PressRequest: Decodable {
+    let key: String
+}
+
+// MARK: - Screenshot Request
+
+/// JSON body for `POST /screenshot`.
+struct ScreenshotRequest: Decodable {
+    let outputPath: String
+}

package/browse-ios-runner/BrowseRunnerUITests/RunnerServer.swift

@@ -0,0 +1,246 @@
+import Foundation
+import FlyingFox
+import XCTest
+
+// MARK: - HTTP Server
+
+/// Async HTTP server built on FlyingFox for the XCUITest runner.
+///
+/// Runs inside the XCUITest process, listening on a configurable port.
+/// Uses `onMain {}` to dispatch XCUITest API calls to the main thread
+/// via DispatchQueue.main + withCheckedContinuation, since @MainActor
+/// doesn't execute reliably inside XCUITest async contexts.
+final class RunnerServer {
+
+    private let port: UInt16
+    private var handlers: [String: any HTTPHandler] = [:]
+
+    init(port: UInt16 = 9820) {
+        self.port = port
+    }
+
+    func route(_ path: String, handler: some HTTPHandler) {
+        handlers[path] = handler
+    }
+
+    func start() async throws {
+        let server = HTTPServer(
+            address: try .inet(ip4: "127.0.0.1", port: port),
+            timeout: 100
+        )
+
+        for (path, handler) in handlers {
+            await server.appendRoute(HTTPRoute(path), to: handler)
+        }
+
+        NSLog("[BrowseRunner] HTTP server listening on port %d", port)
+        try await server.run()
+    }
+}
+
+// MARK: - Main Thread Dispatch
+
+/// Dispatch a block to the main thread and await its result.
+/// XCUITest APIs must run on the main thread. We can't use @MainActor
+/// because the Swift concurrency main actor executor doesn't drain
+/// reliably inside XCUITest async test methods. Instead we use
+/// DispatchQueue.main + withCheckedContinuation, which always works.
+func onMain<T: Sendable>(_ block: @escaping @Sendable () -> T) async -> T {
+    await withCheckedContinuation { continuation in
+        DispatchQueue.main.async {
+            continuation.resume(returning: block())
+        }
+    }
+}
+
+// MARK: - JSON Response Helpers
+
+enum JSONResponse {
+
+    static func success(_ data: Any? = nil) -> FlyingFox.HTTPResponse {
+        var obj: [String: Any] = ["success": true]
+        if let data = data {
+            obj["data"] = data
+        }
+        return jsonResponse(obj, status: 200)
+    }
+
+    static func error(_ message: String, status: Int = 400) -> FlyingFox.HTTPResponse {
+        return jsonResponse(["success": false, "error": message], status: status)
+    }
+
+    private static func jsonResponse(_ object: Any, status: Int) -> FlyingFox.HTTPResponse {
+        let data = (try? JSONSerialization.data(withJSONObject: object)) ?? Data()
+        return FlyingFox.HTTPResponse(
+            statusCode: HTTPStatusCode(status, phrase: ""),
+            headers: [HTTPHeader("Content-Type"): "application/json"],
+            body: data
+        )
+    }
+}
+
+// MARK: - Route Handlers
+
+struct HealthHandler: HTTPHandler {
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        return JSONResponse.success(["status": "healthy"])
+    }
+}
+
+struct ConfigureHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let json = try? JSONSerialization.jsonObject(with: body) as? [String: Any],
+              let bundleId = json["targetBundleId"] as? String else {
+            return JSONResponse.error("Expected: {\"targetBundleId\": \"com.apple.Preferences\"}")
+        }
+
+        await onMain {
+            context.targetBundleId = bundleId
+            let app = XCUIApplication(bundleIdentifier: bundleId)
+            app.activate()
+            context.targetApp = app
+        }
+        NSLog("[BrowseRunner] Target app switched to: %@", bundleId)
+        return JSONResponse.success(["configured": bundleId])
+    }
+}
+
+struct TreeHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let tree = await onMain {
+            TreeBuilder.buildTree(from: context.targetApp)
+        }
+        guard let data = try? JSONEncoder().encode(tree),
+              let dict = try? JSONSerialization.jsonObject(with: data) else {
+            return JSONResponse.error("Failed to encode tree")
+        }
+        return JSONResponse.success(dict)
+    }
+}
+
+struct ActionRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let req = try? JSONDecoder().decode(ActionRequest.self, from: body) else {
+            return JSONResponse.error("Invalid request body. Expected: {\"path\":[0,1],\"actionName\":\"tap\"}")
+        }
+        let result = await onMain {
+            ActionHandler.performAction(app: context.targetApp, path: req.path, actionName: req.actionName)
+        }
+        if let ok = result["success"] as? Bool, ok {
+            return JSONResponse.success(result)
+        } else {
+            return JSONResponse.error(result["error"] as? String ?? "Action failed")
+        }
+    }
+}
+
+struct SetValueRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let req = try? JSONDecoder().decode(SetValueRequest.self, from: body) else {
+            return JSONResponse.error("Invalid request body. Expected: {\"path\":[0,1],\"value\":\"text\"}")
+        }
+        let result = await onMain {
+            ActionHandler.setValue(app: context.targetApp, path: req.path, value: req.value)
+        }
+        if let ok = result["success"] as? Bool, ok {
+            return JSONResponse.success(result)
+        } else {
+            return JSONResponse.error(result["error"] as? String ?? "Set value failed")
+        }
+    }
+}
+
+struct TypeRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let req = try? JSONDecoder().decode(TypeRequest.self, from: body) else {
+            return JSONResponse.error("Invalid request body. Expected: {\"text\":\"hello\"}")
+        }
+        let result = await onMain {
+            ActionHandler.typeText(app: context.targetApp, text: req.text)
+        }
+        if let ok = result["success"] as? Bool, ok {
+            return JSONResponse.success(result)
+        } else {
+            return JSONResponse.error(result["error"] as? String ?? "Type failed")
+        }
+    }
+}
+
+struct PressRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let req = try? JSONDecoder().decode(PressRequest.self, from: body) else {
+            return JSONResponse.error("Invalid request body. Expected: {\"key\":\"return\"}")
+        }
+        let result = await onMain {
+            ActionHandler.pressKey(app: context.targetApp, key: req.key)
+        }
+        if let ok = result["success"] as? Bool, ok {
+            return JSONResponse.success(result)
+        } else {
+            return JSONResponse.error(result["error"] as? String ?? "Press key failed")
+        }
+    }
+}
+
+struct ScreenshotRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let body = try await request.bodyData
+        guard let req = try? JSONDecoder().decode(ScreenshotRequest.self, from: body) else {
+            return JSONResponse.error("Invalid request body. Expected: {\"outputPath\":\"/tmp/shot.png\"}")
+        }
+        let result = await onMain {
+            ScreenshotHandler.captureScreenshot(app: context.targetApp, outputPath: req.outputPath)
+        }
+        if let ok = result["success"] as? Bool, ok {
+            return JSONResponse.success(result)
+        } else {
+            return JSONResponse.error(result["error"] as? String ?? "Screenshot failed")
+        }
+    }
+}
+
+struct StateRouteHandler: HTTPHandler {
+    let context: RunnerContext
+
+    func handleRequest(_ request: FlyingFox.HTTPRequest) async throws -> FlyingFox.HTTPResponse {
+        let state = await onMain {
+            StateHandler.captureState(app: context.targetApp, bundleId: context.targetBundleId)
+        }
+        guard let data = try? JSONEncoder().encode(state),
+              let dict = try? JSONSerialization.jsonObject(with: data) else {
+            return JSONResponse.error("Failed to encode state")
+        }
+        return JSONResponse.success(dict)
+    }
+}
+
+// MARK: - Runner Context
+
+final class RunnerContext: @unchecked Sendable {
+    var targetApp: XCUIApplication
+    var targetBundleId: String
+
+    init(targetApp: XCUIApplication, targetBundleId: String) {
+        self.targetApp = targetApp
+        self.targetBundleId = targetBundleId
+    }
+}

package/browse-ios-runner/BrowseRunnerUITests/ScreenshotHandler.swift

@@ -0,0 +1,35 @@
+import XCTest
+
+// MARK: - Screenshot Handler
+
+/// Captures screenshots of the running application.
+///
+/// Uses XCUIScreen to capture the full simulator screen, then writes
+/// the PNG data to the specified output path on the host filesystem.
+/// (The simulator shares the host filesystem, so paths like `/tmp/` work.)
+enum ScreenshotHandler {
+
+    /// Capture a screenshot and save it to the specified path.
+    ///
+    /// - Parameters:
+    ///   - app: The XCUIApplication (used for app-specific screenshots if needed).
+    ///   - outputPath: Absolute path on the host filesystem to write the PNG file.
+    /// - Returns: A dictionary with `success` and optional `error` keys.
+    static func captureScreenshot(
+        app: XCUIApplication,
+        outputPath: String
+    ) -> [String: Any] {
+        let screenshot = app.screenshot()
+        let pngData = screenshot.pngRepresentation
+
+        do {
+            try pngData.write(to: URL(fileURLWithPath: outputPath))
+            return ["success": true]
+        } catch {
+            return [
+                "success": false,
+                "error": "Failed to write screenshot to \(outputPath): \(error.localizedDescription)",
+            ]
+        }
+    }
+}