agent-device 0.7.3 → 0.7.5

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -0,0 +1,258 @@
+import XCTest
+
+extension RunnerTests {
+  // MARK: - Navigation Gestures
+
+  func tapNavigationBack(app: XCUIApplication) -> Bool {
+    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
+    if let back = buttons.first(where: { $0.isHittable }) {
+      back.tap()
+      return true
+    }
+    return pressTvRemoteMenuIfAvailable()
+  }
+
+  func performBackGesture(app: XCUIApplication) {
+    if pressTvRemoteMenuIfAvailable() {
+      return
+    }
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
+    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
+    start.press(forDuration: 0.05, thenDragTo: end)
+  }
+
+  func performAppSwitcherGesture(app: XCUIApplication) {
+    if performTvRemoteAppSwitcherIfAvailable() {
+      return
+    }
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
+    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
+    start.press(forDuration: 0.6, thenDragTo: end)
+  }
+
+  func pressHomeButton() {
+    if pressTvRemoteHomeIfAvailable() {
+      return
+    }
+    XCUIDevice.shared.press(.home)
+  }
+
+  private func pressTvRemoteMenuIfAvailable() -> Bool {
+#if os(tvOS)
+    XCUIRemote.shared.press(.menu)
+    return true
+#else
+    return false
+#endif
+  }
+
+  private func pressTvRemoteHomeIfAvailable() -> Bool {
+#if os(tvOS)
+    XCUIRemote.shared.press(.home)
+    return true
+#else
+    return false
+#endif
+  }
+
+  private func performTvRemoteAppSwitcherIfAvailable() -> Bool {
+#if os(tvOS)
+    XCUIRemote.shared.press(.home)
+    sleepFor(resolveTvRemoteDoublePressDelay())
+    XCUIRemote.shared.press(.home)
+    return true
+#else
+    return false
+#endif
+  }
+
+  private func resolveTvRemoteDoublePressDelay() -> TimeInterval {
+    guard
+      let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],
+      !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+    else {
+      return tvRemoteDoublePressDelayDefault
+    }
+    guard let parsedMs = Double(raw), parsedMs >= 0 else {
+      return tvRemoteDoublePressDelayDefault
+    }
+    return min(parsedMs, 1000) / 1000.0
+  }
+
+  func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
+    let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
+  }
+
+  func clearTextInput(_ element: XCUIElement) {
+    moveCaretToEnd(element: element)
+    let count = estimatedDeleteCount(for: element)
+    let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
+    element.typeText(deletes)
+  }
+
+  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+    let focused = app
+      .descendants(matching: .any)
+      .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+      .firstMatch
+    guard focused.exists else { return nil }
+
+    switch focused.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      return focused
+    default:
+      return nil
+    }
+  }
+
+  private func moveCaretToEnd(element: XCUIElement) {
+    let frame = element.frame
+    guard !frame.isEmpty else {
+      element.tap()
+      return
+    }
+    let origin = element.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let target = origin.withOffset(
+      CGVector(dx: max(2, frame.width - 4), dy: max(2, frame.height / 2))
+    )
+    target.tap()
+  }
+
+  private func estimatedDeleteCount(for element: XCUIElement) -> Int {
+    let valueText = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    let base = valueText.isEmpty ? 24 : (valueText.count + 8)
+    return max(24, min(120, base))
+  }
+
+  func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
+    let predicate = NSPredicate(
+      format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
+      scope,
+      scope
+    )
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
+  }
+
+  func tapAt(app: XCUIApplication, x: Double, y: Double) {
+    let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
+    coordinate.tap()
+  }
+
+  func doubleTapAt(app: XCUIApplication, x: Double, y: Double) {
+    let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
+    coordinate.doubleTap()
+  }
+
+  func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) {
+    let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
+    coordinate.press(forDuration: duration)
+  }
+
+  func dragAt(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    holdDuration: TimeInterval
+  ) {
+    let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let start = origin.withOffset(CGVector(dx: x, dy: y))
+    let end = origin.withOffset(CGVector(dx: x2, dy: y2))
+    start.press(forDuration: holdDuration, thenDragTo: end)
+  }
+
+  func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
+    let total = max(count, 1)
+    let pause = max(pauseMs, 0)
+    for idx in 0..<total {
+      operation(idx)
+      if idx < total - 1 && pause > 0 {
+        Thread.sleep(forTimeInterval: pause / 1000.0)
+      }
+    }
+  }
+
+  func swipe(app: XCUIApplication, direction: SwipeDirection) {
+    if performTvRemoteSwipeIfAvailable(direction: direction) {
+      return
+    }
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
+    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
+    let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
+    let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
+
+    switch direction {
+    case .up:
+      end.press(forDuration: 0.1, thenDragTo: start)
+    case .down:
+      start.press(forDuration: 0.1, thenDragTo: end)
+    case .left:
+      right.press(forDuration: 0.1, thenDragTo: left)
+    case .right:
+      left.press(forDuration: 0.1, thenDragTo: right)
+    }
+  }
+
+  private func performTvRemoteSwipeIfAvailable(direction: SwipeDirection) -> Bool {
+#if os(tvOS)
+    switch direction {
+    case .up:
+      XCUIRemote.shared.press(.up)
+    case .down:
+      XCUIRemote.shared.press(.down)
+    case .left:
+      XCUIRemote.shared.press(.left)
+    case .right:
+      XCUIRemote.shared.press(.right)
+    }
+    return true
+#else
+    return false
+#endif
+  }
+
+  func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) {
+    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+
+    // Use double-tap + drag gesture for reliable map zoom
+    // Zoom in (scale > 1): tap then drag UP
+    // Zoom out (scale < 1): tap then drag DOWN
+
+    // Determine center point (use provided x/y or screen center)
+    let centerX = x.map { $0 / target.frame.width } ?? 0.5
+    let centerY = y.map { $0 / target.frame.height } ?? 0.5
+    let center = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: centerY))
+
+    // Calculate drag distance based on scale (clamped to reasonable range)
+    // Larger scale = more drag distance
+    let dragAmount: CGFloat
+    if scale > 1.0 {
+      // Zoom in: drag up (negative Y direction in normalized coords)
+      dragAmount = min(0.4, CGFloat(scale - 1.0) * 0.2)
+    } else {
+      // Zoom out: drag down (positive Y direction)
+      dragAmount = min(0.4, CGFloat(1.0 - scale) * 0.4)
+    }
+
+    let endY = scale > 1.0 ? (centerY - Double(dragAmount)) : (centerY + Double(dragAmount))
+    let endPoint = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: max(0.1, min(0.9, endY))))
+
+    // Tap first (first tap of double-tap)
+    center.tap()
+
+    // Immediately press and drag (second tap + drag)
+    center.press(forDuration: 0.05, thenDragTo: endPoint)
+  }
+
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -0,0 +1,174 @@
+import XCTest
+import UIKit
+
+extension RunnerTests {
+  // MARK: - Recording
+
+  func captureRunnerFrame() -> UIImage? {
+    var image: UIImage?
+    let capture = {
+      let screenshot = XCUIScreen.main.screenshot()
+      image = screenshot.image
+    }
+    if Thread.isMainThread {
+      capture()
+    } else {
+      DispatchQueue.main.sync(execute: capture)
+    }
+    return image
+  }
+
+  func stopRecordingIfNeeded() {
+    guard let recorder = activeRecording else { return }
+    do {
+      try recorder.stop()
+    } catch {
+      NSLog("AGENT_DEVICE_RUNNER_RECORD_STOP_FAILED=%@", String(describing: error))
+    }
+    activeRecording = nil
+  }
+
+  func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
+    let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
+    let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
+    let safeFileName = fileName.isEmpty ? fallbackName : fileName
+    return (NSTemporaryDirectory() as NSString).appendingPathComponent(safeFileName)
+  }
+
+  // MARK: - Target Activation
+
+  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
+    switch target.state {
+    case .unknown, .notRunning, .runningBackground, .runningBackgroundSuspended:
+      return true
+    default:
+      return false
+    }
+  }
+
+  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
+    let target = XCUIApplication(bundleIdentifier: bundleId)
+    NSLog(
+      "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
+      bundleId,
+      target.state.rawValue,
+      reason
+    )
+    // activate avoids terminating and relaunching the target app
+    target.activate()
+    currentApp = target
+    currentBundleId = bundleId
+    needsFirstInteractionDelay = true
+    return target
+  }
+
+  func withTemporaryScrollIdleTimeoutIfSupported(
+    _ target: XCUIApplication,
+    operation: () -> Void
+  ) {
+    let setter = NSSelectorFromString("setWaitForIdleTimeout:")
+    guard target.responds(to: setter) else {
+      operation()
+      return
+    }
+    let previous = target.value(forKey: "waitForIdleTimeout") as? NSNumber
+    target.setValue(resolveScrollInteractionIdleTimeout(), forKey: "waitForIdleTimeout")
+    defer {
+      if let previous {
+        target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout")
+      }
+    }
+    operation()
+  }
+
+  private func resolveScrollInteractionIdleTimeout() -> TimeInterval {
+    guard
+      let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_IOS_INTERACTION_IDLE_TIMEOUT"],
+      !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+    else {
+      return scrollInteractionIdleTimeoutDefault
+    }
+    guard let parsed = Double(raw), parsed >= 0 else {
+      return scrollInteractionIdleTimeoutDefault
+    }
+    return min(parsed, 30)
+  }
+
+  func shouldRetryCommand(_ command: Command) -> Bool {
+    if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") {
+      return false
+    }
+    return isReadOnlyCommand(command)
+  }
+
+  func shouldRetryException(_ command: Command, message: String) -> Bool {
+    guard shouldRetryCommand(command) else { return false }
+    let normalized = message.lowercased()
+    if normalized.contains("kaxerrorservernotfound") {
+      return true
+    }
+    if normalized.contains("main thread execution timed out") {
+      return true
+    }
+    if normalized.contains("timed out") && command.command == .snapshot {
+      return true
+    }
+    return false
+  }
+
+  // MARK: - Command Classification
+
+  func isReadOnlyCommand(_ command: Command) -> Bool {
+    switch command.command {
+    case .findText, .snapshot, .screenshot:
+      return true
+    case .alert:
+      let action = (command.action ?? "get").lowercased()
+      return action == "get"
+    default:
+      return false
+    }
+  }
+
+  func shouldRetryResponse(_ response: Response) -> Bool {
+    guard response.ok == false else { return false }
+    guard let message = response.error?.message.lowercased() else { return false }
+    return message.contains("is not available")
+  }
+
+  func isInteractionCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case .tap, .longPress, .drag, .type, .swipe, .back, .appSwitcher, .pinch:
+      return true
+    default:
+      return false
+    }
+  }
+
+  func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case .shutdown, .recordStop, .screenshot:
+      return true
+    default:
+      return false
+    }
+  }
+
+  // MARK: - Interaction Stabilization
+
+  func applyInteractionStabilizationIfNeeded() {
+    if needsPostSnapshotInteractionDelay {
+      sleepFor(postSnapshotInteractionDelay)
+      needsPostSnapshotInteractionDelay = false
+    }
+    if needsFirstInteractionDelay {
+      sleepFor(firstInteractionAfterActivateDelay)
+      needsFirstInteractionDelay = false
+    }
+  }
+
+  func sleepFor(_ delay: TimeInterval) {
+    guard delay > 0 else { return }
+    usleep(useconds_t(delay * 1_000_000))
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -0,0 +1,121 @@
+// MARK: - Wire Models
+
+enum CommandType: String, Codable {
+  case tap
+  case tapSeries
+  case longPress
+  case drag
+  case dragSeries
+  case type
+  case swipe
+  case findText
+  case snapshot
+  case screenshot
+  case back
+  case home
+  case appSwitcher
+  case alert
+  case pinch
+  case recordStart
+  case recordStop
+  case shutdown
+}
+
+enum SwipeDirection: String, Codable {
+  case up
+  case down
+  case left
+  case right
+}
+
+struct Command: Codable {
+  let command: CommandType
+  let appBundleId: String?
+  let text: String?
+  let clearFirst: Bool?
+  let action: String?
+  let x: Double?
+  let y: Double?
+  let count: Double?
+  let intervalMs: Double?
+  let doubleTap: Bool?
+  let pauseMs: Double?
+  let pattern: String?
+  let x2: Double?
+  let y2: Double?
+  let durationMs: Double?
+  let direction: SwipeDirection?
+  let scale: Double?
+  let outPath: String?
+  let fps: Int?
+  let interactiveOnly: Bool?
+  let compact: Bool?
+  let depth: Int?
+  let scope: String?
+  let raw: Bool?
+}
+
+struct Response: Codable {
+  let ok: Bool
+  let data: DataPayload?
+  let error: ErrorPayload?
+
+  init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
+    self.ok = ok
+    self.data = data
+    self.error = error
+  }
+}
+
+struct DataPayload: Codable {
+  let message: String?
+  let found: Bool?
+  let items: [String]?
+  let nodes: [SnapshotNode]?
+  let truncated: Bool?
+
+  init(
+    message: String? = nil,
+    found: Bool? = nil,
+    items: [String]? = nil,
+    nodes: [SnapshotNode]? = nil,
+    truncated: Bool? = nil
+  ) {
+    self.message = message
+    self.found = found
+    self.items = items
+    self.nodes = nodes
+    self.truncated = truncated
+  }
+}
+
+struct ErrorPayload: Codable {
+  let message: String
+}
+
+struct SnapshotRect: Codable {
+  let x: Double
+  let y: Double
+  let width: Double
+  let height: Double
+}
+
+struct SnapshotNode: Codable {
+  let index: Int
+  let type: String
+  let label: String?
+  let identifier: String?
+  let value: String?
+  let rect: SnapshotRect
+  let enabled: Bool
+  let hittable: Bool
+  let depth: Int
+}
+
+struct SnapshotOptions {
+  let interactiveOnly: Bool
+  let compact: Bool
+  let depth: Int?
+  let scope: String?
+  let raw: Bool
+}