agent-device 0.10.2 → 0.11.0

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -422,38 +422,81 @@ extension RunnerTests {
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "type requires text"))
       }
+      let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
+      let target: XCUIElement?
+      if let x = command.x, let y = command.y {
+        target = textInputAt(app: activeApp, x: x, y: y) ?? focusedTextInput(app: activeApp)
+      } else {
+        target = focusedTextInput(app: activeApp)
+      }
+      func typeIntoTarget(_ value: String) {
+        if let focused = target {
+          focused.typeText(value)
+        } else {
+          activeApp.typeText(value)
+        }
+      }
       if command.clearFirst == true {
-        guard let focused = focusedTextInput(app: activeApp) else {
-          return Response(ok: false, error: ErrorPayload(message: "no focused text input to clear"))
+        guard let focused = target else {
+          let message =
+            (command.x != nil && command.y != nil)
+            ? "no text input found at the provided coordinates to clear"
+            : "no focused text input to clear"
+          return Response(ok: false, error: ErrorPayload(message: message))
         }
         clearTextInput(focused)
-        focused.typeText(text)
-        return Response(ok: true, data: DataPayload(message: "typed"))
       }
-      if let focused = focusedTextInput(app: activeApp) {
-        focused.typeText(text)
+      if delaySeconds > 0 && text.count > 1 {
+        let chunks = Array(text)
+        for (index, character) in chunks.enumerated() {
+          typeIntoTarget(String(character))
+          if index + 1 < chunks.count {
+            Thread.sleep(forTimeInterval: delaySeconds)
+          }
+        }
       } else {
-        activeApp.typeText(text)
+        typeIntoTarget(text)
       }
       return Response(ok: true, data: DataPayload(message: "typed"))
+    case .interactionFrame:
+      let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
+      return Response(
+        ok: true,
+        data: DataPayload(
+          x: frame.minX,
+          y: frame.minY,
+          referenceWidth: frame.width,
+          referenceHeight: frame.height
+        )
+      )
     case .swipe:
       guard let direction = command.direction else {
         return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
       }
-      let referenceFrame = resolvedGestureReferenceFrame(app: activeApp)
+      var executedFrame: DragVisualizationFrame?
       let timing = measureGesture {
         withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-          swipe(app: activeApp, direction: direction)
+          executedFrame = swipe(
+            app: activeApp,
+            direction: direction
+          )
         }
       }
+      guard let dragFrame = executedFrame else {
+        return Response(ok: false, error: ErrorPayload(message: "swipe is only supported on tvOS"))
+      }
       return Response(
         ok: true,
         data: DataPayload(
           message: "swiped",
           gestureStartUptimeMs: timing.gestureStartUptimeMs,
           gestureEndUptimeMs: timing.gestureEndUptimeMs,
-          referenceWidth: referenceFrame.referenceWidth,
-          referenceHeight: referenceFrame.referenceHeight
+          x: dragFrame.x,
+          y: dragFrame.y,
+          x2: dragFrame.x2,
+          y2: dragFrame.y2,
+          referenceWidth: dragFrame.referenceWidth,
+          referenceHeight: dragFrame.referenceHeight
         )
       )
     case .findText:
@@ -510,18 +553,43 @@ extension RunnerTests {
       // Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
       return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
 #endif
-    case .back:
-      if tapNavigationBack(app: activeApp) {
-        return Response(ok: true, data: DataPayload(message: "back"))
+    case .back, .backInApp:
+      if tapInAppBackControl(app: activeApp) {
+        let message = command.command == .back ? "back" : "backInApp"
+        return Response(ok: true, data: DataPayload(message: message))
+      }
+      return Response(ok: false, error: ErrorPayload(message: "in-app back control is not available"))
+    case .backSystem:
+      if performSystemBackAction(app: activeApp) {
+        return Response(ok: true, data: DataPayload(message: "backSystem"))
       }
-      performBackGesture(app: activeApp)
-      return Response(ok: true, data: DataPayload(message: "back"))
+      return Response(ok: false, error: ErrorPayload(message: "system back is not available"))
     case .home:
       pressHomeButton()
       return Response(ok: true, data: DataPayload(message: "home"))
     case .appSwitcher:
       performAppSwitcherGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "appSwitcher"))
+    case .keyboardDismiss:
+      let result = dismissKeyboard(app: activeApp)
+      if result.wasVisible && !result.dismissed {
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: "UNSUPPORTED_OPERATION",
+            message: "Unable to dismiss the iOS keyboard without a native dismiss gesture or control"
+          )
+        )
+      }
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "keyboardDismiss",
+          visible: result.visible,
+          wasVisible: result.wasVisible,
+          dismissed: result.dismissed
+        )
+      )
     case .alert:
       let action = (command.action ?? "get").lowercased()
       let alert = activeApp.alerts.firstMatch

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -17,14 +17,9 @@ extension RunnerTests {
     let referenceHeight: Double
   }
 
-  struct GestureReferenceFrame {
-    let referenceWidth: Double
-    let referenceHeight: Double
-  }
-
   // MARK: - Navigation Gestures
 
-  func tapNavigationBack(app: XCUIApplication) -> Bool {
+  func tapInAppBackControl(app: XCUIApplication) -> Bool {
 #if os(macOS)
     if let back = macOSNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
@@ -37,7 +32,7 @@ extension RunnerTests {
       back.tap()
       return true
     }
-    return pressTvRemoteMenuIfAvailable()
+    return false
 #endif
   }
 
@@ -51,6 +46,18 @@ extension RunnerTests {
     start.press(forDuration: 0.05, thenDragTo: end)
   }
 
+  func performSystemBackAction(app: XCUIApplication) -> Bool {
+#if os(macOS)
+    return false
+#else
+    if pressTvRemoteMenuIfAvailable() {
+      return true
+    }
+    performBackGesture(app: app)
+    return true
+#endif
+  }
+
   func performAppSwitcherGesture(app: XCUIApplication) {
     if performTvRemoteAppSwitcherIfAvailable() {
       return
@@ -161,19 +168,114 @@ extension RunnerTests {
     element.typeText(deletes)
   }
 
-  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
-    let focused = app
-      .descendants(matching: .any)
-      .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
-      .firstMatch
-    guard focused.exists else { return nil }
+  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    let point = CGPoint(x: x, y: y)
+    var matched: XCUIElement?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      // Prefer the smallest matching field so nested editable controls win over large containers.
+      let candidates = app.descendants(matching: .any).allElementsBoundByIndex
+        .filter { element in
+          guard element.exists else { return false }
+          switch element.elementType {
+          case .textField, .secureTextField, .searchField, .textView:
+            let frame = element.frame
+            return !frame.isEmpty && frame.contains(point)
+          default:
+            return false
+          }
+        }
+        .sorted { left, right in
+          let leftArea = max(1, left.frame.width * left.frame.height)
+          let rightArea = max(1, right.frame.width * right.frame.height)
+          if leftArea != rightArea {
+            return leftArea < rightArea
+          }
+          if left.frame.minY != right.frame.minY {
+            return left.frame.minY < right.frame.minY
+          }
+          if left.frame.minX != right.frame.minX {
+            return left.frame.minX < right.frame.minX
+          }
+          return left.elementType.rawValue < right.elementType.rawValue
+        }
+      matched = candidates.first
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_TEXT_INPUT_AT_POINT_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return matched
+  }
 
-    switch focused.elementType {
-    case .textField, .secureTextField, .searchField, .textView:
-      return focused
-    default:
+  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+    var focused: XCUIElement?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      let candidate = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .firstMatch
+      guard candidate.exists else { return }
+
+      switch candidate.elementType {
+      case .textField, .secureTextField, .searchField, .textView:
+        focused = candidate
+      default:
+        return
+      }
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_FOCUSED_INPUT_QUERY_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
       return nil
     }
+    return focused
+  }
+
+  func isKeyboardVisible(app: XCUIApplication) -> Bool {
+    let keyboard = app.keyboards.firstMatch
+    return keyboard.exists && !keyboard.frame.isEmpty
+  }
+
+  func dismissKeyboard(app: XCUIApplication) -> (wasVisible: Bool, dismissed: Bool, visible: Bool) {
+    let wasVisible = isKeyboardVisible(app: app)
+    guard wasVisible else {
+      return (wasVisible: false, dismissed: false, visible: false)
+    }
+
+    let keyboard = app.keyboards.firstMatch
+    keyboard.swipeDown()
+    sleepFor(0.2)
+    if !isKeyboardVisible(app: app) {
+      return (wasVisible: true, dismissed: true, visible: false)
+    }
+
+    if tapKeyboardDismissControl(app: app) {
+      sleepFor(0.2)
+      let visible = isKeyboardVisible(app: app)
+      return (wasVisible: true, dismissed: !visible, visible: visible)
+    }
+
+    return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
+  }
+
+  private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool {
+    for label in ["Hide keyboard", "Dismiss keyboard"] {
+      let candidates = [
+        app.keyboards.buttons[label],
+        app.keyboards.keys[label],
+        app.toolbars.buttons[label],
+      ]
+      if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) {
+        hittable.tap()
+        return true
+      }
+    }
+    return false
   }
 
   private func moveCaretToEnd(element: XCUIElement) {
@@ -322,7 +424,7 @@ extension RunnerTests {
     )
   }
 
-  private func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
+  func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
     let window = app.windows.firstMatch
     let windowFrame = window.frame
     if window.exists && !windowFrame.isEmpty {
@@ -334,14 +436,6 @@ extension RunnerTests {
     return CGRect(x: 0, y: 0, width: 0, height: 0)
   }
 
-  func resolvedGestureReferenceFrame(app: XCUIApplication) -> GestureReferenceFrame {
-    let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
-    return GestureReferenceFrame(
-      referenceWidth: frame.width,
-      referenceHeight: frame.height
-    )
-  }
-
   func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
     let total = max(count, 1)
     let pause = max(pauseMs, 0)
@@ -353,39 +447,36 @@ extension RunnerTests {
     }
   }
 
-  func swipe(app: XCUIApplication, direction: SwipeDirection) {
+  func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {
-      return
-    }
-    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
-    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
-    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
-    let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
-    let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
-
-    switch direction {
-    case .up:
-      end.press(forDuration: 0.1, thenDragTo: start)
-    case .down:
-      start.press(forDuration: 0.1, thenDragTo: end)
-    case .left:
-      right.press(forDuration: 0.1, thenDragTo: left)
-    case .right:
-      left.press(forDuration: 0.1, thenDragTo: right)
+      let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
+      let midX = frame.midX
+      let midY = frame.midY
+      return DragVisualizationFrame(
+        x: midX,
+        y: midY,
+        x2: midX,
+        y2: midY,
+        referenceWidth: frame.width,
+        referenceHeight: frame.height
+      )
     }
+    return nil
   }
 
-  private func performTvRemoteSwipeIfAvailable(direction: SwipeDirection) -> Bool {
+  private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
 #if os(tvOS)
     switch direction {
-    case .up:
+    case "up":
       XCUIRemote.shared.press(.up)
-    case .down:
+    case "down":
       XCUIRemote.shared.press(.down)
-    case .left:
+    case "left":
       XCUIRemote.shared.press(.left)
-    case .right:
+    case "right":
       XCUIRemote.shared.press(.right)
+    default:
+      return false
     }
     return true
 #else
@@ -461,5 +552,4 @@ extension RunnerTests {
     let element = app.descendants(matching: .any).matching(predicate).firstMatch
     return element.exists ? element : nil
   }
-
 }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -152,7 +152,7 @@ extension RunnerTests {
 
   func isReadOnlyCommand(_ command: Command) -> Bool {
     switch command.command {
-    case .findText, .readText, .snapshot, .screenshot:
+    case .interactionFrame, .findText, .readText, .snapshot, .screenshot:
       return true
     case .alert:
       let action = (command.action ?? "get").lowercased()
@@ -170,7 +170,18 @@ extension RunnerTests {
 
   func isInteractionCommand(_ command: CommandType) -> Bool {
     switch command {
-    case .tap, .longPress, .drag, .type, .swipe, .back, .appSwitcher, .pinch:
+    case
+      .tap,
+      .longPress,
+      .drag,
+      .type,
+      .swipe,
+      .back,
+      .backInApp,
+      .backSystem,
+      .appSwitcher,
+      .keyboardDismiss,
+      .pinch:
       return true
     default:
       return false

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -5,6 +5,7 @@ enum CommandType: String, Codable {
   case mouseClick
   case tapSeries
   case longPress
+  case interactionFrame
   case drag
   case dragSeries
   case type
@@ -14,8 +15,11 @@ enum CommandType: String, Codable {
   case snapshot
   case screenshot
   case back
+  case backInApp
+  case backSystem
   case home
   case appSwitcher
+  case keyboardDismiss
   case alert
   case pinch
   case recordStart
@@ -24,17 +28,11 @@ enum CommandType: String, Codable {
   case shutdown
 }
 
-enum SwipeDirection: String, Codable {
-  case up
-  case down
-  case left
-  case right
-}
-
 struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
   let text: String?
+  let delayMs: Int?
   let clearFirst: Bool?
   let action: String?
   let x: Double?
@@ -48,7 +46,7 @@ struct Command: Codable {
   let x2: Double?
   let y2: Double?
   let durationMs: Double?
-  let direction: SwipeDirection?
+  let direction: String?
   let scale: Double?
   let outPath: String?
   let fps: Int?
@@ -87,6 +85,9 @@ struct DataPayload: Codable {
   let referenceWidth: Double?
   let referenceHeight: Double?
   let currentUptimeMs: Double?
+  let visible: Bool?
+  let wasVisible: Bool?
+  let dismissed: Bool?
 
   init(
     message: String? = nil,
@@ -103,7 +104,10 @@ struct DataPayload: Codable {
     y2: Double? = nil,
     referenceWidth: Double? = nil,
     referenceHeight: Double? = nil,
-    currentUptimeMs: Double? = nil
+    currentUptimeMs: Double? = nil,
+    visible: Bool? = nil,
+    wasVisible: Bool? = nil,
+    dismissed: Bool? = nil
   ) {
     self.message = message
     self.text = text
@@ -120,11 +124,20 @@ struct DataPayload: Codable {
     self.referenceWidth = referenceWidth
     self.referenceHeight = referenceHeight
     self.currentUptimeMs = currentUptimeMs
+    self.visible = visible
+    self.wasVisible = wasVisible
+    self.dismissed = dismissed
   }
 }
 
 struct ErrorPayload: Codable {
+  let code: String?
   let message: String
+
+  init(code: String? = nil, message: String) {
+    self.code = code
+    self.message = message
+  }
 }
 
 struct SnapshotRect: Codable {