agent-device 0.10.3 → 0.11.1

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -17,14 +17,9 @@ extension RunnerTests {
     let referenceHeight: Double
   }
 
-  struct GestureReferenceFrame {
-    let referenceWidth: Double
-    let referenceHeight: Double
-  }
-
   // MARK: - Navigation Gestures
 
-  func tapNavigationBack(app: XCUIApplication) -> Bool {
+  func tapInAppBackControl(app: XCUIApplication) -> Bool {
 #if os(macOS)
     if let back = macOSNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
@@ -37,7 +32,7 @@ extension RunnerTests {
       back.tap()
       return true
     }
-    return pressTvRemoteMenuIfAvailable()
+    return false
 #endif
   }
 
@@ -51,6 +46,18 @@ extension RunnerTests {
     start.press(forDuration: 0.05, thenDragTo: end)
   }
 
+  func performSystemBackAction(app: XCUIApplication) -> Bool {
+#if os(macOS)
+    return false
+#else
+    if pressTvRemoteMenuIfAvailable() {
+      return true
+    }
+    performBackGesture(app: app)
+    return true
+#endif
+  }
+
   func performAppSwitcherGesture(app: XCUIApplication) {
     if performTvRemoteAppSwitcherIfAvailable() {
       return
@@ -161,19 +168,114 @@ extension RunnerTests {
     element.typeText(deletes)
   }
 
-  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
-    let focused = app
-      .descendants(matching: .any)
-      .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
-      .firstMatch
-    guard focused.exists else { return nil }
+  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    let point = CGPoint(x: x, y: y)
+    var matched: XCUIElement?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      // Prefer the smallest matching field so nested editable controls win over large containers.
+      let candidates = app.descendants(matching: .any).allElementsBoundByIndex
+        .filter { element in
+          guard element.exists else { return false }
+          switch element.elementType {
+          case .textField, .secureTextField, .searchField, .textView:
+            let frame = element.frame
+            return !frame.isEmpty && frame.contains(point)
+          default:
+            return false
+          }
+        }
+        .sorted { left, right in
+          let leftArea = max(1, left.frame.width * left.frame.height)
+          let rightArea = max(1, right.frame.width * right.frame.height)
+          if leftArea != rightArea {
+            return leftArea < rightArea
+          }
+          if left.frame.minY != right.frame.minY {
+            return left.frame.minY < right.frame.minY
+          }
+          if left.frame.minX != right.frame.minX {
+            return left.frame.minX < right.frame.minX
+          }
+          return left.elementType.rawValue < right.elementType.rawValue
+        }
+      matched = candidates.first
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_TEXT_INPUT_AT_POINT_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return matched
+  }
 
-    switch focused.elementType {
-    case .textField, .secureTextField, .searchField, .textView:
-      return focused
-    default:
+  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+    var focused: XCUIElement?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      let candidate = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .firstMatch
+      guard candidate.exists else { return }
+
+      switch candidate.elementType {
+      case .textField, .secureTextField, .searchField, .textView:
+        focused = candidate
+      default:
+        return
+      }
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_FOCUSED_INPUT_QUERY_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
       return nil
     }
+    return focused
+  }
+
+  func isKeyboardVisible(app: XCUIApplication) -> Bool {
+    let keyboard = app.keyboards.firstMatch
+    return keyboard.exists && !keyboard.frame.isEmpty
+  }
+
+  func dismissKeyboard(app: XCUIApplication) -> (wasVisible: Bool, dismissed: Bool, visible: Bool) {
+    let wasVisible = isKeyboardVisible(app: app)
+    guard wasVisible else {
+      return (wasVisible: false, dismissed: false, visible: false)
+    }
+
+    let keyboard = app.keyboards.firstMatch
+    keyboard.swipeDown()
+    sleepFor(0.2)
+    if !isKeyboardVisible(app: app) {
+      return (wasVisible: true, dismissed: true, visible: false)
+    }
+
+    if tapKeyboardDismissControl(app: app) {
+      sleepFor(0.2)
+      let visible = isKeyboardVisible(app: app)
+      return (wasVisible: true, dismissed: !visible, visible: visible)
+    }
+
+    return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
+  }
+
+  private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool {
+    for label in ["Hide keyboard", "Dismiss keyboard"] {
+      let candidates = [
+        app.keyboards.buttons[label],
+        app.keyboards.keys[label],
+        app.toolbars.buttons[label],
+      ]
+      if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) {
+        hittable.tap()
+        return true
+      }
+    }
+    return false
   }
 
   private func moveCaretToEnd(element: XCUIElement) {
@@ -322,7 +424,7 @@ extension RunnerTests {
     )
   }
 
-  private func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
+  func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
     let window = app.windows.firstMatch
     let windowFrame = window.frame
     if window.exists && !windowFrame.isEmpty {
@@ -334,14 +436,6 @@ extension RunnerTests {
     return CGRect(x: 0, y: 0, width: 0, height: 0)
   }
 
-  func resolvedGestureReferenceFrame(app: XCUIApplication) -> GestureReferenceFrame {
-    let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
-    return GestureReferenceFrame(
-      referenceWidth: frame.width,
-      referenceHeight: frame.height
-    )
-  }
-
   func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
     let total = max(count, 1)
     let pause = max(pauseMs, 0)
@@ -353,39 +447,36 @@ extension RunnerTests {
     }
   }
 
-  func swipe(app: XCUIApplication, direction: SwipeDirection) {
+  func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {
-      return
-    }
-    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
-    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
-    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
-    let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
-    let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
-
-    switch direction {
-    case .up:
-      end.press(forDuration: 0.1, thenDragTo: start)
-    case .down:
-      start.press(forDuration: 0.1, thenDragTo: end)
-    case .left:
-      right.press(forDuration: 0.1, thenDragTo: left)
-    case .right:
-      left.press(forDuration: 0.1, thenDragTo: right)
+      let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
+      let midX = frame.midX
+      let midY = frame.midY
+      return DragVisualizationFrame(
+        x: midX,
+        y: midY,
+        x2: midX,
+        y2: midY,
+        referenceWidth: frame.width,
+        referenceHeight: frame.height
+      )
     }
+    return nil
   }
 
-  private func performTvRemoteSwipeIfAvailable(direction: SwipeDirection) -> Bool {
+  private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
 #if os(tvOS)
     switch direction {
-    case .up:
+    case "up":
       XCUIRemote.shared.press(.up)
-    case .down:
+    case "down":
       XCUIRemote.shared.press(.down)
-    case .left:
+    case "left":
       XCUIRemote.shared.press(.left)
-    case .right:
+    case "right":
       XCUIRemote.shared.press(.right)
+    default:
+      return false
     }
     return true
 #else
@@ -461,5 +552,4 @@ extension RunnerTests {
     let element = app.descendants(matching: .any).matching(predicate).firstMatch
     return element.exists ? element : nil
   }
-
 }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -152,7 +152,7 @@ extension RunnerTests {
 
   func isReadOnlyCommand(_ command: Command) -> Bool {
     switch command.command {
-    case .findText, .readText, .snapshot, .screenshot:
+    case .interactionFrame, .findText, .readText, .snapshot, .screenshot:
       return true
     case .alert:
       let action = (command.action ?? "get").lowercased()
@@ -170,7 +170,18 @@ extension RunnerTests {
 
   func isInteractionCommand(_ command: CommandType) -> Bool {
     switch command {
-    case .tap, .longPress, .drag, .type, .swipe, .back, .appSwitcher, .pinch:
+    case
+      .tap,
+      .longPress,
+      .drag,
+      .type,
+      .swipe,
+      .back,
+      .backInApp,
+      .backSystem,
+      .appSwitcher,
+      .keyboardDismiss,
+      .pinch:
       return true
     default:
       return false

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -5,6 +5,7 @@ enum CommandType: String, Codable {
   case mouseClick
   case tapSeries
   case longPress
+  case interactionFrame
   case drag
   case dragSeries
   case type
@@ -14,8 +15,11 @@ enum CommandType: String, Codable {
   case snapshot
   case screenshot
   case back
+  case backInApp
+  case backSystem
   case home
   case appSwitcher
+  case keyboardDismiss
   case alert
   case pinch
   case recordStart
@@ -24,17 +28,11 @@ enum CommandType: String, Codable {
   case shutdown
 }
 
-enum SwipeDirection: String, Codable {
-  case up
-  case down
-  case left
-  case right
-}
-
 struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
   let text: String?
+  let delayMs: Int?
   let clearFirst: Bool?
   let action: String?
   let x: Double?
@@ -48,7 +46,7 @@ struct Command: Codable {
   let x2: Double?
   let y2: Double?
   let durationMs: Double?
-  let direction: SwipeDirection?
+  let direction: String?
   let scale: Double?
   let outPath: String?
   let fps: Int?
@@ -87,6 +85,9 @@ struct DataPayload: Codable {
   let referenceWidth: Double?
   let referenceHeight: Double?
   let currentUptimeMs: Double?
+  let visible: Bool?
+  let wasVisible: Bool?
+  let dismissed: Bool?
 
   init(
     message: String? = nil,
@@ -103,7 +104,10 @@ struct DataPayload: Codable {
     y2: Double? = nil,
     referenceWidth: Double? = nil,
     referenceHeight: Double? = nil,
-    currentUptimeMs: Double? = nil
+    currentUptimeMs: Double? = nil,
+    visible: Bool? = nil,
+    wasVisible: Bool? = nil,
+    dismissed: Bool? = nil
   ) {
     self.message = message
     self.text = text
@@ -120,11 +124,20 @@ struct DataPayload: Codable {
     self.referenceWidth = referenceWidth
     self.referenceHeight = referenceHeight
     self.currentUptimeMs = currentUptimeMs
+    self.visible = visible
+    self.wasVisible = wasVisible
+    self.dismissed = dismissed
   }
 }
 
 struct ErrorPayload: Codable {
+  let code: String?
   let message: String
+
+  init(code: String? = nil, message: String) {
+    self.code = code
+    self.message = message
+  }
 }
 
 struct SnapshotRect: Codable {

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -1,7 +1,16 @@
 import XCTest
 
 extension RunnerTests {
+  private static let collapsedTabCandidateTypes: Set<XCUIElement.ElementType> = [
+    .button,
+    .link,
+    .menuItem,
+    .other,
+    .staticText
+  ]
+
   private struct SnapshotTraversalContext {
+    let queryRoot: XCUIElement
     let rootSnapshot: XCUIElementSnapshot
     let viewport: CGRect
     let flatSnapshots: [XCUIElementSnapshot]
@@ -68,12 +77,34 @@ extension RunnerTests {
       return DataPayload(nodes: [], truncated: false)
     }
 
+    var cachedDescendantElements: [XCUIElement]?
+    func collapsedTabDescendants() -> [XCUIElement] {
+      if let cachedDescendantElements {
+        return cachedDescendantElements
+      }
+      let fetched = safeSnapshotElementsQuery {
+        context.queryRoot.descendants(matching: .any).allElementsBoundByIndex
+      }
+      cachedDescendantElements = fetched
+      return fetched
+    }
+
     var nodes: [SnapshotNode] = []
     var truncated = false
     let rootEvaluation = evaluateSnapshot(context.rootSnapshot, in: context)
     nodes.append(
       makeSnapshotNode(snapshot: context.rootSnapshot, evaluation: rootEvaluation, depth: 0, index: 0)
     )
+    if context.maxDepth > 0 {
+      let didTruncateFallback = appendCollapsedTabFallbackNodes(
+        to: &nodes,
+        containerSnapshot: context.rootSnapshot,
+        resolveElements: collapsedTabDescendants,
+        depth: 1,
+        nodeLimit: fastSnapshotLimit
+      )
+      truncated = truncated || didTruncateFallback
+    }
 
     var seen = Set<String>()
     var stack: [(XCUIElementSnapshot, Int, Int)] = context.rootSnapshot.children.map { ($0, 1, 1) }
@@ -119,6 +150,16 @@ extension RunnerTests {
           index: nodes.count
         )
       )
+      if visibleDepth < context.maxDepth {
+        let didTruncateFallback = appendCollapsedTabFallbackNodes(
+          to: &nodes,
+          containerSnapshot: snapshot,
+          resolveElements: collapsedTabDescendants,
+          depth: visibleDepth + 1,
+          nodeLimit: fastSnapshotLimit
+        )
+        truncated = truncated || didTruncateFallback
+      }
 
     }
 
@@ -247,6 +288,7 @@ extension RunnerTests {
 
     let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
     return SnapshotTraversalContext(
+      queryRoot: queryRoot,
       rootSnapshot: rootSnapshot,
       viewport: viewport,
       flatSnapshots: flatSnapshots,
@@ -376,4 +418,183 @@ extension RunnerTests {
     if rect.isNull || rect.isEmpty { return false }
     return rect.intersects(viewport)
   }
+
+  private func appendCollapsedTabFallbackNodes(
+    to nodes: inout [SnapshotNode],
+    containerSnapshot: XCUIElementSnapshot,
+    resolveElements: () -> [XCUIElement],
+    depth: Int,
+    nodeLimit: Int
+  ) -> Bool {
+    let fallbackNodes = collapsedTabFallbackNodes(
+      for: containerSnapshot,
+      resolveElements: resolveElements,
+      startingIndex: nodes.count,
+      depth: depth
+    )
+    if fallbackNodes.isEmpty { return false }
+    let remaining = max(0, nodeLimit - nodes.count)
+    if remaining == 0 { return true }
+    nodes.append(contentsOf: fallbackNodes.prefix(remaining))
+    return fallbackNodes.count > remaining
+  }
+
+  private func collapsedTabFallbackNodes(
+    for containerSnapshot: XCUIElementSnapshot,
+    resolveElements: () -> [XCUIElement],
+    startingIndex: Int,
+    depth: Int
+  ) -> [SnapshotNode] {
+    if !containerSnapshot.children.isEmpty { return [] }
+    guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] }
+    let containerFrame = containerSnapshot.frame
+    if containerFrame.isNull || containerFrame.isEmpty { return [] }
+
+    // Collapsed tab containers should be rare, so a full descendant scan is acceptable once per
+    // snapshot as a fallback for XCTest omitting the tab children from the snapshot tree.
+    let elements = resolveElements()
+    let candidates = elements.compactMap { element in
+      collapsedTabCandidateNode(
+        element: element,
+        containerSnapshot: containerSnapshot,
+        containerFrame: containerFrame
+      )
+    }
+    .sorted { left, right in
+      if left.rect.x != right.rect.x {
+        return left.rect.x < right.rect.x
+      }
+      return left.rect.y < right.rect.y
+    }
+
+    if candidates.count < 2 { return [] }
+    let rowMidpoints = candidates.map { $0.rect.y + ($0.rect.height / 2) }
+    let rowSpread = (rowMidpoints.max() ?? 0) - (rowMidpoints.min() ?? 0)
+    // Allow modest vertical jitter and short two-row wraps while still rejecting unrelated controls.
+    if rowSpread > max(24.0, Double(containerFrame.height) * 0.6) { return [] }
+
+    var seen = Set<String>()
+    let uniqueCandidates = candidates.filter { node in
+      let key = "\(node.type)-\(node.label ?? "")-\(node.identifier ?? "")-\(node.value ?? "")-\(node.rect.x)-\(node.rect.y)-\(node.rect.width)-\(node.rect.height)"
+      if seen.contains(key) { return false }
+      seen.insert(key)
+      return true
+    }
+    if uniqueCandidates.count < 2 { return [] }
+
+    return uniqueCandidates.enumerated().map { offset, node in
+      SnapshotNode(
+        index: startingIndex + offset,
+        type: node.type,
+        label: node.label,
+        identifier: node.identifier,
+        value: node.value,
+        rect: node.rect,
+        enabled: node.enabled,
+        hittable: node.hittable,
+        depth: depth
+      )
+    }
+  }
+
+  private func collapsedTabCandidateNode(
+    element: XCUIElement,
+    containerSnapshot: XCUIElementSnapshot,
+    containerFrame: CGRect
+  ) -> SnapshotNode? {
+    var node: SnapshotNode?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      if !element.exists { return }
+      let elementType = element.elementType
+      if !Self.collapsedTabCandidateTypes.contains(elementType) { return }
+      let frame = element.frame
+      if frame.isNull || frame.isEmpty { return }
+      if frame.equalTo(containerFrame) { return }
+      let area = max(CGFloat(1), frame.width * frame.height)
+      let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height)
+      if area >= containerArea * 0.9 { return }
+      let center = CGPoint(x: frame.midX, y: frame.midY)
+      if !containerFrame.contains(center) { return }
+
+      let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+      let valueText = snapshotValueText(element)
+      let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil
+      if !hasContent { return }
+      if sameSemanticElement(
+        containerSnapshot: containerSnapshot,
+        elementType: elementType,
+        label: label,
+        identifier: identifier
+      ) {
+        return
+      }
+
+      node = SnapshotNode(
+        index: 0,
+        type: elementTypeName(elementType),
+        label: label.isEmpty ? nil : label,
+        identifier: identifier.isEmpty ? nil : identifier,
+        value: valueText,
+        rect: snapshotRect(from: frame),
+        enabled: element.isEnabled,
+        hittable: element.isHittable,
+        depth: 0
+      )
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return node
+  }
+
+  private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool {
+    let frame = snapshot.frame
+    if frame.isNull || frame.isEmpty { return false }
+    if frame.width < max(CGFloat(160), frame.height * 1.75) { return false }
+    switch snapshot.elementType {
+    case .tabBar, .segmentedControl, .slider:
+      return true
+    default:
+      return false
+    }
+  }
+
+  private func snapshotValueText(_ element: XCUIElement) -> String? {
+    let text = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    return text.isEmpty ? nil : text
+  }
+
+  private func sameSemanticElement(
+    containerSnapshot: XCUIElementSnapshot,
+    elementType: XCUIElement.ElementType,
+    label: String,
+    identifier: String
+  ) -> Bool {
+    if containerSnapshot.elementType != elementType { return false }
+    let containerLabel = containerSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let containerIdentifier = containerSnapshot.identifier
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    return containerLabel == label && containerIdentifier == identifier
+  }
+
+  private func safeSnapshotElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
+    var elements: [XCUIElement] = []
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      elements = fetch()
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_SNAPSHOT_QUERY_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return []
+    }
+    return elements
+  }
 }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -21,27 +21,39 @@ extension RunnerTests {
           status: 413,
           response: Response(ok: false, error: ErrorPayload(message: "request too large"))
         )
-        connection.send(content: response, completion: .contentProcessed { [weak self] _ in
-          connection.cancel()
+        self.sendResponse(response, over: connection) { [weak self] in
           self?.finish()
-        })
+        }
         return
       }
       let combined = buffer + data
       if let body = self.parseRequest(data: combined) {
         let result = self.handleRequestBody(body)
-        connection.send(content: result.data, completion: .contentProcessed { _ in
-          connection.cancel()
+        self.sendResponse(result.data, over: connection) { [weak self] in
           if result.shouldFinish {
-            self.finish()
+            self?.finish()
           }
-        })
+        }
       } else {
         self.receiveRequest(connection: connection, buffer: combined)
       }
     }
   }
 
+  private func sendResponse(
+    _ response: Data,
+    over connection: NWConnection,
+    afterSend: @escaping () -> Void = {}
+  ) {
+    connection.send(content: response, isComplete: true, completion: .contentProcessed { error in
+      if let error {
+        NSLog("AGENT_DEVICE_RUNNER_SEND_FAILED=%@", String(describing: error))
+      }
+      connection.cancel()
+      afterSend()
+    })
+  }
+
   private func parseRequest(data: Data) -> Data? {
     guard let headerEnd = data.range(of: Data("\r\n\r\n".utf8)) else {
       return nil