agent-device 0.20.10 → 0.21.1

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -1,11 +1,20 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 #if os(macOS)
 import CoreGraphics
 #endif
 
+private struct RunnerUnsupportedOperationError: LocalizedError {
+  let message: String
+
+  var errorDescription: String? { message }
+}
+
 private enum RunnerInterfaceOrientation {
-  static let unknown = 0
+
+
+
   static let portrait = 1
   static let portraitUpsideDown = 2
   static let landscapeRight = 3
@@ -18,11 +27,18 @@ extension RunnerTests {
     case sampled
   }
 
-  enum SynthesizedDragProfile {
+  enum SynthesizedDragProfile: Equatable {
     case continuous
+    case controlledScroll
     case fastSwipe
   }
 
+  func scrollDragProfile(
+    releaseBehavior: ScrollReleaseBehavior?
+  ) -> SynthesizedDragProfile {
+    releaseBehavior == .inertial ? .fastSwipe : .controlledScroll
+  }
+
   struct TouchVisualizationFrame {
     let x: Double
     let y: Double
@@ -178,11 +194,11 @@ extension RunnerTests {
     }
   }
 
-  // Maestro-compat gate for the non-hittable coordinate fallback: an element
-  // with no frame at all cannot be coordinate-tapped, otherwise the decision
-  // is the shared TapPointPolicy center-in-frame rule (golden parity table
-  // with the TS twin). app.frame is the frame source here — replay taps
-  // resolved bounds Maestro-style, so the union frame is intentional.
+
+
+
+
+
   private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool {
     let frame = element.frame
     if frame.isEmpty {
@@ -191,10 +207,10 @@ extension RunnerTests {
     return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: app.frame)
   }
 
-  // The tappable on-screen viewport. app.frame is unsuitable: it unions
-  // transformed subtrees, so a closed drawer at negative x inflates it and
-  // out-of-window coordinates still pass containment. Falls back to app.frame
-  // when no window frame is readable.
+
+
+
+
   func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
     let window = app.windows.element(boundBy: 0)
     if window.exists {
@@ -207,11 +223,11 @@ extension RunnerTests {
   }
 
   func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
-    // querySelector is a read — it backs get/is/wait and the offscreen-refusal
-    // double-check, none of which mutate. The fail-closed raw-match rule exists
-    // to stop a mutation acting on an unseen duplicate; applying it here would
-    // instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH
-    // for readers that previously resolved the hittable element.
+
+
+
+
+
     let match = findElement(
       app: app,
       selectorKey: selectorKey,
@@ -257,6 +273,23 @@ extension RunnerTests {
     )
   }
 
+
+
+  func smallestElementFirst(_ left: XCUIElement, _ right: XCUIElement) -> Bool {
+    let leftArea = max(1, left.frame.width * left.frame.height)
+    let rightArea = max(1, right.frame.width * right.frame.height)
+    if leftArea != rightArea {
+      return leftArea < rightArea
+    }
+    if left.frame.minY != right.frame.minY {
+      return left.frame.minY < right.frame.minY
+    }
+    if left.frame.minX != right.frame.minX {
+      return left.frame.minX < right.frame.minX
+    }
+    return left.elementType.rawValue < right.elementType.rawValue
+  }
+
   func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
     let point = CGPoint(x: x, y: y)
     let textInputCandidates = textInputCandidatesAt(app: app, point: point)
@@ -270,20 +303,7 @@ extension RunnerTests {
       .filter { element in
         element.exists && !element.frame.isEmpty && element.frame.contains(point)
       }
-      .sorted { left, right in
-        let leftArea = max(1, left.frame.width * left.frame.height)
-        let rightArea = max(1, right.frame.width * right.frame.height)
-        if leftArea != rightArea {
-          return leftArea < rightArea
-        }
-        if left.frame.minY != right.frame.minY {
-          return left.frame.minY < right.frame.minY
-        }
-        if left.frame.minX != right.frame.minX {
-          return left.frame.minX < right.frame.minX
-        }
-        return left.elementType.rawValue < right.elementType.rawValue
-      }
+      .sorted(by: smallestElementFirst)
 
     for element in candidates where prefersExpandedTextRead(element) {
       if let text = readableText(for: element) {
@@ -298,50 +318,6 @@ extension RunnerTests {
     return nil
   }
 
-  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
-    return textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
-  }
-
-  private func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
-    safely("TEXT_INPUT_AT_POINT", []) {
-      // Query the text-input element types directly instead of enumerating the entire tree
-      // (app.descendants(.any).allElementsBoundByIndex snapshots every element and is ~10x
-      // slower — it dominated fill latency because resolveTextEntryElement re-runs this on
-      // each verify/repair poll once the focused field reference goes stale).
-      // Prefer the smallest matching field so nested editable controls win over large containers.
-      [
-        app.textFields,
-        app.secureTextFields,
-        app.searchFields,
-        app.textViews,
-      ]
-        .flatMap { $0.allElementsBoundByIndex }
-        .filter { element in
-          guard element.exists else { return false }
-          let frame = element.frame
-          return isCoordinateTextInputCandidate(
-            enabled: element.isEnabled,
-            frame: frame,
-            point: point
-          )
-        }
-        .sorted { left, right in
-          let leftArea = max(1, left.frame.width * left.frame.height)
-          let rightArea = max(1, right.frame.width * right.frame.height)
-          if leftArea != rightArea {
-            return leftArea < rightArea
-          }
-          if left.frame.minY != right.frame.minY {
-            return left.frame.minY < right.frame.minY
-          }
-          if left.frame.minX != right.frame.minX {
-            return left.frame.minX < right.frame.minX
-          }
-          return left.elementType.rawValue < right.elementType.rawValue
-        }
-    }
-  }
-
   private func readableText(for element: XCUIElement) -> String? {
     let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
     let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -368,16 +344,6 @@ extension RunnerTests {
     }
   }
 
-  func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
-    let predicate = NSPredicate(
-      format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
-      scope,
-      scope
-    )
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
-  }
-
   func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
     if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") {
       return outcome
@@ -394,30 +360,14 @@ extension RunnerTests {
     case "secondary":
       coordinate.rightClick()
     case "middle":
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
-      )
+      throw RunnerUnsupportedOperationError(message: "middle mouse button is not supported")
     default:
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
-      )
+      throw RunnerUnsupportedOperationError(message: "unsupported mouse button: \(button)")
     }
 #elseif os(tvOS)
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "mouseClick is not supported on tvOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "mouseClick is not supported on tvOS")
 #else
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "mouseClick is only supported on macOS")
 #endif
   }
 
@@ -425,28 +375,22 @@ extension RunnerTests {
     app: XCUIApplication,
     x: Double,
     y: Double,
-    direction: String,
+    direction: RunnerScrollDirection,
     pixels: Double,
     durationMs: Double?
   ) throws {
 #if os(macOS)
-    guard let events = desktopScrollWheelDeltaEvents(
+    let events = desktopScrollWheelDeltaEvents(
       direction: direction,
       pixels: pixels,
       durationMs: durationMs
-    ) else {
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "unsupported desktop scroll direction: \(direction)"]
-      )
-    }
+    )
 
     let coordinate = interactionCoordinate(app: app, x: x, y: y)
     let interval = desktopScrollEventIntervalSeconds(durationMs: durationMs, eventCount: events.count)
     for (index, deltas) in events.enumerated() {
-      // Keep desktop scrolling on XCTest's coordinate API so macOS owns wheel synthesis, natural
-      // scrolling preference handling, and cursor placement instead of posting raw CGEvents.
+
+
       coordinate.scroll(
         byDeltaX: CGFloat(deltas.horizontal),
         deltaY: CGFloat(deltas.vertical)
@@ -456,44 +400,35 @@ extension RunnerTests {
       }
     }
 #elseif os(tvOS)
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "desktopScroll is not supported on tvOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "desktopScroll is not supported on tvOS")
 #else
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "desktopScroll is only supported on macOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "desktopScroll is only supported on macOS")
 #endif
   }
 
-  func desktopScrollWheelDeltas(direction: String, pixels: Double) -> (vertical: Int32, horizontal: Int32)? {
+  func desktopScrollWheelDeltas(
+    direction: RunnerScrollDirection,
+    pixels: Double
+  ) -> (vertical: Int32, horizontal: Int32) {
     let magnitude = Int32(max(1, min(Double(Int32.max), pixels.rounded())))
     switch direction {
-    case "up":
+    case .up:
       return (vertical: magnitude, horizontal: 0)
-    case "down":
+    case .down:
       return (vertical: -magnitude, horizontal: 0)
-    case "left":
+    case .left:
       return (vertical: 0, horizontal: magnitude)
-    case "right":
+    case .right:
       return (vertical: 0, horizontal: -magnitude)
-    default:
-      return nil
     }
   }
 
   func desktopScrollWheelDeltaEvents(
-    direction: String,
+    direction: RunnerScrollDirection,
     pixels: Double,
     durationMs: Double?
-  ) -> [(vertical: Int32, horizontal: Int32)]? {
-    guard let totalDeltas = desktopScrollWheelDeltas(direction: direction, pixels: pixels) else {
-      return nil
-    }
+  ) -> [(vertical: Int32, horizontal: Int32)] {
+    let totalDeltas = desktopScrollWheelDeltas(direction: direction, pixels: pixels)
     let magnitude = max(abs(Int(totalDeltas.vertical)), abs(Int(totalDeltas.horizontal)))
     let duration = max(0, durationMs ?? 0)
     let requestedEventCount = duration > 0 ? Int(ceil(duration / 16.0)) : 1
@@ -550,7 +485,7 @@ extension RunnerTests {
     y2: Double,
     holdDuration: TimeInterval
   ) -> RunnerInteractionOutcome {
-    // tvOS has no coordinate drag. Preserve the direction as a focus move.
+
     let dx = x2 - x
     let dy = y2 - y
     let button: TvRemoteButton = abs(dx) > abs(dy)
@@ -562,9 +497,9 @@ extension RunnerTests {
     return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
   }
 
-  /// Rotates an interface-oriented point into the device-native (portrait) space the
-  /// synthesized event path consumes — synthesized events skip XCTest's orientation
-  /// handling, so without this a landscape tap lands in the wrong place.
+
+
+
   func nativeSynthesizedPoint(
     orientedX x: Double,
     orientedY y: Double,
@@ -582,13 +517,13 @@ extension RunnerTests {
       return CGPoint(x: localY, y: width - localX)
     case RunnerInterfaceOrientation.portraitUpsideDown:
       return CGPoint(x: width - localX, y: height - localY)
-    default:  // portrait or unknown
+    default:
       return CGPoint(x: localX, y: localY)
     }
   }
 
-  /// Rotates an interface-oriented translation vector into the same native
-  /// coordinate space as `nativeSynthesizedPoint`.
+
+
   func nativeSynthesizedVector(
     orientedDx dx: Double,
     orientedDy dy: Double,
@@ -601,7 +536,7 @@ extension RunnerTests {
       return CGVector(dx: dy, dy: -dx)
     case RunnerInterfaceOrientation.portraitUpsideDown:
       return CGVector(dx: -dx, dy: -dy)
-    default:  // portrait or unknown
+    default:
       return CGVector(dx: dx, dy: dy)
     }
   }
@@ -646,6 +581,15 @@ extension RunnerTests {
         y2: Double(end.y),
         durationMs: durationMs
       )
+    case .controlledScroll:
+      RunnerSynthesizedGesture.synthesizeControlledScroll(
+        withApplication: app,
+        x: Double(start.x),
+        y: Double(start.y),
+        x2: Double(end.x),
+        y2: Double(end.y),
+        durationMs: durationMs
+      )
     case .fastSwipe:
       RunnerSynthesizedGesture.synthesizeSwipe(
         withApplication: app,
@@ -938,8 +882,8 @@ extension RunnerTests {
     screenshotSize: CGSize,
     interfaceOrientation: Int
   ) -> CGRect? {
-    // Physical iOS screenshots can retain portrait dimensions after the interface rotates,
-    // while accessibility frames remain in the logical landscape coordinate space.
+
+
     guard screenshotSize.width.isFinite, screenshotSize.height.isFinite,
       screenshotSize.width > 0,
       screenshotSize.height > 0
@@ -1113,8 +1057,8 @@ extension RunnerTests {
   ) -> RunnerInteractionOutcome {
 #if os(iOS)
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    // The portable planner and validation use this exact viewport. Using app.frame here can
-    // diverge when XCTest unions transformed/off-screen descendants into the application frame.
+
+
     let frame = CGRect(
       x: plan.viewport.x,
       y: plan.viewport.y,
@@ -1245,4 +1189,220 @@ extension RunnerTests {
   }
 #endif
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 }