agent-device 0.20.10 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -1,11 +1,17 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 #if os(macOS)
 import CoreGraphics
 #endif
 
+private struct RunnerUnsupportedOperationError: LocalizedError {
+  let message: String
+
+  var errorDescription: String? { message }
+}
+
 private enum RunnerInterfaceOrientation {
-  static let unknown = 0
   static let portrait = 1
   static let portraitUpsideDown = 2
   static let landscapeRight = 3
@@ -18,11 +24,18 @@ extension RunnerTests {
     case sampled
   }
 
-  enum SynthesizedDragProfile {
+  enum SynthesizedDragProfile: Equatable {
     case continuous
+    case controlledScroll
     case fastSwipe
   }
 
+  func scrollDragProfile(
+    releaseBehavior: ScrollReleaseBehavior?
+  ) -> SynthesizedDragProfile {
+    releaseBehavior == .inertial ? .fastSwipe : .controlledScroll
+  }
+
   struct TouchVisualizationFrame {
     let x: Double
     let y: Double
@@ -257,6 +270,23 @@ extension RunnerTests {
     )
   }
 
+  /// Shared ordering for point-hit candidates: smallest area wins, then top-to-bottom,
+  /// left-to-right, then stable element-type order for ties.
+  func smallestElementFirst(_ left: XCUIElement, _ right: XCUIElement) -> Bool {
+    let leftArea = max(1, left.frame.width * left.frame.height)
+    let rightArea = max(1, right.frame.width * right.frame.height)
+    if leftArea != rightArea {
+      return leftArea < rightArea
+    }
+    if left.frame.minY != right.frame.minY {
+      return left.frame.minY < right.frame.minY
+    }
+    if left.frame.minX != right.frame.minX {
+      return left.frame.minX < right.frame.minX
+    }
+    return left.elementType.rawValue < right.elementType.rawValue
+  }
+
   func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
     let point = CGPoint(x: x, y: y)
     let textInputCandidates = textInputCandidatesAt(app: app, point: point)
@@ -270,20 +300,7 @@ extension RunnerTests {
       .filter { element in
         element.exists && !element.frame.isEmpty && element.frame.contains(point)
       }
-      .sorted { left, right in
-        let leftArea = max(1, left.frame.width * left.frame.height)
-        let rightArea = max(1, right.frame.width * right.frame.height)
-        if leftArea != rightArea {
-          return leftArea < rightArea
-        }
-        if left.frame.minY != right.frame.minY {
-          return left.frame.minY < right.frame.minY
-        }
-        if left.frame.minX != right.frame.minX {
-          return left.frame.minX < right.frame.minX
-        }
-        return left.elementType.rawValue < right.elementType.rawValue
-      }
+      .sorted(by: smallestElementFirst)
 
     for element in candidates where prefersExpandedTextRead(element) {
       if let text = readableText(for: element) {
@@ -325,20 +342,7 @@ extension RunnerTests {
             point: point
           )
         }
-        .sorted { left, right in
-          let leftArea = max(1, left.frame.width * left.frame.height)
-          let rightArea = max(1, right.frame.width * right.frame.height)
-          if leftArea != rightArea {
-            return leftArea < rightArea
-          }
-          if left.frame.minY != right.frame.minY {
-            return left.frame.minY < right.frame.minY
-          }
-          if left.frame.minX != right.frame.minX {
-            return left.frame.minX < right.frame.minX
-          }
-          return left.elementType.rawValue < right.elementType.rawValue
-        }
+        .sorted(by: smallestElementFirst)
     }
   }
 
@@ -368,16 +372,6 @@ extension RunnerTests {
     }
   }
 
-  func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
-    let predicate = NSPredicate(
-      format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
-      scope,
-      scope
-    )
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
-  }
-
   func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
     if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") {
       return outcome
@@ -394,30 +388,14 @@ extension RunnerTests {
     case "secondary":
       coordinate.rightClick()
     case "middle":
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
-      )
+      throw RunnerUnsupportedOperationError(message: "middle mouse button is not supported")
     default:
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
-      )
+      throw RunnerUnsupportedOperationError(message: "unsupported mouse button: \(button)")
     }
 #elseif os(tvOS)
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "mouseClick is not supported on tvOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "mouseClick is not supported on tvOS")
 #else
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "mouseClick is only supported on macOS")
 #endif
   }
 
@@ -425,22 +403,16 @@ extension RunnerTests {
     app: XCUIApplication,
     x: Double,
     y: Double,
-    direction: String,
+    direction: RunnerScrollDirection,
     pixels: Double,
     durationMs: Double?
   ) throws {
 #if os(macOS)
-    guard let events = desktopScrollWheelDeltaEvents(
+    let events = desktopScrollWheelDeltaEvents(
       direction: direction,
       pixels: pixels,
       durationMs: durationMs
-    ) else {
-      throw NSError(
-        domain: "AgentDeviceRunner",
-        code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "unsupported desktop scroll direction: \(direction)"]
-      )
-    }
+    )
 
     let coordinate = interactionCoordinate(app: app, x: x, y: y)
     let interval = desktopScrollEventIntervalSeconds(durationMs: durationMs, eventCount: events.count)
@@ -456,44 +428,35 @@ extension RunnerTests {
       }
     }
 #elseif os(tvOS)
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "desktopScroll is not supported on tvOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "desktopScroll is not supported on tvOS")
 #else
-    throw NSError(
-      domain: "AgentDeviceRunner",
-      code: 1,
-      userInfo: [NSLocalizedDescriptionKey: "desktopScroll is only supported on macOS"]
-    )
+    throw RunnerUnsupportedOperationError(message: "desktopScroll is only supported on macOS")
 #endif
   }
 
-  func desktopScrollWheelDeltas(direction: String, pixels: Double) -> (vertical: Int32, horizontal: Int32)? {
+  func desktopScrollWheelDeltas(
+    direction: RunnerScrollDirection,
+    pixels: Double
+  ) -> (vertical: Int32, horizontal: Int32) {
     let magnitude = Int32(max(1, min(Double(Int32.max), pixels.rounded())))
     switch direction {
-    case "up":
+    case .up:
       return (vertical: magnitude, horizontal: 0)
-    case "down":
+    case .down:
       return (vertical: -magnitude, horizontal: 0)
-    case "left":
+    case .left:
       return (vertical: 0, horizontal: magnitude)
-    case "right":
+    case .right:
       return (vertical: 0, horizontal: -magnitude)
-    default:
-      return nil
     }
   }
 
   func desktopScrollWheelDeltaEvents(
-    direction: String,
+    direction: RunnerScrollDirection,
     pixels: Double,
     durationMs: Double?
-  ) -> [(vertical: Int32, horizontal: Int32)]? {
-    guard let totalDeltas = desktopScrollWheelDeltas(direction: direction, pixels: pixels) else {
-      return nil
-    }
+  ) -> [(vertical: Int32, horizontal: Int32)] {
+    let totalDeltas = desktopScrollWheelDeltas(direction: direction, pixels: pixels)
     let magnitude = max(abs(Int(totalDeltas.vertical)), abs(Int(totalDeltas.horizontal)))
     let duration = max(0, durationMs ?? 0)
     let requestedEventCount = duration > 0 ? Int(ceil(duration / 16.0)) : 1
@@ -646,6 +609,15 @@ extension RunnerTests {
         y2: Double(end.y),
         durationMs: durationMs
       )
+    case .controlledScroll:
+      RunnerSynthesizedGesture.synthesizeControlledScroll(
+        withApplication: app,
+        x: Double(start.x),
+        y: Double(start.y),
+        x2: Double(end.x),
+        y2: Double(end.y),
+        durationMs: durationMs
+      )
     case .fastSwipe:
       RunnerSynthesizedGesture.synthesizeSwipe(
         withApplication: app,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -108,7 +108,7 @@ extension RunnerTests {
     // old PID before refreshCachedTargetIfProcessChanged can observe it.
     clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch")
     clearPrivateAXAcceptedDepth(reason: "external_app_relaunch")
-    needsFirstInteractionDelay = true
+    beginFirstInteractionStabilization()
     return Response(ok: true, data: DataPayload(message: "target reset"))
   }
 
@@ -132,7 +132,7 @@ extension RunnerTests {
     clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
     clearPrivateAXAcceptedDepth(reason: "target_process_changed")
     snapshotXCTestPenaltyWarmupExemptionPending = true
-    needsFirstInteractionDelay = true
+    beginFirstInteractionStabilization()
   }
 
   static func processIdentifier(of target: XCUIApplication) -> Int? {
@@ -205,7 +205,7 @@ extension RunnerTests {
     currentAppProcessIdentifier = Self.processIdentifier(of: target)
     clearRememberedTextEntryTap()
     snapshotXCTestPenaltyWarmupExemptionPending = false
-    needsFirstInteractionDelay = true
+    beginFirstInteractionStabilization()
     return target
   }
 
@@ -320,12 +320,19 @@ extension RunnerTests {
       sleepFor(postSnapshotInteractionDelay)
       needsPostSnapshotInteractionDelay = false
     }
-    if needsFirstInteractionDelay {
-      sleepFor(firstInteractionAfterActivateDelay)
-      needsFirstInteractionDelay = false
+    if let readyUptime = firstInteractionReadyUptime {
+      sleepFor(readyUptime - ProcessInfo.processInfo.systemUptime)
+      firstInteractionReadyUptime = nil
     }
   }
 
+  /// Start the post-activation settling window. Measured from now, so the time the caller spends
+  /// getting back to us counts towards it instead of being charged twice.
+  func beginFirstInteractionStabilization() {
+    firstInteractionReadyUptime =
+      ProcessInfo.processInfo.systemUptime + firstInteractionAfterActivateDelay
+  }
+
   func sleepFor(_ delay: TimeInterval) {
     guard delay > 0 else { return }
     // Keep XCTest/UI sources moving during command-local pauses such as delayed typing.

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -1,3 +1,5 @@
+import AgentDeviceSnapshotPresentation
+
 // MARK: - Wire Models
 
 enum CommandType: String, Codable {
@@ -132,6 +134,7 @@ struct Command: Codable {
   let direction: String?
   let amount: Double?
   let pixels: Double?
+  let scrollReleaseBehavior: ScrollReleaseBehavior?
   let orientation: String?
   let gesturePlan: RunnerGesturePlan?
   let outPath: String?
@@ -148,6 +151,11 @@ struct Command: Codable {
   let steps: [SequenceStep]?
 }
 
+enum ScrollReleaseBehavior: String, Codable {
+  case controlled
+  case inertial
+}
+
 /// Canonical one- or two-pointer plan produced by the portable TypeScript planner.
 struct RunnerGesturePlan: Codable {
   let topology: String
@@ -207,14 +215,8 @@ struct SequenceStepResult: Codable {
 
 struct Response: Codable {
   let ok: Bool
-  let data: DataPayload?
-  let error: ErrorPayload?
-
-  init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
-    self.ok = ok
-    self.data = data
-    self.error = error
-  }
+  var data: DataPayload?
+  var error: ErrorPayload?
 }
 
 extension Response {
@@ -230,161 +232,76 @@ extension Response {
 }
 
 struct DataPayload: Codable {
-  let message: String?
-  let imageBase64: String?
-  let text: String?
-  let found: Bool?
-  let items: [String]?
-  let nodes: [PresentedNode]?
-  let truncated: Bool?
-  let snapshotQuality: SnapshotQuality?
-  let gestureStartUptimeMs: Double?
-  let gestureEndUptimeMs: Double?
-  let x: Double?
-  let y: Double?
-  let x2: Double?
-  let y2: Double?
-  let referenceWidth: Double?
-  let referenceHeight: Double?
+  var message: String?
+  var imageBase64: String?
+  var text: String?
+  var found: Bool?
+  var items: [String]?
+  var nodes: [PresentedNode]?
+  var truncated: Bool?
+  var qualityPayload: SnapshotQualityPayload? = nil
+  var snapshotQuality: SnapshotQuality?
+  var gestureStartUptimeMs: Double?
+  var gestureEndUptimeMs: Double?
+  var x: Double?
+  var y: Double?
+  var x2: Double?
+  var y2: Double?
+  var referenceWidth: Double?
+  var referenceHeight: Double?
   var currentUptimeMs: Double?
-  let commandId: String?
-  let lifecycleState: String?
-  let lifecycleCommand: String?
-  let lifecycleResponseOk: Bool?
-  let lifecycleResponseJson: String?
-  let lifecycleErrorCode: String?
-  let lifecycleErrorMessage: String?
-  let lifecycleErrorHint: String?
-  let visible: Bool?
-  let wasVisible: Bool?
-  let dismissed: Bool?
-  let keyboardDismissMechanism: String?
-  let orientation: String?
-  let gestureFallback: String?
-  let gestureFallbackMessage: String?
-  let gestureFallbackHint: String?
-  let maestroNonHittableCoordinateFallbackUsed: Bool?
-  let textEntryRoute: String?
-  let runnerFatal: Bool?
-  let runnerFatalReason: String?
-  let completedSteps: Int?
-  let failedStepIndex: Int?
-  let sequenceResults: [SequenceStepResult]?
+  var commandId: String?
+  var lifecycleState: String?
+  var lifecycleCommand: String?
+  var lifecycleResponseOk: Bool?
+  var lifecycleResponseJson: String?
+  var lifecycleErrorCode: String?
+  var lifecycleErrorMessage: String?
+  var lifecycleErrorHint: String?
+  var visible: Bool?
+  var wasVisible: Bool?
+  var dismissed: Bool?
+  var keyboardDismissMechanism: String?
+  var orientation: String?
+  var gestureFallback: String?
+  var gestureFallbackMessage: String?
+  var gestureFallbackHint: String?
+  var maestroNonHittableCoordinateFallbackUsed: Bool?
+  var textEntryRoute: String?
+  var runnerFatal: Bool?
+  var runnerFatalReason: String?
+  var completedSteps: Int?
+  var failedStepIndex: Int?
+  var sequenceResults: [SequenceStepResult]?
+}
 
-  init(
-    message: String? = nil,
-    imageBase64: String? = nil,
-    text: String? = nil,
-    found: Bool? = nil,
-    items: [String]? = nil,
-    nodes: [PresentedNode]? = nil,
-    truncated: Bool? = nil,
-    snapshotQuality: SnapshotQuality? = nil,
-    gestureStartUptimeMs: Double? = nil,
-    gestureEndUptimeMs: Double? = nil,
-    x: Double? = nil,
-    y: Double? = nil,
-    x2: Double? = nil,
-    y2: Double? = nil,
-    referenceWidth: Double? = nil,
-    referenceHeight: Double? = nil,
-    currentUptimeMs: Double? = nil,
-    commandId: String? = nil,
-    lifecycleState: String? = nil,
-    lifecycleCommand: String? = nil,
-    lifecycleResponseOk: Bool? = nil,
-    lifecycleResponseJson: String? = nil,
-    lifecycleErrorCode: String? = nil,
-    lifecycleErrorMessage: String? = nil,
-    lifecycleErrorHint: String? = nil,
-    visible: Bool? = nil,
-    wasVisible: Bool? = nil,
-    dismissed: Bool? = nil,
-    keyboardDismissMechanism: String? = nil,
-    orientation: String? = nil,
-    gestureFallback: String? = nil,
-    gestureFallbackMessage: String? = nil,
-    gestureFallbackHint: String? = nil,
-    maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
-    textEntryRoute: String? = nil,
-    runnerFatal: Bool? = nil,
-    runnerFatalReason: String? = nil,
-    completedSteps: Int? = nil,
-    failedStepIndex: Int? = nil,
-    sequenceResults: [SequenceStepResult]? = nil
-  ) {
-    self.message = message
-    self.imageBase64 = imageBase64
-    self.text = text
-    self.found = found
-    self.items = items
+struct SnapshotQualityPayload: Codable {
+  let nodes: [PresentedNode]
+  let truncated: Bool
+  let scope: String?
+
+  init(nodes: [PresentedNode], truncated: Bool) {
     self.nodes = nodes
     self.truncated = truncated
-    self.snapshotQuality = snapshotQuality
-    self.gestureStartUptimeMs = gestureStartUptimeMs
-    self.gestureEndUptimeMs = gestureEndUptimeMs
-    self.x = x
-    self.y = y
-    self.x2 = x2
-    self.y2 = y2
-    self.referenceWidth = referenceWidth
-    self.referenceHeight = referenceHeight
-    self.currentUptimeMs = currentUptimeMs
-    self.commandId = commandId
-    self.lifecycleState = lifecycleState
-    self.lifecycleCommand = lifecycleCommand
-    self.lifecycleResponseOk = lifecycleResponseOk
-    self.lifecycleResponseJson = lifecycleResponseJson
-    self.lifecycleErrorCode = lifecycleErrorCode
-    self.lifecycleErrorMessage = lifecycleErrorMessage
-    self.lifecycleErrorHint = lifecycleErrorHint
-    self.visible = visible
-    self.wasVisible = wasVisible
-    self.dismissed = dismissed
-    self.keyboardDismissMechanism = keyboardDismissMechanism
-    self.orientation = orientation
-    self.gestureFallback = gestureFallback
-    self.gestureFallbackMessage = gestureFallbackMessage
-    self.gestureFallbackHint = gestureFallbackHint
-    self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
-    self.textEntryRoute = textEntryRoute
-    self.runnerFatal = runnerFatal
-    self.runnerFatalReason = runnerFatalReason
-    self.completedSteps = completedSteps
-    self.failedStepIndex = failedStepIndex
-    self.sequenceResults = sequenceResults
+    self.scope = nil
   }
-}
-
-struct ErrorPayload: Codable {
-  let code: String?
-  let message: String
-  let hint: String?
 
-  init(code: String? = nil, message: String, hint: String? = nil) {
-    self.code = code
-    self.message = message
-    self.hint = hint
+  private enum CodingKeys: String, CodingKey {
+    case nodes
+    case truncated
+    case scope
   }
-}
 
-struct SnapshotRect: Codable {
-  let x: Double
-  let y: Double
-  let width: Double
-  let height: Double
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    try container.encode(nodes, forKey: .nodes)
+    try container.encode(truncated, forKey: .truncated)
+    try container.encodeNil(forKey: .scope)
+  }
 }
 
-struct PresentationOptions {
-  let interactiveOnly: Bool
-  let depth: Int?
-  let scope: String?
-  let raw: Bool
-  /// Internal daemon ask: capture with this backend first regardless of channel
-  /// health ("private-ax"). Same-backend evidence probes (tap-outcome
-  /// corroboration) must be captured the way their baseline was.
-  var preferredBackend: String? = nil
-  /// Read UIAccessibilityCustomActions for merged leaves. Opt-in: each element
-  /// costs its own AX round trip (see RunnerAXSnapshotBridge).
-  var customActions: Bool = false
+struct ErrorPayload: Codable {
+  var code: String?
+  let message: String
+  var hint: String?
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXElementTypes.swift

@@ -0,0 +1,12 @@
+import XCTest
+
+extension RunnerTests {
+  func privateAXElementType(rawElementType: Int) -> XCUIElement.ElementType? {
+    guard let raw = UInt(exactly: rawElementType),
+      let type = XCUIElement.ElementType(rawValue: raw)
+    else {
+      return nil
+    }
+    return type
+  }
+}