agent-device 0.21.6 → 0.21.8

package/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift

@@ -0,0 +1,61 @@
+import Foundation
+
+/// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a
+/// drag, and a mouse-up posted in the same event tick as its mouse-down is never
+/// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15
+/// mouse-ups and activated 0 actions, 20 ms activated 14 of 15, and 40 ms and above
+/// activated 15 of 15. Holds shorter than this are raised to it.
+public let minimumMouseClickHoldMs = 40
+
+/// Hold used when the caller does not name one. Comfortably above
+/// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff.
+public let defaultMouseClickHoldMs = 60
+
+/// The gap between the two presses of one double-click. Well inside the system
+/// double-click interval (500 ms by default), and independent of the caller's repeat
+/// interval, which separates whole presses rather than the halves of one.
+public let mouseClickPairGapMs = 80
+
+public func mouseClickHoldMs(requestedMs: Int) -> Int {
+  if requestedMs <= 0 {
+    return defaultMouseClickHoldMs
+  }
+  return max(requestedMs, minimumMouseClickHoldMs)
+}
+
+/// One press of the button: the click state it is posted with, and how long after the
+/// previous release it starts.
+public struct MouseClickPress: Equatable, Sendable {
+  /// `1` for an independent click; `2` for the second half of a double-click.
+  public let clickState: Int
+  public let delayBeforeMs: Int
+
+  public init(clickState: Int, delayBeforeMs: Int) {
+    self.clickState = clickState
+    self.delayBeforeMs = delayBeforeMs
+  }
+}
+
+/// The presses one request becomes. `clicks` is always a count of independent presses at
+/// click state 1, which is what every other platform means by `--count`; only
+/// `doubleClick` raises the state, and it does so per press, so `doubleClick` with three
+/// clicks is three double-clicks rather than one triple-click.
+public func mouseClickPresses(clicks: Int, doubleClick: Bool, intervalMs: Int) -> [MouseClickPress] {
+  let gap = max(intervalMs, 0)
+  var presses: [MouseClickPress] = []
+  for index in 0..<max(clicks, 1) {
+    presses.append(MouseClickPress(clickState: 1, delayBeforeMs: index == 0 ? 0 : gap))
+    if doubleClick {
+      presses.append(MouseClickPress(clickState: 2, delayBeforeMs: mouseClickPairGapMs))
+    }
+  }
+  return presses
+}
+
+/// How long the schedule keeps the helper busy: every hold plus every gap. The caller's
+/// process timeout must cover this, or the helper is killed mid-schedule.
+public func mouseClickScheduleMs(holdMs: Int, clicks: Int, doubleClick: Bool, intervalMs: Int) -> Int {
+  let hold = mouseClickHoldMs(requestedMs: holdMs)
+  return mouseClickPresses(clicks: clicks, doubleClick: doubleClick, intervalMs: intervalMs)
+    .reduce(0) { $0 + $1.delayBeforeMs + hold }
+}

package/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift

@@ -0,0 +1,30 @@
+import XCTest
+
+@testable import AgentDeviceMacOSDeviceHub
+
+final class DeviceHubPoseTests: XCTestCase {
+  func testEveryPoseNamesTheActionBarControlDeviceHubDescribes() {
+    XCTAssertEqual(DeviceHubPose(argument: "closed")?.controlDescription, "Closed")
+    XCTAssertEqual(DeviceHubPose(argument: "book")?.controlDescription, "Book")
+    XCTAssertEqual(DeviceHubPose(argument: "OPEN")?.controlDescription, "Open")
+    XCTAssertNil(DeviceHubPose(argument: "half-open"), "the helper takes Device Hub's names, not the CLI's")
+    XCTAssertEqual(Set(DeviceHubPose.allCases.map(\.controlDescription)), ["Closed", "Book", "Open"])
+  }
+
+  // Device Hub titles the window `<name> – iOS 27.1`; a simulator whose name merely starts
+  // with another's must not claim that window.
+  func testWindowTitleMatchesTheDeviceNameAsAPrefixWord() {
+    XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo – iOS 27.1"))
+    XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo Lab – iOS 27.1"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "bench-golden-v1 – iOS 27.0"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: nil))
+  }
+
+  func testSidebarRowIsKeyedByUdid() {
+    XCTAssertEqual(
+      deviceHubDeviceRowIdentifier(udid: "4F879835-4AB3-4046-B033-5AB769209DD4"),
+      "TableRow.Device.4F879835-4AB3-4046-B033-5AB769209DD4"
+    )
+  }
+}

package/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift

@@ -0,0 +1,58 @@
+import XCTest
+
+@testable import AgentDeviceMacOSInput
+
+final class MouseClickScheduleTests: XCTestCase {
+  // The original schedule posted move, down and up back to back. AppKit and SwiftUI
+  // never delivered the release, so every surface press opened a tracking session that
+  // no control ever completed: 0 of 15 measured clicks activated their control.
+  func testHoldShorterThanTheDeliverableFloorIsRaised() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 0), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 1), minimumMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 39), minimumMouseClickHoldMs)
+    XCTAssertGreaterThan(defaultMouseClickHoldMs, minimumMouseClickHoldMs)
+  }
+
+  func testNamedLongPressHoldIsKept() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 800), 800)
+  }
+
+  // `--count N` means N independent presses on every platform; a rising click state would
+  // turn `--count 2` into a double-click and `--count 3` into a triple-click.
+  func testRepeatClicksAreIndependentPressesAtClickStateOne() {
+    let presses = mouseClickPresses(clicks: 3, doubleClick: false, intervalMs: 150)
+    XCTAssertEqual(presses.map(\.clickState), [1, 1, 1])
+    XCTAssertEqual(presses.map(\.delayBeforeMs), [0, 150, 150])
+  }
+
+  // Only `--double-tap` raises the click state, and it does so per press, so it composes
+  // with `--count` into that many double-clicks rather than one triple-click.
+  func testDoubleClickRaisesTheStateInsideEachPress() {
+    XCTAssertEqual(
+      mouseClickPresses(clicks: 1, doubleClick: true, intervalMs: 120).map(\.clickState),
+      [1, 2]
+    )
+    let three = mouseClickPresses(clicks: 3, doubleClick: true, intervalMs: 500)
+    XCTAssertEqual(three.map(\.clickState), [1, 2, 1, 2, 1, 2])
+    XCTAssertEqual(
+      three.map(\.delayBeforeMs),
+      [0, mouseClickPairGapMs, 500, mouseClickPairGapMs, 500, mouseClickPairGapMs]
+    )
+    XCTAssertLessThan(mouseClickPairGapMs, 500, "a pair must land inside the system double-click interval")
+  }
+
+  func testZeroClicksStillPostsOnePress() {
+    XCTAssertEqual(mouseClickPresses(clicks: 0, doubleClick: false, intervalMs: 0).count, 1)
+  }
+
+  // The process timeout the caller sets is derived from this: a schedule the timeout does
+  // not cover is killed mid-hold.
+  func testScheduleDurationSumsEveryHoldAndGap() {
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 0, clicks: 1, doubleClick: false, intervalMs: 120), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 10_000, clicks: 4, doubleClick: false, intervalMs: 120), 4 * 10_000 + 3 * 120)
+    XCTAssertEqual(
+      mouseClickScheduleMs(holdMs: 60, clicks: 2, doubleClick: true, intervalMs: 100),
+      4 * 60 + 2 * mouseClickPairGapMs + 100
+    )
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h

@@ -118,6 +118,10 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey;
 
 + (NSInteger)processIdentifierForApplication:(XCUIApplication *)application;
 
+/// Pids of the applications the AX client currently reports as active. Same private accessor the
+/// snapshot target match uses; pids only — the client resolves no bundle id for an arbitrary app.
++ (NSArray<NSNumber *> *)activeApplicationProcessIdentifiers;
+
 /// The frontier-extension loop, exposed for the runner unit bundle: a frontier
 /// whose element vanished or whose re-rooted request fails must count as
 /// missed (see the deepExtension keys above) — an all-miss extension reporting

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m

@@ -707,6 +707,26 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
   return [self integerFrom:application selectorName:@"processID"];
 }
 
++ (NSArray<NSNumber *> *)activeApplicationProcessIdentifiers
+{
+  id axClient = [self accessibilityClient];
+  if (nil == axClient) {
+    return @[];
+  }
+  id activeApplications = [self objectFrom:axClient selectorName:@"activeApplications"];
+  if (![activeApplications isKindOfClass:NSArray.class]) {
+    return @[];
+  }
+  NSMutableArray<NSNumber *> *processIdentifiers = [NSMutableArray array];
+  for (id candidate in (NSArray *)activeApplications) {
+    NSInteger processIdentifier = [self integerFrom:candidate selectorName:@"processIdentifier"];
+    if (processIdentifier > 0) {
+      [processIdentifiers addObject:@(processIdentifier)];
+    }
+  }
+  return processIdentifiers;
+}
+
 + (id)accessibilityApplicationForApplication:(XCUIApplication *)application axClient:(id)axClient
 {
   NSInteger targetProcessID = [self integerFrom:application selectorName:@"processID"];

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift

@@ -1,4 +1,5 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 
 
@@ -406,8 +407,6 @@ extension RunnerTests {
 
 
 
-
-
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

@@ -216,11 +216,11 @@ extension RunnerTests {
       }
 
       let rootFrame = privateAXRect(root["frame"])
-      let viewport = privateAXSnapshotViewport(app: app, rootFrame: rootFrame)
+      let geometry = privateAXSnapshotGeometry(app: app, rootFrame: rootFrame)
+      let viewport = geometry.viewport
       let nodes = privateAXAcquisition(
         rawRoot: root,
-        hint: hint,
-        viewport: viewport
+        hint: hint
       )
 
 
@@ -254,7 +254,8 @@ extension RunnerTests {
         customActions: Self.privateAXCustomActionCoverage(
           response[RunnerAXSnapshotCustomActionsKey]
         ),
-        viewport: viewport
+        viewport: viewport,
+        interfaceOrientation: geometry.interfaceOrientation
       )
     #else
       return nil
@@ -269,26 +270,36 @@ extension RunnerTests {
     !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
   }
 
-  private func privateAXSnapshotViewport(app: XCUIApplication, rootFrame: CGRect) -> CGRect {
+
+
+
+
+  private func privateAXSnapshotGeometry(
+    app: XCUIApplication,
+    rootFrame: CGRect
+  ) -> (viewport: CGRect, interfaceOrientation: Int) {
     let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame
     guard shouldReadPrivateAXViewportViaXCTest() else {
-      return fallback
+      return (fallback, RunnerInterfaceOrientation.unknown)
     }
     do {
-      let viewport = try runMainThreadWork(
+      let anchor = try runMainThreadWork(
         "private_ax_viewport",
         timeout: 1,
         timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport")
       ) {
-        self.safeSnapshotViewport(app: app)
+        (
+          viewport: self.safeSnapshotViewport(app: app),
+          interfaceOrientation: self.capturedInterfaceOrientation(app: app)
+        )
       }
-      if viewport.isInfinite || viewport.isNull || viewport.isEmpty {
-        return fallback
+      if anchor.viewport.isInfinite || anchor.viewport.isNull || anchor.viewport.isEmpty {
+        return (fallback, RunnerInterfaceOrientation.unknown)
       }
-      return viewport
+      return (anchor.viewport, anchor.interfaceOrientation)
     } catch {
       NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_VIEWPORT_FALLBACK=%@", String(describing: error))
-      return fallback
+      return (fallback, RunnerInterfaceOrientation.unknown)
     }
   }
 
@@ -857,6 +868,8 @@ extension RunnerTests {
 
 
 
+
+
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -898,6 +898,34 @@ extension RunnerTests {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -921,11 +949,18 @@ extension RunnerTests {
 
   func executeAccepted(command: Command) throws -> Response {
     commandJournal.start(command: command)
+    pendingTargetActivation = nil
     do {
       let response = try executeDispatched(command: command)
       commandJournal.finish(command: command, response: response)
-      return response
+      guard let fact = pendingTargetActivation else { return response }
+
+
+
+      pendingTargetActivation = nil
+      return response.stampingTargetActivation(fact)
     } catch {
+      pendingTargetActivation = nil
       commandJournal.fail(command: command, error: error)
       throw error
     }
@@ -2080,6 +2115,18 @@ extension RunnerTests {
     case .appSwitcher:
       performAppSwitcherGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "appSwitcher"))
+    case .actionButton:
+      guard pressActionButton() else {
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: "UNSUPPORTED_OPERATION",
+            message: "actionButton requires a device model with an Action Button",
+            hint: "The Action Button is on iPhone 15 Pro and later and iPad Pro (M4) and later. Assign a Shortcut or App Intent to it in Settings > Action Button."
+          )
+        )
+      }
+      return Response(ok: true, data: DataPayload(message: "actionButton"))
     case .keyboardDismiss:
       let result = dismissKeyboard(app: activeApp)
       if result.wasVisible && !result.dismissed {
@@ -2544,6 +2591,14 @@ extension RunnerTests {
 
 
 
+
+    if command.command == .actionButton {
+      return true
+    }
+
+
+
+
     guard command.text == nil, command.selectorKey == nil else { return false }
     guard hasCachedTargetForActivationSkip(command: command) else { return false }
     return isCoordinateOnlyTap(command)

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -137,7 +137,7 @@ final class RunnerCommandJournal {
       return false
     case .tap, .mouseClick, .longPress, .drag,
          .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .back,
-         .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn,
+         .backInApp, .backSystem, .home, .rotate, .appSwitcher, .actionButton, .keyboardDismiss, .keyboardReturn,
          .alert, .sequence, .gesture, .gestureViewport, .recordStart, .recordStop,
          .status, .uptime, .activate, .terminate, .targetReset, .shutdown:
       return true

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -11,16 +11,6 @@ private struct RunnerUnsupportedOperationError: LocalizedError {
   var errorDescription: String? { message }
 }
 
-enum RunnerInterfaceOrientation {
-
-
-
-  static let portrait = 1
-  static let portraitUpsideDown = 2
-  static let landscapeRight = 3
-  static let landscapeLeft = 4
-}
-
 extension RunnerTests {
   enum PlannedGestureExecution: Equatable {
     case fastSwipe
@@ -134,6 +124,24 @@ extension RunnerTests {
 #endif
   }
 
+
+
+
+
+
+
+  @discardableResult
+  func pressActionButton() -> Bool {
+#if os(iOS)
+    guard #available(iOS 16.0, *) else { return false }
+    guard XCUIDevice.shared.hasHardwareButton(.action) else { return false }
+    XCUIDevice.shared.press(.action)
+    return true
+#else
+    return false
+#endif
+  }
+
   func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
     let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
     let element = app.descendants(matching: .any).matching(predicate).firstMatch
@@ -252,7 +260,7 @@ extension RunnerTests {
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: valueText.isEmpty ? nil : valueText,
-        rect: snapshotRect(from: element.frame),
+        rect: SnapshotRect(element.frame),
         enabled: element.isEnabled,
         focused: nil,
         selected: element.isSelected ? true : nil,
@@ -497,50 +505,6 @@ extension RunnerTests {
     return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
   }
 
-
-
-
-  func nativeSynthesizedPoint(
-    orientedX x: Double,
-    orientedY y: Double,
-    in frame: CGRect,
-    interfaceOrientation: Int
-  ) -> CGPoint {
-    let localX = x - Double(frame.minX)
-    let localY = y - Double(frame.minY)
-    let width = Double(frame.width)
-    let height = Double(frame.height)
-    switch interfaceOrientation {
-    case RunnerInterfaceOrientation.landscapeRight:
-      return CGPoint(x: height - localY, y: localX)
-    case RunnerInterfaceOrientation.landscapeLeft:
-      return CGPoint(x: localY, y: width - localX)
-    case RunnerInterfaceOrientation.portraitUpsideDown:
-      return CGPoint(x: width - localX, y: height - localY)
-    default:
-      return CGPoint(x: localX, y: localY)
-    }
-  }
-
-
-
-  func nativeSynthesizedVector(
-    orientedDx dx: Double,
-    orientedDy dy: Double,
-    interfaceOrientation: Int
-  ) -> CGVector {
-    switch interfaceOrientation {
-    case RunnerInterfaceOrientation.landscapeRight:
-      return CGVector(dx: -dy, dy: dx)
-    case RunnerInterfaceOrientation.landscapeLeft:
-      return CGVector(dx: dy, dy: -dx)
-    case RunnerInterfaceOrientation.portraitUpsideDown:
-      return CGVector(dx: -dx, dy: -dy)
-    default:
-      return CGVector(dx: dx, dy: dy)
-    }
-  }
-
   func synthesizedDragAt(
     app: XCUIApplication,
     x: Double,
@@ -569,8 +533,16 @@ extension RunnerTests {
       )
     }
     let frame = context.referenceFrame
-    let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
-    let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
+    let start = CoordinateSpaceRotation.native(
+      point: CGPoint(x: x, y: y),
+      in: frame,
+      interfaceOrientation: orientation
+    )
+    let end = CoordinateSpaceRotation.native(
+      point: CGPoint(x: x2, y: y2),
+      in: frame,
+      interfaceOrientation: orientation
+    )
     let message = switch profile {
     case .continuous:
       RunnerSynthesizedGesture.synthesizeContinuousDrag(
@@ -644,7 +616,11 @@ extension RunnerTests {
       )
     }
     let frame = context.referenceFrame
-    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
+    let point = CoordinateSpaceRotation.native(
+      point: CGPoint(x: x, y: y),
+      in: frame,
+      interfaceOrientation: orientation
+    )
     if let message = RunnerSynthesizedGesture.synthesizeTap(
       withApplication: app,
       x: Double(point.x),
@@ -1055,9 +1031,8 @@ extension RunnerTests {
     )
     let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
       pointer.samples.map { sample in
-        let point = nativeSynthesizedPoint(
-          orientedX: sample.point.x,
-          orientedY: sample.point.y,
+        let point = CoordinateSpaceRotation.native(
+          point: CGPoint(x: sample.point.x, y: sample.point.y),
           in: frame,
           interfaceOrientation: orientation
         )
@@ -1325,53 +1300,6 @@ extension RunnerTests {
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-