agent-device 0.21.7 → 0.21.9

package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -1,3 +1,4 @@
+import AgentDeviceMacOSInput
 import AppKit
 import ApplicationServices
 import CoreGraphics
@@ -63,6 +64,10 @@ struct ReadResponse: Encodable {
 struct PressResponse: Encodable {
   let x: Double
   let y: Double
+  /// Hold actually posted; a short request is raised to the deliverable floor.
+  let holdMs: Int
+  let clicks: Int
+  let doubleClick: Bool
   let bundleId: String?
   let surface: String?
 }
@@ -122,6 +127,8 @@ struct AgentDeviceMacOSHelper {
       return try handleScreenshot(arguments: Array(arguments.dropFirst()))
     case "audio-probe":
       return try handleAudioProbe(arguments: Array(arguments.dropFirst()))
+    case "device-hub":
+      return try handleDeviceHub(arguments: Array(arguments.dropFirst()))
     default:
       throw HelperError.invalidArgs("unknown command: \(command)")
     }
@@ -378,10 +385,48 @@ struct AgentDeviceMacOSHelper {
       throw HelperError.invalidArgs("press requires --x <number> --y <number>")
     }
 
+    let holdMs = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--hold-ms"),
+      name: "--hold-ms",
+      minimum: 0,
+      default: 0
+    )
+    let clicks = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--clicks"),
+      name: "--clicks",
+      minimum: 1,
+      default: 1
+    )
+    let intervalMs = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--interval-ms"),
+      name: "--interval-ms",
+      minimum: 0,
+      default: 120
+    )
+
+    let doubleClick = arguments.contains("--double-click")
     let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
     let surface = optionValue(arguments: arguments, name: "--surface")
-    try pressAtPosition(bundleId: bundleId, surface: surface, x: x, y: y)
-    return SuccessEnvelope(data: PressResponse(x: x, y: y, bundleId: bundleId, surface: surface))
+    let request = MouseClickRequest(
+      x: x,
+      y: y,
+      holdMs: holdMs,
+      clicks: clicks,
+      doubleClick: doubleClick,
+      intervalMs: intervalMs
+    )
+    try pressAtPosition(request)
+    return SuccessEnvelope(
+      data: PressResponse(
+        x: x,
+        y: y,
+        holdMs: mouseClickHoldMs(requestedMs: holdMs),
+        clicks: clicks,
+        doubleClick: doubleClick,
+        bundleId: bundleId,
+        surface: surface
+      )
+    )
   }
 
   static func handleScreenshot(arguments: [String]) throws -> any Encodable {
@@ -434,6 +479,21 @@ private func intOption(arguments: [String], name: String) -> Int? {
   return Int(value)
 }
 
+private func validatedPressInt(
+  _ raw: String?,
+  name: String,
+  minimum: Int,
+  default fallback: Int
+) throws -> Int {
+  guard let raw else {
+    return fallback
+  }
+  guard let value = Int(raw), value >= minimum else {
+    throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)")
+  }
+  return value
+}
+
 private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String {
   let targetApp: NSRunningApplication?
   if surface == "frontmost-app" || (surface == nil && bundleId != nil) {
@@ -481,19 +541,12 @@ private func readTextAtPosition(bundleId: String?, surface: String?, x: Double,
   throw HelperError.commandFailed("read did not resolve text")
 }
 
-private func pressAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws {
-  _ = bundleId
-  _ = surface
-  let point = CGPoint(x: x, y: y)
-  guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left),
-        let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
-        let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)
-  else {
+private func pressAtPosition(_ request: MouseClickRequest) throws {
+  do {
+    try postMouseClick(request)
+  } catch MouseClickDeliveryError.eventCreationFailed {
     throw HelperError.commandFailed("press action failed", details: ["reason": "event_creation_failed"])
   }
-  move.post(tap: .cghidEventTap)
-  down.post(tap: .cghidEventTap)
-  up.post(tap: .cghidEventTap)
 }
 
 private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws {

package/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift

@@ -0,0 +1,116 @@
+import CoreGraphics
+import Foundation
+
+public struct MouseClickRequest: Equatable, Sendable {
+  public let x: Double
+  public let y: Double
+  /// How long the button stays down. Zero asks for the default hold.
+  public let holdMs: Int
+  /// Independent presses, each at click state 1.
+  public let clicks: Int
+  /// Post every press as a double-click pair with a rising click state.
+  public let doubleClick: Bool
+  public let intervalMs: Int
+
+  public init(
+    x: Double,
+    y: Double,
+    holdMs: Int = 0,
+    clicks: Int = 1,
+    doubleClick: Bool = false,
+    intervalMs: Int = 120
+  ) {
+    self.x = x
+    self.y = y
+    self.holdMs = holdMs
+    self.clicks = clicks
+    self.doubleClick = doubleClick
+    self.intervalMs = intervalMs
+  }
+}
+
+public enum MouseClickDeliveryError: Error, Equatable {
+  case eventCreationFailed
+}
+
+/// The button the helper currently holds down, kept where a signal handler can reach it.
+/// A helper killed between a mouse-down and its mouse-up would otherwise leave the system's
+/// primary button stuck down for whatever the user touches next. The host stops the helper
+/// with SIGTERM before SIGKILL (`runMacOsHelper` in `helper.ts`) so that this handler runs
+/// on a deadline and on a cancelled request, not only on a signal sent by hand.
+nonisolated(unsafe) private var heldMouseButton: (point: CGPoint, clickState: Int)?
+
+private func releaseHeldMouseButton() {
+  guard let held = heldMouseButton else { return }
+  heldMouseButton = nil
+  let up = CGEvent(
+    mouseEventSource: nil,
+    mouseType: .leftMouseUp,
+    mouseCursorPosition: held.point,
+    mouseButton: .left
+  )
+  up?.setIntegerValueField(.mouseEventClickState, value: Int64(held.clickState))
+  up?.post(tap: .cghidEventTap)
+}
+
+private func installMouseReleaseOnTermination() {
+  for terminationSignal in [SIGTERM, SIGINT, SIGHUP] {
+    signal(terminationSignal) { received in
+      releaseHeldMouseButton()
+      _exit(128 + received)
+    }
+  }
+}
+
+/// Posts a click the way a trackpad would: one motion to the point, then each press held
+/// long enough for the app to accept the release. A termination signal that lands inside a
+/// hold releases the button before the process exits.
+public func postMouseClick(_ request: MouseClickRequest) throws {
+  installMouseReleaseOnTermination()
+  let point = CGPoint(x: request.x, y: request.y)
+  let hold = mouseClickHoldMs(requestedMs: request.holdMs)
+  let presses = mouseClickPresses(
+    clicks: request.clicks,
+    doubleClick: request.doubleClick,
+    intervalMs: request.intervalMs
+  )
+
+  guard let move = CGEvent(
+    mouseEventSource: nil,
+    mouseType: .mouseMoved,
+    mouseCursorPosition: point,
+    mouseButton: .left
+  ) else {
+    throw MouseClickDeliveryError.eventCreationFailed
+  }
+  move.post(tap: .cghidEventTap)
+
+  for press in presses {
+    if press.delayBeforeMs > 0 {
+      usleep(UInt32(press.delayBeforeMs) * 1000)
+    }
+    guard let down = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .leftMouseDown,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ), let up = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .leftMouseUp,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ) else {
+      throw MouseClickDeliveryError.eventCreationFailed
+    }
+    down.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState))
+    up.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState))
+    heldMouseButton = (point, press.clickState)
+    down.post(tap: .cghidEventTap)
+    usleep(UInt32(hold) * 1000)
+    // The record clears only after the up is posted: a signal that lands between the two
+    // would otherwise find nothing to release and exit with the button still down. A signal
+    // that lands after the post releases a button already up, which is harmless.
+    up.post(tap: .cghidEventTap)
+    heldMouseButton = nil
+  }
+}

package/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift

@@ -0,0 +1,61 @@
+import Foundation
+
+/// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a
+/// drag, and a mouse-up posted in the same event tick as its mouse-down is never
+/// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15
+/// mouse-ups and activated 0 actions, 20 ms activated 14 of 15, and 40 ms and above
+/// activated 15 of 15. Holds shorter than this are raised to it.
+public let minimumMouseClickHoldMs = 40
+
+/// Hold used when the caller does not name one. Comfortably above
+/// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff.
+public let defaultMouseClickHoldMs = 60
+
+/// The gap between the two presses of one double-click. Well inside the system
+/// double-click interval (500 ms by default), and independent of the caller's repeat
+/// interval, which separates whole presses rather than the halves of one.
+public let mouseClickPairGapMs = 80
+
+public func mouseClickHoldMs(requestedMs: Int) -> Int {
+  if requestedMs <= 0 {
+    return defaultMouseClickHoldMs
+  }
+  return max(requestedMs, minimumMouseClickHoldMs)
+}
+
+/// One press of the button: the click state it is posted with, and how long after the
+/// previous release it starts.
+public struct MouseClickPress: Equatable, Sendable {
+  /// `1` for an independent click; `2` for the second half of a double-click.
+  public let clickState: Int
+  public let delayBeforeMs: Int
+
+  public init(clickState: Int, delayBeforeMs: Int) {
+    self.clickState = clickState
+    self.delayBeforeMs = delayBeforeMs
+  }
+}
+
+/// The presses one request becomes. `clicks` is always a count of independent presses at
+/// click state 1, which is what every other platform means by `--count`; only
+/// `doubleClick` raises the state, and it does so per press, so `doubleClick` with three
+/// clicks is three double-clicks rather than one triple-click.
+public func mouseClickPresses(clicks: Int, doubleClick: Bool, intervalMs: Int) -> [MouseClickPress] {
+  let gap = max(intervalMs, 0)
+  var presses: [MouseClickPress] = []
+  for index in 0..<max(clicks, 1) {
+    presses.append(MouseClickPress(clickState: 1, delayBeforeMs: index == 0 ? 0 : gap))
+    if doubleClick {
+      presses.append(MouseClickPress(clickState: 2, delayBeforeMs: mouseClickPairGapMs))
+    }
+  }
+  return presses
+}
+
+/// How long the schedule keeps the helper busy: every hold plus every gap. The caller's
+/// process timeout must cover this, or the helper is killed mid-schedule.
+public func mouseClickScheduleMs(holdMs: Int, clicks: Int, doubleClick: Bool, intervalMs: Int) -> Int {
+  let hold = mouseClickHoldMs(requestedMs: holdMs)
+  return mouseClickPresses(clicks: clicks, doubleClick: doubleClick, intervalMs: intervalMs)
+    .reduce(0) { $0 + $1.delayBeforeMs + hold }
+}

package/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift

@@ -0,0 +1,30 @@
+import XCTest
+
+@testable import AgentDeviceMacOSDeviceHub
+
+final class DeviceHubPoseTests: XCTestCase {
+  func testEveryPoseNamesTheActionBarControlDeviceHubDescribes() {
+    XCTAssertEqual(DeviceHubPose(argument: "closed")?.controlDescription, "Closed")
+    XCTAssertEqual(DeviceHubPose(argument: "book")?.controlDescription, "Book")
+    XCTAssertEqual(DeviceHubPose(argument: "OPEN")?.controlDescription, "Open")
+    XCTAssertNil(DeviceHubPose(argument: "half-open"), "the helper takes Device Hub's names, not the CLI's")
+    XCTAssertEqual(Set(DeviceHubPose.allCases.map(\.controlDescription)), ["Closed", "Book", "Open"])
+  }
+
+  // Device Hub titles the window `<name> – iOS 27.1`; a simulator whose name merely starts
+  // with another's must not claim that window.
+  func testWindowTitleMatchesTheDeviceNameAsAPrefixWord() {
+    XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo – iOS 27.1"))
+    XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo Lab – iOS 27.1"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "bench-golden-v1 – iOS 27.0"))
+    XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: nil))
+  }
+
+  func testSidebarRowIsKeyedByUdid() {
+    XCTAssertEqual(
+      deviceHubDeviceRowIdentifier(udid: "4F879835-4AB3-4046-B033-5AB769209DD4"),
+      "TableRow.Device.4F879835-4AB3-4046-B033-5AB769209DD4"
+    )
+  }
+}

package/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift

@@ -0,0 +1,65 @@
+import ApplicationServices
+import XCTest
+@testable import AgentDeviceMacOSDeviceHub
+
+final class DeviceHubWindowInventoryTests: XCTestCase {
+  func testLaterWindowKeepsItsProcessAndDoesNotPreventEmptyProcessReopen() {
+    var inventory = DeviceHubWindowInventory()
+    let window = AXUIElementCreateApplication(20)
+    inventory.record(processID: 10, read: .available([]))
+    inventory.record(processID: 20, read: .available([window]))
+    XCTAssertEqual(inventory.candidates.map(\.processID), [20])
+    XCTAssertTrue(CFEqual(inventory.candidates[0].window, window))
+    inventory.record(candidate: 0, outcome: .sidebarUnavailable)
+    XCTAssertEqual(inventory.processesToReopen(attempted: []), [10])
+    XCTAssertEqual(inventory.failureReason(), "device-hub-identity-unconfirmed")
+    XCTAssertEqual(inventory.candidateOutcomes, ["20:0:sidebar-unavailable"])
+  }
+
+  func testFailedReadsAreNotReopenedOrMisreportedAsMissingWindows() {
+    var inventory = DeviceHubWindowInventory()
+    inventory.record(processID: 10, read: .failed(-25204))
+    inventory.record(processID: 20, read: .available([]))
+    XCTAssertEqual(inventory.processesToReopen(attempted: []), [20])
+    XCTAssertEqual(inventory.failures, [10: -25204])
+    XCTAssertEqual(inventory.failureReason(), "device-hub-window-read-failed")
+  }
+
+  func testSuccessfulEmptyReadIsMissingAndReopenIsAttemptedOnlyOnce() {
+    var inventory = DeviceHubWindowInventory()
+    inventory.record(processID: 10, read: .available([]))
+    XCTAssertEqual(inventory.failureReason(), "device-hub-window-missing")
+    XCTAssertEqual(inventory.processesToReopen(attempted: []), [10])
+    XCTAssertEqual(inventory.processesToReopen(attempted: [10]), [])
+  }
+
+  func testSharedBudgetExhaustionStopsReopenWithoutClaimingMissingDevice() {
+    var inventory = DeviceHubWindowInventory()
+    inventory.record(processID: 10, read: .available([]))
+    inventory.expireBudget()
+    XCTAssertEqual(inventory.processesToReopen(attempted: []), [])
+    XCTAssertEqual(inventory.failureReason(), "device-hub-discovery-timeout")
+  }
+
+  func testCandidateOutcomesRetainUncertaintyAndOwningProcess() {
+    var inventory = DeviceHubWindowInventory()
+    inventory.record(processID: 10, read: .available([AXUIElementCreateApplication(10)]))
+    inventory.record(processID: 20, read: .available([AXUIElementCreateApplication(20)]))
+    inventory.record(candidate: 0, outcome: .timedOut)
+    inventory.record(candidate: 1, outcome: .unconfirmed)
+    XCTAssertEqual(inventory.candidateOutcomes, ["10:0:timed-out", "20:1:row-unconfirmed"])
+    XCTAssertEqual(inventory.failureReason(), "device-hub-identity-unconfirmed")
+  }
+  func testUnrelatedReopenFailureDoesNotMaskCandidateUncertainty() {
+    var inventory = DeviceHubWindowInventory()
+    inventory.record(processID: 10, read: .available([]))
+    let failures: [Int32: String] = [10: "automation-permission"]
+    XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "automation-permission")
+    inventory.record(processID: 20, read: .available([AXUIElementCreateApplication(20)]))
+    inventory.record(candidate: 0, outcome: .timedOut)
+    XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "device-hub-identity-unconfirmed")
+    inventory.expireBudget()
+    XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "device-hub-discovery-timeout")
+  }
+
+}

package/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift

@@ -0,0 +1,58 @@
+import XCTest
+
+@testable import AgentDeviceMacOSInput
+
+final class MouseClickScheduleTests: XCTestCase {
+  // The original schedule posted move, down and up back to back. AppKit and SwiftUI
+  // never delivered the release, so every surface press opened a tracking session that
+  // no control ever completed: 0 of 15 measured clicks activated their control.
+  func testHoldShorterThanTheDeliverableFloorIsRaised() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 0), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 1), minimumMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 39), minimumMouseClickHoldMs)
+    XCTAssertGreaterThan(defaultMouseClickHoldMs, minimumMouseClickHoldMs)
+  }
+
+  func testNamedLongPressHoldIsKept() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 800), 800)
+  }
+
+  // `--count N` means N independent presses on every platform; a rising click state would
+  // turn `--count 2` into a double-click and `--count 3` into a triple-click.
+  func testRepeatClicksAreIndependentPressesAtClickStateOne() {
+    let presses = mouseClickPresses(clicks: 3, doubleClick: false, intervalMs: 150)
+    XCTAssertEqual(presses.map(\.clickState), [1, 1, 1])
+    XCTAssertEqual(presses.map(\.delayBeforeMs), [0, 150, 150])
+  }
+
+  // Only `--double-tap` raises the click state, and it does so per press, so it composes
+  // with `--count` into that many double-clicks rather than one triple-click.
+  func testDoubleClickRaisesTheStateInsideEachPress() {
+    XCTAssertEqual(
+      mouseClickPresses(clicks: 1, doubleClick: true, intervalMs: 120).map(\.clickState),
+      [1, 2]
+    )
+    let three = mouseClickPresses(clicks: 3, doubleClick: true, intervalMs: 500)
+    XCTAssertEqual(three.map(\.clickState), [1, 2, 1, 2, 1, 2])
+    XCTAssertEqual(
+      three.map(\.delayBeforeMs),
+      [0, mouseClickPairGapMs, 500, mouseClickPairGapMs, 500, mouseClickPairGapMs]
+    )
+    XCTAssertLessThan(mouseClickPairGapMs, 500, "a pair must land inside the system double-click interval")
+  }
+
+  func testZeroClicksStillPostsOnePress() {
+    XCTAssertEqual(mouseClickPresses(clicks: 0, doubleClick: false, intervalMs: 0).count, 1)
+  }
+
+  // The process timeout the caller sets is derived from this: a schedule the timeout does
+  // not cover is killed mid-hold.
+  func testScheduleDurationSumsEveryHoldAndGap() {
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 0, clicks: 1, doubleClick: false, intervalMs: 120), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 10_000, clicks: 4, doubleClick: false, intervalMs: 120), 4 * 10_000 + 3 * 120)
+    XCTAssertEqual(
+      mouseClickScheduleMs(holdMs: 60, clicks: 2, doubleClick: true, intervalMs: 100),
+      4 * 60 + 2 * mouseClickPairGapMs + 100
+    )
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h

@@ -2,3 +2,4 @@
 #import "RunnerAXSnapshotBridge.h"
 #import "RunnerSynthesizedGesture.h"
 #import "RunnerSynthesizedTextEntry.h"
+#import "RunnerXCTestEventBridge.h"

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift

@@ -0,0 +1,175 @@
+import XCTest
+#if canImport(UIKit)
+import UIKit
+#endif
+
+
+
+
+struct CapturedAppScreen {
+  let image: RunnerImage
+  let displayID: UInt
+  let pixelWidth: Int
+  let pixelHeight: Int
+  let pixelsPerPoint: Double
+}
+
+
+
+
+enum RunnerAppScreenCaptureFailure: String, Error {
+  case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED"
+  case unresolvedScreen = "APP_SCREEN_UNRESOLVED"
+  case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE"
+
+  init(_ failure: RunnerApplicationScreenFailure) {
+    switch failure {
+    case .unresolvedWindow:
+      self = .unresolvedWindow
+    default:
+      self = .unresolvedScreen
+    }
+  }
+
+  var message: String {
+    switch self {
+    case .unresolvedWindow:
+      return "Neither the target app nor the system surface resolved a window, so no display could be named."
+    case .unresolvedScreen:
+      return "The resolved app window did not report the display it is shown on."
+    case .unrenderableImage:
+      return "The resolved display handed back an image that could not be drawn upright."
+    }
+  }
+
+  var hint: String {
+    switch self {
+    case .unresolvedWindow:
+      return "The capture follows the display that owns a window, so this is a runtime that is answering no window for any process; runner.log names the window frame and the query answer it refused."
+    case .unresolvedScreen:
+      return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved."
+    case .unrenderableImage:
+      return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question."
+    }
+  }
+}
+
+#if canImport(UIKit) && os(iOS)
+extension RunnerTests {
+
+
+
+
+
+
+
+
+
+
+
+  func captureResolvedAppScreen(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
+      .unresolvedScreen
+    )
+    let capture = {
+      outcome = Self.resolveCapturedAppScreen(app: app)
+    }
+    if Thread.isMainThread {
+      capture()
+    } else {
+      DispatchQueue.main.sync(execute: capture)
+    }
+    return outcome
+  }
+
+
+
+
+
+
+
+
+
+
+
+  func captureObservedScreen(app: XCUIApplication) -> Result<
+    CapturedAppScreen, RunnerAppScreenCaptureFailure
+  > {
+    selectObservedScreenCapture(
+      resolving: { captureResolvedAppScreen(app: app) },
+      fallingBack: { captureResolvedAppScreen(app: springboard) }
+    )
+  }
+
+
+
+  func selectObservedScreenCapture(
+    resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    let outcome = resolving()
+    if case .failure(.unresolvedWindow) = outcome {
+      return fallingBack()
+    }
+    return outcome
+  }
+
+  private static func resolveCapturedAppScreen(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    var screen: AnyObject?
+    var displayID: UInt = 0
+    var resolution: RunnerApplicationScreenFailure = .unresolvedScreen
+    guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution),
+      let captureScreen = screen as? XCUIScreen
+    else {
+      return .failure(RunnerAppScreenCaptureFailure(resolution))
+    }
+    let image = captureScreen.screenshot().image
+    guard let upright = runnerUprightCaptureImage(image) else {
+      return .failure(.unrenderableImage)
+    }
+
+
+
+    guard upright.scale.isFinite, upright.scale > 0 else {
+      return .failure(.unrenderableImage)
+    }
+    guard let cgImage = runnerCGImage(from: upright) else {
+      return .failure(.unrenderableImage)
+    }
+    return .success(
+      CapturedAppScreen(
+        image: upright,
+        displayID: displayID,
+        pixelWidth: cgImage.width,
+        pixelHeight: cgImage.height,
+        pixelsPerPoint: Double(upright.scale)
+      )
+    )
+  }
+
+
+
+
+
+
+
+  private static func runnerUprightCaptureImage(_ image: UIImage) -> UIImage? {
+    if image.imageOrientation == .up {
+      return image
+    }
+    guard image.size.width > 0, image.size.height > 0, image.scale.isFinite, image.scale > 0 else {
+      return nil
+    }
+    let format = UIGraphicsImageRendererFormat()
+    format.scale = image.scale
+    let renderer = UIGraphicsImageRenderer(size: image.size, format: format)
+    return renderer.image { _ in
+      image.draw(in: CGRect(origin: .zero, size: image.size))
+    }
+  }
+}
+#endif

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h

@@ -8,7 +8,11 @@ FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t);
 
 @interface RunnerSynthesizedGesture : NSObject
 
+// `resolvedWindow` is the app window the caller already resolved for the gesture's reference
+// frame. The record's display ID is read from that same window so geometry and routing can never
+// name different windows; pass nil to fall back to the application's first window.
 + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
+                                        resolvedWindow:(id _Nullable)resolvedWindow
                                                     x:(double)x
                                                     y:(double)y
                                                    x2:(double)x2
@@ -16,26 +20,30 @@ FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t);
                                             durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
+                                                resolvedWindow:(id _Nullable)resolvedWindow
                                                              x:(double)x
                                                              y:(double)y
                                                             x2:(double)x2
-                                                             y2:(double)y2
+                                                            y2:(double)y2
                                                      durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
+                                                 resolvedWindow:(id _Nullable)resolvedWindow
                                                                 x:(double)x
                                                                 y:(double)y
                                                                x2:(double)x2
-                                                                y2:(double)y2
+                                                               y2:(double)y2
                                                         durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
                                                    x:(double)x
                                                    y:(double)y;
 
 // Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample
 // starts contact; subsequent samples move it; all pointers lift at their final offset.
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
+                                          resolvedWindow:(id _Nullable)resolvedWindow
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples;
 
 // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight,