agent-device 0.21.13 → 0.21.15

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -180,6 +180,7 @@ extension RunnerTests {
 
 
 
+  @MainActor
   func ensureRunnerHostAppActive(reason: String) {
     NSLog(
       "AGENT_DEVICE_RUNNER_HOST_ACTIVATE state=%d reason=%@",
@@ -191,9 +192,9 @@ extension RunnerTests {
     } else if app.state != .runningForeground {
       app.activate()
     }
-    currentApp = app
-    currentBundleId = nil
-    currentAppProcessIdentifier = nil
+    mainOwned.app = app
+    mainOwned.bundleId = nil
+    mainOwned.processIdentifier = nil
     resetTargetBoundState()
   }
 
@@ -207,16 +208,18 @@ extension RunnerTests {
     lastLoggedGesturePolicyLines.removeAll()
   }
 
+  @MainActor
   func invalidateCachedTarget(reason: String) {
-    if currentApp != nil || currentBundleId != nil {
+    if mainOwned.app != nil || mainOwned.bundleId != nil {
       NSLog("AGENT_DEVICE_RUNNER_TARGET_CACHE_INVALIDATE reason=%@", reason)
     }
-    currentApp = nil
-    currentBundleId = nil
-    currentAppProcessIdentifier = nil
+    mainOwned.app = nil
+    mainOwned.bundleId = nil
+    mainOwned.processIdentifier = nil
     resetTargetBoundState()
   }
 
+  @MainActor
   func resetTargetAfterExternalRelaunch() -> Response {
     invalidateCachedTarget(reason: "external_app_relaunch")
 
@@ -228,22 +231,23 @@ extension RunnerTests {
     return Response(ok: true, data: DataPayload(message: "target reset"))
   }
 
+  @MainActor
   func refreshCachedTargetIfProcessChanged(bundleId: String) {
-    guard currentBundleId == bundleId, currentApp != nil else { return }
+    guard mainOwned.bundleId == bundleId, mainOwned.app != nil else { return }
     let candidate = XCUIApplication(bundleIdentifier: bundleId)
     let observedProcessIdentifier = Self.processIdentifier(of: candidate)
     guard Self.shouldRefreshCachedTarget(
-      cachedProcessIdentifier: currentAppProcessIdentifier,
+      cachedProcessIdentifier: mainOwned.processIdentifier,
       observedProcessIdentifier: observedProcessIdentifier
     ) else { return }
     NSLog(
       "AGENT_DEVICE_RUNNER_TARGET_CACHE_REFRESH bundle=%@ previousPid=%d currentPid=%d",
       bundleId,
-      currentAppProcessIdentifier ?? 0,
+      mainOwned.processIdentifier ?? 0,
       observedProcessIdentifier ?? 0
     )
-    currentApp = candidate
-    currentAppProcessIdentifier = observedProcessIdentifier
+    mainOwned.app = candidate
+    mainOwned.processIdentifier = observedProcessIdentifier
     resetTargetBoundState()
     clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
     clearPrivateAXAcceptedDepth(reason: "target_process_changed")
@@ -280,11 +284,12 @@ extension RunnerTests {
     return false
   }
 
+  @MainActor
   func canUseFastForegroundAppGuard(
     activeApp: XCUIApplication,
     requestedBundleId: String?
   ) -> Bool {
-    guard let requestedBundleId, currentBundleId == requestedBundleId, currentApp != nil else {
+    guard let requestedBundleId, mainOwned.bundleId == requestedBundleId, mainOwned.app != nil else {
       return false
     }
     guard activeApp.state == .runningForeground else { return false }
@@ -292,6 +297,7 @@ extension RunnerTests {
     return true
   }
 
+  @MainActor
   func writeFastAppGuardMarker(bundleId: String, state: XCUIApplication.State) {
 
 
@@ -318,9 +324,10 @@ extension RunnerTests {
 
 
 
-  func notRunningReadResponse(command: Command, bundleId: String) -> Response? {
+
+  func notRunningRefusal(command: Command, bundleId: String) -> Response? {
 #if os(iOS)
-    guard isReadOnlyCommand(command),
+    guard command.traits.launchPolicy == .existingApp,
       XCUIApplication(bundleIdentifier: bundleId).state == .notRunning
     else { return nil }
     NSLog(
@@ -341,6 +348,7 @@ extension RunnerTests {
 #endif
   }
 
+  @MainActor
   func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
     let target = XCUIApplication(bundleIdentifier: bundleId)
     let initialState = target.state
@@ -374,9 +382,9 @@ extension RunnerTests {
         otherActiveApplicationPid.map(String.init) ?? "-"
       )
     }
-    currentApp = target
-    currentBundleId = bundleId
-    currentAppProcessIdentifier = Self.processIdentifier(of: target)
+    mainOwned.app = target
+    mainOwned.bundleId = bundleId
+    mainOwned.processIdentifier = Self.processIdentifier(of: target)
     resetTargetBoundState()
     beginFirstInteractionStabilization()
     return target
@@ -387,10 +395,11 @@ extension RunnerTests {
 
 
 
+  @MainActor
   func withBoundedInteractionIdleTimeoutIfSupported(
     _ target: XCUIApplication,
     waits: RunnerInteractionIdleWaits,
-    operation: () -> Void
+    operation: @MainActor () -> Void
   ) {
     let setter = NSSelectorFromString("setWaitForIdleTimeout:")
     let supportsWaitForIdleTimeout = target.responds(to: setter)
@@ -409,10 +418,11 @@ extension RunnerTests {
   }
 
 
+  @MainActor
   private func performWithQuiescenceSkippedIfSupported(
     _ target: XCUIApplication,
     waits: RunnerInteractionIdleWaits,
-    operation: () -> Void
+    operation: @MainActor () -> Void
   ) {
     let selector = NSSelectorFromString("_performWithInteractionOptions:block:")
     guard target.responds(to: selector) else {
@@ -440,7 +450,7 @@ extension RunnerTests {
       options = skipPreEventQuiescence
     }
     withoutActuallyEscaping(operation) { escapableOperation in
-      let block: @convention(block) () -> Void = escapableOperation
+      let block: @convention(block) () -> Void = { _ = runOnMainActor(escapableOperation) }
       performWithOptions(
         target,
         selector,
@@ -450,49 +460,26 @@ extension RunnerTests {
     }
   }
 
-  func shouldRetryCommand(_ command: Command) -> Bool {
-    isReadOnlyCommand(command)
-  }
+
 
   func shouldRetryException(_ command: Command, message: String) -> Bool {
-    guard shouldRetryCommand(command) else { return false }
+    guard command.traits.retryOnSessionLoss else { return false }
 
     return message.lowercased().contains("kaxerrorservernotfound")
   }
 
-
-
-  func isReadOnlyCommand(_ command: Command) -> Bool {
-    switch command.command.traits.readOnly {
-    case .always:
-      return true
-    case .never:
-      return false
-    case .conditional:
-
-      return (command.action ?? "get").lowercased() == "get"
-    }
-  }
-
   func shouldRetryResponse(_ response: Response) -> Bool {
     guard response.ok == false else { return false }
     return response.error?.retryableFailure != nil
   }
 
-  func isInteractionCommand(_ command: CommandType) -> Bool {
-    return command.traits.isInteraction
-  }
-
-  func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
-    return command.traits.isLifecycle
-  }
-
 
 
+  @MainActor
   func applyInteractionStabilizationIfNeeded() {
-    if needsPostSnapshotInteractionDelay {
+    if mainOwned.needsPostSnapshotInteractionDelay {
       sleepFor(postSnapshotInteractionDelay)
-      needsPostSnapshotInteractionDelay = false
+      mainOwned.needsPostSnapshotInteractionDelay = false
     }
     if let readyUptime = firstInteractionReadyUptime {
       sleepFor(readyUptime - ProcessInfo.processInfo.systemUptime)

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift

@@ -57,12 +57,12 @@ extension RunnerTests {
   func runMainThreadWork<T>(
     _ operation: String,
     timeout: TimeInterval,
-    timeoutError: @escaping () -> Error,
-    onAbandoned: (() -> Void)? = nil,
-    _ work: @escaping () throws -> T
+    timeoutError: @escaping @Sendable () -> Error,
+    onAbandoned: (@Sendable () -> Void)? = nil,
+    _ work: @escaping @MainActor () throws -> T
   ) throws -> T {
     if Thread.isMainThread {
-      return try work()
+      return try runOnMainActor(work).get()
     }
     mainThreadWorkLock.lock()
     let state = enqueueMainThreadWorkLocked(operation, work)
@@ -85,8 +85,8 @@ extension RunnerTests {
   func runMainThreadWorkIfIdle<T>(
     _ operation: String,
     timeout: TimeInterval,
-    timeoutError: @escaping () -> Error,
-    _ work: @escaping () throws -> T
+    timeoutError: @escaping @Sendable () -> Error,
+    _ work: @escaping @MainActor () throws -> T
   ) throws -> T? {
     if Thread.isMainThread {
       return nil
@@ -109,16 +109,12 @@ extension RunnerTests {
 
   private func enqueueMainThreadWorkLocked<T>(
     _ operation: String,
-    _ work: @escaping () throws -> T
+    _ work: @escaping @MainActor () throws -> T
   ) -> MainThreadWorkState<T> {
     let state = MainThreadWorkState<T>()
     mainThreadWorkInFlightCount += 1
     DispatchQueue.main.async {
-      do {
-        state.result = .success(try work())
-      } catch {
-        state.result = .failure(error)
-      }
+      state.result = runOnMainActor(work)
       self.mainThreadWorkLock.lock()
       self.mainThreadWorkInFlightCount -= 1
       let abandoned = state.abandoned
@@ -147,8 +143,8 @@ extension RunnerTests {
     _ state: MainThreadWorkState<T>,
     operation: String,
     timeout: TimeInterval,
-    timeoutError: @escaping () -> Error,
-    onAbandoned: (() -> Void)?
+    timeoutError: @escaping @Sendable () -> Error,
+    onAbandoned: (@Sendable () -> Void)?
   ) throws -> T {
     let waitResult = state.completed.wait(timeout: .now() + timeout)
     if waitResult == .timedOut {
@@ -191,7 +187,7 @@ extension RunnerTests {
     }
   }
 
-  func mainThreadExecutionTimeoutError() -> Error {
+  static func mainThreadExecutionTimeoutError() -> Error {
     NSError(
       domain: RunnerErrorDomain.general,
       code: RunnerErrorCode.mainThreadExecutionTimedOut,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -2,7 +2,7 @@ import AgentDeviceSnapshotPresentation
 
 
 
-enum CommandType: String, Codable {
+enum CommandType: String, Codable, CaseIterable {
   case tap
   case mouseClick
   case longPress
@@ -33,6 +33,7 @@ enum CommandType: String, Codable {
   case recordStop
   case status
   case uptime
+  case appState
   case activate
   case terminate
   case targetReset
@@ -42,6 +43,46 @@ enum CommandType: String, Codable {
 
 
 
+enum CommandLaunchPolicy: Equatable {
+
+
+
+
+
+
+
+
+
+  case noApp
+
+
+
+
+  case presentedSurface
+
+
+
+
+
+
+  case existingApp
+
+  case mayLaunch
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -50,72 +91,94 @@ struct CommandTraits {
 
   let isInteraction: Bool
 
+  let retryOnSessionLoss: Bool
 
-  let readOnly: ReadOnly
 
-  let isLifecycle: Bool
+  let launchPolicy: CommandLaunchPolicy
 
-  enum ReadOnly {
-    case always
-    case never
 
 
 
-    case conditional
+  let convertsRecordedFailure: Bool
+
+  init(
+    isInteraction: Bool = false,
+    retryOnSessionLoss: Bool = false,
+    launchPolicy: CommandLaunchPolicy,
+    convertsRecordedFailure: Bool = false
+  ) {
+    self.isInteraction = isInteraction
+    self.retryOnSessionLoss = retryOnSessionLoss
+    self.launchPolicy = launchPolicy
+    self.convertsRecordedFailure = convertsRecordedFailure
   }
 }
 
-extension CommandType {
+fileprivate extension CommandTraits {
 
 
+  static let interaction = CommandTraits(
+    isInteraction: true,
+    launchPolicy: .mayLaunch,
+    convertsRecordedFailure: true
+  )
 
-  var traits: CommandTraits {
-    switch self {
 
 
 
 
+  static let appMutation = CommandTraits(launchPolicy: .mayLaunch, convertsRecordedFailure: true)
 
-    case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll,
-         .backInApp, .backSystem, .rotate, .appSwitcher,
-         .keyboardDismiss, .keyboardReturn, .sequence, .gesture:
-      return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
 
 
-    case .findText, .readText, .snapshot, .gestureViewport:
-      return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
+  static let appRead = CommandTraits(retryOnSessionLoss: true, launchPolicy: .existingApp)
 
 
-    case .screenshot:
-      return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
 
 
-    case .alert:
-      return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
 
+  static let selectorResolution = CommandTraits(
+    launchPolicy: .existingApp,
+    convertsRecordedFailure: true
+  )
 
-    case .recordStop, .uptime, .terminate, .targetReset, .shutdown:
-      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
 
 
+  static let runnerCaptureRead = CommandTraits(retryOnSessionLoss: true, launchPolicy: .noApp)
 
 
+  static let runnerLifecycle = CommandTraits(launchPolicy: .noApp)
 
 
-    case .actionButton:
-      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
 
-    case .status:
-      return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
 
+  static let presentedSurfaceMutation = CommandTraits(
+    launchPolicy: .presentedSurface,
+    convertsRecordedFailure: true
+  )
 
 
+  static let presentedSurfaceQuery = CommandTraits(
+    retryOnSessionLoss: true,
+    launchPolicy: .presentedSurface
+  )
+}
 
+extension CommandTraits {
 
 
-    case .mouseClick, .querySelector, .home, .recordStart, .activate:
-      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
-    }
+
+  static let textEntryWitnessOwners: Set<CommandType> = [.tap, .type]
+}
+
+extension Command {
+
+
+
+
+
+  var invalidatesRememberedTextEntryTap: Bool {
+    traits.convertsRecordedFailure && !CommandTraits.textEntryWitnessOwners.contains(command)
   }
 }
 
@@ -159,6 +222,49 @@ struct Command: Codable {
   let steps: [SequenceStep]?
 }
 
+extension Command {
+
+
+
+  var traits: CommandTraits {
+    switch command {
+
+
+
+
+    case .tap, .type, .longPress, .drag, .remotePress, .swipe, .scroll, .desktopScroll,
+         .backInApp, .backSystem, .rotate, .appSwitcher,
+         .keyboardDismiss, .keyboardReturn, .sequence, .gesture:
+      return .interaction
+
+    case .findText, .readText, .snapshot, .gestureViewport:
+      return .appRead
+
+
+
+    case .screenshot, .status, .appState:
+      return .runnerCaptureRead
+
+    case .alert:
+      return (action ?? "get").lowercased() == "get"
+        ? .presentedSurfaceQuery
+        : .presentedSurfaceMutation
+
+    case .recordStop, .uptime, .terminate, .targetReset, .shutdown:
+      return .runnerLifecycle
+
+    case .actionButton:
+      return .presentedSurfaceMutation
+
+    case .querySelector:
+      return .selectorResolution
+
+    case .mouseClick, .home, .recordStart, .activate:
+      return .appMutation
+    }
+  }
+}
+
 enum ScrollReleaseBehavior: String, Codable {
   case controlled
   case inertial
@@ -306,6 +412,7 @@ struct DataPayload: Codable {
   var referenceWidth: Double?
   var referenceHeight: Double?
   var currentUptimeMs: Double?
+  var applicationState: String?
   var commandId: String?
   var lifecycleState: String?
   var lifecycleCommand: String?

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -1,3 +1,4 @@
+import AgentDeviceSnapshotPresentation
 import XCTest
 
 extension RunnerTests {
@@ -43,6 +44,7 @@ extension RunnerTests {
     }
   }
 
+  @MainActor
   func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome {
 #if os(macOS)
     if let back = macOSNavigationBackElement(app: app) {
@@ -59,8 +61,8 @@ extension RunnerTests {
       back.tap()
       return .performed
     }
-    if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) {
-      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "")
+    if isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) {
+      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "")
     } else if let back = topNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
       return .performed
@@ -120,16 +122,9 @@ extension RunnerTests {
     return navigationBackKeywords.firstIndex { text.contains($0) }
   }
 
-
-  static func isUsableNavigationFrame(_ frame: CGRect) -> Bool {
-    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
-      return false
-    }
-    return !frame.isInfinite
-  }
-
   static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
-    guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else {
+    guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window)
+    else {
       return false
     }
 
@@ -138,7 +133,7 @@ extension RunnerTests {
   }
 
   static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
-    guard isUsableNavigationFrame(frame) else {
+    guard SnapshotGeometry.isPositiveFinite(frame) else {
       return nil
     }
 
@@ -148,6 +143,7 @@ extension RunnerTests {
     return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
   }
 
+  @MainActor
   private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> InAppBackOutcome {
 #if os(iOS)
     let frame = onScreenWindowFrame(app: app)
@@ -159,17 +155,20 @@ extension RunnerTests {
       app: app,
       policy: synthesizedGesturePolicy(.coordinateTap)
     )?.withReferenceFrame(frame)
-    let synthesized = performGesture(app, idleTimeout: false) {
+    switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: {
       synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
-    }
-    if case .performed = synthesized.outcome {
-      return verifyNavigationFallbackOutcome(app: app, before: before)
-    }
-    let fallback = performGesture(app) {
-      tapAt(app: app, x: point.x, y: point.y)
-    }
-    if case .performed = fallback.outcome {
+    }) {
+    case .performed:
       return verifyNavigationFallbackOutcome(app: app, before: before)
+    case .refused:
+      return .unavailable
+    case .xctestFallback:
+      let fallback = performGesture(app) {
+        tapAt(app: app, x: point.x, y: point.y)
+      }
+      if case .performed = fallback.outcome {
+        return verifyNavigationFallbackOutcome(app: app, before: before)
+      }
     }
 #endif
     return .unavailable

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -8,6 +8,7 @@ struct PrivateAXFields {
   let label: String
   let identifier: String
   let value: String
+  let placeholder: String
   let rawType: Int
   let elementType: XCUIElement.ElementType?
   let enabled: Bool
@@ -52,6 +53,7 @@ extension RunnerTests {
       label: privateAXPresentationString(raw["label"]),
       identifier: privateAXPresentationString(raw["identifier"]),
       value: privateAXPresentationString(raw["value"]),
+      placeholder: privateAXPresentationString(raw["placeholder"]),
       rawType: rawType,
       elementType: privateAXElementType(rawElementType: rawType),
       enabled: privateAXPresentationBool(raw["enabled"]) ?? true,
@@ -70,6 +72,7 @@ extension RunnerTests {
       label: fields.label.isEmpty ? nil : fields.label,
       identifier: fields.identifier.isEmpty ? nil : fields.identifier,
       value: fields.value.isEmpty ? nil : fields.value,
+      placeholder: fields.placeholder.isEmpty ? nil : fields.placeholder,
       rect: SnapshotRect(fields.rect), enabled: fields.enabled,
       focused: fields.focused, selected: fields.selected,
       hittable: false,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -36,8 +36,8 @@ extension RunnerTests {
 
 
     func start(
-      bootstrap: @escaping () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
-      frame: @escaping () -> RunnerImage?
+      bootstrap: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+      frame: @escaping @Sendable () -> RunnerImage?
     ) throws {
       let url = URL(fileURLWithPath: outputPath)
       let directory = url.deletingLastPathComponent()
@@ -132,7 +132,7 @@ extension RunnerTests {
 
       let timer = DispatchSource.makeTimerSource(queue: queue)
       timer.schedule(deadline: .now() + frameInterval, repeating: frameInterval)
-      timer.setEventHandler { [weak self] in
+      timer.setEventHandler { @Sendable [weak self] in
         guard let self else { return }
         if self.shouldStop() { return }
         guard let image = frame() else { return }
@@ -299,16 +299,17 @@ extension RunnerTests {
 
 
 
+  @MainActor
   func startRecording(
     _ recorder: ScreenRecorder,
-    capture: @escaping () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
+    capture: @escaping @MainActor () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
   ) throws {
     try recorder.start(bootstrap: capture) { [weak self] in
       guard let self else { return nil }
       return try? self.runMainThreadWorkIfIdle(
         "recording_frame",
         timeout: self.recordingFrameCaptureTimeout,
-        timeoutError: self.mainThreadExecutionTimeoutError
+        timeoutError: Self.mainThreadExecutionTimeoutError
       ) {
         try capture().get().image
       }