agent-device 0.19.0 → 0.19.1

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -36,34 +36,21 @@ extension RunnerTests {
     let y2: Double
   }
 
+  struct SynthesizedDragPlan {
+    let points: DragPoints
+    let context: SynthesizedCoordinateContext
+
+    var referenceFrame: CGRect {
+      context.referenceFrame
+    }
+  }
+
   struct SelectorElementMatch {
     let element: XCUIElement?
     let isAmbiguous: Bool
     let usedNonHittableFallback: Bool
   }
 
-  // MARK: - Navigation Gestures
-
-  func tapInAppBackControl(app: XCUIApplication) -> Bool {
-#if os(macOS)
-    if let back = macOSNavigationBackElement(app: app) {
-      tapElementCenter(app: app, element: back)
-      return true
-    }
-    return false
-#elseif os(tvOS)
-    _ = pressTvRemote(.menu)
-    return true
-#else
-    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
-    if let back = buttons.first(where: { $0.isHittable }) {
-      back.tap()
-      return true
-    }
-    return false
-#endif
-  }
-
   func performBackGesture(app: XCUIApplication) {
     if pressTvRemote(.menu) {
       return
@@ -787,11 +774,24 @@ extension RunnerTests {
     y: Double,
     x2: Double,
     y2: Double,
-    durationMs: Double
+    durationMs: Double,
+    context: SynthesizedCoordinateContext? = nil
   ) -> RunnerInteractionOutcome {
 #if os(iOS)
+    guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
+      return .unsupported(
+        message: "synthesized coordinate drag requires finite coordinates",
+        hint: "Retry with finite x, y, x2, and y2 values."
+      )
+    }
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    let frame = app.frame
+    guard let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.synthesizedDrag)) else {
+      return .unsupported(
+        message: "synthesized coordinate drag could not resolve a finite screen frame",
+        hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+      )
+    }
+    let frame = context.referenceFrame
     let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
     let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
     if let message = RunnerSynthesizedGesture.synthesizeSwipe(
@@ -804,7 +804,7 @@ extension RunnerTests {
     ) {
       return .unsupported(
         message: message,
-        hint: "Falling back to XCTest coordinate drag may be slower; update Xcode if this persists."
+        hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists."
       )
     }
     return .performed
@@ -821,10 +821,28 @@ extension RunnerTests {
 #endif
   }
 
-  func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+  func synthesizedTapAt(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    context: SynthesizedCoordinateContext? = nil
+  ) -> RunnerInteractionOutcome {
 #if os(iOS)
+    guard x.isFinite, y.isFinite else {
+      return .unsupported(
+        message: "synthesized coordinate tap requires finite coordinates",
+        hint: "Retry with finite x and y values."
+      )
+    }
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation)
+    guard let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.coordinateTap)) else {
+      return .unsupported(
+        message: "synthesized coordinate tap could not resolve a finite screen frame",
+        hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+      )
+    }
+    let frame = context.referenceFrame
+    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
     if let message = RunnerSynthesizedGesture.synthesizeTap(
       withApplication: app,
       x: Double(point.x),
@@ -994,6 +1012,114 @@ extension RunnerTests {
 #endif
   }
 
+  func axFreeSynthesizedDragPlan(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    context: SynthesizedCoordinateContext? = nil
+  ) -> SynthesizedDragPlan? {
+#if os(iOS)
+    let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.synthesizedDrag))
+    guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
+      let context
+    else {
+      return nil
+    }
+    let points = keyboardAvoidingSynthesizedDragPoints(
+      app: app,
+      x: x,
+      y: y,
+      x2: x2,
+      y2: y2,
+      context: context
+    )
+    return SynthesizedDragPlan(
+      points: points,
+      context: context
+    )
+#else
+    return nil
+#endif
+  }
+
+  func axFreeDragVisualizationFrame(
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    referenceFrame: CGRect
+  ) -> DragVisualizationFrame {
+    return DragVisualizationFrame(
+      x: x,
+      y: y,
+      x2: x2,
+      y2: y2,
+      referenceWidth: Double(referenceFrame.width),
+      referenceHeight: Double(referenceFrame.height)
+    )
+  }
+
+  func synthesizedCoordinateContext(policy: SynthesizedGesturePolicy) -> SynthesizedCoordinateContext? {
+#if os(iOS)
+    let health = runnerAccessibilityHealth
+    guard let referenceFrame = synthesizedScreenshotReferenceFrame(
+      screenshotSize: { XCUIScreen.main.screenshot().image.size }
+    ) else {
+      return nil
+    }
+    return SynthesizedCoordinateContext(
+      referenceFrame: referenceFrame,
+      keyboardPolicy: policy.keyboardPolicy,
+      fallbackPolicy: policy.fallbackPolicy,
+      accessibilityHealth: health
+    )
+#else
+    return nil
+#endif
+  }
+
+  func synthesizedScreenshotReferenceFrame(screenshotSize: () -> CGSize) -> CGRect? {
+    let screenshotSize = screenshotSize()
+    guard screenshotSize.width.isFinite, screenshotSize.height.isFinite,
+      screenshotSize.width > 0,
+      screenshotSize.height > 0
+    else {
+      return nil
+    }
+    return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height)
+  }
+
+  func synthesizedFrameAvoidingKeyboardWhenAllowed(
+    app: XCUIApplication,
+    context: SynthesizedCoordinateContext
+  ) -> CGRect {
+#if os(iOS)
+    guard context.allowsKeyboardProbe else { return context.referenceFrame }
+    return frameAvoidingKeyboard(app: app, frame: context.referenceFrame)
+#else
+    return context.referenceFrame
+#endif
+  }
+
+  func keyboardAvoidingSynthesizedDragPoints(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    context: SynthesizedCoordinateContext
+  ) -> DragPoints {
+#if os(iOS)
+    guard context.allowsKeyboardProbe else {
+      return DragPoints(x: x, y: y, x2: x2, y2: y2)
+    }
+    return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
+#else
+    return DragPoints(x: x, y: y, x2: x2, y2: y2)
+#endif
+  }
 
   func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {
@@ -1227,25 +1353,4 @@ extension RunnerTests {
   }
 #endif
 
-  private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
-    let frame = element.frame
-    if !frame.isEmpty {
-      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
-      return
-    }
-#if !os(tvOS)
-    element.tap()
-#endif
-  }
-
-  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
-    let predicate = NSPredicate(
-      format: "identifier == %@ OR label == %@",
-      "go back",
-      "Back"
-    )
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
-  }
-
 }

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -0,0 +1,172 @@
+import XCTest
+
+extension RunnerTests {
+  static let navigationBackKeywords = ["back", "close", "cancel"]
+  static let navigationFallbackVerificationDelay: TimeInterval = 0.25
+
+  func tapInAppBackControl(app: XCUIApplication) -> Bool {
+#if os(macOS)
+    if let back = macOSNavigationBackElement(app: app) {
+      tapElementCenter(app: app, element: back)
+      return true
+    }
+    return false
+#elseif os(tvOS)
+    _ = pressTvRemote(.menu)
+    return true
+#else
+    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
+    if let back = buttons.first(where: { $0.isHittable }) {
+      back.tap()
+      return true
+    }
+    if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) {
+      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "")
+    } else if let back = topNavigationBackElement(app: app) {
+      tapElementCenter(app: app, element: back)
+      return true
+    }
+    return tapTopLeadingNavigationFallback(app: app)
+#endif
+  }
+
+  private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
+    let frame = element.frame
+    if !frame.isEmpty {
+      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      return
+    }
+#if !os(tvOS)
+    element.tap()
+#endif
+  }
+
+  private func topNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
+#if os(iOS)
+    let frame = onScreenWindowFrame(app: app)
+    let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap {
+      element -> (XCUIElement, Int)? in
+      guard element.exists, element.isHittable else { return nil }
+      guard Self.isTopNavigationControlFrame(element.frame, in: frame) else { return nil }
+      guard let rank = Self.navigationBackControlRank(
+        label: element.label,
+        identifier: element.identifier
+      ) else {
+        return nil
+      }
+      return (element, rank)
+    }
+    return candidates.sorted { lhs, rhs in
+      if lhs.1 != rhs.1 { return lhs.1 < rhs.1 }
+      let leftFrame = lhs.0.frame
+      let rightFrame = rhs.0.frame
+      if leftFrame.minY != rightFrame.minY { return leftFrame.minY < rightFrame.minY }
+      return leftFrame.minX < rightFrame.minX
+    }.first?.0
+#else
+    return nil
+#endif
+  }
+
+  static func navigationBackPredicate() -> NSPredicate {
+    let clauses = navigationBackKeywords.flatMap { _ in
+      ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"]
+    }.joined(separator: " OR ")
+    let arguments = navigationBackKeywords.flatMap { [$0, $0] }
+    return NSPredicate(format: clauses, argumentArray: arguments)
+  }
+
+  static func navigationBackControlRank(label: String, identifier: String) -> Int? {
+    let text = "\(label) \(identifier)".lowercased()
+    return navigationBackKeywords.firstIndex { text.contains($0) }
+  }
+
+  static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
+    guard
+      candidate.width.isFinite,
+      candidate.height.isFinite,
+      window.width.isFinite,
+      window.height.isFinite,
+      candidate.width > 0,
+      candidate.height > 0,
+      window.width > 0,
+      window.height > 0
+    else {
+      return false
+    }
+    // Accept the compact navigation/search header band without matching deep content controls.
+    let maxY = window.minY + min(max(window.height * 0.22, 96), 180)
+    return candidate.midY >= window.minY && candidate.midY <= maxY
+  }
+
+  static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
+    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
+      return nil
+    }
+    // Aim at the standard leading navigation slot, bounded for compact and tablet widths.
+    let xOffset = min(max(frame.width * 0.08, 28), 44)
+    // Sit below the status/dynamic-island region and inside common custom RN search headers.
+    let yOffset = min(max(frame.height * 0.155, 56), 132)
+    return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
+  }
+
+  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> Bool {
+#if os(iOS)
+    let frame = onScreenWindowFrame(app: app)
+    guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else {
+      return false
+    }
+    let before = captureNavigationFallbackVisualState()
+    let context = synthesizedCoordinateContext(
+      policy: synthesizedGesturePolicy(.coordinateTap)
+    )?.withReferenceFrame(frame)
+    let synthesized = performGesture(app, idleTimeout: false) {
+      synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
+    }
+    if case .performed = synthesized.outcome {
+      return didNavigationFallbackChangeVisualState(before: before)
+    }
+    let fallback = performGesture(app) {
+      tapAt(app: app, x: point.x, y: point.y)
+    }
+    if case .performed = fallback.outcome {
+      return didNavigationFallbackChangeVisualState(before: before)
+    }
+#endif
+    return false
+  }
+
+  private func captureNavigationFallbackVisualState() -> Data? {
+#if os(iOS)
+    runnerPngData(for: XCUIScreen.main.screenshot().image)
+#else
+    return nil
+#endif
+  }
+
+  private func didNavigationFallbackChangeVisualState(before: Data?) -> Bool {
+    sleepFor(Self.navigationFallbackVerificationDelay)
+    let after = captureNavigationFallbackVisualState()
+    let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after)
+    if !changed {
+      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_FALLBACK_NO_STATE_CHANGE")
+    }
+    return changed
+  }
+
+  static func didNavigationFallbackChangeVisualState(before: Data?, after: Data?) -> Bool {
+    guard let before, let after else { return false }
+    return before != after
+  }
+
+  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
+    let predicate = NSPredicate(
+      format: "identifier == %@ OR label == %@",
+      "go back",
+      "Back"
+    )
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
+  }
+
+}

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift

@@ -40,8 +40,10 @@ extension RunnerTests {
       ? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!)
       : nil
 
+    let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps)
+
     let execution = assembleSequenceExecution(steps: steps) { _, step in
-      performSequenceStep(step, activeApp: activeApp)
+      performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext)
     }
     return sequenceResponse(execution: execution, touchFrame: firstFrame)
   }
@@ -141,7 +143,8 @@ extension RunnerTests {
 
   private func performSequenceStep(
     _ step: SequenceStep,
-    activeApp: XCUIApplication
+    activeApp: XCUIApplication,
+    synthesizedContext: SynthesizedCoordinateContext? = nil
   ) -> SequenceStepOutcome {
     let x = step.x ?? 0
     let y = step.y ?? 0
@@ -149,10 +152,26 @@ extension RunnerTests {
     // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not
     // change the touch mechanism for these inputs.
     if step.kind == "tap", step.synthesized == true {
+      let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
+#if os(iOS)
+      guard let synthesizedContext else {
+        let nowMs = ProcessInfo.processInfo.systemUptime * 1000
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: .unsupported(
+            message: "synthesized coordinate tap could not resolve a finite coordinate frame",
+            hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+          ),
+          gestureStartUptimeMs: nowMs,
+          gestureEndUptimeMs: nowMs
+        )
+      }
+#endif
       let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
-        synthesizedTapAt(app: activeApp, x: x, y: y)
+        synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext)
       }
       if case .performed = outcome {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
         if let pauseMs = step.pauseMs, pauseMs > 0 {
           sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
         }
@@ -162,11 +181,50 @@ extension RunnerTests {
           gestureEndUptimeMs: timing.gestureEndUptimeMs
         )
       }
+#if os(iOS)
+      guard synthesizedContext.allowsXCTestCoordinateFallback else {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: outcome,
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      }
+      logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true)
+#endif
       // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
     }
     if step.kind == "drag", step.synthesized == true {
-      let dragPoints = keyboardAvoidingDragPoints(
+      let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
+      let dragPoints: DragPoints
+      let dragContext: SynthesizedCoordinateContext?
+#if os(iOS)
+      guard let dragPlan = axFreeSynthesizedDragPlan(
+        app: activeApp,
+        x: x,
+        y: y,
+        x2: step.x2 ?? x,
+        y2: step.y2 ?? y,
+        context: synthesizedContext
+      ) else {
+        let nowMs = ProcessInfo.processInfo.systemUptime * 1000
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: .unsupported(
+            message: "synthesized coordinate drag could not resolve a finite coordinate frame",
+            hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+          ),
+          gestureStartUptimeMs: nowMs,
+          gestureEndUptimeMs: nowMs
+        )
+      }
+      dragPoints = dragPlan.points
+      dragContext = dragPlan.context
+#else
+      dragPoints = keyboardAvoidingDragPoints(
         app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
+      dragContext = nil
+#endif
       let durationMs = min(max(step.durationMs ?? 250, 16), 10000)
       let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
         synthesizedDragAt(
@@ -175,10 +233,12 @@ extension RunnerTests {
           y: dragPoints.y,
           x2: dragPoints.x2,
           y2: dragPoints.y2,
-          durationMs: durationMs
+          durationMs: durationMs,
+          context: dragContext
         )
       }
       if case .performed = outcome {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
         if let pauseMs = step.pauseMs, pauseMs > 0 {
           sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
         }
@@ -188,6 +248,17 @@ extension RunnerTests {
           gestureEndUptimeMs: timing.gestureEndUptimeMs
         )
       }
+#if os(iOS)
+      guard dragContext?.allowsXCTestCoordinateFallback == true else {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: outcome,
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      }
+#endif
+      logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: true)
       let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250)
       let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) {
         dragAt(