agent-device 0.21.8 → 0.21.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.8.apk → agent-device-android-ime-helper-0.21.9.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.9.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.8.manifest.json → agent-device-android-ime-helper-0.21.9.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.8.apk → agent-device-android-snapshot-helper-0.21.9.apk} +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.9.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.8.manifest.json → agent-device-android-snapshot-helper-0.21.9.manifest.json} +6 -6
  7. package/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift +65 -0
  8. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift +45 -140
  9. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubSidebar.swift +57 -0
  10. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift +108 -0
  11. package/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift +65 -0
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +175 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +10 -2
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +42 -15
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +8 -0
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +45 -22
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +39 -734
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +20 -4
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +63 -1
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +13 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +25 -10
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +5 -562
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +566 -0
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +5 -1
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +594 -0
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +35 -1
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +160 -0
  29. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +3 -0
  30. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift +203 -0
  31. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +5 -0
  32. package/dist/src/absence-observation-errors.js +1 -1
  33. package/dist/src/adapter.js +1 -1
  34. package/dist/src/agent-browser-lifecycle.js +1 -1
  35. package/dist/src/agent-browser-process-record.js +1 -1
  36. package/dist/src/agent-device-client.js +1 -1
  37. package/dist/src/alert-runtime.js +1 -1
  38. package/dist/src/android-adb.js +1 -1
  39. package/dist/src/android-input-ownership.js +1 -1
  40. package/dist/src/android.js +1 -1
  41. package/dist/src/app-launch.js +1 -1
  42. package/dist/src/app-log-runtime2.js +3 -3
  43. package/dist/src/app-resolution.js +2 -2
  44. package/dist/src/apple-runner-platform.js +1 -1
  45. package/dist/src/application-lifecycle-interaction.js +1 -1
  46. package/dist/src/artifact-paths.js +1 -0
  47. package/dist/src/back-mode.js +1 -1
  48. package/dist/src/cli-config.js +1 -1
  49. package/dist/src/cli-help.js +8 -7
  50. package/dist/src/cli.js +4 -4
  51. package/dist/src/client-types.d.ts +23 -5
  52. package/dist/src/command-tools.js +3 -3
  53. package/dist/src/connection-runtime.js +1 -1
  54. package/dist/src/device-claim-inspection.js +1 -1
  55. package/dist/src/device-claim-rule.js +2 -2
  56. package/dist/src/device-input-state.d.ts +1 -1
  57. package/dist/src/device-selection-resolver.js +1 -1
  58. package/dist/src/device.js +1 -1
  59. package/dist/src/device2.js +1 -1
  60. package/dist/src/dispatch-resolve.js +1 -1
  61. package/dist/src/doctor-facade.js +1 -1
  62. package/dist/src/durable-resource.js +1 -1
  63. package/dist/src/find.js +1 -1
  64. package/dist/src/finders.js +1 -1
  65. package/dist/src/fold-runtime.js +1 -0
  66. package/dist/src/gesture-admission.js +1 -1
  67. package/dist/src/interaction-positionals.js +1 -1
  68. package/dist/src/interaction-ref-policy.js +1 -1
  69. package/dist/src/interaction.js +1 -1
  70. package/dist/src/interaction2.js +1 -1
  71. package/dist/src/interactor-operation-catalog.js +1 -1
  72. package/dist/src/interactor.js +4 -4
  73. package/dist/src/interactors.js +1 -1
  74. package/dist/src/internal/bin.js +1 -1
  75. package/dist/src/internal/daemon.js +2 -2
  76. package/dist/src/inventory2.js +1 -1
  77. package/dist/src/ios-snapshot-acquisition.js +1 -1
  78. package/dist/src/ios-snapshot-runtime.js +1 -1
  79. package/dist/src/lease.js +1 -1
  80. package/dist/src/local-interactor-operation-set.js +1 -1
  81. package/dist/src/location-coordinates.js +1 -1
  82. package/dist/src/managed-device-scope.js +1 -1
  83. package/dist/src/mechanics.js +17 -17
  84. package/dist/src/mobile-snapshot-semantics.js +1 -1
  85. package/dist/src/mp4-duration.js +1 -0
  86. package/dist/src/multitouch-support.js +1 -1
  87. package/dist/src/owner-identity.js +1 -1
  88. package/dist/src/parameterized-recorded-fill.js +1 -1
  89. package/dist/src/perf-facade.js +2 -2
  90. package/dist/src/perf-runtime-plan.js +1 -1
  91. package/dist/src/platform-request-scope.js +1 -1
  92. package/dist/src/platform-runtime-managed-owner.js +1 -1
  93. package/dist/src/platform-runtime-open-target.js +1 -1
  94. package/dist/src/platform-runtime-operation-host.js +2 -2
  95. package/dist/src/platform-runtime-runtime-hints.js +1 -1
  96. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  97. package/dist/src/platform-runtime-unavailable.js +1 -1
  98. package/dist/src/platform-runtime.js +1 -1
  99. package/dist/src/platform-runtime2.js +1 -1
  100. package/dist/src/png-crop-bytes.js +1 -1
  101. package/dist/src/png-encode.js +1 -0
  102. package/dist/src/png-size.js +1 -0
  103. package/dist/src/provider-webdriver.js +1 -1
  104. package/dist/src/quality-warnings.js +1 -1
  105. package/dist/src/react-native.js +1 -1
  106. package/dist/src/record-runtime.js +1 -1
  107. package/dist/src/recording-export-quality.js +1 -1
  108. package/dist/src/recording-scope.js +1 -0
  109. package/dist/src/recording.js +3 -0
  110. package/dist/src/registry.js +35 -35
  111. package/dist/src/replay.js +2 -2
  112. package/dist/src/request-admission.js +1 -1
  113. package/dist/src/request-runtime-binding.js +1 -1
  114. package/dist/src/result-serialization.js +1 -1
  115. package/dist/src/runner-cache-metadata.js +2 -2
  116. package/dist/src/runner-client.js +1 -1
  117. package/dist/src/runner-contract.js +1 -1
  118. package/dist/src/runner-dev-tools-security.js +1 -1
  119. package/dist/src/runner-device-readiness.js +1 -1
  120. package/dist/src/runner-disposal.js +1 -1
  121. package/dist/src/runner-operations-facade.js +1 -1
  122. package/dist/src/runner-presentation.js +1 -1
  123. package/dist/src/runner.js +1 -1
  124. package/dist/src/runtime.js +2 -2
  125. package/dist/src/runtime10.js +2 -1
  126. package/dist/src/runtime2.js +4 -4
  127. package/dist/src/runtime4.js +5 -5
  128. package/dist/src/runtime5.js +1 -1
  129. package/dist/src/runtime6.js +1 -1
  130. package/dist/src/runtime7.js +1 -1
  131. package/dist/src/runtime8.js +1 -1
  132. package/dist/src/runtime9.js +1 -1
  133. package/dist/src/screen-recording-admission-ledger.js +1 -1
  134. package/dist/src/screen-recording-resource-recovery.js +1 -1
  135. package/dist/src/screen-recording-transport.js +1 -1
  136. package/dist/src/screenshot-density.js +1 -1
  137. package/dist/src/screenshot-overlay-draw.js +1 -0
  138. package/dist/src/screenshot-runtime.js +1 -1
  139. package/dist/src/script.js +1 -1
  140. package/dist/src/scroll-gesture.js +1 -1
  141. package/dist/src/sdk-batch.js +2 -2
  142. package/dist/src/sdk-contracts.d.ts +7 -5
  143. package/dist/src/sdk-device.js +1 -1
  144. package/dist/src/sdk-finders.js +1 -1
  145. package/dist/src/sdk-selectors.d.ts +44 -1
  146. package/dist/src/sdk-selectors.js +1 -1
  147. package/dist/src/selection.js +0 -0
  148. package/dist/src/selector-runtime.js +1 -1
  149. package/dist/src/selectors.js +1 -1
  150. package/dist/src/semantic-index.js +1 -1
  151. package/dist/src/session-routing.js +1 -1
  152. package/dist/src/session-snapshot.js +1 -1
  153. package/dist/src/session-store.js +3 -3
  154. package/dist/src/session2.js +3 -3
  155. package/dist/src/simctl.js +1 -1
  156. package/dist/src/simulator-boot.js +1 -1
  157. package/dist/src/snapshot-runtime-binding.js +1 -1
  158. package/dist/src/snapshot-runtime.js +1 -1
  159. package/dist/src/snapshot-state.js +1 -1
  160. package/dist/src/snapshot4.js +1 -1
  161. package/dist/src/src.js +1 -1
  162. package/dist/src/src7.js +1 -1
  163. package/dist/src/stop-sequence.js +1 -1
  164. package/dist/src/text-entry-focus.js +1 -1
  165. package/dist/src/tree.js +1 -1
  166. package/dist/src/type-text-runtime.js +1 -1
  167. package/dist/src/video.js +2 -2
  168. package/package.json +1 -1
  169. package/server.json +2 -2
  170. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.8.apk.sha256 +0 -1
  171. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.8.apk.sha256 +0 -1
@@ -76,7 +76,7 @@ extension RunnerTests {
76
76
 
77
77
  private func performCoordinateBackGesture(app: XCUIApplication) {
78
78
  #if !os(tvOS)
79
- let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
79
+ let target = resolveRunnerWindow(app: app).window ?? app
80
80
  let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
81
81
  let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
82
82
  start.press(forDuration: 0.05, thenDragTo: end)
@@ -106,7 +106,7 @@ extension RunnerTests {
106
106
 
107
107
  private func performCoordinateAppSwitcherGesture(app: XCUIApplication) {
108
108
  #if !os(tvOS)
109
- let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
109
+ let target = resolveRunnerWindow(app: app).window ?? app
110
110
  let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
111
111
  let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
112
112
  start.press(forDuration: 0.6, thenDragTo: end)
@@ -219,15 +219,35 @@ extension RunnerTests {
219
219
 
220
220
 
221
221
 
222
- func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
223
- let window = app.windows.element(boundBy: 0)
224
- if window.exists {
225
- let frame = window.frame
226
- if !frame.isEmpty {
227
- return frame
222
+
223
+
224
+
225
+
226
+
227
+
228
+ func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) {
229
+ let windows = app.windows.allElementsBoundByIndex
230
+ var frames: [CGRect?] = []
231
+ for window in windows {
232
+ frames.append(window.exists ? window.frame : nil)
233
+ }
234
+ if let index = Self.firstUsableWindowIndex(frames: frames) {
235
+ return (windows[index], frames[index] ?? .zero)
236
+ }
237
+ return (nil, app.frame)
238
+ }
239
+
240
+ static func firstUsableWindowIndex(frames: [CGRect?]) -> Int? {
241
+ for (index, frame) in frames.enumerated() {
242
+ if let frame, !frame.isEmpty {
243
+ return index
228
244
  }
229
245
  }
230
- return app.frame
246
+ return nil
247
+ }
248
+
249
+ func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
250
+ resolveRunnerWindow(app: app).frame
231
251
  }
232
252
 
233
253
  func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
@@ -505,580 +525,6 @@ extension RunnerTests {
505
525
  return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
506
526
  }
507
527
 
508
- func synthesizedDragAt(
509
- app: XCUIApplication,
510
- x: Double,
511
- y: Double,
512
- x2: Double,
513
- y2: Double,
514
- durationMs: Double,
515
- profile: SynthesizedDragProfile = .continuous,
516
- context: SynthesizedCoordinateContext? = nil
517
- ) -> RunnerInteractionOutcome {
518
- #if os(iOS)
519
- guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
520
- return .unsupported(
521
- message: "synthesized coordinate drag requires finite coordinates",
522
- hint: "Retry with finite x, y, x2, and y2 values."
523
- )
524
- }
525
- let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
526
- guard let context = context ?? synthesizedCoordinateContext(
527
- app: app,
528
- policy: synthesizedGesturePolicy(.synthesizedDrag)
529
- ) else {
530
- return .unsupported(
531
- message: "synthesized coordinate drag could not resolve a finite screen frame",
532
- hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
533
- )
534
- }
535
- let frame = context.referenceFrame
536
- let start = CoordinateSpaceRotation.native(
537
- point: CGPoint(x: x, y: y),
538
- in: frame,
539
- interfaceOrientation: orientation
540
- )
541
- let end = CoordinateSpaceRotation.native(
542
- point: CGPoint(x: x2, y: y2),
543
- in: frame,
544
- interfaceOrientation: orientation
545
- )
546
- let message = switch profile {
547
- case .continuous:
548
- RunnerSynthesizedGesture.synthesizeContinuousDrag(
549
- withApplication: app,
550
- x: Double(start.x),
551
- y: Double(start.y),
552
- x2: Double(end.x),
553
- y2: Double(end.y),
554
- durationMs: durationMs
555
- )
556
- case .controlledScroll:
557
- RunnerSynthesizedGesture.synthesizeControlledScroll(
558
- withApplication: app,
559
- x: Double(start.x),
560
- y: Double(start.y),
561
- x2: Double(end.x),
562
- y2: Double(end.y),
563
- durationMs: durationMs
564
- )
565
- case .fastSwipe:
566
- RunnerSynthesizedGesture.synthesizeSwipe(
567
- withApplication: app,
568
- x: Double(start.x),
569
- y: Double(start.y),
570
- x2: Double(end.x),
571
- y2: Double(end.y),
572
- durationMs: durationMs
573
- )
574
- }
575
- if let message {
576
- return .unsupported(
577
- message: message,
578
- hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists."
579
- )
580
- }
581
- return .performed
582
- #elseif os(tvOS)
583
- return .unsupported(
584
- message: "coordinate drag is not supported on tvOS",
585
- hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
586
- )
587
- #else
588
- return .unsupported(
589
- message: "coordinate drag is not supported on macOS",
590
- hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
591
- )
592
- #endif
593
- }
594
-
595
- func synthesizedTapAt(
596
- app: XCUIApplication,
597
- x: Double,
598
- y: Double,
599
- context: SynthesizedCoordinateContext? = nil
600
- ) -> RunnerInteractionOutcome {
601
- #if os(iOS)
602
- guard x.isFinite, y.isFinite else {
603
- return .unsupported(
604
- message: "synthesized coordinate tap requires finite coordinates",
605
- hint: "Retry with finite x and y values."
606
- )
607
- }
608
- let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
609
- guard let context = context ?? synthesizedCoordinateContext(
610
- app: app,
611
- policy: synthesizedGesturePolicy(.coordinateTap)
612
- ) else {
613
- return .unsupported(
614
- message: "synthesized coordinate tap could not resolve a finite screen frame",
615
- hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
616
- )
617
- }
618
- let frame = context.referenceFrame
619
- let point = CoordinateSpaceRotation.native(
620
- point: CGPoint(x: x, y: y),
621
- in: frame,
622
- interfaceOrientation: orientation
623
- )
624
- if let message = RunnerSynthesizedGesture.synthesizeTap(
625
- withApplication: app,
626
- x: Double(point.x),
627
- y: Double(point.y)
628
- ) {
629
- return .unsupported(
630
- message: message,
631
- hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree."
632
- )
633
- }
634
- return .performed
635
- #elseif os(tvOS)
636
- return .unsupported(
637
- message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
638
- hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
639
- )
640
- #else
641
- return .unsupported(
642
- message: "synthesized coordinate tap is not supported on macOS",
643
- hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
644
- )
645
- #endif
646
- }
647
-
648
- func keyboardAvoidingDragPoints(
649
- app: XCUIApplication,
650
- x: Double,
651
- y: Double,
652
- x2: Double,
653
- y2: Double
654
- ) -> DragPoints {
655
- let original = DragPoints(x: x, y: y, x2: x2, y2: y2)
656
- #if os(iOS)
657
- guard let keyboardFrame = visibleKeyboardFrame(app: app) else {
658
- return original
659
- }
660
- let minX = min(x, x2)
661
- let minY = min(y, y2)
662
- let gestureBounds = CGRect(
663
- x: CGFloat(minX),
664
- y: CGFloat(minY),
665
- width: CGFloat(max(abs(x2 - x), 1)),
666
- height: CGFloat(max(abs(y2 - y), 1))
667
- )
668
- guard gestureBounds.intersects(keyboardFrame) else {
669
- return original
670
- }
671
-
672
- let window = app.windows.firstMatch
673
- let appFrame = window.exists && !window.frame.isEmpty ? window.frame : app.frame
674
- guard !appFrame.isEmpty else {
675
- return original
676
- }
677
-
678
- let padding: Double = 12
679
- let targetMaxY = Double(keyboardFrame.minY) - padding
680
- let currentMaxY = max(y, y2)
681
- let shift = currentMaxY - targetMaxY
682
- guard shift > 0 else {
683
- return original
684
- }
685
-
686
- let adjustedY = y - shift
687
- let adjustedY2 = y2 - shift
688
- guard min(adjustedY, adjustedY2) >= Double(appFrame.minY) + padding else {
689
- return original
690
- }
691
-
692
- NSLog(
693
- "AGENT_DEVICE_RUNNER_KEYBOARD_AVOIDING_DRAG from=(%.1f,%.1f)->(%.1f,%.1f) adjusted=(%.1f,%.1f)->(%.1f,%.1f) keyboardMinY=%.1f",
694
- x,
695
- y,
696
- x2,
697
- y2,
698
- x,
699
- adjustedY,
700
- x2,
701
- adjustedY2,
702
- Double(keyboardFrame.minY)
703
- )
704
- return DragPoints(x: x, y: adjustedY, x2: x2, y2: adjustedY2)
705
- #else
706
- return original
707
- #endif
708
- }
709
-
710
- func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame {
711
- let appFrame = app.frame
712
- let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame)
713
- let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX
714
- let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY
715
- return TouchVisualizationFrame(
716
- x: originX + x,
717
- y: originY + y,
718
- referenceWidth: referenceFrame.width,
719
- referenceHeight: referenceFrame.height
720
- )
721
- }
722
-
723
- func resolvedDragVisualizationFrame(
724
- app: XCUIApplication,
725
- x: Double,
726
- y: Double,
727
- x2: Double,
728
- y2: Double
729
- ) -> DragVisualizationFrame {
730
- let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y)
731
- let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2)
732
- return DragVisualizationFrame(
733
- x: start.x,
734
- y: start.y,
735
- x2: end.x,
736
- y2: end.y,
737
- referenceWidth: start.referenceWidth,
738
- referenceHeight: start.referenceHeight
739
- )
740
- }
741
-
742
- func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
743
- let window = app.windows.firstMatch
744
- if window.exists {
745
- let windowFrame = window.frame
746
- if !windowFrame.isEmpty {
747
- return frameAvoidingKeyboard(app: app, frame: windowFrame)
748
- }
749
- }
750
- if !appFrame.isEmpty {
751
- return frameAvoidingKeyboard(app: app, frame: appFrame)
752
- }
753
- return CGRect(x: 0, y: 0, width: 0, height: 0)
754
- }
755
-
756
- private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect {
757
- #if os(iOS)
758
- guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else {
759
- return frame
760
- }
761
- let intersection = frame.intersection(keyboardFrame)
762
- guard !intersection.isNull && intersection.height > 0 else {
763
- return frame
764
- }
765
- let keyboardCoverage = intersection.width / max(frame.width, 1)
766
- guard keyboardCoverage >= 0.5 else {
767
- return frame
768
- }
769
- let safeHeight = keyboardFrame.minY - frame.minY
770
- guard safeHeight >= frame.height * 0.25 else {
771
- return frame
772
- }
773
- return CGRect(x: frame.minX, y: frame.minY, width: frame.width, height: safeHeight)
774
- #else
775
- return frame
776
- #endif
777
- }
778
-
779
- func axFreeSynthesizedDragPlan(
780
- app: XCUIApplication,
781
- x: Double,
782
- y: Double,
783
- x2: Double,
784
- y2: Double,
785
- context: SynthesizedCoordinateContext? = nil
786
- ) -> SynthesizedDragPlan? {
787
- #if os(iOS)
788
- let context = context ?? synthesizedCoordinateContext(
789
- app: app,
790
- policy: synthesizedGesturePolicy(.synthesizedDrag)
791
- )
792
- guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
793
- let context
794
- else {
795
- return nil
796
- }
797
- let points = keyboardAvoidingSynthesizedDragPoints(
798
- app: app,
799
- x: x,
800
- y: y,
801
- x2: x2,
802
- y2: y2,
803
- context: context
804
- )
805
- return SynthesizedDragPlan(
806
- points: points,
807
- context: context
808
- )
809
- #else
810
- return nil
811
- #endif
812
- }
813
-
814
- func axFreeDragVisualizationFrame(
815
- x: Double,
816
- y: Double,
817
- x2: Double,
818
- y2: Double,
819
- referenceFrame: CGRect
820
- ) -> DragVisualizationFrame {
821
- return DragVisualizationFrame(
822
- x: x,
823
- y: y,
824
- x2: x2,
825
- y2: y2,
826
- referenceWidth: Double(referenceFrame.width),
827
- referenceHeight: Double(referenceFrame.height)
828
- )
829
- }
830
-
831
- func synthesizedCoordinateContext(
832
- app: XCUIApplication,
833
- policy: SynthesizedGesturePolicy
834
- ) -> SynthesizedCoordinateContext? {
835
- #if os(iOS)
836
- let health = runnerAccessibilityHealth
837
- let orientation = Int(
838
- RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)
839
- )
840
- guard let referenceFrame = orientedSynthesizedScreenshotReferenceFrame(
841
- screenshotSize: XCUIScreen.main.screenshot().image.size,
842
- interfaceOrientation: orientation
843
- ) else {
844
- return nil
845
- }
846
- return SynthesizedCoordinateContext(
847
- referenceFrame: referenceFrame,
848
- keyboardPolicy: policy.keyboardPolicy,
849
- fallbackPolicy: policy.fallbackPolicy,
850
- accessibilityHealth: health
851
- )
852
- #else
853
- return nil
854
- #endif
855
- }
856
-
857
- func orientedSynthesizedScreenshotReferenceFrame(
858
- screenshotSize: CGSize,
859
- interfaceOrientation: Int
860
- ) -> CGRect? {
861
-
862
-
863
- guard screenshotSize.width.isFinite, screenshotSize.height.isFinite,
864
- screenshotSize.width > 0,
865
- screenshotSize.height > 0
866
- else {
867
- return nil
868
- }
869
- let isLandscape = interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft
870
- || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight
871
- let isPortrait = interfaceOrientation == RunnerInterfaceOrientation.portrait
872
- || interfaceOrientation == RunnerInterfaceOrientation.portraitUpsideDown
873
- let width: CGFloat
874
- let height: CGFloat
875
- if isLandscape {
876
- width = max(screenshotSize.width, screenshotSize.height)
877
- height = min(screenshotSize.width, screenshotSize.height)
878
- } else if isPortrait {
879
- width = min(screenshotSize.width, screenshotSize.height)
880
- height = max(screenshotSize.width, screenshotSize.height)
881
- } else {
882
- width = screenshotSize.width
883
- height = screenshotSize.height
884
- }
885
- return CGRect(x: 0, y: 0, width: width, height: height)
886
- }
887
-
888
- func keyboardAvoidingSynthesizedDragPoints(
889
- app: XCUIApplication,
890
- x: Double,
891
- y: Double,
892
- x2: Double,
893
- y2: Double,
894
- context: SynthesizedCoordinateContext
895
- ) -> DragPoints {
896
- #if os(iOS)
897
- guard context.allowsKeyboardProbe else {
898
- return DragPoints(x: x, y: y, x2: x2, y2: y2)
899
- }
900
- return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
901
- #else
902
- return DragPoints(x: x, y: y, x2: x2, y2: y2)
903
- #endif
904
- }
905
-
906
- func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
907
- if performTvRemoteSwipeIfAvailable(direction: direction) {
908
- let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
909
- let midX = frame.midX
910
- let midY = frame.midY
911
- return DragVisualizationFrame(
912
- x: midX,
913
- y: midY,
914
- x2: midX,
915
- y2: midY,
916
- referenceWidth: frame.width,
917
- referenceHeight: frame.height
918
- )
919
- }
920
- return nil
921
- }
922
-
923
- private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
924
- switch direction {
925
- case "up":
926
- return pressTvRemote(.up)
927
- case "down":
928
- return pressTvRemote(.down)
929
- case "left":
930
- return pressTvRemote(.left)
931
- case "right":
932
- return pressTvRemote(.right)
933
- default:
934
- return false
935
- }
936
- }
937
-
938
- func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? {
939
- guard plan.topology == "single" || plan.topology == "two" else {
940
- return "planned gesture topology must be single or two"
941
- }
942
- let supportedIntent = plan.topology == "single"
943
- ? plan.intent == "fling" || plan.intent == "pan"
944
- : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate"
945
- || plan.intent == "transform"
946
- guard supportedIntent else { return "planned gesture has unsupported intent for its topology" }
947
- if plan.topology == "single" {
948
- guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else {
949
- return "single-pointer gesture requires a supported execution profile"
950
- }
951
- } else if plan.executionProfile != nil {
952
- return "multi-touch gesture cannot define a single-pointer execution profile"
953
- }
954
- guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else {
955
- return "planned gesture durationMs must be between 16 and 10000"
956
- }
957
- let viewport = plan.viewport
958
- guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite,
959
- viewport.height.isFinite, viewport.width > 0, viewport.height > 0
960
- else {
961
- return "planned gesture viewport must be finite and positive"
962
- }
963
- let expectedPointerCount = plan.topology == "single" ? 1 : 2
964
- guard plan.pointers.count == expectedPointerCount else {
965
- return "planned gesture pointer count does not match topology"
966
- }
967
- for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index {
968
- return "planned gesture requires ordered pointer ids"
969
- }
970
- let firstSamples = plan.pointers[0].samples
971
- guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" }
972
- for pointer in plan.pointers {
973
- guard pointer.samples.count == firstSamples.count else {
974
- return "planned pointer paths require matching samples"
975
- }
976
- var previousOffset = -1.0
977
- for (index, sample) in pointer.samples.enumerated() {
978
- guard sample.offsetMs.isFinite,
979
- sample.offsetMs == firstSamples[index].offsetMs,
980
- sample.offsetMs > previousOffset
981
- else {
982
- return "planned pointer sample offsets must match and strictly increase"
983
- }
984
- let point = sample.point
985
- guard point.x.isFinite, point.y.isFinite,
986
- point.x >= viewport.x,
987
- point.x <= viewport.x + viewport.width,
988
- point.y >= viewport.y,
989
- point.y <= viewport.y + viewport.height
990
- else {
991
- return "planned pointer sample lies outside the viewport"
992
- }
993
- previousOffset = sample.offsetMs
994
- }
995
- guard pointer.samples.first?.offsetMs == 0,
996
- pointer.samples.last?.offsetMs == plan.durationMs
997
- else {
998
- return "planned pointer paths must start at 0 and end at durationMs"
999
- }
1000
- }
1001
- if plan.topology == "two" {
1002
- guard let firstStart = firstSamples.first?.point,
1003
- let secondStart = plan.pointers[1].samples.first?.point,
1004
- hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0
1005
- else {
1006
- return "planned pointer paths require a positive initial span"
1007
- }
1008
- }
1009
- return nil
1010
- }
1011
-
1012
- func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution {
1013
- plan.topology == "single" && plan.executionProfile == "endpoint-hold"
1014
- ? .fastSwipe
1015
- : .sampled
1016
- }
1017
-
1018
- func sampledPlannedGesture(
1019
- app: XCUIApplication,
1020
- plan: RunnerGesturePlan
1021
- ) -> RunnerInteractionOutcome {
1022
- #if os(iOS)
1023
- let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
1024
-
1025
-
1026
- let frame = CGRect(
1027
- x: plan.viewport.x,
1028
- y: plan.viewport.y,
1029
- width: plan.viewport.width,
1030
- height: plan.viewport.height
1031
- )
1032
- let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
1033
- pointer.samples.map { sample in
1034
- let point = CoordinateSpaceRotation.native(
1035
- point: CGPoint(x: sample.point.x, y: sample.point.y),
1036
- in: frame,
1037
- interfaceOrientation: orientation
1038
- )
1039
- return [
1040
- "x": NSNumber(value: Double(point.x)),
1041
- "y": NSNumber(value: Double(point.y)),
1042
- "offsetMs": NSNumber(value: sample.offsetMs),
1043
- ]
1044
- }
1045
- }
1046
- if let message = RunnerSynthesizedGesture.synthesizeGesture(
1047
- withApplication: app,
1048
- pointerSamples: pointerSamples
1049
- ) {
1050
- return .unsupported(
1051
- message: message,
1052
- hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists."
1053
- )
1054
- }
1055
- return .performed
1056
- #elseif os(tvOS)
1057
- return .unsupported(
1058
- message: "two-finger gestures are not supported on tvOS",
1059
- hint: "tvOS has no touch input; use remote-driven navigation."
1060
- )
1061
- #elseif os(visionOS)
1062
- return .unsupported(
1063
- message: "two-finger touch gestures are not supported on visionOS",
1064
- hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only."
1065
- )
1066
- #else
1067
- return .unsupported(
1068
- message: "two-finger gestures are not supported on macOS",
1069
- hint: "macOS automation has no multi-touch input; run on an iOS simulator."
1070
- )
1071
- #endif
1072
- }
1073
-
1074
- private func interactionRoot(app: XCUIApplication) -> XCUIElement {
1075
- let windows = app.windows.allElementsBoundByIndex
1076
- if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
1077
- return window
1078
- }
1079
- return app
1080
- }
1081
-
1082
528
  private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
1083
529
  #if os(tvOS)
1084
530
  return .unsupported(
@@ -1139,14 +585,20 @@ extension RunnerTests {
1139
585
  #if !os(tvOS)
1140
586
  private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
1141
587
  #if os(iOS)
588
+
589
+
590
+
591
+
592
+
593
+
1142
594
  let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
1143
595
  return origin.withOffset(CGVector(dx: x, dy: y))
1144
596
  #else
1145
- let root = interactionRoot(app: app)
597
+ let resolved = resolveRunnerWindow(app: app)
598
+ let root = resolved.window ?? app
1146
599
  let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
1147
- let rootFrame = root.frame
1148
- let offsetX = x - Double(rootFrame.origin.x)
1149
- let offsetY = y - Double(rootFrame.origin.y)
600
+ let offsetX = x - Double(resolved.frame.origin.x)
601
+ let offsetY = y - Double(resolved.frame.origin.y)
1150
602
  return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
1151
603
  #endif
1152
604
  }
@@ -1171,153 +623,6 @@ extension RunnerTests {
1171
623
 
1172
624
 
1173
625
 
1174
-
1175
-
1176
-
1177
-
1178
-
1179
-
1180
-
1181
-
1182
-
1183
-
1184
-
1185
-
1186
-
1187
-
1188
-
1189
-
1190
-
1191
-
1192
-
1193
-
1194
-
1195
-
1196
-
1197
-
1198
-
1199
-
1200
-
1201
-
1202
-
1203
-
1204
-
1205
-
1206
-
1207
-
1208
-
1209
-
1210
-
1211
-
1212
-
1213
-
1214
-
1215
-
1216
-
1217
-
1218
-
1219
-
1220
-
1221
-
1222
-
1223
-
1224
-
1225
-
1226
-
1227
-
1228
-
1229
-
1230
-
1231
-
1232
-
1233
-
1234
-
1235
-
1236
-
1237
-
1238
-
1239
-
1240
-
1241
-
1242
-
1243
-
1244
-
1245
-
1246
-
1247
-
1248
-
1249
-
1250
-
1251
-
1252
-
1253
-
1254
-
1255
-
1256
-
1257
-
1258
-
1259
-
1260
-
1261
-
1262
-
1263
-
1264
-
1265
-
1266
-
1267
-
1268
-
1269
-
1270
-
1271
-
1272
-
1273
-
1274
-
1275
-
1276
-
1277
-
1278
-
1279
-
1280
-
1281
-
1282
-
1283
-
1284
-
1285
-
1286
-
1287
-
1288
-
1289
-
1290
-
1291
-
1292
-
1293
-
1294
-
1295
-
1296
-
1297
-
1298
-
1299
-
1300
-
1301
-
1302
-
1303
-
1304
-
1305
-
1306
-
1307
-
1308
-
1309
-
1310
-
1311
-
1312
-
1313
-
1314
-
1315
-
1316
-
1317
-
1318
-
1319
-
1320
-
1321
626
 
1322
627
 
1323
628