agent-device 0.19.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/README.md +61 -70
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.manifest.json +10 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk.sha256 +1 -0
  7. package/{android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.manifest.json → android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.manifest.json} +6 -6
  8. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +2 -0
  9. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +5 -0
  10. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +12 -10
  11. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +154 -185
  12. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +113 -19
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +82 -0
  14. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +275 -113
  15. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +2 -2
  16. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +150 -104
  17. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +56 -2
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +73 -0
  19. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +44 -14
  20. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +2 -2
  21. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +2 -106
  22. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +126 -11
  23. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +51 -8
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +24 -0
  25. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +1 -1
  26. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +43 -35
  27. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +14 -0
  28. package/dist/src/agent-browser-tool.js +1 -1
  29. package/dist/src/agent-device-client.js +3 -0
  30. package/dist/src/android-adb.d.ts +1 -1
  31. package/dist/src/android-adb.js +1 -1
  32. package/dist/src/android-input-method-overlays.js +1 -0
  33. package/dist/src/android-input-ownership.js +1 -6
  34. package/dist/src/android-system-chrome.js +1 -0
  35. package/dist/src/android.js +1 -1
  36. package/dist/src/app-lifecycle.js +5 -5
  37. package/dist/src/app-log-request-scope.js +1 -0
  38. package/dist/src/app-resolution.js +1 -0
  39. package/dist/src/apple-multitouch-support.js +1 -0
  40. package/dist/src/apps.js +5 -5
  41. package/dist/src/audio-probe.js +1 -1
  42. package/dist/src/boot-diagnostics.js +2 -0
  43. package/dist/src/cli-config.js +1 -0
  44. package/dist/src/cli-help.js +64 -30
  45. package/dist/src/cli-runner.js +1 -1
  46. package/dist/src/cli.js +15 -11
  47. package/dist/src/command-schema.js +3 -3
  48. package/dist/src/config.js +1 -0
  49. package/dist/src/context.js +1 -0
  50. package/dist/src/contracts.d.ts +1 -1
  51. package/dist/src/contracts.js +1 -1
  52. package/dist/src/daemon-shutdown-report.js +1 -0
  53. package/dist/src/debug-symbols.js +1 -1
  54. package/dist/src/device-input-state.js +1 -1
  55. package/dist/src/devices2.js +1 -1
  56. package/dist/src/find.js +1 -1
  57. package/dist/src/finders.js +1 -1
  58. package/dist/src/generic.js +1 -1
  59. package/dist/src/helper.js +1 -1
  60. package/dist/src/index.d.ts +412 -289
  61. package/dist/src/index.js +1 -1
  62. package/dist/src/input-actions.js +4 -3
  63. package/dist/src/input-actions2.js +1 -1
  64. package/dist/src/install-artifact.js +1 -0
  65. package/dist/src/install-source.d.ts +5 -1
  66. package/dist/src/install-source.js +1 -1
  67. package/dist/src/install-source2.js +1 -1
  68. package/dist/src/interaction-snapshot.js +1 -1
  69. package/dist/src/interaction.d.ts +230 -0
  70. package/dist/src/interaction.js +1 -1
  71. package/dist/src/interactor.js +2 -2
  72. package/dist/src/internal/bin.js +1 -1
  73. package/dist/src/internal/daemon.js +43 -17
  74. package/dist/src/internal/png-worker.d.ts +22 -0
  75. package/dist/src/internal/png-worker.js +1 -1
  76. package/dist/src/launch-console.js +1 -0
  77. package/dist/src/lease-context.js +1 -0
  78. package/dist/src/lease.js +1 -1
  79. package/dist/src/linux.js +1 -1
  80. package/dist/src/logcat.js +5 -0
  81. package/dist/src/metro.d.ts +2 -2
  82. package/dist/src/notifications.js +1 -1
  83. package/dist/src/png-worker-contract.js +1 -1
  84. package/dist/src/process-lock.js +1 -0
  85. package/dist/src/react-native.js +1 -1
  86. package/dist/src/record-trace.js +1 -1
  87. package/dist/src/register-builtins.js +1 -0
  88. package/dist/src/registry.js +24 -24
  89. package/dist/src/reporting.js +8 -7
  90. package/dist/src/runner-client.js +1 -1
  91. package/dist/src/runner-disposal.js +2 -0
  92. package/dist/src/runner-lease.js +1 -0
  93. package/dist/src/runner-sequence.js +1 -1
  94. package/dist/src/runtime-transport.js +1 -1
  95. package/dist/src/runtime.js +3 -3
  96. package/dist/src/runtime2.js +1 -0
  97. package/dist/src/sdk-android-adb.d.ts +119 -33
  98. package/dist/src/sdk-android-adb.js +4 -4
  99. package/dist/src/sdk-app-inventory.js +1 -1
  100. package/dist/src/sdk-batch-runner.d.ts +2 -1
  101. package/dist/src/sdk-batch-runner.js +2 -2
  102. package/dist/src/sdk-batch.js +2 -2
  103. package/dist/src/sdk-contracts.d.ts +105 -98
  104. package/dist/src/sdk-contracts.js +2 -2
  105. package/dist/src/sdk-device.js +1 -0
  106. package/dist/src/sdk-finders.d.ts +3 -3
  107. package/dist/src/sdk-finders.js +1 -1
  108. package/dist/src/sdk-io.d.ts +1 -1
  109. package/dist/src/sdk-io.js +1 -1
  110. package/dist/src/sdk-remote-config.d.ts +9 -1
  111. package/dist/src/sdk-remote-config.js +1 -1
  112. package/dist/src/sdk-selectors.js +1 -1
  113. package/dist/src/selector-runtime.js +1 -1
  114. package/dist/src/selector-vocabulary.js +1 -0
  115. package/dist/src/selectors.d.ts +12 -3
  116. package/dist/src/selectors.js +1 -1
  117. package/dist/src/server.js +3 -3
  118. package/dist/src/session-target-evidence.js +1 -0
  119. package/dist/src/session.js +8 -10
  120. package/dist/src/setting-state.js +1 -1
  121. package/dist/src/simulator.js +1 -1
  122. package/dist/src/snapshot-diagnostics.js +5 -2
  123. package/dist/src/snapshot.js +1 -1
  124. package/dist/src/snapshot2.js +1 -2
  125. package/dist/src/tool-provider.js +2 -2
  126. package/dist/src/tool-provider2.js +1 -1
  127. package/dist/src/tv-remote.js +13 -12
  128. package/dist/src/url.js +1 -1
  129. package/dist/src/web.js +1 -1
  130. package/package.json +33 -20
  131. package/server.json +2 -2
  132. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.apk +0 -0
  133. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.apk.sha256 +0 -1
  134. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.manifest.json +0 -10
  135. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.apk +0 -0
  136. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.apk.sha256 +0 -1
  137. package/dist/src/apple-runner-options.js +0 -1
  138. package/dist/src/cli-output.js +0 -1
  139. package/dist/src/client.js +0 -3
  140. package/dist/src/handler-utils.js +0 -1
  141. package/dist/src/output-path.js +0 -1
  142. package/dist/src/points.js +0 -4
  143. package/dist/src/provider.js +0 -1
  144. package/dist/src/providers.js +0 -1
  145. package/dist/src/record-trace-recording.js +0 -23
  146. package/dist/src/recording-provider.js +0 -1
  147. package/dist/src/selectors2.js +0 -1
  148. package/dist/src/simctl.js +0 -1
  149. package/dist/src/snapshot-helper.js +0 -5
  150. package/dist/src/ttl-memo.js +0 -1
  151. package/dist/src/xml.js +0 -2
  152. /package/{macos-helper → apple/macos-helper}/Package.swift +0 -0
  153. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +0 -0
  154. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +0 -0
  155. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/main.swift +0 -0
  156. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +0 -0
  157. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +0 -0
  158. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -0
  159. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +0 -0
  160. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +0 -0
  161. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +0 -0
  162. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.h +0 -0
  163. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.m +0 -0
  164. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +0 -0
  165. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +0 -0
  166. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +0 -0
  167. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +0 -0
  168. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +0 -0
  169. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +0 -0
  170. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +0 -0
  171. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +0 -0
  172. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +0 -0
  173. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +0 -0
  174. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +0 -0
  175. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +0 -0
  176. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -0
  177. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-trim.swift +0 -0
  178. /package/{src/platforms/linux → linux}/atspi-dump.py +0 -0
@@ -13,6 +13,16 @@ private enum RunnerInterfaceOrientation {
13
13
  }
14
14
 
15
15
  extension RunnerTests {
16
+ enum PlannedGestureExecution: Equatable {
17
+ case fastSwipe
18
+ case sampled
19
+ }
20
+
21
+ enum SynthesizedDragProfile {
22
+ case continuous
23
+ case fastSwipe
24
+ }
25
+
16
26
  struct TouchVisualizationFrame {
17
27
  let x: Double
18
28
  let y: Double
@@ -139,7 +149,8 @@ extension RunnerTests {
139
149
  app: XCUIApplication,
140
150
  selectorKey: String,
141
151
  selectorValue: String,
142
- allowNonHittableFallback: Bool = false
152
+ allowNonHittableFallback: Bool = false,
153
+ expectedPoint: CGPoint? = nil
143
154
  ) -> SelectorElementMatch {
144
155
  let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
145
156
  guard !value.isEmpty else {
@@ -163,6 +174,9 @@ extension RunnerTests {
163
174
  var nonHittableElement: XCUIElement?
164
175
  let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
165
176
  for element in matches where element.exists {
177
+ if let expectedPoint, !element.frame.contains(expectedPoint) {
178
+ continue
179
+ }
166
180
  if !element.isHittable {
167
181
  if allowNonHittableFallback && hasTappableFrame(app: app, element: element) {
168
182
  guard nonHittableElement == nil else {
@@ -775,6 +789,7 @@ extension RunnerTests {
775
789
  x2: Double,
776
790
  y2: Double,
777
791
  durationMs: Double,
792
+ profile: SynthesizedDragProfile = .continuous,
778
793
  context: SynthesizedCoordinateContext? = nil
779
794
  ) -> RunnerInteractionOutcome {
780
795
  #if os(iOS)
@@ -794,14 +809,27 @@ extension RunnerTests {
794
809
  let frame = context.referenceFrame
795
810
  let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
796
811
  let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
797
- if let message = RunnerSynthesizedGesture.synthesizeSwipe(
798
- withApplication: app,
799
- x: Double(start.x),
800
- y: Double(start.y),
801
- x2: Double(end.x),
802
- y2: Double(end.y),
803
- durationMs: durationMs
804
- ) {
812
+ let message = switch profile {
813
+ case .continuous:
814
+ RunnerSynthesizedGesture.synthesizeContinuousDrag(
815
+ withApplication: app,
816
+ x: Double(start.x),
817
+ y: Double(start.y),
818
+ x2: Double(end.x),
819
+ y2: Double(end.y),
820
+ durationMs: durationMs
821
+ )
822
+ case .fastSwipe:
823
+ RunnerSynthesizedGesture.synthesizeSwipe(
824
+ withApplication: app,
825
+ x: Double(start.x),
826
+ y: Double(start.y),
827
+ x2: Double(end.x),
828
+ y2: Double(end.y),
829
+ durationMs: durationMs
830
+ )
831
+ }
832
+ if let message {
805
833
  return .unsupported(
806
834
  message: message,
807
835
  hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists."
@@ -1153,125 +1181,143 @@ extension RunnerTests {
1153
1181
  }
1154
1182
  }
1155
1183
 
1156
- func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
1157
- #if os(iOS)
1158
- // A coordinate tap+drag is a single-finger gesture: React Native reads it as a pan
1159
- // and the pinch scale never changes (#629). Drive the two-finger XCTest synthesis
1160
- // path (the same one transformGesture uses) with zero translation/rotation so RN's
1161
- // pinch recognizer actually fires.
1162
- let frame = interactionRoot(app: app).frame
1163
- let centerX = x ?? Double(frame.midX)
1164
- let centerY = y ?? Double(frame.midY)
1165
- return transformGesture(
1166
- app: app,
1167
- x: centerX,
1168
- y: centerY,
1169
- dx: 0,
1170
- dy: 0,
1171
- scale: scale,
1172
- degrees: 0,
1173
- durationMs: 300
1174
- )
1175
- #elseif os(tvOS)
1176
- return .unsupported(
1177
- message: "pinch is not supported on tvOS",
1178
- hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)."
1179
- )
1180
- #else
1181
- return .unsupported(
1182
- message: "pinch is not supported on macOS",
1183
- hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)."
1184
- )
1185
- #endif
1184
+ func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? {
1185
+ guard plan.topology == "single" || plan.topology == "two" else {
1186
+ return "planned gesture topology must be single or two"
1187
+ }
1188
+ let supportedIntent = plan.topology == "single"
1189
+ ? plan.intent == "fling" || plan.intent == "pan"
1190
+ : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate"
1191
+ || plan.intent == "transform"
1192
+ guard supportedIntent else { return "planned gesture has unsupported intent for its topology" }
1193
+ if plan.topology == "single" {
1194
+ guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else {
1195
+ return "single-pointer gesture requires a supported execution profile"
1196
+ }
1197
+ } else if plan.executionProfile != nil {
1198
+ return "multi-touch gesture cannot define a single-pointer execution profile"
1199
+ }
1200
+ guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else {
1201
+ return "planned gesture durationMs must be between 16 and 10000"
1202
+ }
1203
+ let viewport = plan.viewport
1204
+ guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite,
1205
+ viewport.height.isFinite, viewport.width > 0, viewport.height > 0
1206
+ else {
1207
+ return "planned gesture viewport must be finite and positive"
1208
+ }
1209
+ let expectedPointerCount = plan.topology == "single" ? 1 : 2
1210
+ guard plan.pointers.count == expectedPointerCount else {
1211
+ return "planned gesture pointer count does not match topology"
1212
+ }
1213
+ for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index {
1214
+ return "planned gesture requires ordered pointer ids"
1215
+ }
1216
+ let firstSamples = plan.pointers[0].samples
1217
+ guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" }
1218
+ for pointer in plan.pointers {
1219
+ guard pointer.samples.count == firstSamples.count else {
1220
+ return "planned pointer paths require matching samples"
1221
+ }
1222
+ var previousOffset = -1.0
1223
+ for (index, sample) in pointer.samples.enumerated() {
1224
+ guard sample.offsetMs.isFinite,
1225
+ sample.offsetMs == firstSamples[index].offsetMs,
1226
+ sample.offsetMs > previousOffset
1227
+ else {
1228
+ return "planned pointer sample offsets must match and strictly increase"
1229
+ }
1230
+ let point = sample.point
1231
+ guard point.x.isFinite, point.y.isFinite,
1232
+ point.x >= viewport.x,
1233
+ point.x <= viewport.x + viewport.width,
1234
+ point.y >= viewport.y,
1235
+ point.y <= viewport.y + viewport.height
1236
+ else {
1237
+ return "planned pointer sample lies outside the viewport"
1238
+ }
1239
+ previousOffset = sample.offsetMs
1240
+ }
1241
+ guard pointer.samples.first?.offsetMs == 0,
1242
+ pointer.samples.last?.offsetMs == plan.durationMs
1243
+ else {
1244
+ return "planned pointer paths must start at 0 and end at durationMs"
1245
+ }
1246
+ }
1247
+ if plan.topology == "two" {
1248
+ guard let firstStart = firstSamples.first?.point,
1249
+ let secondStart = plan.pointers[1].samples.first?.point,
1250
+ hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0
1251
+ else {
1252
+ return "planned pointer paths require a positive initial span"
1253
+ }
1254
+ }
1255
+ return nil
1186
1256
  }
1187
1257
 
1188
- func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
1189
- #if os(iOS)
1190
- // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634)
1191
- // with zero translation/scale so React Native's rotation recognizer actually fires. The native
1192
- // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture
1193
- // handler does not read reliably — the same class of problem #629/#634 fixed for pinch.
1194
- // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract
1195
- // keeps it for compatibility and direction is carried entirely by the sign of `degrees`.
1196
- let frame = interactionRoot(app: app).frame
1197
- let centerX = x ?? Double(frame.midX)
1198
- let centerY = y ?? Double(frame.midY)
1199
- return transformGesture(
1200
- app: app,
1201
- x: centerX,
1202
- y: centerY,
1203
- dx: 0,
1204
- dy: 0,
1205
- scale: 1,
1206
- degrees: degrees,
1207
- durationMs: 300
1208
- )
1209
- #elseif os(tvOS)
1210
- return .unsupported(
1211
- message: "rotate-gesture is not supported on tvOS",
1212
- hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)."
1213
- )
1214
- #else
1215
- return .unsupported(
1216
- message: "rotate-gesture is not supported on macOS",
1217
- hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)."
1218
- )
1219
- #endif
1258
+ func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution {
1259
+ plan.topology == "single" && plan.executionProfile == "endpoint-hold"
1260
+ ? .fastSwipe
1261
+ : .sampled
1220
1262
  }
1221
1263
 
1222
- func transformGesture(
1264
+ func sampledPlannedGesture(
1223
1265
  app: XCUIApplication,
1224
- x: Double,
1225
- y: Double,
1226
- dx: Double,
1227
- dy: Double,
1228
- scale: Double,
1229
- degrees: Double,
1230
- durationMs: Double
1266
+ plan: RunnerGesturePlan
1231
1267
  ) -> RunnerInteractionOutcome {
1232
1268
  #if os(iOS)
1233
- let target = interactionRoot(app: app)
1234
1269
  let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
1235
- let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation)
1236
- let vector = nativeSynthesizedVector(orientedDx: dx, orientedDy: dy, interfaceOrientation: orientation)
1237
- if let message = RunnerSynthesizedGesture.synthesizeTransform(
1270
+ // The portable planner and validation use this exact viewport. Using app.frame here can
1271
+ // diverge when XCTest unions transformed/off-screen descendants into the application frame.
1272
+ let frame = CGRect(
1273
+ x: plan.viewport.x,
1274
+ y: plan.viewport.y,
1275
+ width: plan.viewport.width,
1276
+ height: plan.viewport.height
1277
+ )
1278
+ let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
1279
+ pointer.samples.map { sample in
1280
+ let point = nativeSynthesizedPoint(
1281
+ orientedX: sample.point.x,
1282
+ orientedY: sample.point.y,
1283
+ in: frame,
1284
+ interfaceOrientation: orientation
1285
+ )
1286
+ return [
1287
+ "x": NSNumber(value: Double(point.x)),
1288
+ "y": NSNumber(value: Double(point.y)),
1289
+ "offsetMs": NSNumber(value: sample.offsetMs),
1290
+ ]
1291
+ }
1292
+ }
1293
+ if let message = RunnerSynthesizedGesture.synthesizeGesture(
1238
1294
  withApplication: app,
1239
- x: Double(point.x),
1240
- y: Double(point.y),
1241
- dx: Double(vector.dx),
1242
- dy: Double(vector.dy),
1243
- scale: scale,
1244
- degrees: degrees,
1245
- radius: transformGestureRadius(frame: target.frame, scale: scale),
1246
- durationMs: durationMs
1295
+ pointerSamples: pointerSamples
1247
1296
  ) {
1248
1297
  return .unsupported(
1249
1298
  message: message,
1250
- hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)."
1299
+ hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists."
1251
1300
  )
1252
1301
  }
1253
1302
  return .performed
1254
1303
  #elseif os(tvOS)
1255
1304
  return .unsupported(
1256
- message: "transformGesture is not supported on tvOS",
1257
- hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)."
1305
+ message: "two-finger gestures are not supported on tvOS",
1306
+ hint: "tvOS has no touch input; use remote-driven navigation."
1307
+ )
1308
+ #elseif os(visionOS)
1309
+ return .unsupported(
1310
+ message: "two-finger touch gestures are not supported on visionOS",
1311
+ hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only."
1258
1312
  )
1259
1313
  #else
1260
1314
  return .unsupported(
1261
- message: "transformGesture is not supported on macOS",
1262
- hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)."
1315
+ message: "two-finger gestures are not supported on macOS",
1316
+ hint: "macOS automation has no multi-touch input; run on an iOS simulator."
1263
1317
  )
1264
1318
  #endif
1265
1319
  }
1266
1320
 
1267
- private func transformGestureRadius(frame: CGRect, scale: Double) -> Double {
1268
- let shorterSide = Double(min(frame.width, frame.height))
1269
- let frameRadius = shorterSide * 0.20
1270
- let minimumEndRadius = shorterSide * 0.08
1271
- let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius
1272
- return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
1273
- }
1274
-
1275
1321
  private func interactionRoot(app: XCUIApplication) -> XCUIElement {
1276
1322
  let windows = app.windows.allElementsBoundByIndex
1277
1323
  if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
@@ -85,6 +85,8 @@ extension RunnerTests {
85
85
  }
86
86
  currentApp = app
87
87
  currentBundleId = nil
88
+ currentAppProcessIdentifier = nil
89
+ snapshotXCTestPenaltyWarmupExemptionPending = false
88
90
  }
89
91
 
90
92
  func invalidateCachedTarget(reason: String) {
@@ -93,6 +95,48 @@ extension RunnerTests {
93
95
  }
94
96
  currentApp = nil
95
97
  currentBundleId = nil
98
+ currentAppProcessIdentifier = nil
99
+ snapshotXCTestPenaltyWarmupExemptionPending = false
100
+ }
101
+
102
+ func resetTargetAfterExternalRelaunch() -> Response {
103
+ invalidateCachedTarget(reason: "external_app_relaunch")
104
+ needsFirstInteractionDelay = true
105
+ return Response(ok: true, data: DataPayload(message: "target reset"))
106
+ }
107
+
108
+ func refreshCachedTargetIfProcessChanged(bundleId: String) {
109
+ guard currentBundleId == bundleId, currentApp != nil else { return }
110
+ let candidate = XCUIApplication(bundleIdentifier: bundleId)
111
+ let observedProcessIdentifier = Self.processIdentifier(of: candidate)
112
+ guard Self.shouldRefreshCachedTarget(
113
+ cachedProcessIdentifier: currentAppProcessIdentifier,
114
+ observedProcessIdentifier: observedProcessIdentifier
115
+ ) else { return }
116
+ NSLog(
117
+ "AGENT_DEVICE_RUNNER_TARGET_CACHE_REFRESH bundle=%@ previousPid=%d currentPid=%d",
118
+ bundleId,
119
+ currentAppProcessIdentifier ?? 0,
120
+ observedProcessIdentifier ?? 0
121
+ )
122
+ currentApp = candidate
123
+ currentAppProcessIdentifier = observedProcessIdentifier
124
+ clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
125
+ snapshotXCTestPenaltyWarmupExemptionPending = true
126
+ needsFirstInteractionDelay = true
127
+ }
128
+
129
+ static func processIdentifier(of target: XCUIApplication) -> Int? {
130
+ let value = RunnerAXSnapshotBridge.processIdentifier(for: target)
131
+ return value > 0 ? value : nil
132
+ }
133
+
134
+ static func shouldRefreshCachedTarget(
135
+ cachedProcessIdentifier: Int?,
136
+ observedProcessIdentifier: Int?
137
+ ) -> Bool {
138
+ guard let cachedProcessIdentifier, let observedProcessIdentifier else { return false }
139
+ return cachedProcessIdentifier != observedProcessIdentifier
96
140
  }
97
141
 
98
142
  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
@@ -131,16 +175,26 @@ extension RunnerTests {
131
175
 
132
176
  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
133
177
  let target = XCUIApplication(bundleIdentifier: bundleId)
178
+ let initialState = target.state
134
179
  NSLog(
135
180
  "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
136
181
  bundleId,
137
- target.state.rawValue,
182
+ initialState.rawValue,
138
183
  reason
139
184
  )
140
185
  // activate avoids terminating and relaunching the target app
141
- target.activate()
186
+ if initialState == .runningForeground {
187
+ NSLog(
188
+ "AGENT_DEVICE_RUNNER_ACTIVATE_SKIPPED bundle=%@ reason=already_foreground",
189
+ bundleId
190
+ )
191
+ } else {
192
+ target.activate()
193
+ }
142
194
  currentApp = target
143
195
  currentBundleId = bundleId
196
+ currentAppProcessIdentifier = Self.processIdentifier(of: target)
197
+ snapshotXCTestPenaltyWarmupExemptionPending = false
144
198
  needsFirstInteractionDelay = true
145
199
  return target
146
200
  }
@@ -0,0 +1,73 @@
1
+ import XCTest
2
+
3
+
4
+ extension RunnerTests {
5
+
6
+ func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
7
+ XCTAssertFalse(
8
+ Self.shouldRefreshCachedTarget(
9
+ cachedProcessIdentifier: nil,
10
+ observedProcessIdentifier: 42
11
+ )
12
+ )
13
+ XCTAssertFalse(
14
+ Self.shouldRefreshCachedTarget(
15
+ cachedProcessIdentifier: 42,
16
+ observedProcessIdentifier: 42
17
+ )
18
+ )
19
+ XCTAssertTrue(
20
+ Self.shouldRefreshCachedTarget(
21
+ cachedProcessIdentifier: 41,
22
+ observedProcessIdentifier: 42
23
+ )
24
+ )
25
+ }
26
+
27
+ func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
28
+ snapshotXCTestPenaltyWarmupExemptionPending = true
29
+
30
+ XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
31
+ XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
32
+ }
33
+
34
+ func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
35
+ penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
36
+ XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
37
+
38
+ clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
39
+
40
+ XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
41
+ }
42
+
43
+ func testCachedTargetInvalidationClearsProcessBoundState() {
44
+ currentApp = app
45
+ currentBundleId = "com.example.app"
46
+ currentAppProcessIdentifier = 42
47
+ snapshotXCTestPenaltyWarmupExemptionPending = true
48
+
49
+ invalidateCachedTarget(reason: "unit_test")
50
+
51
+ XCTAssertNil(currentApp)
52
+ XCTAssertNil(currentBundleId)
53
+ XCTAssertNil(currentAppProcessIdentifier)
54
+ XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
55
+ }
56
+
57
+ func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
58
+ currentApp = app
59
+ currentBundleId = "com.example.app"
60
+ currentAppProcessIdentifier = 42
61
+ snapshotXCTestPenaltyWarmupExemptionPending = true
62
+ needsFirstInteractionDelay = false
63
+
64
+ let response = resetTargetAfterExternalRelaunch()
65
+
66
+ XCTAssertTrue(response.ok)
67
+ XCTAssertNil(currentApp)
68
+ XCTAssertNil(currentBundleId)
69
+ XCTAssertNil(currentAppProcessIdentifier)
70
+ XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
71
+ XCTAssertTrue(needsFirstInteractionDelay)
72
+ }
73
+ }
@@ -24,14 +24,14 @@ enum CommandType: String, Codable {
24
24
  case keyboardDismiss
25
25
  case keyboardReturn
26
26
  case alert
27
- case pinch
28
27
  case sequence
29
- case rotateGesture
30
- case transformGesture
28
+ case gesture
29
+ case gestureViewport
31
30
  case recordStart
32
31
  case recordStop
33
32
  case status
34
33
  case uptime
34
+ case targetReset
35
35
  case shutdown
36
36
  }
37
37
 
@@ -74,11 +74,11 @@ extension CommandType {
74
74
  // .sequence is the fused multi-step gesture batch.
75
75
  case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll,
76
76
  .back, .backInApp, .backSystem, .rotate, .appSwitcher,
77
- .keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture:
77
+ .keyboardDismiss, .keyboardReturn, .sequence, .gesture:
78
78
  return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
79
79
 
80
80
  // Read-only reads: eligible for the session-invalidating retry.
81
- case .findText, .readText, .snapshot:
81
+ case .findText, .readText, .snapshot, .gestureViewport:
82
82
  return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
83
83
 
84
84
  // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
@@ -90,7 +90,7 @@ extension CommandType {
90
90
  return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
91
91
 
92
92
  // Runner-lifecycle commands: skip the app-activation preflight.
93
- case .recordStop, .uptime, .shutdown:
93
+ case .recordStop, .uptime, .targetReset, .shutdown:
94
94
  return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
95
95
 
96
96
  case .status:
@@ -126,16 +126,13 @@ struct Command: Codable {
126
126
  let remoteButton: String?
127
127
  let x2: Double?
128
128
  let y2: Double?
129
- let dx: Double?
130
- let dy: Double?
131
129
  let durationMs: Double?
130
+ let timeoutMs: Double?
132
131
  let direction: String?
133
132
  let amount: Double?
134
133
  let pixels: Double?
135
134
  let orientation: String?
136
- let scale: Double?
137
- let degrees: Double?
138
- let velocity: Double?
135
+ let gesturePlan: RunnerGesturePlan?
139
136
  let outPath: String?
140
137
  let fps: Int?
141
138
  let maxSize: Int?
@@ -148,6 +145,38 @@ struct Command: Codable {
148
145
  let steps: [SequenceStep]?
149
146
  }
150
147
 
148
+ /// Canonical one- or two-pointer plan produced by the portable TypeScript planner.
149
+ struct RunnerGesturePlan: Codable {
150
+ let topology: String
151
+ let intent: String
152
+ let executionProfile: String?
153
+ let durationMs: Double
154
+ let viewport: RunnerGestureViewport
155
+ let pointers: [RunnerGesturePointer]
156
+ }
157
+
158
+ struct RunnerGestureViewport: Codable {
159
+ let x: Double
160
+ let y: Double
161
+ let width: Double
162
+ let height: Double
163
+ }
164
+
165
+ struct RunnerGesturePointer: Codable {
166
+ let pointerId: Int
167
+ let samples: [RunnerGestureSample]
168
+ }
169
+
170
+ struct RunnerGestureSample: Codable {
171
+ let offsetMs: Double
172
+ let point: RunnerGesturePoint
173
+ }
174
+
175
+ struct RunnerGesturePoint: Codable {
176
+ let x: Double
177
+ let y: Double
178
+ }
179
+
151
180
  /// One allowlisted coordinate gesture step inside a fused `sequence` command.
152
181
  /// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
153
182
  /// INVALID_ARGS for an unknown kind instead of a generic decode failure.
@@ -155,11 +184,9 @@ struct SequenceStep: Codable {
155
184
  let kind: String
156
185
  let x: Double?
157
186
  let y: Double?
158
- let x2: Double?
159
- let y2: Double?
160
187
  let durationMs: Double?
161
188
  let pauseMs: Double?
162
- /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the
189
+ /// For `tap` steps on iOS non-tv: use the synthesized HID fast path instead of the
163
190
  /// drag-based XCUICoordinate path, matching the individual command behavior.
164
191
  let synthesized: Bool?
165
192
  }
@@ -231,6 +258,7 @@ struct DataPayload: Codable {
231
258
  let gestureFallback: String?
232
259
  let gestureFallbackMessage: String?
233
260
  let gestureFallbackHint: String?
261
+ let maestroNonHittableCoordinateFallbackUsed: Bool?
234
262
  let runnerFatal: Bool?
235
263
  let runnerFatalReason: String?
236
264
  let completedSteps: Int?
@@ -269,6 +297,7 @@ struct DataPayload: Codable {
269
297
  gestureFallback: String? = nil,
270
298
  gestureFallbackMessage: String? = nil,
271
299
  gestureFallbackHint: String? = nil,
300
+ maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
272
301
  runnerFatal: Bool? = nil,
273
302
  runnerFatalReason: String? = nil,
274
303
  completedSteps: Int? = nil,
@@ -306,6 +335,7 @@ struct DataPayload: Codable {
306
335
  self.gestureFallback = gestureFallback
307
336
  self.gestureFallbackMessage = gestureFallbackMessage
308
337
  self.gestureFallbackHint = gestureFallbackHint
338
+ self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
309
339
  self.runnerFatal = runnerFatal
310
340
  self.runnerFatalReason = runnerFatalReason
311
341
  self.completedSteps = completedSteps
@@ -1,10 +1,10 @@
1
1
  import XCTest
2
2
 
3
- // Swift port of buildScrollGesturePlan from src/core/scroll-gesture.ts.
3
+ // Swift port of buildScrollGesturePlan from src/contracts/scroll-gesture.ts.
4
4
  //
5
5
  // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android,
6
6
  // recording, and reported-pixels), and the runner places the gesture with this Swift copy. The
7
- // parity test vectors at the bottom of this file mirror src/core/__tests__/scroll-gesture.test.ts —
7
+ // parity test vectors at the bottom of this file mirror src/contracts/scroll-gesture.test.ts —
8
8
  // if you change the math in either language, update the other and both vector sets.
9
9
  //
10
10
  // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()`