agent-device 0.19.3 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +61 -70
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.manifest.json +10 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk.sha256 +1 -0
  7. package/{android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.manifest.json → android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.manifest.json} +6 -6
  8. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +2 -0
  9. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +5 -0
  10. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +5 -10
  11. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +74 -174
  12. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +113 -19
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +82 -0
  14. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +272 -118
  15. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +2 -2
  16. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +134 -101
  17. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +56 -2
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +73 -0
  19. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +44 -24
  20. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +2 -2
  21. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +2 -107
  22. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +126 -11
  23. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +51 -8
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +24 -0
  25. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +1 -1
  26. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +43 -35
  27. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +14 -0
  28. package/dist/src/agent-browser-tool.js +1 -1
  29. package/dist/src/agent-device-client.js +3 -0
  30. package/dist/src/android-adb.d.ts +1 -1
  31. package/dist/src/android-adb.js +1 -1
  32. package/dist/src/android-input-method-overlays.js +1 -0
  33. package/dist/src/android-input-ownership.js +1 -6
  34. package/dist/src/android-system-chrome.js +1 -0
  35. package/dist/src/android.js +1 -1
  36. package/dist/src/app-lifecycle.js +5 -5
  37. package/dist/src/app-log-request-scope.js +1 -0
  38. package/dist/src/app-resolution.js +1 -0
  39. package/dist/src/apple-multitouch-support.js +1 -0
  40. package/dist/src/apps.js +5 -5
  41. package/dist/src/boot-diagnostics.js +2 -0
  42. package/dist/src/cli-config.js +1 -0
  43. package/dist/src/cli-help.js +61 -27
  44. package/dist/src/cli-runner.js +1 -1
  45. package/dist/src/cli.js +15 -11
  46. package/dist/src/command-schema.js +3 -3
  47. package/dist/src/config.js +1 -0
  48. package/dist/src/context.js +1 -0
  49. package/dist/src/contracts.d.ts +1 -1
  50. package/dist/src/daemon-shutdown-report.js +1 -0
  51. package/dist/src/debug-symbols.js +1 -1
  52. package/dist/src/device-input-state.js +1 -1
  53. package/dist/src/devices2.js +1 -1
  54. package/dist/src/find.js +1 -1
  55. package/dist/src/finders.js +1 -1
  56. package/dist/src/generic.js +1 -1
  57. package/dist/src/helper.js +1 -1
  58. package/dist/src/index.d.ts +412 -293
  59. package/dist/src/index.js +1 -1
  60. package/dist/src/input-actions.js +4 -3
  61. package/dist/src/install-artifact.js +1 -0
  62. package/dist/src/install-source.d.ts +5 -1
  63. package/dist/src/install-source.js +1 -1
  64. package/dist/src/install-source2.js +1 -1
  65. package/dist/src/interaction-snapshot.js +1 -1
  66. package/dist/src/interaction.d.ts +230 -0
  67. package/dist/src/interaction.js +1 -1
  68. package/dist/src/interactor.js +2 -2
  69. package/dist/src/internal/bin.js +1 -1
  70. package/dist/src/internal/daemon.js +42 -20
  71. package/dist/src/internal/png-worker.d.ts +22 -0
  72. package/dist/src/internal/png-worker.js +1 -1
  73. package/dist/src/launch-console.js +1 -0
  74. package/dist/src/lease-context.js +1 -0
  75. package/dist/src/lease.js +1 -1
  76. package/dist/src/linux.js +1 -1
  77. package/dist/src/logcat.js +5 -0
  78. package/dist/src/metro.d.ts +2 -2
  79. package/dist/src/notifications.js +1 -1
  80. package/dist/src/png-worker-contract.js +1 -1
  81. package/dist/src/process-lock.js +1 -0
  82. package/dist/src/react-native.js +1 -1
  83. package/dist/src/record-trace.js +1 -1
  84. package/dist/src/register-builtins.js +1 -0
  85. package/dist/src/registry.js +24 -24
  86. package/dist/src/reporting.js +8 -7
  87. package/dist/src/runner-client.js +1 -1
  88. package/dist/src/runner-disposal.js +2 -0
  89. package/dist/src/runner-lease.js +1 -0
  90. package/dist/src/runner-sequence.js +1 -1
  91. package/dist/src/runtime.js +3 -3
  92. package/dist/src/runtime2.js +1 -0
  93. package/dist/src/sdk-android-adb.d.ts +119 -33
  94. package/dist/src/sdk-android-adb.js +2 -2
  95. package/dist/src/sdk-batch-runner.d.ts +2 -1
  96. package/dist/src/sdk-batch-runner.js +2 -2
  97. package/dist/src/sdk-batch.js +2 -2
  98. package/dist/src/sdk-contracts.d.ts +105 -98
  99. package/dist/src/sdk-contracts.js +1 -1
  100. package/dist/src/sdk-device.js +1 -1
  101. package/dist/src/sdk-finders.d.ts +3 -3
  102. package/dist/src/sdk-finders.js +1 -1
  103. package/dist/src/sdk-io.d.ts +1 -1
  104. package/dist/src/sdk-remote-config.d.ts +9 -1
  105. package/dist/src/sdk-remote-config.js +1 -1
  106. package/dist/src/sdk-selectors.js +1 -1
  107. package/dist/src/selector-runtime.js +1 -1
  108. package/dist/src/selector-vocabulary.js +1 -0
  109. package/dist/src/selectors.d.ts +12 -3
  110. package/dist/src/selectors.js +1 -1
  111. package/dist/src/server.js +3 -3
  112. package/dist/src/session-target-evidence.js +1 -0
  113. package/dist/src/session.js +8 -10
  114. package/dist/src/simulator.js +1 -1
  115. package/dist/src/snapshot-diagnostics.js +5 -2
  116. package/dist/src/snapshot.js +1 -1
  117. package/dist/src/snapshot2.js +1 -2
  118. package/dist/src/tv-remote.js +13 -12
  119. package/dist/src/url.js +1 -1
  120. package/dist/src/web.js +1 -1
  121. package/package.json +28 -23
  122. package/server.json +2 -2
  123. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.apk +0 -0
  124. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.apk.sha256 +0 -1
  125. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.manifest.json +0 -10
  126. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.apk +0 -0
  127. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.apk.sha256 +0 -1
  128. package/dist/src/apple-runner-options.js +0 -1
  129. package/dist/src/cli-output.js +0 -1
  130. package/dist/src/client.js +0 -3
  131. package/dist/src/handler-utils.js +0 -1
  132. package/dist/src/output-path.js +0 -1
  133. package/dist/src/points.js +0 -4
  134. package/dist/src/provider.js +0 -1
  135. package/dist/src/providers.js +0 -1
  136. package/dist/src/record-trace-recording.js +0 -23
  137. package/dist/src/recording-provider.js +0 -1
  138. package/dist/src/selectors2.js +0 -1
  139. package/dist/src/simctl.js +0 -1
  140. package/dist/src/ttl-memo.js +0 -1
  141. package/dist/src/xml.js +0 -2
  142. /package/{macos-helper → apple/macos-helper}/Package.swift +0 -0
  143. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +0 -0
  144. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +0 -0
  145. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/main.swift +0 -0
  146. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +0 -0
  147. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +0 -0
  148. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -0
  149. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +0 -0
  150. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +0 -0
  151. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +0 -0
  152. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.h +0 -0
  153. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.m +0 -0
  154. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +0 -0
  155. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +0 -0
  156. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +0 -0
  157. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +0 -0
  158. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +0 -0
  159. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +0 -0
  160. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +0 -0
  161. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +0 -0
  162. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +0 -0
  163. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +0 -0
  164. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +0 -0
  165. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +0 -0
  166. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -0
  167. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-trim.swift +0 -0
  168. /package/{src/platforms/linux → linux}/atspi-dump.py +0 -0
@@ -13,6 +13,16 @@ private enum RunnerInterfaceOrientation {
13
13
  }
14
14
 
15
15
  extension RunnerTests {
16
+ enum PlannedGestureExecution: Equatable {
17
+ case fastSwipe
18
+ case sampled
19
+ }
20
+
21
+ enum SynthesizedDragProfile {
22
+ case continuous
23
+ case fastSwipe
24
+ }
25
+
16
26
  struct TouchVisualizationFrame {
17
27
  let x: Double
18
28
  let y: Double
@@ -139,7 +149,8 @@ extension RunnerTests {
139
149
  app: XCUIApplication,
140
150
  selectorKey: String,
141
151
  selectorValue: String,
142
- allowNonHittableFallback: Bool = false
152
+ allowNonHittableFallback: Bool = false,
153
+ expectedPoint: CGPoint? = nil
143
154
  ) -> SelectorElementMatch {
144
155
  let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
145
156
  guard !value.isEmpty else {
@@ -163,6 +174,9 @@ extension RunnerTests {
163
174
  var nonHittableElement: XCUIElement?
164
175
  let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
165
176
  for element in matches where element.exists {
177
+ if let expectedPoint, !element.frame.contains(expectedPoint) {
178
+ continue
179
+ }
166
180
  if !element.isHittable {
167
181
  if allowNonHittableFallback && hasTappableFrame(app: app, element: element) {
168
182
  guard nonHittableElement == nil else {
@@ -775,7 +789,7 @@ extension RunnerTests {
775
789
  x2: Double,
776
790
  y2: Double,
777
791
  durationMs: Double,
778
- semantics: SynthesizedDragSemantics?,
792
+ profile: SynthesizedDragProfile = .continuous,
779
793
  context: SynthesizedCoordinateContext? = nil
780
794
  ) -> RunnerInteractionOutcome {
781
795
  #if os(iOS)
@@ -795,8 +809,9 @@ extension RunnerTests {
795
809
  let frame = context.referenceFrame
796
810
  let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
797
811
  let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
798
- let message = if semantics == .swipe {
799
- RunnerSynthesizedGesture.synthesizeSwipe(
812
+ let message = switch profile {
813
+ case .continuous:
814
+ RunnerSynthesizedGesture.synthesizeContinuousDrag(
800
815
  withApplication: app,
801
816
  x: Double(start.x),
802
817
  y: Double(start.y),
@@ -804,8 +819,8 @@ extension RunnerTests {
804
819
  y2: Double(end.y),
805
820
  durationMs: durationMs
806
821
  )
807
- } else {
808
- RunnerSynthesizedGesture.synthesizeContinuousDrag(
822
+ case .fastSwipe:
823
+ RunnerSynthesizedGesture.synthesizeSwipe(
809
824
  withApplication: app,
810
825
  x: Double(start.x),
811
826
  y: Double(start.y),
@@ -1166,125 +1181,143 @@ extension RunnerTests {
1166
1181
  }
1167
1182
  }
1168
1183
 
1169
- func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
1170
- #if os(iOS)
1171
- // A coordinate tap+drag is a single-finger gesture: React Native reads it as a pan
1172
- // and the pinch scale never changes (#629). Drive the two-finger XCTest synthesis
1173
- // path (the same one transformGesture uses) with zero translation/rotation so RN's
1174
- // pinch recognizer actually fires.
1175
- let frame = interactionRoot(app: app).frame
1176
- let centerX = x ?? Double(frame.midX)
1177
- let centerY = y ?? Double(frame.midY)
1178
- return transformGesture(
1179
- app: app,
1180
- x: centerX,
1181
- y: centerY,
1182
- dx: 0,
1183
- dy: 0,
1184
- scale: scale,
1185
- degrees: 0,
1186
- durationMs: 300
1187
- )
1188
- #elseif os(tvOS)
1189
- return .unsupported(
1190
- message: "pinch is not supported on tvOS",
1191
- hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)."
1192
- )
1193
- #else
1194
- return .unsupported(
1195
- message: "pinch is not supported on macOS",
1196
- hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)."
1197
- )
1198
- #endif
1184
+ func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? {
1185
+ guard plan.topology == "single" || plan.topology == "two" else {
1186
+ return "planned gesture topology must be single or two"
1187
+ }
1188
+ let supportedIntent = plan.topology == "single"
1189
+ ? plan.intent == "fling" || plan.intent == "pan"
1190
+ : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate"
1191
+ || plan.intent == "transform"
1192
+ guard supportedIntent else { return "planned gesture has unsupported intent for its topology" }
1193
+ if plan.topology == "single" {
1194
+ guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else {
1195
+ return "single-pointer gesture requires a supported execution profile"
1196
+ }
1197
+ } else if plan.executionProfile != nil {
1198
+ return "multi-touch gesture cannot define a single-pointer execution profile"
1199
+ }
1200
+ guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else {
1201
+ return "planned gesture durationMs must be between 16 and 10000"
1202
+ }
1203
+ let viewport = plan.viewport
1204
+ guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite,
1205
+ viewport.height.isFinite, viewport.width > 0, viewport.height > 0
1206
+ else {
1207
+ return "planned gesture viewport must be finite and positive"
1208
+ }
1209
+ let expectedPointerCount = plan.topology == "single" ? 1 : 2
1210
+ guard plan.pointers.count == expectedPointerCount else {
1211
+ return "planned gesture pointer count does not match topology"
1212
+ }
1213
+ for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index {
1214
+ return "planned gesture requires ordered pointer ids"
1215
+ }
1216
+ let firstSamples = plan.pointers[0].samples
1217
+ guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" }
1218
+ for pointer in plan.pointers {
1219
+ guard pointer.samples.count == firstSamples.count else {
1220
+ return "planned pointer paths require matching samples"
1221
+ }
1222
+ var previousOffset = -1.0
1223
+ for (index, sample) in pointer.samples.enumerated() {
1224
+ guard sample.offsetMs.isFinite,
1225
+ sample.offsetMs == firstSamples[index].offsetMs,
1226
+ sample.offsetMs > previousOffset
1227
+ else {
1228
+ return "planned pointer sample offsets must match and strictly increase"
1229
+ }
1230
+ let point = sample.point
1231
+ guard point.x.isFinite, point.y.isFinite,
1232
+ point.x >= viewport.x,
1233
+ point.x <= viewport.x + viewport.width,
1234
+ point.y >= viewport.y,
1235
+ point.y <= viewport.y + viewport.height
1236
+ else {
1237
+ return "planned pointer sample lies outside the viewport"
1238
+ }
1239
+ previousOffset = sample.offsetMs
1240
+ }
1241
+ guard pointer.samples.first?.offsetMs == 0,
1242
+ pointer.samples.last?.offsetMs == plan.durationMs
1243
+ else {
1244
+ return "planned pointer paths must start at 0 and end at durationMs"
1245
+ }
1246
+ }
1247
+ if plan.topology == "two" {
1248
+ guard let firstStart = firstSamples.first?.point,
1249
+ let secondStart = plan.pointers[1].samples.first?.point,
1250
+ hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0
1251
+ else {
1252
+ return "planned pointer paths require a positive initial span"
1253
+ }
1254
+ }
1255
+ return nil
1199
1256
  }
1200
1257
 
1201
- func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
1202
- #if os(iOS)
1203
- // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634)
1204
- // with zero translation/scale so React Native's rotation recognizer actually fires. The native
1205
- // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture
1206
- // handler does not read reliably — the same class of problem #629/#634 fixed for pinch.
1207
- // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract
1208
- // keeps it for compatibility and direction is carried entirely by the sign of `degrees`.
1209
- let frame = interactionRoot(app: app).frame
1210
- let centerX = x ?? Double(frame.midX)
1211
- let centerY = y ?? Double(frame.midY)
1212
- return transformGesture(
1213
- app: app,
1214
- x: centerX,
1215
- y: centerY,
1216
- dx: 0,
1217
- dy: 0,
1218
- scale: 1,
1219
- degrees: degrees,
1220
- durationMs: 300
1221
- )
1222
- #elseif os(tvOS)
1223
- return .unsupported(
1224
- message: "rotate-gesture is not supported on tvOS",
1225
- hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)."
1226
- )
1227
- #else
1228
- return .unsupported(
1229
- message: "rotate-gesture is not supported on macOS",
1230
- hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)."
1231
- )
1232
- #endif
1258
+ func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution {
1259
+ plan.topology == "single" && plan.executionProfile == "endpoint-hold"
1260
+ ? .fastSwipe
1261
+ : .sampled
1233
1262
  }
1234
1263
 
1235
- func transformGesture(
1264
+ func sampledPlannedGesture(
1236
1265
  app: XCUIApplication,
1237
- x: Double,
1238
- y: Double,
1239
- dx: Double,
1240
- dy: Double,
1241
- scale: Double,
1242
- degrees: Double,
1243
- durationMs: Double
1266
+ plan: RunnerGesturePlan
1244
1267
  ) -> RunnerInteractionOutcome {
1245
1268
  #if os(iOS)
1246
- let target = interactionRoot(app: app)
1247
1269
  let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
1248
- let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation)
1249
- let vector = nativeSynthesizedVector(orientedDx: dx, orientedDy: dy, interfaceOrientation: orientation)
1250
- if let message = RunnerSynthesizedGesture.synthesizeTransform(
1270
+ // The portable planner and validation use this exact viewport. Using app.frame here can
1271
+ // diverge when XCTest unions transformed/off-screen descendants into the application frame.
1272
+ let frame = CGRect(
1273
+ x: plan.viewport.x,
1274
+ y: plan.viewport.y,
1275
+ width: plan.viewport.width,
1276
+ height: plan.viewport.height
1277
+ )
1278
+ let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
1279
+ pointer.samples.map { sample in
1280
+ let point = nativeSynthesizedPoint(
1281
+ orientedX: sample.point.x,
1282
+ orientedY: sample.point.y,
1283
+ in: frame,
1284
+ interfaceOrientation: orientation
1285
+ )
1286
+ return [
1287
+ "x": NSNumber(value: Double(point.x)),
1288
+ "y": NSNumber(value: Double(point.y)),
1289
+ "offsetMs": NSNumber(value: sample.offsetMs),
1290
+ ]
1291
+ }
1292
+ }
1293
+ if let message = RunnerSynthesizedGesture.synthesizeGesture(
1251
1294
  withApplication: app,
1252
- x: Double(point.x),
1253
- y: Double(point.y),
1254
- dx: Double(vector.dx),
1255
- dy: Double(vector.dy),
1256
- scale: scale,
1257
- degrees: degrees,
1258
- radius: transformGestureRadius(frame: target.frame, scale: scale),
1259
- durationMs: durationMs
1295
+ pointerSamples: pointerSamples
1260
1296
  ) {
1261
1297
  return .unsupported(
1262
1298
  message: message,
1263
- hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)."
1299
+ hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists."
1264
1300
  )
1265
1301
  }
1266
1302
  return .performed
1267
1303
  #elseif os(tvOS)
1268
1304
  return .unsupported(
1269
- message: "transformGesture is not supported on tvOS",
1270
- hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)."
1305
+ message: "two-finger gestures are not supported on tvOS",
1306
+ hint: "tvOS has no touch input; use remote-driven navigation."
1307
+ )
1308
+ #elseif os(visionOS)
1309
+ return .unsupported(
1310
+ message: "two-finger touch gestures are not supported on visionOS",
1311
+ hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only."
1271
1312
  )
1272
1313
  #else
1273
1314
  return .unsupported(
1274
- message: "transformGesture is not supported on macOS",
1275
- hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)."
1315
+ message: "two-finger gestures are not supported on macOS",
1316
+ hint: "macOS automation has no multi-touch input; run on an iOS simulator."
1276
1317
  )
1277
1318
  #endif
1278
1319
  }
1279
1320
 
1280
- private func transformGestureRadius(frame: CGRect, scale: Double) -> Double {
1281
- let shorterSide = Double(min(frame.width, frame.height))
1282
- let frameRadius = shorterSide * 0.20
1283
- let minimumEndRadius = shorterSide * 0.08
1284
- let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius
1285
- return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
1286
- }
1287
-
1288
1321
  private func interactionRoot(app: XCUIApplication) -> XCUIElement {
1289
1322
  let windows = app.windows.allElementsBoundByIndex
1290
1323
  if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
@@ -85,6 +85,8 @@ extension RunnerTests {
85
85
  }
86
86
  currentApp = app
87
87
  currentBundleId = nil
88
+ currentAppProcessIdentifier = nil
89
+ snapshotXCTestPenaltyWarmupExemptionPending = false
88
90
  }
89
91
 
90
92
  func invalidateCachedTarget(reason: String) {
@@ -93,6 +95,48 @@ extension RunnerTests {
93
95
  }
94
96
  currentApp = nil
95
97
  currentBundleId = nil
98
+ currentAppProcessIdentifier = nil
99
+ snapshotXCTestPenaltyWarmupExemptionPending = false
100
+ }
101
+
102
+ func resetTargetAfterExternalRelaunch() -> Response {
103
+ invalidateCachedTarget(reason: "external_app_relaunch")
104
+ needsFirstInteractionDelay = true
105
+ return Response(ok: true, data: DataPayload(message: "target reset"))
106
+ }
107
+
108
+ func refreshCachedTargetIfProcessChanged(bundleId: String) {
109
+ guard currentBundleId == bundleId, currentApp != nil else { return }
110
+ let candidate = XCUIApplication(bundleIdentifier: bundleId)
111
+ let observedProcessIdentifier = Self.processIdentifier(of: candidate)
112
+ guard Self.shouldRefreshCachedTarget(
113
+ cachedProcessIdentifier: currentAppProcessIdentifier,
114
+ observedProcessIdentifier: observedProcessIdentifier
115
+ ) else { return }
116
+ NSLog(
117
+ "AGENT_DEVICE_RUNNER_TARGET_CACHE_REFRESH bundle=%@ previousPid=%d currentPid=%d",
118
+ bundleId,
119
+ currentAppProcessIdentifier ?? 0,
120
+ observedProcessIdentifier ?? 0
121
+ )
122
+ currentApp = candidate
123
+ currentAppProcessIdentifier = observedProcessIdentifier
124
+ clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
125
+ snapshotXCTestPenaltyWarmupExemptionPending = true
126
+ needsFirstInteractionDelay = true
127
+ }
128
+
129
+ static func processIdentifier(of target: XCUIApplication) -> Int? {
130
+ let value = RunnerAXSnapshotBridge.processIdentifier(for: target)
131
+ return value > 0 ? value : nil
132
+ }
133
+
134
+ static func shouldRefreshCachedTarget(
135
+ cachedProcessIdentifier: Int?,
136
+ observedProcessIdentifier: Int?
137
+ ) -> Bool {
138
+ guard let cachedProcessIdentifier, let observedProcessIdentifier else { return false }
139
+ return cachedProcessIdentifier != observedProcessIdentifier
96
140
  }
97
141
 
98
142
  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
@@ -131,16 +175,26 @@ extension RunnerTests {
131
175
 
132
176
  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
133
177
  let target = XCUIApplication(bundleIdentifier: bundleId)
178
+ let initialState = target.state
134
179
  NSLog(
135
180
  "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
136
181
  bundleId,
137
- target.state.rawValue,
182
+ initialState.rawValue,
138
183
  reason
139
184
  )
140
185
  // activate avoids terminating and relaunching the target app
141
- target.activate()
186
+ if initialState == .runningForeground {
187
+ NSLog(
188
+ "AGENT_DEVICE_RUNNER_ACTIVATE_SKIPPED bundle=%@ reason=already_foreground",
189
+ bundleId
190
+ )
191
+ } else {
192
+ target.activate()
193
+ }
142
194
  currentApp = target
143
195
  currentBundleId = bundleId
196
+ currentAppProcessIdentifier = Self.processIdentifier(of: target)
197
+ snapshotXCTestPenaltyWarmupExemptionPending = false
144
198
  needsFirstInteractionDelay = true
145
199
  return target
146
200
  }
@@ -0,0 +1,73 @@
1
+ import XCTest
2
+
3
+
4
+ extension RunnerTests {
5
+
6
+ func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
7
+ XCTAssertFalse(
8
+ Self.shouldRefreshCachedTarget(
9
+ cachedProcessIdentifier: nil,
10
+ observedProcessIdentifier: 42
11
+ )
12
+ )
13
+ XCTAssertFalse(
14
+ Self.shouldRefreshCachedTarget(
15
+ cachedProcessIdentifier: 42,
16
+ observedProcessIdentifier: 42
17
+ )
18
+ )
19
+ XCTAssertTrue(
20
+ Self.shouldRefreshCachedTarget(
21
+ cachedProcessIdentifier: 41,
22
+ observedProcessIdentifier: 42
23
+ )
24
+ )
25
+ }
26
+
27
+ func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
28
+ snapshotXCTestPenaltyWarmupExemptionPending = true
29
+
30
+ XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
31
+ XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
32
+ }
33
+
34
+ func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
35
+ penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
36
+ XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
37
+
38
+ clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
39
+
40
+ XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
41
+ }
42
+
43
+ func testCachedTargetInvalidationClearsProcessBoundState() {
44
+ currentApp = app
45
+ currentBundleId = "com.example.app"
46
+ currentAppProcessIdentifier = 42
47
+ snapshotXCTestPenaltyWarmupExemptionPending = true
48
+
49
+ invalidateCachedTarget(reason: "unit_test")
50
+
51
+ XCTAssertNil(currentApp)
52
+ XCTAssertNil(currentBundleId)
53
+ XCTAssertNil(currentAppProcessIdentifier)
54
+ XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
55
+ }
56
+
57
+ func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
58
+ currentApp = app
59
+ currentBundleId = "com.example.app"
60
+ currentAppProcessIdentifier = 42
61
+ snapshotXCTestPenaltyWarmupExemptionPending = true
62
+ needsFirstInteractionDelay = false
63
+
64
+ let response = resetTargetAfterExternalRelaunch()
65
+
66
+ XCTAssertTrue(response.ok)
67
+ XCTAssertNil(currentApp)
68
+ XCTAssertNil(currentBundleId)
69
+ XCTAssertNil(currentAppProcessIdentifier)
70
+ XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
71
+ XCTAssertTrue(needsFirstInteractionDelay)
72
+ }
73
+ }
@@ -24,23 +24,17 @@ enum CommandType: String, Codable {
24
24
  case keyboardDismiss
25
25
  case keyboardReturn
26
26
  case alert
27
- case pinch
28
27
  case sequence
29
- case rotateGesture
30
- case transformGesture
28
+ case gesture
29
+ case gestureViewport
31
30
  case recordStart
32
31
  case recordStop
33
32
  case status
34
33
  case uptime
34
+ case targetReset
35
35
  case shutdown
36
36
  }
37
37
 
38
- enum SynthesizedDragSemantics: String, Codable {
39
- case swipe
40
- case pan
41
- case fling
42
- }
43
-
44
38
  /// Runner command traits — see CONTEXT.md ("Runner command traits").
45
39
  ///
46
40
  /// Single source of truth for how the runner classifies a command across three
@@ -80,11 +74,11 @@ extension CommandType {
80
74
  // .sequence is the fused multi-step gesture batch.
81
75
  case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll,
82
76
  .back, .backInApp, .backSystem, .rotate, .appSwitcher,
83
- .keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture:
77
+ .keyboardDismiss, .keyboardReturn, .sequence, .gesture:
84
78
  return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
85
79
 
86
80
  // Read-only reads: eligible for the session-invalidating retry.
87
- case .findText, .readText, .snapshot:
81
+ case .findText, .readText, .snapshot, .gestureViewport:
88
82
  return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
89
83
 
90
84
  // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
@@ -96,7 +90,7 @@ extension CommandType {
96
90
  return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
97
91
 
98
92
  // Runner-lifecycle commands: skip the app-activation preflight.
99
- case .recordStop, .uptime, .shutdown:
93
+ case .recordStop, .uptime, .targetReset, .shutdown:
100
94
  return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
101
95
 
102
96
  case .status:
@@ -132,16 +126,13 @@ struct Command: Codable {
132
126
  let remoteButton: String?
133
127
  let x2: Double?
134
128
  let y2: Double?
135
- let dx: Double?
136
- let dy: Double?
137
129
  let durationMs: Double?
130
+ let timeoutMs: Double?
138
131
  let direction: String?
139
132
  let amount: Double?
140
133
  let pixels: Double?
141
134
  let orientation: String?
142
- let scale: Double?
143
- let degrees: Double?
144
- let velocity: Double?
135
+ let gesturePlan: RunnerGesturePlan?
145
136
  let outPath: String?
146
137
  let fps: Int?
147
138
  let maxSize: Int?
@@ -151,11 +142,41 @@ struct Command: Codable {
151
142
  let raw: Bool?
152
143
  let fullscreen: Bool?
153
144
  let synthesized: Bool?
154
- /// Preserves the public gesture's timing semantics after it is lowered to runner `drag`.
155
- let dragSemantics: SynthesizedDragSemantics?
156
145
  let steps: [SequenceStep]?
157
146
  }
158
147
 
148
+ /// Canonical one- or two-pointer plan produced by the portable TypeScript planner.
149
+ struct RunnerGesturePlan: Codable {
150
+ let topology: String
151
+ let intent: String
152
+ let executionProfile: String?
153
+ let durationMs: Double
154
+ let viewport: RunnerGestureViewport
155
+ let pointers: [RunnerGesturePointer]
156
+ }
157
+
158
+ struct RunnerGestureViewport: Codable {
159
+ let x: Double
160
+ let y: Double
161
+ let width: Double
162
+ let height: Double
163
+ }
164
+
165
+ struct RunnerGesturePointer: Codable {
166
+ let pointerId: Int
167
+ let samples: [RunnerGestureSample]
168
+ }
169
+
170
+ struct RunnerGestureSample: Codable {
171
+ let offsetMs: Double
172
+ let point: RunnerGesturePoint
173
+ }
174
+
175
+ struct RunnerGesturePoint: Codable {
176
+ let x: Double
177
+ let y: Double
178
+ }
179
+
159
180
  /// One allowlisted coordinate gesture step inside a fused `sequence` command.
160
181
  /// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
161
182
  /// INVALID_ARGS for an unknown kind instead of a generic decode failure.
@@ -163,15 +184,11 @@ struct SequenceStep: Codable {
163
184
  let kind: String
164
185
  let x: Double?
165
186
  let y: Double?
166
- let x2: Double?
167
- let y2: Double?
168
187
  let durationMs: Double?
169
188
  let pauseMs: Double?
170
- /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the
189
+ /// For `tap` steps on iOS non-tv: use the synthesized HID fast path instead of the
171
190
  /// drag-based XCUICoordinate path, matching the individual command behavior.
172
191
  let synthesized: Bool?
173
- /// Preserves one-shot swipe timing when repeated swipes are fused into a sequence.
174
- let dragSemantics: SynthesizedDragSemantics?
175
192
  }
176
193
 
177
194
  /// Per-step result for a `sequence` response. `ok:false` carries the failing step's
@@ -241,6 +258,7 @@ struct DataPayload: Codable {
241
258
  let gestureFallback: String?
242
259
  let gestureFallbackMessage: String?
243
260
  let gestureFallbackHint: String?
261
+ let maestroNonHittableCoordinateFallbackUsed: Bool?
244
262
  let runnerFatal: Bool?
245
263
  let runnerFatalReason: String?
246
264
  let completedSteps: Int?
@@ -279,6 +297,7 @@ struct DataPayload: Codable {
279
297
  gestureFallback: String? = nil,
280
298
  gestureFallbackMessage: String? = nil,
281
299
  gestureFallbackHint: String? = nil,
300
+ maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
282
301
  runnerFatal: Bool? = nil,
283
302
  runnerFatalReason: String? = nil,
284
303
  completedSteps: Int? = nil,
@@ -316,6 +335,7 @@ struct DataPayload: Codable {
316
335
  self.gestureFallback = gestureFallback
317
336
  self.gestureFallbackMessage = gestureFallbackMessage
318
337
  self.gestureFallbackHint = gestureFallbackHint
338
+ self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
319
339
  self.runnerFatal = runnerFatal
320
340
  self.runnerFatalReason = runnerFatalReason
321
341
  self.completedSteps = completedSteps
@@ -1,10 +1,10 @@
1
1
  import XCTest
2
2
 
3
- // Swift port of buildScrollGesturePlan from src/core/scroll-gesture.ts.
3
+ // Swift port of buildScrollGesturePlan from src/contracts/scroll-gesture.ts.
4
4
  //
5
5
  // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android,
6
6
  // recording, and reported-pixels), and the runner places the gesture with this Swift copy. The
7
- // parity test vectors at the bottom of this file mirror src/core/__tests__/scroll-gesture.test.ts —
7
+ // parity test vectors at the bottom of this file mirror src/contracts/scroll-gesture.test.ts —
8
8
  // if you change the math in either language, update the other and both vector sets.
9
9
  //
10
10
  // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()`