agent-device 0.21.0 → 0.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (328) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.0.apk → agent-device-android-ime-helper-0.21.2.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.2.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.0.manifest.json → agent-device-android-ime-helper-0.21.2.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.2.apk +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.2.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.0.manifest.json → agent-device-android-snapshot-helper-0.21.2.manifest.json} +6 -6
  7. package/apple/snapshot-bridge/README.md +58 -1
  8. package/apple/snapshot-bridge/SnapshotBridge.m +6 -0
  9. package/apple/snapshot-bridge/SnapshotBridgeCapture.h +25 -0
  10. package/apple/snapshot-bridge/SnapshotBridgeCapture.m +138 -0
  11. package/apple/snapshot-bridge/SnapshotBridgeRuntime.h +1 -0
  12. package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +32 -3
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +12 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +20 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +413 -0
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +15 -15
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +72 -0
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +63 -14
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +668 -82
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +48 -4
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +39 -4
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +1078 -211
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +290 -0
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +1 -1
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +10 -10
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +250 -74
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +101 -42
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +13 -13
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +55 -43
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +71 -4
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +144 -7
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +13 -1
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +216 -17
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +206 -22
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +289 -42
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +92 -22
  37. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +617 -122
  38. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +79 -4
  39. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +71 -71
  40. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +92 -6
  41. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +67 -67
  42. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +4 -4
  43. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +17 -17
  44. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +30 -30
  45. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +31 -31
  46. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +28 -0
  47. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +101 -0
  48. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +19 -19
  49. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +55 -17
  50. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +26 -5
  51. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +126 -75
  52. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +8 -12
  53. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +9 -9
  54. package/dist/src/absence-observation-errors.js +1 -0
  55. package/dist/src/adapter.js +1 -1
  56. package/dist/src/adb-executor.js +1 -1
  57. package/dist/src/adb-failure.js +1 -1
  58. package/dist/src/agent-browser-lifecycle.js +1 -1
  59. package/dist/src/agent-browser-process-record.js +1 -1
  60. package/dist/src/agent-browser-provider.js +1 -1
  61. package/dist/src/agent-browser-tool.js +1 -1
  62. package/dist/src/agent-device-client.js +2 -2
  63. package/dist/src/ai-sdk.d.ts +1 -1
  64. package/dist/src/ai-sdk.js +1 -1
  65. package/dist/src/alert-runtime.js +1 -1
  66. package/dist/src/android-input-ownership.js +1 -1
  67. package/dist/src/app-catalog.js +1 -1
  68. package/dist/src/app-deployment-resolution.js +1 -1
  69. package/dist/src/app-helpers.js +1 -1
  70. package/dist/src/app-inventory-contract.js +1 -1
  71. package/dist/src/app-inventory.js +1 -1
  72. package/dist/src/app-launch.js +2 -2
  73. package/dist/src/app-lifecycle.js +3 -3
  74. package/dist/src/app-lifecycle2.js +1 -1
  75. package/dist/src/app-log-files.js +1 -1
  76. package/dist/src/app-log-runtime2.js +3 -3
  77. package/dist/src/app-resolution.js +1 -1
  78. package/dist/src/apple-runner-platform.js +1 -0
  79. package/dist/src/application-lifecycle-interaction.js +1 -1
  80. package/dist/src/archive.js +1 -1
  81. package/dist/src/artifact-download.js +1 -1
  82. package/dist/src/auth-session.js +1 -1
  83. package/dist/src/auth.js +1 -1
  84. package/dist/src/back-mode.js +1 -1
  85. package/dist/src/backend-snapshot-options.js +1 -1
  86. package/dist/src/boot-diagnostics.js +1 -1
  87. package/dist/src/browser-launch.js +1 -0
  88. package/dist/src/capture.js +1 -1
  89. package/dist/src/catalog.js +1 -1
  90. package/dist/src/cli-config.js +1 -1
  91. package/dist/src/cli-help.js +30 -26
  92. package/dist/src/cli.js +4 -4
  93. package/dist/src/click-button.js +1 -1
  94. package/dist/src/client-metro-companion.js +1 -1
  95. package/dist/src/client-metro.js +1 -1
  96. package/dist/src/client-types.d.ts +28 -13
  97. package/dist/src/code-signature-cache.js +1 -1
  98. package/dist/src/code-signature.js +1 -1
  99. package/dist/src/command-metadata.js +1 -1
  100. package/dist/src/command-schema.js +3 -3
  101. package/dist/src/command-tools.js +2 -2
  102. package/dist/src/command.js +1 -1
  103. package/dist/src/config.js +1 -1
  104. package/dist/src/connection-runtime.js +1 -1
  105. package/dist/src/connection.js +3 -3
  106. package/dist/src/contracts.d.ts +1 -1
  107. package/dist/src/contracts.js +1 -1
  108. package/dist/src/daemon-client-lifecycle.js +1 -1
  109. package/dist/src/daemon-process.js +1 -1
  110. package/dist/src/daemon-shutdown-report.js +1 -1
  111. package/dist/src/daemon.js +1 -1
  112. package/dist/src/debug-symbols-facade.js +1 -1
  113. package/dist/src/device-boot.js +1 -0
  114. package/dist/src/device-claim-inspection.js +1 -1
  115. package/dist/src/device-claim-rule.js +1 -1
  116. package/dist/src/device-input-state.d.ts +1 -1
  117. package/dist/src/device-inventory-context.js +1 -1
  118. package/dist/src/device-rotation.js +1 -1
  119. package/dist/src/device-selection-resolver.js +1 -1
  120. package/dist/src/device-session.js +1 -1
  121. package/dist/src/device2.js +1 -1
  122. package/dist/src/diagnostics.js +1 -1
  123. package/dist/src/dispatch-resolve.js +1 -1
  124. package/dist/src/doctor2.js +1 -1
  125. package/dist/src/durable-resource.js +1 -1
  126. package/dist/src/engine-eval-script.js +1 -0
  127. package/dist/src/error.js +1 -1
  128. package/dist/src/exec.js +1 -1
  129. package/dist/src/file.js +1 -1
  130. package/dist/src/find.js +1 -1
  131. package/dist/src/finders.js +1 -1
  132. package/dist/src/format.js +1 -2
  133. package/dist/src/generic-settle.js +1 -1
  134. package/dist/src/gesture-plan.js +1 -1
  135. package/dist/src/gesture-runtime.js +1 -1
  136. package/dist/src/harmonyos.js +1 -1
  137. package/dist/src/hdc.js +1 -1
  138. package/dist/src/helper.js +2 -2
  139. package/dist/src/human-control-contract.js +1 -1
  140. package/dist/src/human-control.js +1 -1
  141. package/dist/src/index.d.ts +1 -1
  142. package/dist/src/index.js +1 -1
  143. package/dist/src/input-actions.js +1 -1
  144. package/dist/src/input-actions2.js +2 -2
  145. package/dist/src/install-artifact-facade.js +1 -1
  146. package/dist/src/install-source-network-transport.js +1 -1
  147. package/dist/src/install-source2.js +1 -1
  148. package/dist/src/interaction-positionals.js +1 -1
  149. package/dist/src/interaction-ref-policy.js +1 -1
  150. package/dist/src/interaction.js +1 -1
  151. package/dist/src/interaction2.js +1 -1
  152. package/dist/src/interactor.js +5 -5
  153. package/dist/src/interactor2.js +1 -1
  154. package/dist/src/interactors.js +1 -1
  155. package/dist/src/internal/bin.js +1 -1
  156. package/dist/src/internal/daemon.js +2 -2
  157. package/dist/src/internal/png-worker.d.ts +3 -0
  158. package/dist/src/internal/png-worker.js +1 -1
  159. package/dist/src/internal/run-script-http-child.js +1 -1
  160. package/dist/src/inventory.js +2 -2
  161. package/dist/src/inventory2.js +1 -1
  162. package/dist/src/inventory3.js +1 -1
  163. package/dist/src/inventory5.js +1 -1
  164. package/dist/src/ios-snapshot-acquisition.js +1 -1
  165. package/dist/src/ios-snapshot-planning.js +1 -1
  166. package/dist/src/ios-snapshot-runtime.js +1 -1
  167. package/dist/src/ios-system-surface.js +1 -0
  168. package/dist/src/lease-scope.js +1 -1
  169. package/dist/src/lease.js +1 -1
  170. package/dist/src/limrun-runtime-dependencies.js +1 -1
  171. package/dist/src/limrun.d.ts +1 -1
  172. package/dist/src/linux.js +1 -1
  173. package/dist/src/local-interactor-operation-set.js +1 -1
  174. package/dist/src/location-coordinates.js +1 -1
  175. package/dist/src/managed-device-scope.js +1 -1
  176. package/dist/src/mechanics.js +16 -16
  177. package/dist/src/mobile-snapshot-semantics.js +1 -1
  178. package/dist/src/mp4-atoms.js +1 -0
  179. package/dist/src/multitouch-support.js +1 -1
  180. package/dist/src/observation.js +1 -1
  181. package/dist/src/open-target-resolution.js +1 -0
  182. package/dist/src/owner-identity.js +3 -3
  183. package/dist/src/parameterized-recorded-fill.js +1 -1
  184. package/dist/src/perf-facade.js +2 -2
  185. package/dist/src/perf-process-identity.js +1 -0
  186. package/dist/src/perf-runtime-operation-builder.js +1 -1
  187. package/dist/src/perf-runtime-plan.js +1 -1
  188. package/dist/src/perf.js +1 -1
  189. package/dist/src/physical-device-control.js +1 -1
  190. package/dist/src/platform-request-scope.js +1 -1
  191. package/dist/src/platform-runtime-android-adb-host.js +1 -1
  192. package/dist/src/platform-runtime-app-log-process.js +1 -1
  193. package/dist/src/platform-runtime-host.js +1 -1
  194. package/dist/src/platform-runtime-managed-owner.js +1 -1
  195. package/dist/src/platform-runtime-open-target.js +1 -1
  196. package/dist/src/platform-runtime-operation-host.js +2 -2
  197. package/dist/src/platform-runtime-runtime-hints.js +2 -2
  198. package/dist/src/platform-runtime-screen-recording-android-host.js +1 -1
  199. package/dist/src/platform-runtime-screen-recording-apple-runner-host.js +1 -1
  200. package/dist/src/platform-runtime-screen-recording-apple-runner-transport.js +1 -1
  201. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  202. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  203. package/dist/src/platform-runtime-screen-recording-process-host.js +1 -1
  204. package/dist/src/platform-runtime-unavailable.js +1 -1
  205. package/dist/src/platform-runtime.js +1 -1
  206. package/dist/src/platform-runtime2.js +1 -1
  207. package/dist/src/png-crop-bytes.js +1 -0
  208. package/dist/src/png-worker-contract.js +1 -1
  209. package/dist/src/png.js +1 -0
  210. package/dist/src/prepare-kind.js +1 -1
  211. package/dist/src/process.js +1 -1
  212. package/dist/src/provider-device-runtime.js +1 -1
  213. package/dist/src/provider-session-ownership.js +1 -1
  214. package/dist/src/provider-webdriver.js +1 -1
  215. package/dist/src/proxy.js +1 -1
  216. package/dist/src/quality-warnings.js +1 -0
  217. package/dist/src/react-native.js +1 -1
  218. package/dist/src/record-runtime.js +1 -1
  219. package/dist/src/rect-center.js +1 -1
  220. package/dist/src/register-builtins.js +1 -1
  221. package/dist/src/registry.js +36 -36
  222. package/dist/src/remote-config2.js +1 -1
  223. package/dist/src/replay-divergence.js +1 -1
  224. package/dist/src/replay.js +2 -2
  225. package/dist/src/reporting.js +9 -8
  226. package/dist/src/request-admission.js +1 -1
  227. package/dist/src/request-cancel.js +1 -1
  228. package/dist/src/request-cancellation.js +1 -1
  229. package/dist/src/request-runtime-binding.js +1 -1
  230. package/dist/src/retry.js +1 -1
  231. package/dist/src/runner-cache-metadata.js +2 -2
  232. package/dist/src/runner-client.js +1 -1
  233. package/dist/src/runner-disposal.js +1 -1
  234. package/dist/src/runner-operations-facade.js +1 -1
  235. package/dist/src/runner-presentation.js +1 -1
  236. package/dist/src/runner.js +1 -1
  237. package/dist/src/runtime-transport-hints.js +1 -1
  238. package/dist/src/runtime.js +3 -3
  239. package/dist/src/runtime10.js +1 -0
  240. package/dist/src/runtime2.js +4 -4
  241. package/dist/src/runtime3.js +1 -1
  242. package/dist/src/runtime4.js +5 -4
  243. package/dist/src/runtime5.js +1 -1
  244. package/dist/src/runtime6.js +1 -1
  245. package/dist/src/runtime7.js +1 -1
  246. package/dist/src/runtime8.js +1 -1
  247. package/dist/src/runtime9.js +1 -1
  248. package/dist/src/screen-recording-live-handle.js +1 -1
  249. package/dist/src/screen-recording-options.js +1 -1
  250. package/dist/src/screen-recording-resource-recovery.js +1 -1
  251. package/dist/src/screenshot-density.js +1 -1
  252. package/dist/src/screenshot-runtime.js +1 -1
  253. package/dist/src/screenshot.js +1 -1
  254. package/dist/src/screenshot3.js +3 -3
  255. package/dist/src/script.js +1 -1
  256. package/dist/src/scroll-command.js +1 -1
  257. package/dist/src/scroll-gesture.js +1 -1
  258. package/dist/src/sdk-batch-runner.js +2 -2
  259. package/dist/src/sdk-batch.js +2 -2
  260. package/dist/src/sdk-contracts.d.ts +69 -26
  261. package/dist/src/sdk-contracts.js +2 -2
  262. package/dist/src/sdk-device.js +1 -1
  263. package/dist/src/sdk-finders.d.ts +1 -1
  264. package/dist/src/sdk-finders.js +1 -1
  265. package/dist/src/sdk-io.js +1 -1
  266. package/dist/src/sdk-remote-config.d.ts +1 -1
  267. package/dist/src/sdk-remote-config.js +1 -1
  268. package/dist/src/sdk-selectors.d.ts +80 -16
  269. package/dist/src/sdk-selectors.js +1 -1
  270. package/dist/src/selection.js +0 -0
  271. package/dist/src/selector-runtime.js +1 -1
  272. package/dist/src/selectors.js +1 -1
  273. package/dist/src/semantic-index.js +1 -1
  274. package/dist/src/server.js +3 -3
  275. package/dist/src/session-allocation.js +1 -1
  276. package/dist/src/session-observation.js +1 -0
  277. package/dist/src/session-routing.js +1 -1
  278. package/dist/src/session-runtime-admission.js +1 -1
  279. package/dist/src/session-snapshot.js +1 -1
  280. package/dist/src/session-store.js +3 -3
  281. package/dist/src/session-target-evidence.js +1 -1
  282. package/dist/src/session2.js +5 -5
  283. package/dist/src/settings.js +1 -1
  284. package/dist/src/simulator-boot.js +3 -0
  285. package/dist/src/simulator.js +1 -1
  286. package/dist/src/snapshot-quality-backend-capabilities.js +1 -1
  287. package/dist/src/snapshot-runtime-binding.js +1 -1
  288. package/dist/src/snapshot-runtime.js +1 -1
  289. package/dist/src/snapshot-state.js +1 -1
  290. package/dist/src/snapshot2.js +1 -1
  291. package/dist/src/snapshot3.js +1 -1
  292. package/dist/src/snapshot4.js +1 -1
  293. package/dist/src/src.js +1 -1
  294. package/dist/src/src7.js +2 -2
  295. package/dist/src/src8.js +2 -2
  296. package/dist/src/string-enum.js +1 -1
  297. package/dist/src/takeover.js +1 -1
  298. package/dist/src/telemetry.js +1 -1
  299. package/dist/src/tool-provider.js +2 -2
  300. package/dist/src/tool-provider2.js +1 -1
  301. package/dist/src/tool-provider3.js +1 -1
  302. package/dist/src/toolchain-probe.js +1 -1
  303. package/dist/src/touch-reference-frame.js +1 -0
  304. package/dist/src/touch-runtime.js +1 -1
  305. package/dist/src/transport.js +1 -1
  306. package/dist/src/tree.js +1 -1
  307. package/dist/src/tv-remote.js +1 -1
  308. package/dist/src/type-text-runtime.js +1 -1
  309. package/dist/src/update-check.js +1 -1
  310. package/dist/src/vars.js +2 -0
  311. package/dist/src/verified-file.js +1 -1
  312. package/dist/src/version.js +1 -1
  313. package/dist/src/video.js +2 -2
  314. package/dist/src/viewport.js +1 -0
  315. package/dist/src/web.js +1 -1
  316. package/dist/src/web2.js +1 -1
  317. package/dist/src/webdriver-ios-snapshot.js +1 -1
  318. package/dist/src/webdriver-source.js +1 -1
  319. package/dist/src/window-state.js +1 -0
  320. package/package.json +4 -2
  321. package/server.json +2 -2
  322. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.0.apk.sha256 +0 -1
  323. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.0.apk +0 -0
  324. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.0.apk.sha256 +0 -1
  325. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift +0 -93
  326. package/dist/src/platform-runtime-apple-resources.js +0 -1
  327. package/dist/src/runner-selector-query.js +0 -1
  328. package/dist/src/snapshot-quality-verdict.js +0 -1
@@ -2,8 +2,32 @@ import XCTest
2
2
  import AgentDeviceSnapshotPresentation
3
3
 
4
4
 
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
5
29
  extension RunnerTests {
6
- // MARK: - Main Thread Dispatch
30
+
7
31
 
8
32
  private func currentUptimeMs() -> Double {
9
33
  ProcessInfo.processInfo.systemUptime * 1000
@@ -35,7 +59,7 @@ extension RunnerTests {
35
59
  }
36
60
  }
37
61
 
38
- /// Optional visualization frame returned with a gesture response.
62
+
39
63
  enum GestureFrame {
40
64
  case none
41
65
  case touch(TouchVisualizationFrame?)
@@ -58,111 +82,861 @@ extension RunnerTests {
58
82
  }
59
83
 
60
84
 
61
- /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
62
- /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis
63
- /// pointer-plan gestures pass `false` because RunnerSynthesizedGesture governs their
64
- /// own timing. Returns the captured timing and the action's outcome.
65
- ///
66
- /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap
67
- /// it in the scroll idle-timeout/quiescence-skip path and change its runtime behavior.
68
- func performGesture(
69
- _ app: XCUIApplication,
70
- idleTimeout: Bool = true,
71
- _ action: () -> RunnerInteractionOutcome
72
- ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) {
73
- var outcome = RunnerInteractionOutcome.performed
74
- let timing = measureGesture {
75
- if idleTimeout {
76
- withTemporaryScrollIdleTimeoutIfSupported(app) { outcome = action() }
77
- } else {
78
- outcome = action()
79
- }
80
- }
81
- return (timing, outcome)
82
- }
83
85
 
84
- /// Single factory for the success payload every gesture returns (message + gesture timing +
85
- /// an optional touch/drag visualization frame), so the field shape lives in one place.
86
- private func gestureResponse(
87
- message: String,
88
- timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
89
- frame: GestureFrame = .none,
90
- fallback: GestureFallback? = nil,
91
- maestroNonHittableCoordinateFallbackUsed: Bool? = nil
92
- ) -> Response {
93
- let data: DataPayload
94
- switch frame {
95
- case .none:
96
- data = DataPayload(
97
- message: message,
98
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
99
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
100
- gestureFallback: fallback?.strategy,
101
- gestureFallbackMessage: fallback?.message,
102
- gestureFallbackHint: fallback?.hint,
103
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
104
- )
105
- case .touch(let f):
106
- data = DataPayload(
107
- message: message,
108
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
109
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
110
- x: f?.x,
111
- y: f?.y,
112
- referenceWidth: f?.referenceWidth,
113
- referenceHeight: f?.referenceHeight,
114
- gestureFallback: fallback?.strategy,
115
- gestureFallbackMessage: fallback?.message,
116
- gestureFallbackHint: fallback?.hint,
117
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
118
- )
119
- case .drag(let f):
120
- data = DataPayload(
121
- message: message,
122
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
123
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
124
- x: f.x,
125
- y: f.y,
126
- x2: f.x2,
127
- y2: f.y2,
128
- referenceWidth: f.referenceWidth,
129
- referenceHeight: f.referenceHeight,
130
- gestureFallback: fallback?.strategy,
131
- gestureFallbackMessage: fallback?.message,
132
- gestureFallbackHint: fallback?.hint
133
- )
134
- }
135
- return Response(ok: true, data: data)
136
- }
137
86
 
138
- /// Gesture plans already return canonical centroid endpoints from the portable runtime.
139
- /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's
140
- /// visualization frame into only the fast-fling response shape.
141
- private func canonicalPlannedGestureResponse(_ response: Response) -> Response {
142
- guard response.ok, let data = response.data else { return response }
143
- return Response(
144
- ok: true,
145
- data: DataPayload(
146
- message: data.message,
147
- gestureStartUptimeMs: data.gestureStartUptimeMs,
148
- gestureEndUptimeMs: data.gestureEndUptimeMs,
149
- gestureFallback: data.gestureFallback,
150
- gestureFallbackMessage: data.gestureFallbackMessage,
151
- gestureFallbackHint: data.gestureFallbackHint
152
- )
153
- )
154
- }
155
87
 
156
- private func plannedGestureResponse(
157
- plan: RunnerGesturePlan,
158
- timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
159
- outcome: RunnerInteractionOutcome
160
- ) -> Response {
161
- if let response = unsupportedResponse(for: outcome) {
162
- return response
163
- }
164
- return gestureResponse(message: plan.intent, timing: timing)
165
- }
88
+
89
+
90
+
91
+
92
+ func performGesture(
93
+ _ app: XCUIApplication,
94
+ idleTimeout: Bool = true,
95
+ _ action: () -> RunnerInteractionOutcome
96
+ ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) {
97
+ var outcome = RunnerInteractionOutcome.performed
98
+ let timing = measureGesture {
99
+ if idleTimeout {
100
+ withTemporaryScrollIdleTimeoutIfSupported(app) { outcome = action() }
101
+ } else {
102
+ outcome = action()
103
+ }
104
+ }
105
+ return (timing, outcome)
106
+ }
107
+
108
+
109
+
110
+ private func gestureResponse(
111
+ message: String,
112
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
113
+ frame: GestureFrame = .none,
114
+ fallback: GestureFallback? = nil,
115
+ maestroNonHittableCoordinateFallbackUsed: Bool? = nil
116
+ ) -> Response {
117
+ let data: DataPayload
118
+ switch frame {
119
+ case .none:
120
+ data = DataPayload(
121
+ message: message,
122
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
123
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
124
+ gestureFallback: fallback?.strategy,
125
+ gestureFallbackMessage: fallback?.message,
126
+ gestureFallbackHint: fallback?.hint,
127
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
128
+ )
129
+ case .touch(let f):
130
+ data = DataPayload(
131
+ message: message,
132
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
133
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
134
+ x: f?.x,
135
+ y: f?.y,
136
+ referenceWidth: f?.referenceWidth,
137
+ referenceHeight: f?.referenceHeight,
138
+ gestureFallback: fallback?.strategy,
139
+ gestureFallbackMessage: fallback?.message,
140
+ gestureFallbackHint: fallback?.hint,
141
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
142
+ )
143
+ case .drag(let f):
144
+ data = DataPayload(
145
+ message: message,
146
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
147
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
148
+ x: f.x,
149
+ y: f.y,
150
+ x2: f.x2,
151
+ y2: f.y2,
152
+ referenceWidth: f.referenceWidth,
153
+ referenceHeight: f.referenceHeight,
154
+ gestureFallback: fallback?.strategy,
155
+ gestureFallbackMessage: fallback?.message,
156
+ gestureFallbackHint: fallback?.hint
157
+ )
158
+ }
159
+ return Response(ok: true, data: data)
160
+ }
161
+
162
+
163
+
164
+
165
+ private func canonicalPlannedGestureResponse(_ response: Response) -> Response {
166
+ guard response.ok, let data = response.data else { return response }
167
+ return Response(
168
+ ok: true,
169
+ data: DataPayload(
170
+ message: data.message,
171
+ gestureStartUptimeMs: data.gestureStartUptimeMs,
172
+ gestureEndUptimeMs: data.gestureEndUptimeMs,
173
+ gestureFallback: data.gestureFallback,
174
+ gestureFallbackMessage: data.gestureFallbackMessage,
175
+ gestureFallbackHint: data.gestureFallbackHint
176
+ )
177
+ )
178
+ }
179
+
180
+ private func plannedGestureResponse(
181
+ plan: RunnerGesturePlan,
182
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
183
+ outcome: RunnerInteractionOutcome
184
+ ) -> Response {
185
+ if let response = unsupportedResponse(for: outcome) {
186
+ return response
187
+ }
188
+ return gestureResponse(message: plan.intent, timing: timing)
189
+ }
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+
204
+
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+
233
+
234
+
235
+
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
+
255
+
256
+
257
+
258
+
259
+
260
+
261
+
262
+
263
+
264
+
265
+
266
+
267
+
268
+
269
+
270
+
271
+
272
+
273
+
274
+
275
+
276
+
277
+
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+
292
+
293
+
294
+
295
+
296
+
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+
307
+
308
+
309
+
310
+
311
+
312
+
313
+
314
+
315
+
316
+
317
+
318
+
319
+
320
+
321
+
322
+
323
+
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+
333
+
334
+
335
+
336
+
337
+
338
+
339
+
340
+
341
+
342
+
343
+
344
+
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+
386
+
387
+
388
+
389
+
390
+
391
+
392
+
393
+
394
+
395
+
396
+
397
+
398
+
399
+
400
+
401
+
402
+
403
+
404
+
405
+
406
+
407
+
408
+
409
+
410
+
411
+
412
+
413
+
414
+
415
+
416
+
417
+
418
+
419
+
420
+
421
+
422
+
423
+
424
+
425
+
426
+
427
+
428
+
429
+
430
+
431
+
432
+
433
+
434
+
435
+
436
+
437
+
438
+
439
+
440
+
441
+
442
+
443
+
444
+
445
+
446
+
447
+
448
+
449
+
450
+
451
+
452
+
453
+
454
+
455
+
456
+
457
+
458
+
459
+
460
+
461
+
462
+
463
+
464
+
465
+
466
+
467
+
468
+
469
+
470
+
471
+
472
+
473
+
474
+
475
+
476
+
477
+
478
+
479
+
480
+
481
+
482
+
483
+
484
+
485
+
486
+
487
+
488
+
489
+
490
+
491
+
492
+
493
+
494
+
495
+
496
+
497
+
498
+
499
+
500
+
501
+
502
+
503
+
504
+
505
+
506
+
507
+
508
+
509
+
510
+
511
+
512
+
513
+
514
+
515
+
516
+
517
+
518
+
519
+
520
+
521
+
522
+
523
+
524
+
525
+
526
+
527
+
528
+
529
+
530
+
531
+
532
+
533
+
534
+
535
+
536
+
537
+
538
+
539
+
540
+
541
+
542
+
543
+
544
+
545
+
546
+
547
+
548
+
549
+
550
+
551
+
552
+
553
+
554
+
555
+
556
+
557
+
558
+
559
+
560
+
561
+
562
+
563
+
564
+
565
+
566
+
567
+
568
+
569
+
570
+
571
+
572
+
573
+
574
+
575
+
576
+
577
+
578
+
579
+
580
+
581
+
582
+
583
+
584
+
585
+
586
+
587
+
588
+
589
+
590
+
591
+
592
+
593
+
594
+
595
+
596
+
597
+
598
+
599
+
600
+
601
+
602
+
603
+
604
+
605
+
606
+
607
+
608
+
609
+
610
+
611
+
612
+
613
+
614
+
615
+
616
+
617
+
618
+
619
+
620
+
621
+
622
+
623
+
624
+
625
+
626
+
627
+
628
+
629
+
630
+
631
+
632
+
633
+
634
+
635
+
636
+
637
+
638
+
639
+
640
+
641
+
642
+
643
+
644
+
645
+
646
+
647
+
648
+
649
+
650
+
651
+
652
+
653
+
654
+
655
+
656
+
657
+
658
+
659
+
660
+
661
+
662
+
663
+
664
+
665
+
666
+
667
+
668
+
669
+
670
+
671
+
672
+
673
+
674
+
675
+
676
+
677
+
678
+
679
+
680
+
681
+
682
+
683
+
684
+
685
+
686
+
687
+
688
+
689
+
690
+
691
+
692
+
693
+
694
+
695
+
696
+
697
+
698
+
699
+
700
+
701
+
702
+
703
+
704
+
705
+
706
+
707
+
708
+
709
+
710
+
711
+
712
+
713
+
714
+
715
+
716
+
717
+
718
+
719
+
720
+
721
+
722
+
723
+
724
+
725
+
726
+
727
+
728
+
729
+
730
+
731
+
732
+
733
+
734
+
735
+
736
+
737
+
738
+
739
+
740
+
741
+
742
+
743
+
744
+
745
+
746
+
747
+
748
+
749
+
750
+
751
+
752
+
753
+
754
+
755
+
756
+
757
+
758
+
759
+
760
+
761
+
762
+
763
+
764
+
765
+
766
+
767
+
768
+
769
+
770
+
771
+
772
+
773
+
774
+
775
+
776
+
777
+
778
+
779
+
780
+
781
+
782
+
783
+
784
+
785
+
786
+
787
+
788
+
789
+
790
+
791
+
792
+
793
+
794
+
795
+
796
+
797
+
798
+
799
+
800
+
801
+
802
+
803
+
804
+
805
+
806
+
807
+
808
+
809
+
810
+
811
+
812
+
813
+
814
+
815
+
816
+
817
+
818
+
819
+
820
+
821
+
822
+
823
+
824
+
825
+
826
+
827
+
828
+
829
+
830
+
831
+
832
+
833
+
834
+
835
+
836
+
837
+
838
+
839
+
840
+
841
+
842
+
843
+
844
+
845
+
846
+
847
+
848
+
849
+
850
+
851
+
852
+
853
+
854
+
855
+
856
+
857
+
858
+
859
+
860
+
861
+
862
+
863
+
864
+
865
+
866
+
867
+
868
+
869
+
870
+
871
+
872
+
873
+
874
+
875
+
876
+
877
+
878
+
879
+
880
+
881
+
882
+
883
+
884
+
885
+
886
+
887
+
888
+
889
+
890
+
891
+
892
+
893
+
894
+
895
+
896
+
897
+
898
+
899
+
900
+
901
+
902
+
903
+
904
+
905
+
906
+
907
+
908
+
909
+
910
+
911
+
912
+
913
+
914
+
915
+
916
+
917
+
918
+
919
+
920
+
921
+
922
+
923
+
924
+
925
+
926
+
927
+
928
+
929
+
930
+
931
+
932
+
933
+
934
+
935
+
936
+
937
+
938
+
939
+
166
940
 
167
941
 
168
942
 
@@ -195,16 +969,16 @@ extension RunnerTests {
195
969
  }
196
970
 
197
971
  func executeUptime() -> Response {
198
- // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a
199
- // fresher send-time stamp on every ok response; kept so direct callers still get a value.
972
+
973
+
200
974
  Response(
201
975
  ok: true,
202
976
  data: DataPayload(currentUptimeMs: currentUptimeMs())
203
977
  )
204
978
  }
205
979
 
206
- /// Tracks one main-queue dispatch so the watchdog and the dispatched block can agree —
207
- /// under `mainThreadWorkLock` — on exactly one of: finished in time, or abandoned.
980
+
981
+
208
982
  private final class MainThreadWorkState {
209
983
  var finished = false
210
984
  var abandoned = false
@@ -212,6 +986,8 @@ extension RunnerTests {
212
986
 
213
987
  struct ActiveCommandContext {
214
988
  let app: XCUIApplication
989
+
990
+ var systemSurface: SystemSurfaceHost? = nil
215
991
  }
216
992
 
217
993
  enum ActiveCommandPreparation {
@@ -286,10 +1062,10 @@ extension RunnerTests {
286
1062
  }
287
1063
 
288
1064
  private func executeDispatched(command: Command) throws -> Response {
289
- // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue
290
- // block, queueing more main-thread commands behind it only buries the runner deeper.
291
- // Refuse fast instead so the daemon backs off while the abandoned work drains; past the
292
- // wedge threshold, escalate so the daemon recycles this runner (#1105).
1065
+
1066
+
1067
+
1068
+
293
1069
  if let unavailable = runnerUnavailableResponse(command: command) {
294
1070
  return unavailable
295
1071
  }
@@ -304,9 +1080,9 @@ extension RunnerTests {
304
1080
  routeToSpringboard: routeToSpringboard
305
1081
  )
306
1082
  }
307
- // Resolve this before the command's outer main-thread block. If the bounded probe abandons
308
- // slow XCTest enumeration, return the established recoverable response instead of queueing
309
- // command preparation behind work that may outlive the 30-second command watchdog.
1083
+
1084
+
1085
+
310
1086
  let routeToSpringboard = shouldRouteToSpringboardBlockingSystemModal(command)
311
1087
  if let unavailable = runnerUnavailableResponse(command: command) {
312
1088
  return unavailable
@@ -405,7 +1181,7 @@ extension RunnerTests {
405
1181
  )
406
1182
  }
407
1183
 
408
- // MARK: - Command Handling
1184
+
409
1185
 
410
1186
  private func executeOnMainSafely(
411
1187
  command: Command,
@@ -456,6 +1232,20 @@ extension RunnerTests {
456
1232
  userInfo: [NSLocalizedDescriptionKey: "command returned no response"]
457
1233
  )
458
1234
  }
1235
+
1236
+
1237
+
1238
+
1239
+
1240
+
1241
+
1242
+
1243
+
1244
+
1245
+
1246
+
1247
+
1248
+
459
1249
  if didRecordXCTestFailure(since: failureCountBefore),
460
1250
  let failureResponse = xctestRecordedFailureResponse(command: command, response: response)
461
1251
  {
@@ -482,9 +1272,9 @@ extension RunnerTests {
482
1272
  }
483
1273
  }
484
1274
 
485
- /// The dispatched snapshot recovery loop: read-only retry + XCTest-recorded-failure invalidation,
486
- /// matching what `executeOnMainSafely` gives the generic path. `perform` runs the capture and its
487
- /// own bounded main-thread work.
1275
+
1276
+
1277
+
488
1278
  func executeDispatchedWithRecovery(
489
1279
  command: Command,
490
1280
  perform: () throws -> Response
@@ -498,8 +1288,8 @@ extension RunnerTests {
498
1288
  self.currentXCTestFailureCount()
499
1289
  }
500
1290
  let response = try perform()
501
- // Recovered independently — re-entering main for bookkeeping would queue behind the still-
502
- // abandoned XCTest query and re-stall the command (#1244), so skip it until that work drains.
1291
+
1292
+
503
1293
  if hasAbandonedTreeCapture() {
504
1294
  NSLog(
505
1295
  "AGENT_DEVICE_RUNNER_DISPATCH_RECOVERY_SKIPPED_XCTEST_OCCUPIED command=%@",
@@ -554,12 +1344,16 @@ extension RunnerTests {
554
1344
  case .response(let response):
555
1345
  return response
556
1346
  case .context(let context):
557
- return try executeSnapshotPrepared(command: command, activeApp: context.app)
1347
+ return try executeSnapshotPrepared(
1348
+ command: command,
1349
+ activeApp: context.app,
1350
+ systemSurface: context.systemSurface
1351
+ )
558
1352
  }
559
1353
  }
560
1354
 
561
- /// Pure command→options projection, extracted so the runner unit bundle can
562
- /// prove the decoded wire field actually reaches presentation options (#1634 P2).
1355
+
1356
+
563
1357
  static func presentationOptions(from command: Command) -> PresentationOptions {
564
1358
  let customActions = command.customActions ?? false
565
1359
  return PresentationOptions(
@@ -567,24 +1361,34 @@ extension RunnerTests {
567
1361
  depth: command.depth,
568
1362
  scope: command.scope,
569
1363
  raw: command.raw ?? false,
570
- // Custom actions are only readable through the private AX client, so
571
- // asking for them pins that backend rather than silently returning a
572
- // capture that structurally cannot carry them. An explicit pin wins.
1364
+
1365
+
1366
+
573
1367
  preferredBackend: command.preferredBackend
574
1368
  ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil),
575
1369
  customActions: customActions
576
1370
  )
577
1371
  }
578
1372
 
579
- private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
1373
+ private func executeSnapshotPrepared(
1374
+ command: Command,
1375
+ activeApp: XCUIApplication,
1376
+ systemSurface: SystemSurfaceHost? = nil
1377
+ ) throws -> Response {
580
1378
  let options = Self.presentationOptions(from: command)
581
1379
  do {
582
- let payload: DataPayload
1380
+ var payload: DataPayload
583
1381
  if options.raw {
584
1382
  payload = try snapshotRaw(app: activeApp, options: options)
585
1383
  } else {
586
1384
  payload = try snapshotFast(app: activeApp, options: options)
587
1385
  }
1386
+ if let systemSurface {
1387
+ payload.systemSurface = SystemSurfaceProvenancePayload(
1388
+ bundleId: systemSurface.bundleId,
1389
+ kind: systemSurface.kind.rawValue
1390
+ )
1391
+ }
588
1392
  setNeedsPostSnapshotInteractionDelay()
589
1393
  return Response(ok: true, data: payload)
590
1394
  } catch let failure as SnapshotCaptureFailure {
@@ -734,10 +1538,10 @@ extension RunnerTests {
734
1538
  }
735
1539
  case .recordStop:
736
1540
  guard let recorder = activeRecording else {
737
- // The runner protocol is the durable cleanup primitive. A daemon may crash after the
738
- // native stop succeeds but before it commits the resource transition, so exact-owner
739
- // recovery must be able to repeat this command safely. Public `record stop` still owns
740
- // its user-facing no-active validation through the daemon session manifest.
1541
+
1542
+
1543
+
1544
+
741
1545
  return Response(ok: true, data: DataPayload(message: "recording already stopped"))
742
1546
  }
743
1547
  do {
@@ -757,8 +1561,8 @@ extension RunnerTests {
757
1561
  else {
758
1562
  return Response(ok: false, error: ErrorPayload(message: "activate requires appBundleId"))
759
1563
  }
760
- // prepareActiveCommandContext already activated this bundle. Keep this case as the
761
- // explicit acknowledgement after that preflight, not as a second activation.
1564
+
1565
+
762
1566
  return Response(ok: true, data: DataPayload(message: "app activated"))
763
1567
  case .terminate:
764
1568
  guard
@@ -787,10 +1591,20 @@ extension RunnerTests {
787
1591
  routeToSpringboard: Bool = false
788
1592
  ) -> ActiveCommandPreparation {
789
1593
  var activeApp = currentApp ?? app
1594
+ var systemSurface: SystemSurfaceHost? = nil
790
1595
  if routeToSpringboard {
791
1596
  activeApp = springboard
792
1597
  } else if shouldSkipAppActivationPreflight(command) {
793
1598
  activeApp = resolveAppWithoutActivation(command: command)
1599
+ } else if let presented = presentedSystemSurfaceHost() {
1600
+
1601
+
1602
+
1603
+ activeApp = presented.app
1604
+ systemSurface = presented.host
1605
+ if isInteractionCommand(command.command) {
1606
+ applyInteractionStabilizationIfNeeded()
1607
+ }
794
1608
  } else if !isRunnerLifecycleCommand(command.command) {
795
1609
  let normalizedBundleId = command.appBundleId?
796
1610
  .trimmingCharacters(in: .whitespacesAndNewlines)
@@ -802,7 +1616,7 @@ extension RunnerTests {
802
1616
  refreshCachedTargetIfProcessChanged(bundleId: bundleId)
803
1617
  }
804
1618
  } else {
805
- // Do not reuse stale bundle targets when the caller does not explicitly request one.
1619
+
806
1620
  invalidateCachedTarget(reason: "missing_app_bundle")
807
1621
  }
808
1622
 
@@ -851,7 +1665,25 @@ extension RunnerTests {
851
1665
  applyInteractionStabilizationIfNeeded()
852
1666
  }
853
1667
  }
854
- return .context(ActiveCommandContext(app: activeApp))
1668
+ return .context(ActiveCommandContext(app: activeApp, systemSurface: systemSurface))
1669
+ }
1670
+
1671
+
1672
+
1673
+
1674
+
1675
+ private func presentedSystemSurfaceHost() -> (host: SystemSurfaceHost, app: XCUIApplication)? {
1676
+ #if os(iOS)
1677
+ for host in SystemSurfaceHostRegistry.hosts {
1678
+ let candidate = XCUIApplication(bundleIdentifier: host.bundleId)
1679
+ if candidate.state == .runningForeground {
1680
+ return (host, candidate)
1681
+ }
1682
+ }
1683
+ return nil
1684
+ #else
1685
+ return nil
1686
+ #endif
855
1687
  }
856
1688
 
857
1689
  func executeOnMainPrepared(
@@ -896,15 +1728,15 @@ extension RunnerTests {
896
1728
  }
897
1729
  if let element = match.element {
898
1730
  let frame = element.frame
899
- // XCTest reports closed-drawer/off-viewport items as hittable, then
900
- // "taps" coordinates outside the visible window as a silent no-op.
901
- // Refuse instead; the daemon falls back to tree-based resolution,
902
- // which can prefer an on-screen candidate or explain the off-screen
903
- // state. The check uses the main window frame, not app.frame: on RN
904
- // apps app.frame unions transformed subtrees (a closed drawer at
905
- // negative x), so it happily "contains" unreachable coordinates.
906
- // The DECISION lives in TapPointPolicy (golden parity table with the
907
- // TS twin); onScreenWindowFrame only supplies the frame.
1731
+
1732
+
1733
+
1734
+
1735
+
1736
+
1737
+
1738
+
1739
+
908
1740
  if !match.usedNonHittableFallback
909
1741
  && !TapPointPolicy.isAllowed(
910
1742
  elementFrame: frame,
@@ -968,10 +1800,10 @@ extension RunnerTests {
968
1800
  }
969
1801
  let (timing, outcome) = performGesture(activeApp) {
970
1802
  if expectedPoint != nil || match.usedNonHittableFallback {
971
- // Maestro compatibility: RN E2E backdoor controls can be 1x1 and
972
- // reported non-hittable by XCTest. Snapshot-resolved targets also
973
- // need coordinate delivery because XCUIElement.activate() does not
974
- // invoke every React Native accessibility wrapper.
1803
+
1804
+
1805
+
1806
+
975
1807
  return tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y)
976
1808
  }
977
1809
  return activateElement(app: activeApp, element: element, action: "tap by selector")
@@ -1007,9 +1839,9 @@ extension RunnerTests {
1007
1839
  )
1008
1840
  let textInput: XCUIElement?
1009
1841
  if !xCTestTextInputProbeSkipped {
1010
- textInput = textInputAt(app: activeApp, x: x, y: y)
1842
+ textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y)
1011
1843
  } else {
1012
- // A process-scoped tap cannot authorize later typing without concrete element identity.
1844
+
1013
1845
  textInput = nil
1014
1846
  NSLog(
1015
1847
  "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@",
@@ -1056,8 +1888,8 @@ extension RunnerTests {
1056
1888
  }
1057
1889
  let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
1058
1890
  do {
1059
- // mouseClick throws (it has no RunnerInteractionOutcome), so it keeps raw measureGesture
1060
- // and only routes the success payload through gestureResponse.
1891
+
1892
+
1061
1893
  var clickError: Error?
1062
1894
  let timing = measureGesture {
1063
1895
  do {
@@ -1103,9 +1935,9 @@ extension RunnerTests {
1103
1935
  synthesizedPolicyKind: .synthesizedDrag
1104
1936
  )
1105
1937
  case .scroll:
1106
- // Fused frame-resolve + drag scroll for non-tvOS. On iOS this intentionally stays on the
1107
- // AX-free synthesized coordinate lane so scroll keeps working when XCTest cannot serialize
1108
- // the accessibility tree.
1938
+
1939
+
1940
+
1109
1941
  guard let rawDirection = command.direction,
1110
1942
  let direction = RunnerScrollDirection(rawValue: rawDirection)
1111
1943
  else {
@@ -1121,21 +1953,25 @@ extension RunnerTests {
1121
1953
  error: ErrorPayload(message: "scroll could not resolve a usable interaction frame")
1122
1954
  )
1123
1955
  }
1124
- let frame = scrollReferenceFrame(app: activeApp, context: scrollContext)
1125
- guard frame.width > 0, frame.height > 0 else {
1956
+ let viewport = resolvedScrollViewport(app: activeApp, context: scrollContext)
1957
+ let defaults = runnerDragCommandDefaults(command)
1958
+ switch viewport.gestureDispatch(
1959
+ direction: direction,
1960
+ amount: defaults.scrollAmount,
1961
+ pixels: command.pixels
1962
+ ) {
1963
+ case .occluded(let occlusionKeyboardMinY, let visibleHeight):
1964
+ return scrollKeyboardOccludedResponse(
1965
+ direction: direction.rawValue,
1966
+ keyboardMinY: occlusionKeyboardMinY,
1967
+ visibleHeight: visibleHeight
1968
+ )
1969
+ case .unusableFrame:
1126
1970
  return Response(
1127
1971
  ok: false,
1128
1972
  error: ErrorPayload(message: "scroll could not resolve a usable interaction frame")
1129
1973
  )
1130
- }
1131
- let defaults = runnerDragCommandDefaults(command)
1132
- guard let plan = runnerScrollGesturePlan(
1133
- direction: direction,
1134
- amount: defaults.scrollAmount,
1135
- pixels: command.pixels,
1136
- referenceWidth: frame.width,
1137
- referenceHeight: frame.height
1138
- ) else {
1974
+ case .unusablePlan:
1139
1975
  return Response(
1140
1976
  ok: false,
1141
1977
  error: ErrorPayload(
@@ -1143,21 +1979,24 @@ extension RunnerTests {
1143
1979
  message: "scroll could not compute a gesture plan"
1144
1980
  )
1145
1981
  )
1982
+ case .gesture(let gesture):
1983
+ guard scrollDurationIsValid(command.durationMs) else {
1984
+ return invalidScrollDurationResponse(commandName: "scroll")
1985
+ }
1986
+ return gesture.attachingEvidence(
1987
+ to: executeScrollDragGesture(
1988
+ activeApp: activeApp,
1989
+ x: gesture.planFrame.minX + gesture.plan.x1,
1990
+ y: gesture.planFrame.minY + gesture.plan.y1,
1991
+ x2: gesture.planFrame.minX + gesture.plan.x2,
1992
+ y2: gesture.planFrame.minY + gesture.plan.y2,
1993
+ durationMs: defaults.durationMs,
1994
+ message: "scrolled",
1995
+ context: scrollContext.withReferenceFrame(gesture.coordinateFrame),
1996
+ releaseBehavior: command.scrollReleaseBehavior
1997
+ )
1998
+ )
1146
1999
  }
1147
- guard scrollDurationIsValid(command.durationMs) else {
1148
- return invalidScrollDurationResponse(commandName: "scroll")
1149
- }
1150
- return executeScrollDragGesture(
1151
- activeApp: activeApp,
1152
- x: frame.minX + plan.x1,
1153
- y: frame.minY + plan.y1,
1154
- x2: frame.minX + plan.x2,
1155
- y2: frame.minY + plan.y2,
1156
- durationMs: defaults.durationMs,
1157
- message: "scrolled",
1158
- context: scrollContext.withReferenceFrame(frame),
1159
- releaseBehavior: command.scrollReleaseBehavior
1160
- )
1161
2000
  case .desktopScroll:
1162
2001
  guard let rawDirection = command.direction,
1163
2002
  let direction = RunnerScrollDirection(rawValue: rawDirection)
@@ -1244,8 +2083,8 @@ extension RunnerTests {
1244
2083
  guard let direction = command.direction else {
1245
2084
  return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
1246
2085
  }
1247
- // swipe returns an optional frame (tvOS-only) rather than a RunnerInteractionOutcome, so it
1248
- // keeps raw measureGesture and only routes the success payload through gestureResponse.
2086
+
2087
+
1249
2088
  var executedFrame: DragVisualizationFrame?
1250
2089
  let timing = measureGesture {
1251
2090
  withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
@@ -1280,12 +2119,12 @@ extension RunnerTests {
1280
2119
  case .screenshot:
1281
2120
  let screenshot: XCUIScreenshot
1282
2121
  #if os(macOS)
1283
- // macOS keeps the app-targeted capture behavior for window-level screenshots.
2122
+
1284
2123
  if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
1285
2124
  let targetApp = XCUIApplication(bundleIdentifier: bundleId)
1286
2125
  targetApp.activate()
1287
2126
  activeApp = targetApp
1288
- // Brief wait for the app transition animation to complete
2127
+
1289
2128
  sleepFor(0.5)
1290
2129
  }
1291
2130
  if command.fullscreen == true {
@@ -1317,7 +2156,7 @@ extension RunnerTests {
1317
2156
  #if os(macOS)
1318
2157
  return Response(ok: true, data: DataPayload(message: filePath))
1319
2158
  #else
1320
- // Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
2159
+
1321
2160
  return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
1322
2161
  #endif
1323
2162
  case .back, .backInApp:
@@ -1387,8 +2226,8 @@ extension RunnerTests {
1387
2226
  Self.alertCommandTimeout(timeoutMs: command.timeoutMs)
1388
2227
  )
1389
2228
  guard let alert = resolveAlert(app: activeApp, deadline: deadline) else {
1390
- // Typed so the host retries on absence alone: a transport or runner failure carries no
1391
- // code and must not be mistaken for "no alert yet" (ALERT_NOT_FOUND_RUNNER_CODE).
2229
+
2230
+
1392
2231
  return Response(
1393
2232
  ok: false,
1394
2233
  error: ErrorPayload(code: "ALERT_NOT_FOUND", message: "alert not found")
@@ -1410,7 +2249,7 @@ extension RunnerTests {
1410
2249
  }
1411
2250
  switch plannedGestureExecution(for: plan) {
1412
2251
  case .fastSwipe:
1413
- // Validation above guarantees a non-empty, single-pointer path for this execution kind.
2252
+
1414
2253
  let first = plan.pointers[0].samples.first!.point
1415
2254
  let last = plan.pointers[0].samples.last!.point
1416
2255
  return canonicalPlannedGestureResponse(
@@ -1509,8 +2348,8 @@ extension RunnerTests {
1509
2348
  #endif
1510
2349
  }
1511
2350
 
1512
- /// Shared drag execution for explicit drag commands. The iOS synthesized lane keeps its
1513
- /// fallback policy explicit; viewport scrolling owns a separate single drag specification.
2351
+
2352
+
1514
2353
  private func executeDragGesture(
1515
2354
  activeApp: XCUIApplication,
1516
2355
  x: Double,
@@ -1732,12 +2571,30 @@ extension RunnerTests {
1732
2571
  )
1733
2572
  }
1734
2573
 
1735
- private func scrollReferenceFrame(app: XCUIApplication, context: SynthesizedCoordinateContext) -> CGRect {
1736
- #if os(iOS)
1737
- return synthesizedFrameAvoidingKeyboardWhenAllowed(app: app, context: context)
1738
- #else
1739
- return resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
1740
- #endif
2574
+
2575
+
2576
+
2577
+
2578
+
2579
+
2580
+
2581
+ private func scrollKeyboardOccludedResponse(
2582
+ direction: String,
2583
+ keyboardMinY: Double,
2584
+ visibleHeight: Double
2585
+ ) -> Response {
2586
+ return Response(
2587
+ ok: false,
2588
+ error: ErrorPayload(
2589
+ code: ScrollViewportPolicy.occlusionRunnerCode,
2590
+ message: String(
2591
+ format:
2592
+ "scroll %@ refused: the keyboard leaves %.0fpt of visible surface above it, too little to swipe",
2593
+ direction,
2594
+ visibleHeight
2595
+ )
2596
+ )
2597
+ )
1741
2598
  }
1742
2599
 
1743
2600
  private func dragCommandName(message: String) -> String {
@@ -1771,15 +2628,20 @@ extension RunnerTests {
1771
2628
  }
1772
2629
 
1773
2630
 
2631
+
2632
+
2633
+
2634
+
2635
+
1774
2636
  private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool {
1775
2637
  #if os(iOS)
1776
2638
  if command.command == .alert {
1777
2639
  return true
1778
2640
  }
1779
- // Coordinate-only synthesized taps can run after an AX-fatal foreground screen because they do not
1780
- // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text
1781
- // interactions intentionally stay on the normal AX path because they need an element query.
1782
- // Scroll/drag/sequence keep the normal foreground guard and stabilization path.
2641
+
2642
+
2643
+
2644
+
1783
2645
  guard command.text == nil, command.selectorKey == nil else { return false }
1784
2646
  guard hasCachedTargetForActivationSkip(command: command) else { return false }
1785
2647
  return isCoordinateOnlyTap(command)
@@ -1795,10 +2657,15 @@ extension RunnerTests {
1795
2657
  guard isCoordinateOnlyTap(command) else {
1796
2658
  return false
1797
2659
  }
2660
+
2661
+
2662
+
2663
+
2664
+
1798
2665
  let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget)
1799
- // `runMainThreadWork` executes inline for a main-thread caller, so that path cannot use its
1800
- // timeout machinery. Direct main-thread dispatch keeps the prior synchronous modal check;
1801
- // normal off-main command dispatch uses the bounded probe and post-probe busy recovery.
2666
+
2667
+
2668
+
1802
2669
  if Thread.isMainThread {
1803
2670
  return firstBlockingSystemModal(
1804
2671
  in: springboard,
@@ -1857,8 +2724,8 @@ extension RunnerTests {
1857
2724
  command.x != nil,
1858
2725
  command.y != nil
1859
2726
  {
1860
- // The shared runtime has already resolved this node as non-hittable and
1861
- // deliberately selected Maestro's coordinate compatibility route.
2727
+
2728
+
1862
2729
  maestroNonHittableCoordinateFallbackUsed = true
1863
2730
  }
1864
2731
  let focusStartedAt = Date()
@@ -1906,8 +2773,8 @@ extension RunnerTests {
1906
2773
  if let resolvedCoordinateTarget {
1907
2774
  target = resolvedCoordinateTarget
1908
2775
  } else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1909
- // Released daemons may still send selector-keyed type commands even though current
1910
- // daemons resolve fill selectors through the runtime tree before reaching the runner.
2776
+
2777
+
1911
2778
  let match = findElement(
1912
2779
  app: activeApp,
1913
2780
  selectorKey: selectorKey,
@@ -1989,8 +2856,8 @@ extension RunnerTests {
1989
2856
  } else if point != nil {
1990
2857
  frame = activeApp.frame
1991
2858
  } else {
1992
- // Bare `type` has no coordinate response to normalize. Avoid serializing the
1993
- // application AX tree only to emit unused reference dimensions.
2859
+
2860
+
1994
2861
  frame = .zero
1995
2862
  }
1996
2863
  return Response(