agent-device 0.21.0 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (282) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.0.apk → agent-device-android-ime-helper-0.21.1.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.1.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.0.manifest.json → agent-device-android-ime-helper-0.21.1.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.0.apk → agent-device-android-snapshot-helper-0.21.1.apk} +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.1.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.0.manifest.json → agent-device-android-snapshot-helper-0.21.1.manifest.json} +6 -6
  7. package/apple/snapshot-bridge/README.md +58 -1
  8. package/apple/snapshot-bridge/SnapshotBridge.m +6 -0
  9. package/apple/snapshot-bridge/SnapshotBridgeCapture.h +25 -0
  10. package/apple/snapshot-bridge/SnapshotBridgeCapture.m +138 -0
  11. package/apple/snapshot-bridge/SnapshotBridgeRuntime.h +1 -0
  12. package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +32 -3
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +12 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +20 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +15 -15
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +72 -0
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +63 -14
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +668 -82
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +14 -4
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +39 -4
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +1065 -223
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +290 -0
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +1 -1
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +10 -10
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +249 -61
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +101 -42
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +13 -13
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +51 -43
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +71 -4
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +144 -7
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +13 -1
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +216 -17
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +206 -22
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +289 -42
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +92 -22
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +617 -122
  37. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +79 -4
  38. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +71 -71
  39. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +83 -5
  40. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +67 -67
  41. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +4 -4
  42. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +17 -17
  43. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +30 -30
  44. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +31 -31
  45. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +28 -0
  46. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +101 -0
  47. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +19 -19
  48. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +55 -17
  49. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +26 -5
  50. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +125 -75
  51. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +9 -9
  52. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +9 -9
  53. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift +4 -4
  54. package/dist/src/adapter.js +1 -1
  55. package/dist/src/adb-executor.js +1 -1
  56. package/dist/src/adb-failure.js +1 -1
  57. package/dist/src/agent-browser-process-record.js +1 -1
  58. package/dist/src/agent-browser-provider.js +1 -1
  59. package/dist/src/agent-browser-tool.js +1 -1
  60. package/dist/src/agent-device-client.js +2 -2
  61. package/dist/src/ai-sdk.d.ts +1 -1
  62. package/dist/src/ai-sdk.js +1 -1
  63. package/dist/src/alert-runtime.js +1 -1
  64. package/dist/src/android-input-ownership.js +1 -1
  65. package/dist/src/app-catalog.js +1 -1
  66. package/dist/src/app-deployment-resolution.js +1 -1
  67. package/dist/src/app-inventory-contract.js +1 -1
  68. package/dist/src/app-inventory.js +1 -1
  69. package/dist/src/app-launch.js +2 -2
  70. package/dist/src/app-lifecycle.js +3 -3
  71. package/dist/src/app-lifecycle2.js +1 -1
  72. package/dist/src/app-log-files.js +1 -1
  73. package/dist/src/app-log-runtime2.js +3 -3
  74. package/dist/src/app-resolution.js +1 -1
  75. package/dist/src/apple-runner-platform.js +1 -0
  76. package/dist/src/application-lifecycle-interaction.js +1 -1
  77. package/dist/src/archive.js +1 -1
  78. package/dist/src/artifact-download.js +1 -1
  79. package/dist/src/auth-session.js +1 -1
  80. package/dist/src/auth.js +1 -1
  81. package/dist/src/backend-snapshot-options.js +1 -1
  82. package/dist/src/boot-diagnostics.js +1 -1
  83. package/dist/src/capture.js +1 -1
  84. package/dist/src/cli-config.js +1 -1
  85. package/dist/src/cli-help.js +25 -21
  86. package/dist/src/cli-runner.js +1 -1
  87. package/dist/src/cli.js +4 -4
  88. package/dist/src/click-button.js +1 -1
  89. package/dist/src/client-metro-companion.js +1 -1
  90. package/dist/src/client-metro.js +1 -1
  91. package/dist/src/client-types.d.ts +27 -13
  92. package/dist/src/code-signature-cache.js +1 -1
  93. package/dist/src/code-signature.js +1 -1
  94. package/dist/src/command-schema.js +3 -3
  95. package/dist/src/command-tools.js +2 -2
  96. package/dist/src/command.js +1 -1
  97. package/dist/src/connection-runtime.js +1 -1
  98. package/dist/src/connection.js +3 -3
  99. package/dist/src/contracts.d.ts +1 -1
  100. package/dist/src/contracts.js +1 -1
  101. package/dist/src/daemon-client-lifecycle.js +1 -1
  102. package/dist/src/daemon.js +1 -1
  103. package/dist/src/debug-symbols-facade.js +1 -1
  104. package/dist/src/device-claim-rule.js +1 -1
  105. package/dist/src/device-inventory-context.js +1 -1
  106. package/dist/src/device-rotation.js +1 -1
  107. package/dist/src/device-selection-resolver.js +1 -1
  108. package/dist/src/device-session.js +1 -1
  109. package/dist/src/device2.js +1 -1
  110. package/dist/src/diagnostics.js +1 -1
  111. package/dist/src/dispatch-resolve.js +1 -1
  112. package/dist/src/doctor2.js +1 -1
  113. package/dist/src/error.js +1 -1
  114. package/dist/src/exec.js +1 -1
  115. package/dist/src/file.js +1 -1
  116. package/dist/src/find.js +1 -1
  117. package/dist/src/generic-settle.js +1 -1
  118. package/dist/src/gesture-plan.js +1 -1
  119. package/dist/src/gesture-runtime.js +1 -1
  120. package/dist/src/harmonyos.js +1 -1
  121. package/dist/src/hdc.js +1 -1
  122. package/dist/src/helper.js +2 -2
  123. package/dist/src/human-control-contract.js +1 -1
  124. package/dist/src/human-control.js +1 -1
  125. package/dist/src/index.d.ts +1 -1
  126. package/dist/src/index.js +1 -1
  127. package/dist/src/input-actions.js +1 -1
  128. package/dist/src/input-actions2.js +2 -2
  129. package/dist/src/install-artifact-facade.js +1 -1
  130. package/dist/src/install-source-network-transport.js +1 -1
  131. package/dist/src/install-source2.js +1 -1
  132. package/dist/src/interaction-positionals.js +1 -1
  133. package/dist/src/interaction-ref-policy.js +1 -1
  134. package/dist/src/interaction2.js +1 -1
  135. package/dist/src/interactor.js +5 -5
  136. package/dist/src/interactor2.js +1 -1
  137. package/dist/src/interactors.js +1 -1
  138. package/dist/src/internal/bin.js +1 -1
  139. package/dist/src/internal/daemon.js +2 -2
  140. package/dist/src/internal/png-worker.js +1 -1
  141. package/dist/src/internal/run-script-http-child.js +1 -1
  142. package/dist/src/inventory.js +2 -2
  143. package/dist/src/inventory2.js +1 -1
  144. package/dist/src/inventory3.js +1 -1
  145. package/dist/src/inventory5.js +1 -1
  146. package/dist/src/ios-snapshot-runtime.js +1 -1
  147. package/dist/src/ios-system-surface.js +1 -0
  148. package/dist/src/lease-scope.js +1 -1
  149. package/dist/src/lease.js +1 -1
  150. package/dist/src/limrun-runtime-dependencies.js +1 -1
  151. package/dist/src/limrun.d.ts +1 -1
  152. package/dist/src/linux.js +1 -1
  153. package/dist/src/location-coordinates.js +1 -1
  154. package/dist/src/managed-device-scope.js +1 -1
  155. package/dist/src/mechanics.js +16 -16
  156. package/dist/src/multitouch-support.js +1 -1
  157. package/dist/src/observation.js +1 -1
  158. package/dist/src/open-target-resolution.js +1 -0
  159. package/dist/src/owner-identity.js +3 -3
  160. package/dist/src/parameterized-recorded-fill.js +1 -1
  161. package/dist/src/perf-facade.js +2 -2
  162. package/dist/src/perf-process-identity.js +1 -0
  163. package/dist/src/perf-runtime-operation-builder.js +1 -1
  164. package/dist/src/perf-runtime-plan.js +1 -1
  165. package/dist/src/perf.js +1 -1
  166. package/dist/src/physical-device-control.js +1 -1
  167. package/dist/src/platform-runtime-android-adb-host.js +1 -1
  168. package/dist/src/platform-runtime-app-log-process.js +1 -1
  169. package/dist/src/platform-runtime-host.js +1 -1
  170. package/dist/src/platform-runtime-managed-owner.js +1 -1
  171. package/dist/src/platform-runtime-open-target.js +1 -1
  172. package/dist/src/platform-runtime-operation-host.js +2 -2
  173. package/dist/src/platform-runtime-runtime-hints.js +2 -2
  174. package/dist/src/platform-runtime-screen-recording-android-host.js +1 -1
  175. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  176. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  177. package/dist/src/platform-runtime-unavailable.js +1 -1
  178. package/dist/src/platform-runtime.js +1 -1
  179. package/dist/src/platform-runtime2.js +1 -1
  180. package/dist/src/png-worker-contract.js +1 -1
  181. package/dist/src/prepare-kind.js +1 -1
  182. package/dist/src/provider-device-runtime.js +1 -1
  183. package/dist/src/provider-session-ownership.js +1 -1
  184. package/dist/src/provider-webdriver.js +1 -1
  185. package/dist/src/proxy.js +1 -1
  186. package/dist/src/react-native.js +1 -1
  187. package/dist/src/record-runtime.js +1 -1
  188. package/dist/src/rect-center.js +1 -1
  189. package/dist/src/register-builtins.js +1 -1
  190. package/dist/src/registry.js +37 -36
  191. package/dist/src/remote-config2.js +1 -1
  192. package/dist/src/replay-divergence.js +1 -1
  193. package/dist/src/replay.js +2 -2
  194. package/dist/src/reporting.js +9 -8
  195. package/dist/src/request-admission.js +1 -1
  196. package/dist/src/request-cancel.js +1 -1
  197. package/dist/src/request-runtime-binding.js +1 -1
  198. package/dist/src/retry.js +1 -1
  199. package/dist/src/runner-cache-metadata.js +2 -2
  200. package/dist/src/runner-client.js +1 -1
  201. package/dist/src/runner-disposal.js +1 -1
  202. package/dist/src/runner-operations-facade.js +1 -1
  203. package/dist/src/runner.js +1 -1
  204. package/dist/src/runtime-transport-hints.js +1 -1
  205. package/dist/src/runtime.js +3 -3
  206. package/dist/src/runtime2.js +4 -4
  207. package/dist/src/runtime3.js +1 -1
  208. package/dist/src/runtime4.js +5 -4
  209. package/dist/src/runtime5.js +1 -1
  210. package/dist/src/runtime6.js +1 -1
  211. package/dist/src/runtime7.js +1 -1
  212. package/dist/src/runtime8.js +1 -1
  213. package/dist/src/runtime9.js +1 -1
  214. package/dist/src/screen-recording-live-handle.js +1 -1
  215. package/dist/src/screen-recording-options.js +1 -1
  216. package/dist/src/screen-recording-resource-recovery.js +1 -1
  217. package/dist/src/screenshot-density.js +1 -1
  218. package/dist/src/screenshot-runtime.js +1 -1
  219. package/dist/src/screenshot.js +1 -1
  220. package/dist/src/screenshot3.js +3 -3
  221. package/dist/src/script.js +1 -1
  222. package/dist/src/scroll-command.js +1 -1
  223. package/dist/src/scroll-gesture.js +1 -1
  224. package/dist/src/sdk-batch-runner.js +2 -2
  225. package/dist/src/sdk-batch.js +2 -2
  226. package/dist/src/sdk-contracts.d.ts +44 -1
  227. package/dist/src/sdk-contracts.js +2 -2
  228. package/dist/src/sdk-device.js +1 -1
  229. package/dist/src/sdk-finders.d.ts +1 -1
  230. package/dist/src/sdk-io.js +1 -1
  231. package/dist/src/sdk-remote-config.d.ts +1 -1
  232. package/dist/src/sdk-remote-config.js +1 -1
  233. package/dist/src/sdk-selectors.d.ts +69 -13
  234. package/dist/src/selector-runtime.js +1 -1
  235. package/dist/src/server.js +3 -3
  236. package/dist/src/session-allocation.js +1 -1
  237. package/dist/src/session-observation.js +1 -0
  238. package/dist/src/session-routing.js +1 -1
  239. package/dist/src/session-runtime-admission.js +1 -1
  240. package/dist/src/session-snapshot.js +1 -1
  241. package/dist/src/session-store.js +3 -3
  242. package/dist/src/session-target-evidence.js +1 -1
  243. package/dist/src/session2.js +5 -5
  244. package/dist/src/settings.js +1 -1
  245. package/dist/src/simulator.js +1 -1
  246. package/dist/src/snapshot-quality-backend-capabilities.js +1 -1
  247. package/dist/src/snapshot-quality-verdict.js +1 -1
  248. package/dist/src/snapshot-runtime-binding.js +1 -1
  249. package/dist/src/snapshot-runtime.js +1 -1
  250. package/dist/src/snapshot-state.js +1 -1
  251. package/dist/src/snapshot2.js +1 -1
  252. package/dist/src/snapshot3.js +1 -1
  253. package/dist/src/snapshot4.js +1 -1
  254. package/dist/src/src.js +1 -1
  255. package/dist/src/src7.js +2 -2
  256. package/dist/src/src8.js +2 -2
  257. package/dist/src/string-enum.js +1 -1
  258. package/dist/src/takeover.js +1 -1
  259. package/dist/src/tool-provider.js +2 -2
  260. package/dist/src/tool-provider2.js +1 -1
  261. package/dist/src/tool-provider3.js +1 -1
  262. package/dist/src/toolchain-probe.js +1 -1
  263. package/dist/src/touch-reference-frame.js +1 -0
  264. package/dist/src/touch-runtime.js +1 -1
  265. package/dist/src/transport.js +1 -1
  266. package/dist/src/tv-remote.js +1 -1
  267. package/dist/src/type-text-runtime.js +1 -1
  268. package/dist/src/update-check.js +1 -1
  269. package/dist/src/verified-file.js +1 -1
  270. package/dist/src/video.js +1 -1
  271. package/dist/src/web.js +1 -1
  272. package/dist/src/web2.js +1 -1
  273. package/dist/src/webdriver-source.js +1 -1
  274. package/dist/src/window-state.js +1 -0
  275. package/package.json +3 -2
  276. package/server.json +2 -2
  277. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.0.apk.sha256 +0 -1
  278. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.0.apk.sha256 +0 -1
  279. package/dist/src/back-mode.js +0 -1
  280. package/dist/src/format.js +0 -2
  281. package/dist/src/platform-runtime-apple-resources.js +0 -1
  282. package/dist/src/runner-selector-query.js +0 -1
@@ -2,167 +2,941 @@ import XCTest
2
2
  import AgentDeviceSnapshotPresentation
3
3
 
4
4
 
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
5
29
  extension RunnerTests {
6
- // MARK: - Main Thread Dispatch
7
30
 
8
- private func currentUptimeMs() -> Double {
9
- ProcessInfo.processInfo.systemUptime * 1000
10
- }
11
31
 
12
- private func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) {
13
- let gestureStartUptimeMs = currentUptimeMs()
14
- action()
15
- return (gestureStartUptimeMs, currentUptimeMs())
16
- }
32
+ private func currentUptimeMs() -> Double {
33
+ ProcessInfo.processInfo.systemUptime * 1000
34
+ }
35
+
36
+ private func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) {
37
+ let gestureStartUptimeMs = currentUptimeMs()
38
+ action()
39
+ return (gestureStartUptimeMs, currentUptimeMs())
40
+ }
41
+
42
+ func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
43
+ min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120)
44
+ }
45
+
46
+ func coordinateDragHoldDuration() -> TimeInterval {
47
+ 0.050
48
+ }
49
+
50
+ func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
51
+ switch outcome {
52
+ case .performed:
53
+ return nil
54
+ case .unsupported(let message, let hint):
55
+ return Response(
56
+ ok: false,
57
+ error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
58
+ )
59
+ }
60
+ }
61
+
62
+
63
+ enum GestureFrame {
64
+ case none
65
+ case touch(TouchVisualizationFrame?)
66
+ case drag(DragVisualizationFrame)
67
+ }
68
+
69
+ struct GestureFallback {
70
+ let strategy: String
71
+ let message: String
72
+ let hint: String?
73
+ }
74
+
75
+ private func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
76
+ switch outcome {
77
+ case .performed:
78
+ return nil
79
+ case .unsupported(let message, let hint):
80
+ return GestureFallback(strategy: strategy, message: message, hint: hint)
81
+ }
82
+ }
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+ func performGesture(
93
+ _ app: XCUIApplication,
94
+ idleTimeout: Bool = true,
95
+ _ action: () -> RunnerInteractionOutcome
96
+ ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) {
97
+ var outcome = RunnerInteractionOutcome.performed
98
+ let timing = measureGesture {
99
+ if idleTimeout {
100
+ withTemporaryScrollIdleTimeoutIfSupported(app) { outcome = action() }
101
+ } else {
102
+ outcome = action()
103
+ }
104
+ }
105
+ return (timing, outcome)
106
+ }
107
+
108
+
109
+
110
+ private func gestureResponse(
111
+ message: String,
112
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
113
+ frame: GestureFrame = .none,
114
+ fallback: GestureFallback? = nil,
115
+ maestroNonHittableCoordinateFallbackUsed: Bool? = nil
116
+ ) -> Response {
117
+ let data: DataPayload
118
+ switch frame {
119
+ case .none:
120
+ data = DataPayload(
121
+ message: message,
122
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
123
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
124
+ gestureFallback: fallback?.strategy,
125
+ gestureFallbackMessage: fallback?.message,
126
+ gestureFallbackHint: fallback?.hint,
127
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
128
+ )
129
+ case .touch(let f):
130
+ data = DataPayload(
131
+ message: message,
132
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
133
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
134
+ x: f?.x,
135
+ y: f?.y,
136
+ referenceWidth: f?.referenceWidth,
137
+ referenceHeight: f?.referenceHeight,
138
+ gestureFallback: fallback?.strategy,
139
+ gestureFallbackMessage: fallback?.message,
140
+ gestureFallbackHint: fallback?.hint,
141
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
142
+ )
143
+ case .drag(let f):
144
+ data = DataPayload(
145
+ message: message,
146
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
147
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
148
+ x: f.x,
149
+ y: f.y,
150
+ x2: f.x2,
151
+ y2: f.y2,
152
+ referenceWidth: f.referenceWidth,
153
+ referenceHeight: f.referenceHeight,
154
+ gestureFallback: fallback?.strategy,
155
+ gestureFallbackMessage: fallback?.message,
156
+ gestureFallbackHint: fallback?.hint
157
+ )
158
+ }
159
+ return Response(ok: true, data: data)
160
+ }
161
+
162
+
163
+
164
+
165
+ private func canonicalPlannedGestureResponse(_ response: Response) -> Response {
166
+ guard response.ok, let data = response.data else { return response }
167
+ return Response(
168
+ ok: true,
169
+ data: DataPayload(
170
+ message: data.message,
171
+ gestureStartUptimeMs: data.gestureStartUptimeMs,
172
+ gestureEndUptimeMs: data.gestureEndUptimeMs,
173
+ gestureFallback: data.gestureFallback,
174
+ gestureFallbackMessage: data.gestureFallbackMessage,
175
+ gestureFallbackHint: data.gestureFallbackHint
176
+ )
177
+ )
178
+ }
179
+
180
+ private func plannedGestureResponse(
181
+ plan: RunnerGesturePlan,
182
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
183
+ outcome: RunnerInteractionOutcome
184
+ ) -> Response {
185
+ if let response = unsupportedResponse(for: outcome) {
186
+ return response
187
+ }
188
+ return gestureResponse(message: plan.intent, timing: timing)
189
+ }
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+
204
+
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+
233
+
234
+
235
+
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
+
255
+
256
+
257
+
258
+
259
+
260
+
261
+
262
+
263
+
264
+
265
+
266
+
267
+
268
+
269
+
270
+
271
+
272
+
273
+
274
+
275
+
276
+
277
+
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+
292
+
293
+
294
+
295
+
296
+
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+
307
+
308
+
309
+
310
+
311
+
312
+
313
+
314
+
315
+
316
+
317
+
318
+
319
+
320
+
321
+
322
+
323
+
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+
333
+
334
+
335
+
336
+
337
+
338
+
339
+
340
+
341
+
342
+
343
+
344
+
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+
386
+
387
+
388
+
389
+
390
+
391
+
392
+
393
+
394
+
395
+
396
+
397
+
398
+
399
+
400
+
401
+
402
+
403
+
404
+
405
+
406
+
407
+
408
+
409
+
410
+
411
+
412
+
413
+
414
+
415
+
416
+
417
+
418
+
419
+
420
+
421
+
422
+
423
+
424
+
425
+
426
+
427
+
428
+
429
+
430
+
431
+
432
+
433
+
434
+
435
+
436
+
437
+
438
+
439
+
440
+
441
+
442
+
443
+
444
+
445
+
446
+
447
+
448
+
449
+
450
+
451
+
452
+
453
+
454
+
455
+
456
+
457
+
458
+
459
+
460
+
461
+
462
+
463
+
464
+
465
+
466
+
467
+
468
+
469
+
470
+
471
+
472
+
473
+
474
+
475
+
476
+
477
+
478
+
479
+
480
+
481
+
482
+
483
+
484
+
485
+
486
+
487
+
488
+
489
+
490
+
491
+
492
+
493
+
494
+
495
+
496
+
497
+
498
+
499
+
500
+
501
+
502
+
503
+
504
+
505
+
506
+
507
+
508
+
509
+
510
+
511
+
512
+
513
+
514
+
515
+
516
+
517
+
518
+
519
+
520
+
521
+
522
+
523
+
524
+
525
+
526
+
527
+
528
+
529
+
530
+
531
+
532
+
533
+
534
+
535
+
536
+
537
+
538
+
539
+
540
+
541
+
542
+
543
+
544
+
545
+
546
+
547
+
548
+
549
+
550
+
551
+
552
+
553
+
554
+
555
+
556
+
557
+
558
+
559
+
560
+
561
+
562
+
563
+
564
+
565
+
566
+
567
+
568
+
569
+
570
+
571
+
572
+
573
+
574
+
575
+
576
+
577
+
578
+
579
+
580
+
581
+
582
+
583
+
584
+
585
+
586
+
587
+
588
+
589
+
590
+
591
+
592
+
593
+
594
+
595
+
596
+
597
+
598
+
599
+
600
+
601
+
602
+
603
+
604
+
605
+
606
+
607
+
608
+
609
+
610
+
611
+
612
+
613
+
614
+
615
+
616
+
617
+
618
+
619
+
620
+
621
+
622
+
623
+
624
+
625
+
626
+
627
+
628
+
629
+
630
+
631
+
632
+
633
+
634
+
635
+
636
+
637
+
638
+
639
+
640
+
641
+
642
+
643
+
644
+
645
+
646
+
647
+
648
+
649
+
650
+
651
+
652
+
653
+
654
+
655
+
656
+
657
+
658
+
659
+
660
+
661
+
662
+
663
+
664
+
665
+
666
+
667
+
668
+
669
+
670
+
671
+
672
+
673
+
674
+
675
+
676
+
677
+
678
+
679
+
680
+
681
+
682
+
683
+
684
+
685
+
686
+
687
+
688
+
689
+
690
+
691
+
692
+
693
+
694
+
695
+
696
+
697
+
698
+
699
+
700
+
701
+
702
+
703
+
704
+
705
+
706
+
707
+
708
+
709
+
710
+
711
+
712
+
713
+
714
+
715
+
716
+
717
+
718
+
719
+
720
+
721
+
722
+
723
+
724
+
725
+
726
+
727
+
728
+
729
+
730
+
731
+
732
+
733
+
734
+
735
+
736
+
737
+
738
+
739
+
740
+
741
+
742
+
743
+
744
+
745
+
746
+
747
+
748
+
749
+
750
+
751
+
752
+
753
+
754
+
755
+
756
+
757
+
758
+
759
+
760
+
761
+
762
+
763
+
764
+
765
+
766
+
767
+
768
+
769
+
770
+
771
+
772
+
773
+
774
+
775
+
776
+
777
+
778
+
779
+
780
+
781
+
782
+
783
+
784
+
785
+
786
+
787
+
788
+
789
+
790
+
791
+
792
+
793
+
794
+
795
+
796
+
797
+
798
+
799
+
800
+
801
+
802
+
803
+
804
+
805
+
806
+
807
+
808
+
809
+
810
+
811
+
812
+
813
+
814
+
815
+
816
+
817
+
818
+
819
+
820
+
821
+
822
+
823
+
824
+
825
+
826
+
827
+
828
+
829
+
830
+
831
+
832
+
833
+
834
+
835
+
836
+
837
+
838
+
839
+
840
+
841
+
842
+
843
+
844
+
845
+
846
+
847
+
848
+
849
+
850
+
851
+
852
+
853
+
854
+
855
+
856
+
857
+
858
+
859
+
860
+
861
+
862
+
863
+
864
+
865
+
866
+
867
+
868
+
869
+
870
+
871
+
872
+
873
+
874
+
875
+
876
+
877
+
878
+
879
+
880
+
881
+
882
+
883
+
884
+
885
+
886
+
887
+
888
+
889
+
890
+
891
+
892
+
893
+
894
+
895
+
896
+
897
+
898
+
899
+
900
+
901
+
902
+
903
+
904
+
905
+
906
+
907
+
908
+
909
+
910
+
911
+
912
+
913
+
914
+
915
+
916
+
917
+
918
+
919
+
920
+
921
+
922
+
923
+
924
+
925
+
926
+
927
+
928
+
17
929
 
18
- func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
19
- min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120)
20
- }
21
930
 
22
- func coordinateDragHoldDuration() -> TimeInterval {
23
- 0.050
24
- }
25
931
 
26
- func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
27
- switch outcome {
28
- case .performed:
29
- return nil
30
- case .unsupported(let message, let hint):
31
- return Response(
32
- ok: false,
33
- error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
34
- )
35
- }
36
- }
37
932
 
38
- /// Optional visualization frame returned with a gesture response.
39
- enum GestureFrame {
40
- case none
41
- case touch(TouchVisualizationFrame?)
42
- case drag(DragVisualizationFrame)
43
- }
44
933
 
45
- struct GestureFallback {
46
- let strategy: String
47
- let message: String
48
- let hint: String?
49
- }
50
934
 
51
- private func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
52
- switch outcome {
53
- case .performed:
54
- return nil
55
- case .unsupported(let message, let hint):
56
- return GestureFallback(strategy: strategy, message: message, hint: hint)
57
- }
58
- }
59
935
 
60
936
 
61
- /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
62
- /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis
63
- /// pointer-plan gestures pass `false` because RunnerSynthesizedGesture governs their
64
- /// own timing. Returns the captured timing and the action's outcome.
65
- ///
66
- /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap
67
- /// it in the scroll idle-timeout/quiescence-skip path and change its runtime behavior.
68
- func performGesture(
69
- _ app: XCUIApplication,
70
- idleTimeout: Bool = true,
71
- _ action: () -> RunnerInteractionOutcome
72
- ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) {
73
- var outcome = RunnerInteractionOutcome.performed
74
- let timing = measureGesture {
75
- if idleTimeout {
76
- withTemporaryScrollIdleTimeoutIfSupported(app) { outcome = action() }
77
- } else {
78
- outcome = action()
79
- }
80
- }
81
- return (timing, outcome)
82
- }
83
937
 
84
- /// Single factory for the success payload every gesture returns (message + gesture timing +
85
- /// an optional touch/drag visualization frame), so the field shape lives in one place.
86
- private func gestureResponse(
87
- message: String,
88
- timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
89
- frame: GestureFrame = .none,
90
- fallback: GestureFallback? = nil,
91
- maestroNonHittableCoordinateFallbackUsed: Bool? = nil
92
- ) -> Response {
93
- let data: DataPayload
94
- switch frame {
95
- case .none:
96
- data = DataPayload(
97
- message: message,
98
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
99
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
100
- gestureFallback: fallback?.strategy,
101
- gestureFallbackMessage: fallback?.message,
102
- gestureFallbackHint: fallback?.hint,
103
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
104
- )
105
- case .touch(let f):
106
- data = DataPayload(
107
- message: message,
108
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
109
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
110
- x: f?.x,
111
- y: f?.y,
112
- referenceWidth: f?.referenceWidth,
113
- referenceHeight: f?.referenceHeight,
114
- gestureFallback: fallback?.strategy,
115
- gestureFallbackMessage: fallback?.message,
116
- gestureFallbackHint: fallback?.hint,
117
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
118
- )
119
- case .drag(let f):
120
- data = DataPayload(
121
- message: message,
122
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
123
- gestureEndUptimeMs: timing.gestureEndUptimeMs,
124
- x: f.x,
125
- y: f.y,
126
- x2: f.x2,
127
- y2: f.y2,
128
- referenceWidth: f.referenceWidth,
129
- referenceHeight: f.referenceHeight,
130
- gestureFallback: fallback?.strategy,
131
- gestureFallbackMessage: fallback?.message,
132
- gestureFallbackHint: fallback?.hint
133
- )
134
- }
135
- return Response(ok: true, data: data)
136
- }
137
938
 
138
- /// Gesture plans already return canonical centroid endpoints from the portable runtime.
139
- /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's
140
- /// visualization frame into only the fast-fling response shape.
141
- private func canonicalPlannedGestureResponse(_ response: Response) -> Response {
142
- guard response.ok, let data = response.data else { return response }
143
- return Response(
144
- ok: true,
145
- data: DataPayload(
146
- message: data.message,
147
- gestureStartUptimeMs: data.gestureStartUptimeMs,
148
- gestureEndUptimeMs: data.gestureEndUptimeMs,
149
- gestureFallback: data.gestureFallback,
150
- gestureFallbackMessage: data.gestureFallbackMessage,
151
- gestureFallbackHint: data.gestureFallbackHint
152
- )
153
- )
154
- }
155
939
 
156
- private func plannedGestureResponse(
157
- plan: RunnerGesturePlan,
158
- timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
159
- outcome: RunnerInteractionOutcome
160
- ) -> Response {
161
- if let response = unsupportedResponse(for: outcome) {
162
- return response
163
- }
164
- return gestureResponse(message: plan.intent, timing: timing)
165
- }
166
940
 
167
941
 
168
942
 
@@ -195,16 +969,16 @@ extension RunnerTests {
195
969
  }
196
970
 
197
971
  func executeUptime() -> Response {
198
- // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a
199
- // fresher send-time stamp on every ok response; kept so direct callers still get a value.
972
+
973
+
200
974
  Response(
201
975
  ok: true,
202
976
  data: DataPayload(currentUptimeMs: currentUptimeMs())
203
977
  )
204
978
  }
205
979
 
206
- /// Tracks one main-queue dispatch so the watchdog and the dispatched block can agree —
207
- /// under `mainThreadWorkLock` — on exactly one of: finished in time, or abandoned.
980
+
981
+
208
982
  private final class MainThreadWorkState {
209
983
  var finished = false
210
984
  var abandoned = false
@@ -212,6 +986,8 @@ extension RunnerTests {
212
986
 
213
987
  struct ActiveCommandContext {
214
988
  let app: XCUIApplication
989
+
990
+ var systemSurface: SystemSurfaceHost? = nil
215
991
  }
216
992
 
217
993
  enum ActiveCommandPreparation {
@@ -286,10 +1062,10 @@ extension RunnerTests {
286
1062
  }
287
1063
 
288
1064
  private func executeDispatched(command: Command) throws -> Response {
289
- // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue
290
- // block, queueing more main-thread commands behind it only buries the runner deeper.
291
- // Refuse fast instead so the daemon backs off while the abandoned work drains; past the
292
- // wedge threshold, escalate so the daemon recycles this runner (#1105).
1065
+
1066
+
1067
+
1068
+
293
1069
  if let unavailable = runnerUnavailableResponse(command: command) {
294
1070
  return unavailable
295
1071
  }
@@ -304,9 +1080,9 @@ extension RunnerTests {
304
1080
  routeToSpringboard: routeToSpringboard
305
1081
  )
306
1082
  }
307
- // Resolve this before the command's outer main-thread block. If the bounded probe abandons
308
- // slow XCTest enumeration, return the established recoverable response instead of queueing
309
- // command preparation behind work that may outlive the 30-second command watchdog.
1083
+
1084
+
1085
+
310
1086
  let routeToSpringboard = shouldRouteToSpringboardBlockingSystemModal(command)
311
1087
  if let unavailable = runnerUnavailableResponse(command: command) {
312
1088
  return unavailable
@@ -405,7 +1181,7 @@ extension RunnerTests {
405
1181
  )
406
1182
  }
407
1183
 
408
- // MARK: - Command Handling
1184
+
409
1185
 
410
1186
  private func executeOnMainSafely(
411
1187
  command: Command,
@@ -456,6 +1232,20 @@ extension RunnerTests {
456
1232
  userInfo: [NSLocalizedDescriptionKey: "command returned no response"]
457
1233
  )
458
1234
  }
1235
+
1236
+
1237
+
1238
+
1239
+
1240
+
1241
+
1242
+
1243
+
1244
+
1245
+
1246
+
1247
+
1248
+
459
1249
  if didRecordXCTestFailure(since: failureCountBefore),
460
1250
  let failureResponse = xctestRecordedFailureResponse(command: command, response: response)
461
1251
  {
@@ -482,9 +1272,9 @@ extension RunnerTests {
482
1272
  }
483
1273
  }
484
1274
 
485
- /// The dispatched snapshot recovery loop: read-only retry + XCTest-recorded-failure invalidation,
486
- /// matching what `executeOnMainSafely` gives the generic path. `perform` runs the capture and its
487
- /// own bounded main-thread work.
1275
+
1276
+
1277
+
488
1278
  func executeDispatchedWithRecovery(
489
1279
  command: Command,
490
1280
  perform: () throws -> Response
@@ -498,8 +1288,8 @@ extension RunnerTests {
498
1288
  self.currentXCTestFailureCount()
499
1289
  }
500
1290
  let response = try perform()
501
- // Recovered independently — re-entering main for bookkeeping would queue behind the still-
502
- // abandoned XCTest query and re-stall the command (#1244), so skip it until that work drains.
1291
+
1292
+
503
1293
  if hasAbandonedTreeCapture() {
504
1294
  NSLog(
505
1295
  "AGENT_DEVICE_RUNNER_DISPATCH_RECOVERY_SKIPPED_XCTEST_OCCUPIED command=%@",
@@ -554,12 +1344,16 @@ extension RunnerTests {
554
1344
  case .response(let response):
555
1345
  return response
556
1346
  case .context(let context):
557
- return try executeSnapshotPrepared(command: command, activeApp: context.app)
1347
+ return try executeSnapshotPrepared(
1348
+ command: command,
1349
+ activeApp: context.app,
1350
+ systemSurface: context.systemSurface
1351
+ )
558
1352
  }
559
1353
  }
560
1354
 
561
- /// Pure command→options projection, extracted so the runner unit bundle can
562
- /// prove the decoded wire field actually reaches presentation options (#1634 P2).
1355
+
1356
+
563
1357
  static func presentationOptions(from command: Command) -> PresentationOptions {
564
1358
  let customActions = command.customActions ?? false
565
1359
  return PresentationOptions(
@@ -567,24 +1361,34 @@ extension RunnerTests {
567
1361
  depth: command.depth,
568
1362
  scope: command.scope,
569
1363
  raw: command.raw ?? false,
570
- // Custom actions are only readable through the private AX client, so
571
- // asking for them pins that backend rather than silently returning a
572
- // capture that structurally cannot carry them. An explicit pin wins.
1364
+
1365
+
1366
+
573
1367
  preferredBackend: command.preferredBackend
574
1368
  ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil),
575
1369
  customActions: customActions
576
1370
  )
577
1371
  }
578
1372
 
579
- private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
1373
+ private func executeSnapshotPrepared(
1374
+ command: Command,
1375
+ activeApp: XCUIApplication,
1376
+ systemSurface: SystemSurfaceHost? = nil
1377
+ ) throws -> Response {
580
1378
  let options = Self.presentationOptions(from: command)
581
1379
  do {
582
- let payload: DataPayload
1380
+ var payload: DataPayload
583
1381
  if options.raw {
584
1382
  payload = try snapshotRaw(app: activeApp, options: options)
585
1383
  } else {
586
1384
  payload = try snapshotFast(app: activeApp, options: options)
587
1385
  }
1386
+ if let systemSurface {
1387
+ payload.systemSurface = SystemSurfaceProvenancePayload(
1388
+ bundleId: systemSurface.bundleId,
1389
+ kind: systemSurface.kind.rawValue
1390
+ )
1391
+ }
588
1392
  setNeedsPostSnapshotInteractionDelay()
589
1393
  return Response(ok: true, data: payload)
590
1394
  } catch let failure as SnapshotCaptureFailure {
@@ -734,10 +1538,10 @@ extension RunnerTests {
734
1538
  }
735
1539
  case .recordStop:
736
1540
  guard let recorder = activeRecording else {
737
- // The runner protocol is the durable cleanup primitive. A daemon may crash after the
738
- // native stop succeeds but before it commits the resource transition, so exact-owner
739
- // recovery must be able to repeat this command safely. Public `record stop` still owns
740
- // its user-facing no-active validation through the daemon session manifest.
1541
+
1542
+
1543
+
1544
+
741
1545
  return Response(ok: true, data: DataPayload(message: "recording already stopped"))
742
1546
  }
743
1547
  do {
@@ -757,8 +1561,8 @@ extension RunnerTests {
757
1561
  else {
758
1562
  return Response(ok: false, error: ErrorPayload(message: "activate requires appBundleId"))
759
1563
  }
760
- // prepareActiveCommandContext already activated this bundle. Keep this case as the
761
- // explicit acknowledgement after that preflight, not as a second activation.
1564
+
1565
+
762
1566
  return Response(ok: true, data: DataPayload(message: "app activated"))
763
1567
  case .terminate:
764
1568
  guard
@@ -787,10 +1591,20 @@ extension RunnerTests {
787
1591
  routeToSpringboard: Bool = false
788
1592
  ) -> ActiveCommandPreparation {
789
1593
  var activeApp = currentApp ?? app
1594
+ var systemSurface: SystemSurfaceHost? = nil
790
1595
  if routeToSpringboard {
791
1596
  activeApp = springboard
792
1597
  } else if shouldSkipAppActivationPreflight(command) {
793
1598
  activeApp = resolveAppWithoutActivation(command: command)
1599
+ } else if let presented = presentedSystemSurfaceHost() {
1600
+
1601
+
1602
+
1603
+ activeApp = presented.app
1604
+ systemSurface = presented.host
1605
+ if isInteractionCommand(command.command) {
1606
+ applyInteractionStabilizationIfNeeded()
1607
+ }
794
1608
  } else if !isRunnerLifecycleCommand(command.command) {
795
1609
  let normalizedBundleId = command.appBundleId?
796
1610
  .trimmingCharacters(in: .whitespacesAndNewlines)
@@ -802,7 +1616,7 @@ extension RunnerTests {
802
1616
  refreshCachedTargetIfProcessChanged(bundleId: bundleId)
803
1617
  }
804
1618
  } else {
805
- // Do not reuse stale bundle targets when the caller does not explicitly request one.
1619
+
806
1620
  invalidateCachedTarget(reason: "missing_app_bundle")
807
1621
  }
808
1622
 
@@ -851,7 +1665,25 @@ extension RunnerTests {
851
1665
  applyInteractionStabilizationIfNeeded()
852
1666
  }
853
1667
  }
854
- return .context(ActiveCommandContext(app: activeApp))
1668
+ return .context(ActiveCommandContext(app: activeApp, systemSurface: systemSurface))
1669
+ }
1670
+
1671
+
1672
+
1673
+
1674
+
1675
+ private func presentedSystemSurfaceHost() -> (host: SystemSurfaceHost, app: XCUIApplication)? {
1676
+ #if os(iOS)
1677
+ for host in SystemSurfaceHostRegistry.hosts {
1678
+ let candidate = XCUIApplication(bundleIdentifier: host.bundleId)
1679
+ if candidate.state == .runningForeground {
1680
+ return (host, candidate)
1681
+ }
1682
+ }
1683
+ return nil
1684
+ #else
1685
+ return nil
1686
+ #endif
855
1687
  }
856
1688
 
857
1689
  func executeOnMainPrepared(
@@ -896,15 +1728,15 @@ extension RunnerTests {
896
1728
  }
897
1729
  if let element = match.element {
898
1730
  let frame = element.frame
899
- // XCTest reports closed-drawer/off-viewport items as hittable, then
900
- // "taps" coordinates outside the visible window as a silent no-op.
901
- // Refuse instead; the daemon falls back to tree-based resolution,
902
- // which can prefer an on-screen candidate or explain the off-screen
903
- // state. The check uses the main window frame, not app.frame: on RN
904
- // apps app.frame unions transformed subtrees (a closed drawer at
905
- // negative x), so it happily "contains" unreachable coordinates.
906
- // The DECISION lives in TapPointPolicy (golden parity table with the
907
- // TS twin); onScreenWindowFrame only supplies the frame.
1731
+
1732
+
1733
+
1734
+
1735
+
1736
+
1737
+
1738
+
1739
+
908
1740
  if !match.usedNonHittableFallback
909
1741
  && !TapPointPolicy.isAllowed(
910
1742
  elementFrame: frame,
@@ -968,10 +1800,10 @@ extension RunnerTests {
968
1800
  }
969
1801
  let (timing, outcome) = performGesture(activeApp) {
970
1802
  if expectedPoint != nil || match.usedNonHittableFallback {
971
- // Maestro compatibility: RN E2E backdoor controls can be 1x1 and
972
- // reported non-hittable by XCTest. Snapshot-resolved targets also
973
- // need coordinate delivery because XCUIElement.activate() does not
974
- // invoke every React Native accessibility wrapper.
1803
+
1804
+
1805
+
1806
+
975
1807
  return tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y)
976
1808
  }
977
1809
  return activateElement(app: activeApp, element: element, action: "tap by selector")
@@ -1007,9 +1839,9 @@ extension RunnerTests {
1007
1839
  )
1008
1840
  let textInput: XCUIElement?
1009
1841
  if !xCTestTextInputProbeSkipped {
1010
- textInput = textInputAt(app: activeApp, x: x, y: y)
1842
+ textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y)
1011
1843
  } else {
1012
- // A process-scoped tap cannot authorize later typing without concrete element identity.
1844
+
1013
1845
  textInput = nil
1014
1846
  NSLog(
1015
1847
  "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@",
@@ -1056,8 +1888,8 @@ extension RunnerTests {
1056
1888
  }
1057
1889
  let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
1058
1890
  do {
1059
- // mouseClick throws (it has no RunnerInteractionOutcome), so it keeps raw measureGesture
1060
- // and only routes the success payload through gestureResponse.
1891
+
1892
+
1061
1893
  var clickError: Error?
1062
1894
  let timing = measureGesture {
1063
1895
  do {
@@ -1103,9 +1935,9 @@ extension RunnerTests {
1103
1935
  synthesizedPolicyKind: .synthesizedDrag
1104
1936
  )
1105
1937
  case .scroll:
1106
- // Fused frame-resolve + drag scroll for non-tvOS. On iOS this intentionally stays on the
1107
- // AX-free synthesized coordinate lane so scroll keeps working when XCTest cannot serialize
1108
- // the accessibility tree.
1938
+
1939
+
1940
+
1109
1941
  guard let rawDirection = command.direction,
1110
1942
  let direction = RunnerScrollDirection(rawValue: rawDirection)
1111
1943
  else {
@@ -1244,8 +2076,8 @@ extension RunnerTests {
1244
2076
  guard let direction = command.direction else {
1245
2077
  return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
1246
2078
  }
1247
- // swipe returns an optional frame (tvOS-only) rather than a RunnerInteractionOutcome, so it
1248
- // keeps raw measureGesture and only routes the success payload through gestureResponse.
2079
+
2080
+
1249
2081
  var executedFrame: DragVisualizationFrame?
1250
2082
  let timing = measureGesture {
1251
2083
  withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
@@ -1280,12 +2112,12 @@ extension RunnerTests {
1280
2112
  case .screenshot:
1281
2113
  let screenshot: XCUIScreenshot
1282
2114
  #if os(macOS)
1283
- // macOS keeps the app-targeted capture behavior for window-level screenshots.
2115
+
1284
2116
  if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
1285
2117
  let targetApp = XCUIApplication(bundleIdentifier: bundleId)
1286
2118
  targetApp.activate()
1287
2119
  activeApp = targetApp
1288
- // Brief wait for the app transition animation to complete
2120
+
1289
2121
  sleepFor(0.5)
1290
2122
  }
1291
2123
  if command.fullscreen == true {
@@ -1317,7 +2149,7 @@ extension RunnerTests {
1317
2149
  #if os(macOS)
1318
2150
  return Response(ok: true, data: DataPayload(message: filePath))
1319
2151
  #else
1320
- // Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
2152
+
1321
2153
  return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
1322
2154
  #endif
1323
2155
  case .back, .backInApp:
@@ -1387,8 +2219,8 @@ extension RunnerTests {
1387
2219
  Self.alertCommandTimeout(timeoutMs: command.timeoutMs)
1388
2220
  )
1389
2221
  guard let alert = resolveAlert(app: activeApp, deadline: deadline) else {
1390
- // Typed so the host retries on absence alone: a transport or runner failure carries no
1391
- // code and must not be mistaken for "no alert yet" (ALERT_NOT_FOUND_RUNNER_CODE).
2222
+
2223
+
1392
2224
  return Response(
1393
2225
  ok: false,
1394
2226
  error: ErrorPayload(code: "ALERT_NOT_FOUND", message: "alert not found")
@@ -1410,7 +2242,7 @@ extension RunnerTests {
1410
2242
  }
1411
2243
  switch plannedGestureExecution(for: plan) {
1412
2244
  case .fastSwipe:
1413
- // Validation above guarantees a non-empty, single-pointer path for this execution kind.
2245
+
1414
2246
  let first = plan.pointers[0].samples.first!.point
1415
2247
  let last = plan.pointers[0].samples.last!.point
1416
2248
  return canonicalPlannedGestureResponse(
@@ -1509,8 +2341,8 @@ extension RunnerTests {
1509
2341
  #endif
1510
2342
  }
1511
2343
 
1512
- /// Shared drag execution for explicit drag commands. The iOS synthesized lane keeps its
1513
- /// fallback policy explicit; viewport scrolling owns a separate single drag specification.
2344
+
2345
+
1514
2346
  private func executeDragGesture(
1515
2347
  activeApp: XCUIApplication,
1516
2348
  x: Double,
@@ -1771,15 +2603,20 @@ extension RunnerTests {
1771
2603
  }
1772
2604
 
1773
2605
 
2606
+
2607
+
2608
+
2609
+
2610
+
1774
2611
  private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool {
1775
2612
  #if os(iOS)
1776
2613
  if command.command == .alert {
1777
2614
  return true
1778
2615
  }
1779
- // Coordinate-only synthesized taps can run after an AX-fatal foreground screen because they do not
1780
- // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text
1781
- // interactions intentionally stay on the normal AX path because they need an element query.
1782
- // Scroll/drag/sequence keep the normal foreground guard and stabilization path.
2616
+
2617
+
2618
+
2619
+
1783
2620
  guard command.text == nil, command.selectorKey == nil else { return false }
1784
2621
  guard hasCachedTargetForActivationSkip(command: command) else { return false }
1785
2622
  return isCoordinateOnlyTap(command)
@@ -1795,10 +2632,15 @@ extension RunnerTests {
1795
2632
  guard isCoordinateOnlyTap(command) else {
1796
2633
  return false
1797
2634
  }
2635
+
2636
+
2637
+
2638
+
2639
+
1798
2640
  let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget)
1799
- // `runMainThreadWork` executes inline for a main-thread caller, so that path cannot use its
1800
- // timeout machinery. Direct main-thread dispatch keeps the prior synchronous modal check;
1801
- // normal off-main command dispatch uses the bounded probe and post-probe busy recovery.
2641
+
2642
+
2643
+
1802
2644
  if Thread.isMainThread {
1803
2645
  return firstBlockingSystemModal(
1804
2646
  in: springboard,
@@ -1857,8 +2699,8 @@ extension RunnerTests {
1857
2699
  command.x != nil,
1858
2700
  command.y != nil
1859
2701
  {
1860
- // The shared runtime has already resolved this node as non-hittable and
1861
- // deliberately selected Maestro's coordinate compatibility route.
2702
+
2703
+
1862
2704
  maestroNonHittableCoordinateFallbackUsed = true
1863
2705
  }
1864
2706
  let focusStartedAt = Date()
@@ -1906,8 +2748,8 @@ extension RunnerTests {
1906
2748
  if let resolvedCoordinateTarget {
1907
2749
  target = resolvedCoordinateTarget
1908
2750
  } else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1909
- // Released daemons may still send selector-keyed type commands even though current
1910
- // daemons resolve fill selectors through the runtime tree before reaching the runner.
2751
+
2752
+
1911
2753
  let match = findElement(
1912
2754
  app: activeApp,
1913
2755
  selectorKey: selectorKey,
@@ -1989,8 +2831,8 @@ extension RunnerTests {
1989
2831
  } else if point != nil {
1990
2832
  frame = activeApp.frame
1991
2833
  } else {
1992
- // Bare `type` has no coordinate response to normalize. Avoid serializing the
1993
- // application AX tree only to emit unused reference dimensions.
2834
+
2835
+
1994
2836
  frame = .zero
1995
2837
  }
1996
2838
  return Response(