agent-device 0.21.30 → 0.21.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.apk → agent-device-android-ime-helper-0.21.31.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.31.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.manifest.json → agent-device-android-ime-helper-0.21.31.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.30.manifest.json → agent-device-android-snapshot-helper-0.21.31.manifest.json} +7 -7
  7. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +20 -6
  8. package/apple/snapshot-bridge/README.md +16 -3
  9. package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +33 -1
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +110 -8
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +20 -0
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +4 -4
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +1 -1
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +30 -4
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +14 -4
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +81 -41
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +40 -6
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +5 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +74 -10
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +33 -4
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +2 -2
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +23 -2
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +158 -6
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +165 -70
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift +2 -3
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +49 -27
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +17 -9
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +30 -28
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +87 -87
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +4 -3
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +13 -0
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +45 -0
  33. package/dist/src/absence-observation-errors.js +1 -1
  34. package/dist/src/adapter.js +1 -1
  35. package/dist/src/adb-executor.js +1 -1
  36. package/dist/src/adb-failure.js +2 -2
  37. package/dist/src/agent-browser-lifecycle.js +1 -1
  38. package/dist/src/agent-browser-process-record.js +1 -1
  39. package/dist/src/agent-browser-provider.js +1 -1
  40. package/dist/src/agent-browser-tool.js +1 -1
  41. package/dist/src/agent-device-client.js +1 -1
  42. package/dist/src/alert-tree-driver.js +1 -1
  43. package/dist/src/alert.js +1 -1
  44. package/dist/src/alert2.js +1 -1
  45. package/dist/src/android.js +1 -1
  46. package/dist/src/app-backend.js +1 -1
  47. package/dist/src/app-launch.js +1 -1
  48. package/dist/src/app-lifecycle.js +4 -4
  49. package/dist/src/app-log-runtime2.js +3 -3
  50. package/dist/src/app-log-session-resource.js +1 -1
  51. package/dist/src/app-resolution-facade.js +1 -1
  52. package/dist/src/app-resolution.js +2 -2
  53. package/dist/src/apple-runner-platform.js +1 -1
  54. package/dist/src/application-lifecycle-interaction.js +1 -1
  55. package/dist/src/artifact-tracking.js +1 -1
  56. package/dist/src/automation-helper-artifact.js +1 -1
  57. package/dist/src/automation-helper.js +1 -1
  58. package/dist/src/back-mode.js +1 -1
  59. package/dist/src/boot-diagnostics.js +1 -1
  60. package/dist/src/browser-launch.js +1 -1
  61. package/dist/src/cli-config.js +1 -1
  62. package/dist/src/cli-help.js +35 -29
  63. package/dist/src/cli-runner.js +1 -1
  64. package/dist/src/cli.js +3 -3
  65. package/dist/src/client-metro-companion.js +1 -1
  66. package/dist/src/client-metro.js +1 -1
  67. package/dist/src/client-types.d.ts +3 -3
  68. package/dist/src/cloud-artifacts.d.ts +5 -4
  69. package/dist/src/code-signature-cache.js +1 -1
  70. package/dist/src/command-metadata.js +10 -10
  71. package/dist/src/command-tools.js +2 -2
  72. package/dist/src/command.js +1 -1
  73. package/dist/src/composed-inventory-gateways.js +1 -1
  74. package/dist/src/connection-runtime.js +1 -1
  75. package/dist/src/connection.js +5 -5
  76. package/dist/src/contracts.js +1 -1
  77. package/dist/src/daemon-client-ensure.js +1 -1
  78. package/dist/src/daemon-client-lease-beat.js +1 -1
  79. package/dist/src/daemon-client-lifecycle.js +1 -1
  80. package/dist/src/daemon-client-liveness-probe.js +1 -1
  81. package/dist/src/daemon-client.js +1 -1
  82. package/dist/src/daemon-command-registry.js +1 -1
  83. package/dist/src/daemon-composition.js +1 -1
  84. package/dist/src/daemon-policy-file.js +1 -1
  85. package/dist/src/daemon-registration-owner.js +3 -3
  86. package/dist/src/daemon-resolution.js +1 -1
  87. package/dist/src/daemon-startup-diagnostics.js +1 -1
  88. package/dist/src/debug-symbols-facade.js +3 -3
  89. package/dist/src/device-claim-recovery.js +1 -1
  90. package/dist/src/device-claims.js +1 -1
  91. package/dist/src/device-inventory-context.js +1 -1
  92. package/dist/src/device-readiness-runtime.js +1 -1
  93. package/dist/src/device-selection-resolver.js +1 -1
  94. package/dist/src/device.js +1 -1
  95. package/dist/src/devicectl.js +1 -1
  96. package/dist/src/diagnostics.js +1 -1
  97. package/dist/src/display-inventory.js +1 -1
  98. package/dist/src/doctor-facade.js +1 -1
  99. package/dist/src/durable-capture-resource.js +1 -1
  100. package/dist/src/durable-resource.js +1 -1
  101. package/dist/src/engine-eval-script.js +1 -1
  102. package/dist/src/error.js +1 -1
  103. package/dist/src/explicit-session-resolution.js +1 -1
  104. package/dist/src/expo-dev-client-launch.js +1 -1
  105. package/dist/src/file.js +1 -1
  106. package/dist/src/find.js +1 -1
  107. package/dist/src/finders.js +1 -1
  108. package/dist/src/gesture-admission.js +1 -1
  109. package/dist/src/gesture-plan.js +1 -1
  110. package/dist/src/gesture-runtime.js +1 -1
  111. package/dist/src/hdc.js +1 -1
  112. package/dist/src/helper.js +2 -2
  113. package/dist/src/host.js +1 -1
  114. package/dist/src/host2.js +1 -1
  115. package/dist/src/host3.js +1 -1
  116. package/dist/src/host4.js +1 -1
  117. package/dist/src/host5.js +1 -0
  118. package/dist/src/human-control.js +1 -1
  119. package/dist/src/ime-rebind.js +1 -1
  120. package/dist/src/ime-state.js +1 -1
  121. package/dist/src/index.js +1 -1
  122. package/dist/src/input-actions.js +1 -1
  123. package/dist/src/input-actions2.js +1 -1
  124. package/dist/src/install-source-network-transport.js +1 -1
  125. package/dist/src/install-source2.js +1 -1
  126. package/dist/src/interaction-positionals.js +1 -1
  127. package/dist/src/interaction.js +1 -1
  128. package/dist/src/interaction2.js +1 -1
  129. package/dist/src/interactor-operation-catalog.js +1 -1
  130. package/dist/src/interactor.js +5 -5
  131. package/dist/src/interactor2.js +1 -1
  132. package/dist/src/interactor3.js +1 -1
  133. package/dist/src/interactor4.js +1 -1
  134. package/dist/src/interactor5.js +1 -1
  135. package/dist/src/interactors.js +1 -1
  136. package/dist/src/internal/bin.js +1 -1
  137. package/dist/src/internal/daemon.js +1 -1
  138. package/dist/src/internal/png-worker.js +1 -1
  139. package/dist/src/inventory.js +1 -1
  140. package/dist/src/inventory2.js +1 -1
  141. package/dist/src/inventory6.js +1 -1
  142. package/dist/src/ios-node-interpretation.js +1 -1
  143. package/dist/src/ios-settings.js +1 -1
  144. package/dist/src/ios-snapshot-acquisition.js +1 -1
  145. package/dist/src/ios-snapshot-engine.js +1 -1
  146. package/dist/src/ios-snapshot-runtime.js +1 -1
  147. package/dist/src/keyboard-occlusion.js +1 -1
  148. package/dist/src/keyed-lock.js +1 -1
  149. package/dist/src/launch-confirmation.js +1 -1
  150. package/dist/src/launch-services-registration.js +4 -4
  151. package/dist/src/lease.js +1 -1
  152. package/dist/src/legacy.js +1 -1
  153. package/dist/src/limrun-runtime-dependencies.js +1 -1
  154. package/dist/src/local-interactor-operation-set.js +1 -1
  155. package/dist/src/mechanics.js +12 -12
  156. package/dist/src/mobile-snapshot-semantics.js +1 -1
  157. package/dist/src/observability.js +1 -1
  158. package/dist/src/output-path.js +1 -1
  159. package/dist/src/parameterized-recorded-fill.js +1 -1
  160. package/dist/src/perf-facade.js +1 -1
  161. package/dist/src/perf-runtime-operation-builder.js +1 -1
  162. package/dist/src/perf-target.js +1 -1
  163. package/dist/src/physical-device-control.js +1 -1
  164. package/dist/src/platform-runtime-android-adb-host.js +1 -1
  165. package/dist/src/platform-runtime-claim-recovery.js +1 -1
  166. package/dist/src/platform-runtime-daemon-services.js +1 -1
  167. package/dist/src/platform-runtime-host.js +1 -1
  168. package/dist/src/platform-runtime-open-target.js +1 -1
  169. package/dist/src/platform-runtime-operation-host.js +5 -5
  170. package/dist/src/platform-runtime-runtime-hints.js +2 -2
  171. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  172. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  173. package/dist/src/platform-runtime-unavailable.js +1 -1
  174. package/dist/src/platform-runtime.js +1 -1
  175. package/dist/src/platform-runtime2.js +1 -1
  176. package/dist/src/png-resize.js +1 -1
  177. package/dist/src/png-worker-client.js +1 -1
  178. package/dist/src/process-exit.js +1 -1
  179. package/dist/src/provider-device-runtime.d.ts +28 -0
  180. package/dist/src/provider-plugin-catalog.js +2 -0
  181. package/dist/src/provider-plugin.d.ts +1 -1
  182. package/dist/src/provider-webdriver.js +1 -1
  183. package/dist/src/proxy.js +1 -1
  184. package/dist/src/react-native-overlay2.js +1 -1
  185. package/dist/src/react-native.js +1 -1
  186. package/dist/src/react-profile-command.js +1 -1
  187. package/dist/src/record-runtime.js +1 -1
  188. package/dist/src/record-trace.js +1 -1
  189. package/dist/src/recording-scope.js +1 -1
  190. package/dist/src/recording.js +1 -1
  191. package/dist/src/registry.js +1 -1
  192. package/dist/src/remote-config-core.js +1 -1
  193. package/dist/src/replay-backend-host.js +1 -1
  194. package/dist/src/replay-divergence.js +1 -1
  195. package/dist/src/replay-maestro-backend.js +3 -3
  196. package/dist/src/replay-script-selection.js +1 -1
  197. package/dist/src/replay2.js +1 -1
  198. package/dist/src/request-runtime-binding.js +1 -1
  199. package/dist/src/request.js +1 -1
  200. package/dist/src/result-serialization.js +1 -1
  201. package/dist/src/retry.js +1 -1
  202. package/dist/src/runner-cache-retention.js +1 -1
  203. package/dist/src/runner-client.js +1 -1
  204. package/dist/src/runner-dev-tools-security.js +1 -1
  205. package/dist/src/runner-device-readiness.js +1 -1
  206. package/dist/src/runner-device-set.js +1 -1
  207. package/dist/src/runner-error-classification.js +2 -2
  208. package/dist/src/runner-exchange.js +1 -1
  209. package/dist/src/runner-lease.js +1 -1
  210. package/dist/src/runner-operations-facade.js +1 -1
  211. package/dist/src/runner-retention.js +1 -1
  212. package/dist/src/runner-selector-query.js +1 -1
  213. package/dist/src/runner-start-budget.js +1 -1
  214. package/dist/src/runner-startup-transport.js +1 -1
  215. package/dist/src/runner-toolchain-probe.js +1 -1
  216. package/dist/src/runner.js +2 -2
  217. package/dist/src/runtime-admission.js +1 -1
  218. package/dist/src/runtime-deployment.js +1 -1
  219. package/dist/src/runtime-factory.js +1 -1
  220. package/dist/src/runtime-session.js +1 -1
  221. package/dist/src/runtime.js +4 -4
  222. package/dist/src/runtime10.js +2 -2
  223. package/dist/src/runtime2.js +1 -1
  224. package/dist/src/runtime3.js +5 -5
  225. package/dist/src/runtime4.js +1 -1
  226. package/dist/src/runtime5.js +1 -1
  227. package/dist/src/runtime6.js +1 -1
  228. package/dist/src/runtime7.js +1 -1
  229. package/dist/src/runtime8.js +1 -1
  230. package/dist/src/runtime9.js +2 -2
  231. package/dist/src/screen-recording-transport.js +1 -1
  232. package/dist/src/screenshot-density.js +1 -1
  233. package/dist/src/screenshot-runtime.js +1 -1
  234. package/dist/src/screenshot3.js +1 -1
  235. package/dist/src/sdk-batch-runner.js +2 -2
  236. package/dist/src/sdk-batch.js +2 -2
  237. package/dist/src/sdk-contracts.d.ts +9 -0
  238. package/dist/src/sdk-contracts.js +2 -2
  239. package/dist/src/sdk-device.js +1 -1
  240. package/dist/src/sdk-finders.js +1 -1
  241. package/dist/src/sdk-io.js +5 -5
  242. package/dist/src/sdk-remote-config.js +1 -1
  243. package/dist/src/sdk-selectors.js +1 -1
  244. package/dist/src/selection.js +0 -0
  245. package/dist/src/selector-pipeline-policy.js +1 -1
  246. package/dist/src/selector-runtime.js +1 -1
  247. package/dist/src/selectors.js +1 -1
  248. package/dist/src/server-startup.js +1 -1
  249. package/dist/src/session-paths.js +1 -1
  250. package/dist/src/session-replay-vars.js +1 -1
  251. package/dist/src/session-runtime.js +1 -1
  252. package/dist/src/session-store.js +3 -3
  253. package/dist/src/session.js +1 -1
  254. package/dist/src/session2.js +5 -5
  255. package/dist/src/settings-bluetooth.js +1 -1
  256. package/dist/src/settings.js +1 -1
  257. package/dist/src/settings2.js +1 -1
  258. package/dist/src/settle-observation-capture.js +1 -1
  259. package/dist/src/simctl-settings.js +1 -1
  260. package/dist/src/simctl.js +1 -1
  261. package/dist/src/simulator-boot.js +1 -1
  262. package/dist/src/simulator-state.js +1 -1
  263. package/dist/src/simulator.js +1 -1
  264. package/dist/src/snapshot-chrome.js +1 -1
  265. package/dist/src/snapshot-desktop-projection.js +1 -1
  266. package/dist/src/snapshot-helper-install.js +6 -6
  267. package/dist/src/snapshot-node-lookup.js +1 -1
  268. package/dist/src/snapshot-occlusion.js +1 -1
  269. package/dist/src/snapshot-runtime-binding.js +1 -1
  270. package/dist/src/snapshot-runtime-capture-input.js +1 -1
  271. package/dist/src/snapshot-runtime.js +1 -1
  272. package/dist/src/snapshot-state.js +1 -1
  273. package/dist/src/snapshot.js +1 -1
  274. package/dist/src/snapshot3.js +1 -1
  275. package/dist/src/src.js +1 -1
  276. package/dist/src/src3.js +1 -1
  277. package/dist/src/src4.js +1 -1
  278. package/dist/src/store.js +1 -1
  279. package/dist/src/tap-keyboard-occlusion.js +1 -1
  280. package/dist/src/target-evidence-tree.js +1 -1
  281. package/dist/src/target-evidence.js +1 -1
  282. package/dist/src/telephony-runtime.js +1 -1
  283. package/dist/src/text-entry-focus.js +1 -1
  284. package/dist/src/tool-provider.js +3 -3
  285. package/dist/src/tool-provider2.js +1 -1
  286. package/dist/src/tool-provider3.js +1 -1
  287. package/dist/src/toolchain-probe.js +1 -1
  288. package/dist/src/touch-reference-frame.js +1 -1
  289. package/dist/src/touch-runtime.js +1 -1
  290. package/dist/src/touch-runtime2.js +1 -1
  291. package/dist/src/tree.js +1 -1
  292. package/dist/src/tv-remote-runtime.js +1 -1
  293. package/dist/src/tv-remote.js +2 -2
  294. package/dist/src/type-text-runtime.js +1 -1
  295. package/dist/src/update-check.js +1 -1
  296. package/dist/src/vega-executable.js +1 -1
  297. package/dist/src/video.js +1 -1
  298. package/dist/src/webdriver-source.js +1 -1
  299. package/package.json +7 -6
  300. package/server.json +2 -2
  301. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.30.apk.sha256 +0 -1
  302. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk +0 -0
  303. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk.sha256 +0 -1
  304. package/dist/src/physical-device-apps.js +0 -1
@@ -0,0 +1 @@
1
+ 55be3a5a0f49adcb17cd8ea5a93e69530baa444f723cfb47520c3f65953ce683 agent-device-android-ime-helper-0.21.31.apk
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "android-ime-helper",
3
- "version": "0.21.30",
4
- "assetName": "agent-device-android-ime-helper-0.21.30.apk",
5
- "sha256": "a0ebbbfbeec5d64f99ff481b73471805a5c689ccb346cb3fe51385ba772ebedf",
3
+ "version": "0.21.31",
4
+ "assetName": "agent-device-android-ime-helper-0.21.31.apk",
5
+ "sha256": "55be3a5a0f49adcb17cd8ea5a93e69530baa444f723cfb47520c3f65953ce683",
6
6
  "packageName": "com.callstack.agentdevice.imehelper",
7
- "versionCode": 21030,
7
+ "versionCode": 21031,
8
8
  "serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
9
9
  "broadcastProtocol": "android-ime-helper-v1"
10
10
  }
@@ -0,0 +1 @@
1
+ 93dd1c19a865573dbceb442e6745e655ce6999a7f0d504a7f7cba31848624514 agent-device-android-snapshot-helper-0.21.31.apk
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "android-snapshot-helper",
3
- "version": "0.21.30",
4
- "releaseTag": "v0.21.30",
5
- "assetName": "agent-device-android-snapshot-helper-0.21.30.apk",
6
- "apkUrl": "https://github.com/callstack/agent-device/releases/download/v0.21.30/agent-device-android-snapshot-helper-0.21.30.apk",
7
- "sha256": "13b710b0fb1b4e5a0258d10cf7a91c702a528a2e032bb0f8518aca6a8217b48d",
8
- "checksumName": "agent-device-android-snapshot-helper-0.21.30.apk.sha256",
3
+ "version": "0.21.31",
4
+ "releaseTag": "v0.21.31",
5
+ "assetName": "agent-device-android-snapshot-helper-0.21.31.apk",
6
+ "apkUrl": "https://github.com/callstack/agent-device/releases/download/v0.21.31/agent-device-android-snapshot-helper-0.21.31.apk",
7
+ "sha256": "93dd1c19a865573dbceb442e6745e655ce6999a7f0d504a7f7cba31848624514",
8
+ "checksumName": "agent-device-android-snapshot-helper-0.21.31.apk.sha256",
9
9
  "packageName": "com.callstack.agentdevice.snapshothelper",
10
- "versionCode": 21030,
10
+ "versionCode": 21031,
11
11
  "instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
12
12
  "minSdk": 23,
13
13
  "targetSdk": 36,
@@ -259,6 +259,18 @@ func scrollBarValue(current: Double, signedTravel: Double, overflow: Double) ->
259
259
  min(1, max(0, current + signedTravel / overflow))
260
260
  }
261
261
 
262
+ /// A SwiftUI `Form` lists each row as a child of the scroll area, with no single document view as
263
+ /// tall as the content, so the content is the span all of the area's children cover.
264
+ func scrollContentOverflow(area: RectResponse, contentFrames: [RectResponse], isVertical: Bool)
265
+ -> Double
266
+ {
267
+ let spans = contentFrames
268
+ .filter { $0.width > 0 && $0.height > 0 }
269
+ .map { isVertical ? ($0.y, $0.y + $0.height) : ($0.x, $0.x + $0.width) }
270
+ guard let start = spans.map(\.0).min(), let end = spans.map(\.1).max() else { return 0 }
271
+ return end - start - (isVertical ? area.height : area.width)
272
+ }
273
+
262
274
  func isPressableControlRole(_ role: String) -> Bool {
263
275
  pressableControlRoles.contains(role)
264
276
  }
@@ -426,15 +438,17 @@ private func performScrollBarScroll(
426
438
  guard let bar = elementAttribute(element, attribute: barAttribute as String),
427
439
  isAttributeSettable(bar, attribute: kAXValueAttribute as String),
428
440
  let value = numberAttribute(bar, attribute: kAXValueAttribute as String),
429
- let area = rectAttribute(element),
430
- let content = children(of: element)
431
- .filter({ role(of: $0) != "AXScrollBar" })
432
- .compactMap(rectAttribute)
433
- .max(by: { $0.width * $0.height < $1.width * $1.height })
441
+ let area = rectAttribute(element)
434
442
  else {
435
443
  continue
436
444
  }
437
- let overflow = isVertical ? content.height - area.height : content.width - area.width
445
+ let overflow = scrollContentOverflow(
446
+ area: area,
447
+ contentFrames: children(of: element)
448
+ .filter({ role(of: $0) != "AXScrollBar" })
449
+ .compactMap(rectAttribute),
450
+ isVertical: isVertical
451
+ )
438
452
  guard overflow > 0 else { continue }
439
453
  let next = scrollBarValue(current: value, signedTravel: signedTravel, overflow: overflow)
440
454
  return AXUIElementSetAttributeValue(bar, kAXValueAttribute as CFString, NSNumber(value: next))
@@ -4,7 +4,8 @@ This directory contains the small private accessibility reader used by the
4
4
  Apple platform acquisition facet. The framed server and request validation
5
5
  live in `SnapshotBridge.m`; private runtime binding lives in
6
6
  `SnapshotBridgeRuntime.m`. It is compiled for the iOS Simulator on first use
7
- and is never downloaded, pre-signed, or built by npm installation.
7
+ and is never downloaded, pre-signed, or built by npm installation. It serves
8
+ iPhone and iPad Simulators, which both run the iOS Simulator runtime (#3844).
8
9
 
9
10
  The guest process uses the `XCTAccessibilityFramework` remote-access client
10
11
  from the simulator runtime and the `userTestingSnapshotForElement:options:error:`
@@ -45,8 +46,20 @@ acquisition. If the target is covered by system UI, ownership is unavailable,
45
46
  or the owner changes during capture, it returns a typed failure without the
46
47
  app tree. The existing route then uses XCTest, which owns system-modal
47
48
  resolution. Secondary owners such as the return-to-app status-bar control do
48
- not replace the native primary owner. The route's generation circuit remains
49
- disabled after fallback until that app relaunches.
49
+ not replace the native primary owner. An ownership refusal is evidence about
50
+ the screen, so the route sends only that capture to XCTest and keeps the app
51
+ generation on the bridge.
52
+
53
+ ## Window placement
54
+
55
+ The snapshot API reports the frames of an app that runs in an iPadOS window
56
+ (Windowed Apps or Stage Manager) relative to that window, not to the screen.
57
+ Before acquisition, the reader compares the AXRuntime frame of the primary app
58
+ with the frame of the system app. Both frames use the native space of the
59
+ screen, also in landscape. If they differ, the reader returns `app-window-not-full-screen` without
60
+ the app tree. Like an ownership refusal, this is evidence about the screen, so
61
+ the route sends only that capture to XCTest. If the runtime cannot report either
62
+ frame, the reader keeps the capture.
50
63
 
51
64
  ## Remote content
52
65
 
@@ -18,7 +18,7 @@
18
18
  NSString *const kProtocolVersionKey = @"protocolVersion";
19
19
  NSString *const kSourceVersionKey = @"sourceVersion";
20
20
  NSString *const kRequestIdKey = @"requestId";
21
- NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.8.0";
21
+ NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.9.0";
22
22
  const NSUInteger kProtocolVersion = 1;
23
23
  const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024;
24
24
  const NSUInteger kMaximumDepth = 128;
@@ -69,7 +69,9 @@ typedef bool (*AutomationEnabledFn)(void);
69
69
 
70
70
  @protocol AXForegroundElement <NSObject>
71
71
  + (nullable id<AXForegroundElement>)primaryApp;
72
+ + (nullable id<AXForegroundElement>)systemApplication;
72
73
  - (pid_t)pid;
74
+ - (CGRect)frame;
73
75
  @end
74
76
 
75
77
  static NSNumber *finiteNumber(double value)
@@ -187,6 +189,31 @@ static void finishRequestWatchdog(dispatch_source_t watchdog, SnapshotWatchdogSt
187
189
  return [application respondsToSelector:@selector(pid)] && [application pid] == pid;
188
190
  }
189
191
 
192
+ // The snapshot API reports the frames of an app that runs in an iPadOS window relative to that
193
+ // window, so its tree cannot be placed on the screen. AXRuntime reports the primary app's window and
194
+ // the system app, which spans the screen, in one native space. A window that differs from the screen
195
+ // is refused; a runtime that cannot report either frame keeps the capture.
196
+ - (BOOL)primaryAppWindowDiffersFromScreen
197
+ {
198
+ Class<AXForegroundElement> elementClass = (Class<AXForegroundElement>)objc_lookUpClass("AXElement");
199
+ if (![elementClass respondsToSelector:@selector(primaryApp)] ||
200
+ ![elementClass respondsToSelector:@selector(systemApplication)]) {
201
+ return NO;
202
+ }
203
+ id<AXForegroundElement> application = [elementClass primaryApp];
204
+ id<AXForegroundElement> system = [elementClass systemApplication];
205
+ if (![application respondsToSelector:@selector(frame)] || ![system respondsToSelector:@selector(frame)]) {
206
+ return NO;
207
+ }
208
+ CGRect window = [application frame];
209
+ CGRect screen = [system frame];
210
+ if (CGRectIsEmpty(window) || CGRectIsEmpty(screen)) return NO;
211
+ return fabs(CGRectGetMinX(window) - CGRectGetMinX(screen)) > 0.5 ||
212
+ fabs(CGRectGetMinY(window) - CGRectGetMinY(screen)) > 0.5 ||
213
+ fabs(CGRectGetWidth(window) - CGRectGetWidth(screen)) > 0.5 ||
214
+ fabs(CGRectGetHeight(window) - CGRectGetHeight(screen)) > 0.5;
215
+ }
216
+
190
217
  - (nullable id)jsonValue:(id)value name:(NSString *)name
191
218
  {
192
219
  if (!value || value == [NSNull null]) return nil;
@@ -383,6 +410,11 @@ static void finishRequestWatchdog(dispatch_source_t watchdog, SnapshotWatchdogSt
383
410
  finishRequestWatchdog(watchdog, watchdogState);
384
411
  return nil;
385
412
  }
413
+ if ([self primaryAppWindowDiffersFromScreen]) {
414
+ if (error) *error = failureResponse(requestId, @"unsupported", @"app-window-not-full-screen", @"target app window does not fill the screen");
415
+ finishRequestWatchdog(watchdog, watchdogState);
416
+ return nil;
417
+ }
386
418
  snapshot = captureSnapshotTree((__bridge id)raw, maxDepth, maxNodes, nativeLevelsHint,
387
419
  ^id(id element, NSUInteger depth, NSUInteger nodes, NSError **captureError) {
388
420
  if (![self isPrimaryForegroundProcess:pid]) {
@@ -83,6 +83,7 @@ int main(int argc, const char *argv[]) {
83
83
  #endif
84
84
 
85
85
  @interface AgentDeviceRunnerViewController : UIViewController
86
+ @property(nonatomic, strong) UILabel *navigationBackStatus;
86
87
  @property(nonatomic, strong) UILabel *maskedToggleStatus;
87
88
  @property(nonatomic, strong) UILabel *maskedDeliveryStatus;
88
89
  @property(nonatomic, assign) NSUInteger maskedTogglePresses;
@@ -109,6 +110,7 @@ int main(int argc, const char *argv[]) {
109
110
  @property(nonatomic, assign) NSTimeInterval textEntryBurstMinGap;
110
111
  @property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds;
111
112
  @property(nonatomic, strong) NSLayoutConstraint *textEntryFieldTop;
113
+ @property(nonatomic, weak) UITextField *textEntryField;
112
114
  @property(nonatomic, assign) BOOL alertFixtureStarted;
113
115
  @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop;
114
116
  @property(nonatomic, strong) NSTimer *alertBannerRepost;
@@ -129,6 +131,39 @@ int main(int argc, const char *argv[]) {
129
131
  }
130
132
  @end
131
133
 
134
+ // XCTest reads every element's accessibility value each time it snapshots the app, which it does
135
+ // for every element query and every attribute read from a live element. Counting this field's
136
+ // value reads therefore counts the accessibility queries a command makes. The value is masked, so a
137
+ // fill's read-back ends at its first read instead of polling through a stability window whose
138
+ // poll count depends on how fast the host answers.
139
+ static NSUInteger AgentDeviceTextEntryValueReads = 0;
140
+
141
+ @interface AgentDeviceValueReadCountingTextField : AgentDeviceMaskedTextField
142
+ @end
143
+
144
+ @implementation AgentDeviceValueReadCountingTextField
145
+ - (NSString *)accessibilityValue {
146
+ AgentDeviceTextEntryValueReads += 1;
147
+ return [super accessibilityValue];
148
+ }
149
+ @end
150
+
151
+ // Reports the read count as its accessibility label and the counted field's text as its value, so
152
+ // reading either changes no view state and the masked text stays checkable.
153
+ @interface AgentDeviceValueReadCountLabel : UILabel
154
+ @property(nonatomic, weak) UITextField *countedField;
155
+ @end
156
+
157
+ @implementation AgentDeviceValueReadCountLabel
158
+ - (NSString *)accessibilityLabel {
159
+ return [NSString stringWithFormat:@"%lu", (unsigned long)AgentDeviceTextEntryValueReads];
160
+ }
161
+
162
+ - (NSString *)accessibilityValue {
163
+ return self.countedField.text;
164
+ }
165
+ @end
166
+
132
167
  // A field that reformats its own text on every edit into `## ### ## ##` digit groups, the way a
133
168
  // digit-grouping formatter in a Flutter or React Native input does. The accessibility value is
134
169
  // what the user sees — the formatted string — so it never equals the digits a `fill` sent (#2634).
@@ -512,20 +547,33 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
512
547
  }
513
548
  if ((codeComplete && [arguments containsObject:@"--agent-device-text-entry-auto-submit"]) ||
514
549
  (textField.text.length > 0 && [arguments containsObject:@"--agent-device-text-entry-replace-after-input"])) {
515
- UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame];
516
- if (![arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) {
517
- nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input";
518
- }
519
- nextScreenField.borderStyle = UITextBorderStyleRoundedRect;
520
- [textField.superview addSubview:nextScreenField];
521
- [textField removeFromSuperview];
522
- [nextScreenField becomeFirstResponder];
550
+ [self replaceTextEntryFieldWithNextScreenInput:textField];
523
551
  } else if (codeComplete &&
524
552
  [arguments containsObject:@"--agent-device-text-entry-auto-submit-without-successor"]) {
525
553
  [textField removeFromSuperview];
526
554
  }
527
555
  }
528
556
 
557
+ - (void)replaceTextEntryFieldWithNextScreenInput:(UITextField *)textField {
558
+ UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame];
559
+ if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) {
560
+ nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input";
561
+ }
562
+ nextScreenField.borderStyle = UITextBorderStyleRoundedRect;
563
+ [textField.superview addSubview:nextScreenField];
564
+ [textField removeFromSuperview];
565
+ [nextScreenField becomeFirstResponder];
566
+ }
567
+
568
+ // The same navigation as `replace-after-input`, landing when a test presses the button instead of on
569
+ // an edit, so the test decides which runner step it lands between.
570
+ - (void)agentDeviceTextEntryReplaceDidPress:(UIButton *)sender {
571
+ (void)sender;
572
+ if (self.textEntryField.superview != nil) {
573
+ [self replaceTextEntryFieldWithNextScreenInput:self.textEntryField];
574
+ }
575
+ }
576
+
529
577
  // Stops answering the element-type query that found it, the moment it takes focus, under
530
578
  // `--agent-device-text-entry-unqueryable-on-focus`. #3060's Flutter password field reaches this shape
531
579
  // because its two accessibility channels disagree once focused — `TextField` through the legacy
@@ -586,6 +634,10 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
586
634
  }
587
635
  #endif
588
636
 
637
+ - (void)navigationBackPressed {
638
+ self.navigationBackStatus.text = @"Back performed";
639
+ }
640
+
589
641
  - (void)viewDidLoad {
590
642
  [super viewDidLoad];
591
643
 
@@ -605,6 +657,24 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
605
657
 
606
658
  // Keep the fixture behind a launch argument so normal runner snapshots remain unchanged.
607
659
  #if TARGET_OS_IOS
660
+ if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-back-regression"]) {
661
+ self.navigationBackStatus = label;
662
+ label.accessibilityIdentifier = @"agent-device-back-status";
663
+ if ([NSProcessInfo.processInfo.arguments containsObject:@"--navigation-bar"]) {
664
+ UINavigationBar *bar = [[UINavigationBar alloc] initWithFrame:CGRectMake(0, 62, self.view.bounds.size.width, 44)];
665
+ UINavigationItem *item = [[UINavigationItem alloc] initWithTitle:@"Thread"];
666
+ item.leftBarButtonItem = [[UIBarButtonItem alloc] initWithTitle:@"Go back" style:UIBarButtonItemStylePlain target:self action:@selector(navigationBackPressed)];
667
+ bar.items = @[item];
668
+ [self.view addSubview:bar];
669
+ } else {
670
+ UIButton *back = [UIButton buttonWithType:UIButtonTypeSystem];
671
+ back.frame = CGRectMake(12, 62, 88, 44);
672
+ [back setTitle:@"Go back" forState:UIControlStateNormal];
673
+ [back addTarget:self action:@selector(navigationBackPressed) forControlEvents:UIControlEventTouchUpInside];
674
+ [self.view addSubview:back];
675
+ }
676
+ }
677
+
608
678
  if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) {
609
679
  self.alertActionStatus = label;
610
680
  label.accessibilityIdentifier = @"agent-device-alert-actions";
@@ -667,6 +737,8 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
667
737
  [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-submit-clears"];
668
738
  BOOL maskedValue =
669
739
  [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-masked-value"];
740
+ BOOL countsValueReads =
741
+ [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-counts-value-reads"];
670
742
  UITextField *textField;
671
743
  if (digitCountValue) {
672
744
  textField = [[AgentDeviceDigitCountTextField alloc] init];
@@ -679,6 +751,8 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
679
751
  textField = [[AgentDeviceSubmitClearingTextField alloc] init];
680
752
  } else if (maskedValue) {
681
753
  textField = [[AgentDeviceMaskedTextField alloc] init];
754
+ } else if (countsValueReads) {
755
+ textField = [[AgentDeviceValueReadCountingTextField alloc] init];
682
756
  } else {
683
757
  textField = [[UITextField alloc] init];
684
758
  }
@@ -701,6 +775,7 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
701
775
  forControlEvents:UIControlEventEditingChanged];
702
776
  textField.translatesAutoresizingMaskIntoConstraints = NO;
703
777
  [self.view addSubview:textField];
778
+ self.textEntryField = textField;
704
779
  self.textEntryFieldTop = [textField.topAnchor constraintEqualToAnchor:label.bottomAnchor
705
780
  constant:AgentDeviceTextEntryFieldTopInset];
706
781
  [NSLayoutConstraint activateConstraints:@[
@@ -709,6 +784,18 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
709
784
  [textField.widthAnchor constraintEqualToConstant:240],
710
785
  [textField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
711
786
  ]];
787
+ if (countsValueReads) {
788
+ AgentDeviceValueReadCountLabel *valueReads = [[AgentDeviceValueReadCountLabel alloc] init];
789
+ valueReads.countedField = textField;
790
+ valueReads.accessibilityIdentifier = @"agent-device-text-entry-value-reads";
791
+ valueReads.text = @"value reads";
792
+ valueReads.translatesAutoresizingMaskIntoConstraints = NO;
793
+ [self.view addSubview:valueReads];
794
+ [NSLayoutConstraint activateConstraints:@[
795
+ [valueReads.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
796
+ [valueReads.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:12],
797
+ ]];
798
+ }
712
799
  if (currencyValue || submitClears) {
713
800
  UILabel *entryStatus = [[UILabel alloc] init];
714
801
  entryStatus.accessibilityIdentifier = @"agent-device-text-entry-events";
@@ -751,6 +838,21 @@ static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;
751
838
  ]];
752
839
  [self updateTapWitnessStatus];
753
840
  }
841
+ if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-replace-on-request"]) {
842
+ // Above the title, so the software keyboard never covers it.
843
+ UIButton *replace = [UIButton buttonWithType:UIButtonTypeSystem];
844
+ replace.accessibilityIdentifier = @"agent-device-text-entry-replace";
845
+ [replace setTitle:@"Replace input" forState:UIControlStateNormal];
846
+ replace.translatesAutoresizingMaskIntoConstraints = NO;
847
+ [replace addTarget:self
848
+ action:@selector(agentDeviceTextEntryReplaceDidPress:)
849
+ forControlEvents:UIControlEventTouchUpInside];
850
+ [self.view addSubview:replace];
851
+ [NSLayoutConstraint activateConstraints:@[
852
+ [replace.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
853
+ [replace.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-24],
854
+ ]];
855
+ }
754
856
  if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) {
755
857
  self.textEntryFocusWitness = [[UILabel alloc] init];
756
858
  self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus";
@@ -129,6 +129,26 @@ extension RunnerTests {
129
129
  else {
130
130
  return .failure(RunnerAppScreenCaptureFailure(resolution))
131
131
  }
132
+ return capturedScreen(captureScreen, displayID: displayID)
133
+ }
134
+
135
+
136
+
137
+ func captureSoleAppScreen() -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
138
+ var screen: AnyObject?
139
+ var displayID: UInt = 0
140
+ guard RunnerResolveSoleScreen(&screen, &displayID) == nil,
141
+ let captureScreen = screen as? XCUIScreen
142
+ else {
143
+ return .failure(.unresolvedScreen)
144
+ }
145
+ return Self.capturedScreen(captureScreen, displayID: displayID)
146
+ }
147
+
148
+ private static func capturedScreen(
149
+ _ captureScreen: XCUIScreen,
150
+ displayID: UInt
151
+ ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
132
152
  let image = captureScreen.screenshot().image
133
153
  guard let upright = runnerUprightCaptureImage(image) else {
134
154
  return .failure(.unrenderableImage)
@@ -331,10 +331,10 @@ static NSString * _Nullable RunnerCreateEventRecord(
331
331
  if (missing != nil) return missing;
332
332
 
333
333
  NSUInteger displayID = 0;
334
- if (resolvedWindow == nil) {
335
- return @"private XCTest event synthesis unavailable: no resolved application window";
336
- }
337
- NSString *displayError = RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
334
+ // A nil window is the caller's sole-screen route: the app tree could not name its window.
335
+ NSString *displayError = resolvedWindow == nil
336
+ ? RunnerResolveSoleScreenDisplayID(&displayID)
337
+ : RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
338
338
  if (displayError != nil) return displayError;
339
339
  NSInteger interfaceOrientation =
340
340
  ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);
@@ -132,7 +132,7 @@ extension RunnerTests {
132
132
  privateAXAcceptedDepthBundleId = bundleId
133
133
  privateAXAcceptedDepthProcessIdentifier = processIdentifier
134
134
  privateAXAcceptedDepth = depth
135
- privateAXAcceptedDepthUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration)
135
+ privateAXAcceptedDepthUntil = Date().addingTimeInterval(SnapshotXCTestChannelPenalty.duration)
136
136
  privateAXAcceptedDepthLock.unlock()
137
137
  NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_DEPTH_REMEMBERED depth=%ld bundle=%@", depth, bundleId ?? "")
138
138
  }
@@ -223,15 +223,41 @@ extension RunnerTests {
223
223
  guard let x = command.x, let y = command.y else {
224
224
  return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y"))
225
225
  }
226
- let duration = (command.durationMs ?? 800) / 1000.0
226
+ let durationMs = command.durationMs ?? 800
227
+ let fallback: GestureFallback?
228
+ #if os(iOS)
229
+ let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress
230
+ let context = synthesizedCoordinateContext(
231
+ app: activeApp,
232
+ policy: synthesizedGesturePolicy(policyKind)
233
+ )
234
+ switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
235
+ synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context)
236
+ }) {
237
+ case .performed(let timing):
238
+ let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) }
239
+ return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame))
240
+ case .xctestFallback(let message, let hint):
241
+ fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint)
242
+ case .refused(_, let message, let hint):
243
+ return unsupportedResponse(message: message, hint: hint)
244
+ }
245
+ #else
246
+ fallback = nil
247
+ #endif
227
248
  let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
228
249
  let (timing, outcome) = performGesture(activeApp) {
229
- longPressAt(app: activeApp, x: x, y: y, duration: duration)
250
+ longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0)
230
251
  }
231
252
  if let response = unsupportedResponse(for: outcome) {
232
253
  return response
233
254
  }
234
- return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame))
255
+ return gestureResponse(
256
+ message: "long pressed",
257
+ timing: timing,
258
+ frame: .touch(touchFrame),
259
+ fallback: fallback
260
+ )
235
261
  case .drag:
236
262
  guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
237
263
  return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
@@ -644,7 +670,7 @@ extension RunnerTests {
644
670
  return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
645
671
  }
646
672
  case .gestureViewport:
647
- let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
673
+ let frame = gestureViewportFrame(app: activeApp)
648
674
  guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else {
649
675
  return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable"))
650
676
  }
@@ -259,13 +259,23 @@ extension RunnerTests {
259
259
 
260
260
 
261
261
  func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) {
262
+ guard let resolved = resolveRunnerWindowElement(app: app) else {
263
+ return (nil, app.frame)
264
+ }
265
+ return (resolved.window, resolved.frame)
266
+ }
267
+
268
+
269
+
270
+
271
+ func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? {
262
272
  for window in app.windows.allElementsBoundByIndex where window.exists {
263
273
  let frame = window.frame
264
274
  if !frame.isEmpty {
265
275
  return (window, frame)
266
276
  }
267
277
  }
268
- return (nil, app.frame)
278
+ return nil
269
279
  }
270
280
 
271
281
  func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
@@ -325,7 +335,7 @@ extension RunnerTests {
325
335
 
326
336
 
327
337
 
328
- func smallestElementFirst(_ left: XCUIElement, _ right: XCUIElement) -> Bool {
338
+ func smallestElementFirst(_ left: any XCUIElementAttributes, _ right: any XCUIElementAttributes) -> Bool {
329
339
  let leftArea = max(1, left.frame.width * left.frame.height)
330
340
  let rightArea = max(1, right.frame.width * right.frame.height)
331
341
  if leftArea != rightArea {
@@ -342,7 +352,7 @@ extension RunnerTests {
342
352
 
343
353
  func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
344
354
  let point = CGPoint(x: x, y: y)
345
- let textInputCandidates = textInputCandidatesAt(app: app, point: point)
355
+ let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element)
346
356
  for element in textInputCandidates where prefersExpandedTextRead(element) {
347
357
  if let text = readableText(for: element) {
348
358
  return text
@@ -353,7 +363,7 @@ extension RunnerTests {
353
363
  .filter { element in
354
364
  element.exists && !element.frame.isEmpty && element.frame.contains(point)
355
365
  }
356
- .sorted(by: smallestElementFirst)
366
+ .sorted { smallestElementFirst($0, $1) }
357
367
 
358
368
  for element in candidates where prefersExpandedTextRead(element) {
359
369
  if let text = readableText(for: element) {