agent-device 0.21.30 → 0.21.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.apk → agent-device-android-ime-helper-0.21.31.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.31.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.manifest.json → agent-device-android-ime-helper-0.21.31.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.30.manifest.json → agent-device-android-snapshot-helper-0.21.31.manifest.json} +7 -7
  7. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +20 -6
  8. package/apple/snapshot-bridge/README.md +16 -3
  9. package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +33 -1
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +110 -8
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +20 -0
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +4 -4
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +1 -1
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +30 -4
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +14 -4
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +81 -41
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +40 -6
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +5 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +74 -10
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +33 -4
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +2 -2
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +23 -2
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +158 -6
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +165 -70
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift +2 -3
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +49 -27
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +17 -9
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +30 -28
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +87 -87
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +4 -3
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +13 -0
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +45 -0
  33. package/dist/src/absence-observation-errors.js +1 -1
  34. package/dist/src/adapter.js +1 -1
  35. package/dist/src/adb-executor.js +1 -1
  36. package/dist/src/adb-failure.js +2 -2
  37. package/dist/src/agent-browser-lifecycle.js +1 -1
  38. package/dist/src/agent-browser-process-record.js +1 -1
  39. package/dist/src/agent-browser-provider.js +1 -1
  40. package/dist/src/agent-browser-tool.js +1 -1
  41. package/dist/src/agent-device-client.js +1 -1
  42. package/dist/src/alert-tree-driver.js +1 -1
  43. package/dist/src/alert.js +1 -1
  44. package/dist/src/alert2.js +1 -1
  45. package/dist/src/android.js +1 -1
  46. package/dist/src/app-backend.js +1 -1
  47. package/dist/src/app-launch.js +1 -1
  48. package/dist/src/app-lifecycle.js +4 -4
  49. package/dist/src/app-log-runtime2.js +3 -3
  50. package/dist/src/app-log-session-resource.js +1 -1
  51. package/dist/src/app-resolution-facade.js +1 -1
  52. package/dist/src/app-resolution.js +2 -2
  53. package/dist/src/apple-runner-platform.js +1 -1
  54. package/dist/src/application-lifecycle-interaction.js +1 -1
  55. package/dist/src/artifact-tracking.js +1 -1
  56. package/dist/src/automation-helper-artifact.js +1 -1
  57. package/dist/src/automation-helper.js +1 -1
  58. package/dist/src/back-mode.js +1 -1
  59. package/dist/src/boot-diagnostics.js +1 -1
  60. package/dist/src/browser-launch.js +1 -1
  61. package/dist/src/cli-config.js +1 -1
  62. package/dist/src/cli-help.js +35 -29
  63. package/dist/src/cli-runner.js +1 -1
  64. package/dist/src/cli.js +3 -3
  65. package/dist/src/client-metro-companion.js +1 -1
  66. package/dist/src/client-metro.js +1 -1
  67. package/dist/src/client-types.d.ts +3 -3
  68. package/dist/src/cloud-artifacts.d.ts +5 -4
  69. package/dist/src/code-signature-cache.js +1 -1
  70. package/dist/src/command-metadata.js +10 -10
  71. package/dist/src/command-tools.js +2 -2
  72. package/dist/src/command.js +1 -1
  73. package/dist/src/composed-inventory-gateways.js +1 -1
  74. package/dist/src/connection-runtime.js +1 -1
  75. package/dist/src/connection.js +5 -5
  76. package/dist/src/contracts.js +1 -1
  77. package/dist/src/daemon-client-ensure.js +1 -1
  78. package/dist/src/daemon-client-lease-beat.js +1 -1
  79. package/dist/src/daemon-client-lifecycle.js +1 -1
  80. package/dist/src/daemon-client-liveness-probe.js +1 -1
  81. package/dist/src/daemon-client.js +1 -1
  82. package/dist/src/daemon-command-registry.js +1 -1
  83. package/dist/src/daemon-composition.js +1 -1
  84. package/dist/src/daemon-policy-file.js +1 -1
  85. package/dist/src/daemon-registration-owner.js +3 -3
  86. package/dist/src/daemon-resolution.js +1 -1
  87. package/dist/src/daemon-startup-diagnostics.js +1 -1
  88. package/dist/src/debug-symbols-facade.js +3 -3
  89. package/dist/src/device-claim-recovery.js +1 -1
  90. package/dist/src/device-claims.js +1 -1
  91. package/dist/src/device-inventory-context.js +1 -1
  92. package/dist/src/device-readiness-runtime.js +1 -1
  93. package/dist/src/device-selection-resolver.js +1 -1
  94. package/dist/src/device.js +1 -1
  95. package/dist/src/devicectl.js +1 -1
  96. package/dist/src/diagnostics.js +1 -1
  97. package/dist/src/display-inventory.js +1 -1
  98. package/dist/src/doctor-facade.js +1 -1
  99. package/dist/src/durable-capture-resource.js +1 -1
  100. package/dist/src/durable-resource.js +1 -1
  101. package/dist/src/engine-eval-script.js +1 -1
  102. package/dist/src/error.js +1 -1
  103. package/dist/src/explicit-session-resolution.js +1 -1
  104. package/dist/src/expo-dev-client-launch.js +1 -1
  105. package/dist/src/file.js +1 -1
  106. package/dist/src/find.js +1 -1
  107. package/dist/src/finders.js +1 -1
  108. package/dist/src/gesture-admission.js +1 -1
  109. package/dist/src/gesture-plan.js +1 -1
  110. package/dist/src/gesture-runtime.js +1 -1
  111. package/dist/src/hdc.js +1 -1
  112. package/dist/src/helper.js +2 -2
  113. package/dist/src/host.js +1 -1
  114. package/dist/src/host2.js +1 -1
  115. package/dist/src/host3.js +1 -1
  116. package/dist/src/host4.js +1 -1
  117. package/dist/src/host5.js +1 -0
  118. package/dist/src/human-control.js +1 -1
  119. package/dist/src/ime-rebind.js +1 -1
  120. package/dist/src/ime-state.js +1 -1
  121. package/dist/src/index.js +1 -1
  122. package/dist/src/input-actions.js +1 -1
  123. package/dist/src/input-actions2.js +1 -1
  124. package/dist/src/install-source-network-transport.js +1 -1
  125. package/dist/src/install-source2.js +1 -1
  126. package/dist/src/interaction-positionals.js +1 -1
  127. package/dist/src/interaction.js +1 -1
  128. package/dist/src/interaction2.js +1 -1
  129. package/dist/src/interactor-operation-catalog.js +1 -1
  130. package/dist/src/interactor.js +5 -5
  131. package/dist/src/interactor2.js +1 -1
  132. package/dist/src/interactor3.js +1 -1
  133. package/dist/src/interactor4.js +1 -1
  134. package/dist/src/interactor5.js +1 -1
  135. package/dist/src/interactors.js +1 -1
  136. package/dist/src/internal/bin.js +1 -1
  137. package/dist/src/internal/daemon.js +1 -1
  138. package/dist/src/internal/png-worker.js +1 -1
  139. package/dist/src/inventory.js +1 -1
  140. package/dist/src/inventory2.js +1 -1
  141. package/dist/src/inventory6.js +1 -1
  142. package/dist/src/ios-node-interpretation.js +1 -1
  143. package/dist/src/ios-settings.js +1 -1
  144. package/dist/src/ios-snapshot-acquisition.js +1 -1
  145. package/dist/src/ios-snapshot-engine.js +1 -1
  146. package/dist/src/ios-snapshot-runtime.js +1 -1
  147. package/dist/src/keyboard-occlusion.js +1 -1
  148. package/dist/src/keyed-lock.js +1 -1
  149. package/dist/src/launch-confirmation.js +1 -1
  150. package/dist/src/launch-services-registration.js +4 -4
  151. package/dist/src/lease.js +1 -1
  152. package/dist/src/legacy.js +1 -1
  153. package/dist/src/limrun-runtime-dependencies.js +1 -1
  154. package/dist/src/local-interactor-operation-set.js +1 -1
  155. package/dist/src/mechanics.js +12 -12
  156. package/dist/src/mobile-snapshot-semantics.js +1 -1
  157. package/dist/src/observability.js +1 -1
  158. package/dist/src/output-path.js +1 -1
  159. package/dist/src/parameterized-recorded-fill.js +1 -1
  160. package/dist/src/perf-facade.js +1 -1
  161. package/dist/src/perf-runtime-operation-builder.js +1 -1
  162. package/dist/src/perf-target.js +1 -1
  163. package/dist/src/physical-device-control.js +1 -1
  164. package/dist/src/platform-runtime-android-adb-host.js +1 -1
  165. package/dist/src/platform-runtime-claim-recovery.js +1 -1
  166. package/dist/src/platform-runtime-daemon-services.js +1 -1
  167. package/dist/src/platform-runtime-host.js +1 -1
  168. package/dist/src/platform-runtime-open-target.js +1 -1
  169. package/dist/src/platform-runtime-operation-host.js +5 -5
  170. package/dist/src/platform-runtime-runtime-hints.js +2 -2
  171. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  172. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  173. package/dist/src/platform-runtime-unavailable.js +1 -1
  174. package/dist/src/platform-runtime.js +1 -1
  175. package/dist/src/platform-runtime2.js +1 -1
  176. package/dist/src/png-resize.js +1 -1
  177. package/dist/src/png-worker-client.js +1 -1
  178. package/dist/src/process-exit.js +1 -1
  179. package/dist/src/provider-device-runtime.d.ts +28 -0
  180. package/dist/src/provider-plugin-catalog.js +2 -0
  181. package/dist/src/provider-plugin.d.ts +1 -1
  182. package/dist/src/provider-webdriver.js +1 -1
  183. package/dist/src/proxy.js +1 -1
  184. package/dist/src/react-native-overlay2.js +1 -1
  185. package/dist/src/react-native.js +1 -1
  186. package/dist/src/react-profile-command.js +1 -1
  187. package/dist/src/record-runtime.js +1 -1
  188. package/dist/src/record-trace.js +1 -1
  189. package/dist/src/recording-scope.js +1 -1
  190. package/dist/src/recording.js +1 -1
  191. package/dist/src/registry.js +1 -1
  192. package/dist/src/remote-config-core.js +1 -1
  193. package/dist/src/replay-backend-host.js +1 -1
  194. package/dist/src/replay-divergence.js +1 -1
  195. package/dist/src/replay-maestro-backend.js +3 -3
  196. package/dist/src/replay-script-selection.js +1 -1
  197. package/dist/src/replay2.js +1 -1
  198. package/dist/src/request-runtime-binding.js +1 -1
  199. package/dist/src/request.js +1 -1
  200. package/dist/src/result-serialization.js +1 -1
  201. package/dist/src/retry.js +1 -1
  202. package/dist/src/runner-cache-retention.js +1 -1
  203. package/dist/src/runner-client.js +1 -1
  204. package/dist/src/runner-dev-tools-security.js +1 -1
  205. package/dist/src/runner-device-readiness.js +1 -1
  206. package/dist/src/runner-device-set.js +1 -1
  207. package/dist/src/runner-error-classification.js +2 -2
  208. package/dist/src/runner-exchange.js +1 -1
  209. package/dist/src/runner-lease.js +1 -1
  210. package/dist/src/runner-operations-facade.js +1 -1
  211. package/dist/src/runner-retention.js +1 -1
  212. package/dist/src/runner-selector-query.js +1 -1
  213. package/dist/src/runner-start-budget.js +1 -1
  214. package/dist/src/runner-startup-transport.js +1 -1
  215. package/dist/src/runner-toolchain-probe.js +1 -1
  216. package/dist/src/runner.js +2 -2
  217. package/dist/src/runtime-admission.js +1 -1
  218. package/dist/src/runtime-deployment.js +1 -1
  219. package/dist/src/runtime-factory.js +1 -1
  220. package/dist/src/runtime-session.js +1 -1
  221. package/dist/src/runtime.js +4 -4
  222. package/dist/src/runtime10.js +2 -2
  223. package/dist/src/runtime2.js +1 -1
  224. package/dist/src/runtime3.js +5 -5
  225. package/dist/src/runtime4.js +1 -1
  226. package/dist/src/runtime5.js +1 -1
  227. package/dist/src/runtime6.js +1 -1
  228. package/dist/src/runtime7.js +1 -1
  229. package/dist/src/runtime8.js +1 -1
  230. package/dist/src/runtime9.js +2 -2
  231. package/dist/src/screen-recording-transport.js +1 -1
  232. package/dist/src/screenshot-density.js +1 -1
  233. package/dist/src/screenshot-runtime.js +1 -1
  234. package/dist/src/screenshot3.js +1 -1
  235. package/dist/src/sdk-batch-runner.js +2 -2
  236. package/dist/src/sdk-batch.js +2 -2
  237. package/dist/src/sdk-contracts.d.ts +9 -0
  238. package/dist/src/sdk-contracts.js +2 -2
  239. package/dist/src/sdk-device.js +1 -1
  240. package/dist/src/sdk-finders.js +1 -1
  241. package/dist/src/sdk-io.js +5 -5
  242. package/dist/src/sdk-remote-config.js +1 -1
  243. package/dist/src/sdk-selectors.js +1 -1
  244. package/dist/src/selection.js +0 -0
  245. package/dist/src/selector-pipeline-policy.js +1 -1
  246. package/dist/src/selector-runtime.js +1 -1
  247. package/dist/src/selectors.js +1 -1
  248. package/dist/src/server-startup.js +1 -1
  249. package/dist/src/session-paths.js +1 -1
  250. package/dist/src/session-replay-vars.js +1 -1
  251. package/dist/src/session-runtime.js +1 -1
  252. package/dist/src/session-store.js +3 -3
  253. package/dist/src/session.js +1 -1
  254. package/dist/src/session2.js +5 -5
  255. package/dist/src/settings-bluetooth.js +1 -1
  256. package/dist/src/settings.js +1 -1
  257. package/dist/src/settings2.js +1 -1
  258. package/dist/src/settle-observation-capture.js +1 -1
  259. package/dist/src/simctl-settings.js +1 -1
  260. package/dist/src/simctl.js +1 -1
  261. package/dist/src/simulator-boot.js +1 -1
  262. package/dist/src/simulator-state.js +1 -1
  263. package/dist/src/simulator.js +1 -1
  264. package/dist/src/snapshot-chrome.js +1 -1
  265. package/dist/src/snapshot-desktop-projection.js +1 -1
  266. package/dist/src/snapshot-helper-install.js +6 -6
  267. package/dist/src/snapshot-node-lookup.js +1 -1
  268. package/dist/src/snapshot-occlusion.js +1 -1
  269. package/dist/src/snapshot-runtime-binding.js +1 -1
  270. package/dist/src/snapshot-runtime-capture-input.js +1 -1
  271. package/dist/src/snapshot-runtime.js +1 -1
  272. package/dist/src/snapshot-state.js +1 -1
  273. package/dist/src/snapshot.js +1 -1
  274. package/dist/src/snapshot3.js +1 -1
  275. package/dist/src/src.js +1 -1
  276. package/dist/src/src3.js +1 -1
  277. package/dist/src/src4.js +1 -1
  278. package/dist/src/store.js +1 -1
  279. package/dist/src/tap-keyboard-occlusion.js +1 -1
  280. package/dist/src/target-evidence-tree.js +1 -1
  281. package/dist/src/target-evidence.js +1 -1
  282. package/dist/src/telephony-runtime.js +1 -1
  283. package/dist/src/text-entry-focus.js +1 -1
  284. package/dist/src/tool-provider.js +3 -3
  285. package/dist/src/tool-provider2.js +1 -1
  286. package/dist/src/tool-provider3.js +1 -1
  287. package/dist/src/toolchain-probe.js +1 -1
  288. package/dist/src/touch-reference-frame.js +1 -1
  289. package/dist/src/touch-runtime.js +1 -1
  290. package/dist/src/touch-runtime2.js +1 -1
  291. package/dist/src/tree.js +1 -1
  292. package/dist/src/tv-remote-runtime.js +1 -1
  293. package/dist/src/tv-remote.js +2 -2
  294. package/dist/src/type-text-runtime.js +1 -1
  295. package/dist/src/update-check.js +1 -1
  296. package/dist/src/vega-executable.js +1 -1
  297. package/dist/src/video.js +1 -1
  298. package/dist/src/webdriver-source.js +1 -1
  299. package/package.json +7 -6
  300. package/server.json +2 -2
  301. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.30.apk.sha256 +0 -1
  302. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk +0 -0
  303. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk.sha256 +0 -1
  304. package/dist/src/physical-device-apps.js +0 -1
@@ -165,6 +165,63 @@ extension RunnerTests {
165
165
  #endif
166
166
  }
167
167
 
168
+ #if os(iOS)
169
+
170
+
171
+ @MainActor
172
+ func synthesizedLongPressAt(
173
+ app: XCUIApplication,
174
+ x: Double,
175
+ y: Double,
176
+ durationMs: Double,
177
+ context: SynthesizedCoordinateContext?
178
+ ) -> RunnerInteractionOutcome {
179
+ guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else {
180
+ return .unsupported(
181
+ message: "synthesized coordinate long press requires finite coordinates and a positive duration",
182
+ hint: "Retry with finite x and y values and a positive durationMs."
183
+ )
184
+ }
185
+ guard let context else {
186
+ return .unsupported(
187
+ message: "synthesized coordinate long press could not resolve an app window with a finite screen frame",
188
+ hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
189
+ )
190
+ }
191
+ let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
192
+ let point = CoordinateSpaceRotation.native(
193
+ point: CGPoint(x: x, y: y),
194
+ in: context.referenceFrame,
195
+ interfaceOrientation: orientation
196
+ )
197
+ logSynthesizedDispatch(
198
+ kind: "longPress",
199
+ start: point,
200
+ end: nil,
201
+ referenceFrame: context.referenceFrame,
202
+ orientation: orientation
203
+ )
204
+ let samples: [[String: NSNumber]] = [0, durationMs].map { offsetMs in
205
+ [
206
+ "x": NSNumber(value: Double(point.x)),
207
+ "y": NSNumber(value: Double(point.y)),
208
+ "offsetMs": NSNumber(value: offsetMs),
209
+ ]
210
+ }
211
+ if let message = RunnerSynthesizedGesture.synthesizeGesture(
212
+ withApplication: app,
213
+ resolvedWindow: context.resolvedWindow,
214
+ pointerSamples: [samples]
215
+ ) {
216
+ return .unsupported(
217
+ message: message,
218
+ hint: "Private XCTest event synthesis is required for an AX-free long press on iOS; update Xcode if this persists."
219
+ )
220
+ }
221
+ return .performed
222
+ }
223
+ #endif
224
+
168
225
  func keyboardAvoidingDragPoints(
169
226
  app: XCUIApplication,
170
227
  x: Double,
@@ -239,6 +296,21 @@ extension RunnerTests {
239
296
  )
240
297
  }
241
298
 
299
+
300
+
301
+ func touchVisualizationFrame(
302
+ x: Double,
303
+ y: Double,
304
+ context: SynthesizedCoordinateContext
305
+ ) -> TouchVisualizationFrame {
306
+ TouchVisualizationFrame(
307
+ x: Double(context.referenceFrame.minX) + x,
308
+ y: Double(context.referenceFrame.minY) + y,
309
+ referenceWidth: Double(context.referenceFrame.width),
310
+ referenceHeight: Double(context.referenceFrame.height)
311
+ )
312
+ }
313
+
242
314
  func resolvedDragVisualizationFrame(
243
315
  app: XCUIApplication,
244
316
  x: Double,
@@ -269,6 +341,27 @@ extension RunnerTests {
269
341
  return CGRect(x: 0, y: 0, width: 0, height: 0)
270
342
  }
271
343
 
344
+
345
+
346
+
347
+ @MainActor
348
+ func gestureViewportFrame(app: XCUIApplication) -> CGRect {
349
+ #if os(iOS)
350
+ guard let context = synthesizedCoordinateContext(
351
+ app: app,
352
+ policy: synthesizedGesturePolicy(.synthesizedDrag)
353
+ ) else {
354
+ return frameAvoidingKeyboard(app: app, frame: app.frame)
355
+ }
356
+
357
+ return context.resolvedWindow == nil
358
+ ? context.referenceFrame
359
+ : frameAvoidingKeyboard(app: app, frame: context.referenceFrame)
360
+ #else
361
+ return resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
362
+ #endif
363
+ }
364
+
272
365
  private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect {
273
366
  #if os(iOS)
274
367
  guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else {
@@ -384,15 +477,28 @@ extension RunnerTests {
384
477
  @MainActor
385
478
  func synthesizedCoordinateContext(
386
479
  app: XCUIApplication,
387
- policy: SynthesizedGesturePolicy
480
+ policy: SynthesizedGesturePolicy,
481
+ windowQueryRejected: Bool = false
388
482
  ) -> SynthesizedCoordinateContext? {
389
483
  #if os(iOS)
390
- let health = mainOwned.accessibilityHealth
391
- let resolved = resolveRunnerWindow(app: app)
392
- guard let window = resolved.window else {
484
+ var health = mainOwned.accessibilityHealth
485
+ let window: XCUIElement?
486
+ let referenceFrame: CGRect
487
+
488
+
489
+ let rejectedTreeSoleScreenFrame = windowQueryRejected ? soleScreenReferenceFrame(app: app) : nil
490
+ if rejectedTreeSoleScreenFrame == nil, let resolved = resolveRunnerWindowElement(app: app) {
491
+ window = resolved.window
492
+ referenceFrame = resolved.frame
493
+ } else if let soleScreenFrame = rejectedTreeSoleScreenFrame ?? soleScreenReferenceFrame(app: app) {
494
+ window = nil
495
+ referenceFrame = soleScreenFrame
496
+
497
+
498
+ health = .unavailable
499
+ } else {
393
500
  return nil
394
501
  }
395
- let referenceFrame = resolved.frame
396
502
  guard referenceFrame.width.isFinite, referenceFrame.height.isFinite,
397
503
  referenceFrame.width > 0, referenceFrame.height > 0
398
504
  else {
@@ -410,6 +516,52 @@ extension RunnerTests {
410
516
  }
411
517
 
412
518
 
519
+ #if os(iOS)
520
+
521
+
522
+
523
+
524
+
525
+ @MainActor
526
+ func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? {
527
+ guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else {
528
+ return nil
529
+ }
530
+ let screen = UIScreen.main
531
+ let scale = screen.nativeScale
532
+ guard scale > 0 else { return nil }
533
+ let frame = Self.soleScreenReferenceFrame(
534
+ nativePortraitSize: CGSize(
535
+ width: screen.nativeBounds.width / scale,
536
+ height: screen.nativeBounds.height / scale
537
+ ),
538
+ interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
539
+ )
540
+ NSLog(
541
+ "AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@",
542
+ frame.map { NSCoder.string(for: $0) } ?? "none"
543
+ )
544
+ return frame
545
+ }
546
+ #endif
547
+
548
+
549
+
550
+ static func soleScreenReferenceFrame(
551
+ nativePortraitSize size: CGSize,
552
+ interfaceOrientation: Int
553
+ ) -> CGRect? {
554
+ guard size.width > 0, size.height > 0 else { return nil }
555
+ switch interfaceOrientation {
556
+ case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown:
557
+ return CGRect(origin: .zero, size: size)
558
+ case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight:
559
+ return CGRect(x: 0, y: 0, width: size.height, height: size.width)
560
+ default:
561
+ return nil
562
+ }
563
+ }
564
+
413
565
  func keyboardAvoidingSynthesizedDragPoints(
414
566
  app: XCUIApplication,
415
567
  x: Double,
@@ -594,7 +746,7 @@ extension RunnerTests {
594
746
  ]
595
747
  }
596
748
  }
597
- let resolvedWindow = resolveRunnerWindow(app: app).window
749
+ let resolvedWindow = resolveRunnerWindowElement(app: app)?.window
598
750
  if let message = RunnerSynthesizedGesture.synthesizeGesture(
599
751
  withApplication: app,
600
752
  resolvedWindow: resolvedWindow,
@@ -159,13 +159,17 @@ extension RunnerTests {
159
159
 
160
160
  let inputAtRefreshPoint: TextInputAtPoint?
161
161
 
162
+
163
+ let postFocusRead: ResolvedTextEntryInput?
164
+
162
165
  init(
163
166
  element: XCUIElement?,
164
167
  refreshPoint: CGPoint?,
165
168
  prefersFocusedElement: Bool,
166
169
  fromTapWitness: Bool = false,
167
170
  boundIdentity: TextEntryInputIdentity? = nil,
168
- inputAtRefreshPoint: TextInputAtPoint? = nil
171
+ inputAtRefreshPoint: TextInputAtPoint? = nil,
172
+ postFocusRead: ResolvedTextEntryInput? = nil
169
173
  ) {
170
174
  self.element = element
171
175
  self.refreshPoint = refreshPoint
@@ -173,32 +177,113 @@ extension RunnerTests {
173
177
  self.fromTapWitness = fromTapWitness
174
178
  self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity
175
179
  self.inputAtRefreshPoint = inputAtRefreshPoint
180
+ self.postFocusRead = postFocusRead
176
181
  }
177
182
 
178
- func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
179
- guard let nextElement else {
180
- return self
181
- }
182
-
183
183
 
184
184
 
185
- let snapshot = nextElement.exists ? try? nextElement.snapshot() : nil
186
- let frame = snapshot?.frame ?? .zero
187
- let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY)
185
+ func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget {
186
+ guard let input else {
187
+ return self
188
+ }
189
+ let frame = input.snapshot?.frame ?? .zero
188
190
  return TextEntryTarget(
189
- element: nextElement,
190
- refreshPoint: point,
191
+ element: input.element,
192
+ refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY),
191
193
  prefersFocusedElement: prefersFocusedElement,
192
194
  fromTapWitness: fromTapWitness,
193
- boundIdentity: boundIdentity ?? snapshot.map {
194
- TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier)
195
- },
195
+ boundIdentity: boundIdentity ?? input.identity,
196
196
  inputAtRefreshPoint: inputAtRefreshPoint
197
197
  )
198
198
  }
199
199
  }
200
200
 
201
201
 
202
+
203
+
204
+
205
+
206
+
207
+ struct ResolvedTextEntryInput {
208
+ let element: XCUIElement
209
+
210
+ let snapshot: XCUIElementSnapshot?
211
+
212
+ private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) {
213
+ self.element = element
214
+ self.snapshot = snapshot
215
+ }
216
+
217
+
218
+
219
+
220
+ init?(reading element: XCUIElement) {
221
+ var exists = false
222
+ var snapshot: XCUIElementSnapshot?
223
+ if let exception = RunnerObjCExceptionCatcher.catchException({
224
+ exists = element.exists
225
+ snapshot = exists ? try? element.snapshot() : nil
226
+ }) {
227
+ NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_READ_IGNORED_EXCEPTION=%@", exception)
228
+ return nil
229
+ }
230
+ guard exists else { return nil }
231
+ self.init(element: element, snapshot: snapshot)
232
+ }
233
+
234
+
235
+
236
+ init?(readingListed element: XCUIElement) {
237
+ let snapshot = try? element.snapshot()
238
+ guard snapshot != nil || element.exists else { return nil }
239
+ self.init(element: element, snapshot: snapshot)
240
+ }
241
+
242
+ var identity: TextEntryInputIdentity? {
243
+ snapshot.map { TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier) }
244
+ }
245
+
246
+ var attributes: any XCUIElementAttributes {
247
+ if let snapshot {
248
+ return snapshot
249
+ }
250
+ return element
251
+ }
252
+ }
253
+
254
+
255
+
256
+
257
+
258
+ struct TextEntryReader {
259
+ private(set) var target: TextEntryTarget
260
+ private var lastRead: ResolvedTextEntryInput?
261
+ private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput?
262
+
263
+ init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) {
264
+ self.target = target.withResolvedInput(target.postFocusRead)
265
+ lastRead = target.postFocusRead
266
+ self.resolve = resolve
267
+ }
268
+
269
+ mutating func input() -> ResolvedTextEntryInput? {
270
+ if let lastRead {
271
+ return lastRead
272
+ }
273
+ let read = resolve(target)
274
+ target = target.withResolvedInput(read)
275
+ lastRead = read
276
+ return read
277
+ }
278
+
279
+
280
+ mutating func dispatch<T>(_ dispatch: () -> T) -> T {
281
+ lastRead = nil
282
+ return dispatch()
283
+ }
284
+ }
285
+
286
+
202
287
  enum TextEntryInputProbe: Equatable {
203
288
  case input(TextEntryInputIdentity)
204
289
 
@@ -226,9 +311,38 @@ extension RunnerTests {
226
311
  boundIdentity?.isDistinguishable ?? true
227
312
  }
228
313
 
314
+
315
+ enum TextEntryFocusEvidence {
316
+
317
+ case focusEngine
318
+
319
+ case focusedInput
320
+
321
+
322
+ case inputReportsKeyboardFocus(ResolvedTextEntryInput)
323
+
324
+ case keyboardAppeared
325
+
326
+ case focusQueryMatchesInput
327
+
328
+
329
+ var name: String {
330
+ switch self {
331
+ case .focusEngine: return "focus-engine"
332
+ case .focusedInput: return "focused-input"
333
+ case .inputReportsKeyboardFocus: return "input-reports-keyboard-focus"
334
+ case .keyboardAppeared: return "keyboard-appeared"
335
+ case .focusQueryMatchesInput: return "focus-query-matches-input"
336
+ }
337
+ }
338
+ }
339
+
229
340
  struct TextEntryStabilization {
230
341
  let element: XCUIElement?
231
- let focusConfirmed: Bool
342
+
343
+ let evidence: TextEntryFocusEvidence?
344
+
345
+ var focusConfirmed: Bool { evidence != nil }
232
346
  }
233
347
 
234
348
  struct TextEntryTapWitness {
@@ -241,7 +355,8 @@ extension RunnerTests {
241
355
  }
242
356
  }
243
357
 
244
- func clearTextInput(app: XCUIApplication, _ element: XCUIElement) {
358
+
359
+ func clearTextInput(app: XCUIApplication, _ input: ResolvedTextEntryInput) {
245
360
 
246
361
 
247
362
 
@@ -250,16 +365,17 @@ extension RunnerTests {
250
365
 
251
366
 
252
367
 
253
- if let existing = editableTextValue(for: element, treatingPlaceholderAsEmpty: true),
368
+ if let existing = editableTextValue(for: input.attributes, treatingPlaceholderAsEmpty: true),
254
369
  existing.isEmpty {
255
370
  return
256
371
  }
372
+
373
+ let count = estimatedDeleteCount(for: input.attributes)
257
374
  #if !os(tvOS)
258
- moveCaretToEnd(app: app, element: element)
375
+ moveCaretToEnd(app: app, input: input)
259
376
  #endif
260
- let count = estimatedDeleteCount(for: element)
261
377
  let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
262
- element.typeText(deletes)
378
+ input.element.typeText(deletes)
263
379
  }
264
380
 
265
381
  func isTextEntryElement(_ element: XCUIElement) -> Bool {
@@ -285,33 +401,35 @@ extension RunnerTests {
285
401
 
286
402
 
287
403
 
288
- func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
404
+ func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
289
405
  guard let boundIdentity = target.boundIdentity else {
290
- return resolveUnboundTextEntryElement(app: app, target: target)
406
+ return resolveUnboundTextEntryInput(app: app, target: target)
291
407
  }
292
- func isBoundInput(_ candidate: XCUIElement) -> Bool {
293
- candidate.exists && probeTextEntryInput(candidate) == .input(boundIdentity)
408
+ func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? {
409
+ guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil }
410
+ return input
294
411
  }
295
- if target.prefersFocusedElement, let focused = focusedTextInput(app: app), isBoundInput(focused) {
296
- return focused
412
+ if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) {
413
+ return input
297
414
  }
298
- if let element = target.element, isBoundInput(element) {
299
- return element
415
+ if let element = target.element, let input = boundInput(element) {
416
+ return input
300
417
  }
301
- if let input = target.inputAtRefreshPoint, textInputStillResolves(input) {
302
- return input.element
418
+ if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) {
419
+ return read
303
420
  }
304
421
  if target.inputAtRefreshPoint == nil,
305
422
  let refreshPoint = target.refreshPoint,
306
423
  case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint),
307
- let match = candidates.first(where: isBoundInput) {
424
+ let match = candidates.first(where: { $0.identity == boundIdentity }) {
308
425
  return match
309
426
  }
310
- if let focused = focusedTextInput(app: app), isBoundInput(focused) {
311
- return focused
427
+ if let focused = focusedTextInput(app: app), let input = boundInput(focused) {
428
+ return input
312
429
  }
313
- if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), isBoundInput(byIdentifier) {
314
- return byIdentifier
430
+ if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity),
431
+ let input = boundInput(byIdentifier) {
432
+ return input
315
433
  }
316
434
  return nil
317
435
  }
@@ -341,7 +459,7 @@ extension RunnerTests {
341
459
  case .absent:
342
460
  return true
343
461
  case .matches(let candidates):
344
- return candidates.allSatisfy { probeTextEntryInput($0) != .input(boundIdentity) }
462
+ return candidates.allSatisfy { $0.identity != boundIdentity }
345
463
  case .unavailable:
346
464
  return false
347
465
  }
@@ -356,21 +474,6 @@ extension RunnerTests {
356
474
  }
357
475
 
358
476
 
359
-
360
-
361
-
362
-
363
-
364
-
365
-
366
-
367
- func textEntrySnapshotFrame(_ element: XCUIElement) -> CGRect? {
368
-
369
-
370
- safely("TEXT_ENTRY_SNAPSHOT_FRAME", { element.exists ? try? element.snapshot() : nil })?.frame
371
- }
372
-
373
-
374
477
  func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
375
478
  var probe = TextEntryInputProbe.unavailable
376
479
  let (_, exception) = catchingObjCException(fallback: ()) {
@@ -385,30 +488,21 @@ extension RunnerTests {
385
488
  }
386
489
 
387
490
 
388
- private func resolveUnboundTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
389
- if target.prefersFocusedElement {
390
- if let focused = focusedTextInput(app: app) {
391
- return focused
392
- }
393
- if let element = target.element, element.exists {
394
- return element
395
- }
396
- } else {
397
- if let element = target.element, element.exists {
398
- return element
399
- }
491
+ private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
492
+ if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) {
493
+ return focused
494
+ }
495
+ if let element = target.element, let input = ResolvedTextEntryInput(reading: element) {
496
+ return input
400
497
  }
401
498
  if let refreshPoint = target.refreshPoint,
402
499
  let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) {
403
500
  return refreshed
404
501
  }
405
- if let focused = focusedTextInput(app: app) {
406
- return focused
407
- }
408
- return nil
502
+ return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:))
409
503
  }
410
504
 
411
- private func moveCaretToEnd(app: XCUIApplication, element: XCUIElement) {
505
+ private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) {
412
506
  #if os(tvOS)
413
507
  return
414
508
  #else
@@ -423,7 +517,8 @@ extension RunnerTests {
423
517
 
424
518
 
425
519
 
426
- guard let frame = textEntrySnapshotFrame(element), !frame.isEmpty else {
520
+
521
+ guard let frame = input.snapshot?.frame, !frame.isEmpty else {
427
522
  return
428
523
  }
429
524
  _ = tapAt(
@@ -434,7 +529,7 @@ extension RunnerTests {
434
529
  #endif
435
530
  }
436
531
 
437
- private func estimatedDeleteCount(for element: XCUIElement) -> Int {
532
+ private func estimatedDeleteCount(for element: any XCUIElementAttributes) -> Int {
438
533
  let valueText = normalizedElementText(element.value)
439
534
  let base = valueText.isEmpty ? 24 : (valueText.count + 8)
440
535
  return max(24, min(120, base))
@@ -125,9 +125,8 @@ extension RunnerTests {
125
125
 
126
126
 
127
127
 
128
- func textEntryObservation(for element: XCUIElement?) -> TextEntryObservation? {
129
- guard let element,
130
- let snapshot = try? element.snapshot(),
128
+ func textEntryObservation(for input: ResolvedTextEntryInput?) -> TextEntryObservation? {
129
+ guard let snapshot = input?.snapshot,
131
130
  let value = editableTextValue(for: snapshot, treatingPlaceholderAsEmpty: true)
132
131
  else {
133
132
  return nil