agent-device 0.21.13 → 0.21.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.13.apk → agent-device-android-ime-helper-0.21.14.apk} +0 -0
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.14.apk.sha256 +1 -0
  3. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.13.manifest.json → agent-device-android-ime-helper-0.21.14.manifest.json} +4 -4
  4. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.14.apk +0 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.14.apk.sha256 +1 -0
  6. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.13.manifest.json → agent-device-android-snapshot-helper-0.21.14.manifest.json} +6 -6
  7. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +23 -0
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +12 -19
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +19 -15
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +2 -3
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +16 -20
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +8 -13
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +16 -19
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +8 -24
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +20 -42
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +7 -14
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +17 -17
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +2 -3
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +48 -19
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +0 -1
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +0 -7
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +7 -14
  23. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +16 -19
  24. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift +21 -9
  25. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +59 -10
  26. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift +12 -10
  27. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift +13 -8
  28. package/dist/src/absence-observation-errors.js +1 -1
  29. package/dist/src/adapter.js +1 -1
  30. package/dist/src/adb-executor.js +1 -1
  31. package/dist/src/adb-failure.js +1 -1
  32. package/dist/src/adb-transport.js +1 -1
  33. package/dist/src/agent-browser-provider.js +1 -1
  34. package/dist/src/agent-browser-tool.js +1 -1
  35. package/dist/src/agent-device-client.js +2 -2
  36. package/dist/src/ai-sdk.js +1 -1
  37. package/dist/src/alert-runtime.js +1 -1
  38. package/dist/src/app-catalog.js +1 -1
  39. package/dist/src/app-deployment-resolution.js +1 -1
  40. package/dist/src/app-inventory.js +1 -1
  41. package/dist/src/app-launch.js +1 -1
  42. package/dist/src/app-lifecycle.js +1 -1
  43. package/dist/src/app-lifecycle2.js +1 -1
  44. package/dist/src/app-log-files.js +1 -1
  45. package/dist/src/app-log-runtime2.js +1 -1
  46. package/dist/src/app-resolution.js +2 -2
  47. package/dist/src/apple-runner-platform.js +1 -1
  48. package/dist/src/application-lifecycle-interaction.js +1 -1
  49. package/dist/src/archive.js +1 -1
  50. package/dist/src/artifact-download.js +1 -1
  51. package/dist/src/auth-session.js +1 -1
  52. package/dist/src/auth.js +1 -1
  53. package/dist/src/backend-snapshot-options.js +1 -1
  54. package/dist/src/boot-diagnostics.js +1 -1
  55. package/dist/src/capture-disclosure.js +1 -1
  56. package/dist/src/capture.js +1 -1
  57. package/dist/src/cli-config.js +1 -1
  58. package/dist/src/cli-help.js +2 -2
  59. package/dist/src/cli.js +4 -4
  60. package/dist/src/click-button.js +1 -1
  61. package/dist/src/client-metro.js +1 -1
  62. package/dist/src/clipboard-shell-response.js +1 -1
  63. package/dist/src/command-schema.js +1 -1
  64. package/dist/src/command-tools.js +2 -2
  65. package/dist/src/connection-runtime.js +1 -1
  66. package/dist/src/connection.js +1 -1
  67. package/dist/src/contracts.js +1 -1
  68. package/dist/src/daemon-client-lifecycle.js +1 -1
  69. package/dist/src/daemon.js +1 -1
  70. package/dist/src/debug-symbols-facade.js +1 -1
  71. package/dist/src/device-claim-rule.js +2 -2
  72. package/dist/src/device-inventory-context.js +1 -1
  73. package/dist/src/device-rotation.js +1 -1
  74. package/dist/src/device-selection-resolver.js +1 -1
  75. package/dist/src/device-session.js +1 -1
  76. package/dist/src/device-shell.js +1 -1
  77. package/dist/src/device2.js +1 -1
  78. package/dist/src/devicectl.js +1 -1
  79. package/dist/src/diagnostics.js +1 -1
  80. package/dist/src/dispatch-resolve.js +1 -1
  81. package/dist/src/doctor2.js +1 -1
  82. package/dist/src/engine-eval-script.js +1 -1
  83. package/dist/src/error.js +1 -1
  84. package/dist/src/exec.js +1 -1
  85. package/dist/src/file.js +1 -1
  86. package/dist/src/find.js +1 -1
  87. package/dist/src/format.js +1 -1
  88. package/dist/src/gesture-plan.js +1 -1
  89. package/dist/src/gesture-runtime.js +1 -1
  90. package/dist/src/harmonyos.js +1 -1
  91. package/dist/src/helper.js +1 -1
  92. package/dist/src/human-control-contract.js +1 -1
  93. package/dist/src/human-control.js +1 -1
  94. package/dist/src/index.js +1 -1
  95. package/dist/src/input-actions.js +1 -1
  96. package/dist/src/input-actions2.js +1 -1
  97. package/dist/src/install-artifact-facade.js +1 -1
  98. package/dist/src/install-source-network-transport.js +1 -1
  99. package/dist/src/install-source2.js +1 -1
  100. package/dist/src/interaction-positionals.js +1 -1
  101. package/dist/src/interaction-ref-policy.js +1 -1
  102. package/dist/src/interaction2.js +1 -1
  103. package/dist/src/interactor-operation-binding.js +1 -1
  104. package/dist/src/interactor.js +1 -1
  105. package/dist/src/interactor2.js +1 -1
  106. package/dist/src/interactors.js +1 -1
  107. package/dist/src/internal/bin.js +1 -1
  108. package/dist/src/internal/daemon.js +2 -2
  109. package/dist/src/internal/png-worker.js +1 -1
  110. package/dist/src/internal/run-script-http-child.js +1 -1
  111. package/dist/src/inventory.js +1 -1
  112. package/dist/src/inventory2.js +1 -1
  113. package/dist/src/inventory3.js +1 -1
  114. package/dist/src/inventory5.js +1 -1
  115. package/dist/src/ios-snapshot-runtime.js +1 -1
  116. package/dist/src/lease.js +1 -1
  117. package/dist/src/limrun-download-file.js +1 -1
  118. package/dist/src/limrun-runtime-dependencies.js +1 -1
  119. package/dist/src/linux.js +1 -1
  120. package/dist/src/location-coordinates.js +1 -1
  121. package/dist/src/managed-device-scope.js +1 -1
  122. package/dist/src/mechanics.js +5 -5
  123. package/dist/src/multitouch-support.js +1 -1
  124. package/dist/src/native-build-cache.js +1 -0
  125. package/dist/src/observation.js +1 -1
  126. package/dist/src/parameterized-recorded-fill.js +1 -1
  127. package/dist/src/perf-facade.js +1 -1
  128. package/dist/src/perf-runtime-operation-builder.js +1 -1
  129. package/dist/src/perf-runtime-plan.js +1 -1
  130. package/dist/src/perf-target.js +2 -2
  131. package/dist/src/perf.js +1 -1
  132. package/dist/src/physical-device-control.js +1 -1
  133. package/dist/src/platform-runtime-host.js +1 -1
  134. package/dist/src/platform-runtime-managed-owner.js +1 -1
  135. package/dist/src/platform-runtime-open-target.js +1 -1
  136. package/dist/src/platform-runtime-operation-host.js +2 -2
  137. package/dist/src/platform-runtime-runtime-hints.js +1 -1
  138. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  139. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  140. package/dist/src/platform-runtime-screen-recording-process-host.js +1 -1
  141. package/dist/src/platform-runtime.js +1 -1
  142. package/dist/src/platform-runtime2.js +1 -1
  143. package/dist/src/png-crop-bytes.js +1 -1
  144. package/dist/src/png-size.js +1 -1
  145. package/dist/src/png-worker-client.js +1 -1
  146. package/dist/src/png.js +1 -1
  147. package/dist/src/prepare-kind.js +1 -1
  148. package/dist/src/provider-session-ownership.js +1 -1
  149. package/dist/src/provider-webdriver.js +1 -1
  150. package/dist/src/proxy.js +1 -1
  151. package/dist/src/quality-warnings.js +1 -1
  152. package/dist/src/react-native.js +1 -1
  153. package/dist/src/record-runtime.js +1 -1
  154. package/dist/src/recording.js +1 -1
  155. package/dist/src/register-builtins.js +1 -1
  156. package/dist/src/registry.js +13 -13
  157. package/dist/src/remote-config2.js +1 -1
  158. package/dist/src/replay-divergence.js +1 -1
  159. package/dist/src/replay.js +1 -1
  160. package/dist/src/reporting.js +1 -1
  161. package/dist/src/request-admission.js +1 -1
  162. package/dist/src/request-cancel.js +1 -1
  163. package/dist/src/request-runtime-binding.js +1 -1
  164. package/dist/src/retry.js +1 -1
  165. package/dist/src/runner-cache-metadata.js +1 -1
  166. package/dist/src/runner-client.js +1 -1
  167. package/dist/src/runner-dev-tools-security.js +1 -1
  168. package/dist/src/runner-device-readiness.js +1 -1
  169. package/dist/src/runner-disposal.js +1 -1
  170. package/dist/src/runner-error-classification.js +2 -2
  171. package/dist/src/runner.js +1 -1
  172. package/dist/src/runtime-transport-hints.js +1 -1
  173. package/dist/src/runtime.js +3 -3
  174. package/dist/src/runtime10.js +2 -2
  175. package/dist/src/runtime2.js +1 -1
  176. package/dist/src/runtime3.js +1 -1
  177. package/dist/src/runtime4.js +5 -5
  178. package/dist/src/runtime5.js +1 -1
  179. package/dist/src/runtime6.js +1 -1
  180. package/dist/src/runtime7.js +1 -1
  181. package/dist/src/runtime8.js +1 -1
  182. package/dist/src/runtime9.js +1 -1
  183. package/dist/src/screen-recording-admission-ledger.js +1 -1
  184. package/dist/src/screen-recording-completion.js +1 -1
  185. package/dist/src/screen-recording-options.js +1 -1
  186. package/dist/src/screen-recording-resource-recovery.js +1 -1
  187. package/dist/src/screenshot-density.js +1 -1
  188. package/dist/src/screenshot-image.js +1 -1
  189. package/dist/src/screenshot-runtime.js +1 -1
  190. package/dist/src/screenshot.js +1 -1
  191. package/dist/src/screenshot3.js +1 -1
  192. package/dist/src/script.js +1 -1
  193. package/dist/src/scroll-command.js +1 -1
  194. package/dist/src/scroll-gesture.js +1 -1
  195. package/dist/src/sdk-batch-runner.js +2 -2
  196. package/dist/src/sdk-batch.js +2 -2
  197. package/dist/src/sdk-contracts.d.ts +21 -2
  198. package/dist/src/sdk-contracts.js +1 -1
  199. package/dist/src/sdk-device.js +1 -1
  200. package/dist/src/sdk-io.js +1 -1
  201. package/dist/src/sdk-remote-config.js +1 -1
  202. package/dist/src/selector-runtime.js +1 -1
  203. package/dist/src/server.js +1 -1
  204. package/dist/src/session-allocation.js +1 -1
  205. package/dist/src/session-routing.js +1 -1
  206. package/dist/src/session-runtime-admission.js +1 -1
  207. package/dist/src/session-snapshot.js +1 -1
  208. package/dist/src/session-store.js +2 -2
  209. package/dist/src/session2.js +3 -3
  210. package/dist/src/settings.js +1 -1
  211. package/dist/src/simctl.js +1 -1
  212. package/dist/src/simulator-state.js +1 -1
  213. package/dist/src/simulator.js +1 -1
  214. package/dist/src/snapshot-runtime-binding.js +1 -1
  215. package/dist/src/snapshot-runtime.js +1 -1
  216. package/dist/src/snapshot-state.js +1 -1
  217. package/dist/src/snapshot2.js +1 -1
  218. package/dist/src/snapshot3.js +1 -1
  219. package/dist/src/snapshot4.js +1 -1
  220. package/dist/src/src.js +1 -1
  221. package/dist/src/src7.js +1 -1
  222. package/dist/src/src8.js +2 -2
  223. package/dist/src/string-enum.js +1 -1
  224. package/dist/src/takeover.js +1 -1
  225. package/dist/src/text-entry-focus.js +1 -1
  226. package/dist/src/tool-provider-facade.js +1 -1
  227. package/dist/src/tool-provider.js +2 -2
  228. package/dist/src/tool-provider2.js +1 -1
  229. package/dist/src/touch-runtime.js +1 -1
  230. package/dist/src/transport.js +1 -1
  231. package/dist/src/tv-remote.js +1 -1
  232. package/dist/src/type-text-runtime.js +1 -1
  233. package/dist/src/verified-file.js +1 -1
  234. package/dist/src/video.js +1 -1
  235. package/dist/src/viewport.js +1 -1
  236. package/dist/src/web.js +1 -1
  237. package/dist/src/web2.js +1 -1
  238. package/dist/src/webdriver-source.js +1 -1
  239. package/package.json +1 -1
  240. package/server.json +2 -2
  241. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.13.apk.sha256 +0 -1
  242. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.13.apk +0 -0
  243. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.13.apk.sha256 +0 -1
  244. package/dist/src/snapshot-process.js +0 -1
@@ -41,18 +41,13 @@ extension RunnerTests {
41
41
 
42
42
 
43
43
 
44
- let geometry = try runMainThreadWork(
44
+ let viewport = try runMainThreadWork(
45
45
  "snapshot_viewport",
46
46
  timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)),
47
47
  timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot")
48
48
  ) {
49
- (
50
- viewport: self.safeSnapshotViewport(app: app),
51
- interfaceOrientation: self.capturedInterfaceOrientation(app: app)
52
- )
49
+ self.safeSnapshotViewport(app: app, readingOrientation: true)
53
50
  }
54
- let viewport = geometry.viewport
55
- let interfaceOrientation = geometry.interfaceOrientation
56
51
  let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget
57
52
  let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow))
58
53
  guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else {
@@ -70,7 +65,6 @@ extension RunnerTests {
70
65
  queryRoot: app,
71
66
  rootSnapshot: rootSnapshot,
72
67
  viewport: viewport,
73
- interfaceOrientation: interfaceOrientation,
74
68
  keyboardBand: keyboardBand
75
69
  )
76
70
  }
@@ -140,8 +134,18 @@ extension RunnerTests {
140
134
  return nil
141
135
  }
142
136
 
143
- func safeSnapshotViewport(app: XCUIApplication) -> CGRect {
144
- safely("SNAPSHOT_VIEWPORT", CGRect.infinite) { snapshotViewport(app: app) }
137
+
138
+
139
+
140
+ func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) -> SnapshotViewport {
141
+ safely("SNAPSHOT_VIEWPORT", .missing(reason: .notProvided)) {
142
+ .reported(
143
+ box: snapshotAppFrame(app: app),
144
+ interfaceOrientation: readingOrientation
145
+ ? capturedInterfaceOrientation(app: app)
146
+ : RunnerInterfaceOrientation.unknown
147
+ )
148
+ }
145
149
  }
146
150
 
147
151
  private func describeSnapshotError(_ error: Error) -> String {
@@ -231,16 +235,12 @@ extension RunnerTests {
231
235
  return text.isEmpty ? nil : text
232
236
  }
233
237
 
234
- private func snapshotViewport(app: XCUIApplication) -> CGRect {
238
+ private func snapshotAppFrame(app: XCUIApplication) -> CGRect {
235
239
  #if os(iOS)
236
- let appFrame = onScreenWindowFrame(app: app)
240
+ return onScreenWindowFrame(app: app)
237
241
  #else
238
- let appFrame = app.frame
242
+ return app.frame
239
243
  #endif
240
- if !appFrame.isNull && !appFrame.isEmpty {
241
- return appFrame
242
- }
243
- return .infinite
244
244
  }
245
245
 
246
246
  static func snapshotTraversalIdentity(
@@ -513,8 +513,7 @@ extension RunnerTests {
513
513
  let normalizedAcquisition = acquisition.replacingNodes(
514
514
  SnapshotGeometrySpace.normalized(
515
515
  nodes: acquisition.nodes,
516
- viewport: acquisition.viewport,
517
- interfaceOrientation: acquisition.interfaceOrientation
516
+ viewport: acquisition.viewport
518
517
  )
519
518
  )
520
519
 
@@ -641,7 +640,7 @@ extension RunnerTests {
641
640
  guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false }
642
641
  guard !isRootContainer else { return true }
643
642
 
644
- let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in
643
+ let isFullScreenContainer = node.hittable != true && rootRects.contains { rootRect in
645
644
  rootRect.x == node.rect.x && rootRect.y == node.rect.y
646
645
  && rootRect.width == node.rect.width && rootRect.height == node.rect.height
647
646
  }
@@ -14,15 +14,12 @@ enum RunnerAccessibilityHealth: String, Equatable {
14
14
 
15
15
  enum SynthesizedKeyboardPolicy: String, Equatable, Hashable {
16
16
  case never
17
- case whenAccessibilityHealthy
18
17
  case requiredWhenAvailable
19
18
 
20
19
  func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool {
21
20
  switch self {
22
21
  case .never:
23
22
  return false
24
- case .whenAccessibilityHealthy:
25
- return accessibilityHealth == .healthy
26
23
  case .requiredWhenAvailable:
27
24
  return accessibilityHealth != .unavailable
28
25
  }
@@ -63,7 +60,6 @@ struct SynthesizedCoordinateContext {
63
60
 
64
61
  let resolvedWindow: XCUIElement
65
62
  let keyboardPolicy: SynthesizedKeyboardPolicy
66
- let fallbackPolicy: SynthesizedFallbackPolicy
67
63
  let accessibilityHealth: RunnerAccessibilityHealth
68
64
 
69
65
  func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext {
@@ -71,15 +67,10 @@ struct SynthesizedCoordinateContext {
71
67
  referenceFrame: frame,
72
68
  resolvedWindow: resolvedWindow,
73
69
  keyboardPolicy: keyboardPolicy,
74
- fallbackPolicy: fallbackPolicy,
75
70
  accessibilityHealth: accessibilityHealth
76
71
  )
77
72
  }
78
73
 
79
- var allowsXCTestCoordinateFallback: Bool {
80
- fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: accessibilityHealth)
81
- }
82
-
83
74
  var allowsKeyboardProbe: Bool {
84
75
  keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth)
85
76
  }
@@ -115,10 +106,22 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool {
115
106
  !xCTestChannelPenalized
116
107
  }
117
108
 
118
- func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
119
- steps.contains { step in
120
- step.synthesized == true && step.kind == "tap"
121
- }
109
+
110
+ func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? {
111
+ step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil
112
+ }
113
+
114
+
115
+ enum SynthesizedGestureAttempt {
116
+ case performed(timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double))
117
+
118
+ case xctestFallback(message: String, hint: String?)
119
+
120
+ case refused(
121
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
122
+ message: String,
123
+ hint: String?
124
+ )
122
125
  }
123
126
 
124
127
  extension RunnerTests {
@@ -126,11 +129,36 @@ extension RunnerTests {
126
129
  steps: [SequenceStep],
127
130
  app: XCUIApplication
128
131
  ) -> SynthesizedCoordinateContext? {
129
- guard sequenceHasSynthesizedCoordinateStep(steps) else { return nil }
130
- return synthesizedCoordinateContext(
131
- app: app,
132
- policy: synthesizedGesturePolicy(.synthesizedDrag)
132
+ guard let kind = steps.lazy.compactMap(synthesizedPolicyKind(forSequenceStep:)).first else {
133
+ return nil
134
+ }
135
+ return synthesizedCoordinateContext(app: app, policy: synthesizedGesturePolicy(kind))
136
+ }
137
+
138
+
139
+
140
+ func performSynthesizedGesture(
141
+ _ app: XCUIApplication,
142
+ kind: SynthesizedGesturePolicyKind,
143
+ context: SynthesizedCoordinateContext?,
144
+ synthesize: () -> RunnerInteractionOutcome
145
+ ) -> SynthesizedGestureAttempt {
146
+ let (timing, outcome) = performGesture(app, idleTimeout: false, synthesize)
147
+ guard case .unsupported(let message, let hint) = outcome else {
148
+ logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false)
149
+ return .performed(timing: timing)
150
+ }
151
+ let fallbackAllowed = synthesizedGesturePolicy(kind).fallbackPolicy.allowsXCTestCoordinateFallback(
152
+ accessibilityHealth: context?.accessibilityHealth ?? runnerAccessibilityHealth
153
+ )
154
+ logSynthesizedGesturePolicyDecision(
155
+ kind: kind,
156
+ context: context,
157
+ fallbackAttempted: fallbackAllowed
133
158
  )
159
+ return fallbackAllowed
160
+ ? .xctestFallback(message: message, hint: hint)
161
+ : .refused(timing: timing, message: message, hint: hint)
134
162
  }
135
163
 
136
164
  func logSynthesizedGesturePolicyDecision(
@@ -162,11 +190,12 @@ extension RunnerTests {
162
190
  return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
163
191
  + " context=unavailable fallbackAttempted=\(fallbackAttempted)"
164
192
  }
193
+ let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy
165
194
  return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
166
195
  + " axHealth=\(context.accessibilityHealth.rawValue) frameSource=window"
167
196
  + " keyboardPolicy=\(context.keyboardPolicy.rawValue)"
168
- + " fallbackPolicy=\(context.fallbackPolicy.rawValue)"
169
- + " fallbackAllowed=\(context.allowsXCTestCoordinateFallback)"
197
+ + " fallbackPolicy=\(fallbackPolicy.rawValue)"
198
+ + " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))"
170
199
  + " fallbackAttempted=\(fallbackAttempted)"
171
200
  }
172
201
  }
@@ -384,7 +384,6 @@ extension RunnerTests {
384
384
  referenceFrame: referenceFrame,
385
385
  resolvedWindow: window,
386
386
  keyboardPolicy: policy.keyboardPolicy,
387
- fallbackPolicy: policy.fallbackPolicy,
388
387
  accessibilityHealth: health
389
388
  )
390
389
  #else
@@ -270,11 +270,4 @@ extension RunnerTests {
270
270
  ) -> Bool {
271
271
  repairMode == .replacement && hasX && hasY && xCTestChannelPenalized
272
272
  }
273
-
274
- static func shouldFallbackFromSynthesizedTextEntryFocus(
275
- _ outcome: RunnerInteractionOutcome
276
- ) -> Bool {
277
- if case .unsupported = outcome { return true }
278
- return false
279
- }
280
273
  }
@@ -31,27 +31,20 @@ extension RunnerTests {
31
31
  app: activeApp,
32
32
  policy: synthesizedGesturePolicy(policyKind)
33
33
  )
34
- let (_, outcome) = performGesture(activeApp, idleTimeout: false) {
34
+ switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
35
35
  synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
36
- }
37
- if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) {
38
- logSynthesizedGesturePolicyDecision(
39
- kind: policyKind,
40
- context: context,
41
- fallbackAttempted: true
42
- )
43
- } else {
44
- logSynthesizedGesturePolicyDecision(
45
- kind: policyKind,
46
- context: context,
47
- fallbackAttempted: false
48
- )
36
+ }) {
37
+ case .performed:
49
38
  resolvedCoordinateContext = context
50
39
  resolvedCoordinateTarget = TextEntryTarget(
51
40
  element: nil,
52
41
  refreshPoint: CGPoint(x: x, y: y),
53
42
  prefersFocusedElement: false
54
43
  )
44
+ case .xctestFallback:
45
+ break
46
+ case .refused(_, let message, let hint):
47
+ return unsupportedResponse(message: message, hint: hint)
55
48
  }
56
49
  }
57
50
  #else
@@ -105,7 +105,7 @@ public enum CoordinateSpaceRotation {
105
105
 
106
106
  public enum SnapshotGeometrySpace: Equatable {
107
107
  case appOrientation
108
- case deviceNative(appFrame: CGRect, interfaceOrientation: Int)
108
+ case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int)
109
109
 
110
110
  public static let quarterTurnTolerance: Double = 1
111
111
 
@@ -116,22 +116,26 @@ public enum SnapshotGeometrySpace: Equatable {
116
116
  case .deviceNative(let appFrame, let interfaceOrientation):
117
117
  return CoordinateSpaceRotation.oriented(
118
118
  rect: reportedFrame,
119
- in: appFrame,
119
+ in: appFrame.rect,
120
120
  interfaceOrientation: interfaceOrientation
121
121
  )
122
122
  }
123
123
  }
124
124
 
125
+ /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation.
125
126
  public static func space(
126
127
  reportedBySurfaceHost isSurfaceHost: Bool,
127
128
  reportedFrame: CGRect,
128
129
  inheritedFrom inherited: SnapshotGeometrySpace,
129
- appFrame: CGRect,
130
- interfaceOrientation: Int
130
+ viewport: SnapshotViewport
131
131
  ) -> SnapshotGeometrySpace {
132
132
  guard isSurfaceHost else { return inherited }
133
- guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation }
134
- guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation }
133
+ guard case .reported(let appFrame, let interfaceOrientation) = viewport,
134
+ namesQuarterTurn(interfaceOrientation),
135
+ isQuarterTurned(reportedFrame, relativeTo: appFrame.rect)
136
+ else {
137
+ return .appOrientation
138
+ }
135
139
  return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation)
136
140
  }
137
141
 
@@ -145,7 +149,7 @@ public enum SnapshotGeometrySpace: Equatable {
145
149
  }
146
150
 
147
151
  private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool {
148
- guard isPlottable(frame), isPlottable(appFrame),
152
+ guard SnapshotGeometry.isPositiveFinite(frame),
149
153
  abs(appFrame.width - appFrame.height) > quarterTurnTolerance
150
154
  else {
151
155
  return false
@@ -153,19 +157,12 @@ public enum SnapshotGeometrySpace: Equatable {
153
157
  return abs(frame.width - appFrame.height) <= quarterTurnTolerance
154
158
  && abs(frame.height - appFrame.width) <= quarterTurnTolerance
155
159
  }
156
-
157
- private static func isPlottable(_ frame: CGRect) -> Bool {
158
- frame.origin.x.isFinite && frame.origin.y.isFinite
159
- && frame.width.isFinite && frame.height.isFinite
160
- && frame.width > 0 && frame.height > 0
161
- }
162
160
  }
163
161
 
164
162
  extension SnapshotGeometrySpace {
165
163
  public static func normalized(
166
164
  nodes: [RawAXNode],
167
- viewport: CGRect,
168
- interfaceOrientation: Int
165
+ viewport: SnapshotViewport
169
166
  ) -> [RawAXNode] {
170
167
  let carriers = SnapshotVisibilityFold.visibilityExemptCarrierTypes
171
168
  var spaces = [SnapshotGeometrySpace](repeating: .appOrientation, count: nodes.count)
@@ -180,16 +177,16 @@ extension SnapshotGeometrySpace {
180
177
  ),
181
178
  reportedFrame: node.rect.cgRect,
182
179
  inheritedFrom: parentIndex.map { spaces[$0] } ?? .appOrientation,
183
- appFrame: viewport,
184
- interfaceOrientation: interfaceOrientation
180
+ viewport: viewport
185
181
  )
186
182
  spaces[position] = nodeSpace
187
183
  let frame = nodeSpace.orientedFrame(of: node.rect.cgRect)
188
184
  result.append(
189
185
  node.replacing(
190
186
  rect: SnapshotRect(frame),
191
- hittable: node.parentIndex != nil
192
- && SnapshotGeometry.isGeometricallyActionable(
187
+ hittable: node.parentIndex == nil
188
+ ? false
189
+ : SnapshotGeometry.isGeometricallyActionable(
193
190
  enabled: node.enabled,
194
191
  frame: frame,
195
192
  viewport: viewport
@@ -2,14 +2,23 @@ import Foundation
2
2
  import CoreGraphics
3
3
 
4
4
  public enum SnapshotGeometry {
5
+ /// Twin of `isPositiveFiniteRect` in `packages/kernel/src/rect.ts`. `CGRect.infinite` is built
6
+ /// from finite components, so it is refused by identity.
7
+ public static func isPositiveFinite(_ rect: CGRect) -> Bool {
8
+ !rect.isInfinite
9
+ && rect.origin.x.isFinite && rect.origin.y.isFinite
10
+ && rect.size.width.isFinite && rect.size.height.isFinite
11
+ && rect.size.width > 0 && rect.size.height > 0
12
+ }
13
+
5
14
  public static func effectiveFrame(
6
15
  reportedFrame: CGRect,
7
- viewport: CGRect,
16
+ viewport: SnapshotViewport,
8
17
  ancestorClip: CGRect?
9
18
  ) -> CGRect {
10
19
  var frame = reportedFrame
11
- if !viewport.isInfinite {
12
- frame = clipped(frame, to: viewport)
20
+ if let box = viewport.rect {
21
+ frame = clipped(frame, to: box)
13
22
  }
14
23
  if let ancestorClip {
15
24
  frame = clipped(frame, to: ancestorClip)
@@ -34,15 +43,18 @@ public enum SnapshotGeometry {
34
43
  )
35
44
  }
36
45
 
46
+ /// The shared `hittable` predicate (#1933), twin of `isGeometricallyActionable` in
47
+ /// `packages/kernel/src/rect.ts`, with `CGRect.contains`'s half-open right and bottom edges.
48
+ /// `nil` when only containment is left to decide and the capture has no viewport box: the node's
49
+ /// `hittable` is then absent on the wire, as it is on the host bridge (#2891).
37
50
  public static func isGeometricallyActionable(
38
51
  enabled: Bool,
39
52
  frame: CGRect,
40
- viewport: CGRect
41
- ) -> Bool {
42
- guard enabled, !frame.isNull, !frame.isEmpty else { return false }
43
- if viewport.isInfinite { return true }
44
- let center = CGPoint(x: frame.midX, y: frame.midY)
45
- return viewport.contains(center)
53
+ viewport: SnapshotViewport
54
+ ) -> Bool? {
55
+ guard enabled, isPositiveFinite(frame) else { return false }
56
+ guard let box = viewport.rect else { return nil }
57
+ return box.contains(CGPoint(x: frame.midX, y: frame.midY))
46
58
  }
47
59
 
48
60
  private static func clipped(_ frame: CGRect, to clip: CGRect) -> CGRect {
@@ -38,7 +38,8 @@ public struct RawAXNode: Equatable {
38
38
  public let enabled: Bool
39
39
  public let focused: Bool?
40
40
  public let selected: Bool?
41
- public var hittable: Bool
41
+ /// Geometric actionability; `nil` when the capture has no viewport box to decide it against.
42
+ public var hittable: Bool?
42
43
  public let depth: Int
43
44
  public let parentIndex: Int?
44
45
  public let hiddenContentAbove: Bool?
@@ -55,7 +56,7 @@ public struct RawAXNode: Equatable {
55
56
  enabled: Bool,
56
57
  focused: Bool?,
57
58
  selected: Bool?,
58
- hittable: Bool,
59
+ hittable: Bool?,
59
60
  depth: Int,
60
61
  parentIndex: Int?,
61
62
  hiddenContentAbove: Bool?,
@@ -79,7 +80,7 @@ public struct RawAXNode: Equatable {
79
80
  self.actions = actions
80
81
  }
81
82
 
82
- func replacing(rect: SnapshotRect, hittable: Bool) -> RawAXNode {
83
+ func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode {
83
84
  var updated = self
84
85
  updated.rect = rect
85
86
  updated.hittable = hittable
@@ -153,15 +154,65 @@ public struct PresentationOptions: Equatable {
153
154
  }
154
155
  }
155
156
 
157
+ /// What a capture knows about the viewport hosting its tree: the three cases of the host's
158
+ /// `IosViewportEvidence` (#2891). No rectangle stands for "unknown".
159
+ public enum SnapshotViewport: Equatable {
160
+ /// A box `SnapshotGeometry.isPositiveFinite` accepted. Only the factories below construct one.
161
+ public struct Box: Equatable {
162
+ public let rect: CGRect
163
+
164
+ init(positiveFinite rect: CGRect) {
165
+ self.rect = rect
166
+ }
167
+ }
168
+
169
+ /// The platform's box for the app's surface, with the interface orientation read in the same hop.
170
+ /// Only this case can anchor a rotation (#2612).
171
+ case reported(Box, interfaceOrientation: Int)
172
+ /// A box the capture inferred from its own root element. It clips and contains, and carries no
173
+ /// orientation, so it cannot anchor a rotation.
174
+ case derived(Box)
175
+ case missing(reason: MissingReason)
176
+
177
+ public enum MissingReason: Equatable {
178
+ /// The read was skipped or raised.
179
+ case notProvided
180
+ /// The box read is one `SnapshotGeometry.isPositiveFinite` refuses.
181
+ case invalid
182
+ }
183
+
184
+ public var rect: CGRect? {
185
+ switch self {
186
+ case .reported(let box, _), .derived(let box):
187
+ return box.rect
188
+ case .missing:
189
+ return nil
190
+ }
191
+ }
192
+
193
+ public static func reported(
194
+ box: CGRect,
195
+ interfaceOrientation: Int = RunnerInterfaceOrientation.unknown
196
+ ) -> SnapshotViewport {
197
+ SnapshotGeometry.isPositiveFinite(box)
198
+ ? .reported(Box(positiveFinite: box), interfaceOrientation: interfaceOrientation)
199
+ : .missing(reason: .invalid)
200
+ }
201
+
202
+ public static func derived(box: CGRect) -> SnapshotViewport {
203
+ SnapshotGeometry.isPositiveFinite(box)
204
+ ? .derived(Box(positiveFinite: box))
205
+ : .missing(reason: .invalid)
206
+ }
207
+ }
208
+
156
209
  public struct SnapshotAcquisition {
157
210
  public let hint: CaptureHint
158
211
  public var nodes: [RawAXNode]
159
212
  public let truncated: Bool
160
213
  public let effectiveDepth: Int?
161
214
  public var customActions: SnapshotCustomActionCoverage?
162
- public let viewport: CGRect
163
- /// The app's interface orientation, consumed by the one `normalized` pass; `unknown` turns nothing.
164
- public let interfaceOrientation: Int
215
+ public let viewport: SnapshotViewport
165
216
 
166
217
  public init(
167
218
  hint: CaptureHint,
@@ -169,8 +220,7 @@ public struct SnapshotAcquisition {
169
220
  truncated: Bool,
170
221
  effectiveDepth: Int?,
171
222
  customActions: SnapshotCustomActionCoverage? = nil,
172
- viewport: CGRect,
173
- interfaceOrientation: Int = 0
223
+ viewport: SnapshotViewport
174
224
  ) {
175
225
  self.hint = hint
176
226
  self.nodes = nodes
@@ -178,7 +228,6 @@ public struct SnapshotAcquisition {
178
228
  self.effectiveDepth = effectiveDepth
179
229
  self.customActions = customActions
180
230
  self.viewport = viewport
181
- self.interfaceOrientation = interfaceOrientation
182
231
  }
183
232
 
184
233
  public func replacingNodes(_ nodes: [RawAXNode]) -> SnapshotAcquisition {
@@ -226,7 +275,7 @@ public struct PresentedNode: Codable, Equatable {
226
275
  public let enabled: Bool
227
276
  public let focused: Bool?
228
277
  public let selected: Bool?
229
- public let hittable: Bool
278
+ public let hittable: Bool?
230
279
  public let depth: Int
231
280
  public let parentIndex: Int?
232
281
  public let hiddenContentAbove: Bool?
@@ -1,6 +1,7 @@
1
1
  import Foundation
2
2
  import CoreGraphics
3
3
 
4
+ /// With no viewport box the cumulative clip has no root, so containment has nothing to violate.
4
5
  public enum SnapshotPresentationInvariant {
5
6
  struct ValidationStats: Equatable {
6
7
  let parentClipLookups: Int
@@ -12,7 +13,7 @@ public enum SnapshotPresentationInvariant {
12
13
 
13
14
  public static func validateRegular(
14
15
  _ nodes: [SnapshotPresentationNode],
15
- viewport: CGRect,
16
+ viewport: SnapshotViewport,
16
17
  policy: SnapshotVisibilityFold.Policy
17
18
  ) throws {
18
19
  _ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy)
@@ -20,24 +21,25 @@ public enum SnapshotPresentationInvariant {
20
21
 
21
22
  static func validateRegularWithStats(
22
23
  _ nodes: [SnapshotPresentationNode],
23
- viewport: CGRect,
24
+ viewport: SnapshotViewport,
24
25
  policy: SnapshotVisibilityFold.Policy
25
26
  ) throws -> ValidationStats {
26
27
  var parentClipLookups = 0
27
- var clipIncludingNodeByIndex: [Int: CGRect] = [:]
28
+ var clipIncludingNodeByIndex: [Int: CGRect?] = [:]
28
29
  clipIncludingNodeByIndex.reserveCapacity(nodes.count)
30
+ let rootClip = viewport.rect
29
31
 
30
32
  for node in nodes {
31
- let ancestorClip: CGRect
33
+ let ancestorClip: CGRect?
32
34
  if let parentIndex = node.raw.parentIndex {
33
35
  parentClipLookups += 1
34
- ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? viewport
36
+ ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? rootClip
35
37
  } else {
36
- ancestorClip = viewport
38
+ ancestorClip = rootClip
37
39
  }
38
40
 
39
41
  let frame = node.effectiveRect.cgRect
40
- let clipIncludingNode: CGRect
42
+ let clipIncludingNode: CGRect?
41
43
  if policy == .cursorProjected,
42
44
  SnapshotVisibilityFold.scrollContainerTypeNames.contains(node.raw.type),
43
45
  !frame.isNull,
@@ -50,7 +52,7 @@ public enum SnapshotPresentationInvariant {
50
52
  clipIncludingNodeByIndex[node.raw.index] = clipIncludingNode
51
53
 
52
54
  guard !frame.isNull, !frame.isEmpty else {
53
- if node.raw.hittable {
55
+ if node.raw.hittable == true {
54
56
  throw SnapshotPresentationFailure.regularDegenerateNodeIsActionable(
55
57
  index: node.raw.index,
56
58
  frame: node.effectiveRect
@@ -59,11 +61,11 @@ public enum SnapshotPresentationInvariant {
59
61
  continue
60
62
  }
61
63
 
62
- guard contains(frame, in: ancestorClip) else {
64
+ if let clip = ancestorClip, !contains(frame, in: clip) {
63
65
  throw SnapshotPresentationFailure.regularNodeOutsideCumulativeClip(
64
66
  index: node.raw.index,
65
67
  frame: node.effectiveRect,
66
- clip: SnapshotRect(ancestorClip)
68
+ clip: SnapshotRect(clip)
67
69
  )
68
70
  }
69
71
  }
@@ -100,7 +100,7 @@ public enum SnapshotVisibilityFold {
100
100
  public static func traversalDecision(
101
101
  for node: RawAXNode,
102
102
  parent: TraversalState,
103
- viewport: CGRect,
103
+ viewport: SnapshotViewport,
104
104
  interactiveOnly: Bool,
105
105
  hasChildren: Bool,
106
106
  policy: Policy
@@ -153,7 +153,7 @@ public enum SnapshotVisibilityFold {
153
153
 
154
154
  public static func fold(
155
155
  _ nodes: [RawAXNode],
156
- viewport: CGRect,
156
+ viewport: SnapshotViewport,
157
157
  interactiveOnly: Bool,
158
158
  policy: Policy
159
159
  ) -> [SnapshotPresentationNode] {
@@ -203,12 +203,17 @@ public enum SnapshotVisibilityFold {
203
203
  enabled: node.enabled,
204
204
  focused: node.focused,
205
205
  selected: node.selected,
206
- hittable: node.parentIndex != nil && node.hittable
207
- && SnapshotGeometry.isGeometricallyActionable(
208
- enabled: node.enabled,
209
- frame: decision.effectiveFrame,
210
- viewport: viewport
211
- ),
206
+ hittable: node.parentIndex == nil
207
+ ? false
208
+ : node.hittable.flatMap { sourceHittable in
209
+ sourceHittable
210
+ ? SnapshotGeometry.isGeometricallyActionable(
211
+ enabled: node.enabled,
212
+ frame: decision.effectiveFrame,
213
+ viewport: viewport
214
+ )
215
+ : false
216
+ },
212
217
  depth: outDepth,
213
218
  parentIndex: keptIndex,
214
219
  hiddenContentAbove: node.hiddenContentAbove,