agent-device 0.20.5 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +5 -3
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.7.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.7.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.7.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.7.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +48 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +108 -1
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +630 -89
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +78 -0
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +149 -10
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +161 -32
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +4 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +36 -28
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +18 -1
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SelectorMatchPolicyTests.swift +4 -0
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +2 -1
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +145 -20
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +4 -0
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +234 -0
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +106 -448
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +602 -0
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
  37. package/dist/src/agent-browser-tool.js +1 -1
  38. package/dist/src/agent-device-client.js +2 -2
  39. package/dist/src/android-adb.js +1 -1
  40. package/dist/src/android.js +1 -1
  41. package/dist/src/app-helpers.js +1 -1
  42. package/dist/src/app-lifecycle.js +7 -7
  43. package/dist/src/app-lifecycle2.js +2 -0
  44. package/dist/src/app-log-files.js +1 -0
  45. package/dist/src/app-log-reconnect.js +1 -0
  46. package/dist/src/app-log-runtime.js +4 -0
  47. package/dist/src/app-resolution.js +2 -1
  48. package/dist/src/apps.js +5 -5
  49. package/dist/src/audio-probe.js +1 -1
  50. package/dist/src/auth-session.js +1 -0
  51. package/dist/src/auth.js +2 -0
  52. package/dist/src/boot-diagnostics.js +2 -2
  53. package/dist/src/cli-config.js +1 -1
  54. package/dist/src/cli-grammar.js +1 -0
  55. package/dist/src/cli-help.js +204 -254
  56. package/dist/src/cli-output.js +1 -0
  57. package/dist/src/cli-runner.js +1 -2
  58. package/dist/src/cli.js +6 -16
  59. package/dist/src/client-metro.js +1 -1
  60. package/dist/src/cloud-artifacts.d.ts +2 -137
  61. package/dist/src/command-metadata.js +1 -1
  62. package/dist/src/command-schema.js +3 -3
  63. package/dist/src/connection-runtime.js +1 -0
  64. package/dist/src/connection.js +5 -0
  65. package/dist/src/context.js +1 -1
  66. package/dist/src/daemon-shutdown-report.js +1 -0
  67. package/dist/src/daemon.js +2 -0
  68. package/dist/src/debug-symbols.js +3 -3
  69. package/dist/src/device-claim-inspection.js +1 -0
  70. package/dist/src/device-input-state.js +2 -2
  71. package/dist/src/device.js +1 -1
  72. package/dist/src/device2.js +2 -0
  73. package/dist/src/find.js +1 -1
  74. package/dist/src/finders.js +1 -1
  75. package/dist/src/generic-settle.js +1 -0
  76. package/dist/src/generic.js +1 -1
  77. package/dist/src/harmonyos.js +1 -0
  78. package/dist/src/helper.js +1 -1
  79. package/dist/src/host-process.js +2 -2
  80. package/dist/src/ime-lifecycle.js +7 -7
  81. package/dist/src/index.d.ts +69 -18
  82. package/dist/src/input-actions.js +4 -4
  83. package/dist/src/input-actions2.js +1 -1
  84. package/dist/src/input-actions3.js +1 -0
  85. package/dist/src/install-artifact.js +1 -5
  86. package/dist/src/install-artifact2.js +1 -1
  87. package/dist/src/install-source2.js +1 -1
  88. package/dist/src/interaction-runtime.js +1 -0
  89. package/dist/src/interaction-snapshot.js +1 -1
  90. package/dist/src/interaction-touch-reference-frame.js +1 -0
  91. package/dist/src/interaction.js +1 -1
  92. package/dist/src/interactor.js +2 -2
  93. package/dist/src/interactor2.js +1 -1
  94. package/dist/src/internal/bin.js +2 -2
  95. package/dist/src/internal/daemon.js +9 -16
  96. package/dist/src/inventory.js +5 -0
  97. package/dist/src/inventory2.js +1 -0
  98. package/dist/src/inventory3.js +2 -0
  99. package/dist/src/inventory4.js +1 -0
  100. package/dist/src/inventory5.js +1 -0
  101. package/dist/src/inventory6.js +1 -0
  102. package/dist/src/lease-context.js +1 -1
  103. package/dist/src/lease.js +1 -1
  104. package/dist/src/limrun-runtime-dependencies.js +1 -0
  105. package/dist/src/limrun.d.ts +10 -104
  106. package/dist/src/limrun.js +1 -1
  107. package/dist/src/linux.js +1 -1
  108. package/dist/src/logcat.js +1 -1
  109. package/dist/src/manifest.js +1 -1
  110. package/dist/src/owner-identity.js +1 -0
  111. package/dist/src/physical-device-control.js +1 -1
  112. package/dist/src/platform-runtime-app-log-android-transport.js +1 -0
  113. package/dist/src/platform-runtime-app-log-process.js +1 -0
  114. package/dist/src/platform-runtime-host.js +1 -0
  115. package/dist/src/platform-runtime-network-web-transport.js +1 -0
  116. package/dist/src/platform-runtime-operation-host.js +5 -0
  117. package/dist/src/platform-runtime-toolchain-host.js +1 -0
  118. package/dist/src/platform-runtime.js +1 -0
  119. package/dist/src/prepare-kind.js +15 -0
  120. package/dist/src/process-entry.js +1 -0
  121. package/dist/src/process-exit.js +1 -0
  122. package/dist/src/process-lock.js +1 -1
  123. package/dist/src/provider-webdriver.js +1 -0
  124. package/dist/src/provider.js +1 -0
  125. package/dist/src/proxy.js +2 -0
  126. package/dist/src/react-native.js +1 -1
  127. package/dist/src/record-trace.js +1 -1
  128. package/dist/src/registry.js +25 -24
  129. package/dist/src/remote-config.js +1 -1
  130. package/dist/src/remote-config2.js +1 -1
  131. package/dist/src/replay.js +3 -0
  132. package/dist/src/reporting.js +4 -4
  133. package/dist/src/runner-client.js +1 -1
  134. package/dist/src/runner-disposal.js +1 -1
  135. package/dist/src/runner-lease.js +1 -1
  136. package/dist/src/runtime.js +3 -3
  137. package/dist/src/runtime2.js +4 -0
  138. package/dist/src/runtime3.js +3 -0
  139. package/dist/src/runtime4.js +1 -0
  140. package/dist/src/runtime5.js +1 -0
  141. package/dist/src/runtime6.js +1 -0
  142. package/dist/src/runtime7.js +1 -0
  143. package/dist/src/screenshot-result.js +4 -4
  144. package/dist/src/screenshot.js +1 -0
  145. package/dist/src/sdk-android-adb.d.ts +20 -2
  146. package/dist/src/sdk-android-adb.js +2 -2
  147. package/dist/src/sdk-batch-runner.js +3 -3
  148. package/dist/src/sdk-contracts.d.ts +47 -6
  149. package/dist/src/sdk-contracts.js +1 -1
  150. package/dist/src/sdk-device.js +1 -1
  151. package/dist/src/sdk-finders.d.ts +4 -20
  152. package/dist/src/sdk-finders.js +1 -1
  153. package/dist/src/sdk-remote-config.js +1 -1
  154. package/dist/src/sdk-selectors.d.ts +601 -0
  155. package/dist/src/sdk-selectors.js +1 -1
  156. package/dist/src/selector-runtime.js +1 -1
  157. package/dist/src/selector-vocabulary.js +1 -1
  158. package/dist/src/selectors.d.ts +1 -46
  159. package/dist/src/selectors.js +1 -1
  160. package/dist/src/server.js +3 -3
  161. package/dist/src/session-target-evidence.js +1 -1
  162. package/dist/src/session.js +10 -9
  163. package/dist/src/shared.js +2 -0
  164. package/dist/src/shell-quote.js +1 -0
  165. package/dist/src/simulator.js +1 -1
  166. package/dist/src/snapshot-runtime.js +1 -0
  167. package/dist/src/snapshot-session.js +1 -0
  168. package/dist/src/snapshot.js +1 -1
  169. package/dist/src/snapshot2.js +1 -1
  170. package/dist/src/src.js +1 -1
  171. package/dist/src/src2.js +2 -0
  172. package/dist/src/src3.js +1 -0
  173. package/dist/src/text-surface.js +1 -0
  174. package/dist/src/timing-safe-equal.js +1 -0
  175. package/dist/src/tool-provider.js +3 -3
  176. package/dist/src/tool-provider2.js +1 -1
  177. package/dist/src/tool-provider3.js +1 -1
  178. package/dist/src/update-check.js +1 -1
  179. package/dist/src/version.js +1 -1
  180. package/dist/src/web.js +1 -1
  181. package/package.json +72 -50
  182. package/server.json +3 -3
  183. package/skills/agent-device/SKILL.md +6 -31
  184. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
  185. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
  186. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
  187. package/dist/src/app-log-request-scope.js +0 -1
  188. package/dist/src/devices.js +0 -1
  189. package/dist/src/devices2.js +0 -1
  190. package/dist/src/devices3.js +0 -1
  191. package/dist/src/devices4.js +0 -1
  192. package/dist/src/interaction.d.ts +0 -230
  193. package/dist/src/provider-limrun-runtime.js +0 -1
  194. package/dist/src/register-builtins.js +0 -1
  195. package/dist/src/screenshot-geometry.js +0 -1
  196. package/dist/src/viewport-dimension.js +0 -14
@@ -136,8 +136,9 @@ struct Command: Codable {
136
136
  let gesturePlan: RunnerGesturePlan?
137
137
  let outPath: String?
138
138
  let fps: Int?
139
- let maxSize: Int?
140
139
  let interactiveOnly: Bool?
140
+ let preferredBackend: String?
141
+ let customActions: Bool?
141
142
  let depth: Int?
142
143
  let scope: String?
143
144
  let raw: Bool?
@@ -257,11 +258,13 @@ struct DataPayload: Codable {
257
258
  let visible: Bool?
258
259
  let wasVisible: Bool?
259
260
  let dismissed: Bool?
261
+ let keyboardDismissMechanism: String?
260
262
  let orientation: String?
261
263
  let gestureFallback: String?
262
264
  let gestureFallbackMessage: String?
263
265
  let gestureFallbackHint: String?
264
266
  let maestroNonHittableCoordinateFallbackUsed: Bool?
267
+ let textEntryRoute: String?
265
268
  let runnerFatal: Bool?
266
269
  let runnerFatalReason: String?
267
270
  let completedSteps: Int?
@@ -297,11 +300,13 @@ struct DataPayload: Codable {
297
300
  visible: Bool? = nil,
298
301
  wasVisible: Bool? = nil,
299
302
  dismissed: Bool? = nil,
303
+ keyboardDismissMechanism: String? = nil,
300
304
  orientation: String? = nil,
301
305
  gestureFallback: String? = nil,
302
306
  gestureFallbackMessage: String? = nil,
303
307
  gestureFallbackHint: String? = nil,
304
308
  maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
309
+ textEntryRoute: String? = nil,
305
310
  runnerFatal: Bool? = nil,
306
311
  runnerFatalReason: String? = nil,
307
312
  completedSteps: Int? = nil,
@@ -336,11 +341,13 @@ struct DataPayload: Codable {
336
341
  self.visible = visible
337
342
  self.wasVisible = wasVisible
338
343
  self.dismissed = dismissed
344
+ self.keyboardDismissMechanism = keyboardDismissMechanism
339
345
  self.orientation = orientation
340
346
  self.gestureFallback = gestureFallback
341
347
  self.gestureFallbackMessage = gestureFallbackMessage
342
348
  self.gestureFallbackHint = gestureFallbackHint
343
349
  self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
350
+ self.textEntryRoute = textEntryRoute
344
351
  self.runnerFatal = runnerFatal
345
352
  self.runnerFatalReason = runnerFatalReason
346
353
  self.completedSteps = completedSteps
@@ -383,6 +390,9 @@ struct SnapshotNode: Codable {
383
390
  let parentIndex: Int?
384
391
  let hiddenContentAbove: Bool?
385
392
  let hiddenContentBelow: Bool?
393
+ /// Names of the element's UIAccessibilityCustomActions, present only when the
394
+ /// capture asked for them and the element carries some.
395
+ var actions: [String]? = nil
386
396
  }
387
397
 
388
398
  struct SnapshotOptions {
@@ -390,4 +400,11 @@ struct SnapshotOptions {
390
400
  let depth: Int?
391
401
  let scope: String?
392
402
  let raw: Bool
403
+ /// Internal daemon ask: capture with this backend first regardless of channel
404
+ /// health ("private-ax"). Same-backend evidence probes (tap-outcome
405
+ /// corroboration) must be captured the way their baseline was.
406
+ var preferredBackend: String? = nil
407
+ /// Read UIAccessibilityCustomActions for merged leaves. Opt-in: each element
408
+ /// costs its own AX round trip (see RunnerAXSnapshotBridge).
409
+ var customActions: Bool = false
393
410
  }
@@ -7,7 +7,6 @@ extension RunnerTests {
7
7
  final class ScreenRecorder {
8
8
  private let outputPath: String
9
9
  private let fps: Int32?
10
- private let maxSize: Int?
11
10
  private var effectiveFps: Int32 {
12
11
  max(1, fps ?? RunnerTests.defaultRecordingFps)
13
12
  }
@@ -26,10 +25,9 @@ extension RunnerTests {
26
25
  private var startedSession = false
27
26
  private var startError: Error?
28
27
 
29
- init(outputPath: String, fps: Int32?, maxSize: Int?) {
28
+ init(outputPath: String, fps: Int32?) {
30
29
  self.outputPath = outputPath
31
30
  self.fps = fps
32
- self.maxSize = maxSize
33
31
  }
34
32
 
35
33
  func start(captureFrame: @escaping () -> RunnerImage?) throws {
@@ -50,7 +48,7 @@ extension RunnerTests {
50
48
  while Date() < bootstrapDeadline {
51
49
  if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
52
50
  bootstrapImage = image
53
- dimensions = scaledDimensions(width: cgImage.width, height: cgImage.height)
51
+ dimensions = CGSize(width: cgImage.width, height: cgImage.height)
54
52
  break
55
53
  }
56
54
  Thread.sleep(forTimeInterval: 0.05)
@@ -261,23 +259,5 @@ extension RunnerTests {
261
259
  return pixelBuffer
262
260
  }
263
261
 
264
- private func scaledDimensions(width: Int, height: Int) -> CGSize {
265
- guard let maxSize, maxSize > 0 else {
266
- return CGSize(width: width, height: height)
267
- }
268
- let longest = max(width, height)
269
- guard longest > maxSize else {
270
- return CGSize(width: width, height: height)
271
- }
272
- let scale = Double(maxSize) / Double(longest)
273
- return CGSize(
274
- width: scaledEvenDimension(width, scale: scale),
275
- height: scaledEvenDimension(height, scale: scale)
276
- )
277
- }
278
-
279
- private func scaledEvenDimension(_ value: Int, scale: Double) -> Int {
280
- max(2, Int((Double(value) * scale / 2.0).rounded()) * 2)
281
- }
282
262
  }
283
263
  }
@@ -493,7 +493,8 @@ extension RunnerTests {
493
493
  reason: failure.message,
494
494
  reasonCode: "ax-rejected",
495
495
  effectiveDepth: nil,
496
- collapsedLeafIndexes: nil
496
+ collapsedLeafIndexes: nil,
497
+ customActions: nil
497
498
  ),
498
499
  runnerFatal: true,
499
500
  runnerFatalReason: Self.axSnapshotUnavailableReason
@@ -22,6 +22,23 @@ struct SnapshotQuality: Codable {
22
22
  let effectiveDepth: Int?
23
23
  /// Leaves that merge many labels — a container marked accessible hides its descendants.
24
24
  let collapsedLeafIndexes: [Int]?
25
+ /// Coverage of the bounded custom-action pass, when the capture asked for one.
26
+ let customActions: SnapshotCustomActionCoverage?
27
+ }
28
+
29
+ /// How much of the merged-element set the custom-action pass actually read. An
30
+ /// unread element renders exactly like one with no actions, so a partial pass
31
+ /// has to say so — otherwise absence reads as proof of absence.
32
+ struct SnapshotCustomActionCoverage: Codable {
33
+ let read: Int
34
+ let candidates: Int
35
+ /// Elements whose action list was clipped by the per-element output caps. A
36
+ /// clipped list looks complete, so it is disclosed on the same principle as
37
+ /// an unread element.
38
+ let truncated: Int
39
+ /// The pass stopped early because an earlier read is still hung. Distinct from
40
+ /// a budget stop: the remedy is waiting, not scrolling.
41
+ let blocked: Bool
25
42
  }
26
43
 
27
44
  enum SnapshotBackendKind: String, CaseIterable {
@@ -78,6 +95,8 @@ struct SnapshotBackendCapture {
78
95
  let payload: DataPayload
79
96
  /// Set by the private AX backend when the ladder accepted a shallower depth than requested.
80
97
  let effectiveDepth: Int?
98
+ /// Set by the private AX backend when the capture asked for custom actions.
99
+ var customActions: SnapshotCustomActionCoverage? = nil
81
100
  }
82
101
 
83
102
  extension RunnerTests {
@@ -144,6 +163,53 @@ extension RunnerTests {
144
163
  return pending
145
164
  }
146
165
 
166
+ /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case
167
+ /// uses the dedicated 'deferred' code: the breaker pre-selected the backend, nothing new
168
+ /// degraded on this capture, and the daemon keys warning suppression and the settle budget
169
+ /// reset off exactly that distinction. The bounded probe keeps 'budget': its short XCTest
170
+ /// slice genuinely constrains what this capture could read.
171
+ ///
172
+ /// `requestPinnedBackend` separates the two ways the plan can arrive at the same
173
+ /// deferred shape. A capture that ASKED for a private-AX-only reading (custom
174
+ /// actions) degraded nothing, so reporting slow accessibility work would name a
175
+ /// cause that does not exist.
176
+ static func xcTestChannelStateFirstFailure(
177
+ _ state: SnapshotXCTestChannelPlanState,
178
+ requestPinnedBackend: Bool = false
179
+ ) -> (reason: String, code: String)? {
180
+ switch state {
181
+ case .normal:
182
+ return nil
183
+ case .deferredToIndependentBackend:
184
+ if requestPinnedBackend {
185
+ return (
186
+ "the private AX backend was selected because this capture asked for accessibility custom actions",
187
+ "requested-backend"
188
+ )
189
+ }
190
+ return (
191
+ "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen",
192
+ "deferred"
193
+ )
194
+ case .boundedXCTestProbe:
195
+ return (
196
+ "XCTest-backed snapshot tiers are running with a short recovery probe after recent slow accessibility work on this screen",
197
+ "budget"
198
+ )
199
+ }
200
+ }
201
+
202
+ /// Pure gate: a capture is planned as penalized when the channel penalty is
203
+ /// active OR the daemon pinned the private-AX backend (same-backend evidence
204
+ /// probe) — both mean "do not enter XCTest tree work first, and stamp the
205
+ /// pre-selection as 'deferred' rather than a degradation".
206
+ static func snapshotXCTestChannelTreatedAsPenalized(
207
+ penalized: Bool,
208
+ preferredBackend: String?
209
+ ) -> Bool {
210
+ penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue
211
+ }
212
+
147
213
  /// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends
148
214
  /// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw
149
215
  /// diagnostic plan keeps tree-first errors, and unknown plans are left untouched.
@@ -198,9 +264,17 @@ extension RunnerTests {
198
264
  // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while
199
265
  // the app remains visually responsive. Simulators can avoid that channel through private AX;
200
266
  // physical devices have no independent semantic backend yet, so they use a bounded probe.
267
+ // A daemon-preferred private-AX capture (same-backend evidence probe) takes the exact
268
+ // penalized route: privateAX-first plan, 'deferred' verdict — the backend was pre-selected
269
+ // deliberately, so no degradation warning should render for it.
201
270
  var xCTestChannelPenalized = false
271
+ var xCTestChannelPenalizedByBreaker = false
202
272
  #if os(iOS)
203
- xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
273
+ xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
274
+ xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized(
275
+ penalized: xCTestChannelPenalizedByBreaker,
276
+ preferredBackend: options.preferredBackend
277
+ )
204
278
  #endif
205
279
  let effective = Self.effectiveSnapshotCapturePlan(
206
280
  plan,
@@ -208,20 +282,18 @@ extension RunnerTests {
208
282
  availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform))
209
283
  )
210
284
  let effectivePlan = effective.plan
285
+ // Only a customActions-implied pin is request-pinned; the daemon's
286
+ // same-backend evidence probe pins for its own reasons and keeps 'deferred'.
287
+ firstFailure = Self.xcTestChannelStateFirstFailure(
288
+ effective.xCTestChannelState,
289
+ requestPinnedBackend: options.customActions && !xCTestChannelPenalizedByBreaker
290
+ )
211
291
  switch effective.xCTestChannelState {
212
292
  case .normal:
213
293
  break
214
294
  case .deferredToIndependentBackend:
215
- firstFailure = (
216
- "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen",
217
- "budget"
218
- )
219
295
  NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", currentBundleId ?? "")
220
296
  case .boundedXCTestProbe:
221
- firstFailure = (
222
- "XCTest-backed snapshot tiers are running with a short recovery probe after recent slow accessibility work on this screen",
223
- "budget"
224
- )
225
297
  NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", currentBundleId ?? "")
226
298
  }
227
299
 
@@ -428,7 +500,7 @@ extension RunnerTests {
428
500
  return ("snapshot returned no nodes", "no-nodes")
429
501
  }
430
502
  if isSparseApplicationWindowTree(nodes) {
431
- return ("snapshot returned only structural application/window nodes", "sparse-tree")
503
+ return ("snapshot returned no semantic controls or content", "sparse-tree")
432
504
  }
433
505
  if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold {
434
506
  return ("snapshot was cut off by its budget with almost nothing collected", "budget")
@@ -459,19 +531,36 @@ extension RunnerTests {
459
531
 
460
532
  static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
461
533
  guard !nodes.isEmpty else { return false }
534
+ let rootRects = nodes.compactMap { node in
535
+ node.type == "Application" || node.type == "Window" ? node.rect : nil
536
+ }
462
537
  return nodes.allSatisfy { node in
463
538
  // Application/Window labels are just the app/window name, and full-screen roots
464
539
  // compute as hittable; neither says anything about tree health.
465
540
  let isRootContainer = node.type == "Application" || node.type == "Window"
466
- let hasContent = (!isRootContainer && node.label?.isEmpty == false)
467
- || node.identifier?.isEmpty == false
468
- || node.value?.isEmpty == false
469
- return !hasContent
470
- && (isRootContainer || !node.hittable)
471
- && Self.structuralOnlyNodeTypes.contains(node.type)
541
+ guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false }
542
+ guard !isRootContainer else { return true }
543
+
544
+ let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in
545
+ rootRect.x == node.rect.x && rootRect.y == node.rect.y
546
+ && rootRect.width == node.rect.width && rootRect.height == node.rect.height
547
+ }
548
+ let hasAddressableIdentifier = node.identifier?.isEmpty == false && !isFullScreenContainer
549
+ return !Self.isSemanticSnapshotText(node.label)
550
+ && !Self.isSemanticSnapshotText(node.value)
551
+ && !hasAddressableIdentifier
472
552
  }
473
553
  }
474
554
 
555
+ /// Private AX can stringify an unserializable accessibility label as the JavaScript object
556
+ /// placeholder. It is transport residue, not UI content, and must not make a shell-only tree
557
+ /// look healthy.
558
+ static func isSemanticSnapshotText(_ text: String?) -> Bool {
559
+ guard let text else { return false }
560
+ let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines)
561
+ return !normalized.isEmpty && normalized.lowercased() != "[object object]"
562
+ }
563
+
475
564
  /// A leaf whose label joins many short segments is a container marked as an accessibility
476
565
  /// element: the platform folds every descendant into one merged node. Nothing below it can
477
566
  /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
@@ -502,7 +591,8 @@ extension RunnerTests {
502
591
  reason: reason?.reason,
503
592
  reasonCode: reason?.code,
504
593
  effectiveDepth: capture.effectiveDepth,
505
- collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? [])
594
+ collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []),
595
+ customActions: capture.customActions
506
596
  )
507
597
  return DataPayload(
508
598
  // Legacy human text for older daemons that read message instead of snapshotQuality.
@@ -515,11 +605,42 @@ extension RunnerTests {
515
605
  )
516
606
  }
517
607
 
608
+ /// Response level, one line per incompleteness. An unread merged element is
609
+ /// byte-identical to one with no actions, and a clipped list looks complete,
610
+ /// so both have to name themselves.
611
+ static func customActionCoverageWarnings(_ coverage: SnapshotCustomActionCoverage) -> [String] {
612
+ var lines: [String] = []
613
+ if coverage.blocked {
614
+ // Scrolling is the remedy for a budget stop, not for this one — saying it
615
+ // here would send the reader off doing something that cannot help.
616
+ lines.append(
617
+ "Custom actions were not read: an earlier accessibility read is still hung, so this "
618
+ + "capture skipped the read pass instead of queueing behind it. No element's actions "
619
+ + "list is authoritative here. Reads resume once that call returns.")
620
+ } else if coverage.read < coverage.candidates {
621
+ lines.append(
622
+ "Custom actions were read for \(coverage.read) of \(coverage.candidates) merged elements, "
623
+ + "on-screen ones first; the remaining \(coverage.candidates - coverage.read) were not read, "
624
+ + "so an absent actions list on those is not evidence that they have none. "
625
+ + "Scroll them into view and re-run to read them.")
626
+ }
627
+ if coverage.truncated > 0 {
628
+ lines.append(
629
+ "\(coverage.truncated) element(s) published more custom actions than are shown; those "
630
+ + "lists are clipped to the first 8 names, and long names are shortened.")
631
+ }
632
+ return lines
633
+ }
634
+
518
635
  static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? {
519
- guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil }
636
+ let customActionWarnings =
637
+ quality.customActions.map { Self.customActionCoverageWarnings($0) } ?? []
638
+ guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil
639
+ || !customActionWarnings.isEmpty
640
+ else { return nil }
520
641
  var parts: [String] = []
521
642
  if quality.state == "recovered" {
522
- let meaning = quality.reasonCode == "budget"
643
+ let meaning = quality.reasonCode == "budget" || quality.reasonCode == "deferred"
523
644
  ? " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen."
524
645
  : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears."
525
646
  parts.append(
@@ -535,9 +656,13 @@ extension RunnerTests {
535
656
  + ". Use screenshot as visual truth and coordinate taps."
536
657
  )
537
658
  }
659
+ parts.append(contentsOf: customActionWarnings)
538
660
  if let depth = quality.effectiveDepth {
661
+ // No --depth remedy here: an explicit --depth capture disables the
662
+ // frontier extension, so following it would return strictly less than
663
+ // this capture did. A plain re-run retries with a fresh extension budget.
539
664
  parts.append(
540
- "The accessibility server rejected deeper requests; this tree is capped at depth \(depth) — re-run with --depth \(depth) --scope <container> for deeper content."
665
+ "The accessibility server rejected deeper requests; content below depth \(depth) may be missing — re-run snapshot to retry deeper content."
541
666
  )
542
667
  }
543
668
  return parts.isEmpty ? nil : parts.joined(separator: " ")
@@ -1,24 +1,4 @@
1
1
  import XCTest
2
2
 
3
3
  extension RunnerTests {
4
- func testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds() {
5
- let wrapper = Self.snapshotTraversalIdentity(
6
- elementType: .other,
7
- label: "Article",
8
- identifier: "",
9
- frame: CGRect(x: 0, y: 97, width: 393, height: 48)
10
- )
11
- let leaf = Self.snapshotTraversalIdentity(
12
- elementType: .other,
13
- label: "Article",
14
- identifier: "",
15
- frame: CGRect(x: 0, y: 97, width: 131, height: 48)
16
- )
17
-
18
- #if os(iOS)
19
- XCTAssertNotEqual(wrapper, leaf)
20
- #else
21
- XCTAssertEqual(wrapper, leaf)
22
- #endif
23
- }
24
4
  }
@@ -101,6 +101,10 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz
101
101
  }
102
102
  }
103
103
 
104
+ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool {
105
+ !xCTestChannelPenalized
106
+ }
107
+
104
108
  func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
105
109
  steps.contains { step in
106
110
  step.synthesized == true && step.kind == "tap"
@@ -0,0 +1,234 @@
1
+ import XCTest
2
+
3
+ // AX-independent text synthesis: private-XCTest boundary, pacing plan, and route policy.
4
+ // The verified XCUIElement path remains in TextEntry/TextTyping.
5
+ extension RunnerTests {
6
+ enum SynthesizedReplacementRouteOutcome {
7
+ case notApplicable
8
+ case completed(TextEntryResult)
9
+ case fallback(TextEntryTarget)
10
+ }
11
+
12
+ struct SynthesizedReplacementRequest {
13
+ let app: XCUIApplication
14
+ let target: TextEntryTarget
15
+ let text: String
16
+ let delaySeconds: Double
17
+ let synthesizer: any TextEntrySynthesizing
18
+ let commandId: String?
19
+ let startedAt: Date
20
+ }
21
+
22
+ enum SynthesizedTextEntryAction: Equatable {
23
+ case continueTyping
24
+ case fallback
25
+ case raise(String?)
26
+ }
27
+
28
+ protocol TextEntrySynthesizing {
29
+ func enterText(
30
+ app: XCUIApplication,
31
+ text: String,
32
+ replacingExistingText: Bool
33
+ ) -> SynthesizedTextEntryAction
34
+ }
35
+
36
+ struct PrivateXCTestTextEntrySynthesizer: TextEntrySynthesizing {
37
+ func enterText(
38
+ app: XCUIApplication,
39
+ text: String,
40
+ replacingExistingText: Bool
41
+ ) -> SynthesizedTextEntryAction {
42
+ #if os(iOS)
43
+ let result = replacingExistingText
44
+ ? RunnerSynthesizedTextEntry.replaceText(withApplication: app, text: text)
45
+ : RunnerSynthesizedTextEntry.synthesizeText(withApplication: app, text: text)
46
+ return Self.action(status: result.status, message: result.message)
47
+ #else
48
+ return .fallback
49
+ #endif
50
+ }
51
+
52
+ #if os(iOS)
53
+ static func action(
54
+ status: RunnerSynthesizedTextEntryStatus,
55
+ message: String?
56
+ ) -> SynthesizedTextEntryAction {
57
+ switch status {
58
+ case .succeeded:
59
+ return .continueTyping
60
+ case .unavailable:
61
+ return .fallback
62
+ case .failed:
63
+ return .raise(message)
64
+ @unknown default:
65
+ return .raise(message)
66
+ }
67
+ }
68
+ #endif
69
+ }
70
+
71
+ struct SynthesizedReplacementStep: Equatable {
72
+ let text: String
73
+ let replacesExistingText: Bool
74
+ }
75
+
76
+ static func synthesizedReplacementSteps(
77
+ text: String,
78
+ delaySeconds: Double
79
+ ) -> [SynthesizedReplacementStep] {
80
+ let characters = Array(text)
81
+ guard delaySeconds > 0, characters.count > 1 else {
82
+ return [SynthesizedReplacementStep(text: text, replacesExistingText: true)]
83
+ }
84
+ return characters.enumerated().map { index, character in
85
+ SynthesizedReplacementStep(
86
+ text: String(character),
87
+ replacesExistingText: index == 0
88
+ )
89
+ }
90
+ }
91
+
92
+ func runSynthesizedReplacementRoute(
93
+ _ request: SynthesizedReplacementRequest
94
+ ) -> SynthesizedReplacementRouteOutcome {
95
+ #if os(iOS)
96
+ NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement")
97
+ let steps = Self.synthesizedReplacementSteps(
98
+ text: request.text,
99
+ delaySeconds: request.delaySeconds
100
+ )
101
+ for (index, step) in steps.enumerated() {
102
+ switch request.synthesizer.enterText(
103
+ app: request.app,
104
+ text: step.text,
105
+ replacingExistingText: step.replacesExistingText
106
+ ) {
107
+ case .fallback:
108
+ NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
109
+ guard let point = request.target.refreshPoint else { return .notApplicable }
110
+ return .fallback(
111
+ focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y)
112
+ )
113
+ case .raise(let message):
114
+ NSException(
115
+ name: NSExceptionName.internalInconsistencyException,
116
+ reason: message ?? "private XCTest text synthesis failed"
117
+ ).raise()
118
+ case .continueTyping:
119
+ break
120
+ }
121
+ if index + 1 < steps.count {
122
+ sleepFor(request.delaySeconds)
123
+ }
124
+ }
125
+ logTextEntryPhase(
126
+ commandId: request.commandId,
127
+ phase: "total",
128
+ startedAt: request.startedAt,
129
+ chars: request.text.count,
130
+ mode: .replacement
131
+ )
132
+ return .completed(
133
+ TextEntryResult(
134
+ verified: nil,
135
+ repaired: false,
136
+ expectedText: request.text,
137
+ observedText: nil,
138
+ textEntryRoute: "synthesized-first-responder-replacement"
139
+ )
140
+ )
141
+ #else
142
+ return .notApplicable
143
+ #endif
144
+ }
145
+
146
+ static func shouldUseSynthesizedFirstResponderReplacement(
147
+ hasResolvedElement: Bool,
148
+ hasRefreshPoint: Bool,
149
+ xCTestChannelPenalized: Bool
150
+ ) -> Bool {
151
+ !hasResolvedElement && hasRefreshPoint && xCTestChannelPenalized
152
+ }
153
+
154
+ static func shouldUseSynthesizedFirstResponderType(
155
+ repairMode: TextTypingRepairMode,
156
+ fromTapWitness: Bool,
157
+ softwareKeyboardVisible: Bool
158
+ ) -> Bool {
159
+ repairMode == .none && fromTapWitness && !softwareKeyboardVisible
160
+ }
161
+
162
+ enum SynthesizedTextCommitProgress: Equatable {
163
+ case committed
164
+ case pending
165
+ case diverged
166
+ }
167
+
168
+ // The private synthesize call returns once the event record is posted, not once the target
169
+ // app has committed the characters, so intermediate reads walk prefix-by-prefix toward the
170
+ // expected value. Anything off that prefix path means the app transformed the input
171
+ // (formatter, mid-text caret, autocomplete) and the runner must not second-guess it. An
172
+ // unreadable value — secure field, or the element stopped resolving — ends the wait the
173
+ // same way.
174
+ static func synthesizedTextCommitProgress(
175
+ observedText: String?,
176
+ expectedText: String
177
+ ) -> SynthesizedTextCommitProgress {
178
+ guard let observedText else {
179
+ return .diverged
180
+ }
181
+ if observedText == expectedText {
182
+ return .committed
183
+ }
184
+ return expectedText.hasPrefix(observedText) ? .pending : .diverged
185
+ }
186
+
187
+ /// Blocks until the synthesized bare-type text is observable in the target field, so `type`
188
+ /// cannot report ok while trailing characters are still uncommitted on a slow simulator.
189
+ ///
190
+ /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the
191
+ /// event stream, so re-synthesizing the difference risks committing it twice after the command
192
+ /// already reported success. Text carrying a submit key is skipped outright: the app may clear
193
+ /// or rewrite the field on submit, so `textBefore + typedText` is not the value to wait for.
194
+ func awaitSynthesizedFirstResponderCommit(
195
+ app: XCUIApplication,
196
+ target: TextEntryTarget,
197
+ textBefore: String?,
198
+ typedText: String
199
+ ) {
200
+ guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else {
201
+ return
202
+ }
203
+ let expectedText = textBefore + typedText
204
+ let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout)
205
+ while Date() < deadline {
206
+ let observedText = editableTextValue(
207
+ for: resolveTextEntryElement(app: app, target: target),
208
+ treatingPlaceholderAsEmpty: true
209
+ )
210
+ switch Self.synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
211
+ case .committed, .diverged:
212
+ return
213
+ case .pending:
214
+ sleepFor(TextEntryTiming.pollInterval)
215
+ }
216
+ }
217
+ }
218
+
219
+ static func shouldUseResolvedCoordinateTextEntryRoute(
220
+ repairMode: TextTypingRepairMode,
221
+ hasX: Bool,
222
+ hasY: Bool,
223
+ xCTestChannelPenalized: Bool
224
+ ) -> Bool {
225
+ repairMode == .replacement && hasX && hasY && xCTestChannelPenalized
226
+ }
227
+
228
+ static func shouldFallbackFromSynthesizedTextEntryFocus(
229
+ _ outcome: RunnerInteractionOutcome
230
+ ) -> Bool {
231
+ if case .unsupported = outcome { return true }
232
+ return false
233
+ }
234
+ }