agent-device 0.20.6 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +3 -3
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.6.apk → agent-device-android-ime-helper-0.20.7.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.6.manifest.json → agent-device-android-ime-helper-0.20.7.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.6.apk → agent-device-android-snapshot-helper-0.20.7.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.6.manifest.json → agent-device-android-snapshot-helper-0.20.7.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +20 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +67 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +339 -4
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +78 -0
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +31 -2
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +29 -7
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +4 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +36 -28
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +7 -0
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SelectorMatchPolicyTests.swift +4 -0
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +2 -1
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +100 -12
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +4 -0
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +57 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +1 -0
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +18 -3
  24. package/dist/src/agent-browser-tool.js +1 -1
  25. package/dist/src/agent-device-client.js +2 -2
  26. package/dist/src/android-adb.js +1 -1
  27. package/dist/src/android.js +1 -1
  28. package/dist/src/app-helpers.js +1 -1
  29. package/dist/src/app-lifecycle.js +7 -7
  30. package/dist/src/app-lifecycle2.js +2 -0
  31. package/dist/src/app-log-files.js +1 -0
  32. package/dist/src/app-log-reconnect.js +1 -0
  33. package/dist/src/app-log-runtime.js +4 -0
  34. package/dist/src/app-resolution.js +2 -1
  35. package/dist/src/apps.js +5 -5
  36. package/dist/src/audio-probe.js +1 -1
  37. package/dist/src/auth-session.js +1 -0
  38. package/dist/src/auth.js +2 -0
  39. package/dist/src/boot-diagnostics.js +2 -2
  40. package/dist/src/cli-config.js +1 -1
  41. package/dist/src/cli-grammar.js +1 -0
  42. package/dist/src/cli-help.js +199 -259
  43. package/dist/src/cli-output.js +1 -0
  44. package/dist/src/cli-runner.js +1 -2
  45. package/dist/src/cli.js +6 -16
  46. package/dist/src/client-metro.js +1 -1
  47. package/dist/src/cloud-artifacts.d.ts +2 -137
  48. package/dist/src/command-schema.js +3 -3
  49. package/dist/src/connection-runtime.js +1 -0
  50. package/dist/src/connection.js +5 -0
  51. package/dist/src/context.js +1 -1
  52. package/dist/src/daemon-shutdown-report.js +1 -0
  53. package/dist/src/daemon.js +2 -0
  54. package/dist/src/debug-symbols.js +3 -3
  55. package/dist/src/device-claim-inspection.js +1 -0
  56. package/dist/src/device-input-state.js +2 -2
  57. package/dist/src/device.js +1 -1
  58. package/dist/src/device2.js +2 -0
  59. package/dist/src/find.js +1 -1
  60. package/dist/src/finders.js +1 -1
  61. package/dist/src/generic-settle.js +1 -1
  62. package/dist/src/generic.js +1 -1
  63. package/dist/src/harmonyos.js +1 -0
  64. package/dist/src/helper.js +1 -1
  65. package/dist/src/host-process.js +2 -2
  66. package/dist/src/ime-lifecycle.js +7 -7
  67. package/dist/src/index.d.ts +43 -12
  68. package/dist/src/input-actions.js +4 -4
  69. package/dist/src/input-actions2.js +1 -1
  70. package/dist/src/input-actions3.js +1 -0
  71. package/dist/src/install-artifact.js +1 -5
  72. package/dist/src/install-artifact2.js +1 -1
  73. package/dist/src/install-source2.js +1 -1
  74. package/dist/src/interaction-runtime.js +1 -1
  75. package/dist/src/interaction-snapshot.js +1 -1
  76. package/dist/src/interaction-touch-reference-frame.js +1 -1
  77. package/dist/src/interaction.js +1 -1
  78. package/dist/src/interactor.js +2 -2
  79. package/dist/src/interactor2.js +1 -1
  80. package/dist/src/internal/bin.js +2 -2
  81. package/dist/src/internal/daemon.js +8 -15
  82. package/dist/src/inventory.js +5 -0
  83. package/dist/src/inventory2.js +1 -0
  84. package/dist/src/inventory3.js +2 -0
  85. package/dist/src/inventory4.js +1 -0
  86. package/dist/src/inventory5.js +1 -0
  87. package/dist/src/inventory6.js +1 -0
  88. package/dist/src/lease-context.js +1 -1
  89. package/dist/src/lease.js +1 -1
  90. package/dist/src/limrun-runtime-dependencies.js +1 -0
  91. package/dist/src/limrun.d.ts +10 -125
  92. package/dist/src/limrun.js +1 -1
  93. package/dist/src/linux.js +1 -1
  94. package/dist/src/logcat.js +1 -1
  95. package/dist/src/manifest.js +1 -1
  96. package/dist/src/owner-identity.js +1 -0
  97. package/dist/src/physical-device-control.js +1 -1
  98. package/dist/src/platform-runtime-app-log-android-transport.js +1 -0
  99. package/dist/src/platform-runtime-app-log-process.js +1 -0
  100. package/dist/src/platform-runtime-host.js +1 -0
  101. package/dist/src/platform-runtime-network-web-transport.js +1 -0
  102. package/dist/src/platform-runtime-operation-host.js +5 -0
  103. package/dist/src/platform-runtime-toolchain-host.js +1 -0
  104. package/dist/src/platform-runtime.js +1 -0
  105. package/dist/src/prepare-kind.js +15 -0
  106. package/dist/src/process-entry.js +1 -0
  107. package/dist/src/process-lock.js +1 -1
  108. package/dist/src/provider-webdriver.js +1 -0
  109. package/dist/src/provider.js +1 -0
  110. package/dist/src/proxy.js +2 -0
  111. package/dist/src/react-native.js +1 -1
  112. package/dist/src/record-trace.js +1 -1
  113. package/dist/src/registry.js +24 -24
  114. package/dist/src/remote-config2.js +1 -1
  115. package/dist/src/replay.js +3 -0
  116. package/dist/src/reporting.js +3 -3
  117. package/dist/src/runner-client.js +1 -1
  118. package/dist/src/runner-disposal.js +1 -1
  119. package/dist/src/runner-lease.js +1 -1
  120. package/dist/src/runtime.js +2 -2
  121. package/dist/src/runtime2.js +4 -0
  122. package/dist/src/runtime3.js +3 -0
  123. package/dist/src/runtime4.js +1 -0
  124. package/dist/src/runtime5.js +1 -0
  125. package/dist/src/runtime6.js +1 -0
  126. package/dist/src/runtime7.js +1 -0
  127. package/dist/src/screenshot-result.js +4 -4
  128. package/dist/src/screenshot.js +1 -0
  129. package/dist/src/sdk-android-adb.js +2 -2
  130. package/dist/src/sdk-batch-runner.js +3 -3
  131. package/dist/src/sdk-contracts.d.ts +31 -5
  132. package/dist/src/sdk-contracts.js +1 -1
  133. package/dist/src/sdk-device.js +1 -1
  134. package/dist/src/sdk-finders.js +1 -1
  135. package/dist/src/sdk-remote-config.js +1 -1
  136. package/dist/src/sdk-selectors.d.ts +280 -4
  137. package/dist/src/sdk-selectors.js +1 -1
  138. package/dist/src/selector-runtime.js +1 -1
  139. package/dist/src/selector-vocabulary.js +1 -1
  140. package/dist/src/selectors.js +1 -1
  141. package/dist/src/server.js +3 -3
  142. package/dist/src/session-target-evidence.js +1 -1
  143. package/dist/src/session.js +10 -9
  144. package/dist/src/shared.js +2 -0
  145. package/dist/src/shell-quote.js +1 -0
  146. package/dist/src/simulator.js +1 -1
  147. package/dist/src/snapshot-runtime.js +1 -0
  148. package/dist/src/snapshot-session.js +1 -0
  149. package/dist/src/snapshot.js +1 -1
  150. package/dist/src/snapshot2.js +1 -1
  151. package/dist/src/src.js +1 -1
  152. package/dist/src/src2.js +2 -0
  153. package/dist/src/src3.js +1 -0
  154. package/dist/src/text-surface.js +1 -1
  155. package/dist/src/timing-safe-equal.js +1 -0
  156. package/dist/src/tool-provider.js +3 -3
  157. package/dist/src/tool-provider2.js +1 -1
  158. package/dist/src/tool-provider3.js +1 -1
  159. package/dist/src/update-check.js +1 -1
  160. package/dist/src/version.js +1 -1
  161. package/dist/src/web.js +1 -1
  162. package/package.json +36 -20
  163. package/server.json +2 -2
  164. package/skills/agent-device/SKILL.md +6 -31
  165. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +0 -1
  166. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +0 -1
  167. package/dist/src/app-log-request-scope.js +0 -1
  168. package/dist/src/devices.js +0 -1
  169. package/dist/src/devices2.js +0 -1
  170. package/dist/src/devices3.js +0 -1
  171. package/dist/src/devices4.js +0 -1
  172. package/dist/src/provider-limrun-runtime.js +0 -1
  173. package/dist/src/register-builtins.js +0 -1
  174. package/dist/src/screenshot-geometry.js +0 -1
  175. package/dist/src/viewport-dimension.js +0 -15
@@ -0,0 +1,78 @@
1
+ struct SelectorCandidateFacts {
2
+ let isHittable: Bool
3
+ let hasTappableFrame: Bool
4
+ let containsExpectedPoint: Bool
5
+
6
+ init(
7
+ isHittable: Bool,
8
+ hasTappableFrame: Bool,
9
+ containsExpectedPoint: Bool = true
10
+ ) {
11
+ self.isHittable = isHittable
12
+ self.hasTappableFrame = hasTappableFrame
13
+ self.containsExpectedPoint = containsExpectedPoint
14
+ }
15
+ }
16
+
17
+ enum DirectSelectorCandidateDecision: Equatable {
18
+ case noMatch
19
+ case selected(index: Int, usedNonHittableFallback: Bool)
20
+ case ambiguous
21
+ }
22
+
23
+ /// How many raw exact matches a dispatch may discard before hittability is
24
+ /// allowed to pick a winner. The two rows differ because the cost of guessing
25
+ /// wrong differs, not because the matching differs.
26
+ enum DirectSelectorRawMatchPolicy: Equatable {
27
+ /// Mutations fail closed: every raw exact match counts, so a hittable
28
+ /// element can never silently win over a same-selector duplicate the caller
29
+ /// never saw and act on the wrong one.
30
+ case rejectDistinctMatches
31
+ /// Reads prefer the single hittable match and ignore non-hittable
32
+ /// same-selector duplicates. A read has no side effect to guard, and
33
+ /// `querySelector` backs `get`/`is`/`wait` — failing those closed turns a
34
+ /// decorative duplicate into an error where the reader previously got its
35
+ /// answer.
36
+ case preferHittableMatch
37
+ }
38
+
39
+ /// Normal direct selector mutations count every raw exact match before
40
+ /// hittability can choose a winner. Reads keep the hittable-preference rule,
41
+ /// and Maestro's explicitly requested coordinate fallback keeps its
42
+ /// point-filtered compatibility behavior.
43
+ func classifyDirectSelectorCandidates(
44
+ _ candidates: [SelectorCandidateFacts],
45
+ allowNonHittableFallback: Bool,
46
+ filtersByExpectedPoint: Bool = false,
47
+ rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches
48
+ ) -> DirectSelectorCandidateDecision {
49
+ let eligible = candidates.indices.filter { index in
50
+ !filtersByExpectedPoint || candidates[index].containsExpectedPoint
51
+ }
52
+
53
+ if !allowNonHittableFallback && rawMatchPolicy == .rejectDistinctMatches {
54
+ guard eligible.count <= 1 else { return .ambiguous }
55
+ guard let index = eligible.first, candidates[index].isHittable else { return .noMatch }
56
+ return .selected(index: index, usedNonHittableFallback: false)
57
+ }
58
+
59
+ var hittableIndex: Int?
60
+ var fallbackIndex: Int?
61
+ for index in eligible {
62
+ let candidate = candidates[index]
63
+ if candidate.isHittable {
64
+ guard hittableIndex == nil else { return .ambiguous }
65
+ hittableIndex = index
66
+ } else if allowNonHittableFallback && candidate.hasTappableFrame {
67
+ guard fallbackIndex == nil else { return .ambiguous }
68
+ fallbackIndex = index
69
+ }
70
+ }
71
+ if let hittableIndex {
72
+ return .selected(index: hittableIndex, usedNonHittableFallback: false)
73
+ }
74
+ if let fallbackIndex {
75
+ return .selected(index: fallbackIndex, usedNonHittableFallback: true)
76
+ }
77
+ return .noMatch
78
+ }
@@ -9,6 +9,12 @@ extension RunnerTests {
9
9
  /// the capture-plan deadline, which is also enforced per call.
10
10
  private static let privateAXDeepExtensionCallLimit = 8
11
11
 
12
+ /// Upper bound on per-element custom-action reads per capture. Each is its own
13
+ /// AX round trip (~100ms on an idle simulator), so this caps the opt-in cost
14
+ /// at roughly a second on top of the capture; the capture-plan deadline stops
15
+ /// it earlier under load. A screenful of merged cards is well under this.
16
+ private static let privateAXCustomActionLimit = 12
17
+
12
18
  /// A capture is depth-limited unless its frontier extension resolved every
13
19
  /// capped node: frontiers left pending (budget/deadline) or missed (element
14
20
  /// vanished, re-rooted request failed) mean subtrees are still absent, and
@@ -42,6 +48,24 @@ extension RunnerTests {
42
48
  return depths.filter { $0 <= remembered }
43
49
  }
44
50
 
51
+ /// The bridge omits the key entirely when the capture did not ask for custom
52
+ /// actions, which is the difference between "nothing to disclose" and "read
53
+ /// none of them" — so a missing key must stay nil, never (0, 0).
54
+ static func privateAXCustomActionCoverage(_ raw: Any?) -> SnapshotCustomActionCoverage? {
55
+ guard let coverage = raw as? [String: Any],
56
+ let read = (coverage[RunnerAXSnapshotCustomActionsReadKey] as? NSNumber)?.intValue,
57
+ let candidates = (coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? NSNumber)?.intValue
58
+ else { return nil }
59
+ // read/candidates are the ratio and must both be present; the truncation
60
+ // count is additive, so an older bridge that omits it reads as zero rather
61
+ // than voiding the whole coverage.
62
+ let truncated =
63
+ (coverage[RunnerAXSnapshotCustomActionsTruncatedKey] as? NSNumber)?.intValue ?? 0
64
+ let blocked = (coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? NSNumber)?.boolValue ?? false
65
+ return SnapshotCustomActionCoverage(
66
+ read: read, candidates: candidates, truncated: truncated, blocked: blocked)
67
+ }
68
+
45
69
  func rememberPrivateAXAcceptedDepth(bundleId: String?, processIdentifier: Int?, depth: Int) {
46
70
  // No PID means no way to notice a relaunch later; record nothing rather than risk serving
47
71
  // a stale shallow rung to a fresh process.
@@ -116,6 +140,7 @@ extension RunnerTests {
116
140
  maxDepth: depth,
117
141
  maxNodes: Self.privateAXSnapshotMaxNodes,
118
142
  deepExtensionCallLimit: exactDepthRequested ? 0 : Self.privateAXDeepExtensionCallLimit,
143
+ customActionLimit: options.customActions ? Self.privateAXCustomActionLimit : 0,
119
144
  deadline: deadline
120
145
  )
121
146
  if response["ok"] as? Bool == true {
@@ -184,7 +209,10 @@ extension RunnerTests {
184
209
  )
185
210
  return SnapshotBackendCapture(
186
211
  payload: DataPayload(nodes: nodes, truncated: (response["truncated"] as? Bool) == true),
187
- effectiveDepth: depthLimited ? effectiveDepth : nil
212
+ effectiveDepth: depthLimited ? effectiveDepth : nil,
213
+ customActions: Self.privateAXCustomActionCoverage(
214
+ response[RunnerAXSnapshotCustomActionsKey]
215
+ )
188
216
  )
189
217
  #else
190
218
  return nil
@@ -274,7 +302,8 @@ extension RunnerTests {
274
302
  depth: depth,
275
303
  parentIndex: parentIndex,
276
304
  hiddenContentAbove: nil,
277
- hiddenContentBelow: nil
305
+ hiddenContentBelow: nil,
306
+ actions: rawNode["actions"] as? [String]
278
307
  )
279
308
  )
280
309
  } else {
@@ -586,12 +586,18 @@ extension RunnerTests {
586
586
  /// Pure command→options projection, extracted so the runner unit bundle can
587
587
  /// prove the decoded wire field actually reaches capture options (#1634 P2).
588
588
  static func snapshotOptions(from command: Command) -> SnapshotOptions {
589
- SnapshotOptions(
589
+ let customActions = command.customActions ?? false
590
+ return SnapshotOptions(
590
591
  interactiveOnly: command.interactiveOnly ?? false,
591
592
  depth: command.depth,
592
593
  scope: command.scope,
593
594
  raw: command.raw ?? false,
595
+ // Custom actions are only readable through the private AX client, so
596
+ // asking for them pins that backend rather than silently returning a
597
+ // capture that structurally cannot carry them. An explicit pin wins.
594
598
  preferredBackend: command.preferredBackend
599
+ ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil),
600
+ customActions: customActions
595
601
  )
596
602
  }
597
603
 
@@ -871,7 +877,7 @@ extension RunnerTests {
871
877
  return .context(ActiveCommandContext(app: activeApp))
872
878
  }
873
879
 
874
- private func executeOnMainPrepared(
880
+ func executeOnMainPrepared(
875
881
  command: Command,
876
882
  activeApp: XCUIApplication,
877
883
  alertDeadline: Date? = nil
@@ -1016,7 +1022,23 @@ extension RunnerTests {
1016
1022
  return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
1017
1023
  }
1018
1024
  if let x = command.x, let y = command.y {
1019
- let textInput = textInputAt(app: activeApp, x: x, y: y)
1025
+ let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(
1026
+ bundleId: currentBundleId
1027
+ )
1028
+ let xCTestTextInputProbeSkipped = !shouldProbeCoordinateTapTextInput(
1029
+ xCTestChannelPenalized: xCTestChannelPenalized
1030
+ )
1031
+ let textInput: XCUIElement?
1032
+ if !xCTestTextInputProbeSkipped {
1033
+ textInput = textInputAt(app: activeApp, x: x, y: y)
1034
+ } else {
1035
+ // A process-scoped tap cannot authorize later typing without concrete element identity.
1036
+ textInput = nil
1037
+ NSLog(
1038
+ "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@",
1039
+ currentBundleId ?? ""
1040
+ )
1041
+ }
1020
1042
  var fallback: GestureFallback?
1021
1043
  if command.synthesized == true {
1022
1044
  let policyKind = SynthesizedGesturePolicyKind.coordinateTap
@@ -1710,11 +1732,11 @@ extension RunnerTests {
1710
1732
  return message == "scrolled" ? "scroll" : "drag"
1711
1733
  }
1712
1734
 
1713
- private func currentXCTestFailureCount() -> Int {
1735
+ func currentXCTestFailureCount() -> Int {
1714
1736
  return testRun?.failureCount ?? 0
1715
1737
  }
1716
1738
 
1717
- private func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
1739
+ func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
1718
1740
  return currentXCTestFailureCount() > failureCountBefore
1719
1741
  }
1720
1742
 
@@ -1736,7 +1758,7 @@ extension RunnerTests {
1736
1758
  )
1737
1759
  }
1738
1760
 
1739
- private func runnerCommandFixture(_ json: String) throws -> Command {
1761
+ func runnerCommandFixture(_ json: String) throws -> Command {
1740
1762
  try JSONDecoder().decode(Command.self, from: Data(json.utf8))
1741
1763
  }
1742
1764
 
@@ -1815,7 +1837,7 @@ extension RunnerTests {
1815
1837
  return XCUIApplication(bundleIdentifier: bundleId)
1816
1838
  }
1817
1839
 
1818
- private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
1840
+ func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
1819
1841
  guard let text = command.text else {
1820
1842
  return Response(ok: false, error: ErrorPayload(message: "type requires text"))
1821
1843
  }
@@ -129,7 +129,8 @@ extension RunnerTests {
129
129
  selectorKey: String,
130
130
  selectorValue: String,
131
131
  allowNonHittableFallback: Bool = false,
132
- expectedPoint: CGPoint? = nil
132
+ expectedPoint: CGPoint? = nil,
133
+ rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches
133
134
  ) -> SelectorElementMatch {
134
135
  let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
135
136
  guard !value.isEmpty else {
@@ -149,35 +150,32 @@ extension RunnerTests {
149
150
  return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
150
151
  }
151
152
 
152
- var matchedElement: XCUIElement?
153
- var nonHittableElement: XCUIElement?
154
153
  let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
155
- for element in matches where element.exists {
156
- if let expectedPoint, !element.frame.contains(expectedPoint) {
157
- continue
158
- }
159
- if !element.isHittable {
160
- if allowNonHittableFallback && hasTappableFrame(app: app, element: element) {
161
- guard nonHittableElement == nil else {
162
- return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
163
- }
164
- nonHittableElement = element
165
- }
166
- continue
167
- }
168
- guard matchedElement == nil else {
169
- return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
170
- }
171
- matchedElement = element
154
+ .filter(\.exists)
155
+ let facts = matches.map { element in
156
+ SelectorCandidateFacts(
157
+ isHittable: element.isHittable,
158
+ hasTappableFrame: hasTappableFrame(app: app, element: element),
159
+ containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true
160
+ )
172
161
  }
173
- if let matchedElement {
174
- return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false)
162
+ switch classifyDirectSelectorCandidates(
163
+ facts,
164
+ allowNonHittableFallback: allowNonHittableFallback,
165
+ filtersByExpectedPoint: expectedPoint != nil,
166
+ rawMatchPolicy: rawMatchPolicy
167
+ ) {
168
+ case .noMatch:
169
+ return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
170
+ case .ambiguous:
171
+ return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
172
+ case let .selected(index, usedNonHittableFallback):
173
+ return SelectorElementMatch(
174
+ element: matches[index],
175
+ isAmbiguous: false,
176
+ usedNonHittableFallback: usedNonHittableFallback
177
+ )
175
178
  }
176
- return SelectorElementMatch(
177
- element: nonHittableElement,
178
- isAmbiguous: false,
179
- usedNonHittableFallback: nonHittableElement != nil
180
- )
181
179
  }
182
180
 
183
181
  // Maestro-compat gate for the non-hittable coordinate fallback: an element
@@ -209,7 +207,17 @@ extension RunnerTests {
209
207
  }
210
208
 
211
209
  func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
212
- let match = findElement(app: app, selectorKey: selectorKey, selectorValue: selectorValue)
210
+ // querySelector is a read — it backs get/is/wait and the offscreen-refusal
211
+ // double-check, none of which mutate. The fail-closed raw-match rule exists
212
+ // to stop a mutation acting on an unseen duplicate; applying it here would
213
+ // instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH
214
+ // for readers that previously resolved the hittable element.
215
+ let match = findElement(
216
+ app: app,
217
+ selectorKey: selectorKey,
218
+ selectorValue: selectorValue,
219
+ rawMatchPolicy: .preferHittableMatch
220
+ )
213
221
  if match.isAmbiguous {
214
222
  return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
215
223
  }
@@ -138,6 +138,7 @@ struct Command: Codable {
138
138
  let fps: Int?
139
139
  let interactiveOnly: Bool?
140
140
  let preferredBackend: String?
141
+ let customActions: Bool?
141
142
  let depth: Int?
142
143
  let scope: String?
143
144
  let raw: Bool?
@@ -389,6 +390,9 @@ struct SnapshotNode: Codable {
389
390
  let parentIndex: Int?
390
391
  let hiddenContentAbove: Bool?
391
392
  let hiddenContentBelow: Bool?
393
+ /// Names of the element's UIAccessibilityCustomActions, present only when the
394
+ /// capture asked for them and the element carries some.
395
+ var actions: [String]? = nil
392
396
  }
393
397
 
394
398
  struct SnapshotOptions {
@@ -400,4 +404,7 @@ struct SnapshotOptions {
400
404
  /// health ("private-ax"). Same-backend evidence probes (tap-outcome
401
405
  /// corroboration) must be captured the way their baseline was.
402
406
  var preferredBackend: String? = nil
407
+ /// Read UIAccessibilityCustomActions for merged leaves. Opt-in: each element
408
+ /// costs its own AX round trip (see RunnerAXSnapshotBridge).
409
+ var customActions: Bool = false
403
410
  }
@@ -493,7 +493,8 @@ extension RunnerTests {
493
493
  reason: failure.message,
494
494
  reasonCode: "ax-rejected",
495
495
  effectiveDepth: nil,
496
- collapsedLeafIndexes: nil
496
+ collapsedLeafIndexes: nil,
497
+ customActions: nil
497
498
  ),
498
499
  runnerFatal: true,
499
500
  runnerFatalReason: Self.axSnapshotUnavailableReason
@@ -22,6 +22,23 @@ struct SnapshotQuality: Codable {
22
22
  let effectiveDepth: Int?
23
23
  /// Leaves that merge many labels — a container marked accessible hides its descendants.
24
24
  let collapsedLeafIndexes: [Int]?
25
+ /// Coverage of the bounded custom-action pass, when the capture asked for one.
26
+ let customActions: SnapshotCustomActionCoverage?
27
+ }
28
+
29
+ /// How much of the merged-element set the custom-action pass actually read. An
30
+ /// unread element renders exactly like one with no actions, so a partial pass
31
+ /// has to say so — otherwise absence reads as proof of absence.
32
+ struct SnapshotCustomActionCoverage: Codable {
33
+ let read: Int
34
+ let candidates: Int
35
+ /// Elements whose action list was clipped by the per-element output caps. A
36
+ /// clipped list looks complete, so it is disclosed on the same principle as
37
+ /// an unread element.
38
+ let truncated: Int
39
+ /// The pass stopped early because an earlier read is still hung. Distinct from
40
+ /// a budget stop: the remedy is waiting, not scrolling.
41
+ let blocked: Bool
25
42
  }
26
43
 
27
44
  enum SnapshotBackendKind: String, CaseIterable {
@@ -78,6 +95,8 @@ struct SnapshotBackendCapture {
78
95
  let payload: DataPayload
79
96
  /// Set by the private AX backend when the ladder accepted a shallower depth than requested.
80
97
  let effectiveDepth: Int?
98
+ /// Set by the private AX backend when the capture asked for custom actions.
99
+ var customActions: SnapshotCustomActionCoverage? = nil
81
100
  }
82
101
 
83
102
  extension RunnerTests {
@@ -149,13 +168,25 @@ extension RunnerTests {
149
168
  /// degraded on this capture, and the daemon keys warning suppression and the settle budget
150
169
  /// reset off exactly that distinction. The bounded probe keeps 'budget': its short XCTest
151
170
  /// slice genuinely constrains what this capture could read.
171
+ ///
172
+ /// `requestPinnedBackend` separates the two ways the plan can arrive at the same
173
+ /// deferred shape. A capture that ASKED for a private-AX-only reading (custom
174
+ /// actions) degraded nothing, so reporting slow accessibility work would name a
175
+ /// cause that does not exist.
152
176
  static func xcTestChannelStateFirstFailure(
153
- _ state: SnapshotXCTestChannelPlanState
177
+ _ state: SnapshotXCTestChannelPlanState,
178
+ requestPinnedBackend: Bool = false
154
179
  ) -> (reason: String, code: String)? {
155
180
  switch state {
156
181
  case .normal:
157
182
  return nil
158
183
  case .deferredToIndependentBackend:
184
+ if requestPinnedBackend {
185
+ return (
186
+ "the private AX backend was selected because this capture asked for accessibility custom actions",
187
+ "requested-backend"
188
+ )
189
+ }
159
190
  return (
160
191
  "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen",
161
192
  "deferred"
@@ -237,9 +268,11 @@ extension RunnerTests {
237
268
  // penalized route: privateAX-first plan, 'deferred' verdict — the backend was pre-selected
238
269
  // deliberately, so no degradation warning should render for it.
239
270
  var xCTestChannelPenalized = false
271
+ var xCTestChannelPenalizedByBreaker = false
240
272
  #if os(iOS)
273
+ xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
241
274
  xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized(
242
- penalized: isSnapshotXCTestChannelPenalized(bundleId: currentBundleId),
275
+ penalized: xCTestChannelPenalizedByBreaker,
243
276
  preferredBackend: options.preferredBackend
244
277
  )
245
278
  #endif
@@ -249,7 +282,12 @@ extension RunnerTests {
249
282
  availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform))
250
283
  )
251
284
  let effectivePlan = effective.plan
252
- firstFailure = Self.xcTestChannelStateFirstFailure(effective.xCTestChannelState)
285
+ // Only a customActions-implied pin is request-pinned; the daemon's
286
+ // same-backend evidence probe pins for its own reasons and keeps 'deferred'.
287
+ firstFailure = Self.xcTestChannelStateFirstFailure(
288
+ effective.xCTestChannelState,
289
+ requestPinnedBackend: options.customActions && !xCTestChannelPenalizedByBreaker
290
+ )
253
291
  switch effective.xCTestChannelState {
254
292
  case .normal:
255
293
  break
@@ -462,7 +500,7 @@ extension RunnerTests {
462
500
  return ("snapshot returned no nodes", "no-nodes")
463
501
  }
464
502
  if isSparseApplicationWindowTree(nodes) {
465
- return ("snapshot returned only structural application/window nodes", "sparse-tree")
503
+ return ("snapshot returned no semantic controls or content", "sparse-tree")
466
504
  }
467
505
  if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold {
468
506
  return ("snapshot was cut off by its budget with almost nothing collected", "budget")
@@ -493,19 +531,36 @@ extension RunnerTests {
493
531
 
494
532
  static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
495
533
  guard !nodes.isEmpty else { return false }
534
+ let rootRects = nodes.compactMap { node in
535
+ node.type == "Application" || node.type == "Window" ? node.rect : nil
536
+ }
496
537
  return nodes.allSatisfy { node in
497
538
  // Application/Window labels are just the app/window name, and full-screen roots
498
539
  // compute as hittable; neither says anything about tree health.
499
540
  let isRootContainer = node.type == "Application" || node.type == "Window"
500
- let hasContent = (!isRootContainer && node.label?.isEmpty == false)
501
- || node.identifier?.isEmpty == false
502
- || node.value?.isEmpty == false
503
- return !hasContent
504
- && (isRootContainer || !node.hittable)
505
- && Self.structuralOnlyNodeTypes.contains(node.type)
541
+ guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false }
542
+ guard !isRootContainer else { return true }
543
+
544
+ let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in
545
+ rootRect.x == node.rect.x && rootRect.y == node.rect.y
546
+ && rootRect.width == node.rect.width && rootRect.height == node.rect.height
547
+ }
548
+ let hasAddressableIdentifier = node.identifier?.isEmpty == false && !isFullScreenContainer
549
+ return !Self.isSemanticSnapshotText(node.label)
550
+ && !Self.isSemanticSnapshotText(node.value)
551
+ && !hasAddressableIdentifier
506
552
  }
507
553
  }
508
554
 
555
+ /// Private AX can stringify an unserializable accessibility label as the JavaScript object
556
+ /// placeholder. It is transport residue, not UI content, and must not make a shell-only tree
557
+ /// look healthy.
558
+ static func isSemanticSnapshotText(_ text: String?) -> Bool {
559
+ guard let text else { return false }
560
+ let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines)
561
+ return !normalized.isEmpty && normalized.lowercased() != "[object object]"
562
+ }
563
+
509
564
  /// A leaf whose label joins many short segments is a container marked as an accessibility
510
565
  /// element: the platform folds every descendant into one merged node. Nothing below it can
511
566
  /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
@@ -536,7 +591,8 @@ extension RunnerTests {
536
591
  reason: reason?.reason,
537
592
  reasonCode: reason?.code,
538
593
  effectiveDepth: capture.effectiveDepth,
539
- collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? [])
594
+ collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []),
595
+ customActions: capture.customActions
540
596
  )
541
597
  return DataPayload(
542
598
  // Legacy human text for older daemons that read message instead of snapshotQuality.
@@ -549,8 +605,39 @@ extension RunnerTests {
549
605
  )
550
606
  }
551
607
 
608
+ /// Response level, one line per incompleteness. An unread merged element is
609
+ /// byte-identical to one with no actions, and a clipped list looks complete,
610
+ /// so both have to name themselves.
611
+ static func customActionCoverageWarnings(_ coverage: SnapshotCustomActionCoverage) -> [String] {
612
+ var lines: [String] = []
613
+ if coverage.blocked {
614
+ // Scrolling is the remedy for a budget stop, not for this one — saying it
615
+ // here would send the reader off doing something that cannot help.
616
+ lines.append(
617
+ "Custom actions were not read: an earlier accessibility read is still hung, so this "
618
+ + "capture skipped the read pass instead of queueing behind it. No element's actions "
619
+ + "list is authoritative here. Reads resume once that call returns.")
620
+ } else if coverage.read < coverage.candidates {
621
+ lines.append(
622
+ "Custom actions were read for \(coverage.read) of \(coverage.candidates) merged elements, "
623
+ + "on-screen ones first; the remaining \(coverage.candidates - coverage.read) were not read, "
624
+ + "so an absent actions list on those is not evidence that they have none. "
625
+ + "Scroll them into view and re-run to read them.")
626
+ }
627
+ if coverage.truncated > 0 {
628
+ lines.append(
629
+ "\(coverage.truncated) element(s) published more custom actions than are shown; those "
630
+ + "lists are clipped to the first 8 names, and long names are shortened.")
631
+ }
632
+ return lines
633
+ }
634
+
552
635
  static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? {
553
- guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil }
636
+ let customActionWarnings =
637
+ quality.customActions.map { Self.customActionCoverageWarnings($0) } ?? []
638
+ guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil
639
+ || !customActionWarnings.isEmpty
640
+ else { return nil }
554
641
  var parts: [String] = []
555
642
  if quality.state == "recovered" {
556
643
  let meaning = quality.reasonCode == "budget" || quality.reasonCode == "deferred"
@@ -569,6 +656,7 @@ extension RunnerTests {
569
656
  + ". Use screenshot as visual truth and coordinate taps."
570
657
  )
571
658
  }
659
+ parts.append(contentsOf: customActionWarnings)
572
660
  if let depth = quality.effectiveDepth {
573
661
  // No --depth remedy here: an explicit --depth capture disables the
574
662
  // frontier extension, so following it would return strictly less than
@@ -101,6 +101,10 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz
101
101
  }
102
102
  }
103
103
 
104
+ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool {
105
+ !xCTestChannelPenalized
106
+ }
107
+
104
108
  func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
105
109
  steps.contains { step in
106
110
  step.synthesized == true && step.kind == "tap"
@@ -159,6 +159,63 @@ extension RunnerTests {
159
159
  repairMode == .none && fromTapWitness && !softwareKeyboardVisible
160
160
  }
161
161
 
162
+ enum SynthesizedTextCommitProgress: Equatable {
163
+ case committed
164
+ case pending
165
+ case diverged
166
+ }
167
+
168
+ // The private synthesize call returns once the event record is posted, not once the target
169
+ // app has committed the characters, so intermediate reads walk prefix-by-prefix toward the
170
+ // expected value. Anything off that prefix path means the app transformed the input
171
+ // (formatter, mid-text caret, autocomplete) and the runner must not second-guess it. An
172
+ // unreadable value — secure field, or the element stopped resolving — ends the wait the
173
+ // same way.
174
+ static func synthesizedTextCommitProgress(
175
+ observedText: String?,
176
+ expectedText: String
177
+ ) -> SynthesizedTextCommitProgress {
178
+ guard let observedText else {
179
+ return .diverged
180
+ }
181
+ if observedText == expectedText {
182
+ return .committed
183
+ }
184
+ return expectedText.hasPrefix(observedText) ? .pending : .diverged
185
+ }
186
+
187
+ /// Blocks until the synthesized bare-type text is observable in the target field, so `type`
188
+ /// cannot report ok while trailing characters are still uncommitted on a slow simulator.
189
+ ///
190
+ /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the
191
+ /// event stream, so re-synthesizing the difference risks committing it twice after the command
192
+ /// already reported success. Text carrying a submit key is skipped outright: the app may clear
193
+ /// or rewrite the field on submit, so `textBefore + typedText` is not the value to wait for.
194
+ func awaitSynthesizedFirstResponderCommit(
195
+ app: XCUIApplication,
196
+ target: TextEntryTarget,
197
+ textBefore: String?,
198
+ typedText: String
199
+ ) {
200
+ guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else {
201
+ return
202
+ }
203
+ let expectedText = textBefore + typedText
204
+ let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout)
205
+ while Date() < deadline {
206
+ let observedText = editableTextValue(
207
+ for: resolveTextEntryElement(app: app, target: target),
208
+ treatingPlaceholderAsEmpty: true
209
+ )
210
+ switch Self.synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
211
+ case .committed, .diverged:
212
+ return
213
+ case .pending:
214
+ sleepFor(TextEntryTiming.pollInterval)
215
+ }
216
+ }
217
+ }
218
+
162
219
  static func shouldUseResolvedCoordinateTextEntryRoute(
163
220
  repairMode: TextTypingRepairMode,
164
221
  hasX: Bool,
@@ -40,6 +40,7 @@ extension RunnerTests {
40
40
  static let pollInterval: TimeInterval = 0.02
41
41
  static let warmupValueTimeout: TimeInterval = 0.4
42
42
  static let verificationStabilityWindow: TimeInterval = 0.2
43
+ static let synthesizedCommitTimeout: TimeInterval = 3.0
43
44
  }
44
45
 
45
46
  struct TextEntryResult {