agent-device 0.20.5 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +5 -3
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.7.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.7.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.7.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.7.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +48 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +108 -1
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +630 -89
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +78 -0
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +149 -10
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +161 -32
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +4 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +36 -28
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +18 -1
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SelectorMatchPolicyTests.swift +4 -0
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +2 -1
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +145 -20
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +4 -0
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +234 -0
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +106 -448
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +602 -0
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
  37. package/dist/src/agent-browser-tool.js +1 -1
  38. package/dist/src/agent-device-client.js +2 -2
  39. package/dist/src/android-adb.js +1 -1
  40. package/dist/src/android.js +1 -1
  41. package/dist/src/app-helpers.js +1 -1
  42. package/dist/src/app-lifecycle.js +7 -7
  43. package/dist/src/app-lifecycle2.js +2 -0
  44. package/dist/src/app-log-files.js +1 -0
  45. package/dist/src/app-log-reconnect.js +1 -0
  46. package/dist/src/app-log-runtime.js +4 -0
  47. package/dist/src/app-resolution.js +2 -1
  48. package/dist/src/apps.js +5 -5
  49. package/dist/src/audio-probe.js +1 -1
  50. package/dist/src/auth-session.js +1 -0
  51. package/dist/src/auth.js +2 -0
  52. package/dist/src/boot-diagnostics.js +2 -2
  53. package/dist/src/cli-config.js +1 -1
  54. package/dist/src/cli-grammar.js +1 -0
  55. package/dist/src/cli-help.js +204 -254
  56. package/dist/src/cli-output.js +1 -0
  57. package/dist/src/cli-runner.js +1 -2
  58. package/dist/src/cli.js +6 -16
  59. package/dist/src/client-metro.js +1 -1
  60. package/dist/src/cloud-artifacts.d.ts +2 -137
  61. package/dist/src/command-metadata.js +1 -1
  62. package/dist/src/command-schema.js +3 -3
  63. package/dist/src/connection-runtime.js +1 -0
  64. package/dist/src/connection.js +5 -0
  65. package/dist/src/context.js +1 -1
  66. package/dist/src/daemon-shutdown-report.js +1 -0
  67. package/dist/src/daemon.js +2 -0
  68. package/dist/src/debug-symbols.js +3 -3
  69. package/dist/src/device-claim-inspection.js +1 -0
  70. package/dist/src/device-input-state.js +2 -2
  71. package/dist/src/device.js +1 -1
  72. package/dist/src/device2.js +2 -0
  73. package/dist/src/find.js +1 -1
  74. package/dist/src/finders.js +1 -1
  75. package/dist/src/generic-settle.js +1 -0
  76. package/dist/src/generic.js +1 -1
  77. package/dist/src/harmonyos.js +1 -0
  78. package/dist/src/helper.js +1 -1
  79. package/dist/src/host-process.js +2 -2
  80. package/dist/src/ime-lifecycle.js +7 -7
  81. package/dist/src/index.d.ts +69 -18
  82. package/dist/src/input-actions.js +4 -4
  83. package/dist/src/input-actions2.js +1 -1
  84. package/dist/src/input-actions3.js +1 -0
  85. package/dist/src/install-artifact.js +1 -5
  86. package/dist/src/install-artifact2.js +1 -1
  87. package/dist/src/install-source2.js +1 -1
  88. package/dist/src/interaction-runtime.js +1 -0
  89. package/dist/src/interaction-snapshot.js +1 -1
  90. package/dist/src/interaction-touch-reference-frame.js +1 -0
  91. package/dist/src/interaction.js +1 -1
  92. package/dist/src/interactor.js +2 -2
  93. package/dist/src/interactor2.js +1 -1
  94. package/dist/src/internal/bin.js +2 -2
  95. package/dist/src/internal/daemon.js +9 -16
  96. package/dist/src/inventory.js +5 -0
  97. package/dist/src/inventory2.js +1 -0
  98. package/dist/src/inventory3.js +2 -0
  99. package/dist/src/inventory4.js +1 -0
  100. package/dist/src/inventory5.js +1 -0
  101. package/dist/src/inventory6.js +1 -0
  102. package/dist/src/lease-context.js +1 -1
  103. package/dist/src/lease.js +1 -1
  104. package/dist/src/limrun-runtime-dependencies.js +1 -0
  105. package/dist/src/limrun.d.ts +10 -104
  106. package/dist/src/limrun.js +1 -1
  107. package/dist/src/linux.js +1 -1
  108. package/dist/src/logcat.js +1 -1
  109. package/dist/src/manifest.js +1 -1
  110. package/dist/src/owner-identity.js +1 -0
  111. package/dist/src/physical-device-control.js +1 -1
  112. package/dist/src/platform-runtime-app-log-android-transport.js +1 -0
  113. package/dist/src/platform-runtime-app-log-process.js +1 -0
  114. package/dist/src/platform-runtime-host.js +1 -0
  115. package/dist/src/platform-runtime-network-web-transport.js +1 -0
  116. package/dist/src/platform-runtime-operation-host.js +5 -0
  117. package/dist/src/platform-runtime-toolchain-host.js +1 -0
  118. package/dist/src/platform-runtime.js +1 -0
  119. package/dist/src/prepare-kind.js +15 -0
  120. package/dist/src/process-entry.js +1 -0
  121. package/dist/src/process-exit.js +1 -0
  122. package/dist/src/process-lock.js +1 -1
  123. package/dist/src/provider-webdriver.js +1 -0
  124. package/dist/src/provider.js +1 -0
  125. package/dist/src/proxy.js +2 -0
  126. package/dist/src/react-native.js +1 -1
  127. package/dist/src/record-trace.js +1 -1
  128. package/dist/src/registry.js +25 -24
  129. package/dist/src/remote-config.js +1 -1
  130. package/dist/src/remote-config2.js +1 -1
  131. package/dist/src/replay.js +3 -0
  132. package/dist/src/reporting.js +4 -4
  133. package/dist/src/runner-client.js +1 -1
  134. package/dist/src/runner-disposal.js +1 -1
  135. package/dist/src/runner-lease.js +1 -1
  136. package/dist/src/runtime.js +3 -3
  137. package/dist/src/runtime2.js +4 -0
  138. package/dist/src/runtime3.js +3 -0
  139. package/dist/src/runtime4.js +1 -0
  140. package/dist/src/runtime5.js +1 -0
  141. package/dist/src/runtime6.js +1 -0
  142. package/dist/src/runtime7.js +1 -0
  143. package/dist/src/screenshot-result.js +4 -4
  144. package/dist/src/screenshot.js +1 -0
  145. package/dist/src/sdk-android-adb.d.ts +20 -2
  146. package/dist/src/sdk-android-adb.js +2 -2
  147. package/dist/src/sdk-batch-runner.js +3 -3
  148. package/dist/src/sdk-contracts.d.ts +47 -6
  149. package/dist/src/sdk-contracts.js +1 -1
  150. package/dist/src/sdk-device.js +1 -1
  151. package/dist/src/sdk-finders.d.ts +4 -20
  152. package/dist/src/sdk-finders.js +1 -1
  153. package/dist/src/sdk-remote-config.js +1 -1
  154. package/dist/src/sdk-selectors.d.ts +601 -0
  155. package/dist/src/sdk-selectors.js +1 -1
  156. package/dist/src/selector-runtime.js +1 -1
  157. package/dist/src/selector-vocabulary.js +1 -1
  158. package/dist/src/selectors.d.ts +1 -46
  159. package/dist/src/selectors.js +1 -1
  160. package/dist/src/server.js +3 -3
  161. package/dist/src/session-target-evidence.js +1 -1
  162. package/dist/src/session.js +10 -9
  163. package/dist/src/shared.js +2 -0
  164. package/dist/src/shell-quote.js +1 -0
  165. package/dist/src/simulator.js +1 -1
  166. package/dist/src/snapshot-runtime.js +1 -0
  167. package/dist/src/snapshot-session.js +1 -0
  168. package/dist/src/snapshot.js +1 -1
  169. package/dist/src/snapshot2.js +1 -1
  170. package/dist/src/src.js +1 -1
  171. package/dist/src/src2.js +2 -0
  172. package/dist/src/src3.js +1 -0
  173. package/dist/src/text-surface.js +1 -0
  174. package/dist/src/timing-safe-equal.js +1 -0
  175. package/dist/src/tool-provider.js +3 -3
  176. package/dist/src/tool-provider2.js +1 -1
  177. package/dist/src/tool-provider3.js +1 -1
  178. package/dist/src/update-check.js +1 -1
  179. package/dist/src/version.js +1 -1
  180. package/dist/src/web.js +1 -1
  181. package/package.json +72 -50
  182. package/server.json +3 -3
  183. package/skills/agent-device/SKILL.md +6 -31
  184. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
  185. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
  186. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
  187. package/dist/src/app-log-request-scope.js +0 -1
  188. package/dist/src/devices.js +0 -1
  189. package/dist/src/devices2.js +0 -1
  190. package/dist/src/devices3.js +0 -1
  191. package/dist/src/devices4.js +0 -1
  192. package/dist/src/interaction.d.ts +0 -230
  193. package/dist/src/provider-limrun-runtime.js +0 -1
  194. package/dist/src/register-builtins.js +0 -1
  195. package/dist/src/screenshot-geometry.js +0 -1
  196. package/dist/src/viewport-dimension.js +0 -14
@@ -0,0 +1,602 @@
1
+ import XCTest
2
+
3
+ // Text typing, verification, and repair. Kept separate from focus/readiness so each
4
+ // text-entry policy remains a bounded review surface.
5
+ extension RunnerTests {
6
+ func typeTextReliably(
7
+ app: XCUIApplication,
8
+ target: TextEntryTarget,
9
+ text: String,
10
+ delaySeconds: Double,
11
+ repairMode: TextTypingRepairMode = .none,
12
+ xCTestChannelPenalized: Bool = false,
13
+ synthesizer: any TextEntrySynthesizing,
14
+ commandId: String? = nil
15
+ ) -> TextEntryResult {
16
+ let totalStartedAt = Date()
17
+ guard !text.isEmpty else {
18
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
19
+ return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
20
+ }
21
+ var activeTarget = target
22
+ if repairMode == .replacement,
23
+ Self.shouldUseSynthesizedFirstResponderReplacement(
24
+ hasResolvedElement: target.element != nil,
25
+ hasRefreshPoint: target.refreshPoint != nil,
26
+ xCTestChannelPenalized: xCTestChannelPenalized
27
+ )
28
+ {
29
+ switch runSynthesizedReplacementRoute(
30
+ SynthesizedReplacementRequest(
31
+ app: app,
32
+ target: target,
33
+ text: text,
34
+ delaySeconds: delaySeconds,
35
+ synthesizer: synthesizer,
36
+ commandId: commandId,
37
+ startedAt: totalStartedAt
38
+ )
39
+ ) {
40
+ case .notApplicable:
41
+ break
42
+ case .completed(let result):
43
+ return result
44
+ case .fallback(let fallbackTarget):
45
+ activeTarget = fallbackTarget
46
+ }
47
+ }
48
+ let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType(
49
+ repairMode: repairMode,
50
+ fromTapWitness: activeTarget.fromTapWitness,
51
+ softwareKeyboardVisible: isKeyboardVisible(app: app)
52
+ )
53
+ let initialResolveStartedAt = Date()
54
+ let initialTarget = resolveTextEntryElement(app: app, target: activeTarget)
55
+ activeTarget = activeTarget.withElement(initialTarget)
56
+ let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true)
57
+ let initialText = repairMode == .append ? currentText : nil
58
+ let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText)
59
+ var textEntryRoute: String?
60
+ logTextEntryPhase(
61
+ commandId: commandId,
62
+ phase: "initial-resolve",
63
+ startedAt: initialResolveStartedAt,
64
+ chars: text.count,
65
+ mode: repairMode
66
+ )
67
+
68
+ // Dispatching text through XCTest without a resolved target or evidence of a focused
69
+ // responder records a test failure. That tears down the long-lived runner and turns a
70
+ // single invalid request into a restart cascade, so fail before entering that channel.
71
+ guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else {
72
+ logTextEntryPhase(
73
+ commandId: commandId,
74
+ phase: "total",
75
+ startedAt: totalStartedAt,
76
+ chars: text.count,
77
+ mode: repairMode
78
+ )
79
+ return TextEntryResult(
80
+ verified: nil,
81
+ repaired: false,
82
+ expectedText: expectedText,
83
+ observedText: nil,
84
+ failure: .notFocused
85
+ )
86
+ }
87
+
88
+ if repairMode == .replacement {
89
+ guard let replacementTarget = initialTarget else {
90
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
91
+ return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
92
+ }
93
+ if currentText == nil || currentText?.isEmpty == false {
94
+ let clearStartedAt = Date()
95
+ clearTextInput(replacementTarget)
96
+ activeTarget = activeTarget.withElement(replacementTarget)
97
+ logTextEntryPhase(
98
+ commandId: commandId,
99
+ phase: "clear",
100
+ startedAt: clearStartedAt,
101
+ chars: text.count,
102
+ mode: repairMode
103
+ )
104
+ }
105
+ }
106
+
107
+ func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) {
108
+ #if os(iOS)
109
+ if shouldUseSynthesizedFirstResponderType {
110
+ guard let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) else {
111
+ return (nil, false, .notFocused)
112
+ }
113
+ textEntryRoute = "synthesized-first-responder"
114
+ NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
115
+ let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true)
116
+ switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
117
+ case .continueTyping:
118
+ // No refresh point: like the tap-witness target itself, the commit wait must observe
119
+ // only the element the tap selected, never rediscover a different field.
120
+ awaitSynthesizedFirstResponderCommit(
121
+ app: app,
122
+ target: TextEntryTarget(
123
+ element: currentTarget,
124
+ refreshPoint: nil,
125
+ prefersFocusedElement: false,
126
+ fromTapWitness: true
127
+ ),
128
+ textBefore: textBefore,
129
+ typedText: value
130
+ )
131
+ return (currentTarget, true, nil)
132
+ case .fallback:
133
+ return (nil, false, .synthesisUnavailable)
134
+ case .raise(let message):
135
+ NSException(
136
+ name: NSExceptionName.internalInconsistencyException,
137
+ reason: message ?? "private XCTest text synthesis failed"
138
+ ).raise()
139
+ }
140
+ }
141
+ #endif
142
+ if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
143
+ textEntryRoute = "xctest-element"
144
+ currentTarget.typeText(value)
145
+ return (currentTarget, true, nil)
146
+ } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
147
+ #if os(iOS)
148
+ textEntryRoute = "synthesized-first-responder"
149
+ NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
150
+ let action = synthesizer.enterText(
151
+ app: app,
152
+ text: value,
153
+ replacingExistingText: false
154
+ )
155
+ switch action {
156
+ case .fallback:
157
+ textEntryRoute = "xctest-application-fallback"
158
+ app.typeText(value)
159
+ case .raise(let message):
160
+ NSException(
161
+ name: NSExceptionName.internalInconsistencyException,
162
+ reason: message ?? "private XCTest text synthesis failed"
163
+ ).raise()
164
+ case .continueTyping:
165
+ break
166
+ }
167
+ #else
168
+ app.typeText(value)
169
+ #endif
170
+ return (resolveTextEntryElement(app: app, target: activeTarget), true, nil)
171
+ }
172
+ return (nil, false, .notFocused)
173
+ }
174
+
175
+ func dispatchFailureResult(_ failure: TextEntryFailure) -> TextEntryResult {
176
+ TextEntryResult(
177
+ verified: nil,
178
+ repaired: false,
179
+ expectedText: expectedText,
180
+ observedText: nil,
181
+ textEntryRoute: textEntryRoute,
182
+ failure: failure
183
+ )
184
+ }
185
+
186
+ func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) {
187
+ guard let expectedValue else {
188
+ sleepFor(TextEntryTiming.pollInterval)
189
+ return
190
+ }
191
+ let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
192
+ while Date() < deadline {
193
+ if editableTextValue(for: resolveTextEntryElement(app: app, target: target)) == expectedValue {
194
+ return
195
+ }
196
+ sleepFor(TextEntryTiming.pollInterval)
197
+ }
198
+ }
199
+
200
+ let characters = Array(text)
201
+ if delaySeconds > 0 && characters.count > 1 {
202
+ var typedTarget: XCUIElement?
203
+ let delayedTypeStartedAt = Date()
204
+ for (index, character) in characters.enumerated() {
205
+ let dispatch = typeIntoCurrentTarget(String(character))
206
+ guard dispatch.dispatched else {
207
+ return dispatchFailureResult(dispatch.failure ?? .notFocused)
208
+ }
209
+ typedTarget = dispatch.element ?? typedTarget
210
+ if index + 1 < characters.count {
211
+ sleepFor(delaySeconds)
212
+ }
213
+ }
214
+ logTextEntryPhase(
215
+ commandId: commandId,
216
+ phase: "type-delayed",
217
+ startedAt: delayedTypeStartedAt,
218
+ chars: characters.count,
219
+ mode: repairMode
220
+ )
221
+ if repairMode == .none {
222
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
223
+ return TextEntryResult(
224
+ verified: nil,
225
+ repaired: false,
226
+ expectedText: nil,
227
+ observedText: nil,
228
+ textEntryRoute: textEntryRoute
229
+ )
230
+ }
231
+ let verifyStartedAt = Date()
232
+ var result = verifyTextEntryWithRepairIfNeeded(
233
+ app: app,
234
+ target: activeTarget.withElement(typedTarget),
235
+ expectedText: expectedText,
236
+ repairMode: repairMode
237
+ )
238
+ logTextEntryPhase(
239
+ commandId: commandId,
240
+ phase: "verify",
241
+ startedAt: verifyStartedAt,
242
+ chars: characters.count,
243
+ mode: repairMode
244
+ )
245
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
246
+ result.textEntryRoute = textEntryRoute
247
+ return result
248
+ }
249
+
250
+ let typedTarget: XCUIElement?
251
+ if repairMode != .none && characters.count > 1 {
252
+ let firstCharacter = String(characters[0])
253
+ let firstStartedAt = Date()
254
+ let firstDispatch = typeIntoCurrentTarget(firstCharacter)
255
+ guard firstDispatch.dispatched else {
256
+ return dispatchFailureResult(firstDispatch.failure ?? .notFocused)
257
+ }
258
+ var firstTypedTarget = firstDispatch.element
259
+ logTextEntryPhase(
260
+ commandId: commandId,
261
+ phase: "type-first",
262
+ startedAt: firstStartedAt,
263
+ chars: 1,
264
+ mode: repairMode
265
+ )
266
+ activeTarget = activeTarget.withElement(firstTypedTarget)
267
+ let warmupExpectedText = expectedTextEntryValue(
268
+ typedText: firstCharacter,
269
+ mode: repairMode,
270
+ initialText: initialText
271
+ )
272
+ let warmupStartedAt = Date()
273
+ waitForWarmupValue(warmupExpectedText, target: activeTarget)
274
+ logTextEntryPhase(
275
+ commandId: commandId,
276
+ phase: "warmup",
277
+ startedAt: warmupStartedAt,
278
+ chars: 1,
279
+ mode: repairMode
280
+ )
281
+ let remainingText = String(characters.dropFirst())
282
+ let remainingStartedAt = Date()
283
+ let remainingDispatch = typeIntoCurrentTarget(remainingText)
284
+ guard remainingDispatch.dispatched else {
285
+ return dispatchFailureResult(remainingDispatch.failure ?? .notFocused)
286
+ }
287
+ firstTypedTarget = remainingDispatch.element ?? firstTypedTarget
288
+ logTextEntryPhase(
289
+ commandId: commandId,
290
+ phase: "type-remaining",
291
+ startedAt: remainingStartedAt,
292
+ chars: characters.count - 1,
293
+ mode: repairMode
294
+ )
295
+ typedTarget = firstTypedTarget
296
+ } else {
297
+ let typeStartedAt = Date()
298
+ let dispatch = typeIntoCurrentTarget(text)
299
+ guard dispatch.dispatched else {
300
+ return dispatchFailureResult(dispatch.failure ?? .notFocused)
301
+ }
302
+ typedTarget = dispatch.element
303
+ logTextEntryPhase(
304
+ commandId: commandId,
305
+ phase: "type-all",
306
+ startedAt: typeStartedAt,
307
+ chars: characters.count,
308
+ mode: repairMode
309
+ )
310
+ }
311
+ if repairMode == .none {
312
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
313
+ return TextEntryResult(
314
+ verified: nil,
315
+ repaired: false,
316
+ expectedText: nil,
317
+ observedText: nil,
318
+ textEntryRoute: textEntryRoute
319
+ )
320
+ }
321
+ let verifyStartedAt = Date()
322
+ var result = verifyTextEntryWithRepairIfNeeded(
323
+ app: app,
324
+ target: activeTarget.withElement(typedTarget),
325
+ expectedText: expectedText,
326
+ repairMode: repairMode
327
+ )
328
+ logTextEntryPhase(
329
+ commandId: commandId,
330
+ phase: "verify",
331
+ startedAt: verifyStartedAt,
332
+ chars: characters.count,
333
+ mode: repairMode
334
+ )
335
+ logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
336
+ result.textEntryRoute = textEntryRoute
337
+ return result
338
+ }
339
+
340
+ func logTextEntryPhase(
341
+ commandId: String?,
342
+ phase: String,
343
+ startedAt: Date,
344
+ chars: Int,
345
+ mode: TextTypingRepairMode
346
+ ) {
347
+ NSLog(
348
+ "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=%@ durationMs=%.1f chars=%d mode=%@",
349
+ commandId ?? "",
350
+ phase,
351
+ Date().timeIntervalSince(startedAt) * 1000.0,
352
+ chars,
353
+ textEntryModeName(mode)
354
+ )
355
+ }
356
+
357
+ func textEntryModeName(_ mode: TextTypingRepairMode) -> String {
358
+ switch mode {
359
+ case .none:
360
+ return "none"
361
+ case .append:
362
+ return "append"
363
+ case .replacement:
364
+ return "replacement"
365
+ }
366
+ }
367
+
368
+ private func verifyTextEntryWithRepairIfNeeded(
369
+ app: XCUIApplication,
370
+ target: TextEntryTarget,
371
+ expectedText: String?,
372
+ repairMode: TextTypingRepairMode
373
+ ) -> TextEntryResult {
374
+ let initialResult = verifyTextEntry(
375
+ app: app,
376
+ target: target,
377
+ expectedText: expectedText,
378
+ repaired: false
379
+ )
380
+ #if os(iOS)
381
+ guard initialResult.verified == false,
382
+ let expectedText = initialResult.expectedText
383
+ else {
384
+ return initialResult
385
+ }
386
+ guard shouldRepairTextEntry(
387
+ app: app,
388
+ target: target,
389
+ expectedText: expectedText,
390
+ repairMode: repairMode
391
+ ) else {
392
+ return verifyTextEntry(
393
+ app: app,
394
+ target: target,
395
+ expectedText: expectedText,
396
+ repaired: false
397
+ )
398
+ }
399
+
400
+ guard let repairTarget = resolveTextEntryElement(app: app, target: target) else {
401
+ return initialResult
402
+ }
403
+ let observedText = editableTextValue(for: repairTarget) ?? ""
404
+ NSLog(
405
+ "AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY expectedLength=%d observedLength=%d",
406
+ expectedText.count,
407
+ observedText.count
408
+ )
409
+ clearTextInput(repairTarget)
410
+ repairTarget.typeText(expectedText)
411
+ return verifyTextEntry(
412
+ app: app,
413
+ target: target.withElement(repairTarget),
414
+ expectedText: expectedText,
415
+ repaired: true
416
+ )
417
+ #else
418
+ return initialResult
419
+ #endif
420
+ }
421
+
422
+ private func verifyTextEntry(
423
+ app: XCUIApplication,
424
+ target: TextEntryTarget,
425
+ expectedText: String?,
426
+ repaired: Bool
427
+ ) -> TextEntryResult {
428
+ let targetElement = resolveTextEntryElement(app: app, target: target)
429
+ guard let expectedText else {
430
+ return TextEntryResult(
431
+ verified: nil,
432
+ repaired: repaired,
433
+ expectedText: nil,
434
+ observedText: editableTextValue(for: targetElement)
435
+ )
436
+ }
437
+ guard let observedText = editableTextValue(for: targetElement) else {
438
+ return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
439
+ }
440
+ guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else {
441
+ return TextEntryResult(
442
+ verified: false,
443
+ repaired: repaired,
444
+ expectedText: expectedText,
445
+ observedText: observedText
446
+ )
447
+ }
448
+ let stableDeadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
449
+ var latestObservedText = observedText
450
+ while Date() < stableDeadline {
451
+ sleepFor(TextEntryTiming.pollInterval)
452
+ guard let nextObservedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
453
+ return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
454
+ }
455
+ latestObservedText = nextObservedText
456
+ guard textEntryValueMatchesExpected(
457
+ resolveTextEntryElement(app: app, target: target),
458
+ observedText: nextObservedText,
459
+ expectedText: expectedText
460
+ ) else {
461
+ return TextEntryResult(
462
+ verified: false,
463
+ repaired: repaired,
464
+ expectedText: expectedText,
465
+ observedText: nextObservedText
466
+ )
467
+ }
468
+ }
469
+ return TextEntryResult(
470
+ verified: true,
471
+ repaired: repaired,
472
+ expectedText: expectedText,
473
+ observedText: latestObservedText
474
+ )
475
+ }
476
+
477
+ private func textEntryValueMatchesExpected(
478
+ _ element: XCUIElement?,
479
+ observedText: String,
480
+ expectedText: String
481
+ ) -> Bool {
482
+ if observedText == expectedText {
483
+ return true
484
+ }
485
+ guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else {
486
+ return false
487
+ }
488
+ var submittedText = expectedText
489
+ while hasTextEntrySubmitSuffix(submittedText) {
490
+ submittedText.removeLast()
491
+ }
492
+ return observedText == submittedText
493
+ }
494
+
495
+ private func hasTextEntrySubmitSuffix(_ text: String) -> Bool {
496
+ text.hasSuffix("\n") || text.hasSuffix("\r")
497
+ }
498
+
499
+ private func expectedTextEntryValue(
500
+ typedText: String,
501
+ mode: TextTypingRepairMode,
502
+ initialText: String?
503
+ ) -> String? {
504
+ switch mode {
505
+ case .none:
506
+ return nil
507
+ case .append:
508
+ guard let initialText else {
509
+ return nil
510
+ }
511
+ return initialText + typedText
512
+ case .replacement:
513
+ return typedText
514
+ }
515
+ }
516
+
517
+ private func shouldRepairTextEntry(
518
+ app: XCUIApplication,
519
+ target: TextEntryTarget,
520
+ expectedText: String,
521
+ repairMode: TextTypingRepairMode
522
+ ) -> Bool {
523
+ #if os(iOS)
524
+ var latestObservedText: String?
525
+ let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
526
+ repeat {
527
+ guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
528
+ return false
529
+ }
530
+ if textEntryValueMatchesExpected(
531
+ resolveTextEntryElement(app: app, target: target),
532
+ observedText: observedText,
533
+ expectedText: expectedText
534
+ ) {
535
+ return false
536
+ }
537
+ latestObservedText = observedText
538
+ if !isRepairableTextEntryMismatch(
539
+ observedText: observedText,
540
+ expectedText: expectedText,
541
+ repairMode: repairMode
542
+ ) {
543
+ return false
544
+ }
545
+ sleepFor(TextEntryTiming.pollInterval)
546
+ } while Date() < deadline
547
+
548
+ guard let latestObservedText else {
549
+ return false
550
+ }
551
+ guard !textEntryValueMatchesExpected(
552
+ resolveTextEntryElement(app: app, target: target),
553
+ observedText: latestObservedText,
554
+ expectedText: expectedText
555
+ ) else {
556
+ return false
557
+ }
558
+ return isRepairableTextEntryMismatch(
559
+ observedText: latestObservedText,
560
+ expectedText: expectedText,
561
+ repairMode: repairMode
562
+ )
563
+ #else
564
+ return false
565
+ #endif
566
+ }
567
+
568
+ private func isRepairableTextEntryMismatch(
569
+ observedText: String,
570
+ expectedText: String,
571
+ repairMode: TextTypingRepairMode
572
+ ) -> Bool {
573
+ guard observedText != expectedText else {
574
+ return false
575
+ }
576
+ if repairMode == .replacement {
577
+ return true
578
+ }
579
+ return observedText.isEmpty || isLikelyDroppedCharacterTextEntryMismatch(
580
+ observedText: observedText,
581
+ expectedText: expectedText
582
+ )
583
+ }
584
+
585
+ private func isLikelyDroppedCharacterTextEntryMismatch(observedText: String, expectedText: String) -> Bool {
586
+ guard observedText.count < expectedText.count else {
587
+ return false
588
+ }
589
+ let missingCharacterCount = expectedText.count - observedText.count
590
+ guard missingCharacterCount <= max(2, expectedText.count / 4) else {
591
+ return false
592
+ }
593
+ var expectedIndex = expectedText.startIndex
594
+ for character in observedText {
595
+ guard let matchIndex = expectedText[expectedIndex...].firstIndex(of: character) else {
596
+ return false
597
+ }
598
+ expectedIndex = expectedText.index(after: matchIndex)
599
+ }
600
+ return true
601
+ }
602
+ }
@@ -48,6 +48,11 @@ final class RunnerTests: XCTestCase {
48
48
  var currentApp: XCUIApplication?
49
49
  var currentBundleId: String?
50
50
  var currentAppProcessIdentifier: Int?
51
+ // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when
52
+ // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text
53
+ // input is a scoped witness for the immediately-following bare type; lifecycle and non-text
54
+ // interactions clear it before it can become stale.
55
+ var textEntryTapWitness: TextEntryTapWitness?
51
56
  let maxRequestBytes = 2 * 1024 * 1024
52
57
  let mainThreadExecutionTimeout: TimeInterval = 30
53
58
  let appExistenceTimeout: TimeInterval = 30
@@ -87,6 +92,19 @@ final class RunnerTests: XCTestCase {
87
92
  var snapshotXCTestChannelPenaltyUntil = Date.distantPast
88
93
  let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
89
94
  var snapshotXCTestPenaltyWarmupExemptionPending = false
95
+ // Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default
96
+ // depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted
97
+ // later captures start there instead of re-paying the rejected deep request (~300ms per
98
+ // capture on the Bluesky feed). Shares the penalty's lifetime model: same duration, cleared
99
+ // on target process change, so screens that regain deep-capture ability are re-probed.
100
+ let privateAXAcceptedDepthLock = NSLock()
101
+ var privateAXAcceptedDepthBundleId: String?
102
+ // PID-bound: a relaunch (external or A->B->A while inactive) changes the process, and the new
103
+ // tree may accept the full depth again. Any invalidation path that drops the cached PID makes
104
+ // the memory unmatchable, so every fresh activation re-probes the full requested depth.
105
+ var privateAXAcceptedDepthProcessIdentifier: Int?
106
+ var privateAXAcceptedDepth: Int?
107
+ var privateAXAcceptedDepthUntil = Date.distantPast
90
108
  // Bluesky-class screens can grind ~4-8s before an XCTest-backed snapshot tier fails; anything
91
109
  // past this threshold marks the screen hostile so the next capture uses non-XCTest recovery.
92
110
  let snapshotXCTestSlowCaptureThreshold: TimeInterval = 3
@@ -0,0 +1,72 @@
1
+ #import <Foundation/Foundation.h>
2
+
3
+ NS_ASSUME_NONNULL_BEGIN
4
+
5
+ // Shared reflection surface for XCTest's private event-synthesis API
6
+ // (XCSynthesizedEventRecord / XCPointerEventPath). Both gesture synthesis
7
+ // (RunnerSynthesizedGesture) and text-entry synthesis (RunnerSynthesizedTextEntry)
8
+ // reflect into this same private API; resolving the part they have in common here
9
+ // means a selector rename only has to be found in one place.
10
+ //
11
+ // Each caller additionally resolves whichever XCSynthesizedEventRecord init
12
+ // overload and XCPointerEventPath factory/mutator selectors its own event kind
13
+ // needs: gesture uses the 2-arg `initWithName:interfaceOrientation:` plus
14
+ // touch-path selectors (`initForTouchAtPoint:offset:`, `moveToPoint:atOffset:`,
15
+ // `liftUpAtOffset:`); text entry uses the 1-arg `initWithName:` plus text-input
16
+ // selectors (`initForTextInput`, `typeText:atOffset:typingSpeed:shouldRedact:`,
17
+ // `typeKey:modifiers:atOffset:`). Those genuinely differ and are resolved by
18
+ // each caller on top of this shared core, not forced into one shape here.
19
+ typedef struct {
20
+ Class recordClass; // XCSynthesizedEventRecord
21
+ Class pathClass; // XCPointerEventPath
22
+ SEL addPathSelector; // addPointerEventPath: (on recordClass)
23
+ SEL setTargetProcessIDSelector; // setTargetProcessID: (on recordClass)
24
+ SEL synthesizeSelector; // synthesizeWithError: (on recordClass)
25
+ SEL processIDSelector; // processID, read on the XCUIApplication
26
+ } RunnerXCTestEventBridge;
27
+
28
+ typedef NSInteger (*RunnerMsgSendInteger)(id, SEL);
29
+ typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger);
30
+ typedef void (*RunnerMsgSendAddPath)(id, SEL, id);
31
+ typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
32
+
33
+ // Resolves the shared core described above against the live XCTest runtime.
34
+ // Returns nil and populates `bridge` on success. Returns a non-nil
35
+ // "private XCTest <surface> synthesis unavailable: ..." message and leaves
36
+ // `bridge` unspecified when a class or selector the private API is expected to
37
+ // expose is missing (the API changed, or is unavailable on this OS/Xcode).
38
+ // `surface` names the calling feature ("event" or "text") so the message text
39
+ // matches what that caller has always reported.
40
+ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveXCTestEventBridge(
41
+ id application,
42
+ NSString *surface,
43
+ RunnerXCTestEventBridge *bridge
44
+ );
45
+
46
+ FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass(
47
+ Class cls,
48
+ NSString *className,
49
+ NSString *surface
50
+ );
51
+ FOUNDATION_EXPORT NSString * _Nullable RunnerRequireSelector(
52
+ Class cls,
53
+ SEL selector,
54
+ NSString *selectorName,
55
+ NSString *surface
56
+ );
57
+ FOUNDATION_EXPORT NSString * _Nullable RunnerRequireApplicationSelector(
58
+ id application,
59
+ SEL selector,
60
+ NSString *selectorName,
61
+ NSString *surface
62
+ );
63
+
64
+ // Formats an NSException the way every synthesis @catch block reports failure:
65
+ // "<name>: <reason>", falling back to "NSException" / `fallbackReason` when
66
+ // either is nil.
67
+ FOUNDATION_EXPORT NSString *RunnerFormatXCTestException(
68
+ NSException *exception,
69
+ NSString *fallbackReason
70
+ );
71
+
72
+ NS_ASSUME_NONNULL_END