agent-device 0.20.5 → 0.20.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.6.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.6.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.6.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.6.manifest.json} +6 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +28 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +41 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +298 -92
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +118 -8
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +134 -27
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +11 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +48 -11
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +177 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +105 -448
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +587 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/android.js +1 -1
- package/dist/src/app-log-request-scope.js +1 -1
- package/dist/src/app-resolution.js +1 -1
- package/dist/src/apps.js +5 -5
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +34 -24
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +15 -15
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +3 -3
- package/dist/src/context.js +1 -1
- package/dist/src/debug-symbols.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/generic-settle.js +1 -0
- package/dist/src/generic.js +1 -1
- package/dist/src/ime-lifecycle.js +3 -3
- package/dist/src/index.d.ts +27 -7
- package/dist/src/input-actions.js +4 -4
- package/dist/src/interaction-runtime.js +1 -0
- package/dist/src/interaction-snapshot.js +1 -1
- package/dist/src/interaction-touch-reference-frame.js +1 -0
- package/dist/src/interaction.js +1 -1
- package/dist/src/interactor.js +2 -2
- package/dist/src/interactor2.js +1 -1
- package/dist/src/internal/bin.js +2 -2
- package/dist/src/internal/daemon.js +15 -15
- package/dist/src/lease-context.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun.d.ts +23 -2
- package/dist/src/linux.js +1 -1
- package/dist/src/process-exit.js +1 -0
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/register-builtins.js +1 -1
- package/dist/src/registry.js +25 -24
- package/dist/src/remote-config.js +1 -1
- package/dist/src/remote-config2.js +1 -1
- package/dist/src/reporting.js +5 -5
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-lease.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/screenshot-geometry.js +1 -1
- package/dist/src/screenshot-result.js +4 -4
- package/dist/src/sdk-android-adb.d.ts +20 -2
- package/dist/src/sdk-android-adb.js +1 -1
- package/dist/src/sdk-batch-runner.js +3 -3
- package/dist/src/sdk-contracts.d.ts +17 -2
- package/dist/src/sdk-finders.d.ts +4 -20
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/{interaction.d.ts → sdk-selectors.d.ts} +98 -3
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selector-vocabulary.js +1 -1
- package/dist/src/selectors.d.ts +1 -46
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +3 -3
- package/dist/src/session-target-evidence.js +1 -1
- package/dist/src/session.js +9 -9
- package/dist/src/snapshot2.js +1 -1
- package/dist/src/text-surface.js +1 -0
- package/dist/src/viewport-dimension.js +13 -12
- package/dist/src/web.js +1 -1
- package/package.json +41 -35
- package/server.json +3 -3
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
- package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift
ADDED
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
// Text typing, verification, and repair. Kept separate from focus/readiness so each
|
|
4
|
+
// text-entry policy remains a bounded review surface.
|
|
5
|
+
extension RunnerTests {
|
|
6
|
+
func typeTextReliably(
|
|
7
|
+
app: XCUIApplication,
|
|
8
|
+
target: TextEntryTarget,
|
|
9
|
+
text: String,
|
|
10
|
+
delaySeconds: Double,
|
|
11
|
+
repairMode: TextTypingRepairMode = .none,
|
|
12
|
+
xCTestChannelPenalized: Bool = false,
|
|
13
|
+
synthesizer: any TextEntrySynthesizing,
|
|
14
|
+
commandId: String? = nil
|
|
15
|
+
) -> TextEntryResult {
|
|
16
|
+
let totalStartedAt = Date()
|
|
17
|
+
guard !text.isEmpty else {
|
|
18
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
|
|
19
|
+
return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
|
|
20
|
+
}
|
|
21
|
+
var activeTarget = target
|
|
22
|
+
if repairMode == .replacement,
|
|
23
|
+
Self.shouldUseSynthesizedFirstResponderReplacement(
|
|
24
|
+
hasResolvedElement: target.element != nil,
|
|
25
|
+
hasRefreshPoint: target.refreshPoint != nil,
|
|
26
|
+
xCTestChannelPenalized: xCTestChannelPenalized
|
|
27
|
+
)
|
|
28
|
+
{
|
|
29
|
+
switch runSynthesizedReplacementRoute(
|
|
30
|
+
SynthesizedReplacementRequest(
|
|
31
|
+
app: app,
|
|
32
|
+
target: target,
|
|
33
|
+
text: text,
|
|
34
|
+
delaySeconds: delaySeconds,
|
|
35
|
+
synthesizer: synthesizer,
|
|
36
|
+
commandId: commandId,
|
|
37
|
+
startedAt: totalStartedAt
|
|
38
|
+
)
|
|
39
|
+
) {
|
|
40
|
+
case .notApplicable:
|
|
41
|
+
break
|
|
42
|
+
case .completed(let result):
|
|
43
|
+
return result
|
|
44
|
+
case .fallback(let fallbackTarget):
|
|
45
|
+
activeTarget = fallbackTarget
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType(
|
|
49
|
+
repairMode: repairMode,
|
|
50
|
+
fromTapWitness: activeTarget.fromTapWitness,
|
|
51
|
+
softwareKeyboardVisible: isKeyboardVisible(app: app)
|
|
52
|
+
)
|
|
53
|
+
let initialResolveStartedAt = Date()
|
|
54
|
+
let initialTarget = resolveTextEntryElement(app: app, target: activeTarget)
|
|
55
|
+
activeTarget = activeTarget.withElement(initialTarget)
|
|
56
|
+
let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true)
|
|
57
|
+
let initialText = repairMode == .append ? currentText : nil
|
|
58
|
+
let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText)
|
|
59
|
+
var textEntryRoute: String?
|
|
60
|
+
logTextEntryPhase(
|
|
61
|
+
commandId: commandId,
|
|
62
|
+
phase: "initial-resolve",
|
|
63
|
+
startedAt: initialResolveStartedAt,
|
|
64
|
+
chars: text.count,
|
|
65
|
+
mode: repairMode
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
// Dispatching text through XCTest without a resolved target or evidence of a focused
|
|
69
|
+
// responder records a test failure. That tears down the long-lived runner and turns a
|
|
70
|
+
// single invalid request into a restart cascade, so fail before entering that channel.
|
|
71
|
+
guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else {
|
|
72
|
+
logTextEntryPhase(
|
|
73
|
+
commandId: commandId,
|
|
74
|
+
phase: "total",
|
|
75
|
+
startedAt: totalStartedAt,
|
|
76
|
+
chars: text.count,
|
|
77
|
+
mode: repairMode
|
|
78
|
+
)
|
|
79
|
+
return TextEntryResult(
|
|
80
|
+
verified: nil,
|
|
81
|
+
repaired: false,
|
|
82
|
+
expectedText: expectedText,
|
|
83
|
+
observedText: nil,
|
|
84
|
+
failure: .notFocused
|
|
85
|
+
)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if repairMode == .replacement {
|
|
89
|
+
guard let replacementTarget = initialTarget else {
|
|
90
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
|
|
91
|
+
return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
|
|
92
|
+
}
|
|
93
|
+
if currentText == nil || currentText?.isEmpty == false {
|
|
94
|
+
let clearStartedAt = Date()
|
|
95
|
+
clearTextInput(replacementTarget)
|
|
96
|
+
activeTarget = activeTarget.withElement(replacementTarget)
|
|
97
|
+
logTextEntryPhase(
|
|
98
|
+
commandId: commandId,
|
|
99
|
+
phase: "clear",
|
|
100
|
+
startedAt: clearStartedAt,
|
|
101
|
+
chars: text.count,
|
|
102
|
+
mode: repairMode
|
|
103
|
+
)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) {
|
|
108
|
+
#if os(iOS)
|
|
109
|
+
if shouldUseSynthesizedFirstResponderType,
|
|
110
|
+
let currentTarget = resolveTextEntryElement(app: app, target: activeTarget)
|
|
111
|
+
{
|
|
112
|
+
textEntryRoute = "synthesized-first-responder"
|
|
113
|
+
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
|
|
114
|
+
switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
|
|
115
|
+
case .continueTyping:
|
|
116
|
+
return (currentTarget, true, nil)
|
|
117
|
+
case .fallback:
|
|
118
|
+
return (nil, false, .synthesisUnavailable)
|
|
119
|
+
case .raise(let message):
|
|
120
|
+
NSException(
|
|
121
|
+
name: NSExceptionName.internalInconsistencyException,
|
|
122
|
+
reason: message ?? "private XCTest text synthesis failed"
|
|
123
|
+
).raise()
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
#endif
|
|
127
|
+
if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
|
|
128
|
+
textEntryRoute = "xctest-element"
|
|
129
|
+
currentTarget.typeText(value)
|
|
130
|
+
return (currentTarget, true, nil)
|
|
131
|
+
} else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
|
|
132
|
+
#if os(iOS)
|
|
133
|
+
textEntryRoute = "synthesized-first-responder"
|
|
134
|
+
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
|
|
135
|
+
let action = synthesizer.enterText(
|
|
136
|
+
app: app,
|
|
137
|
+
text: value,
|
|
138
|
+
replacingExistingText: false
|
|
139
|
+
)
|
|
140
|
+
switch action {
|
|
141
|
+
case .fallback:
|
|
142
|
+
textEntryRoute = "xctest-application-fallback"
|
|
143
|
+
app.typeText(value)
|
|
144
|
+
case .raise(let message):
|
|
145
|
+
NSException(
|
|
146
|
+
name: NSExceptionName.internalInconsistencyException,
|
|
147
|
+
reason: message ?? "private XCTest text synthesis failed"
|
|
148
|
+
).raise()
|
|
149
|
+
case .continueTyping:
|
|
150
|
+
break
|
|
151
|
+
}
|
|
152
|
+
#else
|
|
153
|
+
app.typeText(value)
|
|
154
|
+
#endif
|
|
155
|
+
return (resolveTextEntryElement(app: app, target: activeTarget), true, nil)
|
|
156
|
+
}
|
|
157
|
+
return (nil, false, .notFocused)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
func dispatchFailureResult(_ failure: TextEntryFailure) -> TextEntryResult {
|
|
161
|
+
TextEntryResult(
|
|
162
|
+
verified: nil,
|
|
163
|
+
repaired: false,
|
|
164
|
+
expectedText: expectedText,
|
|
165
|
+
observedText: nil,
|
|
166
|
+
textEntryRoute: textEntryRoute,
|
|
167
|
+
failure: failure
|
|
168
|
+
)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) {
|
|
172
|
+
guard let expectedValue else {
|
|
173
|
+
sleepFor(TextEntryTiming.pollInterval)
|
|
174
|
+
return
|
|
175
|
+
}
|
|
176
|
+
let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
|
|
177
|
+
while Date() < deadline {
|
|
178
|
+
if editableTextValue(for: resolveTextEntryElement(app: app, target: target)) == expectedValue {
|
|
179
|
+
return
|
|
180
|
+
}
|
|
181
|
+
sleepFor(TextEntryTiming.pollInterval)
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
let characters = Array(text)
|
|
186
|
+
if delaySeconds > 0 && characters.count > 1 {
|
|
187
|
+
var typedTarget: XCUIElement?
|
|
188
|
+
let delayedTypeStartedAt = Date()
|
|
189
|
+
for (index, character) in characters.enumerated() {
|
|
190
|
+
let dispatch = typeIntoCurrentTarget(String(character))
|
|
191
|
+
guard dispatch.dispatched else {
|
|
192
|
+
return dispatchFailureResult(dispatch.failure ?? .notFocused)
|
|
193
|
+
}
|
|
194
|
+
typedTarget = dispatch.element ?? typedTarget
|
|
195
|
+
if index + 1 < characters.count {
|
|
196
|
+
sleepFor(delaySeconds)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
logTextEntryPhase(
|
|
200
|
+
commandId: commandId,
|
|
201
|
+
phase: "type-delayed",
|
|
202
|
+
startedAt: delayedTypeStartedAt,
|
|
203
|
+
chars: characters.count,
|
|
204
|
+
mode: repairMode
|
|
205
|
+
)
|
|
206
|
+
if repairMode == .none {
|
|
207
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
|
|
208
|
+
return TextEntryResult(
|
|
209
|
+
verified: nil,
|
|
210
|
+
repaired: false,
|
|
211
|
+
expectedText: nil,
|
|
212
|
+
observedText: nil,
|
|
213
|
+
textEntryRoute: textEntryRoute
|
|
214
|
+
)
|
|
215
|
+
}
|
|
216
|
+
let verifyStartedAt = Date()
|
|
217
|
+
var result = verifyTextEntryWithRepairIfNeeded(
|
|
218
|
+
app: app,
|
|
219
|
+
target: activeTarget.withElement(typedTarget),
|
|
220
|
+
expectedText: expectedText,
|
|
221
|
+
repairMode: repairMode
|
|
222
|
+
)
|
|
223
|
+
logTextEntryPhase(
|
|
224
|
+
commandId: commandId,
|
|
225
|
+
phase: "verify",
|
|
226
|
+
startedAt: verifyStartedAt,
|
|
227
|
+
chars: characters.count,
|
|
228
|
+
mode: repairMode
|
|
229
|
+
)
|
|
230
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
|
|
231
|
+
result.textEntryRoute = textEntryRoute
|
|
232
|
+
return result
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
let typedTarget: XCUIElement?
|
|
236
|
+
if repairMode != .none && characters.count > 1 {
|
|
237
|
+
let firstCharacter = String(characters[0])
|
|
238
|
+
let firstStartedAt = Date()
|
|
239
|
+
let firstDispatch = typeIntoCurrentTarget(firstCharacter)
|
|
240
|
+
guard firstDispatch.dispatched else {
|
|
241
|
+
return dispatchFailureResult(firstDispatch.failure ?? .notFocused)
|
|
242
|
+
}
|
|
243
|
+
var firstTypedTarget = firstDispatch.element
|
|
244
|
+
logTextEntryPhase(
|
|
245
|
+
commandId: commandId,
|
|
246
|
+
phase: "type-first",
|
|
247
|
+
startedAt: firstStartedAt,
|
|
248
|
+
chars: 1,
|
|
249
|
+
mode: repairMode
|
|
250
|
+
)
|
|
251
|
+
activeTarget = activeTarget.withElement(firstTypedTarget)
|
|
252
|
+
let warmupExpectedText = expectedTextEntryValue(
|
|
253
|
+
typedText: firstCharacter,
|
|
254
|
+
mode: repairMode,
|
|
255
|
+
initialText: initialText
|
|
256
|
+
)
|
|
257
|
+
let warmupStartedAt = Date()
|
|
258
|
+
waitForWarmupValue(warmupExpectedText, target: activeTarget)
|
|
259
|
+
logTextEntryPhase(
|
|
260
|
+
commandId: commandId,
|
|
261
|
+
phase: "warmup",
|
|
262
|
+
startedAt: warmupStartedAt,
|
|
263
|
+
chars: 1,
|
|
264
|
+
mode: repairMode
|
|
265
|
+
)
|
|
266
|
+
let remainingText = String(characters.dropFirst())
|
|
267
|
+
let remainingStartedAt = Date()
|
|
268
|
+
let remainingDispatch = typeIntoCurrentTarget(remainingText)
|
|
269
|
+
guard remainingDispatch.dispatched else {
|
|
270
|
+
return dispatchFailureResult(remainingDispatch.failure ?? .notFocused)
|
|
271
|
+
}
|
|
272
|
+
firstTypedTarget = remainingDispatch.element ?? firstTypedTarget
|
|
273
|
+
logTextEntryPhase(
|
|
274
|
+
commandId: commandId,
|
|
275
|
+
phase: "type-remaining",
|
|
276
|
+
startedAt: remainingStartedAt,
|
|
277
|
+
chars: characters.count - 1,
|
|
278
|
+
mode: repairMode
|
|
279
|
+
)
|
|
280
|
+
typedTarget = firstTypedTarget
|
|
281
|
+
} else {
|
|
282
|
+
let typeStartedAt = Date()
|
|
283
|
+
let dispatch = typeIntoCurrentTarget(text)
|
|
284
|
+
guard dispatch.dispatched else {
|
|
285
|
+
return dispatchFailureResult(dispatch.failure ?? .notFocused)
|
|
286
|
+
}
|
|
287
|
+
typedTarget = dispatch.element
|
|
288
|
+
logTextEntryPhase(
|
|
289
|
+
commandId: commandId,
|
|
290
|
+
phase: "type-all",
|
|
291
|
+
startedAt: typeStartedAt,
|
|
292
|
+
chars: characters.count,
|
|
293
|
+
mode: repairMode
|
|
294
|
+
)
|
|
295
|
+
}
|
|
296
|
+
if repairMode == .none {
|
|
297
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
|
|
298
|
+
return TextEntryResult(
|
|
299
|
+
verified: nil,
|
|
300
|
+
repaired: false,
|
|
301
|
+
expectedText: nil,
|
|
302
|
+
observedText: nil,
|
|
303
|
+
textEntryRoute: textEntryRoute
|
|
304
|
+
)
|
|
305
|
+
}
|
|
306
|
+
let verifyStartedAt = Date()
|
|
307
|
+
var result = verifyTextEntryWithRepairIfNeeded(
|
|
308
|
+
app: app,
|
|
309
|
+
target: activeTarget.withElement(typedTarget),
|
|
310
|
+
expectedText: expectedText,
|
|
311
|
+
repairMode: repairMode
|
|
312
|
+
)
|
|
313
|
+
logTextEntryPhase(
|
|
314
|
+
commandId: commandId,
|
|
315
|
+
phase: "verify",
|
|
316
|
+
startedAt: verifyStartedAt,
|
|
317
|
+
chars: characters.count,
|
|
318
|
+
mode: repairMode
|
|
319
|
+
)
|
|
320
|
+
logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
|
|
321
|
+
result.textEntryRoute = textEntryRoute
|
|
322
|
+
return result
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
func logTextEntryPhase(
|
|
326
|
+
commandId: String?,
|
|
327
|
+
phase: String,
|
|
328
|
+
startedAt: Date,
|
|
329
|
+
chars: Int,
|
|
330
|
+
mode: TextTypingRepairMode
|
|
331
|
+
) {
|
|
332
|
+
NSLog(
|
|
333
|
+
"AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=%@ durationMs=%.1f chars=%d mode=%@",
|
|
334
|
+
commandId ?? "",
|
|
335
|
+
phase,
|
|
336
|
+
Date().timeIntervalSince(startedAt) * 1000.0,
|
|
337
|
+
chars,
|
|
338
|
+
textEntryModeName(mode)
|
|
339
|
+
)
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
func textEntryModeName(_ mode: TextTypingRepairMode) -> String {
|
|
343
|
+
switch mode {
|
|
344
|
+
case .none:
|
|
345
|
+
return "none"
|
|
346
|
+
case .append:
|
|
347
|
+
return "append"
|
|
348
|
+
case .replacement:
|
|
349
|
+
return "replacement"
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
private func verifyTextEntryWithRepairIfNeeded(
|
|
354
|
+
app: XCUIApplication,
|
|
355
|
+
target: TextEntryTarget,
|
|
356
|
+
expectedText: String?,
|
|
357
|
+
repairMode: TextTypingRepairMode
|
|
358
|
+
) -> TextEntryResult {
|
|
359
|
+
let initialResult = verifyTextEntry(
|
|
360
|
+
app: app,
|
|
361
|
+
target: target,
|
|
362
|
+
expectedText: expectedText,
|
|
363
|
+
repaired: false
|
|
364
|
+
)
|
|
365
|
+
#if os(iOS)
|
|
366
|
+
guard initialResult.verified == false,
|
|
367
|
+
let expectedText = initialResult.expectedText
|
|
368
|
+
else {
|
|
369
|
+
return initialResult
|
|
370
|
+
}
|
|
371
|
+
guard shouldRepairTextEntry(
|
|
372
|
+
app: app,
|
|
373
|
+
target: target,
|
|
374
|
+
expectedText: expectedText,
|
|
375
|
+
repairMode: repairMode
|
|
376
|
+
) else {
|
|
377
|
+
return verifyTextEntry(
|
|
378
|
+
app: app,
|
|
379
|
+
target: target,
|
|
380
|
+
expectedText: expectedText,
|
|
381
|
+
repaired: false
|
|
382
|
+
)
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
guard let repairTarget = resolveTextEntryElement(app: app, target: target) else {
|
|
386
|
+
return initialResult
|
|
387
|
+
}
|
|
388
|
+
let observedText = editableTextValue(for: repairTarget) ?? ""
|
|
389
|
+
NSLog(
|
|
390
|
+
"AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY expectedLength=%d observedLength=%d",
|
|
391
|
+
expectedText.count,
|
|
392
|
+
observedText.count
|
|
393
|
+
)
|
|
394
|
+
clearTextInput(repairTarget)
|
|
395
|
+
repairTarget.typeText(expectedText)
|
|
396
|
+
return verifyTextEntry(
|
|
397
|
+
app: app,
|
|
398
|
+
target: target.withElement(repairTarget),
|
|
399
|
+
expectedText: expectedText,
|
|
400
|
+
repaired: true
|
|
401
|
+
)
|
|
402
|
+
#else
|
|
403
|
+
return initialResult
|
|
404
|
+
#endif
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
private func verifyTextEntry(
|
|
408
|
+
app: XCUIApplication,
|
|
409
|
+
target: TextEntryTarget,
|
|
410
|
+
expectedText: String?,
|
|
411
|
+
repaired: Bool
|
|
412
|
+
) -> TextEntryResult {
|
|
413
|
+
let targetElement = resolveTextEntryElement(app: app, target: target)
|
|
414
|
+
guard let expectedText else {
|
|
415
|
+
return TextEntryResult(
|
|
416
|
+
verified: nil,
|
|
417
|
+
repaired: repaired,
|
|
418
|
+
expectedText: nil,
|
|
419
|
+
observedText: editableTextValue(for: targetElement)
|
|
420
|
+
)
|
|
421
|
+
}
|
|
422
|
+
guard let observedText = editableTextValue(for: targetElement) else {
|
|
423
|
+
return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
|
|
424
|
+
}
|
|
425
|
+
guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else {
|
|
426
|
+
return TextEntryResult(
|
|
427
|
+
verified: false,
|
|
428
|
+
repaired: repaired,
|
|
429
|
+
expectedText: expectedText,
|
|
430
|
+
observedText: observedText
|
|
431
|
+
)
|
|
432
|
+
}
|
|
433
|
+
let stableDeadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
|
|
434
|
+
var latestObservedText = observedText
|
|
435
|
+
while Date() < stableDeadline {
|
|
436
|
+
sleepFor(TextEntryTiming.pollInterval)
|
|
437
|
+
guard let nextObservedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
|
|
438
|
+
return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
|
|
439
|
+
}
|
|
440
|
+
latestObservedText = nextObservedText
|
|
441
|
+
guard textEntryValueMatchesExpected(
|
|
442
|
+
resolveTextEntryElement(app: app, target: target),
|
|
443
|
+
observedText: nextObservedText,
|
|
444
|
+
expectedText: expectedText
|
|
445
|
+
) else {
|
|
446
|
+
return TextEntryResult(
|
|
447
|
+
verified: false,
|
|
448
|
+
repaired: repaired,
|
|
449
|
+
expectedText: expectedText,
|
|
450
|
+
observedText: nextObservedText
|
|
451
|
+
)
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
return TextEntryResult(
|
|
455
|
+
verified: true,
|
|
456
|
+
repaired: repaired,
|
|
457
|
+
expectedText: expectedText,
|
|
458
|
+
observedText: latestObservedText
|
|
459
|
+
)
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
private func textEntryValueMatchesExpected(
|
|
463
|
+
_ element: XCUIElement?,
|
|
464
|
+
observedText: String,
|
|
465
|
+
expectedText: String
|
|
466
|
+
) -> Bool {
|
|
467
|
+
if observedText == expectedText {
|
|
468
|
+
return true
|
|
469
|
+
}
|
|
470
|
+
guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else {
|
|
471
|
+
return false
|
|
472
|
+
}
|
|
473
|
+
var submittedText = expectedText
|
|
474
|
+
while hasTextEntrySubmitSuffix(submittedText) {
|
|
475
|
+
submittedText.removeLast()
|
|
476
|
+
}
|
|
477
|
+
return observedText == submittedText
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
private func hasTextEntrySubmitSuffix(_ text: String) -> Bool {
|
|
481
|
+
text.hasSuffix("\n") || text.hasSuffix("\r")
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
private func expectedTextEntryValue(
|
|
485
|
+
typedText: String,
|
|
486
|
+
mode: TextTypingRepairMode,
|
|
487
|
+
initialText: String?
|
|
488
|
+
) -> String? {
|
|
489
|
+
switch mode {
|
|
490
|
+
case .none:
|
|
491
|
+
return nil
|
|
492
|
+
case .append:
|
|
493
|
+
guard let initialText else {
|
|
494
|
+
return nil
|
|
495
|
+
}
|
|
496
|
+
return initialText + typedText
|
|
497
|
+
case .replacement:
|
|
498
|
+
return typedText
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
private func shouldRepairTextEntry(
|
|
503
|
+
app: XCUIApplication,
|
|
504
|
+
target: TextEntryTarget,
|
|
505
|
+
expectedText: String,
|
|
506
|
+
repairMode: TextTypingRepairMode
|
|
507
|
+
) -> Bool {
|
|
508
|
+
#if os(iOS)
|
|
509
|
+
var latestObservedText: String?
|
|
510
|
+
let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
|
|
511
|
+
repeat {
|
|
512
|
+
guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
|
|
513
|
+
return false
|
|
514
|
+
}
|
|
515
|
+
if textEntryValueMatchesExpected(
|
|
516
|
+
resolveTextEntryElement(app: app, target: target),
|
|
517
|
+
observedText: observedText,
|
|
518
|
+
expectedText: expectedText
|
|
519
|
+
) {
|
|
520
|
+
return false
|
|
521
|
+
}
|
|
522
|
+
latestObservedText = observedText
|
|
523
|
+
if !isRepairableTextEntryMismatch(
|
|
524
|
+
observedText: observedText,
|
|
525
|
+
expectedText: expectedText,
|
|
526
|
+
repairMode: repairMode
|
|
527
|
+
) {
|
|
528
|
+
return false
|
|
529
|
+
}
|
|
530
|
+
sleepFor(TextEntryTiming.pollInterval)
|
|
531
|
+
} while Date() < deadline
|
|
532
|
+
|
|
533
|
+
guard let latestObservedText else {
|
|
534
|
+
return false
|
|
535
|
+
}
|
|
536
|
+
guard !textEntryValueMatchesExpected(
|
|
537
|
+
resolveTextEntryElement(app: app, target: target),
|
|
538
|
+
observedText: latestObservedText,
|
|
539
|
+
expectedText: expectedText
|
|
540
|
+
) else {
|
|
541
|
+
return false
|
|
542
|
+
}
|
|
543
|
+
return isRepairableTextEntryMismatch(
|
|
544
|
+
observedText: latestObservedText,
|
|
545
|
+
expectedText: expectedText,
|
|
546
|
+
repairMode: repairMode
|
|
547
|
+
)
|
|
548
|
+
#else
|
|
549
|
+
return false
|
|
550
|
+
#endif
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
private func isRepairableTextEntryMismatch(
|
|
554
|
+
observedText: String,
|
|
555
|
+
expectedText: String,
|
|
556
|
+
repairMode: TextTypingRepairMode
|
|
557
|
+
) -> Bool {
|
|
558
|
+
guard observedText != expectedText else {
|
|
559
|
+
return false
|
|
560
|
+
}
|
|
561
|
+
if repairMode == .replacement {
|
|
562
|
+
return true
|
|
563
|
+
}
|
|
564
|
+
return observedText.isEmpty || isLikelyDroppedCharacterTextEntryMismatch(
|
|
565
|
+
observedText: observedText,
|
|
566
|
+
expectedText: expectedText
|
|
567
|
+
)
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
private func isLikelyDroppedCharacterTextEntryMismatch(observedText: String, expectedText: String) -> Bool {
|
|
571
|
+
guard observedText.count < expectedText.count else {
|
|
572
|
+
return false
|
|
573
|
+
}
|
|
574
|
+
let missingCharacterCount = expectedText.count - observedText.count
|
|
575
|
+
guard missingCharacterCount <= max(2, expectedText.count / 4) else {
|
|
576
|
+
return false
|
|
577
|
+
}
|
|
578
|
+
var expectedIndex = expectedText.startIndex
|
|
579
|
+
for character in observedText {
|
|
580
|
+
guard let matchIndex = expectedText[expectedIndex...].firstIndex(of: character) else {
|
|
581
|
+
return false
|
|
582
|
+
}
|
|
583
|
+
expectedIndex = expectedText.index(after: matchIndex)
|
|
584
|
+
}
|
|
585
|
+
return true
|
|
586
|
+
}
|
|
587
|
+
}
|
|
@@ -48,6 +48,11 @@ final class RunnerTests: XCTestCase {
|
|
|
48
48
|
var currentApp: XCUIApplication?
|
|
49
49
|
var currentBundleId: String?
|
|
50
50
|
var currentAppProcessIdentifier: Int?
|
|
51
|
+
// iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when
|
|
52
|
+
// hardware-keyboard input hides the software keyboard. A successful tap on a concrete text
|
|
53
|
+
// input is a scoped witness for the immediately-following bare type; lifecycle and non-text
|
|
54
|
+
// interactions clear it before it can become stale.
|
|
55
|
+
var textEntryTapWitness: TextEntryTapWitness?
|
|
51
56
|
let maxRequestBytes = 2 * 1024 * 1024
|
|
52
57
|
let mainThreadExecutionTimeout: TimeInterval = 30
|
|
53
58
|
let appExistenceTimeout: TimeInterval = 30
|
|
@@ -87,6 +92,19 @@ final class RunnerTests: XCTestCase {
|
|
|
87
92
|
var snapshotXCTestChannelPenaltyUntil = Date.distantPast
|
|
88
93
|
let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
|
|
89
94
|
var snapshotXCTestPenaltyWarmupExemptionPending = false
|
|
95
|
+
// Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default
|
|
96
|
+
// depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted
|
|
97
|
+
// later captures start there instead of re-paying the rejected deep request (~300ms per
|
|
98
|
+
// capture on the Bluesky feed). Shares the penalty's lifetime model: same duration, cleared
|
|
99
|
+
// on target process change, so screens that regain deep-capture ability are re-probed.
|
|
100
|
+
let privateAXAcceptedDepthLock = NSLock()
|
|
101
|
+
var privateAXAcceptedDepthBundleId: String?
|
|
102
|
+
// PID-bound: a relaunch (external or A->B->A while inactive) changes the process, and the new
|
|
103
|
+
// tree may accept the full depth again. Any invalidation path that drops the cached PID makes
|
|
104
|
+
// the memory unmatchable, so every fresh activation re-probes the full requested depth.
|
|
105
|
+
var privateAXAcceptedDepthProcessIdentifier: Int?
|
|
106
|
+
var privateAXAcceptedDepth: Int?
|
|
107
|
+
var privateAXAcceptedDepthUntil = Date.distantPast
|
|
90
108
|
// Bluesky-class screens can grind ~4-8s before an XCTest-backed snapshot tier fails; anything
|
|
91
109
|
// past this threshold marks the screen hostile so the next capture uses non-XCTest recovery.
|
|
92
110
|
let snapshotXCTestSlowCaptureThreshold: TimeInterval = 3
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#import <Foundation/Foundation.h>
|
|
2
|
+
|
|
3
|
+
NS_ASSUME_NONNULL_BEGIN
|
|
4
|
+
|
|
5
|
+
// Shared reflection surface for XCTest's private event-synthesis API
|
|
6
|
+
// (XCSynthesizedEventRecord / XCPointerEventPath). Both gesture synthesis
|
|
7
|
+
// (RunnerSynthesizedGesture) and text-entry synthesis (RunnerSynthesizedTextEntry)
|
|
8
|
+
// reflect into this same private API; resolving the part they have in common here
|
|
9
|
+
// means a selector rename only has to be found in one place.
|
|
10
|
+
//
|
|
11
|
+
// Each caller additionally resolves whichever XCSynthesizedEventRecord init
|
|
12
|
+
// overload and XCPointerEventPath factory/mutator selectors its own event kind
|
|
13
|
+
// needs: gesture uses the 2-arg `initWithName:interfaceOrientation:` plus
|
|
14
|
+
// touch-path selectors (`initForTouchAtPoint:offset:`, `moveToPoint:atOffset:`,
|
|
15
|
+
// `liftUpAtOffset:`); text entry uses the 1-arg `initWithName:` plus text-input
|
|
16
|
+
// selectors (`initForTextInput`, `typeText:atOffset:typingSpeed:shouldRedact:`,
|
|
17
|
+
// `typeKey:modifiers:atOffset:`). Those genuinely differ and are resolved by
|
|
18
|
+
// each caller on top of this shared core, not forced into one shape here.
|
|
19
|
+
typedef struct {
|
|
20
|
+
Class recordClass; // XCSynthesizedEventRecord
|
|
21
|
+
Class pathClass; // XCPointerEventPath
|
|
22
|
+
SEL addPathSelector; // addPointerEventPath: (on recordClass)
|
|
23
|
+
SEL setTargetProcessIDSelector; // setTargetProcessID: (on recordClass)
|
|
24
|
+
SEL synthesizeSelector; // synthesizeWithError: (on recordClass)
|
|
25
|
+
SEL processIDSelector; // processID, read on the XCUIApplication
|
|
26
|
+
} RunnerXCTestEventBridge;
|
|
27
|
+
|
|
28
|
+
typedef NSInteger (*RunnerMsgSendInteger)(id, SEL);
|
|
29
|
+
typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger);
|
|
30
|
+
typedef void (*RunnerMsgSendAddPath)(id, SEL, id);
|
|
31
|
+
typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
|
|
32
|
+
|
|
33
|
+
// Resolves the shared core described above against the live XCTest runtime.
|
|
34
|
+
// Returns nil and populates `bridge` on success. Returns a non-nil
|
|
35
|
+
// "private XCTest <surface> synthesis unavailable: ..." message and leaves
|
|
36
|
+
// `bridge` unspecified when a class or selector the private API is expected to
|
|
37
|
+
// expose is missing (the API changed, or is unavailable on this OS/Xcode).
|
|
38
|
+
// `surface` names the calling feature ("event" or "text") so the message text
|
|
39
|
+
// matches what that caller has always reported.
|
|
40
|
+
FOUNDATION_EXPORT NSString * _Nullable RunnerResolveXCTestEventBridge(
|
|
41
|
+
id application,
|
|
42
|
+
NSString *surface,
|
|
43
|
+
RunnerXCTestEventBridge *bridge
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass(
|
|
47
|
+
Class cls,
|
|
48
|
+
NSString *className,
|
|
49
|
+
NSString *surface
|
|
50
|
+
);
|
|
51
|
+
FOUNDATION_EXPORT NSString * _Nullable RunnerRequireSelector(
|
|
52
|
+
Class cls,
|
|
53
|
+
SEL selector,
|
|
54
|
+
NSString *selectorName,
|
|
55
|
+
NSString *surface
|
|
56
|
+
);
|
|
57
|
+
FOUNDATION_EXPORT NSString * _Nullable RunnerRequireApplicationSelector(
|
|
58
|
+
id application,
|
|
59
|
+
SEL selector,
|
|
60
|
+
NSString *selectorName,
|
|
61
|
+
NSString *surface
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
// Formats an NSException the way every synthesis @catch block reports failure:
|
|
65
|
+
// "<name>: <reason>", falling back to "NSException" / `fallbackReason` when
|
|
66
|
+
// either is nil.
|
|
67
|
+
FOUNDATION_EXPORT NSString *RunnerFormatXCTestException(
|
|
68
|
+
NSException *exception,
|
|
69
|
+
NSString *fallbackReason
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
NS_ASSUME_NONNULL_END
|