agent-device 0.21.32 → 0.21.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.32.apk → agent-device-android-ime-helper-0.21.33.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.33.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.32.manifest.json → agent-device-android-ime-helper-0.21.33.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.32.apk → agent-device-android-snapshot-helper-0.21.33.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.33.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.32.manifest.json → agent-device-android-snapshot-helper-0.21.33.manifest.json} +7 -7
- package/apple/fold-helper/Fold.m +32 -11
- package/apple/snapshot-bridge/ForegroundApplicationGeometry.h +4 -16
- package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +1 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +36 -58
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +66 -111
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +1 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +2 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +5 -29
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +4 -17
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +66 -117
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +4 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +21 -75
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +7 -61
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +1 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +24 -53
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +13 -39
- package/dist/src/app-backend.js +1 -1
- package/dist/src/app-launch.js +1 -1
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/config.js +1 -1
- package/dist/src/device-readiness-runtime.js +1 -1
- package/dist/src/display-inventory.js +1 -1
- package/dist/src/fold-helper.js +2 -2
- package/dist/src/fold-runtime.js +1 -0
- package/dist/src/interactor.js +5 -5
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/mechanics.js +12 -12
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/runtime3.js +5 -5
- package/dist/src/simulator.js +1 -1
- package/package.json +7 -2
- package/server.json +2 -2
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.32.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.32.apk.sha256 +0 -1
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift
CHANGED
|
@@ -18,10 +18,11 @@ extension RunnerTests {
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
|
|
21
22
|
enum NavigationVisualObservation: Equatable {
|
|
22
23
|
case changed
|
|
23
24
|
case unchanged
|
|
24
|
-
case unobserved
|
|
25
|
+
case unobserved(refusal: RunnerAppScreenCaptureFailure)
|
|
25
26
|
|
|
26
27
|
var logToken: String {
|
|
27
28
|
switch self {
|
|
@@ -32,20 +33,6 @@ extension RunnerTests {
|
|
|
32
33
|
}
|
|
33
34
|
}
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
struct NavigationVisualSample {
|
|
38
|
-
let data: Data?
|
|
39
|
-
let refusalCode: String?
|
|
40
|
-
let refusalHint: String?
|
|
41
|
-
|
|
42
|
-
init(data: Data?, refusalCode: String? = nil, refusalHint: String? = nil) {
|
|
43
|
-
self.data = data
|
|
44
|
-
self.refusalCode = refusalCode
|
|
45
|
-
self.refusalHint = refusalHint
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
36
|
@MainActor
|
|
50
37
|
func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome {
|
|
51
38
|
#if os(macOS)
|
|
@@ -94,23 +81,34 @@ extension RunnerTests {
|
|
|
94
81
|
@MainActor
|
|
95
82
|
private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) -> InAppBackOutcome {
|
|
96
83
|
guard SnapshotGeometry.isPositiveFinite(frame) else { return .unavailable }
|
|
97
|
-
let context = synthesizedCoordinateContext(
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
84
|
+
let context = synthesizedCoordinateContext(app: app)
|
|
85
|
+
guard dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else {
|
|
86
|
+
return .unavailable
|
|
87
|
+
}
|
|
88
|
+
return .performed
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@MainActor
|
|
94
|
+
private func dispatchNavigationTap(
|
|
95
|
+
app: XCUIApplication,
|
|
96
|
+
point: CGPoint,
|
|
97
|
+
context: SynthesizedCoordinateContext?
|
|
98
|
+
) -> Bool {
|
|
101
99
|
switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: {
|
|
102
|
-
synthesizedTapAt(app: app, x:
|
|
100
|
+
synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
|
|
103
101
|
}) {
|
|
104
102
|
case .performed:
|
|
105
|
-
return
|
|
103
|
+
return true
|
|
106
104
|
case .refused:
|
|
107
|
-
return
|
|
105
|
+
return false
|
|
108
106
|
case .xctestFallback:
|
|
109
107
|
let fallback = performGesture(app) {
|
|
110
|
-
tapAt(app: app, x:
|
|
108
|
+
tapAt(app: app, x: point.x, y: point.y)
|
|
111
109
|
}
|
|
112
|
-
if case .performed = fallback.outcome { return
|
|
113
|
-
return
|
|
110
|
+
if case .performed = fallback.outcome { return true }
|
|
111
|
+
return false
|
|
114
112
|
}
|
|
115
113
|
}
|
|
116
114
|
|
|
@@ -225,7 +223,6 @@ extension RunnerTests {
|
|
|
225
223
|
#if os(iOS)
|
|
226
224
|
guard let context = synthesizedCoordinateContext(
|
|
227
225
|
app: app,
|
|
228
|
-
policy: synthesizedGesturePolicy(.coordinateTap),
|
|
229
226
|
windowQueryRejected: windowQueryRejected
|
|
230
227
|
), let point = Self.topLeadingNavigationFallbackPoint(
|
|
231
228
|
in: context.referenceFrame,
|
|
@@ -234,23 +231,13 @@ extension RunnerTests {
|
|
|
234
231
|
return .unavailable
|
|
235
232
|
}
|
|
236
233
|
let before = captureNavigationFallbackVisualState(app: app, context: context)
|
|
237
|
-
|
|
238
|
-
synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
|
|
239
|
-
}) {
|
|
240
|
-
case .performed:
|
|
241
|
-
return verifyNavigationFallbackOutcome(app: app, before: before, context: context)
|
|
242
|
-
case .refused:
|
|
234
|
+
guard dispatchNavigationTap(app: app, point: point, context: context) else {
|
|
243
235
|
return .unavailable
|
|
244
|
-
case .xctestFallback:
|
|
245
|
-
let fallback = performGesture(app) {
|
|
246
|
-
tapAt(app: app, x: point.x, y: point.y)
|
|
247
|
-
}
|
|
248
|
-
if case .performed = fallback.outcome {
|
|
249
|
-
return verifyNavigationFallbackOutcome(app: app, before: before, context: context)
|
|
250
|
-
}
|
|
251
236
|
}
|
|
252
|
-
|
|
237
|
+
return verifyNavigationFallbackOutcome(app: app, before: before, context: context)
|
|
238
|
+
#else
|
|
253
239
|
return .unavailable
|
|
240
|
+
#endif
|
|
254
241
|
}
|
|
255
242
|
|
|
256
243
|
private func navigationRejectedQueryCount() -> Int {
|
|
@@ -259,131 +246,93 @@ extension RunnerTests {
|
|
|
259
246
|
return suppressedAxSnapshotIssueCount
|
|
260
247
|
}
|
|
261
248
|
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
|
|
262
252
|
private func captureNavigationFallbackVisualState(
|
|
263
253
|
app: XCUIApplication,
|
|
264
254
|
context: SynthesizedCoordinateContext
|
|
265
|
-
) ->
|
|
255
|
+
) -> Result<Data, RunnerAppScreenCaptureFailure> {
|
|
266
256
|
#if os(iOS)
|
|
267
|
-
return Self.
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
: self.captureResolvedAppScreen(app: app)
|
|
272
|
-
},
|
|
273
|
-
systemSurface: { self.captureResolvedAppScreen(app: self.springboard) },
|
|
257
|
+
return Self.navigationVisualSample(
|
|
258
|
+
from: context.resolvedWindow == nil
|
|
259
|
+
? captureSoleAppScreen(app: app)
|
|
260
|
+
: captureResolvedAppScreen(app: app),
|
|
274
261
|
encoding: { runnerPngData(for: $0.image) }
|
|
275
262
|
)
|
|
276
263
|
#else
|
|
277
|
-
return
|
|
264
|
+
return .failure(.unresolvedScreen)
|
|
278
265
|
#endif
|
|
279
266
|
}
|
|
280
267
|
|
|
281
268
|
|
|
282
269
|
|
|
283
270
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
static func navigationFallbackSample(
|
|
289
|
-
resolvingApp: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
|
|
290
|
-
systemSurface: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
|
|
291
|
-
encoding: (CapturedAppScreen) -> Data?
|
|
292
|
-
) -> NavigationVisualSample {
|
|
293
|
-
_ = systemSurface
|
|
294
|
-
return navigationVisualSample(from: resolvingApp(), encoding: encoding)
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
271
|
static func navigationVisualSample(
|
|
302
|
-
from
|
|
272
|
+
from captured: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
|
|
303
273
|
encoding: (CapturedAppScreen) -> Data?
|
|
304
|
-
) ->
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
let refusal = RunnerAppScreenCaptureFailure.unrenderableImage
|
|
309
|
-
return NavigationVisualSample(
|
|
310
|
-
data: nil,
|
|
311
|
-
refusalCode: refusal.rawValue,
|
|
312
|
-
refusalHint: refusal.hint
|
|
313
|
-
)
|
|
314
|
-
}
|
|
315
|
-
return NavigationVisualSample(data: png)
|
|
316
|
-
case .failure(let failure):
|
|
317
|
-
return NavigationVisualSample(
|
|
318
|
-
data: nil,
|
|
319
|
-
refusalCode: failure.rawValue,
|
|
320
|
-
refusalHint: failure.hint
|
|
321
|
-
)
|
|
274
|
+
) -> Result<Data, RunnerAppScreenCaptureFailure> {
|
|
275
|
+
captured.flatMap { screen in
|
|
276
|
+
guard let png = encoding(screen) else { return .failure(.unrenderableImage) }
|
|
277
|
+
return .success(png)
|
|
322
278
|
}
|
|
323
279
|
}
|
|
324
280
|
|
|
325
281
|
private func verifyNavigationFallbackOutcome(
|
|
326
282
|
app: XCUIApplication,
|
|
327
|
-
before:
|
|
283
|
+
before: Result<Data, RunnerAppScreenCaptureFailure>,
|
|
328
284
|
context: SynthesizedCoordinateContext
|
|
329
285
|
) -> InAppBackOutcome {
|
|
330
286
|
sleepFor(Self.navigationFallbackVerificationDelay)
|
|
331
287
|
let after = captureNavigationFallbackVisualState(app: app, context: context)
|
|
332
|
-
let observation = Self.navigationVisualObservation(before: before
|
|
288
|
+
let observation = Self.navigationVisualObservation(before: before, after: after)
|
|
333
289
|
|
|
334
290
|
|
|
335
291
|
NSLog(
|
|
336
292
|
"AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@",
|
|
337
|
-
before.
|
|
338
|
-
after.
|
|
293
|
+
(try? before.get()).map(\.count) ?? -1,
|
|
294
|
+
(try? after.get()).map(\.count) ?? -1,
|
|
339
295
|
observation.logToken
|
|
340
296
|
)
|
|
341
|
-
return Self.inAppBackOutcome(observation: observation
|
|
297
|
+
return Self.inAppBackOutcome(observation: observation)
|
|
342
298
|
}
|
|
343
299
|
|
|
344
300
|
|
|
345
301
|
|
|
346
|
-
static func inAppBackOutcome(
|
|
347
|
-
observation: NavigationVisualObservation,
|
|
348
|
-
before: NavigationVisualSample,
|
|
349
|
-
after: NavigationVisualSample
|
|
350
|
-
) -> InAppBackOutcome {
|
|
302
|
+
static func inAppBackOutcome(observation: NavigationVisualObservation) -> InAppBackOutcome {
|
|
351
303
|
switch observation {
|
|
352
304
|
case .changed:
|
|
353
305
|
return .performed
|
|
354
306
|
case .unchanged:
|
|
355
307
|
return .unavailable
|
|
356
|
-
case .unobserved:
|
|
308
|
+
case .unobserved(let refusal):
|
|
357
309
|
|
|
358
310
|
|
|
359
|
-
return .unverified(
|
|
311
|
+
return .unverified(
|
|
312
|
+
ErrorPayload(
|
|
313
|
+
code: refusal.rawValue,
|
|
314
|
+
message:
|
|
315
|
+
"The in-app back fallback was dispatched, but no display could be sampled to confirm the result. This is an unknown outcome, not evidence that a back control is absent.",
|
|
316
|
+
hint: refusal.hint
|
|
317
|
+
)
|
|
318
|
+
)
|
|
360
319
|
}
|
|
361
320
|
}
|
|
362
321
|
|
|
363
322
|
static func navigationVisualObservation(
|
|
364
|
-
before: Data
|
|
365
|
-
after: Data
|
|
323
|
+
before: Result<Data, RunnerAppScreenCaptureFailure>,
|
|
324
|
+
after: Result<Data, RunnerAppScreenCaptureFailure>
|
|
366
325
|
) -> NavigationVisualObservation {
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
326
|
+
switch (before, after) {
|
|
327
|
+
case (.success(let before), .success(let after)):
|
|
328
|
+
return before != after ? .changed : .unchanged
|
|
370
329
|
|
|
371
330
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
) -> ErrorPayload {
|
|
378
|
-
ErrorPayload(
|
|
379
|
-
code:
|
|
380
|
-
after.refusalCode
|
|
381
|
-
?? before.refusalCode
|
|
382
|
-
?? RunnerAppScreenCaptureFailure.unresolvedScreen.rawValue,
|
|
383
|
-
message:
|
|
384
|
-
"The in-app back fallback was dispatched, but no display could be sampled to confirm the result. This is an unknown outcome, not evidence that a back control is absent.",
|
|
385
|
-
hint: after.refusalHint ?? before.refusalHint
|
|
386
|
-
)
|
|
331
|
+
case (_, .failure(let refusal)):
|
|
332
|
+
return .unobserved(refusal: refusal)
|
|
333
|
+
case (.failure(let refusal), _):
|
|
334
|
+
return .unobserved(refusal: refusal)
|
|
335
|
+
}
|
|
387
336
|
}
|
|
388
337
|
|
|
389
338
|
private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
|
|
@@ -141,8 +141,7 @@ extension RunnerTests {
|
|
|
141
141
|
profile: SynthesizedDragProfile
|
|
142
142
|
) -> Response? {
|
|
143
143
|
#if os(iOS)
|
|
144
|
-
let
|
|
145
|
-
let context = context ?? synthesizedCoordinateContext(app: activeApp, policy: policy)
|
|
144
|
+
let context = context ?? synthesizedCoordinateContext(app: activeApp)
|
|
146
145
|
guard let plan = axFreeSynthesizedDragPlan(
|
|
147
146
|
app: activeApp,
|
|
148
147
|
x: x,
|
|
@@ -162,12 +161,13 @@ extension RunnerTests {
|
|
|
162
161
|
)
|
|
163
162
|
}
|
|
164
163
|
let durationMs = min(max(durationMs, 16), 10000)
|
|
165
|
-
let dragFrame =
|
|
164
|
+
let dragFrame = DragVisualizationFrame(
|
|
166
165
|
x: plan.points.x,
|
|
167
166
|
y: plan.points.y,
|
|
168
167
|
x2: plan.points.x2,
|
|
169
168
|
y2: plan.points.y2,
|
|
170
|
-
|
|
169
|
+
referenceWidth: Double(plan.referenceFrame.width),
|
|
170
|
+
referenceHeight: Double(plan.referenceFrame.height)
|
|
171
171
|
)
|
|
172
172
|
switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: {
|
|
173
173
|
synthesizedDragAt(
|
|
@@ -13,49 +13,30 @@ enum RunnerAccessibilityHealth: String, Equatable {
|
|
|
13
13
|
case unavailable
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
enum
|
|
17
|
-
case
|
|
18
|
-
case
|
|
16
|
+
enum SynthesizedGesturePolicyKind: String, Equatable, Hashable {
|
|
17
|
+
case coordinateTap
|
|
18
|
+
case coordinateLongPress
|
|
19
|
+
case scroll
|
|
20
|
+
case synthesizedDrag
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
19
25
|
|
|
20
|
-
func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool {
|
|
21
|
-
switch self {
|
|
22
|
-
case .never:
|
|
23
|
-
return false
|
|
24
|
-
case .requiredWhenAvailable:
|
|
25
|
-
return accessibilityHealth != .unavailable
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
26
|
|
|
30
|
-
enum SynthesizedFallbackPolicy: String, Equatable, Hashable {
|
|
31
|
-
case privateSynthesisRequired
|
|
32
|
-
case xctestCoordinateWhenAccessibilityAvailable
|
|
33
|
-
case xctestCoordinateAllowed
|
|
34
27
|
|
|
35
28
|
func allowsXCTestCoordinateFallback(accessibilityHealth: RunnerAccessibilityHealth) -> Bool {
|
|
36
29
|
switch self {
|
|
37
|
-
case .
|
|
38
|
-
return false
|
|
39
|
-
case .xctestCoordinateWhenAccessibilityAvailable:
|
|
40
|
-
return accessibilityHealth != .unavailable
|
|
41
|
-
case .xctestCoordinateAllowed:
|
|
30
|
+
case .coordinateTap:
|
|
42
31
|
return true
|
|
32
|
+
case .coordinateLongPress, .synthesizedDrag:
|
|
33
|
+
return accessibilityHealth != .unavailable
|
|
34
|
+
case .scroll:
|
|
35
|
+
return false
|
|
43
36
|
}
|
|
44
37
|
}
|
|
45
38
|
}
|
|
46
39
|
|
|
47
|
-
enum SynthesizedGesturePolicyKind: String, Equatable, Hashable {
|
|
48
|
-
case coordinateTap
|
|
49
|
-
case coordinateLongPress
|
|
50
|
-
case scroll
|
|
51
|
-
case synthesizedDrag
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
struct SynthesizedGesturePolicy: Equatable, Hashable {
|
|
55
|
-
let keyboardPolicy: SynthesizedKeyboardPolicy
|
|
56
|
-
let fallbackPolicy: SynthesizedFallbackPolicy
|
|
57
|
-
}
|
|
58
|
-
|
|
59
40
|
struct SynthesizedCoordinateContext {
|
|
60
41
|
let referenceFrame: CGRect
|
|
61
42
|
|
|
@@ -63,54 +44,22 @@ struct SynthesizedCoordinateContext {
|
|
|
63
44
|
|
|
64
45
|
|
|
65
46
|
let resolvedWindow: XCUIElement?
|
|
66
|
-
let keyboardPolicy: SynthesizedKeyboardPolicy
|
|
67
47
|
let accessibilityHealth: RunnerAccessibilityHealth
|
|
68
48
|
|
|
69
49
|
func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext {
|
|
70
50
|
SynthesizedCoordinateContext(
|
|
71
51
|
referenceFrame: frame,
|
|
72
52
|
resolvedWindow: resolvedWindow,
|
|
73
|
-
keyboardPolicy: keyboardPolicy,
|
|
74
53
|
accessibilityHealth: accessibilityHealth
|
|
75
54
|
)
|
|
76
55
|
}
|
|
77
56
|
|
|
78
|
-
var allowsKeyboardProbe: Bool {
|
|
79
|
-
keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth)
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
57
|
|
|
83
|
-
func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> SynthesizedGesturePolicy {
|
|
84
|
-
switch kind {
|
|
85
|
-
case .coordinateTap:
|
|
86
|
-
return SynthesizedGesturePolicy(
|
|
87
|
-
keyboardPolicy: .never,
|
|
88
|
-
fallbackPolicy: .xctestCoordinateAllowed
|
|
89
|
-
)
|
|
90
|
-
case .coordinateLongPress:
|
|
91
58
|
|
|
92
59
|
|
|
93
60
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable
|
|
97
|
-
)
|
|
98
|
-
case .scroll:
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
return SynthesizedGesturePolicy(
|
|
106
|
-
keyboardPolicy: .requiredWhenAvailable,
|
|
107
|
-
fallbackPolicy: .privateSynthesisRequired
|
|
108
|
-
)
|
|
109
|
-
case .synthesizedDrag:
|
|
110
|
-
return SynthesizedGesturePolicy(
|
|
111
|
-
keyboardPolicy: .requiredWhenAvailable,
|
|
112
|
-
fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable
|
|
113
|
-
)
|
|
61
|
+
var allowsKeyboardProbe: Bool {
|
|
62
|
+
accessibilityHealth != .unavailable
|
|
114
63
|
}
|
|
115
64
|
}
|
|
116
65
|
|
|
@@ -167,10 +116,10 @@ extension RunnerTests {
|
|
|
167
116
|
steps: [SequenceStep],
|
|
168
117
|
app: XCUIApplication
|
|
169
118
|
) -> SynthesizedCoordinateContext? {
|
|
170
|
-
guard
|
|
119
|
+
guard steps.lazy.contains(where: { synthesizedPolicyKind(forSequenceStep: $0) != nil }) else {
|
|
171
120
|
return nil
|
|
172
121
|
}
|
|
173
|
-
return synthesizedCoordinateContext(app: app
|
|
122
|
+
return synthesizedCoordinateContext(app: app)
|
|
174
123
|
}
|
|
175
124
|
|
|
176
125
|
|
|
@@ -192,7 +141,7 @@ extension RunnerTests {
|
|
|
192
141
|
logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false)
|
|
193
142
|
return .performed(timing: timing)
|
|
194
143
|
}
|
|
195
|
-
let fallbackAllowed =
|
|
144
|
+
let fallbackAllowed = kind.allowsXCTestCoordinateFallback(
|
|
196
145
|
accessibilityHealth: context?.accessibilityHealth ?? mainOwned.accessibilityHealth
|
|
197
146
|
)
|
|
198
147
|
logSynthesizedGesturePolicyDecision(
|
|
@@ -235,12 +184,9 @@ extension RunnerTests {
|
|
|
235
184
|
return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
|
|
236
185
|
+ " context=unavailable fallbackAttempted=\(fallbackAttempted)"
|
|
237
186
|
}
|
|
238
|
-
let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy
|
|
239
187
|
return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
|
|
240
|
-
+ " axHealth=\(context.accessibilityHealth.rawValue)
|
|
241
|
-
+ "
|
|
242
|
-
+ " fallbackPolicy=\(fallbackPolicy.rawValue)"
|
|
243
|
-
+ " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))"
|
|
188
|
+
+ " axHealth=\(context.accessibilityHealth.rawValue)"
|
|
189
|
+
+ " fallbackAllowed=\(kind.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))"
|
|
244
190
|
+ " fallbackAttempted=\(fallbackAttempted)"
|
|
245
191
|
}
|
|
246
192
|
}
|
|
@@ -21,10 +21,7 @@ extension RunnerTests {
|
|
|
21
21
|
)
|
|
22
22
|
}
|
|
23
23
|
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
24
|
-
guard let context = context ?? synthesizedCoordinateContext(
|
|
25
|
-
app: app,
|
|
26
|
-
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
27
|
-
) else {
|
|
24
|
+
guard let context = context ?? synthesizedCoordinateContext(app: app) else {
|
|
28
25
|
return .unsupported(
|
|
29
26
|
message: "synthesized coordinate drag could not resolve an app window with a finite screen frame",
|
|
30
27
|
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
|
|
@@ -107,10 +104,7 @@ extension RunnerTests {
|
|
|
107
104
|
)
|
|
108
105
|
}
|
|
109
106
|
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
110
|
-
guard let context = context ?? synthesizedCoordinateContext(
|
|
111
|
-
app: app,
|
|
112
|
-
policy: synthesizedGesturePolicy(.coordinateTap)
|
|
113
|
-
) else {
|
|
107
|
+
guard let context = context ?? synthesizedCoordinateContext(app: app) else {
|
|
114
108
|
return .unsupported(
|
|
115
109
|
message: "synthesized coordinate tap could not resolve an app window with a finite screen frame",
|
|
116
110
|
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
|
|
@@ -348,10 +342,7 @@ extension RunnerTests {
|
|
|
348
342
|
@MainActor
|
|
349
343
|
func gestureViewportFrame(app: XCUIApplication) -> CGRect {
|
|
350
344
|
#if os(iOS)
|
|
351
|
-
guard let context = synthesizedCoordinateContext(
|
|
352
|
-
app: app,
|
|
353
|
-
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
354
|
-
) else {
|
|
345
|
+
guard let context = synthesizedCoordinateContext(app: app) else {
|
|
355
346
|
return frameAvoidingKeyboard(app: app, frame: app.frame)
|
|
356
347
|
}
|
|
357
348
|
|
|
@@ -396,23 +387,15 @@ extension RunnerTests {
|
|
|
396
387
|
context: SynthesizedCoordinateContext? = nil
|
|
397
388
|
) -> SynthesizedDragPlan? {
|
|
398
389
|
#if os(iOS)
|
|
399
|
-
let context = context ?? synthesizedCoordinateContext(
|
|
400
|
-
app: app,
|
|
401
|
-
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
402
|
-
)
|
|
390
|
+
let context = context ?? synthesizedCoordinateContext(app: app)
|
|
403
391
|
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
|
|
404
392
|
let context
|
|
405
393
|
else {
|
|
406
394
|
return nil
|
|
407
395
|
}
|
|
408
|
-
let points =
|
|
409
|
-
app: app,
|
|
410
|
-
x: x,
|
|
411
|
-
y: y,
|
|
412
|
-
x2: x2,
|
|
413
|
-
y2: y2,
|
|
414
|
-
context: context
|
|
415
|
-
)
|
|
396
|
+
let points = context.allowsKeyboardProbe
|
|
397
|
+
? keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
|
|
398
|
+
: DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
416
399
|
return SynthesizedDragPlan(
|
|
417
400
|
points: points,
|
|
418
401
|
context: context
|
|
@@ -422,23 +405,6 @@ extension RunnerTests {
|
|
|
422
405
|
#endif
|
|
423
406
|
}
|
|
424
407
|
|
|
425
|
-
func axFreeDragVisualizationFrame(
|
|
426
|
-
x: Double,
|
|
427
|
-
y: Double,
|
|
428
|
-
x2: Double,
|
|
429
|
-
y2: Double,
|
|
430
|
-
referenceFrame: CGRect
|
|
431
|
-
) -> DragVisualizationFrame {
|
|
432
|
-
return DragVisualizationFrame(
|
|
433
|
-
x: x,
|
|
434
|
-
y: y,
|
|
435
|
-
x2: x2,
|
|
436
|
-
y2: y2,
|
|
437
|
-
referenceWidth: Double(referenceFrame.width),
|
|
438
|
-
referenceHeight: Double(referenceFrame.height)
|
|
439
|
-
)
|
|
440
|
-
}
|
|
441
|
-
|
|
442
408
|
func logSynthesizedDispatch(
|
|
443
409
|
kind: String,
|
|
444
410
|
start: CGPoint,
|
|
@@ -478,7 +444,6 @@ extension RunnerTests {
|
|
|
478
444
|
@MainActor
|
|
479
445
|
func synthesizedCoordinateContext(
|
|
480
446
|
app: XCUIApplication,
|
|
481
|
-
policy: SynthesizedGesturePolicy,
|
|
482
447
|
windowQueryRejected: Bool = false
|
|
483
448
|
) -> SynthesizedCoordinateContext? {
|
|
484
449
|
#if os(iOS)
|
|
@@ -508,7 +473,6 @@ extension RunnerTests {
|
|
|
508
473
|
return SynthesizedCoordinateContext(
|
|
509
474
|
referenceFrame: referenceFrame,
|
|
510
475
|
resolvedWindow: window,
|
|
511
|
-
keyboardPolicy: policy.keyboardPolicy,
|
|
512
476
|
accessibilityHealth: health
|
|
513
477
|
)
|
|
514
478
|
#else
|
|
@@ -581,24 +545,6 @@ extension RunnerTests {
|
|
|
581
545
|
}
|
|
582
546
|
}
|
|
583
547
|
|
|
584
|
-
func keyboardAvoidingSynthesizedDragPoints(
|
|
585
|
-
app: XCUIApplication,
|
|
586
|
-
x: Double,
|
|
587
|
-
y: Double,
|
|
588
|
-
x2: Double,
|
|
589
|
-
y2: Double,
|
|
590
|
-
context: SynthesizedCoordinateContext
|
|
591
|
-
) -> DragPoints {
|
|
592
|
-
#if os(iOS)
|
|
593
|
-
guard context.allowsKeyboardProbe else {
|
|
594
|
-
return DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
595
|
-
}
|
|
596
|
-
return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
|
|
597
|
-
#else
|
|
598
|
-
return DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
599
|
-
#endif
|
|
600
|
-
}
|
|
601
|
-
|
|
602
548
|
func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
|
|
603
549
|
if performTvRemoteSwipeIfAvailable(direction: direction) {
|
|
604
550
|
let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift
CHANGED
|
@@ -42,10 +42,7 @@ extension RunnerTests {
|
|
|
42
42
|
return textInputProbeUnavailableResponse()
|
|
43
43
|
}
|
|
44
44
|
let policyKind = SynthesizedGesturePolicyKind.coordinateTap
|
|
45
|
-
let context = synthesizedCoordinateContext(
|
|
46
|
-
app: activeApp,
|
|
47
|
-
policy: synthesizedGesturePolicy(policyKind)
|
|
48
|
-
)
|
|
45
|
+
let context = synthesizedCoordinateContext(app: activeApp)
|
|
49
46
|
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
|
|
50
47
|
synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
|
|
51
48
|
}) {
|