agent-device 0.8.6 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -22
- package/dist/src/224.js +2 -2
- package/dist/src/bin.js +65 -58
- package/dist/src/client-normalizers.d.ts +1 -2
- package/dist/src/client-shared.d.ts +1 -2
- package/dist/src/client-types.d.ts +0 -19
- package/dist/src/client.d.ts +1 -1
- package/dist/src/core/capabilities.d.ts +1 -1
- package/dist/src/core/click-button.d.ts +20 -0
- package/dist/src/core/dispatch-resolve.d.ts +7 -6
- package/dist/src/core/dispatch.d.ts +1 -0
- package/dist/src/daemon/context.d.ts +1 -0
- package/dist/src/daemon/handlers/interaction-common.d.ts +12 -0
- package/dist/src/daemon/handlers/interaction-fill.d.ts +3 -0
- package/dist/src/daemon/handlers/interaction-flags.d.ts +4 -0
- package/dist/src/daemon/handlers/interaction-get.d.ts +3 -0
- package/dist/src/daemon/handlers/interaction-is.d.ts +3 -0
- package/dist/src/daemon/handlers/interaction-press.d.ts +3 -0
- package/dist/src/daemon/handlers/interaction-scroll.d.ts +3 -0
- package/dist/src/daemon/handlers/interaction-selector.d.ts +27 -0
- package/dist/src/daemon/handlers/interaction-snapshot.d.ts +8 -0
- package/dist/src/daemon/handlers/interaction-targeting.d.ts +28 -0
- package/dist/src/daemon/handlers/interaction.d.ts +5 -12
- package/dist/src/daemon/handlers/session-device-utils.d.ts +1 -0
- package/dist/src/daemon/handlers/session-runtime.d.ts +3 -8
- package/dist/src/daemon/handlers/session.d.ts +8 -0
- package/dist/src/daemon/handlers/snapshot-alert.d.ts +13 -0
- package/dist/src/daemon/handlers/snapshot-capture.d.ts +27 -0
- package/dist/src/daemon/handlers/snapshot-session.d.ts +15 -0
- package/dist/src/daemon/handlers/snapshot-settings.d.ts +24 -0
- package/dist/src/daemon/handlers/snapshot-wait.d.ts +37 -0
- package/dist/src/daemon/handlers/snapshot.d.ts +4 -20
- package/dist/src/daemon/is-predicates.d.ts +2 -1
- package/dist/src/daemon/script-utils.d.ts +14 -2
- package/dist/src/daemon/selectors-build.d.ts +2 -1
- package/dist/src/daemon/selectors-match.d.ts +3 -2
- package/dist/src/daemon/selectors-resolve.d.ts +3 -2
- package/dist/src/daemon/session-open-script.d.ts +7 -0
- package/dist/src/daemon/session-store.d.ts +1 -0
- package/dist/src/daemon/snapshot-processing.d.ts +2 -1
- package/dist/src/daemon/types.d.ts +6 -5
- package/dist/src/daemon.js +35 -34
- package/dist/src/index.d.ts +1 -1
- package/dist/src/platforms/android/devices.d.ts +4 -0
- package/dist/src/platforms/android/sdk.d.ts +2 -0
- package/dist/src/platforms/ios/app-filter.d.ts +2 -0
- package/dist/src/platforms/ios/devices.d.ts +2 -1
- package/dist/src/platforms/ios/macos-apps.d.ts +12 -0
- package/dist/src/platforms/ios/runner-client.d.ts +3 -1
- package/dist/src/platforms/ios/runner-macos-products.d.ts +3 -0
- package/dist/src/platforms/ios/runner-xctestrun-products.d.ts +2 -0
- package/dist/src/platforms/ios/runner-xctestrun.d.ts +20 -2
- package/dist/src/utils/args.d.ts +1 -1
- package/dist/src/utils/cli-config.d.ts +2 -1
- package/dist/src/utils/command-schema.d.ts +7 -3
- package/dist/src/utils/device.d.ts +13 -5
- package/dist/src/utils/remote-config.d.ts +15 -0
- package/dist/src/utils/remote-open.d.ts +9 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +58 -50
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +10 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +35 -1
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +83 -9
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +39 -7
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +2 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +5 -6
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +132 -112
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +4 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +22 -5
- package/package.json +3 -2
- package/skills/agent-device/SKILL.md +28 -9
- package/skills/agent-device/references/macos-desktop.md +89 -0
- package/skills/agent-device/references/snapshot-refs.md +11 -2
|
@@ -4,12 +4,20 @@ extension RunnerTests {
|
|
|
4
4
|
// MARK: - Navigation Gestures
|
|
5
5
|
|
|
6
6
|
func tapNavigationBack(app: XCUIApplication) -> Bool {
|
|
7
|
+
#if os(macOS)
|
|
8
|
+
if let back = macOSNavigationBackElement(app: app) {
|
|
9
|
+
tapElementCenter(app: app, element: back)
|
|
10
|
+
return true
|
|
11
|
+
}
|
|
12
|
+
return false
|
|
13
|
+
#else
|
|
7
14
|
let buttons = app.navigationBars.buttons.allElementsBoundByIndex
|
|
8
15
|
if let back = buttons.first(where: { $0.isHittable }) {
|
|
9
16
|
back.tap()
|
|
10
17
|
return true
|
|
11
18
|
}
|
|
12
19
|
return pressTvRemoteMenuIfAvailable()
|
|
20
|
+
#endif
|
|
13
21
|
}
|
|
14
22
|
|
|
15
23
|
func performBackGesture(app: XCUIApplication) {
|
|
@@ -33,10 +41,14 @@ extension RunnerTests {
|
|
|
33
41
|
}
|
|
34
42
|
|
|
35
43
|
func pressHomeButton() {
|
|
44
|
+
#if os(macOS)
|
|
45
|
+
return
|
|
46
|
+
#else
|
|
36
47
|
if pressTvRemoteHomeIfAvailable() {
|
|
37
48
|
return
|
|
38
49
|
}
|
|
39
50
|
XCUIDevice.shared.press(.home)
|
|
51
|
+
#endif
|
|
40
52
|
}
|
|
41
53
|
|
|
42
54
|
private func pressTvRemoteMenuIfAvailable() -> Bool {
|
|
@@ -140,20 +152,47 @@ extension RunnerTests {
|
|
|
140
152
|
}
|
|
141
153
|
|
|
142
154
|
func tapAt(app: XCUIApplication, x: Double, y: Double) {
|
|
143
|
-
let
|
|
144
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
155
|
+
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
145
156
|
coordinate.tap()
|
|
146
157
|
}
|
|
147
158
|
|
|
159
|
+
func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws {
|
|
160
|
+
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
161
|
+
#if os(macOS)
|
|
162
|
+
switch button {
|
|
163
|
+
case "primary":
|
|
164
|
+
coordinate.tap()
|
|
165
|
+
case "secondary":
|
|
166
|
+
coordinate.rightClick()
|
|
167
|
+
case "middle":
|
|
168
|
+
throw NSError(
|
|
169
|
+
domain: "AgentDeviceRunner",
|
|
170
|
+
code: 1,
|
|
171
|
+
userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
|
|
172
|
+
)
|
|
173
|
+
default:
|
|
174
|
+
throw NSError(
|
|
175
|
+
domain: "AgentDeviceRunner",
|
|
176
|
+
code: 1,
|
|
177
|
+
userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
#else
|
|
181
|
+
throw NSError(
|
|
182
|
+
domain: "AgentDeviceRunner",
|
|
183
|
+
code: 1,
|
|
184
|
+
userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
|
|
185
|
+
)
|
|
186
|
+
#endif
|
|
187
|
+
}
|
|
188
|
+
|
|
148
189
|
func doubleTapAt(app: XCUIApplication, x: Double, y: Double) {
|
|
149
|
-
let
|
|
150
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
190
|
+
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
151
191
|
coordinate.doubleTap()
|
|
152
192
|
}
|
|
153
193
|
|
|
154
194
|
func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) {
|
|
155
|
-
let
|
|
156
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
195
|
+
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
157
196
|
coordinate.press(forDuration: duration)
|
|
158
197
|
}
|
|
159
198
|
|
|
@@ -165,9 +204,8 @@ extension RunnerTests {
|
|
|
165
204
|
y2: Double,
|
|
166
205
|
holdDuration: TimeInterval
|
|
167
206
|
) {
|
|
168
|
-
let
|
|
169
|
-
let
|
|
170
|
-
let end = origin.withOffset(CGVector(dx: x2, dy: y2))
|
|
207
|
+
let start = interactionCoordinate(app: app, x: x, y: y)
|
|
208
|
+
let end = interactionCoordinate(app: app, x: x2, y: y2)
|
|
171
209
|
start.press(forDuration: holdDuration, thenDragTo: end)
|
|
172
210
|
}
|
|
173
211
|
|
|
@@ -255,4 +293,40 @@ extension RunnerTests {
|
|
|
255
293
|
center.press(forDuration: 0.05, thenDragTo: endPoint)
|
|
256
294
|
}
|
|
257
295
|
|
|
296
|
+
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
|
|
297
|
+
let windows = app.windows.allElementsBoundByIndex
|
|
298
|
+
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
|
|
299
|
+
return window
|
|
300
|
+
}
|
|
301
|
+
return app
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
|
|
305
|
+
let root = interactionRoot(app: app)
|
|
306
|
+
let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
307
|
+
let rootFrame = root.frame
|
|
308
|
+
let offsetX = x - Double(rootFrame.origin.x)
|
|
309
|
+
let offsetY = y - Double(rootFrame.origin.y)
|
|
310
|
+
return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
|
|
314
|
+
let frame = element.frame
|
|
315
|
+
if !frame.isEmpty {
|
|
316
|
+
tapAt(app: app, x: frame.midX, y: frame.midY)
|
|
317
|
+
return
|
|
318
|
+
}
|
|
319
|
+
element.tap()
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
|
|
323
|
+
let predicate = NSPredicate(
|
|
324
|
+
format: "identifier == %@ OR label == %@",
|
|
325
|
+
"go back",
|
|
326
|
+
"Back"
|
|
327
|
+
)
|
|
328
|
+
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
329
|
+
return element.exists ? element : nil
|
|
330
|
+
}
|
|
331
|
+
|
|
258
332
|
}
|
|
@@ -1,11 +1,31 @@
|
|
|
1
1
|
import XCTest
|
|
2
|
-
|
|
2
|
+
#if canImport(AppKit)
|
|
3
|
+
import AppKit
|
|
4
|
+
#endif
|
|
5
|
+
|
|
6
|
+
func runnerPngData(for image: RunnerImage) -> Data? {
|
|
7
|
+
#if canImport(UIKit)
|
|
8
|
+
return image.pngData()
|
|
9
|
+
#elseif canImport(AppKit)
|
|
10
|
+
guard let cgImage = runnerCGImage(from: image) else { return nil }
|
|
11
|
+
let bitmap = NSBitmapImageRep(cgImage: cgImage)
|
|
12
|
+
return bitmap.representation(using: .png, properties: [:])
|
|
13
|
+
#endif
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
func runnerCGImage(from image: RunnerImage) -> CGImage? {
|
|
17
|
+
#if canImport(UIKit)
|
|
18
|
+
return image.cgImage
|
|
19
|
+
#elseif canImport(AppKit)
|
|
20
|
+
return image.cgImage(forProposedRect: nil, context: nil, hints: nil)
|
|
21
|
+
#endif
|
|
22
|
+
}
|
|
3
23
|
|
|
4
24
|
extension RunnerTests {
|
|
5
25
|
// MARK: - Recording
|
|
6
26
|
|
|
7
|
-
func captureRunnerFrame() ->
|
|
8
|
-
var image:
|
|
27
|
+
func captureRunnerFrame() -> RunnerImage? {
|
|
28
|
+
var image: RunnerImage?
|
|
9
29
|
let capture = {
|
|
10
30
|
let screenshot = XCUIScreen.main.screenshot()
|
|
11
31
|
image = screenshot.image
|
|
@@ -29,6 +49,11 @@ extension RunnerTests {
|
|
|
29
49
|
}
|
|
30
50
|
|
|
31
51
|
func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
|
|
52
|
+
#if os(macOS)
|
|
53
|
+
if requestedOutPath.hasPrefix("/") {
|
|
54
|
+
return requestedOutPath
|
|
55
|
+
}
|
|
56
|
+
#endif
|
|
32
57
|
let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
|
|
33
58
|
let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
|
|
34
59
|
let safeFileName = fileName.isEmpty ? fallbackName : fileName
|
|
@@ -38,12 +63,19 @@ extension RunnerTests {
|
|
|
38
63
|
// MARK: - Target Activation
|
|
39
64
|
|
|
40
65
|
func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
|
|
41
|
-
|
|
42
|
-
|
|
66
|
+
let state = target.state
|
|
67
|
+
#if os(macOS)
|
|
68
|
+
if state == .unknown || state == .notRunning || state == .runningBackground {
|
|
69
|
+
return true
|
|
70
|
+
}
|
|
71
|
+
#else
|
|
72
|
+
if state == .unknown || state == .notRunning || state == .runningBackground
|
|
73
|
+
|| state == .runningBackgroundSuspended
|
|
74
|
+
{
|
|
43
75
|
return true
|
|
44
|
-
default:
|
|
45
|
-
return false
|
|
46
76
|
}
|
|
77
|
+
#endif
|
|
78
|
+
return false
|
|
47
79
|
}
|
|
48
80
|
|
|
49
81
|
func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
enum CommandType: String, Codable {
|
|
4
4
|
case tap
|
|
5
|
+
case mouseClick
|
|
5
6
|
case tapSeries
|
|
6
7
|
case longPress
|
|
7
8
|
case drag
|
|
@@ -36,6 +37,7 @@ struct Command: Codable {
|
|
|
36
37
|
let action: String?
|
|
37
38
|
let x: Double?
|
|
38
39
|
let y: Double?
|
|
40
|
+
let button: String?
|
|
39
41
|
let count: Double?
|
|
40
42
|
let intervalMs: Double?
|
|
41
43
|
let doubleTap: Bool?
|
package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import AVFoundation
|
|
2
2
|
import CoreVideo
|
|
3
|
-
import UIKit
|
|
4
3
|
|
|
5
4
|
extension RunnerTests {
|
|
6
5
|
// MARK: - Screen Recorder
|
|
@@ -33,7 +32,7 @@ extension RunnerTests {
|
|
|
33
32
|
self.fps = fps
|
|
34
33
|
}
|
|
35
34
|
|
|
36
|
-
func start(captureFrame: @escaping () ->
|
|
35
|
+
func start(captureFrame: @escaping () -> RunnerImage?) throws {
|
|
37
36
|
let url = URL(fileURLWithPath: outputPath)
|
|
38
37
|
let directory = url.deletingLastPathComponent()
|
|
39
38
|
try FileManager.default.createDirectory(
|
|
@@ -46,10 +45,10 @@ extension RunnerTests {
|
|
|
46
45
|
}
|
|
47
46
|
|
|
48
47
|
var dimensions: CGSize = .zero
|
|
49
|
-
var bootstrapImage:
|
|
48
|
+
var bootstrapImage: RunnerImage?
|
|
50
49
|
let bootstrapDeadline = Date().addingTimeInterval(2.0)
|
|
51
50
|
while Date() < bootstrapDeadline {
|
|
52
|
-
if let image = captureFrame(), let cgImage = image
|
|
51
|
+
if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
|
|
53
52
|
bootstrapImage = image
|
|
54
53
|
dimensions = CGSize(width: cgImage.width, height: cgImage.height)
|
|
55
54
|
break
|
|
@@ -183,8 +182,8 @@ extension RunnerTests {
|
|
|
183
182
|
}
|
|
184
183
|
}
|
|
185
184
|
|
|
186
|
-
private func append(image:
|
|
187
|
-
guard let cgImage = image
|
|
185
|
+
private func append(image: RunnerImage) {
|
|
186
|
+
guard let cgImage = runnerCGImage(from: image) else { return }
|
|
188
187
|
lock.lock()
|
|
189
188
|
defer { lock.unlock() }
|
|
190
189
|
if isStopping { return }
|
|
@@ -1,6 +1,22 @@
|
|
|
1
1
|
import XCTest
|
|
2
2
|
|
|
3
3
|
extension RunnerTests {
|
|
4
|
+
private struct SnapshotTraversalContext {
|
|
5
|
+
let rootSnapshot: XCUIElementSnapshot
|
|
6
|
+
let viewport: CGRect
|
|
7
|
+
let flatSnapshots: [XCUIElementSnapshot]
|
|
8
|
+
let snapshotRanges: [ObjectIdentifier: (Int, Int)]
|
|
9
|
+
let maxDepth: Int
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
private struct SnapshotEvaluation {
|
|
13
|
+
let label: String
|
|
14
|
+
let identifier: String
|
|
15
|
+
let valueText: String?
|
|
16
|
+
let hittable: Bool
|
|
17
|
+
let visible: Bool
|
|
18
|
+
}
|
|
19
|
+
|
|
4
20
|
// MARK: - Snapshot Entry
|
|
5
21
|
|
|
6
22
|
func elementTypeName(_ type: XCUIElement.ElementType) -> String {
|
|
@@ -48,50 +64,19 @@ extension RunnerTests {
|
|
|
48
64
|
return blocking
|
|
49
65
|
}
|
|
50
66
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
let maxDepth = options.depth ?? Int.max
|
|
54
|
-
let viewport = app.frame
|
|
55
|
-
let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
56
|
-
|
|
57
|
-
let rootSnapshot: XCUIElementSnapshot
|
|
58
|
-
do {
|
|
59
|
-
rootSnapshot = try queryRoot.snapshot()
|
|
60
|
-
} catch {
|
|
61
|
-
return DataPayload(nodes: nodes, truncated: truncated)
|
|
67
|
+
guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
|
|
68
|
+
return DataPayload(nodes: [], truncated: false)
|
|
62
69
|
}
|
|
63
70
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
in: flatSnapshots,
|
|
68
|
-
ranges: snapshotRanges
|
|
69
|
-
)
|
|
70
|
-
let rootLabel = aggregatedLabel(for: rootSnapshot) ?? rootSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
71
|
-
let rootIdentifier = rootSnapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
72
|
-
let rootValue = snapshotValueText(rootSnapshot)
|
|
73
|
-
let rootHittable = computedSnapshotHittable(rootSnapshot, viewport: viewport, laterNodes: rootLaterNodes)
|
|
71
|
+
var nodes: [SnapshotNode] = []
|
|
72
|
+
var truncated = false
|
|
73
|
+
let rootEvaluation = evaluateSnapshot(context.rootSnapshot, in: context)
|
|
74
74
|
nodes.append(
|
|
75
|
-
|
|
76
|
-
index: 0,
|
|
77
|
-
type: elementTypeName(rootSnapshot.elementType),
|
|
78
|
-
label: rootLabel.isEmpty ? nil : rootLabel,
|
|
79
|
-
identifier: rootIdentifier.isEmpty ? nil : rootIdentifier,
|
|
80
|
-
value: rootValue,
|
|
81
|
-
rect: SnapshotRect(
|
|
82
|
-
x: Double(rootSnapshot.frame.origin.x),
|
|
83
|
-
y: Double(rootSnapshot.frame.origin.y),
|
|
84
|
-
width: Double(rootSnapshot.frame.size.width),
|
|
85
|
-
height: Double(rootSnapshot.frame.size.height)
|
|
86
|
-
),
|
|
87
|
-
enabled: rootSnapshot.isEnabled,
|
|
88
|
-
hittable: rootHittable,
|
|
89
|
-
depth: 0
|
|
90
|
-
)
|
|
75
|
+
makeSnapshotNode(snapshot: context.rootSnapshot, evaluation: rootEvaluation, depth: 0, index: 0)
|
|
91
76
|
)
|
|
92
77
|
|
|
93
78
|
var seen = Set<String>()
|
|
94
|
-
var stack: [(XCUIElementSnapshot, Int, Int)] = rootSnapshot.children.map { ($0, 1, 1) }
|
|
79
|
+
var stack: [(XCUIElementSnapshot, Int, Int)] = context.rootSnapshot.children.map { ($0, 1, 1) }
|
|
95
80
|
|
|
96
81
|
while let (snapshot, depth, visibleDepth) = stack.popLast() {
|
|
97
82
|
if nodes.count >= fastSnapshotLimit {
|
|
@@ -100,36 +85,24 @@ extension RunnerTests {
|
|
|
100
85
|
}
|
|
101
86
|
if let limit = options.depth, depth > limit { continue }
|
|
102
87
|
|
|
103
|
-
let
|
|
104
|
-
let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
105
|
-
let valueText = snapshotValueText(snapshot)
|
|
106
|
-
let laterNodes = laterSnapshots(
|
|
107
|
-
for: snapshot,
|
|
108
|
-
in: flatSnapshots,
|
|
109
|
-
ranges: snapshotRanges
|
|
110
|
-
)
|
|
111
|
-
let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
|
|
112
|
-
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
113
|
-
if !isVisibleInViewport(snapshot.frame, viewport) && !hasContent {
|
|
114
|
-
continue
|
|
115
|
-
}
|
|
116
|
-
|
|
88
|
+
let evaluation = evaluateSnapshot(snapshot, in: context)
|
|
117
89
|
let include = shouldInclude(
|
|
118
90
|
snapshot: snapshot,
|
|
119
|
-
label: label,
|
|
120
|
-
identifier: identifier,
|
|
121
|
-
valueText: valueText,
|
|
91
|
+
label: evaluation.label,
|
|
92
|
+
identifier: evaluation.identifier,
|
|
93
|
+
valueText: evaluation.valueText,
|
|
122
94
|
options: options,
|
|
123
|
-
hittable: hittable
|
|
95
|
+
hittable: evaluation.hittable,
|
|
96
|
+
visible: evaluation.visible
|
|
124
97
|
)
|
|
125
98
|
|
|
126
|
-
let key = "\(snapshot.elementType)-\(label)-\(identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
|
|
99
|
+
let key = "\(snapshot.elementType)-\(evaluation.label)-\(evaluation.identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
|
|
127
100
|
let isDuplicate = seen.contains(key)
|
|
128
101
|
if !isDuplicate {
|
|
129
102
|
seen.insert(key)
|
|
130
103
|
}
|
|
131
104
|
|
|
132
|
-
if depth < maxDepth {
|
|
105
|
+
if depth < context.maxDepth {
|
|
133
106
|
let nextVisibleDepth = include && !isDuplicate ? visibleDepth + 1 : visibleDepth
|
|
134
107
|
for child in snapshot.children.reversed() {
|
|
135
108
|
stack.append((child, depth + 1, nextVisibleDepth))
|
|
@@ -139,21 +112,11 @@ extension RunnerTests {
|
|
|
139
112
|
if !include || isDuplicate { continue }
|
|
140
113
|
|
|
141
114
|
nodes.append(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
value: valueText,
|
|
148
|
-
rect: SnapshotRect(
|
|
149
|
-
x: Double(snapshot.frame.origin.x),
|
|
150
|
-
y: Double(snapshot.frame.origin.y),
|
|
151
|
-
width: Double(snapshot.frame.size.width),
|
|
152
|
-
height: Double(snapshot.frame.size.height)
|
|
153
|
-
),
|
|
154
|
-
enabled: snapshot.isEnabled,
|
|
155
|
-
hittable: hittable,
|
|
156
|
-
depth: min(maxDepth, visibleDepth)
|
|
115
|
+
makeSnapshotNode(
|
|
116
|
+
snapshot: snapshot,
|
|
117
|
+
evaluation: evaluation,
|
|
118
|
+
depth: min(context.maxDepth, visibleDepth),
|
|
119
|
+
index: nodes.count
|
|
157
120
|
)
|
|
158
121
|
)
|
|
159
122
|
|
|
@@ -167,19 +130,12 @@ extension RunnerTests {
|
|
|
167
130
|
return blocking
|
|
168
131
|
}
|
|
169
132
|
|
|
170
|
-
let
|
|
171
|
-
|
|
172
|
-
var truncated = false
|
|
173
|
-
let viewport = app.frame
|
|
174
|
-
|
|
175
|
-
let rootSnapshot: XCUIElementSnapshot
|
|
176
|
-
do {
|
|
177
|
-
rootSnapshot = try queryRoot.snapshot()
|
|
178
|
-
} catch {
|
|
179
|
-
return DataPayload(nodes: nodes, truncated: truncated)
|
|
133
|
+
guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
|
|
134
|
+
return DataPayload(nodes: [], truncated: false)
|
|
180
135
|
}
|
|
181
136
|
|
|
182
|
-
|
|
137
|
+
var nodes: [SnapshotNode] = []
|
|
138
|
+
var truncated = false
|
|
183
139
|
|
|
184
140
|
func walk(_ snapshot: XCUIElementSnapshot, depth: Int) {
|
|
185
141
|
if nodes.count >= maxSnapshotElements {
|
|
@@ -187,37 +143,19 @@ extension RunnerTests {
|
|
|
187
143
|
return
|
|
188
144
|
}
|
|
189
145
|
if let limit = options.depth, depth > limit { return }
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
193
|
-
let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
194
|
-
let valueText = snapshotValueText(snapshot)
|
|
195
|
-
let laterNodes = laterSnapshots(
|
|
196
|
-
for: snapshot,
|
|
197
|
-
in: flatSnapshots,
|
|
198
|
-
ranges: snapshotRanges
|
|
199
|
-
)
|
|
200
|
-
let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
|
|
146
|
+
|
|
147
|
+
let evaluation = evaluateSnapshot(snapshot, in: context)
|
|
201
148
|
if shouldInclude(
|
|
202
149
|
snapshot: snapshot,
|
|
203
|
-
label: label,
|
|
204
|
-
identifier: identifier,
|
|
205
|
-
valueText: valueText,
|
|
150
|
+
label: evaluation.label,
|
|
151
|
+
identifier: evaluation.identifier,
|
|
152
|
+
valueText: evaluation.valueText,
|
|
206
153
|
options: options,
|
|
207
|
-
hittable: hittable
|
|
154
|
+
hittable: evaluation.hittable,
|
|
155
|
+
visible: evaluation.visible
|
|
208
156
|
) {
|
|
209
157
|
nodes.append(
|
|
210
|
-
|
|
211
|
-
index: nodes.count,
|
|
212
|
-
type: elementTypeName(snapshot.elementType),
|
|
213
|
-
label: label.isEmpty ? nil : label,
|
|
214
|
-
identifier: identifier.isEmpty ? nil : identifier,
|
|
215
|
-
value: valueText,
|
|
216
|
-
rect: snapshotRect(from: snapshot.frame),
|
|
217
|
-
enabled: snapshot.isEnabled,
|
|
218
|
-
hittable: hittable,
|
|
219
|
-
depth: depth
|
|
220
|
-
)
|
|
158
|
+
makeSnapshotNode(snapshot: snapshot, evaluation: evaluation, depth: depth, index: nodes.count)
|
|
221
159
|
)
|
|
222
160
|
}
|
|
223
161
|
|
|
@@ -228,7 +166,7 @@ extension RunnerTests {
|
|
|
228
166
|
}
|
|
229
167
|
}
|
|
230
168
|
|
|
231
|
-
walk(rootSnapshot, depth: 0)
|
|
169
|
+
walk(context.rootSnapshot, depth: 0)
|
|
232
170
|
return DataPayload(nodes: nodes, truncated: truncated)
|
|
233
171
|
}
|
|
234
172
|
|
|
@@ -249,7 +187,8 @@ extension RunnerTests {
|
|
|
249
187
|
identifier: String,
|
|
250
188
|
valueText: String?,
|
|
251
189
|
options: SnapshotOptions,
|
|
252
|
-
hittable: Bool
|
|
190
|
+
hittable: Bool,
|
|
191
|
+
visible: Bool
|
|
253
192
|
) -> Bool {
|
|
254
193
|
let type = snapshot.elementType
|
|
255
194
|
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
@@ -257,6 +196,11 @@ extension RunnerTests {
|
|
|
257
196
|
if snapshot.children.count <= 1 { return false }
|
|
258
197
|
}
|
|
259
198
|
if options.interactiveOnly {
|
|
199
|
+
#if os(macOS)
|
|
200
|
+
if !visible && type != .application {
|
|
201
|
+
return false
|
|
202
|
+
}
|
|
203
|
+
#endif
|
|
260
204
|
if interactiveTypes.contains(type) { return true }
|
|
261
205
|
if hittable && type != .other { return true }
|
|
262
206
|
if hasContent { return true }
|
|
@@ -287,6 +231,70 @@ extension RunnerTests {
|
|
|
287
231
|
return true
|
|
288
232
|
}
|
|
289
233
|
|
|
234
|
+
private func makeSnapshotTraversalContext(
|
|
235
|
+
app: XCUIApplication,
|
|
236
|
+
options: SnapshotOptions
|
|
237
|
+
) -> SnapshotTraversalContext? {
|
|
238
|
+
let viewport = snapshotViewport(app: app)
|
|
239
|
+
let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
240
|
+
|
|
241
|
+
let rootSnapshot: XCUIElementSnapshot
|
|
242
|
+
do {
|
|
243
|
+
rootSnapshot = try queryRoot.snapshot()
|
|
244
|
+
} catch {
|
|
245
|
+
return nil
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
|
|
249
|
+
return SnapshotTraversalContext(
|
|
250
|
+
rootSnapshot: rootSnapshot,
|
|
251
|
+
viewport: viewport,
|
|
252
|
+
flatSnapshots: flatSnapshots,
|
|
253
|
+
snapshotRanges: snapshotRanges,
|
|
254
|
+
maxDepth: options.depth ?? Int.max
|
|
255
|
+
)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
private func evaluateSnapshot(
|
|
259
|
+
_ snapshot: XCUIElementSnapshot,
|
|
260
|
+
in context: SnapshotTraversalContext
|
|
261
|
+
) -> SnapshotEvaluation {
|
|
262
|
+
let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
263
|
+
let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
264
|
+
let valueText = snapshotValueText(snapshot)
|
|
265
|
+
let laterNodes = laterSnapshots(
|
|
266
|
+
for: snapshot,
|
|
267
|
+
in: context.flatSnapshots,
|
|
268
|
+
ranges: context.snapshotRanges
|
|
269
|
+
)
|
|
270
|
+
return SnapshotEvaluation(
|
|
271
|
+
label: label,
|
|
272
|
+
identifier: identifier,
|
|
273
|
+
valueText: valueText,
|
|
274
|
+
hittable: computedSnapshotHittable(snapshot, viewport: context.viewport, laterNodes: laterNodes),
|
|
275
|
+
visible: isVisibleInViewport(snapshot.frame, context.viewport)
|
|
276
|
+
)
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private func makeSnapshotNode(
|
|
280
|
+
snapshot: XCUIElementSnapshot,
|
|
281
|
+
evaluation: SnapshotEvaluation,
|
|
282
|
+
depth: Int,
|
|
283
|
+
index: Int
|
|
284
|
+
) -> SnapshotNode {
|
|
285
|
+
return SnapshotNode(
|
|
286
|
+
index: index,
|
|
287
|
+
type: elementTypeName(snapshot.elementType),
|
|
288
|
+
label: evaluation.label.isEmpty ? nil : evaluation.label,
|
|
289
|
+
identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
|
|
290
|
+
value: evaluation.valueText,
|
|
291
|
+
rect: snapshotRect(from: snapshot.frame),
|
|
292
|
+
enabled: snapshot.isEnabled,
|
|
293
|
+
hittable: evaluation.hittable,
|
|
294
|
+
depth: depth
|
|
295
|
+
)
|
|
296
|
+
}
|
|
297
|
+
|
|
290
298
|
private func isOccludingType(_ type: XCUIElement.ElementType) -> Bool {
|
|
291
299
|
switch type {
|
|
292
300
|
case .application, .window:
|
|
@@ -339,6 +347,18 @@ extension RunnerTests {
|
|
|
339
347
|
return text.isEmpty ? nil : text
|
|
340
348
|
}
|
|
341
349
|
|
|
350
|
+
private func snapshotViewport(app: XCUIApplication) -> CGRect {
|
|
351
|
+
let windows = app.windows.allElementsBoundByIndex
|
|
352
|
+
if let window = windows.first(where: { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty }) {
|
|
353
|
+
return window.frame
|
|
354
|
+
}
|
|
355
|
+
let appFrame = app.frame
|
|
356
|
+
if !appFrame.isNull && !appFrame.isEmpty {
|
|
357
|
+
return appFrame
|
|
358
|
+
}
|
|
359
|
+
return .infinite
|
|
360
|
+
}
|
|
361
|
+
|
|
342
362
|
private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
|
|
343
363
|
if depth > 4 { return nil }
|
|
344
364
|
let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
@@ -4,6 +4,9 @@ extension RunnerTests {
|
|
|
4
4
|
// MARK: - Blocking System Modal Snapshot
|
|
5
5
|
|
|
6
6
|
func blockingSystemAlertSnapshot() -> DataPayload? {
|
|
7
|
+
#if os(macOS)
|
|
8
|
+
return nil
|
|
9
|
+
#else
|
|
7
10
|
guard let modal = firstBlockingSystemModal(in: springboard) else {
|
|
8
11
|
return nil
|
|
9
12
|
}
|
|
@@ -40,6 +43,7 @@ extension RunnerTests {
|
|
|
40
43
|
}
|
|
41
44
|
|
|
42
45
|
return DataPayload(nodes: nodes, truncated: false)
|
|
46
|
+
#endif
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
|
|
@@ -7,6 +7,13 @@
|
|
|
7
7
|
|
|
8
8
|
import XCTest
|
|
9
9
|
import Network
|
|
10
|
+
#if canImport(UIKit)
|
|
11
|
+
import UIKit
|
|
12
|
+
typealias RunnerImage = UIImage
|
|
13
|
+
#elseif canImport(AppKit)
|
|
14
|
+
import AppKit
|
|
15
|
+
typealias RunnerImage = NSImage
|
|
16
|
+
#endif
|
|
10
17
|
|
|
11
18
|
final class RunnerTests: XCTestCase {
|
|
12
19
|
enum RunnerErrorDomain {
|
|
@@ -85,11 +92,7 @@ final class RunnerTests: XCTestCase {
|
|
|
85
92
|
let queue = DispatchQueue(label: "agent-device.runner")
|
|
86
93
|
let desiredPort = RunnerEnv.resolvePort()
|
|
87
94
|
NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", desiredPort)
|
|
88
|
-
|
|
89
|
-
listener = try NWListener(using: .tcp, on: port)
|
|
90
|
-
} else {
|
|
91
|
-
listener = try NWListener(using: .tcp)
|
|
92
|
-
}
|
|
95
|
+
listener = try makeRunnerListener(desiredPort: desiredPort)
|
|
93
96
|
listener?.stateUpdateHandler = { [weak self] state in
|
|
94
97
|
switch state {
|
|
95
98
|
case .ready:
|
|
@@ -123,4 +126,18 @@ final class RunnerTests: XCTestCase {
|
|
|
123
126
|
XCTFail("runner wait ended with \(result)")
|
|
124
127
|
}
|
|
125
128
|
}
|
|
129
|
+
|
|
130
|
+
private func makeRunnerListener(desiredPort: UInt16) throws -> NWListener {
|
|
131
|
+
if desiredPort > 0, let port = NWEndpoint.Port(rawValue: desiredPort) {
|
|
132
|
+
#if os(macOS)
|
|
133
|
+
let parameters = NWParameters.tcp
|
|
134
|
+
parameters.allowLocalEndpointReuse = true
|
|
135
|
+
parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: port)
|
|
136
|
+
return try NWListener(using: parameters)
|
|
137
|
+
#else
|
|
138
|
+
return try NWListener(using: .tcp, on: port)
|
|
139
|
+
#endif
|
|
140
|
+
}
|
|
141
|
+
return try NWListener(using: .tcp)
|
|
142
|
+
}
|
|
126
143
|
}
|