agent-device 0.8.6 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +106 -22
  2. package/dist/src/224.js +2 -2
  3. package/dist/src/bin.js +65 -58
  4. package/dist/src/client-normalizers.d.ts +1 -2
  5. package/dist/src/client-shared.d.ts +1 -2
  6. package/dist/src/client-types.d.ts +0 -19
  7. package/dist/src/client.d.ts +1 -1
  8. package/dist/src/core/capabilities.d.ts +1 -1
  9. package/dist/src/core/click-button.d.ts +20 -0
  10. package/dist/src/core/dispatch-resolve.d.ts +7 -6
  11. package/dist/src/core/dispatch.d.ts +1 -0
  12. package/dist/src/daemon/context.d.ts +1 -0
  13. package/dist/src/daemon/handlers/interaction-common.d.ts +12 -0
  14. package/dist/src/daemon/handlers/interaction-fill.d.ts +3 -0
  15. package/dist/src/daemon/handlers/interaction-flags.d.ts +4 -0
  16. package/dist/src/daemon/handlers/interaction-get.d.ts +3 -0
  17. package/dist/src/daemon/handlers/interaction-is.d.ts +3 -0
  18. package/dist/src/daemon/handlers/interaction-press.d.ts +3 -0
  19. package/dist/src/daemon/handlers/interaction-scroll.d.ts +3 -0
  20. package/dist/src/daemon/handlers/interaction-selector.d.ts +27 -0
  21. package/dist/src/daemon/handlers/interaction-snapshot.d.ts +8 -0
  22. package/dist/src/daemon/handlers/interaction-targeting.d.ts +28 -0
  23. package/dist/src/daemon/handlers/interaction.d.ts +5 -12
  24. package/dist/src/daemon/handlers/session-device-utils.d.ts +1 -0
  25. package/dist/src/daemon/handlers/session-runtime.d.ts +3 -8
  26. package/dist/src/daemon/handlers/session.d.ts +8 -0
  27. package/dist/src/daemon/handlers/snapshot-alert.d.ts +13 -0
  28. package/dist/src/daemon/handlers/snapshot-capture.d.ts +27 -0
  29. package/dist/src/daemon/handlers/snapshot-session.d.ts +15 -0
  30. package/dist/src/daemon/handlers/snapshot-settings.d.ts +24 -0
  31. package/dist/src/daemon/handlers/snapshot-wait.d.ts +37 -0
  32. package/dist/src/daemon/handlers/snapshot.d.ts +4 -20
  33. package/dist/src/daemon/is-predicates.d.ts +2 -1
  34. package/dist/src/daemon/script-utils.d.ts +14 -2
  35. package/dist/src/daemon/selectors-build.d.ts +2 -1
  36. package/dist/src/daemon/selectors-match.d.ts +3 -2
  37. package/dist/src/daemon/selectors-resolve.d.ts +3 -2
  38. package/dist/src/daemon/session-open-script.d.ts +7 -0
  39. package/dist/src/daemon/session-store.d.ts +1 -0
  40. package/dist/src/daemon/snapshot-processing.d.ts +2 -1
  41. package/dist/src/daemon/types.d.ts +6 -5
  42. package/dist/src/daemon.js +35 -34
  43. package/dist/src/index.d.ts +1 -1
  44. package/dist/src/platforms/android/devices.d.ts +4 -0
  45. package/dist/src/platforms/android/sdk.d.ts +2 -0
  46. package/dist/src/platforms/ios/app-filter.d.ts +2 -0
  47. package/dist/src/platforms/ios/devices.d.ts +2 -1
  48. package/dist/src/platforms/ios/macos-apps.d.ts +12 -0
  49. package/dist/src/platforms/ios/runner-client.d.ts +3 -1
  50. package/dist/src/platforms/ios/runner-macos-products.d.ts +3 -0
  51. package/dist/src/platforms/ios/runner-xctestrun-products.d.ts +2 -0
  52. package/dist/src/platforms/ios/runner-xctestrun.d.ts +20 -2
  53. package/dist/src/utils/args.d.ts +1 -1
  54. package/dist/src/utils/cli-config.d.ts +2 -1
  55. package/dist/src/utils/command-schema.d.ts +7 -3
  56. package/dist/src/utils/device.d.ts +13 -5
  57. package/dist/src/utils/remote-config.d.ts +15 -0
  58. package/dist/src/utils/remote-open.d.ts +9 -0
  59. package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +58 -50
  60. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +10 -0
  61. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +35 -1
  62. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +83 -9
  63. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +39 -7
  64. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +2 -0
  65. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +5 -6
  66. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +132 -112
  67. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +4 -0
  68. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +22 -5
  69. package/package.json +3 -2
  70. package/skills/agent-device/SKILL.md +28 -9
  71. package/skills/agent-device/references/macos-desktop.md +89 -0
  72. package/skills/agent-device/references/snapshot-refs.md +11 -2
@@ -4,12 +4,20 @@ extension RunnerTests {
4
4
  // MARK: - Navigation Gestures
5
5
 
6
6
  func tapNavigationBack(app: XCUIApplication) -> Bool {
7
+ #if os(macOS)
8
+ if let back = macOSNavigationBackElement(app: app) {
9
+ tapElementCenter(app: app, element: back)
10
+ return true
11
+ }
12
+ return false
13
+ #else
7
14
  let buttons = app.navigationBars.buttons.allElementsBoundByIndex
8
15
  if let back = buttons.first(where: { $0.isHittable }) {
9
16
  back.tap()
10
17
  return true
11
18
  }
12
19
  return pressTvRemoteMenuIfAvailable()
20
+ #endif
13
21
  }
14
22
 
15
23
  func performBackGesture(app: XCUIApplication) {
@@ -33,10 +41,14 @@ extension RunnerTests {
33
41
  }
34
42
 
35
43
  func pressHomeButton() {
44
+ #if os(macOS)
45
+ return
46
+ #else
36
47
  if pressTvRemoteHomeIfAvailable() {
37
48
  return
38
49
  }
39
50
  XCUIDevice.shared.press(.home)
51
+ #endif
40
52
  }
41
53
 
42
54
  private func pressTvRemoteMenuIfAvailable() -> Bool {
@@ -140,20 +152,47 @@ extension RunnerTests {
140
152
  }
141
153
 
142
154
  func tapAt(app: XCUIApplication, x: Double, y: Double) {
143
- let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
144
- let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
155
+ let coordinate = interactionCoordinate(app: app, x: x, y: y)
145
156
  coordinate.tap()
146
157
  }
147
158
 
159
+ func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws {
160
+ let coordinate = interactionCoordinate(app: app, x: x, y: y)
161
+ #if os(macOS)
162
+ switch button {
163
+ case "primary":
164
+ coordinate.tap()
165
+ case "secondary":
166
+ coordinate.rightClick()
167
+ case "middle":
168
+ throw NSError(
169
+ domain: "AgentDeviceRunner",
170
+ code: 1,
171
+ userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
172
+ )
173
+ default:
174
+ throw NSError(
175
+ domain: "AgentDeviceRunner",
176
+ code: 1,
177
+ userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
178
+ )
179
+ }
180
+ #else
181
+ throw NSError(
182
+ domain: "AgentDeviceRunner",
183
+ code: 1,
184
+ userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
185
+ )
186
+ #endif
187
+ }
188
+
148
189
  func doubleTapAt(app: XCUIApplication, x: Double, y: Double) {
149
- let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
150
- let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
190
+ let coordinate = interactionCoordinate(app: app, x: x, y: y)
151
191
  coordinate.doubleTap()
152
192
  }
153
193
 
154
194
  func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) {
155
- let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
156
- let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
195
+ let coordinate = interactionCoordinate(app: app, x: x, y: y)
157
196
  coordinate.press(forDuration: duration)
158
197
  }
159
198
 
@@ -165,9 +204,8 @@ extension RunnerTests {
165
204
  y2: Double,
166
205
  holdDuration: TimeInterval
167
206
  ) {
168
- let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
169
- let start = origin.withOffset(CGVector(dx: x, dy: y))
170
- let end = origin.withOffset(CGVector(dx: x2, dy: y2))
207
+ let start = interactionCoordinate(app: app, x: x, y: y)
208
+ let end = interactionCoordinate(app: app, x: x2, y: y2)
171
209
  start.press(forDuration: holdDuration, thenDragTo: end)
172
210
  }
173
211
 
@@ -255,4 +293,40 @@ extension RunnerTests {
255
293
  center.press(forDuration: 0.05, thenDragTo: endPoint)
256
294
  }
257
295
 
296
+ private func interactionRoot(app: XCUIApplication) -> XCUIElement {
297
+ let windows = app.windows.allElementsBoundByIndex
298
+ if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
299
+ return window
300
+ }
301
+ return app
302
+ }
303
+
304
+ private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
305
+ let root = interactionRoot(app: app)
306
+ let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
307
+ let rootFrame = root.frame
308
+ let offsetX = x - Double(rootFrame.origin.x)
309
+ let offsetY = y - Double(rootFrame.origin.y)
310
+ return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
311
+ }
312
+
313
+ private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
314
+ let frame = element.frame
315
+ if !frame.isEmpty {
316
+ tapAt(app: app, x: frame.midX, y: frame.midY)
317
+ return
318
+ }
319
+ element.tap()
320
+ }
321
+
322
+ private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
323
+ let predicate = NSPredicate(
324
+ format: "identifier == %@ OR label == %@",
325
+ "go back",
326
+ "Back"
327
+ )
328
+ let element = app.descendants(matching: .any).matching(predicate).firstMatch
329
+ return element.exists ? element : nil
330
+ }
331
+
258
332
  }
@@ -1,11 +1,31 @@
1
1
  import XCTest
2
- import UIKit
2
+ #if canImport(AppKit)
3
+ import AppKit
4
+ #endif
5
+
6
+ func runnerPngData(for image: RunnerImage) -> Data? {
7
+ #if canImport(UIKit)
8
+ return image.pngData()
9
+ #elseif canImport(AppKit)
10
+ guard let cgImage = runnerCGImage(from: image) else { return nil }
11
+ let bitmap = NSBitmapImageRep(cgImage: cgImage)
12
+ return bitmap.representation(using: .png, properties: [:])
13
+ #endif
14
+ }
15
+
16
+ func runnerCGImage(from image: RunnerImage) -> CGImage? {
17
+ #if canImport(UIKit)
18
+ return image.cgImage
19
+ #elseif canImport(AppKit)
20
+ return image.cgImage(forProposedRect: nil, context: nil, hints: nil)
21
+ #endif
22
+ }
3
23
 
4
24
  extension RunnerTests {
5
25
  // MARK: - Recording
6
26
 
7
- func captureRunnerFrame() -> UIImage? {
8
- var image: UIImage?
27
+ func captureRunnerFrame() -> RunnerImage? {
28
+ var image: RunnerImage?
9
29
  let capture = {
10
30
  let screenshot = XCUIScreen.main.screenshot()
11
31
  image = screenshot.image
@@ -29,6 +49,11 @@ extension RunnerTests {
29
49
  }
30
50
 
31
51
  func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
52
+ #if os(macOS)
53
+ if requestedOutPath.hasPrefix("/") {
54
+ return requestedOutPath
55
+ }
56
+ #endif
32
57
  let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
33
58
  let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
34
59
  let safeFileName = fileName.isEmpty ? fallbackName : fileName
@@ -38,12 +63,19 @@ extension RunnerTests {
38
63
  // MARK: - Target Activation
39
64
 
40
65
  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
41
- switch target.state {
42
- case .unknown, .notRunning, .runningBackground, .runningBackgroundSuspended:
66
+ let state = target.state
67
+ #if os(macOS)
68
+ if state == .unknown || state == .notRunning || state == .runningBackground {
69
+ return true
70
+ }
71
+ #else
72
+ if state == .unknown || state == .notRunning || state == .runningBackground
73
+ || state == .runningBackgroundSuspended
74
+ {
43
75
  return true
44
- default:
45
- return false
46
76
  }
77
+ #endif
78
+ return false
47
79
  }
48
80
 
49
81
  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
@@ -2,6 +2,7 @@
2
2
 
3
3
  enum CommandType: String, Codable {
4
4
  case tap
5
+ case mouseClick
5
6
  case tapSeries
6
7
  case longPress
7
8
  case drag
@@ -36,6 +37,7 @@ struct Command: Codable {
36
37
  let action: String?
37
38
  let x: Double?
38
39
  let y: Double?
40
+ let button: String?
39
41
  let count: Double?
40
42
  let intervalMs: Double?
41
43
  let doubleTap: Bool?
@@ -1,6 +1,5 @@
1
1
  import AVFoundation
2
2
  import CoreVideo
3
- import UIKit
4
3
 
5
4
  extension RunnerTests {
6
5
  // MARK: - Screen Recorder
@@ -33,7 +32,7 @@ extension RunnerTests {
33
32
  self.fps = fps
34
33
  }
35
34
 
36
- func start(captureFrame: @escaping () -> UIImage?) throws {
35
+ func start(captureFrame: @escaping () -> RunnerImage?) throws {
37
36
  let url = URL(fileURLWithPath: outputPath)
38
37
  let directory = url.deletingLastPathComponent()
39
38
  try FileManager.default.createDirectory(
@@ -46,10 +45,10 @@ extension RunnerTests {
46
45
  }
47
46
 
48
47
  var dimensions: CGSize = .zero
49
- var bootstrapImage: UIImage?
48
+ var bootstrapImage: RunnerImage?
50
49
  let bootstrapDeadline = Date().addingTimeInterval(2.0)
51
50
  while Date() < bootstrapDeadline {
52
- if let image = captureFrame(), let cgImage = image.cgImage {
51
+ if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
53
52
  bootstrapImage = image
54
53
  dimensions = CGSize(width: cgImage.width, height: cgImage.height)
55
54
  break
@@ -183,8 +182,8 @@ extension RunnerTests {
183
182
  }
184
183
  }
185
184
 
186
- private func append(image: UIImage) {
187
- guard let cgImage = image.cgImage else { return }
185
+ private func append(image: RunnerImage) {
186
+ guard let cgImage = runnerCGImage(from: image) else { return }
188
187
  lock.lock()
189
188
  defer { lock.unlock() }
190
189
  if isStopping { return }
@@ -1,6 +1,22 @@
1
1
  import XCTest
2
2
 
3
3
  extension RunnerTests {
4
+ private struct SnapshotTraversalContext {
5
+ let rootSnapshot: XCUIElementSnapshot
6
+ let viewport: CGRect
7
+ let flatSnapshots: [XCUIElementSnapshot]
8
+ let snapshotRanges: [ObjectIdentifier: (Int, Int)]
9
+ let maxDepth: Int
10
+ }
11
+
12
+ private struct SnapshotEvaluation {
13
+ let label: String
14
+ let identifier: String
15
+ let valueText: String?
16
+ let hittable: Bool
17
+ let visible: Bool
18
+ }
19
+
4
20
  // MARK: - Snapshot Entry
5
21
 
6
22
  func elementTypeName(_ type: XCUIElement.ElementType) -> String {
@@ -48,50 +64,19 @@ extension RunnerTests {
48
64
  return blocking
49
65
  }
50
66
 
51
- var nodes: [SnapshotNode] = []
52
- var truncated = false
53
- let maxDepth = options.depth ?? Int.max
54
- let viewport = app.frame
55
- let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
56
-
57
- let rootSnapshot: XCUIElementSnapshot
58
- do {
59
- rootSnapshot = try queryRoot.snapshot()
60
- } catch {
61
- return DataPayload(nodes: nodes, truncated: truncated)
67
+ guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
68
+ return DataPayload(nodes: [], truncated: false)
62
69
  }
63
70
 
64
- let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
65
- let rootLaterNodes = laterSnapshots(
66
- for: rootSnapshot,
67
- in: flatSnapshots,
68
- ranges: snapshotRanges
69
- )
70
- let rootLabel = aggregatedLabel(for: rootSnapshot) ?? rootSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
71
- let rootIdentifier = rootSnapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
72
- let rootValue = snapshotValueText(rootSnapshot)
73
- let rootHittable = computedSnapshotHittable(rootSnapshot, viewport: viewport, laterNodes: rootLaterNodes)
71
+ var nodes: [SnapshotNode] = []
72
+ var truncated = false
73
+ let rootEvaluation = evaluateSnapshot(context.rootSnapshot, in: context)
74
74
  nodes.append(
75
- SnapshotNode(
76
- index: 0,
77
- type: elementTypeName(rootSnapshot.elementType),
78
- label: rootLabel.isEmpty ? nil : rootLabel,
79
- identifier: rootIdentifier.isEmpty ? nil : rootIdentifier,
80
- value: rootValue,
81
- rect: SnapshotRect(
82
- x: Double(rootSnapshot.frame.origin.x),
83
- y: Double(rootSnapshot.frame.origin.y),
84
- width: Double(rootSnapshot.frame.size.width),
85
- height: Double(rootSnapshot.frame.size.height)
86
- ),
87
- enabled: rootSnapshot.isEnabled,
88
- hittable: rootHittable,
89
- depth: 0
90
- )
75
+ makeSnapshotNode(snapshot: context.rootSnapshot, evaluation: rootEvaluation, depth: 0, index: 0)
91
76
  )
92
77
 
93
78
  var seen = Set<String>()
94
- var stack: [(XCUIElementSnapshot, Int, Int)] = rootSnapshot.children.map { ($0, 1, 1) }
79
+ var stack: [(XCUIElementSnapshot, Int, Int)] = context.rootSnapshot.children.map { ($0, 1, 1) }
95
80
 
96
81
  while let (snapshot, depth, visibleDepth) = stack.popLast() {
97
82
  if nodes.count >= fastSnapshotLimit {
@@ -100,36 +85,24 @@ extension RunnerTests {
100
85
  }
101
86
  if let limit = options.depth, depth > limit { continue }
102
87
 
103
- let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
104
- let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
105
- let valueText = snapshotValueText(snapshot)
106
- let laterNodes = laterSnapshots(
107
- for: snapshot,
108
- in: flatSnapshots,
109
- ranges: snapshotRanges
110
- )
111
- let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
112
- let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
113
- if !isVisibleInViewport(snapshot.frame, viewport) && !hasContent {
114
- continue
115
- }
116
-
88
+ let evaluation = evaluateSnapshot(snapshot, in: context)
117
89
  let include = shouldInclude(
118
90
  snapshot: snapshot,
119
- label: label,
120
- identifier: identifier,
121
- valueText: valueText,
91
+ label: evaluation.label,
92
+ identifier: evaluation.identifier,
93
+ valueText: evaluation.valueText,
122
94
  options: options,
123
- hittable: hittable
95
+ hittable: evaluation.hittable,
96
+ visible: evaluation.visible
124
97
  )
125
98
 
126
- let key = "\(snapshot.elementType)-\(label)-\(identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
99
+ let key = "\(snapshot.elementType)-\(evaluation.label)-\(evaluation.identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
127
100
  let isDuplicate = seen.contains(key)
128
101
  if !isDuplicate {
129
102
  seen.insert(key)
130
103
  }
131
104
 
132
- if depth < maxDepth {
105
+ if depth < context.maxDepth {
133
106
  let nextVisibleDepth = include && !isDuplicate ? visibleDepth + 1 : visibleDepth
134
107
  for child in snapshot.children.reversed() {
135
108
  stack.append((child, depth + 1, nextVisibleDepth))
@@ -139,21 +112,11 @@ extension RunnerTests {
139
112
  if !include || isDuplicate { continue }
140
113
 
141
114
  nodes.append(
142
- SnapshotNode(
143
- index: nodes.count,
144
- type: elementTypeName(snapshot.elementType),
145
- label: label.isEmpty ? nil : label,
146
- identifier: identifier.isEmpty ? nil : identifier,
147
- value: valueText,
148
- rect: SnapshotRect(
149
- x: Double(snapshot.frame.origin.x),
150
- y: Double(snapshot.frame.origin.y),
151
- width: Double(snapshot.frame.size.width),
152
- height: Double(snapshot.frame.size.height)
153
- ),
154
- enabled: snapshot.isEnabled,
155
- hittable: hittable,
156
- depth: min(maxDepth, visibleDepth)
115
+ makeSnapshotNode(
116
+ snapshot: snapshot,
117
+ evaluation: evaluation,
118
+ depth: min(context.maxDepth, visibleDepth),
119
+ index: nodes.count
157
120
  )
158
121
  )
159
122
 
@@ -167,19 +130,12 @@ extension RunnerTests {
167
130
  return blocking
168
131
  }
169
132
 
170
- let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
171
- var nodes: [SnapshotNode] = []
172
- var truncated = false
173
- let viewport = app.frame
174
-
175
- let rootSnapshot: XCUIElementSnapshot
176
- do {
177
- rootSnapshot = try queryRoot.snapshot()
178
- } catch {
179
- return DataPayload(nodes: nodes, truncated: truncated)
133
+ guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
134
+ return DataPayload(nodes: [], truncated: false)
180
135
  }
181
136
 
182
- let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
137
+ var nodes: [SnapshotNode] = []
138
+ var truncated = false
183
139
 
184
140
  func walk(_ snapshot: XCUIElementSnapshot, depth: Int) {
185
141
  if nodes.count >= maxSnapshotElements {
@@ -187,37 +143,19 @@ extension RunnerTests {
187
143
  return
188
144
  }
189
145
  if let limit = options.depth, depth > limit { return }
190
- if !isVisibleInViewport(snapshot.frame, viewport) { return }
191
-
192
- let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
193
- let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
194
- let valueText = snapshotValueText(snapshot)
195
- let laterNodes = laterSnapshots(
196
- for: snapshot,
197
- in: flatSnapshots,
198
- ranges: snapshotRanges
199
- )
200
- let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
146
+
147
+ let evaluation = evaluateSnapshot(snapshot, in: context)
201
148
  if shouldInclude(
202
149
  snapshot: snapshot,
203
- label: label,
204
- identifier: identifier,
205
- valueText: valueText,
150
+ label: evaluation.label,
151
+ identifier: evaluation.identifier,
152
+ valueText: evaluation.valueText,
206
153
  options: options,
207
- hittable: hittable
154
+ hittable: evaluation.hittable,
155
+ visible: evaluation.visible
208
156
  ) {
209
157
  nodes.append(
210
- SnapshotNode(
211
- index: nodes.count,
212
- type: elementTypeName(snapshot.elementType),
213
- label: label.isEmpty ? nil : label,
214
- identifier: identifier.isEmpty ? nil : identifier,
215
- value: valueText,
216
- rect: snapshotRect(from: snapshot.frame),
217
- enabled: snapshot.isEnabled,
218
- hittable: hittable,
219
- depth: depth
220
- )
158
+ makeSnapshotNode(snapshot: snapshot, evaluation: evaluation, depth: depth, index: nodes.count)
221
159
  )
222
160
  }
223
161
 
@@ -228,7 +166,7 @@ extension RunnerTests {
228
166
  }
229
167
  }
230
168
 
231
- walk(rootSnapshot, depth: 0)
169
+ walk(context.rootSnapshot, depth: 0)
232
170
  return DataPayload(nodes: nodes, truncated: truncated)
233
171
  }
234
172
 
@@ -249,7 +187,8 @@ extension RunnerTests {
249
187
  identifier: String,
250
188
  valueText: String?,
251
189
  options: SnapshotOptions,
252
- hittable: Bool
190
+ hittable: Bool,
191
+ visible: Bool
253
192
  ) -> Bool {
254
193
  let type = snapshot.elementType
255
194
  let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
@@ -257,6 +196,11 @@ extension RunnerTests {
257
196
  if snapshot.children.count <= 1 { return false }
258
197
  }
259
198
  if options.interactiveOnly {
199
+ #if os(macOS)
200
+ if !visible && type != .application {
201
+ return false
202
+ }
203
+ #endif
260
204
  if interactiveTypes.contains(type) { return true }
261
205
  if hittable && type != .other { return true }
262
206
  if hasContent { return true }
@@ -287,6 +231,70 @@ extension RunnerTests {
287
231
  return true
288
232
  }
289
233
 
234
+ private func makeSnapshotTraversalContext(
235
+ app: XCUIApplication,
236
+ options: SnapshotOptions
237
+ ) -> SnapshotTraversalContext? {
238
+ let viewport = snapshotViewport(app: app)
239
+ let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
240
+
241
+ let rootSnapshot: XCUIElementSnapshot
242
+ do {
243
+ rootSnapshot = try queryRoot.snapshot()
244
+ } catch {
245
+ return nil
246
+ }
247
+
248
+ let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
249
+ return SnapshotTraversalContext(
250
+ rootSnapshot: rootSnapshot,
251
+ viewport: viewport,
252
+ flatSnapshots: flatSnapshots,
253
+ snapshotRanges: snapshotRanges,
254
+ maxDepth: options.depth ?? Int.max
255
+ )
256
+ }
257
+
258
+ private func evaluateSnapshot(
259
+ _ snapshot: XCUIElementSnapshot,
260
+ in context: SnapshotTraversalContext
261
+ ) -> SnapshotEvaluation {
262
+ let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
263
+ let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
264
+ let valueText = snapshotValueText(snapshot)
265
+ let laterNodes = laterSnapshots(
266
+ for: snapshot,
267
+ in: context.flatSnapshots,
268
+ ranges: context.snapshotRanges
269
+ )
270
+ return SnapshotEvaluation(
271
+ label: label,
272
+ identifier: identifier,
273
+ valueText: valueText,
274
+ hittable: computedSnapshotHittable(snapshot, viewport: context.viewport, laterNodes: laterNodes),
275
+ visible: isVisibleInViewport(snapshot.frame, context.viewport)
276
+ )
277
+ }
278
+
279
+ private func makeSnapshotNode(
280
+ snapshot: XCUIElementSnapshot,
281
+ evaluation: SnapshotEvaluation,
282
+ depth: Int,
283
+ index: Int
284
+ ) -> SnapshotNode {
285
+ return SnapshotNode(
286
+ index: index,
287
+ type: elementTypeName(snapshot.elementType),
288
+ label: evaluation.label.isEmpty ? nil : evaluation.label,
289
+ identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
290
+ value: evaluation.valueText,
291
+ rect: snapshotRect(from: snapshot.frame),
292
+ enabled: snapshot.isEnabled,
293
+ hittable: evaluation.hittable,
294
+ depth: depth
295
+ )
296
+ }
297
+
290
298
  private func isOccludingType(_ type: XCUIElement.ElementType) -> Bool {
291
299
  switch type {
292
300
  case .application, .window:
@@ -339,6 +347,18 @@ extension RunnerTests {
339
347
  return text.isEmpty ? nil : text
340
348
  }
341
349
 
350
+ private func snapshotViewport(app: XCUIApplication) -> CGRect {
351
+ let windows = app.windows.allElementsBoundByIndex
352
+ if let window = windows.first(where: { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty }) {
353
+ return window.frame
354
+ }
355
+ let appFrame = app.frame
356
+ if !appFrame.isNull && !appFrame.isEmpty {
357
+ return appFrame
358
+ }
359
+ return .infinite
360
+ }
361
+
342
362
  private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
343
363
  if depth > 4 { return nil }
344
364
  let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -4,6 +4,9 @@ extension RunnerTests {
4
4
  // MARK: - Blocking System Modal Snapshot
5
5
 
6
6
  func blockingSystemAlertSnapshot() -> DataPayload? {
7
+ #if os(macOS)
8
+ return nil
9
+ #else
7
10
  guard let modal = firstBlockingSystemModal(in: springboard) else {
8
11
  return nil
9
12
  }
@@ -40,6 +43,7 @@ extension RunnerTests {
40
43
  }
41
44
 
42
45
  return DataPayload(nodes: nodes, truncated: false)
46
+ #endif
43
47
  }
44
48
 
45
49
  private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
@@ -7,6 +7,13 @@
7
7
 
8
8
  import XCTest
9
9
  import Network
10
+ #if canImport(UIKit)
11
+ import UIKit
12
+ typealias RunnerImage = UIImage
13
+ #elseif canImport(AppKit)
14
+ import AppKit
15
+ typealias RunnerImage = NSImage
16
+ #endif
10
17
 
11
18
  final class RunnerTests: XCTestCase {
12
19
  enum RunnerErrorDomain {
@@ -85,11 +92,7 @@ final class RunnerTests: XCTestCase {
85
92
  let queue = DispatchQueue(label: "agent-device.runner")
86
93
  let desiredPort = RunnerEnv.resolvePort()
87
94
  NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", desiredPort)
88
- if desiredPort > 0, let port = NWEndpoint.Port(rawValue: desiredPort) {
89
- listener = try NWListener(using: .tcp, on: port)
90
- } else {
91
- listener = try NWListener(using: .tcp)
92
- }
95
+ listener = try makeRunnerListener(desiredPort: desiredPort)
93
96
  listener?.stateUpdateHandler = { [weak self] state in
94
97
  switch state {
95
98
  case .ready:
@@ -123,4 +126,18 @@ final class RunnerTests: XCTestCase {
123
126
  XCTFail("runner wait ended with \(result)")
124
127
  }
125
128
  }
129
+
130
+ private func makeRunnerListener(desiredPort: UInt16) throws -> NWListener {
131
+ if desiredPort > 0, let port = NWEndpoint.Port(rawValue: desiredPort) {
132
+ #if os(macOS)
133
+ let parameters = NWParameters.tcp
134
+ parameters.allowLocalEndpointReuse = true
135
+ parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: port)
136
+ return try NWListener(using: parameters)
137
+ #else
138
+ return try NWListener(using: .tcp, on: port)
139
+ #endif
140
+ }
141
+ return try NWListener(using: .tcp)
142
+ }
126
143
  }