agent-device 0.14.8 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/android-snapshot-helper/README.md +4 -2
- package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.14.8.apk → agent-device-android-snapshot-helper-0.15.0.apk} +0 -0
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.15.0.apk.sha256 +1 -0
- package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.14.8.manifest.json → agent-device-android-snapshot-helper-0.15.0.manifest.json} +6 -6
- package/dist/src/1769.js +7 -0
- package/dist/src/2151.js +429 -0
- package/dist/src/221.js +4 -4
- package/dist/src/2842.js +1 -0
- package/dist/src/3572.js +1 -0
- package/dist/src/4057.js +1 -1
- package/dist/src/840.js +2 -0
- package/dist/src/9542.js +2 -2
- package/dist/src/9639.js +2 -2
- package/dist/src/9818.js +1 -1
- package/dist/src/android-adb.d.ts +49 -11
- package/dist/src/android-adb.js +1 -1
- package/dist/src/android-snapshot-helper.d.ts +35 -2
- package/dist/src/cli.js +60 -57
- package/dist/src/contracts.d.ts +2 -0
- package/dist/src/finders.d.ts +2 -0
- package/dist/src/index.d.ts +25 -22
- package/dist/src/internal/companion-tunnel.js +1 -1
- package/dist/src/internal/daemon.js +51 -23
- package/dist/src/remote-config.d.ts +17 -14
- package/dist/src/selectors.d.ts +3 -0
- package/dist/src/server.js +2 -20
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +7 -1
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +210 -56
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +890 -99
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +94 -7
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +8 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +24 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +2 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +185 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +1 -2
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +26 -0
- package/package.json +25 -11
- package/server.json +3 -3
- package/skills/agent-device/SKILL.md +6 -1
- package/skills/dogfood/SKILL.md +3 -1
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.14.8.apk.sha256 +0 -1
- package/dist/src/180.js +0 -1
- package/dist/src/6108.js +0 -26
- package/dist/src/6642.js +0 -1
- package/dist/src/7462.js +0 -1
- package/dist/src/8809.js +0 -8
- package/dist/src/command-schema.js +0 -381
- package/skills/react-devtools/SKILL.md +0 -48
|
@@ -72,6 +72,21 @@ extension RunnerTests {
|
|
|
72
72
|
|
|
73
73
|
// MARK: - Target Activation
|
|
74
74
|
|
|
75
|
+
func ensureRunnerHostAppActive(reason: String) {
|
|
76
|
+
NSLog(
|
|
77
|
+
"AGENT_DEVICE_RUNNER_HOST_ACTIVATE state=%d reason=%@",
|
|
78
|
+
app.state.rawValue,
|
|
79
|
+
reason
|
|
80
|
+
)
|
|
81
|
+
if app.state == .unknown || app.state == .notRunning {
|
|
82
|
+
app.launch()
|
|
83
|
+
} else if app.state != .runningForeground {
|
|
84
|
+
app.activate()
|
|
85
|
+
}
|
|
86
|
+
currentApp = app
|
|
87
|
+
currentBundleId = nil
|
|
88
|
+
}
|
|
89
|
+
|
|
75
90
|
func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
|
|
76
91
|
let state = target.state
|
|
77
92
|
#if os(macOS)
|
|
@@ -88,6 +103,24 @@ extension RunnerTests {
|
|
|
88
103
|
return false
|
|
89
104
|
}
|
|
90
105
|
|
|
106
|
+
func canUseFastForegroundAppGuard(
|
|
107
|
+
activeApp: XCUIApplication,
|
|
108
|
+
requestedBundleId: String?,
|
|
109
|
+
command: CommandType
|
|
110
|
+
) -> Bool {
|
|
111
|
+
guard let requestedBundleId, currentBundleId == requestedBundleId, currentApp != nil else {
|
|
112
|
+
return false
|
|
113
|
+
}
|
|
114
|
+
guard activeApp.state == .runningForeground else { return false }
|
|
115
|
+
NSLog(
|
|
116
|
+
"AGENT_DEVICE_RUNNER_FAST_APP_GUARD command=%@ bundle=%@ state=%d",
|
|
117
|
+
String(describing: command),
|
|
118
|
+
requestedBundleId,
|
|
119
|
+
activeApp.state.rawValue
|
|
120
|
+
)
|
|
121
|
+
return true
|
|
122
|
+
}
|
|
123
|
+
|
|
91
124
|
func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
|
|
92
125
|
let target = XCUIApplication(bundleIdentifier: bundleId)
|
|
93
126
|
NSLog(
|
|
@@ -109,18 +142,53 @@ extension RunnerTests {
|
|
|
109
142
|
operation: () -> Void
|
|
110
143
|
) {
|
|
111
144
|
let setter = NSSelectorFromString("setWaitForIdleTimeout:")
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
145
|
+
let supportsWaitForIdleTimeout = target.responds(to: setter)
|
|
146
|
+
let previous = supportsWaitForIdleTimeout
|
|
147
|
+
? (target.value(forKey: "waitForIdleTimeout") as? NSNumber)
|
|
148
|
+
: nil
|
|
149
|
+
if supportsWaitForIdleTimeout {
|
|
150
|
+
target.setValue(resolveScrollInteractionIdleTimeout(), forKey: "waitForIdleTimeout")
|
|
115
151
|
}
|
|
116
|
-
let previous = target.value(forKey: "waitForIdleTimeout") as? NSNumber
|
|
117
|
-
target.setValue(resolveScrollInteractionIdleTimeout(), forKey: "waitForIdleTimeout")
|
|
118
152
|
defer {
|
|
119
153
|
if let previous {
|
|
120
154
|
target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout")
|
|
121
155
|
}
|
|
122
156
|
}
|
|
123
|
-
operation
|
|
157
|
+
performWithQuiescenceSkippedIfSupported(target, operation: operation)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Some apps never report post-gesture quiescence, even after XCTest has synthesized the event.
|
|
161
|
+
private func performWithQuiescenceSkippedIfSupported(
|
|
162
|
+
_ target: XCUIApplication,
|
|
163
|
+
operation: () -> Void
|
|
164
|
+
) {
|
|
165
|
+
let selector = NSSelectorFromString("_performWithInteractionOptions:block:")
|
|
166
|
+
guard target.responds(to: selector) else {
|
|
167
|
+
operation()
|
|
168
|
+
return
|
|
169
|
+
}
|
|
170
|
+
typealias PerformWithInteractionOptions = @convention(c) (
|
|
171
|
+
NSObject,
|
|
172
|
+
Selector,
|
|
173
|
+
UInt,
|
|
174
|
+
@convention(block) () -> Void
|
|
175
|
+
) -> Void
|
|
176
|
+
let implementation = target.method(for: selector)
|
|
177
|
+
let performWithOptions = unsafeBitCast(
|
|
178
|
+
implementation,
|
|
179
|
+
to: PerformWithInteractionOptions.self
|
|
180
|
+
)
|
|
181
|
+
let skipPreEventQuiescence = UInt(1)
|
|
182
|
+
let skipPostEventQuiescence = UInt(2)
|
|
183
|
+
withoutActuallyEscaping(operation) { escapableOperation in
|
|
184
|
+
let block: @convention(block) () -> Void = escapableOperation
|
|
185
|
+
performWithOptions(
|
|
186
|
+
target,
|
|
187
|
+
selector,
|
|
188
|
+
skipPreEventQuiescence | skipPostEventQuiescence,
|
|
189
|
+
block
|
|
190
|
+
)
|
|
191
|
+
}
|
|
124
192
|
}
|
|
125
193
|
|
|
126
194
|
private func resolveScrollInteractionIdleTimeout() -> TimeInterval {
|
|
@@ -184,6 +252,7 @@ extension RunnerTests {
|
|
|
184
252
|
.tap,
|
|
185
253
|
.longPress,
|
|
186
254
|
.drag,
|
|
255
|
+
.remotePress,
|
|
187
256
|
.type,
|
|
188
257
|
.swipe,
|
|
189
258
|
.back,
|
|
@@ -201,7 +270,7 @@ extension RunnerTests {
|
|
|
201
270
|
|
|
202
271
|
func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
|
|
203
272
|
switch command {
|
|
204
|
-
case .shutdown, .recordStop, .screenshot:
|
|
273
|
+
case .shutdown, .recordStop, .screenshot, .uptime:
|
|
205
274
|
return true
|
|
206
275
|
default:
|
|
207
276
|
return false
|
|
@@ -223,6 +292,24 @@ extension RunnerTests {
|
|
|
223
292
|
|
|
224
293
|
func sleepFor(_ delay: TimeInterval) {
|
|
225
294
|
guard delay > 0 else { return }
|
|
295
|
+
// Keep XCTest/UI sources moving during command-local pauses such as delayed typing.
|
|
296
|
+
if Thread.isMainThread {
|
|
297
|
+
let deadline = Date().addingTimeInterval(delay)
|
|
298
|
+
while Date() < deadline {
|
|
299
|
+
let slice = min(max(deadline.timeIntervalSinceNow, 0), 0.02)
|
|
300
|
+
if slice <= 0 {
|
|
301
|
+
break
|
|
302
|
+
}
|
|
303
|
+
let handledSource = RunLoop.current.run(
|
|
304
|
+
mode: .default,
|
|
305
|
+
before: Date().addingTimeInterval(slice)
|
|
306
|
+
)
|
|
307
|
+
if !handledSource {
|
|
308
|
+
usleep(useconds_t(slice * 1_000_000))
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
return
|
|
312
|
+
}
|
|
226
313
|
usleep(useconds_t(delay * 1_000_000))
|
|
227
314
|
}
|
|
228
315
|
}
|
|
@@ -8,9 +8,11 @@ enum CommandType: String, Codable {
|
|
|
8
8
|
case interactionFrame
|
|
9
9
|
case drag
|
|
10
10
|
case dragSeries
|
|
11
|
+
case remotePress
|
|
11
12
|
case type
|
|
12
13
|
case swipe
|
|
13
14
|
case findText
|
|
15
|
+
case querySelector
|
|
14
16
|
case readText
|
|
15
17
|
case snapshot
|
|
16
18
|
case screenshot
|
|
@@ -33,12 +35,16 @@ struct Command: Codable {
|
|
|
33
35
|
let command: CommandType
|
|
34
36
|
let appBundleId: String?
|
|
35
37
|
let text: String?
|
|
38
|
+
let selectorKey: String?
|
|
39
|
+
let selectorValue: String?
|
|
36
40
|
let delayMs: Int?
|
|
41
|
+
let textEntryMode: String?
|
|
37
42
|
let clearFirst: Bool?
|
|
38
43
|
let action: String?
|
|
39
44
|
let x: Double?
|
|
40
45
|
let y: Double?
|
|
41
46
|
let button: String?
|
|
47
|
+
let remoteButton: String?
|
|
42
48
|
let count: Double?
|
|
43
49
|
let intervalMs: Double?
|
|
44
50
|
let doubleTap: Bool?
|
|
@@ -162,6 +168,8 @@ struct SnapshotNode: Codable {
|
|
|
162
168
|
let value: String?
|
|
163
169
|
let rect: SnapshotRect
|
|
164
170
|
let enabled: Bool
|
|
171
|
+
let focused: Bool?
|
|
172
|
+
let selected: Bool?
|
|
165
173
|
let hittable: Bool
|
|
166
174
|
let depth: Int
|
|
167
175
|
let parentIndex: Int?
|
|
@@ -28,6 +28,8 @@ extension RunnerTests {
|
|
|
28
28
|
let identifier: String
|
|
29
29
|
let valueText: String?
|
|
30
30
|
let hittable: Bool
|
|
31
|
+
let focused: Bool
|
|
32
|
+
let selected: Bool
|
|
31
33
|
let visible: Bool
|
|
32
34
|
}
|
|
33
35
|
|
|
@@ -341,6 +343,8 @@ extension RunnerTests {
|
|
|
341
343
|
identifier: identifier,
|
|
342
344
|
valueText: valueText,
|
|
343
345
|
hittable: computedSnapshotHittable(snapshot, viewport: context.viewport, laterNodes: laterNodes),
|
|
346
|
+
focused: snapshotHasFocus(snapshot),
|
|
347
|
+
selected: snapshotIsSelected(snapshot),
|
|
344
348
|
visible: isVisibleInViewport(snapshot.frame, context.viewport)
|
|
345
349
|
)
|
|
346
350
|
}
|
|
@@ -360,6 +364,8 @@ extension RunnerTests {
|
|
|
360
364
|
value: evaluation.valueText,
|
|
361
365
|
rect: snapshotRect(from: snapshot.frame),
|
|
362
366
|
enabled: snapshot.isEnabled,
|
|
367
|
+
focused: evaluation.focused ? true : nil,
|
|
368
|
+
selected: evaluation.selected ? true : nil,
|
|
363
369
|
hittable: evaluation.hittable,
|
|
364
370
|
depth: depth,
|
|
365
371
|
parentIndex: parentIndex,
|
|
@@ -525,6 +531,8 @@ extension RunnerTests {
|
|
|
525
531
|
value: node.value,
|
|
526
532
|
rect: node.rect,
|
|
527
533
|
enabled: node.enabled,
|
|
534
|
+
focused: node.focused,
|
|
535
|
+
selected: node.selected,
|
|
528
536
|
hittable: node.hittable,
|
|
529
537
|
depth: depth,
|
|
530
538
|
parentIndex: parentIndex,
|
|
@@ -575,6 +583,8 @@ extension RunnerTests {
|
|
|
575
583
|
value: valueText,
|
|
576
584
|
rect: snapshotRect(from: frame),
|
|
577
585
|
enabled: element.isEnabled,
|
|
586
|
+
focused: elementHasFocus(element) ? true : nil,
|
|
587
|
+
selected: element.isSelected ? true : nil,
|
|
578
588
|
hittable: element.isHittable,
|
|
579
589
|
depth: 0,
|
|
580
590
|
parentIndex: nil,
|
|
@@ -592,6 +602,20 @@ extension RunnerTests {
|
|
|
592
602
|
return node
|
|
593
603
|
}
|
|
594
604
|
|
|
605
|
+
private func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool {
|
|
606
|
+
var focused = false
|
|
607
|
+
_ = RunnerObjCExceptionCatcher.catchException({
|
|
608
|
+
if let value = (snapshot as! NSObject).value(forKey: "hasFocus") as? Bool {
|
|
609
|
+
focused = value
|
|
610
|
+
}
|
|
611
|
+
})
|
|
612
|
+
return focused
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
private func snapshotIsSelected(_ snapshot: XCUIElementSnapshot) -> Bool {
|
|
616
|
+
return snapshot.isSelected
|
|
617
|
+
}
|
|
618
|
+
|
|
595
619
|
private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool {
|
|
596
620
|
let frame = snapshot.frame
|
|
597
621
|
if frame.isNull || frame.isEmpty { return false }
|
|
@@ -186,6 +186,8 @@ extension RunnerTests {
|
|
|
186
186
|
value: nil,
|
|
187
187
|
rect: snapshotRect(from: element.frame),
|
|
188
188
|
enabled: element.isEnabled,
|
|
189
|
+
focused: elementHasFocus(element) ? true : nil,
|
|
190
|
+
selected: element.isSelected ? true : nil,
|
|
189
191
|
hittable: hittableOverride ?? element.isHittable,
|
|
190
192
|
depth: depth,
|
|
191
193
|
parentIndex: nil,
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
enum RunnerInteractionOutcome {
|
|
4
|
+
case performed
|
|
5
|
+
case unsupported(String)
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
enum TvRemoteButton {
|
|
9
|
+
case select
|
|
10
|
+
case menu
|
|
11
|
+
case home
|
|
12
|
+
case up
|
|
13
|
+
case down
|
|
14
|
+
case left
|
|
15
|
+
case right
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
extension RunnerTests {
|
|
19
|
+
func resolveTvRemoteDoublePressDelay() -> TimeInterval {
|
|
20
|
+
guard
|
|
21
|
+
let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],
|
|
22
|
+
!raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
23
|
+
else {
|
|
24
|
+
return tvRemoteDoublePressDelayDefault
|
|
25
|
+
}
|
|
26
|
+
guard let parsedMs = Double(raw), parsedMs >= 0 else {
|
|
27
|
+
return tvRemoteDoublePressDelayDefault
|
|
28
|
+
}
|
|
29
|
+
return min(parsedMs, 1000) / 1000.0
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
@discardableResult
|
|
33
|
+
func pressTvRemote(_ button: TvRemoteButton, duration: TimeInterval? = nil) -> Bool {
|
|
34
|
+
#if os(tvOS)
|
|
35
|
+
let remoteButton = xcuiRemoteButton(button)
|
|
36
|
+
if let duration, duration > 0 {
|
|
37
|
+
XCUIRemote.shared.press(remoteButton, forDuration: duration)
|
|
38
|
+
} else {
|
|
39
|
+
XCUIRemote.shared.press(remoteButton)
|
|
40
|
+
}
|
|
41
|
+
return true
|
|
42
|
+
#else
|
|
43
|
+
return false
|
|
44
|
+
#endif
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
func tvRemoteButton(from raw: String?) -> TvRemoteButton? {
|
|
48
|
+
switch raw?.lowercased() {
|
|
49
|
+
case "select":
|
|
50
|
+
return .select
|
|
51
|
+
case "menu":
|
|
52
|
+
return .menu
|
|
53
|
+
case "home":
|
|
54
|
+
return .home
|
|
55
|
+
case "up":
|
|
56
|
+
return .up
|
|
57
|
+
case "down":
|
|
58
|
+
return .down
|
|
59
|
+
case "left":
|
|
60
|
+
return .left
|
|
61
|
+
case "right":
|
|
62
|
+
return .right
|
|
63
|
+
default:
|
|
64
|
+
return nil
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
func elementHasFocus(_ element: XCUIElement) -> Bool {
|
|
69
|
+
var focused = false
|
|
70
|
+
_ = RunnerObjCExceptionCatcher.catchException({
|
|
71
|
+
if let value = (element as NSObject).value(forKey: "hasFocus") as? Bool {
|
|
72
|
+
focused = value
|
|
73
|
+
}
|
|
74
|
+
})
|
|
75
|
+
return focused
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
func activateElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome {
|
|
79
|
+
if let outcome = selectFocusedTvElement(app: app, element: element, action: action) {
|
|
80
|
+
return outcome
|
|
81
|
+
}
|
|
82
|
+
return performElementTap(element)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
func selectFocusedTvElement(app: XCUIApplication, point: CGPoint, action: String) -> RunnerInteractionOutcome? {
|
|
86
|
+
#if os(tvOS)
|
|
87
|
+
guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
|
|
88
|
+
return .unsupported("\(action) is supported on tvOS only when the requested point is inside the focused element")
|
|
89
|
+
}
|
|
90
|
+
_ = pressTvRemote(.select)
|
|
91
|
+
return .performed
|
|
92
|
+
#else
|
|
93
|
+
return nil
|
|
94
|
+
#endif
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
func longSelectFocusedTvElement(app: XCUIApplication, point: CGPoint, duration: TimeInterval) -> RunnerInteractionOutcome? {
|
|
98
|
+
#if os(tvOS)
|
|
99
|
+
guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
|
|
100
|
+
return .unsupported("long press is supported on tvOS only when the requested point is inside the focused element")
|
|
101
|
+
}
|
|
102
|
+
_ = pressTvRemote(.select, duration: duration)
|
|
103
|
+
return .performed
|
|
104
|
+
#else
|
|
105
|
+
return nil
|
|
106
|
+
#endif
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private func performElementTap(_ element: XCUIElement) -> RunnerInteractionOutcome {
|
|
110
|
+
#if os(tvOS)
|
|
111
|
+
return .unsupported("element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
|
|
112
|
+
#else
|
|
113
|
+
element.tap()
|
|
114
|
+
return .performed
|
|
115
|
+
#endif
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private func selectFocusedTvElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome? {
|
|
119
|
+
#if os(tvOS)
|
|
120
|
+
guard tvFocusedElementMatches(app: app, target: element) else {
|
|
121
|
+
return .unsupported("\(action) is supported on tvOS only when the requested element is focused")
|
|
122
|
+
}
|
|
123
|
+
_ = pressTvRemote(.select)
|
|
124
|
+
return .performed
|
|
125
|
+
#else
|
|
126
|
+
return nil
|
|
127
|
+
#endif
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private func tvFocusedElementMatches(app: XCUIApplication, target: XCUIElement) -> Bool {
|
|
131
|
+
#if os(tvOS)
|
|
132
|
+
if target.hasFocus {
|
|
133
|
+
return true
|
|
134
|
+
}
|
|
135
|
+
guard let focused = focusedTvElement(app: app) else {
|
|
136
|
+
return false
|
|
137
|
+
}
|
|
138
|
+
let targetFrame = target.frame
|
|
139
|
+
let focusedFrame = focused.frame
|
|
140
|
+
guard !targetFrame.isEmpty && !focusedFrame.isEmpty else {
|
|
141
|
+
return false
|
|
142
|
+
}
|
|
143
|
+
let focusedCenter = CGPoint(x: focusedFrame.midX, y: focusedFrame.midY)
|
|
144
|
+
let targetCenter = CGPoint(x: targetFrame.midX, y: targetFrame.midY)
|
|
145
|
+
return targetFrame.contains(focusedCenter)
|
|
146
|
+
|| focusedFrame.contains(targetCenter)
|
|
147
|
+
|| targetFrame.intersects(focusedFrame)
|
|
148
|
+
#else
|
|
149
|
+
return false
|
|
150
|
+
#endif
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private func focusedTvElement(app: XCUIApplication) -> XCUIElement? {
|
|
154
|
+
#if os(tvOS)
|
|
155
|
+
let focused = app
|
|
156
|
+
.descendants(matching: .any)
|
|
157
|
+
.matching(NSPredicate(format: "hasFocus == true"))
|
|
158
|
+
.firstMatch
|
|
159
|
+
return focused.exists ? focused : nil
|
|
160
|
+
#else
|
|
161
|
+
return nil
|
|
162
|
+
#endif
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
#if os(tvOS)
|
|
166
|
+
private func xcuiRemoteButton(_ button: TvRemoteButton) -> XCUIRemote.Button {
|
|
167
|
+
switch button {
|
|
168
|
+
case .select:
|
|
169
|
+
return .select
|
|
170
|
+
case .menu:
|
|
171
|
+
return .menu
|
|
172
|
+
case .home:
|
|
173
|
+
return .home
|
|
174
|
+
case .up:
|
|
175
|
+
return .up
|
|
176
|
+
case .down:
|
|
177
|
+
return .down
|
|
178
|
+
case .left:
|
|
179
|
+
return .left
|
|
180
|
+
case .right:
|
|
181
|
+
return .right
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
#endif
|
|
185
|
+
}
|
|
@@ -90,8 +90,7 @@ final class RunnerTests: XCTestCase {
|
|
|
90
90
|
@MainActor
|
|
91
91
|
func testCommand() throws {
|
|
92
92
|
doneExpectation = expectation(description: "agent-device command handled")
|
|
93
|
-
|
|
94
|
-
currentApp = app
|
|
93
|
+
NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1")
|
|
95
94
|
let queue = DispatchQueue(label: "agent-device.runner")
|
|
96
95
|
let desiredPort = RunnerEnv.resolvePort()
|
|
97
96
|
NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", desiredPort)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"configurations" : [
|
|
3
|
+
{
|
|
4
|
+
"id" : "916C7049-FE79-4C78-B55F-79242F92CB19",
|
|
5
|
+
"name" : "Configuration 1",
|
|
6
|
+
"options" : {
|
|
7
|
+
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"defaultOptions" : {
|
|
12
|
+
"preferredScreenCaptureFormat" : "screenshots",
|
|
13
|
+
"systemAttachmentLifetime" : "keepNever",
|
|
14
|
+
"userAttachmentLifetime" : "keepNever"
|
|
15
|
+
},
|
|
16
|
+
"testTargets" : [
|
|
17
|
+
{
|
|
18
|
+
"target" : {
|
|
19
|
+
"containerPath" : "container:AgentDeviceRunner.xcodeproj",
|
|
20
|
+
"identifier" : "20EA2EDC2F2CFC7C001CF0EF",
|
|
21
|
+
"name" : "AgentDeviceRunnerUITests"
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
],
|
|
25
|
+
"version" : 1
|
|
26
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-device",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Agent-native CLI for AI mobile testing and app automation across iOS, Android, tvOS, Android TV, macOS, and Linux.",
|
|
5
5
|
"mcpName": "io.github.callstackincubator/agent-device",
|
|
6
6
|
"license": "MIT",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"url": "https://github.com/callstackincubator/agent-device/issues"
|
|
15
15
|
},
|
|
16
16
|
"type": "module",
|
|
17
|
-
"packageManager": "pnpm@
|
|
17
|
+
"packageManager": "pnpm@11.1.2",
|
|
18
18
|
"main": "dist/src/index.js",
|
|
19
19
|
"types": "dist/src/index.d.ts",
|
|
20
20
|
"exports": {
|
|
@@ -76,11 +76,17 @@
|
|
|
76
76
|
"scripts": {
|
|
77
77
|
"build": "rslib build",
|
|
78
78
|
"clean:daemon": "rm -f ~/.agent-device/daemon.json && rm -f ~/.agent-device/daemon.lock",
|
|
79
|
+
"clean:xcuitest": "node scripts/clean-xcuitest-derived.mjs",
|
|
80
|
+
"clean:xcuitest:ios": "node scripts/clean-xcuitest-derived.mjs ios",
|
|
81
|
+
"clean:xcuitest:macos": "node scripts/clean-xcuitest-derived.mjs macos",
|
|
82
|
+
"clean:xcuitest:tvos": "node scripts/clean-xcuitest-derived.mjs tvos",
|
|
79
83
|
"build:node": "pnpm build && pnpm clean:daemon",
|
|
80
84
|
"build:xcuitest": "pnpm build:xcuitest:ios && pnpm build:xcuitest:macos",
|
|
81
|
-
"build:xcuitest:ios": "AGENT_DEVICE_XCUITEST_PLATFORM=ios
|
|
85
|
+
"build:xcuitest:ios": "AGENT_DEVICE_XCUITEST_PLATFORM=ios sh ./scripts/build-xcuitest-apple.sh",
|
|
86
|
+
"build:xcuitest:ios:clean": "pnpm clean:xcuitest:ios && pnpm build:xcuitest:ios",
|
|
82
87
|
"build:xcuitest:macos": "AGENT_DEVICE_XCUITEST_PLATFORM=macos sh ./scripts/build-xcuitest-apple.sh",
|
|
83
|
-
"build:xcuitest:tvos": "AGENT_DEVICE_XCUITEST_PLATFORM=tvos
|
|
88
|
+
"build:xcuitest:tvos": "AGENT_DEVICE_XCUITEST_PLATFORM=tvos sh ./scripts/build-xcuitest-apple.sh",
|
|
89
|
+
"build:xcuitest:tvos:clean": "pnpm clean:xcuitest:tvos && pnpm build:xcuitest:tvos",
|
|
84
90
|
"build:android-snapshot-helper": "sh ./scripts/build-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
|
|
85
91
|
"package:android-snapshot-helper": "sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
|
|
86
92
|
"package:android-snapshot-helper:npm": "rm -rf android-snapshot-helper/dist && sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") android-snapshot-helper/dist",
|
|
@@ -90,7 +96,7 @@
|
|
|
90
96
|
"lint": "oxlint . --deny-warnings",
|
|
91
97
|
"format": "oxfmt --write src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
|
|
92
98
|
"fallow": "fallow --summary",
|
|
93
|
-
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow
|
|
99
|
+
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",
|
|
94
100
|
"check:fallow": "fallow audit",
|
|
95
101
|
"check:quick": "pnpm lint && pnpm typecheck",
|
|
96
102
|
"sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",
|
|
@@ -105,11 +111,16 @@
|
|
|
105
111
|
"test-app:ios": "pnpm --dir examples/test-app ios",
|
|
106
112
|
"test-app:android": "pnpm --dir examples/test-app android",
|
|
107
113
|
"test-app:typecheck": "pnpm --dir examples/test-app typecheck",
|
|
108
|
-
"test": "vitest run",
|
|
109
|
-
"test:unit": "vitest run",
|
|
114
|
+
"test": "vitest run --project unit",
|
|
115
|
+
"test:unit": "vitest run --project unit",
|
|
116
|
+
"test:coverage": "vitest run --coverage",
|
|
117
|
+
"test:integration:provider": "vitest run --project provider-integration",
|
|
118
|
+
"test:integration:progress": "node scripts/integration-progress.mjs",
|
|
119
|
+
"test:integration:progress:check": "node scripts/integration-progress.mjs --check",
|
|
110
120
|
"test:skillgym": "pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts",
|
|
111
121
|
"test:smoke": "node --test test/integration/smoke-*.test.ts",
|
|
112
|
-
"test:integration": "node --test test/integration/*.test.ts",
|
|
122
|
+
"test:integration:node": "node --test test/integration/*.test.ts",
|
|
123
|
+
"test:integration": "pnpm test:integration:node && pnpm test:integration:provider",
|
|
113
124
|
"test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator",
|
|
114
125
|
"test:replay:ios-device": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device",
|
|
115
126
|
"test:replay:android": "node --experimental-strip-types src/bin.ts test test/integration/replays/android",
|
|
@@ -183,18 +194,21 @@
|
|
|
183
194
|
],
|
|
184
195
|
"dependencies": {
|
|
185
196
|
"fast-xml-parser": "^5.7.2",
|
|
186
|
-
"pngjs": "^7.0.0"
|
|
197
|
+
"pngjs": "^7.0.0",
|
|
198
|
+
"yaml": "^2.9.0"
|
|
187
199
|
},
|
|
188
200
|
"devDependencies": {
|
|
189
|
-
"
|
|
201
|
+
"@microsoft/api-extractor": "^7.58.7",
|
|
190
202
|
"@rslib/core": "0.20.1",
|
|
191
203
|
"@types/node": "^22.0.0",
|
|
192
204
|
"@types/pngjs": "^6.0.5",
|
|
205
|
+
"@vitest/coverage-v8": "4.1.2",
|
|
193
206
|
"fallow": "^2.52.0",
|
|
194
207
|
"oxfmt": "^0.42.0",
|
|
195
208
|
"oxlint": "^1.57.0",
|
|
196
209
|
"skillgym": "^0.8.0",
|
|
197
210
|
"typescript": "^6.0.2",
|
|
198
|
-
"vite": "^8.0.10"
|
|
211
|
+
"vite": "^8.0.10",
|
|
212
|
+
"vitest": "^4.1.2"
|
|
199
213
|
}
|
|
200
214
|
}
|
package/server.json
CHANGED
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
3
|
"name": "io.github.callstackincubator/agent-device",
|
|
4
4
|
"title": "agent-device",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "Let AI agents inspect, control, and debug real iOS, Android, desktop, and TV apps",
|
|
6
6
|
"repository": {
|
|
7
7
|
"url": "https://github.com/callstackincubator/agent-device",
|
|
8
8
|
"source": "github"
|
|
9
9
|
},
|
|
10
|
-
"version": "0.
|
|
10
|
+
"version": "0.15.0",
|
|
11
11
|
"packages": [
|
|
12
12
|
{
|
|
13
13
|
"registryType": "npm",
|
|
14
14
|
"identifier": "agent-device",
|
|
15
|
-
"version": "0.
|
|
15
|
+
"version": "0.15.0",
|
|
16
16
|
"transport": {
|
|
17
17
|
"type": "stdio"
|
|
18
18
|
}
|
|
@@ -11,7 +11,9 @@ Router only. Private setup before using this skill:
|
|
|
11
11
|
agent-device --version
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
If that fails but the user may have installed `agent-device` globally, check the user's configured login/interactive shell and environment before using `npx`. Resolve the command the same way the user would from a normal terminal session, then run the absolute binary path if found. This may require inspecting shell startup behavior or package-manager/global bin locations; do not assume the Codex process `PATH` is the user's `PATH`.
|
|
15
|
+
|
|
16
|
+
Require `agent-device >= 0.14.0`; older CLIs lack these help topics. If older, stop and tell the user to upgrade the trusted install or approve an exact-version npm command. Do not run `npm install -g agent-device@latest` or `npx -y agent-device@latest` autonomously, and do not include version/upgrade commands in final plans.
|
|
15
17
|
|
|
16
18
|
Before your first agent-device command or plan, read the version-matched CLI guide:
|
|
17
19
|
|
|
@@ -23,6 +25,7 @@ Escalate only when relevant:
|
|
|
23
25
|
|
|
24
26
|
```bash
|
|
25
27
|
agent-device help debugging
|
|
28
|
+
agent-device help react-native
|
|
26
29
|
agent-device help react-devtools
|
|
27
30
|
agent-device help remote
|
|
28
31
|
agent-device help macos
|
|
@@ -32,3 +35,5 @@ agent-device help dogfood
|
|
|
32
35
|
Default loop: `open -> snapshot/-i -> get/is/find or press/fill/scroll/wait -> verify -> close`.
|
|
33
36
|
|
|
34
37
|
Use this skill only to route into version-matched CLI help. Let `help workflow` provide exact command shapes, platform limits, and current workflow guidance.
|
|
38
|
+
|
|
39
|
+
For precise location workflows, read the installed `settings` help before planning so coordinate support and platform limits come from the active CLI version.
|
package/skills/dogfood/SKILL.md
CHANGED
|
@@ -12,7 +12,9 @@ Router for exploratory QA. Private setup before using this skill:
|
|
|
12
12
|
agent-device --version
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
If that fails, stop and tell the user to expose a trusted `agent-device` binary on PATH or approve an exact-version npm command. This skill intentionally keeps allowed tools restricted to `agent-device` and `npx agent-device`.
|
|
16
|
+
|
|
17
|
+
Require `agent-device >= 0.14.0`; older CLIs lack these help topics. If older, stop and tell the user to upgrade the trusted install or approve an exact-version npm command. Do not run `npm install -g agent-device@latest` or `npx -y agent-device@latest` autonomously, and do not include version/upgrade commands in final plans.
|
|
16
18
|
|
|
17
19
|
Read current CLI guidance:
|
|
18
20
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0669bbeb4c3b549a9084dc3d75e4afa8f055408424c40c2fd9db4b75eb1f6e53 agent-device-android-snapshot-helper-0.14.8.apk
|
package/dist/src/180.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import e from"node:path";import t from"node:crypto";import n from"node:fs";import{resolveUserPath as a,expandUserHomePath as i}from"./3267.js";import{findProjectRoot as r}from"./9671.js";function o(t){let n,r=(n=(t??"").trim())?a(n):e.join(i("~"),".agent-device");return{baseDir:r,infoPath:e.join(r,"daemon.json"),lockPath:e.join(r,"daemon.lock"),logPath:e.join(r,"daemon.log"),sessionsDir:e.join(r,"sessions")}}function s(e){let t=(e??"").trim().toLowerCase();return"http"===t?"http":"dual"===t?"dual":"socket"}function l(e){let t=(e??"").trim().toLowerCase();return"auto"===t?"auto":"socket"===t?"socket":"http"===t?"http":"auto"}function d(e){return"tenant"===(e??"").trim().toLowerCase()?"tenant":"none"}function u(e){if(!e)return;let t=e.trim();if(t&&/^[a-zA-Z0-9._-]{1,128}$/.test(t))return t}let p=/(?:^|[^\w$.])(?:import|export)\s+(?:type\s+)?(?:[^'"`]*?\s+from\s+)?['"]([^'"]+)['"]/gm,c=/import\(\s*['"]([^'"]+)['"]\s*\)/gm,m=[".ts",".tsx",".js",".jsx",".mjs",".cjs"];function f(){let e=process.argv[1];return e?h(e):"unknown"}function h(a,i=r()){try{let r=e.resolve(i),o=[e.resolve(a)],s=new Set,l=[];for(;o.length>0;){let t=o.pop();if(!t||s.has(t))continue;s.add(t);let a=n.statSync(t);if(!a.isFile())continue;let i=e.relative(r,t)||t;l.push(`${i}:${a.size}:${Math.trunc(a.mtimeMs)}`);let d=n.readFileSync(t,"utf8");for(let n of function(e){let t=new Set;return g(e,p,t),g(e,c,t),[...t]}(d)){let a=function(t,n){let a=e.resolve(e.dirname(t),n),i=I(a);if(i)return i;for(let e of m){let t=I(`${a}${e}`);if(t)return t}for(let t of m){let n=I(e.join(a,`index${t}`));if(n)return n}return null}(t,n);a&&o.push(a)}}let d=l.sort().join("|"),u=t.createHash("sha1").update(d).digest("hex");return`graph:${l.length}:${u}`}catch{return"unknown"}}function g(e,t,n){t.lastIndex=0;let a=null;for(;null!==(a=t.exec(e));){let e=a[1]?.trim();e?.startsWith(".")&&n.add(e)}}function I(e){try{return n.statSync(e).isFile()?e:null}catch{return null}}function v(e){return e?{message:e}:{}}function S(e,t){return t?{...e,message:t}:e}function b(e){return"string"==typeof e?.message&&e.message.length>0?e.message:null}function k(e){let t=e.appId??e.bundleId??e.packageName;return{session:e.session,appId:t,appBundleId:e.bundleId,package:e.packageName}}function N(e,t,n){return{deviceId:t,deviceName:n,..."android"===e?{serial:t}:"ios"===e?{udid:t}:{}}}function $(e,t={}){let n=t.includeAndroidSerial??!0;return{platform:e.platform,target:e.target,device:e.name,id:e.id,..."ios"===e.platform?{device_udid:e.ios?.udid??e.id,ios_simulator_device_set:e.ios?.simulatorSetPath??null}:{},..."android"===e.platform&&n?{serial:e.android?.serial??e.id}:{}}}function _(e){return{name:e.name,...$(e.device,{includeAndroidSerial:!1}),createdAt:e.createdAt}}function z(e){return{platform:e.platform,id:e.id,name:e.name,kind:e.kind,target:e.target,..."boolean"==typeof e.booted?{booted:e.booted}:{}}}function w(e){let t=e.created?"Created":"Reused",n=e.booted?" (booted)":"";return S({udid:e.udid,device:e.device,runtime:e.runtime,ios_simulator_device_set:e.iosSimulatorDeviceSet??null,created:e.created,booted:e.booted},`${t}: ${e.device} ${e.udid}${n}`)}function P(e){return e.bundleId??e.package??e.app}function y(e){return S({app:e.app,appPath:e.appPath,platform:e.platform,...e.appId?{appId:e.appId}:{},...e.bundleId?{bundleId:e.bundleId}:{},...e.package?{package:e.package}:{}},`Installed: ${P(e)}`)}function C(e){return e.appName??e.bundleId??e.packageName??e.launchTarget}function x(e){return S({launchTarget:e.launchTarget,...e.appName?{appName:e.appName}:{},...e.appId?{appId:e.appId}:{},...e.bundleId?{bundleId:e.bundleId}:{},...e.packageName?{package:e.packageName}:{},...e.installablePath?{installablePath:e.installablePath}:{},...e.archivePath?{archivePath:e.archivePath}:{},...e.materializationId?{materializationId:e.materializationId}:{},...e.materializationExpiresAt?{materializationExpiresAt:e.materializationExpiresAt}:{}},`Installed: ${C(e)}`)}function j(e){let t=e.appName??e.appBundleId??e.session;return S({session:e.session,...e.appName?{appName:e.appName}:{},...e.appBundleId?{appBundleId:e.appBundleId}:{},...e.startup?{startup:e.startup}:{},...e.runtime?{runtime:e.runtime}:{},...e.device?$(e.device):{}},t?`Opened: ${t}`:"Opened")}function A(e){return{session:e.session,...e.shutdown?{shutdown:e.shutdown}:{},...v(e.session?`Closed: ${e.session}`:"Closed")}}function D(e){return{nodes:e.nodes,truncated:e.truncated,...e.appName?{appName:e.appName}:{},...e.appBundleId?{appBundleId:e.appBundleId}:{},...e.visibility?{visibility:e.visibility}:{},...e.androidSnapshot?{androidSnapshot:e.androidSnapshot}:{},...e.warnings&&e.warnings.length>0?{warnings:e.warnings}:{}}}export{k as buildAppIdentifiers,N as buildDeviceIdentifiers,h as computeDaemonCodeSignature,u as normalizeTenantId,b as readCommandMessage,f as resolveDaemonCodeSignature,o as resolveDaemonPaths,s as resolveDaemonServerMode,l as resolveDaemonTransportPreference,P as resolveDeployResultTarget,C as resolveInstallFromSourceResultTarget,d as resolveSessionIsolationMode,A as serializeCloseResult,y as serializeDeployResult,z as serializeDevice,w as serializeEnsureSimulatorResult,x as serializeInstallFromSourceResult,j as serializeOpenResult,_ as serializeSessionListEntry,D as serializeSnapshotResult,v as successText,S as withSuccessText};
|