agent-device 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +99 -0
- package/bin/agent-device.mjs +14 -0
- package/bin/axsnapshot +0 -0
- package/dist/src/861.js +1 -0
- package/dist/src/bin.js +50 -0
- package/dist/src/daemon.js +5 -0
- package/ios-runner/AXSnapshot/Package.swift +18 -0
- package/ios-runner/AXSnapshot/Sources/AXSnapshot/main.swift +167 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.swift +17 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/AccentColor.colorset/Contents.json +11 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/AppIcon.appiconset/Contents.json +36 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/AppIcon.appiconset/logo.jpg +0 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/Contents.json +6 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/Logo.imageset/Contents.json +21 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/Logo.imageset/logo.jpg +0 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/PoweredBy.imageset/Contents.json +21 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/Assets.xcassets/PoweredBy.imageset/powered-by.png +0 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner/ContentView.swift +34 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +461 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +102 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +696 -0
- package/ios-runner/README.md +11 -0
- package/package.json +66 -0
- package/src/bin.ts +3 -0
- package/src/cli.ts +160 -0
- package/src/core/dispatch.ts +259 -0
- package/src/daemon-client.ts +166 -0
- package/src/daemon.ts +842 -0
- package/src/platforms/android/devices.ts +59 -0
- package/src/platforms/android/index.ts +442 -0
- package/src/platforms/ios/ax-snapshot.ts +154 -0
- package/src/platforms/ios/devices.ts +65 -0
- package/src/platforms/ios/index.ts +218 -0
- package/src/platforms/ios/runner-client.ts +534 -0
- package/src/utils/args.ts +175 -0
- package/src/utils/device.ts +84 -0
- package/src/utils/errors.ts +35 -0
- package/src/utils/exec.ts +229 -0
- package/src/utils/interactive.ts +4 -0
- package/src/utils/interactors.ts +72 -0
- package/src/utils/output.ts +146 -0
- package/src/utils/snapshot.ts +63 -0
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Untitled.swift
|
|
3
|
+
// AgentDeviceRunner
|
|
4
|
+
//
|
|
5
|
+
// Created by Michał Pierzchała on 30/01/2026.
|
|
6
|
+
//
|
|
7
|
+
|
|
8
|
+
import XCTest
|
|
9
|
+
import Network
|
|
10
|
+
|
|
11
|
+
final class RunnerTests: XCTestCase {
|
|
12
|
+
private var listener: NWListener?
|
|
13
|
+
private var port: UInt16 = 0
|
|
14
|
+
private var doneExpectation: XCTestExpectation?
|
|
15
|
+
private let app = XCUIApplication()
|
|
16
|
+
private var currentApp: XCUIApplication?
|
|
17
|
+
private var currentBundleId: String?
|
|
18
|
+
private let maxRequestBytes = 2 * 1024 * 1024
|
|
19
|
+
private let maxSnapshotElements = 600
|
|
20
|
+
private let fastSnapshotLimit = 300
|
|
21
|
+
private let interactiveTypes: Set<XCUIElement.ElementType> = [
|
|
22
|
+
.button,
|
|
23
|
+
.cell,
|
|
24
|
+
.checkBox,
|
|
25
|
+
.collectionView,
|
|
26
|
+
.link,
|
|
27
|
+
.menuItem,
|
|
28
|
+
.picker,
|
|
29
|
+
.searchField,
|
|
30
|
+
.segmentedControl,
|
|
31
|
+
.slider,
|
|
32
|
+
.stepper,
|
|
33
|
+
.switch,
|
|
34
|
+
.tabBar,
|
|
35
|
+
.textField,
|
|
36
|
+
.textView,
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
override func setUp() {
|
|
40
|
+
continueAfterFailure = false
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
@MainActor
|
|
44
|
+
func testCommand() throws {
|
|
45
|
+
doneExpectation = expectation(description: "agent-device command handled")
|
|
46
|
+
app.launch()
|
|
47
|
+
currentApp = app
|
|
48
|
+
let queue = DispatchQueue(label: "agent-device.runner")
|
|
49
|
+
let desiredPort = resolveRunnerPort()
|
|
50
|
+
NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", desiredPort)
|
|
51
|
+
if desiredPort > 0, let port = NWEndpoint.Port(rawValue: desiredPort) {
|
|
52
|
+
listener = try NWListener(using: .tcp, on: port)
|
|
53
|
+
} else {
|
|
54
|
+
listener = try NWListener(using: .tcp)
|
|
55
|
+
}
|
|
56
|
+
listener?.stateUpdateHandler = { [weak self] state in
|
|
57
|
+
switch state {
|
|
58
|
+
case .ready:
|
|
59
|
+
NSLog("AGENT_DEVICE_RUNNER_LISTENER_READY")
|
|
60
|
+
if let listenerPort = self?.listener?.port {
|
|
61
|
+
self?.port = listenerPort.rawValue
|
|
62
|
+
NSLog("AGENT_DEVICE_RUNNER_PORT=%d", listenerPort.rawValue)
|
|
63
|
+
} else {
|
|
64
|
+
NSLog("AGENT_DEVICE_RUNNER_PORT_NOT_SET")
|
|
65
|
+
}
|
|
66
|
+
case .failed(let error):
|
|
67
|
+
NSLog("AGENT_DEVICE_RUNNER_LISTENER_FAILED=%@", String(describing: error))
|
|
68
|
+
default:
|
|
69
|
+
break
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
listener?.newConnectionHandler = { [weak self] conn in
|
|
73
|
+
conn.start(queue: queue)
|
|
74
|
+
self?.handle(connection: conn)
|
|
75
|
+
}
|
|
76
|
+
listener?.start(queue: queue)
|
|
77
|
+
|
|
78
|
+
guard let expectation = doneExpectation else {
|
|
79
|
+
XCTFail("runner expectation was not initialized")
|
|
80
|
+
return
|
|
81
|
+
}
|
|
82
|
+
NSLog("AGENT_DEVICE_RUNNER_WAITING")
|
|
83
|
+
let result = XCTWaiter.wait(for: [expectation], timeout: resolveRunnerTimeout())
|
|
84
|
+
NSLog("AGENT_DEVICE_RUNNER_WAIT_RESULT=%@", String(describing: result))
|
|
85
|
+
if result != .completed {
|
|
86
|
+
XCTFail("runner wait ended with \(result)")
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private func handle(connection: NWConnection) {
|
|
91
|
+
receiveRequest(connection: connection, buffer: Data())
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private func receiveRequest(connection: NWConnection, buffer: Data) {
|
|
95
|
+
connection.receive(minimumIncompleteLength: 1, maximumLength: 1024 * 1024) { [weak self] data, _, _, _ in
|
|
96
|
+
guard let self = self, let data = data else {
|
|
97
|
+
connection.cancel()
|
|
98
|
+
return
|
|
99
|
+
}
|
|
100
|
+
if buffer.count + data.count > self.maxRequestBytes {
|
|
101
|
+
let response = self.jsonResponse(
|
|
102
|
+
status: 413,
|
|
103
|
+
response: Response(ok: false, error: ErrorPayload(message: "request too large")),
|
|
104
|
+
)
|
|
105
|
+
connection.send(content: response, completion: .contentProcessed { [weak self] _ in
|
|
106
|
+
connection.cancel()
|
|
107
|
+
self?.finish()
|
|
108
|
+
})
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
let combined = buffer + data
|
|
112
|
+
if let body = self.parseRequest(data: combined) {
|
|
113
|
+
let result = self.handleRequestBody(body)
|
|
114
|
+
connection.send(content: result.data, completion: .contentProcessed { _ in
|
|
115
|
+
connection.cancel()
|
|
116
|
+
if result.shouldFinish {
|
|
117
|
+
self.finish()
|
|
118
|
+
}
|
|
119
|
+
})
|
|
120
|
+
} else {
|
|
121
|
+
self.receiveRequest(connection: connection, buffer: combined)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private func parseRequest(data: Data) -> Data? {
|
|
127
|
+
guard let headerEnd = data.range(of: Data("\r\n\r\n".utf8)) else {
|
|
128
|
+
return nil
|
|
129
|
+
}
|
|
130
|
+
let headerData = data.subdata(in: 0..<headerEnd.lowerBound)
|
|
131
|
+
let bodyStart = headerEnd.upperBound
|
|
132
|
+
let headers = String(decoding: headerData, as: UTF8.self)
|
|
133
|
+
let contentLength = extractContentLength(headers: headers)
|
|
134
|
+
guard let contentLength = contentLength else {
|
|
135
|
+
return nil
|
|
136
|
+
}
|
|
137
|
+
if data.count < bodyStart + contentLength {
|
|
138
|
+
return nil
|
|
139
|
+
}
|
|
140
|
+
let body = data.subdata(in: bodyStart..<(bodyStart + contentLength))
|
|
141
|
+
return body
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private func extractContentLength(headers: String) -> Int? {
|
|
145
|
+
for line in headers.split(separator: "\r\n") {
|
|
146
|
+
let parts = line.split(separator: ":", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }
|
|
147
|
+
if parts.count == 2 && parts[0].lowercased() == "content-length" {
|
|
148
|
+
return Int(parts[1])
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return nil
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private func handleRequestBody(_ body: Data) -> (data: Data, shouldFinish: Bool) {
|
|
155
|
+
guard let json = String(data: body, encoding: .utf8) else {
|
|
156
|
+
return (
|
|
157
|
+
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
158
|
+
false
|
|
159
|
+
)
|
|
160
|
+
}
|
|
161
|
+
guard let data = json.data(using: .utf8) else {
|
|
162
|
+
return (
|
|
163
|
+
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
164
|
+
false
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
do {
|
|
169
|
+
let command = try JSONDecoder().decode(Command.self, from: data)
|
|
170
|
+
let response = try execute(command: command)
|
|
171
|
+
return (jsonResponse(status: 200, response: response), command.command == .shutdown)
|
|
172
|
+
} catch {
|
|
173
|
+
return (
|
|
174
|
+
jsonResponse(status: 500, response: Response(ok: false, error: ErrorPayload(message: "\(error)"))),
|
|
175
|
+
false
|
|
176
|
+
)
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
private func execute(command: Command) throws -> Response {
|
|
181
|
+
if Thread.isMainThread {
|
|
182
|
+
return try executeOnMain(command: command)
|
|
183
|
+
}
|
|
184
|
+
var result: Result<Response, Error>?
|
|
185
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
186
|
+
DispatchQueue.main.async {
|
|
187
|
+
do {
|
|
188
|
+
result = .success(try self.executeOnMain(command: command))
|
|
189
|
+
} catch {
|
|
190
|
+
result = .failure(error)
|
|
191
|
+
}
|
|
192
|
+
semaphore.signal()
|
|
193
|
+
}
|
|
194
|
+
semaphore.wait()
|
|
195
|
+
switch result {
|
|
196
|
+
case .success(let response):
|
|
197
|
+
return response
|
|
198
|
+
case .failure(let error):
|
|
199
|
+
throw error
|
|
200
|
+
case .none:
|
|
201
|
+
throw NSError(domain: "AgentDeviceRunner", code: 1, userInfo: [NSLocalizedDescriptionKey: "no response from main thread"])
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private func executeOnMain(command: Command) throws -> Response {
|
|
206
|
+
let bundleId = command.appBundleId ?? "com.apple.Preferences"
|
|
207
|
+
if currentBundleId != bundleId {
|
|
208
|
+
let target = XCUIApplication(bundleIdentifier: bundleId)
|
|
209
|
+
NSLog("AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d", bundleId, target.state.rawValue)
|
|
210
|
+
// activate avoids terminating and relaunching the target app
|
|
211
|
+
target.activate()
|
|
212
|
+
currentApp = target
|
|
213
|
+
currentBundleId = bundleId
|
|
214
|
+
}
|
|
215
|
+
let activeApp = currentApp ?? app
|
|
216
|
+
_ = activeApp.waitForExistence(timeout: 5)
|
|
217
|
+
|
|
218
|
+
switch command.command {
|
|
219
|
+
case .shutdown:
|
|
220
|
+
return Response(ok: true, data: DataPayload(message: "shutdown"))
|
|
221
|
+
case .tap:
|
|
222
|
+
if let text = command.text {
|
|
223
|
+
if let element = findElement(app: activeApp, text: text) {
|
|
224
|
+
element.tap()
|
|
225
|
+
return Response(ok: true, data: DataPayload(message: "tapped"))
|
|
226
|
+
}
|
|
227
|
+
return Response(ok: false, error: ErrorPayload(message: "element not found"))
|
|
228
|
+
}
|
|
229
|
+
if let x = command.x, let y = command.y {
|
|
230
|
+
tapAt(app: activeApp, x: x, y: y)
|
|
231
|
+
return Response(ok: true, data: DataPayload(message: "tapped"))
|
|
232
|
+
}
|
|
233
|
+
return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
|
|
234
|
+
case .type:
|
|
235
|
+
guard let text = command.text else {
|
|
236
|
+
return Response(ok: false, error: ErrorPayload(message: "type requires text"))
|
|
237
|
+
}
|
|
238
|
+
activeApp.typeText(text)
|
|
239
|
+
return Response(ok: true, data: DataPayload(message: "typed"))
|
|
240
|
+
case .swipe:
|
|
241
|
+
guard let direction = command.direction else {
|
|
242
|
+
return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
|
|
243
|
+
}
|
|
244
|
+
swipe(app: activeApp, direction: direction)
|
|
245
|
+
return Response(ok: true, data: DataPayload(message: "swiped"))
|
|
246
|
+
case .findText:
|
|
247
|
+
guard let text = command.text else {
|
|
248
|
+
return Response(ok: false, error: ErrorPayload(message: "findText requires text"))
|
|
249
|
+
}
|
|
250
|
+
let found = findElement(app: activeApp, text: text) != nil
|
|
251
|
+
return Response(ok: true, data: DataPayload(found: found))
|
|
252
|
+
case .rect:
|
|
253
|
+
guard let text = command.text else {
|
|
254
|
+
return Response(ok: false, error: ErrorPayload(message: "rect requires text"))
|
|
255
|
+
}
|
|
256
|
+
guard let element = findElement(app: activeApp, text: text) else {
|
|
257
|
+
return Response(ok: false, error: ErrorPayload(message: "element not found"))
|
|
258
|
+
}
|
|
259
|
+
let frame = element.frame
|
|
260
|
+
let rect = SnapshotRect(
|
|
261
|
+
x: Double(frame.origin.x),
|
|
262
|
+
y: Double(frame.origin.y),
|
|
263
|
+
width: Double(frame.size.width),
|
|
264
|
+
height: Double(frame.size.height)
|
|
265
|
+
)
|
|
266
|
+
return Response(ok: true, data: DataPayload(rect: rect))
|
|
267
|
+
case .listTappables:
|
|
268
|
+
let elements = activeApp.descendants(matching: .any).allElementsBoundByIndex
|
|
269
|
+
let labels = elements.compactMap { element -> String? in
|
|
270
|
+
guard element.isHittable else { return nil }
|
|
271
|
+
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
272
|
+
if label.isEmpty { return nil }
|
|
273
|
+
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
274
|
+
return identifier.isEmpty ? label : "\(label) [\(identifier)]"
|
|
275
|
+
}
|
|
276
|
+
let unique = Array(Set(labels)).sorted()
|
|
277
|
+
return Response(ok: true, data: DataPayload(items: unique))
|
|
278
|
+
case .snapshot:
|
|
279
|
+
let options = SnapshotOptions(
|
|
280
|
+
interactiveOnly: command.interactiveOnly ?? false,
|
|
281
|
+
compact: command.compact ?? false,
|
|
282
|
+
depth: command.depth,
|
|
283
|
+
scope: command.scope,
|
|
284
|
+
raw: command.raw ?? false,
|
|
285
|
+
)
|
|
286
|
+
if options.raw {
|
|
287
|
+
return Response(ok: true, data: snapshotRaw(app: activeApp, options: options))
|
|
288
|
+
}
|
|
289
|
+
return Response(ok: true, data: snapshotFast(app: activeApp, options: options))
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
private func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
|
|
294
|
+
let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
|
|
295
|
+
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
296
|
+
return element.exists ? element : nil
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
private func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
|
|
300
|
+
let predicate = NSPredicate(
|
|
301
|
+
format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
|
|
302
|
+
scope,
|
|
303
|
+
scope
|
|
304
|
+
)
|
|
305
|
+
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
306
|
+
return element.exists ? element : nil
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
private func tapAt(app: XCUIApplication, x: Double, y: Double) {
|
|
310
|
+
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
311
|
+
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
312
|
+
coordinate.tap()
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
private func swipe(app: XCUIApplication, direction: SwipeDirection) {
|
|
316
|
+
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
317
|
+
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
|
|
318
|
+
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
|
|
319
|
+
let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
|
|
320
|
+
let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
321
|
+
|
|
322
|
+
switch direction {
|
|
323
|
+
case .up:
|
|
324
|
+
end.press(forDuration: 0.1, thenDragTo: start)
|
|
325
|
+
case .down:
|
|
326
|
+
start.press(forDuration: 0.1, thenDragTo: end)
|
|
327
|
+
case .left:
|
|
328
|
+
right.press(forDuration: 0.1, thenDragTo: left)
|
|
329
|
+
case .right:
|
|
330
|
+
left.press(forDuration: 0.1, thenDragTo: right)
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
private func aggregatedLabel(for element: XCUIElement, depth: Int = 0) -> String? {
|
|
335
|
+
if depth > 2 { return nil }
|
|
336
|
+
let text = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
337
|
+
if !text.isEmpty { return text }
|
|
338
|
+
if let value = element.value {
|
|
339
|
+
let valueText = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
340
|
+
if !valueText.isEmpty { return valueText }
|
|
341
|
+
}
|
|
342
|
+
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
343
|
+
for child in children {
|
|
344
|
+
if let childLabel = aggregatedLabel(for: child, depth: depth + 1) {
|
|
345
|
+
return childLabel
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return nil
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
private func elementTypeName(_ type: XCUIElement.ElementType) -> String {
|
|
352
|
+
switch type {
|
|
353
|
+
case .application: return "Application"
|
|
354
|
+
case .window: return "Window"
|
|
355
|
+
case .button: return "Button"
|
|
356
|
+
case .cell: return "Cell"
|
|
357
|
+
case .staticText: return "StaticText"
|
|
358
|
+
case .textField: return "TextField"
|
|
359
|
+
case .textView: return "TextView"
|
|
360
|
+
case .secureTextField: return "SecureTextField"
|
|
361
|
+
case .switch: return "Switch"
|
|
362
|
+
case .slider: return "Slider"
|
|
363
|
+
case .link: return "Link"
|
|
364
|
+
case .image: return "Image"
|
|
365
|
+
case .navigationBar: return "NavigationBar"
|
|
366
|
+
case .tabBar: return "TabBar"
|
|
367
|
+
case .collectionView: return "CollectionView"
|
|
368
|
+
case .table: return "Table"
|
|
369
|
+
case .scrollView: return "ScrollView"
|
|
370
|
+
case .searchField: return "SearchField"
|
|
371
|
+
case .segmentedControl: return "SegmentedControl"
|
|
372
|
+
case .stepper: return "Stepper"
|
|
373
|
+
case .picker: return "Picker"
|
|
374
|
+
case .checkBox: return "CheckBox"
|
|
375
|
+
case .menuItem: return "MenuItem"
|
|
376
|
+
case .other: return "Other"
|
|
377
|
+
default: return "Element(\(type.rawValue))"
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
private func snapshotFast(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
|
|
382
|
+
var nodes: [SnapshotNode] = []
|
|
383
|
+
var truncated = false
|
|
384
|
+
let maxDepth = options.depth ?? 2
|
|
385
|
+
let viewport = app.frame
|
|
386
|
+
let rootLabel = aggregatedLabel(for: app) ?? app.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
387
|
+
let rootNode = SnapshotNode(
|
|
388
|
+
index: 0,
|
|
389
|
+
type: "Application",
|
|
390
|
+
label: rootLabel.isEmpty ? nil : rootLabel,
|
|
391
|
+
identifier: app.identifier.isEmpty ? nil : app.identifier,
|
|
392
|
+
value: nil,
|
|
393
|
+
rect: SnapshotRect(
|
|
394
|
+
x: Double(app.frame.origin.x),
|
|
395
|
+
y: Double(app.frame.origin.y),
|
|
396
|
+
width: Double(app.frame.size.width),
|
|
397
|
+
height: Double(app.frame.size.height),
|
|
398
|
+
),
|
|
399
|
+
enabled: app.isEnabled,
|
|
400
|
+
hittable: app.isHittable,
|
|
401
|
+
depth: 0,
|
|
402
|
+
)
|
|
403
|
+
nodes.append(rootNode)
|
|
404
|
+
|
|
405
|
+
let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
406
|
+
let elements = collectFastElements(root: queryRoot)
|
|
407
|
+
var seen = Set<String>()
|
|
408
|
+
|
|
409
|
+
for element in elements {
|
|
410
|
+
if nodes.count >= fastSnapshotLimit {
|
|
411
|
+
truncated = true
|
|
412
|
+
break
|
|
413
|
+
}
|
|
414
|
+
if !isVisibleInViewport(element.frame, viewport) { continue }
|
|
415
|
+
let label = aggregatedLabel(for: element) ?? element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
416
|
+
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
417
|
+
let valueText: String? = {
|
|
418
|
+
guard let value = element.value else { return nil }
|
|
419
|
+
let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
420
|
+
return text.isEmpty ? nil : text
|
|
421
|
+
}()
|
|
422
|
+
if !shouldInclude(element: element, label: label, identifier: identifier, valueText: valueText, options: options) {
|
|
423
|
+
continue
|
|
424
|
+
}
|
|
425
|
+
let key = "\(element.elementType)-\(label)-\(identifier)-\(element.frame.origin.x)-\(element.frame.origin.y)"
|
|
426
|
+
if seen.contains(key) { continue }
|
|
427
|
+
seen.insert(key)
|
|
428
|
+
nodes.append(
|
|
429
|
+
SnapshotNode(
|
|
430
|
+
index: nodes.count,
|
|
431
|
+
type: elementTypeName(element.elementType),
|
|
432
|
+
label: label.isEmpty ? nil : label,
|
|
433
|
+
identifier: identifier.isEmpty ? nil : identifier,
|
|
434
|
+
value: valueText,
|
|
435
|
+
rect: SnapshotRect(
|
|
436
|
+
x: Double(element.frame.origin.x),
|
|
437
|
+
y: Double(element.frame.origin.y),
|
|
438
|
+
width: Double(element.frame.size.width),
|
|
439
|
+
height: Double(element.frame.size.height),
|
|
440
|
+
),
|
|
441
|
+
enabled: element.isEnabled,
|
|
442
|
+
hittable: element.isHittable,
|
|
443
|
+
depth: min(maxDepth, 1),
|
|
444
|
+
)
|
|
445
|
+
)
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
return DataPayload(nodes: nodes, truncated: truncated)
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
private func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
|
|
452
|
+
let root = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
453
|
+
var nodes: [SnapshotNode] = []
|
|
454
|
+
var truncated = false
|
|
455
|
+
let viewport = app.frame
|
|
456
|
+
|
|
457
|
+
func walk(_ element: XCUIElement, depth: Int) {
|
|
458
|
+
if nodes.count >= maxSnapshotElements {
|
|
459
|
+
truncated = true
|
|
460
|
+
return
|
|
461
|
+
}
|
|
462
|
+
if let limit = options.depth, depth > limit { return }
|
|
463
|
+
if !isVisibleInViewport(element.frame, viewport) { return }
|
|
464
|
+
|
|
465
|
+
let label = aggregatedLabel(for: element) ?? element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
466
|
+
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
467
|
+
let valueText: String? = {
|
|
468
|
+
guard let value = element.value else { return nil }
|
|
469
|
+
let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
470
|
+
return text.isEmpty ? nil : text
|
|
471
|
+
}()
|
|
472
|
+
if shouldInclude(element: element, label: label, identifier: identifier, valueText: valueText, options: options) {
|
|
473
|
+
nodes.append(
|
|
474
|
+
SnapshotNode(
|
|
475
|
+
index: nodes.count,
|
|
476
|
+
type: elementTypeName(element.elementType),
|
|
477
|
+
label: label.isEmpty ? nil : label,
|
|
478
|
+
identifier: identifier.isEmpty ? nil : identifier,
|
|
479
|
+
value: valueText,
|
|
480
|
+
rect: SnapshotRect(
|
|
481
|
+
x: Double(element.frame.origin.x),
|
|
482
|
+
y: Double(element.frame.origin.y),
|
|
483
|
+
width: Double(element.frame.size.width),
|
|
484
|
+
height: Double(element.frame.size.height),
|
|
485
|
+
),
|
|
486
|
+
enabled: element.isEnabled,
|
|
487
|
+
hittable: element.isHittable,
|
|
488
|
+
depth: depth,
|
|
489
|
+
)
|
|
490
|
+
)
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
494
|
+
for child in children {
|
|
495
|
+
walk(child, depth: depth + 1)
|
|
496
|
+
if truncated { return }
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
walk(root, depth: 0)
|
|
501
|
+
return DataPayload(nodes: nodes, truncated: truncated)
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
private func shouldInclude(
|
|
505
|
+
element: XCUIElement,
|
|
506
|
+
label: String,
|
|
507
|
+
identifier: String,
|
|
508
|
+
valueText: String?,
|
|
509
|
+
options: SnapshotOptions
|
|
510
|
+
) -> Bool {
|
|
511
|
+
let type = element.elementType
|
|
512
|
+
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
513
|
+
if options.compact && type == .other && !hasContent && !element.isHittable {
|
|
514
|
+
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
515
|
+
if children.count <= 1 { return false }
|
|
516
|
+
}
|
|
517
|
+
if options.interactiveOnly {
|
|
518
|
+
if interactiveTypes.contains(type) { return true }
|
|
519
|
+
if element.isHittable && type != .other { return true }
|
|
520
|
+
if hasContent && type != .other { return true }
|
|
521
|
+
return false
|
|
522
|
+
}
|
|
523
|
+
if options.compact {
|
|
524
|
+
return hasContent || element.isHittable
|
|
525
|
+
}
|
|
526
|
+
return true
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
private func collectFastElements(root: XCUIElement) -> [XCUIElement] {
|
|
530
|
+
var elements: [XCUIElement] = []
|
|
531
|
+
elements.append(contentsOf: root.buttons.allElementsBoundByIndex)
|
|
532
|
+
elements.append(contentsOf: root.links.allElementsBoundByIndex)
|
|
533
|
+
elements.append(contentsOf: root.cells.allElementsBoundByIndex)
|
|
534
|
+
elements.append(contentsOf: root.staticTexts.allElementsBoundByIndex)
|
|
535
|
+
elements.append(contentsOf: root.switches.allElementsBoundByIndex)
|
|
536
|
+
elements.append(contentsOf: root.textFields.allElementsBoundByIndex)
|
|
537
|
+
elements.append(contentsOf: root.textViews.allElementsBoundByIndex)
|
|
538
|
+
elements.append(contentsOf: root.navigationBars.allElementsBoundByIndex)
|
|
539
|
+
elements.append(contentsOf: root.tabBars.allElementsBoundByIndex)
|
|
540
|
+
elements.append(contentsOf: root.searchFields.allElementsBoundByIndex)
|
|
541
|
+
elements.append(contentsOf: root.segmentedControls.allElementsBoundByIndex)
|
|
542
|
+
elements.append(contentsOf: root.collectionViews.allElementsBoundByIndex)
|
|
543
|
+
elements.append(contentsOf: root.tables.allElementsBoundByIndex)
|
|
544
|
+
return elements
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
private func isVisibleInViewport(_ rect: CGRect, _ viewport: CGRect) -> Bool {
|
|
548
|
+
if rect.isNull || rect.isEmpty { return false }
|
|
549
|
+
return rect.intersects(viewport)
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
private func jsonResponse(status: Int, response: Response) -> Data {
|
|
553
|
+
let encoder = JSONEncoder()
|
|
554
|
+
let body = (try? encoder.encode(response)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
|
|
555
|
+
return httpResponse(status: status, body: body)
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
private func httpResponse(status: Int, body: String) -> Data {
|
|
559
|
+
let headers = [
|
|
560
|
+
"HTTP/1.1 \(status) OK",
|
|
561
|
+
"Content-Type: application/json",
|
|
562
|
+
"Content-Length: \(body.utf8.count)",
|
|
563
|
+
"Connection: close",
|
|
564
|
+
"",
|
|
565
|
+
body,
|
|
566
|
+
].joined(separator: "\r\n")
|
|
567
|
+
return Data(headers.utf8)
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
private func finish() {
|
|
571
|
+
listener?.cancel()
|
|
572
|
+
listener = nil
|
|
573
|
+
doneExpectation?.fulfill()
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
private func resolveRunnerPort() -> UInt16 {
|
|
578
|
+
if let env = ProcessInfo.processInfo.environment["AGENT_DEVICE_RUNNER_PORT"], let port = UInt16(env) {
|
|
579
|
+
return port
|
|
580
|
+
}
|
|
581
|
+
for arg in CommandLine.arguments {
|
|
582
|
+
if arg.hasPrefix("AGENT_DEVICE_RUNNER_PORT=") {
|
|
583
|
+
let value = arg.replacingOccurrences(of: "AGENT_DEVICE_RUNNER_PORT=", with: "")
|
|
584
|
+
if let port = UInt16(value) { return port }
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
return 0
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
private func resolveRunnerTimeout() -> TimeInterval {
|
|
591
|
+
if let env = ProcessInfo.processInfo.environment["AGENT_DEVICE_RUNNER_TIMEOUT"],
|
|
592
|
+
let parsed = Double(env) {
|
|
593
|
+
return parsed
|
|
594
|
+
}
|
|
595
|
+
return 300
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
enum CommandType: String, Codable {
|
|
599
|
+
case tap
|
|
600
|
+
case type
|
|
601
|
+
case swipe
|
|
602
|
+
case findText
|
|
603
|
+
case listTappables
|
|
604
|
+
case snapshot
|
|
605
|
+
case rect
|
|
606
|
+
case shutdown
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
enum SwipeDirection: String, Codable {
|
|
610
|
+
case up
|
|
611
|
+
case down
|
|
612
|
+
case left
|
|
613
|
+
case right
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
struct Command: Codable {
|
|
617
|
+
let command: CommandType
|
|
618
|
+
let appBundleId: String?
|
|
619
|
+
let text: String?
|
|
620
|
+
let x: Double?
|
|
621
|
+
let y: Double?
|
|
622
|
+
let direction: SwipeDirection?
|
|
623
|
+
let interactiveOnly: Bool?
|
|
624
|
+
let compact: Bool?
|
|
625
|
+
let depth: Int?
|
|
626
|
+
let scope: String?
|
|
627
|
+
let raw: Bool?
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
struct Response: Codable {
|
|
631
|
+
let ok: Bool
|
|
632
|
+
let data: DataPayload?
|
|
633
|
+
let error: ErrorPayload?
|
|
634
|
+
|
|
635
|
+
init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
|
|
636
|
+
self.ok = ok
|
|
637
|
+
self.data = data
|
|
638
|
+
self.error = error
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
struct DataPayload: Codable {
|
|
643
|
+
let message: String?
|
|
644
|
+
let found: Bool?
|
|
645
|
+
let items: [String]?
|
|
646
|
+
let nodes: [SnapshotNode]?
|
|
647
|
+
let truncated: Bool?
|
|
648
|
+
let rect: SnapshotRect?
|
|
649
|
+
|
|
650
|
+
init(
|
|
651
|
+
message: String? = nil,
|
|
652
|
+
found: Bool? = nil,
|
|
653
|
+
items: [String]? = nil,
|
|
654
|
+
nodes: [SnapshotNode]? = nil,
|
|
655
|
+
truncated: Bool? = nil,
|
|
656
|
+
rect: SnapshotRect? = nil
|
|
657
|
+
) {
|
|
658
|
+
self.message = message
|
|
659
|
+
self.found = found
|
|
660
|
+
self.items = items
|
|
661
|
+
self.nodes = nodes
|
|
662
|
+
self.truncated = truncated
|
|
663
|
+
self.rect = rect
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
struct ErrorPayload: Codable {
|
|
668
|
+
let message: String
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
struct SnapshotRect: Codable {
|
|
672
|
+
let x: Double
|
|
673
|
+
let y: Double
|
|
674
|
+
let width: Double
|
|
675
|
+
let height: Double
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
struct SnapshotNode: Codable {
|
|
679
|
+
let index: Int
|
|
680
|
+
let type: String
|
|
681
|
+
let label: String?
|
|
682
|
+
let identifier: String?
|
|
683
|
+
let value: String?
|
|
684
|
+
let rect: SnapshotRect
|
|
685
|
+
let enabled: Bool
|
|
686
|
+
let hittable: Bool
|
|
687
|
+
let depth: Int
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
struct SnapshotOptions {
|
|
691
|
+
let interactiveOnly: Bool
|
|
692
|
+
let compact: Bool
|
|
693
|
+
let depth: Int?
|
|
694
|
+
let scope: String?
|
|
695
|
+
let raw: Bool
|
|
696
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# agent-device iOS Runner
|
|
2
|
+
|
|
3
|
+
This folder is reserved for the lightweight XCUITest runner used to provide element-level automation on iOS.
|
|
4
|
+
|
|
5
|
+
## Intent
|
|
6
|
+
- Provide a minimal XCTest target that exposes UI automation over a small HTTP server.
|
|
7
|
+
- Allow local builds via `xcodebuild` and caching for faster subsequent runs.
|
|
8
|
+
- Support simulator prebuilds where compatible.
|
|
9
|
+
|
|
10
|
+
## Status
|
|
11
|
+
Planned for v1 automation layer. See `docs/ios-automation.md` and `docs/ios-runner-protocol.md`.
|