agent-device 0.7.3 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/src/daemon.js +25 -25
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +8 -4
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +381 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +30 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +258 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +174 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +121 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +263 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +359 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +220 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +124 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +30 -1855
- package/ios-runner/README.md +14 -0
- package/package.json +1 -1
- package/skills/agent-device/references/permissions.md +5 -1
|
@@ -1,52 +1,49 @@
|
|
|
1
1
|
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
2
|
+
// RunnerTests.swift
|
|
3
|
+
// AgentDeviceRunnerUITests
|
|
4
4
|
//
|
|
5
5
|
// Created by Michał Pierzchała on 30/01/2026.
|
|
6
6
|
//
|
|
7
7
|
|
|
8
8
|
import XCTest
|
|
9
9
|
import Network
|
|
10
|
-
import AVFoundation
|
|
11
|
-
import CoreVideo
|
|
12
|
-
import UIKit
|
|
13
10
|
|
|
14
11
|
final class RunnerTests: XCTestCase {
|
|
15
|
-
|
|
12
|
+
enum RunnerErrorDomain {
|
|
16
13
|
static let general = "AgentDeviceRunner"
|
|
17
14
|
static let exception = "AgentDeviceRunner.NSException"
|
|
18
15
|
}
|
|
19
16
|
|
|
20
|
-
|
|
17
|
+
enum RunnerErrorCode {
|
|
21
18
|
static let noResponseFromMainThread = 1
|
|
22
19
|
static let commandReturnedNoResponse = 2
|
|
23
20
|
static let mainThreadExecutionTimedOut = 3
|
|
24
21
|
static let objcException = 1
|
|
25
22
|
}
|
|
26
23
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
24
|
+
static let springboardBundleId = "com.apple.springboard"
|
|
25
|
+
var listener: NWListener?
|
|
26
|
+
var doneExpectation: XCTestExpectation?
|
|
27
|
+
let app = XCUIApplication()
|
|
28
|
+
lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId)
|
|
29
|
+
var currentApp: XCUIApplication?
|
|
30
|
+
var currentBundleId: String?
|
|
31
|
+
let maxRequestBytes = 2 * 1024 * 1024
|
|
32
|
+
let maxSnapshotElements = 600
|
|
33
|
+
let fastSnapshotLimit = 300
|
|
34
|
+
let mainThreadExecutionTimeout: TimeInterval = 30
|
|
35
|
+
let appExistenceTimeout: TimeInterval = 30
|
|
36
|
+
let retryCooldown: TimeInterval = 0.2
|
|
37
|
+
let postSnapshotInteractionDelay: TimeInterval = 0.2
|
|
38
|
+
let firstInteractionAfterActivateDelay: TimeInterval = 0.25
|
|
39
|
+
let scrollInteractionIdleTimeoutDefault: TimeInterval = 1.0
|
|
40
|
+
let tvRemoteDoublePressDelayDefault: TimeInterval = 0.0
|
|
41
|
+
let minRecordingFps = 1
|
|
42
|
+
let maxRecordingFps = 120
|
|
43
|
+
var needsPostSnapshotInteractionDelay = false
|
|
44
|
+
var needsFirstInteractionDelay = false
|
|
45
|
+
var activeRecording: ScreenRecorder?
|
|
46
|
+
let interactiveTypes: Set<XCUIElement.ElementType> = [
|
|
50
47
|
.button,
|
|
51
48
|
.cell,
|
|
52
49
|
.checkBox,
|
|
@@ -65,7 +62,7 @@ final class RunnerTests: XCTestCase {
|
|
|
65
62
|
.textView,
|
|
66
63
|
]
|
|
67
64
|
// Keep blocker actions narrow to avoid false positives from generic hittable containers.
|
|
68
|
-
|
|
65
|
+
let actionableTypes: Set<XCUIElement.ElementType> = [
|
|
69
66
|
.button,
|
|
70
67
|
.cell,
|
|
71
68
|
.link,
|
|
@@ -74,261 +71,7 @@ final class RunnerTests: XCTestCase {
|
|
|
74
71
|
.switch,
|
|
75
72
|
]
|
|
76
73
|
|
|
77
|
-
|
|
78
|
-
private let outputPath: String
|
|
79
|
-
private let fps: Int32?
|
|
80
|
-
private let uncappedFrameInterval: TimeInterval = 0.001
|
|
81
|
-
private var uncappedTimestampTimescale: Int32 {
|
|
82
|
-
Int32(max(1, Int((1.0 / uncappedFrameInterval).rounded())))
|
|
83
|
-
}
|
|
84
|
-
private var frameInterval: TimeInterval {
|
|
85
|
-
guard let fps else { return uncappedFrameInterval }
|
|
86
|
-
return 1.0 / Double(fps)
|
|
87
|
-
}
|
|
88
|
-
private let queue = DispatchQueue(label: "agent-device.runner.recorder")
|
|
89
|
-
private let lock = NSLock()
|
|
90
|
-
private var assetWriter: AVAssetWriter?
|
|
91
|
-
private var writerInput: AVAssetWriterInput?
|
|
92
|
-
private var pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor?
|
|
93
|
-
private var timer: DispatchSourceTimer?
|
|
94
|
-
private var recordingStartUptime: TimeInterval?
|
|
95
|
-
private var lastTimestampValue: Int64 = -1
|
|
96
|
-
private var isStopping = false
|
|
97
|
-
private var startedSession = false
|
|
98
|
-
private var startError: Error?
|
|
99
|
-
|
|
100
|
-
init(outputPath: String, fps: Int32?) {
|
|
101
|
-
self.outputPath = outputPath
|
|
102
|
-
self.fps = fps
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
func start(captureFrame: @escaping () -> UIImage?) throws {
|
|
106
|
-
let url = URL(fileURLWithPath: outputPath)
|
|
107
|
-
let directory = url.deletingLastPathComponent()
|
|
108
|
-
try FileManager.default.createDirectory(
|
|
109
|
-
at: directory,
|
|
110
|
-
withIntermediateDirectories: true,
|
|
111
|
-
attributes: nil
|
|
112
|
-
)
|
|
113
|
-
if FileManager.default.fileExists(atPath: outputPath) {
|
|
114
|
-
try FileManager.default.removeItem(atPath: outputPath)
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
var dimensions: CGSize = .zero
|
|
118
|
-
var bootstrapImage: UIImage?
|
|
119
|
-
let bootstrapDeadline = Date().addingTimeInterval(2.0)
|
|
120
|
-
while Date() < bootstrapDeadline {
|
|
121
|
-
if let image = captureFrame(), let cgImage = image.cgImage {
|
|
122
|
-
bootstrapImage = image
|
|
123
|
-
dimensions = CGSize(width: cgImage.width, height: cgImage.height)
|
|
124
|
-
break
|
|
125
|
-
}
|
|
126
|
-
Thread.sleep(forTimeInterval: 0.05)
|
|
127
|
-
}
|
|
128
|
-
guard dimensions.width > 0, dimensions.height > 0 else {
|
|
129
|
-
throw NSError(
|
|
130
|
-
domain: "AgentDeviceRunner.Record",
|
|
131
|
-
code: 1,
|
|
132
|
-
userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"]
|
|
133
|
-
)
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
let writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
|
|
137
|
-
let outputSettings: [String: Any] = [
|
|
138
|
-
AVVideoCodecKey: AVVideoCodecType.h264,
|
|
139
|
-
AVVideoWidthKey: Int(dimensions.width),
|
|
140
|
-
AVVideoHeightKey: Int(dimensions.height),
|
|
141
|
-
]
|
|
142
|
-
let input = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
|
|
143
|
-
input.expectsMediaDataInRealTime = true
|
|
144
|
-
let attributes: [String: Any] = [
|
|
145
|
-
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB,
|
|
146
|
-
kCVPixelBufferWidthKey as String: Int(dimensions.width),
|
|
147
|
-
kCVPixelBufferHeightKey as String: Int(dimensions.height),
|
|
148
|
-
]
|
|
149
|
-
let adaptor = AVAssetWriterInputPixelBufferAdaptor(
|
|
150
|
-
assetWriterInput: input,
|
|
151
|
-
sourcePixelBufferAttributes: attributes
|
|
152
|
-
)
|
|
153
|
-
guard writer.canAdd(input) else {
|
|
154
|
-
throw NSError(
|
|
155
|
-
domain: "AgentDeviceRunner.Record",
|
|
156
|
-
code: 2,
|
|
157
|
-
userInfo: [NSLocalizedDescriptionKey: "failed to add video input"]
|
|
158
|
-
)
|
|
159
|
-
}
|
|
160
|
-
writer.add(input)
|
|
161
|
-
guard writer.startWriting() else {
|
|
162
|
-
throw writer.error ?? NSError(
|
|
163
|
-
domain: "AgentDeviceRunner.Record",
|
|
164
|
-
code: 3,
|
|
165
|
-
userInfo: [NSLocalizedDescriptionKey: "failed to start writing"]
|
|
166
|
-
)
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
lock.lock()
|
|
170
|
-
assetWriter = writer
|
|
171
|
-
writerInput = input
|
|
172
|
-
pixelBufferAdaptor = adaptor
|
|
173
|
-
recordingStartUptime = nil
|
|
174
|
-
lastTimestampValue = -1
|
|
175
|
-
isStopping = false
|
|
176
|
-
startedSession = false
|
|
177
|
-
startError = nil
|
|
178
|
-
lock.unlock()
|
|
179
|
-
|
|
180
|
-
if let firstImage = bootstrapImage {
|
|
181
|
-
append(image: firstImage)
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
let timer = DispatchSource.makeTimerSource(queue: queue)
|
|
185
|
-
timer.schedule(deadline: .now() + frameInterval, repeating: frameInterval)
|
|
186
|
-
timer.setEventHandler { [weak self] in
|
|
187
|
-
guard let self else { return }
|
|
188
|
-
if self.shouldStop() { return }
|
|
189
|
-
guard let image = captureFrame() else { return }
|
|
190
|
-
self.append(image: image)
|
|
191
|
-
}
|
|
192
|
-
self.timer = timer
|
|
193
|
-
timer.resume()
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
func stop() throws {
|
|
197
|
-
var writer: AVAssetWriter?
|
|
198
|
-
var input: AVAssetWriterInput?
|
|
199
|
-
var appendError: Error?
|
|
200
|
-
lock.lock()
|
|
201
|
-
if isStopping {
|
|
202
|
-
lock.unlock()
|
|
203
|
-
return
|
|
204
|
-
}
|
|
205
|
-
isStopping = true
|
|
206
|
-
let activeTimer = timer
|
|
207
|
-
timer = nil
|
|
208
|
-
writer = assetWriter
|
|
209
|
-
input = writerInput
|
|
210
|
-
appendError = startError
|
|
211
|
-
lock.unlock()
|
|
212
|
-
|
|
213
|
-
activeTimer?.cancel()
|
|
214
|
-
input?.markAsFinished()
|
|
215
|
-
guard let writer else { return }
|
|
216
|
-
|
|
217
|
-
let semaphore = DispatchSemaphore(value: 0)
|
|
218
|
-
writer.finishWriting {
|
|
219
|
-
semaphore.signal()
|
|
220
|
-
}
|
|
221
|
-
var stopFailure: Error?
|
|
222
|
-
let waitResult = semaphore.wait(timeout: .now() + 10)
|
|
223
|
-
if waitResult == .timedOut {
|
|
224
|
-
writer.cancelWriting()
|
|
225
|
-
stopFailure = NSError(
|
|
226
|
-
domain: "AgentDeviceRunner.Record",
|
|
227
|
-
code: 6,
|
|
228
|
-
userInfo: [NSLocalizedDescriptionKey: "recording finalization timed out"]
|
|
229
|
-
)
|
|
230
|
-
} else if let appendError {
|
|
231
|
-
stopFailure = appendError
|
|
232
|
-
} else if writer.status == .failed {
|
|
233
|
-
stopFailure = writer.error ?? NSError(
|
|
234
|
-
domain: "AgentDeviceRunner.Record",
|
|
235
|
-
code: 4,
|
|
236
|
-
userInfo: [NSLocalizedDescriptionKey: "failed to finalize recording"]
|
|
237
|
-
)
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
lock.lock()
|
|
241
|
-
assetWriter = nil
|
|
242
|
-
writerInput = nil
|
|
243
|
-
pixelBufferAdaptor = nil
|
|
244
|
-
recordingStartUptime = nil
|
|
245
|
-
lastTimestampValue = -1
|
|
246
|
-
startedSession = false
|
|
247
|
-
startError = nil
|
|
248
|
-
lock.unlock()
|
|
249
|
-
|
|
250
|
-
if let stopFailure {
|
|
251
|
-
throw stopFailure
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
private func append(image: UIImage) {
|
|
256
|
-
guard let cgImage = image.cgImage else { return }
|
|
257
|
-
lock.lock()
|
|
258
|
-
defer { lock.unlock() }
|
|
259
|
-
if isStopping { return }
|
|
260
|
-
if let startError { return }
|
|
261
|
-
guard
|
|
262
|
-
let writer = assetWriter,
|
|
263
|
-
let input = writerInput,
|
|
264
|
-
let adaptor = pixelBufferAdaptor
|
|
265
|
-
else {
|
|
266
|
-
return
|
|
267
|
-
}
|
|
268
|
-
if !startedSession {
|
|
269
|
-
writer.startSession(atSourceTime: .zero)
|
|
270
|
-
startedSession = true
|
|
271
|
-
}
|
|
272
|
-
guard input.isReadyForMoreMediaData else { return }
|
|
273
|
-
guard let pixelBuffer = makePixelBuffer(from: cgImage) else { return }
|
|
274
|
-
let nowUptime = ProcessInfo.processInfo.systemUptime
|
|
275
|
-
if recordingStartUptime == nil {
|
|
276
|
-
recordingStartUptime = nowUptime
|
|
277
|
-
}
|
|
278
|
-
let elapsed = max(0, nowUptime - (recordingStartUptime ?? nowUptime))
|
|
279
|
-
let timescale = fps ?? uncappedTimestampTimescale
|
|
280
|
-
var timestampValue = Int64((elapsed * Double(timescale)).rounded(.down))
|
|
281
|
-
if timestampValue <= lastTimestampValue {
|
|
282
|
-
timestampValue = lastTimestampValue + 1
|
|
283
|
-
}
|
|
284
|
-
let timestamp = CMTime(value: timestampValue, timescale: timescale)
|
|
285
|
-
if !adaptor.append(pixelBuffer, withPresentationTime: timestamp) {
|
|
286
|
-
startError = writer.error ?? NSError(
|
|
287
|
-
domain: "AgentDeviceRunner.Record",
|
|
288
|
-
code: 5,
|
|
289
|
-
userInfo: [NSLocalizedDescriptionKey: "failed to append frame"]
|
|
290
|
-
)
|
|
291
|
-
return
|
|
292
|
-
}
|
|
293
|
-
lastTimestampValue = timestampValue
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
private func shouldStop() -> Bool {
|
|
297
|
-
lock.lock()
|
|
298
|
-
defer { lock.unlock() }
|
|
299
|
-
return isStopping
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
private func makePixelBuffer(from image: CGImage) -> CVPixelBuffer? {
|
|
303
|
-
guard let adaptor = pixelBufferAdaptor else { return nil }
|
|
304
|
-
var pixelBuffer: CVPixelBuffer?
|
|
305
|
-
guard let pool = adaptor.pixelBufferPool else { return nil }
|
|
306
|
-
let status = CVPixelBufferPoolCreatePixelBuffer(
|
|
307
|
-
nil,
|
|
308
|
-
pool,
|
|
309
|
-
&pixelBuffer
|
|
310
|
-
)
|
|
311
|
-
guard status == kCVReturnSuccess, let pixelBuffer else { return nil }
|
|
312
|
-
|
|
313
|
-
CVPixelBufferLockBaseAddress(pixelBuffer, [])
|
|
314
|
-
defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
|
|
315
|
-
guard
|
|
316
|
-
let context = CGContext(
|
|
317
|
-
data: CVPixelBufferGetBaseAddress(pixelBuffer),
|
|
318
|
-
width: image.width,
|
|
319
|
-
height: image.height,
|
|
320
|
-
bitsPerComponent: 8,
|
|
321
|
-
bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
|
|
322
|
-
space: CGColorSpaceCreateDeviceRGB(),
|
|
323
|
-
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue
|
|
324
|
-
)
|
|
325
|
-
else {
|
|
326
|
-
return nil
|
|
327
|
-
}
|
|
328
|
-
context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
|
|
329
|
-
return pixelBuffer
|
|
330
|
-
}
|
|
331
|
-
}
|
|
74
|
+
// MARK: - XCTest Entry
|
|
332
75
|
|
|
333
76
|
override func setUp() {
|
|
334
77
|
continueAfterFailure = true
|
|
@@ -340,7 +83,7 @@ final class RunnerTests: XCTestCase {
|
|
|
340
83
|
app.launch()
|
|
341
84
|
currentApp = app
|
|
342
85
|
let queue = DispatchQueue(label: "agent-device.runner")
|
|
343
|
-
let desiredPort =
|
|
86
|
+
let desiredPort = RunnerEnv.resolvePort()
|
|
344
87
|
NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", desiredPort)
|
|
345
88
|
if desiredPort > 0, let port = NWEndpoint.Port(rawValue: desiredPort) {
|
|
346
89
|
listener = try NWListener(using: .tcp, on: port)
|
|
@@ -380,1572 +123,4 @@ final class RunnerTests: XCTestCase {
|
|
|
380
123
|
XCTFail("runner wait ended with \(result)")
|
|
381
124
|
}
|
|
382
125
|
}
|
|
383
|
-
|
|
384
|
-
private func handle(connection: NWConnection) {
|
|
385
|
-
receiveRequest(connection: connection, buffer: Data())
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
private func receiveRequest(connection: NWConnection, buffer: Data) {
|
|
389
|
-
connection.receive(minimumIncompleteLength: 1, maximumLength: 1024 * 1024) { [weak self] data, _, _, _ in
|
|
390
|
-
guard let self = self, let data = data else {
|
|
391
|
-
connection.cancel()
|
|
392
|
-
return
|
|
393
|
-
}
|
|
394
|
-
if buffer.count + data.count > self.maxRequestBytes {
|
|
395
|
-
let response = self.jsonResponse(
|
|
396
|
-
status: 413,
|
|
397
|
-
response: Response(ok: false, error: ErrorPayload(message: "request too large")),
|
|
398
|
-
)
|
|
399
|
-
connection.send(content: response, completion: .contentProcessed { [weak self] _ in
|
|
400
|
-
connection.cancel()
|
|
401
|
-
self?.finish()
|
|
402
|
-
})
|
|
403
|
-
return
|
|
404
|
-
}
|
|
405
|
-
let combined = buffer + data
|
|
406
|
-
if let body = self.parseRequest(data: combined) {
|
|
407
|
-
let result = self.handleRequestBody(body)
|
|
408
|
-
connection.send(content: result.data, completion: .contentProcessed { _ in
|
|
409
|
-
connection.cancel()
|
|
410
|
-
if result.shouldFinish {
|
|
411
|
-
self.finish()
|
|
412
|
-
}
|
|
413
|
-
})
|
|
414
|
-
} else {
|
|
415
|
-
self.receiveRequest(connection: connection, buffer: combined)
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
private func parseRequest(data: Data) -> Data? {
|
|
421
|
-
guard let headerEnd = data.range(of: Data("\r\n\r\n".utf8)) else {
|
|
422
|
-
return nil
|
|
423
|
-
}
|
|
424
|
-
let headerData = data.subdata(in: 0..<headerEnd.lowerBound)
|
|
425
|
-
let bodyStart = headerEnd.upperBound
|
|
426
|
-
let headers = String(decoding: headerData, as: UTF8.self)
|
|
427
|
-
let contentLength = extractContentLength(headers: headers)
|
|
428
|
-
guard let contentLength = contentLength else {
|
|
429
|
-
return nil
|
|
430
|
-
}
|
|
431
|
-
if data.count < bodyStart + contentLength {
|
|
432
|
-
return nil
|
|
433
|
-
}
|
|
434
|
-
let body = data.subdata(in: bodyStart..<(bodyStart + contentLength))
|
|
435
|
-
return body
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
private func extractContentLength(headers: String) -> Int? {
|
|
439
|
-
for line in headers.split(separator: "\r\n") {
|
|
440
|
-
let parts = line.split(separator: ":", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }
|
|
441
|
-
if parts.count == 2 && parts[0].lowercased() == "content-length" {
|
|
442
|
-
return Int(parts[1])
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
return nil
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
private func handleRequestBody(_ body: Data) -> (data: Data, shouldFinish: Bool) {
|
|
449
|
-
guard let json = String(data: body, encoding: .utf8) else {
|
|
450
|
-
return (
|
|
451
|
-
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
452
|
-
false
|
|
453
|
-
)
|
|
454
|
-
}
|
|
455
|
-
guard let data = json.data(using: .utf8) else {
|
|
456
|
-
return (
|
|
457
|
-
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
458
|
-
false
|
|
459
|
-
)
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
do {
|
|
463
|
-
let command = try JSONDecoder().decode(Command.self, from: data)
|
|
464
|
-
let response = try execute(command: command)
|
|
465
|
-
return (jsonResponse(status: 200, response: response), command.command == .shutdown)
|
|
466
|
-
} catch {
|
|
467
|
-
return (
|
|
468
|
-
jsonResponse(status: 500, response: Response(ok: false, error: ErrorPayload(message: "\(error)"))),
|
|
469
|
-
false
|
|
470
|
-
)
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
private func execute(command: Command) throws -> Response {
|
|
475
|
-
if Thread.isMainThread {
|
|
476
|
-
return try executeOnMainSafely(command: command)
|
|
477
|
-
}
|
|
478
|
-
var result: Result<Response, Error>?
|
|
479
|
-
let semaphore = DispatchSemaphore(value: 0)
|
|
480
|
-
DispatchQueue.main.async {
|
|
481
|
-
do {
|
|
482
|
-
result = .success(try self.executeOnMainSafely(command: command))
|
|
483
|
-
} catch {
|
|
484
|
-
result = .failure(error)
|
|
485
|
-
}
|
|
486
|
-
semaphore.signal()
|
|
487
|
-
}
|
|
488
|
-
let waitResult = semaphore.wait(timeout: .now() + mainThreadExecutionTimeout)
|
|
489
|
-
if waitResult == .timedOut {
|
|
490
|
-
// The main queue work may still be running; we stop waiting and report timeout.
|
|
491
|
-
throw NSError(
|
|
492
|
-
domain: RunnerErrorDomain.general,
|
|
493
|
-
code: RunnerErrorCode.mainThreadExecutionTimedOut,
|
|
494
|
-
userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"]
|
|
495
|
-
)
|
|
496
|
-
}
|
|
497
|
-
switch result {
|
|
498
|
-
case .success(let response):
|
|
499
|
-
return response
|
|
500
|
-
case .failure(let error):
|
|
501
|
-
throw error
|
|
502
|
-
case .none:
|
|
503
|
-
throw NSError(
|
|
504
|
-
domain: RunnerErrorDomain.general,
|
|
505
|
-
code: RunnerErrorCode.noResponseFromMainThread,
|
|
506
|
-
userInfo: [NSLocalizedDescriptionKey: "no response from main thread"]
|
|
507
|
-
)
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
private func executeOnMainSafely(command: Command) throws -> Response {
|
|
512
|
-
var hasRetried = false
|
|
513
|
-
while true {
|
|
514
|
-
var response: Response?
|
|
515
|
-
var swiftError: Error?
|
|
516
|
-
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
517
|
-
do {
|
|
518
|
-
response = try self.executeOnMain(command: command)
|
|
519
|
-
} catch {
|
|
520
|
-
swiftError = error
|
|
521
|
-
}
|
|
522
|
-
})
|
|
523
|
-
|
|
524
|
-
if let exceptionMessage {
|
|
525
|
-
currentApp = nil
|
|
526
|
-
currentBundleId = nil
|
|
527
|
-
if !hasRetried, shouldRetryException(command, message: exceptionMessage) {
|
|
528
|
-
NSLog(
|
|
529
|
-
"AGENT_DEVICE_RUNNER_RETRY command=%@ reason=objc_exception",
|
|
530
|
-
command.command.rawValue
|
|
531
|
-
)
|
|
532
|
-
hasRetried = true
|
|
533
|
-
sleepFor(retryCooldown)
|
|
534
|
-
continue
|
|
535
|
-
}
|
|
536
|
-
throw NSError(
|
|
537
|
-
domain: RunnerErrorDomain.exception,
|
|
538
|
-
code: RunnerErrorCode.objcException,
|
|
539
|
-
userInfo: [NSLocalizedDescriptionKey: exceptionMessage]
|
|
540
|
-
)
|
|
541
|
-
}
|
|
542
|
-
if let swiftError {
|
|
543
|
-
throw swiftError
|
|
544
|
-
}
|
|
545
|
-
guard let response else {
|
|
546
|
-
throw NSError(
|
|
547
|
-
domain: RunnerErrorDomain.general,
|
|
548
|
-
code: RunnerErrorCode.commandReturnedNoResponse,
|
|
549
|
-
userInfo: [NSLocalizedDescriptionKey: "command returned no response"]
|
|
550
|
-
)
|
|
551
|
-
}
|
|
552
|
-
if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) {
|
|
553
|
-
NSLog(
|
|
554
|
-
"AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable",
|
|
555
|
-
command.command.rawValue
|
|
556
|
-
)
|
|
557
|
-
hasRetried = true
|
|
558
|
-
currentApp = nil
|
|
559
|
-
currentBundleId = nil
|
|
560
|
-
sleepFor(retryCooldown)
|
|
561
|
-
continue
|
|
562
|
-
}
|
|
563
|
-
return response
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
private func executeOnMain(command: Command) throws -> Response {
|
|
568
|
-
var activeApp = currentApp ?? app
|
|
569
|
-
if !isRunnerLifecycleCommand(command.command) {
|
|
570
|
-
let normalizedBundleId = command.appBundleId?
|
|
571
|
-
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
572
|
-
let requestedBundleId = (normalizedBundleId?.isEmpty == true) ? nil : normalizedBundleId
|
|
573
|
-
if let bundleId = requestedBundleId {
|
|
574
|
-
if currentBundleId != bundleId || currentApp == nil {
|
|
575
|
-
_ = activateTarget(bundleId: bundleId, reason: "bundle_changed")
|
|
576
|
-
}
|
|
577
|
-
} else {
|
|
578
|
-
// Do not reuse stale bundle targets when the caller does not explicitly request one.
|
|
579
|
-
currentApp = nil
|
|
580
|
-
currentBundleId = nil
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
activeApp = currentApp ?? app
|
|
584
|
-
if let bundleId = requestedBundleId, targetNeedsActivation(activeApp) {
|
|
585
|
-
activeApp = activateTarget(bundleId: bundleId, reason: "stale_target")
|
|
586
|
-
} else if requestedBundleId == nil, targetNeedsActivation(activeApp) {
|
|
587
|
-
app.activate()
|
|
588
|
-
activeApp = app
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
if !activeApp.waitForExistence(timeout: appExistenceTimeout) {
|
|
592
|
-
if let bundleId = requestedBundleId {
|
|
593
|
-
activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait")
|
|
594
|
-
guard activeApp.waitForExistence(timeout: appExistenceTimeout) else {
|
|
595
|
-
return Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))
|
|
596
|
-
}
|
|
597
|
-
} else {
|
|
598
|
-
return Response(ok: false, error: ErrorPayload(message: "runner app is not available"))
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
if isInteractionCommand(command.command) {
|
|
603
|
-
if let bundleId = requestedBundleId, activeApp.state != .runningForeground {
|
|
604
|
-
activeApp = activateTarget(bundleId: bundleId, reason: "interaction_foreground_guard")
|
|
605
|
-
} else if requestedBundleId == nil, activeApp.state != .runningForeground {
|
|
606
|
-
app.activate()
|
|
607
|
-
activeApp = app
|
|
608
|
-
}
|
|
609
|
-
if !activeApp.waitForExistence(timeout: 2) {
|
|
610
|
-
if let bundleId = requestedBundleId {
|
|
611
|
-
return Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))
|
|
612
|
-
}
|
|
613
|
-
return Response(ok: false, error: ErrorPayload(message: "runner app is not available"))
|
|
614
|
-
}
|
|
615
|
-
applyInteractionStabilizationIfNeeded()
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
switch command.command {
|
|
620
|
-
case .shutdown:
|
|
621
|
-
stopRecordingIfNeeded()
|
|
622
|
-
return Response(ok: true, data: DataPayload(message: "shutdown"))
|
|
623
|
-
case .recordStart:
|
|
624
|
-
guard
|
|
625
|
-
let requestedOutPath = command.outPath?.trimmingCharacters(in: .whitespacesAndNewlines),
|
|
626
|
-
!requestedOutPath.isEmpty
|
|
627
|
-
else {
|
|
628
|
-
return Response(ok: false, error: ErrorPayload(message: "recordStart requires outPath"))
|
|
629
|
-
}
|
|
630
|
-
let hasAppBundleId = !(command.appBundleId?
|
|
631
|
-
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
632
|
-
.isEmpty ?? true)
|
|
633
|
-
guard hasAppBundleId else {
|
|
634
|
-
return Response(ok: false, error: ErrorPayload(message: "recordStart requires appBundleId"))
|
|
635
|
-
}
|
|
636
|
-
if activeRecording != nil {
|
|
637
|
-
return Response(ok: false, error: ErrorPayload(message: "recording already in progress"))
|
|
638
|
-
}
|
|
639
|
-
if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) {
|
|
640
|
-
return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)"))
|
|
641
|
-
}
|
|
642
|
-
do {
|
|
643
|
-
let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
|
|
644
|
-
let fpsLabel = command.fps.map(String.init) ?? "max"
|
|
645
|
-
NSLog(
|
|
646
|
-
"AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
|
|
647
|
-
requestedOutPath,
|
|
648
|
-
resolvedOutPath,
|
|
649
|
-
fpsLabel
|
|
650
|
-
)
|
|
651
|
-
let recorder = ScreenRecorder(outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) })
|
|
652
|
-
try recorder.start { [weak self] in
|
|
653
|
-
return self?.captureRunnerFrame()
|
|
654
|
-
}
|
|
655
|
-
activeRecording = recorder
|
|
656
|
-
return Response(ok: true, data: DataPayload(message: "recording started"))
|
|
657
|
-
} catch {
|
|
658
|
-
activeRecording = nil
|
|
659
|
-
return Response(ok: false, error: ErrorPayload(message: "failed to start recording: \(error.localizedDescription)"))
|
|
660
|
-
}
|
|
661
|
-
case .recordStop:
|
|
662
|
-
guard let recorder = activeRecording else {
|
|
663
|
-
return Response(ok: false, error: ErrorPayload(message: "no active recording"))
|
|
664
|
-
}
|
|
665
|
-
do {
|
|
666
|
-
try recorder.stop()
|
|
667
|
-
activeRecording = nil
|
|
668
|
-
return Response(ok: true, data: DataPayload(message: "recording stopped"))
|
|
669
|
-
} catch {
|
|
670
|
-
activeRecording = nil
|
|
671
|
-
return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)"))
|
|
672
|
-
}
|
|
673
|
-
case .tap:
|
|
674
|
-
if let text = command.text {
|
|
675
|
-
if let element = findElement(app: activeApp, text: text) {
|
|
676
|
-
element.tap()
|
|
677
|
-
return Response(ok: true, data: DataPayload(message: "tapped"))
|
|
678
|
-
}
|
|
679
|
-
return Response(ok: false, error: ErrorPayload(message: "element not found"))
|
|
680
|
-
}
|
|
681
|
-
if let x = command.x, let y = command.y {
|
|
682
|
-
tapAt(app: activeApp, x: x, y: y)
|
|
683
|
-
return Response(ok: true, data: DataPayload(message: "tapped"))
|
|
684
|
-
}
|
|
685
|
-
return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
|
|
686
|
-
case .tapSeries:
|
|
687
|
-
guard let x = command.x, let y = command.y else {
|
|
688
|
-
return Response(ok: false, error: ErrorPayload(message: "tapSeries requires x and y"))
|
|
689
|
-
}
|
|
690
|
-
let count = max(Int(command.count ?? 1), 1)
|
|
691
|
-
let intervalMs = max(command.intervalMs ?? 0, 0)
|
|
692
|
-
let doubleTap = command.doubleTap ?? false
|
|
693
|
-
if doubleTap {
|
|
694
|
-
runSeries(count: count, pauseMs: intervalMs) { _ in
|
|
695
|
-
doubleTapAt(app: activeApp, x: x, y: y)
|
|
696
|
-
}
|
|
697
|
-
return Response(ok: true, data: DataPayload(message: "tap series"))
|
|
698
|
-
}
|
|
699
|
-
runSeries(count: count, pauseMs: intervalMs) { _ in
|
|
700
|
-
tapAt(app: activeApp, x: x, y: y)
|
|
701
|
-
}
|
|
702
|
-
return Response(ok: true, data: DataPayload(message: "tap series"))
|
|
703
|
-
case .longPress:
|
|
704
|
-
guard let x = command.x, let y = command.y else {
|
|
705
|
-
return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y"))
|
|
706
|
-
}
|
|
707
|
-
let duration = (command.durationMs ?? 800) / 1000.0
|
|
708
|
-
longPressAt(app: activeApp, x: x, y: y, duration: duration)
|
|
709
|
-
return Response(ok: true, data: DataPayload(message: "long pressed"))
|
|
710
|
-
case .drag:
|
|
711
|
-
guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
|
|
712
|
-
return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
|
|
713
|
-
}
|
|
714
|
-
let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
|
|
715
|
-
withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
|
|
716
|
-
dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
|
|
717
|
-
}
|
|
718
|
-
return Response(ok: true, data: DataPayload(message: "dragged"))
|
|
719
|
-
case .dragSeries:
|
|
720
|
-
guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
|
|
721
|
-
return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))
|
|
722
|
-
}
|
|
723
|
-
let count = max(Int(command.count ?? 1), 1)
|
|
724
|
-
let pauseMs = max(command.pauseMs ?? 0, 0)
|
|
725
|
-
let pattern = command.pattern ?? "one-way"
|
|
726
|
-
if pattern != "one-way" && pattern != "ping-pong" {
|
|
727
|
-
return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
|
|
728
|
-
}
|
|
729
|
-
let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
|
|
730
|
-
withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
|
|
731
|
-
runSeries(count: count, pauseMs: pauseMs) { idx in
|
|
732
|
-
let reverse = pattern == "ping-pong" && (idx % 2 == 1)
|
|
733
|
-
if reverse {
|
|
734
|
-
dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
|
|
735
|
-
} else {
|
|
736
|
-
dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
return Response(ok: true, data: DataPayload(message: "drag series"))
|
|
741
|
-
case .type:
|
|
742
|
-
guard let text = command.text else {
|
|
743
|
-
return Response(ok: false, error: ErrorPayload(message: "type requires text"))
|
|
744
|
-
}
|
|
745
|
-
if command.clearFirst == true {
|
|
746
|
-
guard let focused = focusedTextInput(app: activeApp) else {
|
|
747
|
-
return Response(ok: false, error: ErrorPayload(message: "no focused text input to clear"))
|
|
748
|
-
}
|
|
749
|
-
clearTextInput(focused)
|
|
750
|
-
focused.typeText(text)
|
|
751
|
-
return Response(ok: true, data: DataPayload(message: "typed"))
|
|
752
|
-
}
|
|
753
|
-
if let focused = focusedTextInput(app: activeApp) {
|
|
754
|
-
focused.typeText(text)
|
|
755
|
-
} else {
|
|
756
|
-
activeApp.typeText(text)
|
|
757
|
-
}
|
|
758
|
-
return Response(ok: true, data: DataPayload(message: "typed"))
|
|
759
|
-
case .swipe:
|
|
760
|
-
guard let direction = command.direction else {
|
|
761
|
-
return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
|
|
762
|
-
}
|
|
763
|
-
withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
|
|
764
|
-
swipe(app: activeApp, direction: direction)
|
|
765
|
-
}
|
|
766
|
-
return Response(ok: true, data: DataPayload(message: "swiped"))
|
|
767
|
-
case .findText:
|
|
768
|
-
guard let text = command.text else {
|
|
769
|
-
return Response(ok: false, error: ErrorPayload(message: "findText requires text"))
|
|
770
|
-
}
|
|
771
|
-
let found = findElement(app: activeApp, text: text) != nil
|
|
772
|
-
return Response(ok: true, data: DataPayload(found: found))
|
|
773
|
-
case .snapshot:
|
|
774
|
-
let options = SnapshotOptions(
|
|
775
|
-
interactiveOnly: command.interactiveOnly ?? false,
|
|
776
|
-
compact: command.compact ?? false,
|
|
777
|
-
depth: command.depth,
|
|
778
|
-
scope: command.scope,
|
|
779
|
-
raw: command.raw ?? false,
|
|
780
|
-
)
|
|
781
|
-
if options.raw {
|
|
782
|
-
needsPostSnapshotInteractionDelay = true
|
|
783
|
-
return Response(ok: true, data: snapshotRaw(app: activeApp, options: options))
|
|
784
|
-
}
|
|
785
|
-
needsPostSnapshotInteractionDelay = true
|
|
786
|
-
return Response(ok: true, data: snapshotFast(app: activeApp, options: options))
|
|
787
|
-
case .screenshot:
|
|
788
|
-
// If a target app bundle ID is provided, activate it first so the screenshot
|
|
789
|
-
// captures the target app rather than the AgentDeviceRunner itself.
|
|
790
|
-
if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
791
|
-
let targetApp = XCUIApplication(bundleIdentifier: bundleId)
|
|
792
|
-
targetApp.activate()
|
|
793
|
-
// Brief wait for the app transition animation to complete
|
|
794
|
-
Thread.sleep(forTimeInterval: 0.5)
|
|
795
|
-
}
|
|
796
|
-
let screenshot = XCUIScreen.main.screenshot()
|
|
797
|
-
guard let pngData = screenshot.image.pngData() else {
|
|
798
|
-
return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
|
|
799
|
-
}
|
|
800
|
-
let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
|
|
801
|
-
let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
|
|
802
|
-
do {
|
|
803
|
-
try pngData.write(to: URL(fileURLWithPath: filePath))
|
|
804
|
-
} catch {
|
|
805
|
-
return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)"))
|
|
806
|
-
}
|
|
807
|
-
// Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
|
|
808
|
-
return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
|
|
809
|
-
case .back:
|
|
810
|
-
if tapNavigationBack(app: activeApp) {
|
|
811
|
-
return Response(ok: true, data: DataPayload(message: "back"))
|
|
812
|
-
}
|
|
813
|
-
performBackGesture(app: activeApp)
|
|
814
|
-
return Response(ok: true, data: DataPayload(message: "back"))
|
|
815
|
-
case .home:
|
|
816
|
-
pressHomeButton()
|
|
817
|
-
return Response(ok: true, data: DataPayload(message: "home"))
|
|
818
|
-
case .appSwitcher:
|
|
819
|
-
performAppSwitcherGesture(app: activeApp)
|
|
820
|
-
return Response(ok: true, data: DataPayload(message: "appSwitcher"))
|
|
821
|
-
case .alert:
|
|
822
|
-
let action = (command.action ?? "get").lowercased()
|
|
823
|
-
let alert = activeApp.alerts.firstMatch
|
|
824
|
-
if !alert.exists {
|
|
825
|
-
return Response(ok: false, error: ErrorPayload(message: "alert not found"))
|
|
826
|
-
}
|
|
827
|
-
if action == "accept" {
|
|
828
|
-
let button = alert.buttons.allElementsBoundByIndex.first
|
|
829
|
-
button?.tap()
|
|
830
|
-
return Response(ok: true, data: DataPayload(message: "accepted"))
|
|
831
|
-
}
|
|
832
|
-
if action == "dismiss" {
|
|
833
|
-
let button = alert.buttons.allElementsBoundByIndex.last
|
|
834
|
-
button?.tap()
|
|
835
|
-
return Response(ok: true, data: DataPayload(message: "dismissed"))
|
|
836
|
-
}
|
|
837
|
-
let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
|
|
838
|
-
return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
|
|
839
|
-
case .pinch:
|
|
840
|
-
guard let scale = command.scale, scale > 0 else {
|
|
841
|
-
return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0"))
|
|
842
|
-
}
|
|
843
|
-
pinch(app: activeApp, scale: scale, x: command.x, y: command.y)
|
|
844
|
-
return Response(ok: true, data: DataPayload(message: "pinched"))
|
|
845
|
-
}
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
private func captureRunnerFrame() -> UIImage? {
|
|
849
|
-
var image: UIImage?
|
|
850
|
-
let capture = {
|
|
851
|
-
let screenshot = XCUIScreen.main.screenshot()
|
|
852
|
-
image = screenshot.image
|
|
853
|
-
}
|
|
854
|
-
if Thread.isMainThread {
|
|
855
|
-
capture()
|
|
856
|
-
} else {
|
|
857
|
-
DispatchQueue.main.sync(execute: capture)
|
|
858
|
-
}
|
|
859
|
-
return image
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
private func stopRecordingIfNeeded() {
|
|
863
|
-
guard let recorder = activeRecording else { return }
|
|
864
|
-
do {
|
|
865
|
-
try recorder.stop()
|
|
866
|
-
} catch {
|
|
867
|
-
NSLog("AGENT_DEVICE_RUNNER_RECORD_STOP_FAILED=%@", String(describing: error))
|
|
868
|
-
}
|
|
869
|
-
activeRecording = nil
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
private func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
|
|
873
|
-
let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
|
|
874
|
-
let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
|
|
875
|
-
let safeFileName = fileName.isEmpty ? fallbackName : fileName
|
|
876
|
-
return (NSTemporaryDirectory() as NSString).appendingPathComponent(safeFileName)
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
private func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
|
|
880
|
-
switch target.state {
|
|
881
|
-
case .unknown, .notRunning, .runningBackground, .runningBackgroundSuspended:
|
|
882
|
-
return true
|
|
883
|
-
default:
|
|
884
|
-
return false
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
private func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
|
|
889
|
-
let target = XCUIApplication(bundleIdentifier: bundleId)
|
|
890
|
-
NSLog(
|
|
891
|
-
"AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
|
|
892
|
-
bundleId,
|
|
893
|
-
target.state.rawValue,
|
|
894
|
-
reason
|
|
895
|
-
)
|
|
896
|
-
// activate avoids terminating and relaunching the target app
|
|
897
|
-
target.activate()
|
|
898
|
-
currentApp = target
|
|
899
|
-
currentBundleId = bundleId
|
|
900
|
-
needsFirstInteractionDelay = true
|
|
901
|
-
return target
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
private func withTemporaryScrollIdleTimeoutIfSupported(
|
|
905
|
-
_ target: XCUIApplication,
|
|
906
|
-
operation: () -> Void
|
|
907
|
-
) {
|
|
908
|
-
let setter = NSSelectorFromString("setWaitForIdleTimeout:")
|
|
909
|
-
guard target.responds(to: setter) else {
|
|
910
|
-
operation()
|
|
911
|
-
return
|
|
912
|
-
}
|
|
913
|
-
let previous = target.value(forKey: "waitForIdleTimeout") as? NSNumber
|
|
914
|
-
target.setValue(resolveScrollInteractionIdleTimeout(), forKey: "waitForIdleTimeout")
|
|
915
|
-
defer {
|
|
916
|
-
if let previous {
|
|
917
|
-
target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout")
|
|
918
|
-
}
|
|
919
|
-
}
|
|
920
|
-
operation()
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
private func resolveScrollInteractionIdleTimeout() -> TimeInterval {
|
|
924
|
-
guard
|
|
925
|
-
let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_IOS_INTERACTION_IDLE_TIMEOUT"],
|
|
926
|
-
!raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
927
|
-
else {
|
|
928
|
-
return scrollInteractionIdleTimeoutDefault
|
|
929
|
-
}
|
|
930
|
-
guard let parsed = Double(raw), parsed >= 0 else {
|
|
931
|
-
return scrollInteractionIdleTimeoutDefault
|
|
932
|
-
}
|
|
933
|
-
return min(parsed, 30)
|
|
934
|
-
}
|
|
935
|
-
|
|
936
|
-
private func shouldRetryCommand(_ command: Command) -> Bool {
|
|
937
|
-
if isEnvTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") {
|
|
938
|
-
return false
|
|
939
|
-
}
|
|
940
|
-
return isReadOnlyCommand(command)
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
private func shouldRetryException(_ command: Command, message: String) -> Bool {
|
|
944
|
-
guard shouldRetryCommand(command) else { return false }
|
|
945
|
-
let normalized = message.lowercased()
|
|
946
|
-
if normalized.contains("kaxerrorservernotfound") {
|
|
947
|
-
return true
|
|
948
|
-
}
|
|
949
|
-
if normalized.contains("main thread execution timed out") {
|
|
950
|
-
return true
|
|
951
|
-
}
|
|
952
|
-
if normalized.contains("timed out") && command.command == .snapshot {
|
|
953
|
-
return true
|
|
954
|
-
}
|
|
955
|
-
return false
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
private func isReadOnlyCommand(_ command: Command) -> Bool {
|
|
959
|
-
switch command.command {
|
|
960
|
-
case .findText, .snapshot, .screenshot:
|
|
961
|
-
return true
|
|
962
|
-
case .alert:
|
|
963
|
-
let action = (command.action ?? "get").lowercased()
|
|
964
|
-
return action == "get"
|
|
965
|
-
default:
|
|
966
|
-
return false
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
private func shouldRetryResponse(_ response: Response) -> Bool {
|
|
971
|
-
guard response.ok == false else { return false }
|
|
972
|
-
guard let message = response.error?.message.lowercased() else { return false }
|
|
973
|
-
return message.contains("is not available")
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
private func isInteractionCommand(_ command: CommandType) -> Bool {
|
|
977
|
-
switch command {
|
|
978
|
-
case .tap, .longPress, .drag, .type, .swipe, .back, .appSwitcher, .pinch:
|
|
979
|
-
return true
|
|
980
|
-
default:
|
|
981
|
-
return false
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
|
|
985
|
-
private func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
|
|
986
|
-
switch command {
|
|
987
|
-
case .shutdown, .recordStop, .screenshot:
|
|
988
|
-
return true
|
|
989
|
-
default:
|
|
990
|
-
return false
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
private func applyInteractionStabilizationIfNeeded() {
|
|
995
|
-
if needsPostSnapshotInteractionDelay {
|
|
996
|
-
sleepFor(postSnapshotInteractionDelay)
|
|
997
|
-
needsPostSnapshotInteractionDelay = false
|
|
998
|
-
}
|
|
999
|
-
if needsFirstInteractionDelay {
|
|
1000
|
-
sleepFor(firstInteractionAfterActivateDelay)
|
|
1001
|
-
needsFirstInteractionDelay = false
|
|
1002
|
-
}
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
private func sleepFor(_ delay: TimeInterval) {
|
|
1006
|
-
guard delay > 0 else { return }
|
|
1007
|
-
usleep(useconds_t(delay * 1_000_000))
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
private func tapNavigationBack(app: XCUIApplication) -> Bool {
|
|
1011
|
-
let buttons = app.navigationBars.buttons.allElementsBoundByIndex
|
|
1012
|
-
if let back = buttons.first(where: { $0.isHittable }) {
|
|
1013
|
-
back.tap()
|
|
1014
|
-
return true
|
|
1015
|
-
}
|
|
1016
|
-
return pressTvRemoteMenuIfAvailable()
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
private func performBackGesture(app: XCUIApplication) {
|
|
1020
|
-
if pressTvRemoteMenuIfAvailable() {
|
|
1021
|
-
return
|
|
1022
|
-
}
|
|
1023
|
-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
1024
|
-
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
|
|
1025
|
-
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
1026
|
-
start.press(forDuration: 0.05, thenDragTo: end)
|
|
1027
|
-
}
|
|
1028
|
-
|
|
1029
|
-
private func performAppSwitcherGesture(app: XCUIApplication) {
|
|
1030
|
-
if performTvRemoteAppSwitcherIfAvailable() {
|
|
1031
|
-
return
|
|
1032
|
-
}
|
|
1033
|
-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
1034
|
-
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
|
|
1035
|
-
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
|
|
1036
|
-
start.press(forDuration: 0.6, thenDragTo: end)
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
private func pressHomeButton() {
|
|
1040
|
-
if pressTvRemoteHomeIfAvailable() {
|
|
1041
|
-
return
|
|
1042
|
-
}
|
|
1043
|
-
XCUIDevice.shared.press(.home)
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
private func pressTvRemoteMenuIfAvailable() -> Bool {
|
|
1047
|
-
#if os(tvOS)
|
|
1048
|
-
XCUIRemote.shared.press(.menu)
|
|
1049
|
-
return true
|
|
1050
|
-
#else
|
|
1051
|
-
return false
|
|
1052
|
-
#endif
|
|
1053
|
-
}
|
|
1054
|
-
|
|
1055
|
-
private func pressTvRemoteHomeIfAvailable() -> Bool {
|
|
1056
|
-
#if os(tvOS)
|
|
1057
|
-
XCUIRemote.shared.press(.home)
|
|
1058
|
-
return true
|
|
1059
|
-
#else
|
|
1060
|
-
return false
|
|
1061
|
-
#endif
|
|
1062
|
-
}
|
|
1063
|
-
|
|
1064
|
-
private func performTvRemoteAppSwitcherIfAvailable() -> Bool {
|
|
1065
|
-
#if os(tvOS)
|
|
1066
|
-
XCUIRemote.shared.press(.home)
|
|
1067
|
-
sleepFor(resolveTvRemoteDoublePressDelay())
|
|
1068
|
-
XCUIRemote.shared.press(.home)
|
|
1069
|
-
return true
|
|
1070
|
-
#else
|
|
1071
|
-
return false
|
|
1072
|
-
#endif
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
private func resolveTvRemoteDoublePressDelay() -> TimeInterval {
|
|
1076
|
-
guard
|
|
1077
|
-
let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],
|
|
1078
|
-
!raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
1079
|
-
else {
|
|
1080
|
-
return tvRemoteDoublePressDelayDefault
|
|
1081
|
-
}
|
|
1082
|
-
guard let parsedMs = Double(raw), parsedMs >= 0 else {
|
|
1083
|
-
return tvRemoteDoublePressDelayDefault
|
|
1084
|
-
}
|
|
1085
|
-
return min(parsedMs, 1000) / 1000.0
|
|
1086
|
-
}
|
|
1087
|
-
|
|
1088
|
-
private func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
|
|
1089
|
-
let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
|
|
1090
|
-
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
1091
|
-
return element.exists ? element : nil
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
private func clearTextInput(_ element: XCUIElement) {
|
|
1095
|
-
moveCaretToEnd(element: element)
|
|
1096
|
-
let count = estimatedDeleteCount(for: element)
|
|
1097
|
-
let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
|
|
1098
|
-
element.typeText(deletes)
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
private func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
|
|
1102
|
-
let focused = app
|
|
1103
|
-
.descendants(matching: .any)
|
|
1104
|
-
.matching(NSPredicate(format: "hasKeyboardFocus == 1"))
|
|
1105
|
-
.firstMatch
|
|
1106
|
-
guard focused.exists else { return nil }
|
|
1107
|
-
|
|
1108
|
-
switch focused.elementType {
|
|
1109
|
-
case .textField, .secureTextField, .searchField, .textView:
|
|
1110
|
-
return focused
|
|
1111
|
-
default:
|
|
1112
|
-
return nil
|
|
1113
|
-
}
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
private func moveCaretToEnd(element: XCUIElement) {
|
|
1117
|
-
let frame = element.frame
|
|
1118
|
-
guard !frame.isEmpty else {
|
|
1119
|
-
element.tap()
|
|
1120
|
-
return
|
|
1121
|
-
}
|
|
1122
|
-
let origin = element.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
1123
|
-
let target = origin.withOffset(
|
|
1124
|
-
CGVector(dx: max(2, frame.width - 4), dy: max(2, frame.height / 2))
|
|
1125
|
-
)
|
|
1126
|
-
target.tap()
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
private func estimatedDeleteCount(for element: XCUIElement) -> Int {
|
|
1130
|
-
let valueText = String(describing: element.value ?? "")
|
|
1131
|
-
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1132
|
-
let base = valueText.isEmpty ? 24 : (valueText.count + 8)
|
|
1133
|
-
return max(24, min(120, base))
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
private func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
|
|
1137
|
-
let predicate = NSPredicate(
|
|
1138
|
-
format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
|
|
1139
|
-
scope,
|
|
1140
|
-
scope
|
|
1141
|
-
)
|
|
1142
|
-
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
1143
|
-
return element.exists ? element : nil
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
private func tapAt(app: XCUIApplication, x: Double, y: Double) {
|
|
1147
|
-
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
1148
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
1149
|
-
coordinate.tap()
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
private func doubleTapAt(app: XCUIApplication, x: Double, y: Double) {
|
|
1153
|
-
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
1154
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
1155
|
-
coordinate.doubleTap()
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
private func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) {
|
|
1159
|
-
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
1160
|
-
let coordinate = origin.withOffset(CGVector(dx: x, dy: y))
|
|
1161
|
-
coordinate.press(forDuration: duration)
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
private func dragAt(
|
|
1165
|
-
app: XCUIApplication,
|
|
1166
|
-
x: Double,
|
|
1167
|
-
y: Double,
|
|
1168
|
-
x2: Double,
|
|
1169
|
-
y2: Double,
|
|
1170
|
-
holdDuration: TimeInterval
|
|
1171
|
-
) {
|
|
1172
|
-
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
1173
|
-
let start = origin.withOffset(CGVector(dx: x, dy: y))
|
|
1174
|
-
let end = origin.withOffset(CGVector(dx: x2, dy: y2))
|
|
1175
|
-
start.press(forDuration: holdDuration, thenDragTo: end)
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
private func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
|
|
1179
|
-
let total = max(count, 1)
|
|
1180
|
-
let pause = max(pauseMs, 0)
|
|
1181
|
-
for idx in 0..<total {
|
|
1182
|
-
operation(idx)
|
|
1183
|
-
if idx < total - 1 && pause > 0 {
|
|
1184
|
-
Thread.sleep(forTimeInterval: pause / 1000.0)
|
|
1185
|
-
}
|
|
1186
|
-
}
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
private func swipe(app: XCUIApplication, direction: SwipeDirection) {
|
|
1190
|
-
if performTvRemoteSwipeIfAvailable(direction: direction) {
|
|
1191
|
-
return
|
|
1192
|
-
}
|
|
1193
|
-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
1194
|
-
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
|
|
1195
|
-
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
|
|
1196
|
-
let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
|
|
1197
|
-
let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
1198
|
-
|
|
1199
|
-
switch direction {
|
|
1200
|
-
case .up:
|
|
1201
|
-
end.press(forDuration: 0.1, thenDragTo: start)
|
|
1202
|
-
case .down:
|
|
1203
|
-
start.press(forDuration: 0.1, thenDragTo: end)
|
|
1204
|
-
case .left:
|
|
1205
|
-
right.press(forDuration: 0.1, thenDragTo: left)
|
|
1206
|
-
case .right:
|
|
1207
|
-
left.press(forDuration: 0.1, thenDragTo: right)
|
|
1208
|
-
}
|
|
1209
|
-
}
|
|
1210
|
-
|
|
1211
|
-
private func performTvRemoteSwipeIfAvailable(direction: SwipeDirection) -> Bool {
|
|
1212
|
-
#if os(tvOS)
|
|
1213
|
-
switch direction {
|
|
1214
|
-
case .up:
|
|
1215
|
-
XCUIRemote.shared.press(.up)
|
|
1216
|
-
case .down:
|
|
1217
|
-
XCUIRemote.shared.press(.down)
|
|
1218
|
-
case .left:
|
|
1219
|
-
XCUIRemote.shared.press(.left)
|
|
1220
|
-
case .right:
|
|
1221
|
-
XCUIRemote.shared.press(.right)
|
|
1222
|
-
}
|
|
1223
|
-
return true
|
|
1224
|
-
#else
|
|
1225
|
-
return false
|
|
1226
|
-
#endif
|
|
1227
|
-
}
|
|
1228
|
-
|
|
1229
|
-
private func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) {
|
|
1230
|
-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
1231
|
-
|
|
1232
|
-
// Use double-tap + drag gesture for reliable map zoom
|
|
1233
|
-
// Zoom in (scale > 1): tap then drag UP
|
|
1234
|
-
// Zoom out (scale < 1): tap then drag DOWN
|
|
1235
|
-
|
|
1236
|
-
// Determine center point (use provided x/y or screen center)
|
|
1237
|
-
let centerX = x.map { $0 / target.frame.width } ?? 0.5
|
|
1238
|
-
let centerY = y.map { $0 / target.frame.height } ?? 0.5
|
|
1239
|
-
let center = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: centerY))
|
|
1240
|
-
|
|
1241
|
-
// Calculate drag distance based on scale (clamped to reasonable range)
|
|
1242
|
-
// Larger scale = more drag distance
|
|
1243
|
-
let dragAmount: CGFloat
|
|
1244
|
-
if scale > 1.0 {
|
|
1245
|
-
// Zoom in: drag up (negative Y direction in normalized coords)
|
|
1246
|
-
dragAmount = min(0.4, CGFloat(scale - 1.0) * 0.2)
|
|
1247
|
-
} else {
|
|
1248
|
-
// Zoom out: drag down (positive Y direction)
|
|
1249
|
-
dragAmount = min(0.4, CGFloat(1.0 - scale) * 0.4)
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
let endY = scale > 1.0 ? (centerY - Double(dragAmount)) : (centerY + Double(dragAmount))
|
|
1253
|
-
let endPoint = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: max(0.1, min(0.9, endY))))
|
|
1254
|
-
|
|
1255
|
-
// Tap first (first tap of double-tap)
|
|
1256
|
-
center.tap()
|
|
1257
|
-
|
|
1258
|
-
// Immediately press and drag (second tap + drag)
|
|
1259
|
-
center.press(forDuration: 0.05, thenDragTo: endPoint)
|
|
1260
|
-
}
|
|
1261
|
-
|
|
1262
|
-
private func aggregatedLabel(for element: XCUIElement, depth: Int = 0) -> String? {
|
|
1263
|
-
if depth > 2 { return nil }
|
|
1264
|
-
let text = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1265
|
-
if !text.isEmpty { return text }
|
|
1266
|
-
if let value = element.value {
|
|
1267
|
-
let valueText = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1268
|
-
if !valueText.isEmpty { return valueText }
|
|
1269
|
-
}
|
|
1270
|
-
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
1271
|
-
for child in children {
|
|
1272
|
-
if let childLabel = aggregatedLabel(for: child, depth: depth + 1) {
|
|
1273
|
-
return childLabel
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
return nil
|
|
1277
|
-
}
|
|
1278
|
-
|
|
1279
|
-
private func elementTypeName(_ type: XCUIElement.ElementType) -> String {
|
|
1280
|
-
switch type {
|
|
1281
|
-
case .application: return "Application"
|
|
1282
|
-
case .window: return "Window"
|
|
1283
|
-
case .button: return "Button"
|
|
1284
|
-
case .cell: return "Cell"
|
|
1285
|
-
case .staticText: return "StaticText"
|
|
1286
|
-
case .textField: return "TextField"
|
|
1287
|
-
case .textView: return "TextView"
|
|
1288
|
-
case .secureTextField: return "SecureTextField"
|
|
1289
|
-
case .switch: return "Switch"
|
|
1290
|
-
case .slider: return "Slider"
|
|
1291
|
-
case .link: return "Link"
|
|
1292
|
-
case .image: return "Image"
|
|
1293
|
-
case .navigationBar: return "NavigationBar"
|
|
1294
|
-
case .tabBar: return "TabBar"
|
|
1295
|
-
case .collectionView: return "CollectionView"
|
|
1296
|
-
case .table: return "Table"
|
|
1297
|
-
case .scrollView: return "ScrollView"
|
|
1298
|
-
case .searchField: return "SearchField"
|
|
1299
|
-
case .segmentedControl: return "SegmentedControl"
|
|
1300
|
-
case .stepper: return "Stepper"
|
|
1301
|
-
case .picker: return "Picker"
|
|
1302
|
-
case .checkBox: return "CheckBox"
|
|
1303
|
-
case .menuItem: return "MenuItem"
|
|
1304
|
-
case .other: return "Other"
|
|
1305
|
-
default:
|
|
1306
|
-
switch type.rawValue {
|
|
1307
|
-
case 19:
|
|
1308
|
-
return "Keyboard"
|
|
1309
|
-
case 20:
|
|
1310
|
-
return "Key"
|
|
1311
|
-
case 24:
|
|
1312
|
-
return "SearchField"
|
|
1313
|
-
default:
|
|
1314
|
-
return "Element(\(type.rawValue))"
|
|
1315
|
-
}
|
|
1316
|
-
}
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
private func snapshotFast(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
|
|
1320
|
-
if let blocking = blockingSystemAlertSnapshot() {
|
|
1321
|
-
return blocking
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
var nodes: [SnapshotNode] = []
|
|
1325
|
-
var truncated = false
|
|
1326
|
-
let maxDepth = options.depth ?? Int.max
|
|
1327
|
-
let viewport = app.frame
|
|
1328
|
-
let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
1329
|
-
|
|
1330
|
-
let rootSnapshot: XCUIElementSnapshot
|
|
1331
|
-
do {
|
|
1332
|
-
rootSnapshot = try queryRoot.snapshot()
|
|
1333
|
-
} catch {
|
|
1334
|
-
return DataPayload(nodes: nodes, truncated: truncated)
|
|
1335
|
-
}
|
|
1336
|
-
|
|
1337
|
-
let rootLabel = aggregatedLabel(for: rootSnapshot) ?? rootSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1338
|
-
let rootIdentifier = rootSnapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1339
|
-
let rootValue = snapshotValueText(rootSnapshot)
|
|
1340
|
-
nodes.append(
|
|
1341
|
-
SnapshotNode(
|
|
1342
|
-
index: 0,
|
|
1343
|
-
type: elementTypeName(rootSnapshot.elementType),
|
|
1344
|
-
label: rootLabel.isEmpty ? nil : rootLabel,
|
|
1345
|
-
identifier: rootIdentifier.isEmpty ? nil : rootIdentifier,
|
|
1346
|
-
value: rootValue,
|
|
1347
|
-
rect: SnapshotRect(
|
|
1348
|
-
x: Double(rootSnapshot.frame.origin.x),
|
|
1349
|
-
y: Double(rootSnapshot.frame.origin.y),
|
|
1350
|
-
width: Double(rootSnapshot.frame.size.width),
|
|
1351
|
-
height: Double(rootSnapshot.frame.size.height),
|
|
1352
|
-
),
|
|
1353
|
-
enabled: rootSnapshot.isEnabled,
|
|
1354
|
-
hittable: snapshotHittable(rootSnapshot),
|
|
1355
|
-
depth: 0,
|
|
1356
|
-
)
|
|
1357
|
-
)
|
|
1358
|
-
|
|
1359
|
-
var seen = Set<String>()
|
|
1360
|
-
var stack: [(XCUIElementSnapshot, Int, Int)] = rootSnapshot.children.map { ($0, 1, 1) }
|
|
1361
|
-
|
|
1362
|
-
while let (snapshot, depth, visibleDepth) = stack.popLast() {
|
|
1363
|
-
if nodes.count >= fastSnapshotLimit {
|
|
1364
|
-
truncated = true
|
|
1365
|
-
break
|
|
1366
|
-
}
|
|
1367
|
-
if let limit = options.depth, depth > limit { continue }
|
|
1368
|
-
|
|
1369
|
-
let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1370
|
-
let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1371
|
-
let valueText = snapshotValueText(snapshot)
|
|
1372
|
-
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
1373
|
-
if !isVisibleInViewport(snapshot.frame, viewport) && !hasContent {
|
|
1374
|
-
continue
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
|
-
let include = shouldInclude(
|
|
1378
|
-
snapshot: snapshot,
|
|
1379
|
-
label: label,
|
|
1380
|
-
identifier: identifier,
|
|
1381
|
-
valueText: valueText,
|
|
1382
|
-
options: options
|
|
1383
|
-
)
|
|
1384
|
-
|
|
1385
|
-
let key = "\(snapshot.elementType)-\(label)-\(identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
|
|
1386
|
-
let isDuplicate = seen.contains(key)
|
|
1387
|
-
if !isDuplicate {
|
|
1388
|
-
seen.insert(key)
|
|
1389
|
-
}
|
|
1390
|
-
|
|
1391
|
-
if depth < maxDepth {
|
|
1392
|
-
let nextVisibleDepth = include && !isDuplicate ? visibleDepth + 1 : visibleDepth
|
|
1393
|
-
for child in snapshot.children.reversed() {
|
|
1394
|
-
stack.append((child, depth + 1, nextVisibleDepth))
|
|
1395
|
-
}
|
|
1396
|
-
}
|
|
1397
|
-
|
|
1398
|
-
if !include || isDuplicate { continue }
|
|
1399
|
-
|
|
1400
|
-
nodes.append(
|
|
1401
|
-
SnapshotNode(
|
|
1402
|
-
index: nodes.count,
|
|
1403
|
-
type: elementTypeName(snapshot.elementType),
|
|
1404
|
-
label: label.isEmpty ? nil : label,
|
|
1405
|
-
identifier: identifier.isEmpty ? nil : identifier,
|
|
1406
|
-
value: valueText,
|
|
1407
|
-
rect: SnapshotRect(
|
|
1408
|
-
x: Double(snapshot.frame.origin.x),
|
|
1409
|
-
y: Double(snapshot.frame.origin.y),
|
|
1410
|
-
width: Double(snapshot.frame.size.width),
|
|
1411
|
-
height: Double(snapshot.frame.size.height),
|
|
1412
|
-
),
|
|
1413
|
-
enabled: snapshot.isEnabled,
|
|
1414
|
-
hittable: snapshotHittable(snapshot),
|
|
1415
|
-
depth: min(maxDepth, visibleDepth),
|
|
1416
|
-
)
|
|
1417
|
-
)
|
|
1418
|
-
|
|
1419
|
-
}
|
|
1420
|
-
|
|
1421
|
-
return DataPayload(nodes: nodes, truncated: truncated)
|
|
1422
|
-
}
|
|
1423
|
-
|
|
1424
|
-
private func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
|
|
1425
|
-
if let blocking = blockingSystemAlertSnapshot() {
|
|
1426
|
-
return blocking
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
let root = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
|
|
1430
|
-
var nodes: [SnapshotNode] = []
|
|
1431
|
-
var truncated = false
|
|
1432
|
-
let viewport = app.frame
|
|
1433
|
-
|
|
1434
|
-
func walk(_ element: XCUIElement, depth: Int) {
|
|
1435
|
-
if nodes.count >= maxSnapshotElements {
|
|
1436
|
-
truncated = true
|
|
1437
|
-
return
|
|
1438
|
-
}
|
|
1439
|
-
if let limit = options.depth, depth > limit { return }
|
|
1440
|
-
if !isVisibleInViewport(element.frame, viewport) { return }
|
|
1441
|
-
|
|
1442
|
-
let label = aggregatedLabel(for: element) ?? element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1443
|
-
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1444
|
-
let valueText: String? = {
|
|
1445
|
-
guard let value = element.value else { return nil }
|
|
1446
|
-
let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1447
|
-
return text.isEmpty ? nil : text
|
|
1448
|
-
}()
|
|
1449
|
-
if shouldInclude(element: element, label: label, identifier: identifier, valueText: valueText, options: options) {
|
|
1450
|
-
nodes.append(
|
|
1451
|
-
SnapshotNode(
|
|
1452
|
-
index: nodes.count,
|
|
1453
|
-
type: elementTypeName(element.elementType),
|
|
1454
|
-
label: label.isEmpty ? nil : label,
|
|
1455
|
-
identifier: identifier.isEmpty ? nil : identifier,
|
|
1456
|
-
value: valueText,
|
|
1457
|
-
rect: SnapshotRect(
|
|
1458
|
-
x: Double(element.frame.origin.x),
|
|
1459
|
-
y: Double(element.frame.origin.y),
|
|
1460
|
-
width: Double(element.frame.size.width),
|
|
1461
|
-
height: Double(element.frame.size.height),
|
|
1462
|
-
),
|
|
1463
|
-
enabled: element.isEnabled,
|
|
1464
|
-
hittable: element.isHittable,
|
|
1465
|
-
depth: depth,
|
|
1466
|
-
)
|
|
1467
|
-
)
|
|
1468
|
-
}
|
|
1469
|
-
|
|
1470
|
-
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
1471
|
-
for child in children {
|
|
1472
|
-
walk(child, depth: depth + 1)
|
|
1473
|
-
if truncated { return }
|
|
1474
|
-
}
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
walk(root, depth: 0)
|
|
1478
|
-
return DataPayload(nodes: nodes, truncated: truncated)
|
|
1479
|
-
}
|
|
1480
|
-
|
|
1481
|
-
private func blockingSystemAlertSnapshot() -> DataPayload? {
|
|
1482
|
-
guard let modal = firstBlockingSystemModal(in: springboard) else {
|
|
1483
|
-
return nil
|
|
1484
|
-
}
|
|
1485
|
-
let actions = actionableElements(in: modal)
|
|
1486
|
-
guard !actions.isEmpty else {
|
|
1487
|
-
return nil
|
|
1488
|
-
}
|
|
1489
|
-
|
|
1490
|
-
let title = preferredSystemModalTitle(modal)
|
|
1491
|
-
guard let modalNode = safeMakeSnapshotNode(
|
|
1492
|
-
element: modal,
|
|
1493
|
-
index: 0,
|
|
1494
|
-
type: "Alert",
|
|
1495
|
-
labelOverride: title,
|
|
1496
|
-
identifierOverride: modal.identifier,
|
|
1497
|
-
depth: 0,
|
|
1498
|
-
hittableOverride: true
|
|
1499
|
-
) else {
|
|
1500
|
-
return nil
|
|
1501
|
-
}
|
|
1502
|
-
var nodes: [SnapshotNode] = [modalNode]
|
|
1503
|
-
|
|
1504
|
-
for action in actions {
|
|
1505
|
-
guard let actionNode = safeMakeSnapshotNode(
|
|
1506
|
-
element: action,
|
|
1507
|
-
index: nodes.count,
|
|
1508
|
-
type: elementTypeName(action.elementType),
|
|
1509
|
-
depth: 1,
|
|
1510
|
-
hittableOverride: true
|
|
1511
|
-
) else {
|
|
1512
|
-
continue
|
|
1513
|
-
}
|
|
1514
|
-
nodes.append(actionNode)
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
return DataPayload(nodes: nodes, truncated: false)
|
|
1518
|
-
}
|
|
1519
|
-
|
|
1520
|
-
private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
|
|
1521
|
-
let disableSafeProbe = isEnvTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
|
|
1522
|
-
let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
|
|
1523
|
-
if disableSafeProbe {
|
|
1524
|
-
return fetch()
|
|
1525
|
-
}
|
|
1526
|
-
return self.safeElementsQuery(fetch)
|
|
1527
|
-
}
|
|
1528
|
-
|
|
1529
|
-
let alerts = queryElements {
|
|
1530
|
-
springboard.alerts.allElementsBoundByIndex
|
|
1531
|
-
}
|
|
1532
|
-
for alert in alerts {
|
|
1533
|
-
if safeIsBlockingSystemModal(alert, in: springboard) {
|
|
1534
|
-
return alert
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
|
|
1538
|
-
let sheets = queryElements {
|
|
1539
|
-
springboard.sheets.allElementsBoundByIndex
|
|
1540
|
-
}
|
|
1541
|
-
for sheet in sheets {
|
|
1542
|
-
if safeIsBlockingSystemModal(sheet, in: springboard) {
|
|
1543
|
-
return sheet
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
return nil
|
|
1548
|
-
}
|
|
1549
|
-
|
|
1550
|
-
private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
|
|
1551
|
-
var elements: [XCUIElement] = []
|
|
1552
|
-
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
1553
|
-
elements = fetch()
|
|
1554
|
-
})
|
|
1555
|
-
if let exceptionMessage {
|
|
1556
|
-
NSLog(
|
|
1557
|
-
"AGENT_DEVICE_RUNNER_MODAL_QUERY_IGNORED_EXCEPTION=%@",
|
|
1558
|
-
exceptionMessage
|
|
1559
|
-
)
|
|
1560
|
-
return []
|
|
1561
|
-
}
|
|
1562
|
-
return elements
|
|
1563
|
-
}
|
|
1564
|
-
|
|
1565
|
-
private func safeIsBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
|
|
1566
|
-
var isBlocking = false
|
|
1567
|
-
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
1568
|
-
isBlocking = isBlockingSystemModal(element, in: springboard)
|
|
1569
|
-
})
|
|
1570
|
-
if let exceptionMessage {
|
|
1571
|
-
NSLog(
|
|
1572
|
-
"AGENT_DEVICE_RUNNER_MODAL_CHECK_IGNORED_EXCEPTION=%@",
|
|
1573
|
-
exceptionMessage
|
|
1574
|
-
)
|
|
1575
|
-
return false
|
|
1576
|
-
}
|
|
1577
|
-
return isBlocking
|
|
1578
|
-
}
|
|
1579
|
-
|
|
1580
|
-
private func isBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
|
|
1581
|
-
guard element.exists else { return false }
|
|
1582
|
-
let frame = element.frame
|
|
1583
|
-
if frame.isNull || frame.isEmpty { return false }
|
|
1584
|
-
|
|
1585
|
-
let viewport = springboard.frame
|
|
1586
|
-
if viewport.isNull || viewport.isEmpty { return false }
|
|
1587
|
-
|
|
1588
|
-
let center = CGPoint(x: frame.midX, y: frame.midY)
|
|
1589
|
-
if !viewport.contains(center) { return false }
|
|
1590
|
-
|
|
1591
|
-
return true
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
private func actionableElements(in element: XCUIElement) -> [XCUIElement] {
|
|
1595
|
-
var seen = Set<String>()
|
|
1596
|
-
var actions: [XCUIElement] = []
|
|
1597
|
-
let descendants = safeElementsQuery {
|
|
1598
|
-
element.descendants(matching: .any).allElementsBoundByIndex
|
|
1599
|
-
}
|
|
1600
|
-
for candidate in descendants {
|
|
1601
|
-
if !safeIsActionableCandidate(candidate, seen: &seen) { continue }
|
|
1602
|
-
actions.append(candidate)
|
|
1603
|
-
}
|
|
1604
|
-
return actions
|
|
1605
|
-
}
|
|
1606
|
-
|
|
1607
|
-
private func safeIsActionableCandidate(_ candidate: XCUIElement, seen: inout Set<String>) -> Bool {
|
|
1608
|
-
var include = false
|
|
1609
|
-
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
1610
|
-
if !candidate.exists || !candidate.isHittable { return }
|
|
1611
|
-
if !actionableTypes.contains(candidate.elementType) { return }
|
|
1612
|
-
let frame = candidate.frame
|
|
1613
|
-
if frame.isNull || frame.isEmpty { return }
|
|
1614
|
-
let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)"
|
|
1615
|
-
if seen.contains(key) { return }
|
|
1616
|
-
seen.insert(key)
|
|
1617
|
-
include = true
|
|
1618
|
-
})
|
|
1619
|
-
if let exceptionMessage {
|
|
1620
|
-
NSLog(
|
|
1621
|
-
"AGENT_DEVICE_RUNNER_MODAL_ACTION_IGNORED_EXCEPTION=%@",
|
|
1622
|
-
exceptionMessage
|
|
1623
|
-
)
|
|
1624
|
-
return false
|
|
1625
|
-
}
|
|
1626
|
-
return include
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
private func preferredSystemModalTitle(_ element: XCUIElement) -> String {
|
|
1630
|
-
let label = element.label
|
|
1631
|
-
if !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
1632
|
-
return label
|
|
1633
|
-
}
|
|
1634
|
-
let identifier = element.identifier
|
|
1635
|
-
if !identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
1636
|
-
return identifier
|
|
1637
|
-
}
|
|
1638
|
-
return "System Alert"
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
private func makeSnapshotNode(
|
|
1642
|
-
element: XCUIElement,
|
|
1643
|
-
index: Int,
|
|
1644
|
-
type: String,
|
|
1645
|
-
labelOverride: String? = nil,
|
|
1646
|
-
identifierOverride: String? = nil,
|
|
1647
|
-
depth: Int,
|
|
1648
|
-
hittableOverride: Bool? = nil
|
|
1649
|
-
) -> SnapshotNode {
|
|
1650
|
-
let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1651
|
-
let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1652
|
-
return SnapshotNode(
|
|
1653
|
-
index: index,
|
|
1654
|
-
type: type,
|
|
1655
|
-
label: label.isEmpty ? nil : label,
|
|
1656
|
-
identifier: identifier.isEmpty ? nil : identifier,
|
|
1657
|
-
value: nil,
|
|
1658
|
-
rect: snapshotRect(from: element.frame),
|
|
1659
|
-
enabled: element.isEnabled,
|
|
1660
|
-
hittable: hittableOverride ?? element.isHittable,
|
|
1661
|
-
depth: depth
|
|
1662
|
-
)
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
|
-
private func safeMakeSnapshotNode(
|
|
1666
|
-
element: XCUIElement,
|
|
1667
|
-
index: Int,
|
|
1668
|
-
type: String,
|
|
1669
|
-
labelOverride: String? = nil,
|
|
1670
|
-
identifierOverride: String? = nil,
|
|
1671
|
-
depth: Int,
|
|
1672
|
-
hittableOverride: Bool? = nil
|
|
1673
|
-
) -> SnapshotNode? {
|
|
1674
|
-
var node: SnapshotNode?
|
|
1675
|
-
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
1676
|
-
node = makeSnapshotNode(
|
|
1677
|
-
element: element,
|
|
1678
|
-
index: index,
|
|
1679
|
-
type: type,
|
|
1680
|
-
labelOverride: labelOverride,
|
|
1681
|
-
identifierOverride: identifierOverride,
|
|
1682
|
-
depth: depth,
|
|
1683
|
-
hittableOverride: hittableOverride
|
|
1684
|
-
)
|
|
1685
|
-
})
|
|
1686
|
-
if let exceptionMessage {
|
|
1687
|
-
NSLog(
|
|
1688
|
-
"AGENT_DEVICE_RUNNER_MODAL_NODE_IGNORED_EXCEPTION=%@",
|
|
1689
|
-
exceptionMessage
|
|
1690
|
-
)
|
|
1691
|
-
return nil
|
|
1692
|
-
}
|
|
1693
|
-
return node
|
|
1694
|
-
}
|
|
1695
|
-
|
|
1696
|
-
private func snapshotRect(from frame: CGRect) -> SnapshotRect {
|
|
1697
|
-
return SnapshotRect(
|
|
1698
|
-
x: Double(frame.origin.x),
|
|
1699
|
-
y: Double(frame.origin.y),
|
|
1700
|
-
width: Double(frame.size.width),
|
|
1701
|
-
height: Double(frame.size.height)
|
|
1702
|
-
)
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
private func shouldInclude(
|
|
1706
|
-
element: XCUIElement,
|
|
1707
|
-
label: String,
|
|
1708
|
-
identifier: String,
|
|
1709
|
-
valueText: String?,
|
|
1710
|
-
options: SnapshotOptions
|
|
1711
|
-
) -> Bool {
|
|
1712
|
-
let type = element.elementType
|
|
1713
|
-
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
1714
|
-
if options.compact && type == .other && !hasContent && !element.isHittable {
|
|
1715
|
-
let children = element.children(matching: .any).allElementsBoundByIndex
|
|
1716
|
-
if children.count <= 1 { return false }
|
|
1717
|
-
}
|
|
1718
|
-
if options.interactiveOnly {
|
|
1719
|
-
if interactiveTypes.contains(type) { return true }
|
|
1720
|
-
if element.isHittable && type != .other { return true }
|
|
1721
|
-
if hasContent { return true }
|
|
1722
|
-
return false
|
|
1723
|
-
}
|
|
1724
|
-
if options.compact {
|
|
1725
|
-
return hasContent || element.isHittable
|
|
1726
|
-
}
|
|
1727
|
-
return true
|
|
1728
|
-
}
|
|
1729
|
-
|
|
1730
|
-
private func shouldInclude(
|
|
1731
|
-
snapshot: XCUIElementSnapshot,
|
|
1732
|
-
label: String,
|
|
1733
|
-
identifier: String,
|
|
1734
|
-
valueText: String?,
|
|
1735
|
-
options: SnapshotOptions
|
|
1736
|
-
) -> Bool {
|
|
1737
|
-
let type = snapshot.elementType
|
|
1738
|
-
let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
|
|
1739
|
-
if options.compact && type == .other && !hasContent && !snapshotHittable(snapshot) {
|
|
1740
|
-
if snapshot.children.count <= 1 { return false }
|
|
1741
|
-
}
|
|
1742
|
-
if options.interactiveOnly {
|
|
1743
|
-
if interactiveTypes.contains(type) { return true }
|
|
1744
|
-
if snapshotHittable(snapshot) && type != .other { return true }
|
|
1745
|
-
if hasContent { return true }
|
|
1746
|
-
return false
|
|
1747
|
-
}
|
|
1748
|
-
if options.compact {
|
|
1749
|
-
return hasContent || snapshotHittable(snapshot)
|
|
1750
|
-
}
|
|
1751
|
-
return true
|
|
1752
|
-
}
|
|
1753
|
-
|
|
1754
|
-
private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? {
|
|
1755
|
-
guard let value = snapshot.value else { return nil }
|
|
1756
|
-
let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1757
|
-
return text.isEmpty ? nil : text
|
|
1758
|
-
}
|
|
1759
|
-
|
|
1760
|
-
private func snapshotHittable(_ snapshot: XCUIElementSnapshot) -> Bool {
|
|
1761
|
-
// XCUIElementSnapshot does not expose isHittable; use enabled as a lightweight proxy.
|
|
1762
|
-
return snapshot.isEnabled
|
|
1763
|
-
}
|
|
1764
|
-
|
|
1765
|
-
private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
|
|
1766
|
-
if depth > 4 { return nil }
|
|
1767
|
-
let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1768
|
-
if !text.isEmpty { return text }
|
|
1769
|
-
if let valueText = snapshotValueText(snapshot) { return valueText }
|
|
1770
|
-
for child in snapshot.children {
|
|
1771
|
-
if let childLabel = aggregatedLabel(for: child, depth: depth + 1) {
|
|
1772
|
-
return childLabel
|
|
1773
|
-
}
|
|
1774
|
-
}
|
|
1775
|
-
return nil
|
|
1776
|
-
}
|
|
1777
|
-
|
|
1778
|
-
private func isVisibleInViewport(_ rect: CGRect, _ viewport: CGRect) -> Bool {
|
|
1779
|
-
if rect.isNull || rect.isEmpty { return false }
|
|
1780
|
-
return rect.intersects(viewport)
|
|
1781
|
-
}
|
|
1782
|
-
|
|
1783
|
-
private func jsonResponse(status: Int, response: Response) -> Data {
|
|
1784
|
-
let encoder = JSONEncoder()
|
|
1785
|
-
let body = (try? encoder.encode(response)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
|
|
1786
|
-
return httpResponse(status: status, body: body)
|
|
1787
|
-
}
|
|
1788
|
-
|
|
1789
|
-
private func httpResponse(status: Int, body: String) -> Data {
|
|
1790
|
-
let headers = [
|
|
1791
|
-
"HTTP/1.1 \(status) OK",
|
|
1792
|
-
"Content-Type: application/json",
|
|
1793
|
-
"Content-Length: \(body.utf8.count)",
|
|
1794
|
-
"Connection: close",
|
|
1795
|
-
"",
|
|
1796
|
-
body,
|
|
1797
|
-
].joined(separator: "\r\n")
|
|
1798
|
-
return Data(headers.utf8)
|
|
1799
|
-
}
|
|
1800
|
-
|
|
1801
|
-
private func finish() {
|
|
1802
|
-
listener?.cancel()
|
|
1803
|
-
listener = nil
|
|
1804
|
-
doneExpectation?.fulfill()
|
|
1805
|
-
}
|
|
1806
|
-
}
|
|
1807
|
-
|
|
1808
|
-
private func resolveRunnerPort() -> UInt16 {
|
|
1809
|
-
if let env = ProcessInfo.processInfo.environment["AGENT_DEVICE_RUNNER_PORT"], let port = UInt16(env) {
|
|
1810
|
-
return port
|
|
1811
|
-
}
|
|
1812
|
-
for arg in CommandLine.arguments {
|
|
1813
|
-
if arg.hasPrefix("AGENT_DEVICE_RUNNER_PORT=") {
|
|
1814
|
-
let value = arg.replacingOccurrences(of: "AGENT_DEVICE_RUNNER_PORT=", with: "")
|
|
1815
|
-
if let port = UInt16(value) { return port }
|
|
1816
|
-
}
|
|
1817
|
-
}
|
|
1818
|
-
return 0
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
|
-
private func isEnvTruthy(_ name: String) -> Bool {
|
|
1822
|
-
guard let raw = ProcessInfo.processInfo.environment[name] else {
|
|
1823
|
-
return false
|
|
1824
|
-
}
|
|
1825
|
-
switch raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
|
|
1826
|
-
case "1", "true", "yes", "on":
|
|
1827
|
-
return true
|
|
1828
|
-
default:
|
|
1829
|
-
return false
|
|
1830
|
-
}
|
|
1831
|
-
}
|
|
1832
|
-
|
|
1833
|
-
enum CommandType: String, Codable {
|
|
1834
|
-
case tap
|
|
1835
|
-
case tapSeries
|
|
1836
|
-
case longPress
|
|
1837
|
-
case drag
|
|
1838
|
-
case dragSeries
|
|
1839
|
-
case type
|
|
1840
|
-
case swipe
|
|
1841
|
-
case findText
|
|
1842
|
-
case snapshot
|
|
1843
|
-
case screenshot
|
|
1844
|
-
case back
|
|
1845
|
-
case home
|
|
1846
|
-
case appSwitcher
|
|
1847
|
-
case alert
|
|
1848
|
-
case pinch
|
|
1849
|
-
case recordStart
|
|
1850
|
-
case recordStop
|
|
1851
|
-
case shutdown
|
|
1852
|
-
}
|
|
1853
|
-
|
|
1854
|
-
enum SwipeDirection: String, Codable {
|
|
1855
|
-
case up
|
|
1856
|
-
case down
|
|
1857
|
-
case left
|
|
1858
|
-
case right
|
|
1859
|
-
}
|
|
1860
|
-
|
|
1861
|
-
struct Command: Codable {
|
|
1862
|
-
let command: CommandType
|
|
1863
|
-
let appBundleId: String?
|
|
1864
|
-
let text: String?
|
|
1865
|
-
let clearFirst: Bool?
|
|
1866
|
-
let action: String?
|
|
1867
|
-
let x: Double?
|
|
1868
|
-
let y: Double?
|
|
1869
|
-
let count: Double?
|
|
1870
|
-
let intervalMs: Double?
|
|
1871
|
-
let doubleTap: Bool?
|
|
1872
|
-
let pauseMs: Double?
|
|
1873
|
-
let pattern: String?
|
|
1874
|
-
let x2: Double?
|
|
1875
|
-
let y2: Double?
|
|
1876
|
-
let durationMs: Double?
|
|
1877
|
-
let direction: SwipeDirection?
|
|
1878
|
-
let scale: Double?
|
|
1879
|
-
let outPath: String?
|
|
1880
|
-
let fps: Int?
|
|
1881
|
-
let interactiveOnly: Bool?
|
|
1882
|
-
let compact: Bool?
|
|
1883
|
-
let depth: Int?
|
|
1884
|
-
let scope: String?
|
|
1885
|
-
let raw: Bool?
|
|
1886
|
-
}
|
|
1887
|
-
|
|
1888
|
-
struct Response: Codable {
|
|
1889
|
-
let ok: Bool
|
|
1890
|
-
let data: DataPayload?
|
|
1891
|
-
let error: ErrorPayload?
|
|
1892
|
-
|
|
1893
|
-
init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
|
|
1894
|
-
self.ok = ok
|
|
1895
|
-
self.data = data
|
|
1896
|
-
self.error = error
|
|
1897
|
-
}
|
|
1898
|
-
}
|
|
1899
|
-
|
|
1900
|
-
struct DataPayload: Codable {
|
|
1901
|
-
let message: String?
|
|
1902
|
-
let found: Bool?
|
|
1903
|
-
let items: [String]?
|
|
1904
|
-
let nodes: [SnapshotNode]?
|
|
1905
|
-
let truncated: Bool?
|
|
1906
|
-
|
|
1907
|
-
init(
|
|
1908
|
-
message: String? = nil,
|
|
1909
|
-
found: Bool? = nil,
|
|
1910
|
-
items: [String]? = nil,
|
|
1911
|
-
nodes: [SnapshotNode]? = nil,
|
|
1912
|
-
truncated: Bool? = nil
|
|
1913
|
-
) {
|
|
1914
|
-
self.message = message
|
|
1915
|
-
self.found = found
|
|
1916
|
-
self.items = items
|
|
1917
|
-
self.nodes = nodes
|
|
1918
|
-
self.truncated = truncated
|
|
1919
|
-
}
|
|
1920
|
-
}
|
|
1921
|
-
|
|
1922
|
-
struct ErrorPayload: Codable {
|
|
1923
|
-
let message: String
|
|
1924
|
-
}
|
|
1925
|
-
|
|
1926
|
-
struct SnapshotRect: Codable {
|
|
1927
|
-
let x: Double
|
|
1928
|
-
let y: Double
|
|
1929
|
-
let width: Double
|
|
1930
|
-
let height: Double
|
|
1931
|
-
}
|
|
1932
|
-
|
|
1933
|
-
struct SnapshotNode: Codable {
|
|
1934
|
-
let index: Int
|
|
1935
|
-
let type: String
|
|
1936
|
-
let label: String?
|
|
1937
|
-
let identifier: String?
|
|
1938
|
-
let value: String?
|
|
1939
|
-
let rect: SnapshotRect
|
|
1940
|
-
let enabled: Bool
|
|
1941
|
-
let hittable: Bool
|
|
1942
|
-
let depth: Int
|
|
1943
|
-
}
|
|
1944
|
-
|
|
1945
|
-
struct SnapshotOptions {
|
|
1946
|
-
let interactiveOnly: Bool
|
|
1947
|
-
let compact: Bool
|
|
1948
|
-
let depth: Int?
|
|
1949
|
-
let scope: String?
|
|
1950
|
-
let raw: Bool
|
|
1951
126
|
}
|