agent-device 0.7.3 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/src/daemon.js +25 -25
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +8 -4
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +381 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +30 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +258 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +174 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +121 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +263 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +359 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +220 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +124 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +30 -1855
- package/ios-runner/README.md +14 -0
- package/package.json +1 -1
- package/skills/agent-device/references/permissions.md +5 -1
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
extension RunnerTests {
|
|
4
|
+
// MARK: - Blocking System Modal Snapshot
|
|
5
|
+
|
|
6
|
+
func blockingSystemAlertSnapshot() -> DataPayload? {
|
|
7
|
+
guard let modal = firstBlockingSystemModal(in: springboard) else {
|
|
8
|
+
return nil
|
|
9
|
+
}
|
|
10
|
+
let actions = actionableElements(in: modal)
|
|
11
|
+
guard !actions.isEmpty else {
|
|
12
|
+
return nil
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
let title = preferredSystemModalTitle(modal)
|
|
16
|
+
guard let modalNode = safeMakeSnapshotNode(
|
|
17
|
+
element: modal,
|
|
18
|
+
index: 0,
|
|
19
|
+
type: "Alert",
|
|
20
|
+
labelOverride: title,
|
|
21
|
+
identifierOverride: modal.identifier,
|
|
22
|
+
depth: 0,
|
|
23
|
+
hittableOverride: true
|
|
24
|
+
) else {
|
|
25
|
+
return nil
|
|
26
|
+
}
|
|
27
|
+
var nodes: [SnapshotNode] = [modalNode]
|
|
28
|
+
|
|
29
|
+
for action in actions {
|
|
30
|
+
guard let actionNode = safeMakeSnapshotNode(
|
|
31
|
+
element: action,
|
|
32
|
+
index: nodes.count,
|
|
33
|
+
type: elementTypeName(action.elementType),
|
|
34
|
+
depth: 1,
|
|
35
|
+
hittableOverride: true
|
|
36
|
+
) else {
|
|
37
|
+
continue
|
|
38
|
+
}
|
|
39
|
+
nodes.append(actionNode)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return DataPayload(nodes: nodes, truncated: false)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
|
|
46
|
+
let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
|
|
47
|
+
let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
|
|
48
|
+
if disableSafeProbe {
|
|
49
|
+
return fetch()
|
|
50
|
+
}
|
|
51
|
+
return self.safeElementsQuery(fetch)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
let alerts = queryElements {
|
|
55
|
+
springboard.alerts.allElementsBoundByIndex
|
|
56
|
+
}
|
|
57
|
+
for alert in alerts {
|
|
58
|
+
if safeIsBlockingSystemModal(alert, in: springboard) {
|
|
59
|
+
return alert
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
let sheets = queryElements {
|
|
64
|
+
springboard.sheets.allElementsBoundByIndex
|
|
65
|
+
}
|
|
66
|
+
for sheet in sheets {
|
|
67
|
+
if safeIsBlockingSystemModal(sheet, in: springboard) {
|
|
68
|
+
return sheet
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return nil
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
|
|
76
|
+
var elements: [XCUIElement] = []
|
|
77
|
+
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
78
|
+
elements = fetch()
|
|
79
|
+
})
|
|
80
|
+
if let exceptionMessage {
|
|
81
|
+
NSLog(
|
|
82
|
+
"AGENT_DEVICE_RUNNER_MODAL_QUERY_IGNORED_EXCEPTION=%@",
|
|
83
|
+
exceptionMessage
|
|
84
|
+
)
|
|
85
|
+
return []
|
|
86
|
+
}
|
|
87
|
+
return elements
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private func safeIsBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
|
|
91
|
+
var isBlocking = false
|
|
92
|
+
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
93
|
+
isBlocking = isBlockingSystemModal(element, in: springboard)
|
|
94
|
+
})
|
|
95
|
+
if let exceptionMessage {
|
|
96
|
+
NSLog(
|
|
97
|
+
"AGENT_DEVICE_RUNNER_MODAL_CHECK_IGNORED_EXCEPTION=%@",
|
|
98
|
+
exceptionMessage
|
|
99
|
+
)
|
|
100
|
+
return false
|
|
101
|
+
}
|
|
102
|
+
return isBlocking
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private func isBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
|
|
106
|
+
guard element.exists else { return false }
|
|
107
|
+
let frame = element.frame
|
|
108
|
+
if frame.isNull || frame.isEmpty { return false }
|
|
109
|
+
|
|
110
|
+
let viewport = springboard.frame
|
|
111
|
+
if viewport.isNull || viewport.isEmpty { return false }
|
|
112
|
+
|
|
113
|
+
let center = CGPoint(x: frame.midX, y: frame.midY)
|
|
114
|
+
if !viewport.contains(center) { return false }
|
|
115
|
+
|
|
116
|
+
return true
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private func actionableElements(in element: XCUIElement) -> [XCUIElement] {
|
|
120
|
+
var seen = Set<String>()
|
|
121
|
+
var actions: [XCUIElement] = []
|
|
122
|
+
let descendants = safeElementsQuery {
|
|
123
|
+
element.descendants(matching: .any).allElementsBoundByIndex
|
|
124
|
+
}
|
|
125
|
+
for candidate in descendants {
|
|
126
|
+
if !safeIsActionableCandidate(candidate, seen: &seen) { continue }
|
|
127
|
+
actions.append(candidate)
|
|
128
|
+
}
|
|
129
|
+
return actions
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
private func safeIsActionableCandidate(_ candidate: XCUIElement, seen: inout Set<String>) -> Bool {
|
|
133
|
+
var include = false
|
|
134
|
+
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
135
|
+
if !candidate.exists || !candidate.isHittable { return }
|
|
136
|
+
if !actionableTypes.contains(candidate.elementType) { return }
|
|
137
|
+
let frame = candidate.frame
|
|
138
|
+
if frame.isNull || frame.isEmpty { return }
|
|
139
|
+
let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)"
|
|
140
|
+
if seen.contains(key) { return }
|
|
141
|
+
seen.insert(key)
|
|
142
|
+
include = true
|
|
143
|
+
})
|
|
144
|
+
if let exceptionMessage {
|
|
145
|
+
NSLog(
|
|
146
|
+
"AGENT_DEVICE_RUNNER_MODAL_ACTION_IGNORED_EXCEPTION=%@",
|
|
147
|
+
exceptionMessage
|
|
148
|
+
)
|
|
149
|
+
return false
|
|
150
|
+
}
|
|
151
|
+
return include
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private func preferredSystemModalTitle(_ element: XCUIElement) -> String {
|
|
155
|
+
let label = element.label
|
|
156
|
+
if !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
157
|
+
return label
|
|
158
|
+
}
|
|
159
|
+
let identifier = element.identifier
|
|
160
|
+
if !identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
161
|
+
return identifier
|
|
162
|
+
}
|
|
163
|
+
return "System Alert"
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private func makeSnapshotNode(
|
|
167
|
+
element: XCUIElement,
|
|
168
|
+
index: Int,
|
|
169
|
+
type: String,
|
|
170
|
+
labelOverride: String? = nil,
|
|
171
|
+
identifierOverride: String? = nil,
|
|
172
|
+
depth: Int,
|
|
173
|
+
hittableOverride: Bool? = nil
|
|
174
|
+
) -> SnapshotNode {
|
|
175
|
+
let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
176
|
+
let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
177
|
+
return SnapshotNode(
|
|
178
|
+
index: index,
|
|
179
|
+
type: type,
|
|
180
|
+
label: label.isEmpty ? nil : label,
|
|
181
|
+
identifier: identifier.isEmpty ? nil : identifier,
|
|
182
|
+
value: nil,
|
|
183
|
+
rect: snapshotRect(from: element.frame),
|
|
184
|
+
enabled: element.isEnabled,
|
|
185
|
+
hittable: hittableOverride ?? element.isHittable,
|
|
186
|
+
depth: depth
|
|
187
|
+
)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private func safeMakeSnapshotNode(
|
|
191
|
+
element: XCUIElement,
|
|
192
|
+
index: Int,
|
|
193
|
+
type: String,
|
|
194
|
+
labelOverride: String? = nil,
|
|
195
|
+
identifierOverride: String? = nil,
|
|
196
|
+
depth: Int,
|
|
197
|
+
hittableOverride: Bool? = nil
|
|
198
|
+
) -> SnapshotNode? {
|
|
199
|
+
var node: SnapshotNode?
|
|
200
|
+
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
|
|
201
|
+
node = makeSnapshotNode(
|
|
202
|
+
element: element,
|
|
203
|
+
index: index,
|
|
204
|
+
type: type,
|
|
205
|
+
labelOverride: labelOverride,
|
|
206
|
+
identifierOverride: identifierOverride,
|
|
207
|
+
depth: depth,
|
|
208
|
+
hittableOverride: hittableOverride
|
|
209
|
+
)
|
|
210
|
+
})
|
|
211
|
+
if let exceptionMessage {
|
|
212
|
+
NSLog(
|
|
213
|
+
"AGENT_DEVICE_RUNNER_MODAL_NODE_IGNORED_EXCEPTION=%@",
|
|
214
|
+
exceptionMessage
|
|
215
|
+
)
|
|
216
|
+
return nil
|
|
217
|
+
}
|
|
218
|
+
return node
|
|
219
|
+
}
|
|
220
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
import Network
|
|
3
|
+
|
|
4
|
+
extension RunnerTests {
|
|
5
|
+
// MARK: - Connection Lifecycle
|
|
6
|
+
|
|
7
|
+
func handle(connection: NWConnection) {
|
|
8
|
+
receiveRequest(connection: connection, buffer: Data())
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// MARK: - Request Parsing
|
|
12
|
+
|
|
13
|
+
private func receiveRequest(connection: NWConnection, buffer: Data) {
|
|
14
|
+
connection.receive(minimumIncompleteLength: 1, maximumLength: 1024 * 1024) { [weak self] data, _, _, _ in
|
|
15
|
+
guard let self = self, let data = data else {
|
|
16
|
+
connection.cancel()
|
|
17
|
+
return
|
|
18
|
+
}
|
|
19
|
+
if buffer.count + data.count > self.maxRequestBytes {
|
|
20
|
+
let response = self.jsonResponse(
|
|
21
|
+
status: 413,
|
|
22
|
+
response: Response(ok: false, error: ErrorPayload(message: "request too large")),
|
|
23
|
+
)
|
|
24
|
+
connection.send(content: response, completion: .contentProcessed { [weak self] _ in
|
|
25
|
+
connection.cancel()
|
|
26
|
+
self?.finish()
|
|
27
|
+
})
|
|
28
|
+
return
|
|
29
|
+
}
|
|
30
|
+
let combined = buffer + data
|
|
31
|
+
if let body = self.parseRequest(data: combined) {
|
|
32
|
+
let result = self.handleRequestBody(body)
|
|
33
|
+
connection.send(content: result.data, completion: .contentProcessed { _ in
|
|
34
|
+
connection.cancel()
|
|
35
|
+
if result.shouldFinish {
|
|
36
|
+
self.finish()
|
|
37
|
+
}
|
|
38
|
+
})
|
|
39
|
+
} else {
|
|
40
|
+
self.receiveRequest(connection: connection, buffer: combined)
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private func parseRequest(data: Data) -> Data? {
|
|
46
|
+
guard let headerEnd = data.range(of: Data("\r\n\r\n".utf8)) else {
|
|
47
|
+
return nil
|
|
48
|
+
}
|
|
49
|
+
let headerData = data.subdata(in: 0..<headerEnd.lowerBound)
|
|
50
|
+
let bodyStart = headerEnd.upperBound
|
|
51
|
+
let headers = String(decoding: headerData, as: UTF8.self)
|
|
52
|
+
let contentLength = extractContentLength(headers: headers)
|
|
53
|
+
guard let contentLength = contentLength else {
|
|
54
|
+
return nil
|
|
55
|
+
}
|
|
56
|
+
if data.count < bodyStart + contentLength {
|
|
57
|
+
return nil
|
|
58
|
+
}
|
|
59
|
+
let body = data.subdata(in: bodyStart..<(bodyStart + contentLength))
|
|
60
|
+
return body
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private func extractContentLength(headers: String) -> Int? {
|
|
64
|
+
for line in headers.split(separator: "\r\n") {
|
|
65
|
+
let parts = line.split(separator: ":", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }
|
|
66
|
+
if parts.count == 2 && parts[0].lowercased() == "content-length" {
|
|
67
|
+
return Int(parts[1])
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return nil
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private func handleRequestBody(_ body: Data) -> (data: Data, shouldFinish: Bool) {
|
|
74
|
+
guard let json = String(data: body, encoding: .utf8) else {
|
|
75
|
+
return (
|
|
76
|
+
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
77
|
+
false
|
|
78
|
+
)
|
|
79
|
+
}
|
|
80
|
+
guard let data = json.data(using: .utf8) else {
|
|
81
|
+
return (
|
|
82
|
+
jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
|
|
83
|
+
false
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
do {
|
|
88
|
+
let command = try JSONDecoder().decode(Command.self, from: data)
|
|
89
|
+
let response = try execute(command: command)
|
|
90
|
+
return (jsonResponse(status: 200, response: response), command.command == .shutdown)
|
|
91
|
+
} catch {
|
|
92
|
+
return (
|
|
93
|
+
jsonResponse(status: 500, response: Response(ok: false, error: ErrorPayload(message: "\(error)"))),
|
|
94
|
+
false
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// MARK: - Response Encoding
|
|
100
|
+
|
|
101
|
+
private func jsonResponse(status: Int, response: Response) -> Data {
|
|
102
|
+
let encoder = JSONEncoder()
|
|
103
|
+
let body = (try? encoder.encode(response)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
|
|
104
|
+
return httpResponse(status: status, body: body)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private func httpResponse(status: Int, body: String) -> Data {
|
|
108
|
+
let headers = [
|
|
109
|
+
"HTTP/1.1 \(status) OK",
|
|
110
|
+
"Content-Type: application/json",
|
|
111
|
+
"Content-Length: \(body.utf8.count)",
|
|
112
|
+
"Connection: close",
|
|
113
|
+
"",
|
|
114
|
+
body,
|
|
115
|
+
].joined(separator: "\r\n")
|
|
116
|
+
return Data(headers.utf8)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private func finish() {
|
|
120
|
+
listener?.cancel()
|
|
121
|
+
listener = nil
|
|
122
|
+
doneExpectation?.fulfill()
|
|
123
|
+
}
|
|
124
|
+
}
|