agent-device 0.17.1 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/android-multitouch-helper/dist/{agent-device-android-multitouch-helper-0.17.1.apk → agent-device-android-multitouch-helper-0.17.3.apk} +0 -0
- package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.3.apk.sha256 +1 -0
- package/android-multitouch-helper/dist/{agent-device-android-multitouch-helper-0.17.1.manifest.json → agent-device-android-multitouch-helper-0.17.3.manifest.json} +4 -4
- package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.17.1.apk → agent-device-android-snapshot-helper-0.17.3.apk} +0 -0
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.17.3.apk.sha256 +1 -0
- package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.17.1.manifest.json → agent-device-android-snapshot-helper-0.17.3.manifest.json} +6 -6
- package/dist/src/123.js +1 -0
- package/dist/src/1231.js +1 -1
- package/dist/src/1352.js +1 -1
- package/dist/src/221.js +2 -2
- package/dist/src/2403.js +3 -0
- package/dist/src/2415.js +36 -30
- package/dist/src/4778.js +1 -1
- package/dist/src/5611.js +1 -0
- package/dist/src/5827.js +1 -0
- package/dist/src/6088.js +1 -0
- package/dist/src/6133.js +1 -0
- package/dist/src/7556.js +1 -1
- package/dist/src/7847.js +1 -1
- package/dist/src/8806.js +3 -3
- package/dist/src/9010.js +1 -0
- package/dist/src/9238.js +3 -3
- package/dist/src/9471.js +1 -1
- package/dist/src/9542.js +6 -3
- package/dist/src/9616.js +2 -0
- package/dist/src/9818.js +1 -1
- package/dist/src/android-adb.d.ts +12 -6
- package/dist/src/android-adb.js +1 -1
- package/dist/src/android-snapshot-helper.d.ts +15 -8
- package/dist/src/android.js +5 -5
- package/dist/src/apple.js +1 -1
- package/dist/src/apps.js +10 -10
- package/dist/src/args.js +53 -14
- package/dist/src/batch.d.ts +16 -4
- package/dist/src/batch.js +1 -1
- package/dist/src/cli-output.js +29 -0
- package/dist/src/cli.js +17 -42
- package/dist/src/command-surface.js +1 -1
- package/dist/src/contracts.d.ts +80 -4
- package/dist/src/contracts.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.d.ts +3 -1
- package/dist/src/generic.js +7 -7
- package/dist/src/index.d.ts +250 -79
- package/dist/src/index.js +1 -1
- package/dist/src/input-actions.js +1 -1
- package/dist/src/interaction.js +1 -1
- package/dist/src/internal/png-worker.d.ts +26 -0
- package/dist/src/internal/png-worker.js +1 -0
- package/dist/src/metro.d.ts +3 -1
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-trace-recording.js +3 -3
- package/dist/src/remote-config.d.ts +33 -7
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selectors.d.ts +17 -3
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +2 -2
- package/dist/src/session.js +8 -11
- package/dist/src/snapshot.js +2 -2
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +14 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +297 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +328 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +121 -172
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +153 -5
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +171 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +0 -10
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +64 -18
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +200 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +406 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +82 -169
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +425 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +77 -6
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +10 -1
- package/ios-runner/README.md +5 -5
- package/ios-runner/RUNNER_PROTOCOL.md +0 -1
- package/package.json +8 -5
- package/server.json +2 -2
- package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.1.apk.sha256 +0 -1
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.17.1.apk.sha256 +0 -1
- package/dist/src/1998.js +0 -1
- package/dist/src/2805.js +0 -1
- package/dist/src/4057.js +0 -1
- package/dist/src/5310.js +0 -1
- package/dist/src/5792.js +0 -1
- package/dist/src/6085.js +0 -1
- package/dist/src/6232.js +0 -1
- package/dist/src/7455.js +0 -1
- package/dist/src/8020.js +0 -1
- package/dist/src/8502.js +0 -1
- package/dist/src/8699.js +0 -1
- package/dist/src/940.js +0 -1
- package/dist/src/9404.js +0 -1
- package/dist/src/9533.js +0 -1
- package/dist/src/command-metadata.js +0 -1
- /package/dist/src/{1393.js → 3675.js} +0 -0
package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift
ADDED
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
// MARK: - Snapshot capture plans (ADR 0004)
|
|
4
|
+
//
|
|
5
|
+
// Each snapshot strategy declares an ordered chain of capture backends. One runner walks the
|
|
6
|
+
// chain: capture, classify, accept the first payload the quality classifier calls usable, and
|
|
7
|
+
// stamp the outcome with a structured quality verdict so the daemon renders state instead of
|
|
8
|
+
// re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch.
|
|
9
|
+
|
|
10
|
+
/// Structured quality verdict shipped with every iOS snapshot payload.
|
|
11
|
+
struct SnapshotQuality: Codable {
|
|
12
|
+
/// healthy: first backend produced a usable tree. recovered: a later backend did.
|
|
13
|
+
/// sparse: no backend produced a usable tree; the best attempt is returned as-is.
|
|
14
|
+
let state: String
|
|
15
|
+
/// Backend that produced the returned payload: tree | queries | private-ax.
|
|
16
|
+
let backend: String
|
|
17
|
+
/// Why recovery ran (first failure) or why the payload is degraded.
|
|
18
|
+
let reason: String?
|
|
19
|
+
/// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes.
|
|
20
|
+
let reasonCode: String?
|
|
21
|
+
/// Private AX ladder cap when the accepted tree is shallower than requested.
|
|
22
|
+
let effectiveDepth: Int?
|
|
23
|
+
/// Leaves that merge many labels — a container marked accessible hides its descendants.
|
|
24
|
+
let collapsedLeafIndexes: [Int]?
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
enum SnapshotBackendKind: String, CaseIterable {
|
|
28
|
+
case recursiveTree = "tree"
|
|
29
|
+
case querySweep = "queries"
|
|
30
|
+
case privateAX = "private-ax"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// What the plan runner does when every backend failed or stayed sparse.
|
|
34
|
+
enum SnapshotCaptureTerminalPolicy {
|
|
35
|
+
/// Return the best sparse payload; if the tree backend hit a real AX serialization failure
|
|
36
|
+
/// on an interactive request, fail closed: invalidate the cached target and mark runnerFatal
|
|
37
|
+
/// (AX-unavailable target invalidation, CONTEXT.md).
|
|
38
|
+
case sparseWithFatalOnAXFailure
|
|
39
|
+
/// Re-throw the tree backend's AX failure (raw diagnostics preserve errors, ADR 0004).
|
|
40
|
+
case throwOnAXFailure
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
struct SnapshotBackendCapture {
|
|
44
|
+
let payload: DataPayload
|
|
45
|
+
/// Set by the private AX backend when the ladder accepted a shallower depth than requested.
|
|
46
|
+
let effectiveDepth: Int?
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
extension RunnerTests {
|
|
50
|
+
static let sparseRecoveryTruncatedNodeThreshold = 8
|
|
51
|
+
/// Umbrella wall-clock budget for one capture plan. Individual backends bound themselves,
|
|
52
|
+
/// but chained recovery tiers must never stack past the 30s main-thread watchdog: when the
|
|
53
|
+
/// budget is spent, remaining tiers are skipped and the best payload so far is returned.
|
|
54
|
+
static let snapshotPlanBudget: TimeInterval = 20
|
|
55
|
+
static let collapsedLeafMinimumSegments = 10
|
|
56
|
+
|
|
57
|
+
static func payloadNodeCount(_ payload: DataPayload?) -> Int {
|
|
58
|
+
payload?.nodes?.count ?? 0
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// MARK: Plan definitions
|
|
62
|
+
|
|
63
|
+
static let regularVisiblePlan: [SnapshotBackendKind] = [.recursiveTree, .querySweep, .privateAX]
|
|
64
|
+
static let rawDiagnosticPlan: [SnapshotBackendKind] = [.recursiveTree, .privateAX]
|
|
65
|
+
|
|
66
|
+
// MARK: Plan runner
|
|
67
|
+
|
|
68
|
+
func runSnapshotCapturePlan(
|
|
69
|
+
_ plan: [SnapshotBackendKind],
|
|
70
|
+
app: XCUIApplication,
|
|
71
|
+
options: SnapshotOptions,
|
|
72
|
+
terminal: SnapshotCaptureTerminalPolicy
|
|
73
|
+
) throws -> DataPayload {
|
|
74
|
+
var best: (kind: SnapshotBackendKind, capture: SnapshotBackendCapture)?
|
|
75
|
+
var firstFailure: (reason: String, code: String)?
|
|
76
|
+
var axFailure: SnapshotCaptureFailure?
|
|
77
|
+
let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
|
|
78
|
+
|
|
79
|
+
for kind in plan {
|
|
80
|
+
if kind != plan.first && Date() >= deadline {
|
|
81
|
+
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue)
|
|
82
|
+
if firstFailure == nil {
|
|
83
|
+
firstFailure = ("the capture plan ran out of its time budget", "budget")
|
|
84
|
+
}
|
|
85
|
+
break
|
|
86
|
+
}
|
|
87
|
+
let capture: SnapshotBackendCapture
|
|
88
|
+
do {
|
|
89
|
+
guard let result = try captureWithBackend(kind, app: app, options: options) else {
|
|
90
|
+
continue
|
|
91
|
+
}
|
|
92
|
+
capture = result
|
|
93
|
+
} catch let failure as SnapshotCaptureFailure {
|
|
94
|
+
if Self.isAxSnapshotFailure(failure) { axFailure = failure }
|
|
95
|
+
if firstFailure == nil {
|
|
96
|
+
firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed")
|
|
97
|
+
}
|
|
98
|
+
NSLog(
|
|
99
|
+
"AGENT_DEVICE_RUNNER_SNAPSHOT_BACKEND_FAILED backend=%@ error=%@",
|
|
100
|
+
kind.rawValue,
|
|
101
|
+
failure.message
|
|
102
|
+
)
|
|
103
|
+
continue
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if let sparseReason = Self.sparsePayloadReason(capture.payload) {
|
|
107
|
+
if firstFailure == nil { firstFailure = sparseReason }
|
|
108
|
+
if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) {
|
|
109
|
+
best = (kind, capture)
|
|
110
|
+
}
|
|
111
|
+
continue
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let recovered = kind != plan.first
|
|
115
|
+
if recovered {
|
|
116
|
+
NSLog(
|
|
117
|
+
"AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@",
|
|
118
|
+
kind.rawValue,
|
|
119
|
+
firstFailure?.reason ?? "sparse tree"
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
return stampedSnapshotPayload(
|
|
123
|
+
capture,
|
|
124
|
+
backend: kind,
|
|
125
|
+
state: recovered ? "recovered" : "healthy",
|
|
126
|
+
reason: recovered ? firstFailure : nil
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if let axFailure {
|
|
131
|
+
switch Self.resolveSnapshotPlanTerminal(
|
|
132
|
+
terminal: terminal,
|
|
133
|
+
interactiveOnly: options.interactiveOnly
|
|
134
|
+
) {
|
|
135
|
+
case .throwAxFailure:
|
|
136
|
+
throw axFailure
|
|
137
|
+
case .failClosed:
|
|
138
|
+
// Fail closed on any interactive AX serialization failure that no backend recovered:
|
|
139
|
+
// invalidate the cached target so the next command reacquires it (AX-unavailable target
|
|
140
|
+
// invalidation, CONTEXT.md). A sparse `best` from a later tier (e.g. the query sweep's
|
|
141
|
+
// synthetic root) must NOT suppress this — reaching the terminal already means no backend
|
|
142
|
+
// produced a usable tree.
|
|
143
|
+
return snapshotAccessibilityUnavailable(failure: axFailure)
|
|
144
|
+
case .sparseBest:
|
|
145
|
+
break
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
let fallbackPayload =
|
|
150
|
+
best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) }
|
|
151
|
+
?? stampedSnapshotPayload(
|
|
152
|
+
SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil),
|
|
153
|
+
backend: plan.last ?? .recursiveTree,
|
|
154
|
+
state: "sparse",
|
|
155
|
+
reason: firstFailure
|
|
156
|
+
)
|
|
157
|
+
return fallbackPayload
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private func captureWithBackend(
|
|
161
|
+
_ kind: SnapshotBackendKind,
|
|
162
|
+
app: XCUIApplication,
|
|
163
|
+
options: SnapshotOptions
|
|
164
|
+
) throws -> SnapshotBackendCapture? {
|
|
165
|
+
switch kind {
|
|
166
|
+
case .recursiveTree:
|
|
167
|
+
guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
|
|
168
|
+
return nil
|
|
169
|
+
}
|
|
170
|
+
let payload = options.raw
|
|
171
|
+
? try rawTreeSnapshotPayload(context: context, options: options)
|
|
172
|
+
: recursiveTreeSnapshotPayload(context: context, options: options)
|
|
173
|
+
return SnapshotBackendCapture(payload: payload, effectiveDepth: nil)
|
|
174
|
+
case .querySweep:
|
|
175
|
+
return SnapshotBackendCapture(
|
|
176
|
+
payload: snapshotFlatInteractive(app: app, options: options),
|
|
177
|
+
effectiveDepth: nil
|
|
178
|
+
)
|
|
179
|
+
case .privateAX:
|
|
180
|
+
return privateAXSnapshotCapture(app: app, options: options)
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// MARK: Quality classifier (the single source of "is this snapshot degraded")
|
|
185
|
+
|
|
186
|
+
/// Returns a degradation reason + machine code when the payload is too degraded to accept.
|
|
187
|
+
static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? {
|
|
188
|
+
guard let nodes = payload.nodes, !nodes.isEmpty else {
|
|
189
|
+
return ("snapshot returned no nodes", "no-nodes")
|
|
190
|
+
}
|
|
191
|
+
if isSparseApplicationWindowTree(nodes) {
|
|
192
|
+
return ("snapshot returned only structural application/window nodes", "sparse-tree")
|
|
193
|
+
}
|
|
194
|
+
if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold {
|
|
195
|
+
return ("snapshot was cut off by its budget with almost nothing collected", "budget")
|
|
196
|
+
}
|
|
197
|
+
return nil
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/// Terminal action when a capture plan exhausted every backend with an AX serialization
|
|
201
|
+
/// failure still pending. Pure so the fail-closed-vs-sparse policy is unit-testable without
|
|
202
|
+
/// a live app (the ordering gap the architecture review flagged).
|
|
203
|
+
enum SnapshotPlanTerminalAction: Equatable {
|
|
204
|
+
case throwAxFailure
|
|
205
|
+
case failClosed
|
|
206
|
+
case sparseBest
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
static func resolveSnapshotPlanTerminal(
|
|
210
|
+
terminal: SnapshotCaptureTerminalPolicy,
|
|
211
|
+
interactiveOnly: Bool
|
|
212
|
+
) -> SnapshotPlanTerminalAction {
|
|
213
|
+
switch terminal {
|
|
214
|
+
case .throwOnAXFailure:
|
|
215
|
+
return .throwAxFailure
|
|
216
|
+
case .sparseWithFatalOnAXFailure:
|
|
217
|
+
return interactiveOnly ? .failClosed : .sparseBest
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
|
|
222
|
+
guard !nodes.isEmpty else { return false }
|
|
223
|
+
return nodes.allSatisfy { node in
|
|
224
|
+
// Application/Window labels are just the app/window name, and full-screen roots
|
|
225
|
+
// compute as hittable; neither says anything about tree health.
|
|
226
|
+
let isRootContainer = node.type == "Application" || node.type == "Window"
|
|
227
|
+
let hasContent = (!isRootContainer && node.label?.isEmpty == false)
|
|
228
|
+
|| node.identifier?.isEmpty == false
|
|
229
|
+
|| node.value?.isEmpty == false
|
|
230
|
+
return !hasContent
|
|
231
|
+
&& (isRootContainer || !node.hittable)
|
|
232
|
+
&& Self.structuralOnlyNodeTypes.contains(node.type)
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/// A leaf whose label joins many short segments is a container marked as an accessibility
|
|
237
|
+
/// element: the platform folds every descendant into one merged node. Nothing below it can
|
|
238
|
+
/// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
|
|
239
|
+
static func collapsedLeafIndexes(_ nodes: [SnapshotNode]) -> [Int]? {
|
|
240
|
+
let parents = Set(nodes.compactMap { $0.parentIndex })
|
|
241
|
+
let collapsed = nodes.filter { node in
|
|
242
|
+
guard !parents.contains(node.index) else { return false }
|
|
243
|
+
guard !(node.type.lowercased().contains("text")) else { return false }
|
|
244
|
+
let label = node.label ?? ""
|
|
245
|
+
return label.split(separator: ",").count > collapsedLeafMinimumSegments
|
|
246
|
+
}
|
|
247
|
+
return collapsed.isEmpty ? nil : collapsed.map(\.index)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// MARK: Outcome stamping
|
|
251
|
+
|
|
252
|
+
private func stampedSnapshotPayload(
|
|
253
|
+
_ capture: SnapshotBackendCapture,
|
|
254
|
+
backend: SnapshotBackendKind,
|
|
255
|
+
state: String,
|
|
256
|
+
reason: (reason: String, code: String)?
|
|
257
|
+
) -> DataPayload {
|
|
258
|
+
let payload = capture.payload
|
|
259
|
+
let quality = SnapshotQuality(
|
|
260
|
+
state: state,
|
|
261
|
+
backend: backend.rawValue,
|
|
262
|
+
reason: reason?.reason,
|
|
263
|
+
reasonCode: reason?.code,
|
|
264
|
+
effectiveDepth: capture.effectiveDepth,
|
|
265
|
+
collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? [])
|
|
266
|
+
)
|
|
267
|
+
return DataPayload(
|
|
268
|
+
// Legacy human text for older daemons that read message instead of snapshotQuality.
|
|
269
|
+
message: Self.legacyQualityMessage(quality) ?? payload.message,
|
|
270
|
+
nodes: payload.nodes,
|
|
271
|
+
truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil,
|
|
272
|
+
snapshotQuality: quality,
|
|
273
|
+
runnerFatal: payload.runnerFatal,
|
|
274
|
+
runnerFatalReason: payload.runnerFatalReason
|
|
275
|
+
)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? {
|
|
279
|
+
guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil }
|
|
280
|
+
var parts: [String] = []
|
|
281
|
+
if quality.state == "recovered" {
|
|
282
|
+
let meaning = quality.reasonCode == "budget"
|
|
283
|
+
? " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen."
|
|
284
|
+
: " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears."
|
|
285
|
+
parts.append(
|
|
286
|
+
"Recovered this snapshot with the \(quality.backend) accessibility backend"
|
|
287
|
+
+ (quality.reason.map { " after: \($0)." } ?? ".")
|
|
288
|
+
+ meaning
|
|
289
|
+
)
|
|
290
|
+
}
|
|
291
|
+
if quality.state == "sparse" {
|
|
292
|
+
parts.append(
|
|
293
|
+
"No snapshot backend could read this screen"
|
|
294
|
+
+ (quality.reason.map { " (\($0))" } ?? "")
|
|
295
|
+
+ ". Use screenshot as visual truth and coordinate taps."
|
|
296
|
+
)
|
|
297
|
+
}
|
|
298
|
+
if let depth = quality.effectiveDepth {
|
|
299
|
+
parts.append(
|
|
300
|
+
"The accessibility server rejected deeper requests; this tree is capped at depth \(depth) — re-run with --depth \(depth) --scope <container> for deeper content."
|
|
301
|
+
)
|
|
302
|
+
}
|
|
303
|
+
return parts.isEmpty ? nil : parts.joined(separator: " ")
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// MARK: - In-bundle unit tests
|
|
308
|
+
|
|
309
|
+
extension RunnerTests {
|
|
310
|
+
private func planTestNode(
|
|
311
|
+
index: Int,
|
|
312
|
+
type: String,
|
|
313
|
+
label: String? = nil,
|
|
314
|
+
identifier: String? = nil,
|
|
315
|
+
hittable: Bool = false,
|
|
316
|
+
parentIndex: Int? = nil
|
|
317
|
+
) -> SnapshotNode {
|
|
318
|
+
SnapshotNode(
|
|
319
|
+
index: index,
|
|
320
|
+
type: type,
|
|
321
|
+
label: label,
|
|
322
|
+
identifier: identifier,
|
|
323
|
+
value: nil,
|
|
324
|
+
rect: snapshotRect(from: .zero),
|
|
325
|
+
enabled: true,
|
|
326
|
+
focused: nil,
|
|
327
|
+
selected: nil,
|
|
328
|
+
hittable: hittable,
|
|
329
|
+
depth: parentIndex == nil ? 0 : 1,
|
|
330
|
+
parentIndex: parentIndex,
|
|
331
|
+
hiddenContentAbove: nil,
|
|
332
|
+
hiddenContentBelow: nil
|
|
333
|
+
)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
func testSparsePayloadReasonMatrix() {
|
|
337
|
+
let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true)
|
|
338
|
+
let window = planTestNode(index: 1, type: "Window", parentIndex: 0)
|
|
339
|
+
let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0)
|
|
340
|
+
|
|
341
|
+
// Labeled, hittable root over a bare window is still sparse.
|
|
342
|
+
XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false)))
|
|
343
|
+
// Deadline-truncated near-empty sweep needs recovery even with one real control.
|
|
344
|
+
XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true)))
|
|
345
|
+
// The same tiny tree from a completed sweep is a legitimately minimal screen.
|
|
346
|
+
XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false)))
|
|
347
|
+
// Empty payloads are degraded.
|
|
348
|
+
XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false)))
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
func testCollapsedLeafIndexesFlagsMergedContainersOnly() {
|
|
352
|
+
let root = planTestNode(index: 0, type: "Application", label: "App")
|
|
353
|
+
let merged = planTestNode(
|
|
354
|
+
index: 1,
|
|
355
|
+
type: "Other",
|
|
356
|
+
label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "),
|
|
357
|
+
parentIndex: 0
|
|
358
|
+
)
|
|
359
|
+
let prose = planTestNode(
|
|
360
|
+
index: 2,
|
|
361
|
+
type: "StaticText",
|
|
362
|
+
label: (0...30).map { "clause \($0)" }.joined(separator: ", "),
|
|
363
|
+
parentIndex: 0
|
|
364
|
+
)
|
|
365
|
+
XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1])
|
|
366
|
+
XCTAssertNil(Self.collapsedLeafIndexes([root, prose]))
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
func testLegacyQualityMessageStatesFallbackMeaning() {
|
|
370
|
+
let recovered = SnapshotQuality(
|
|
371
|
+
state: "recovered",
|
|
372
|
+
backend: "queries",
|
|
373
|
+
reason: "snapshot returned only structural application/window nodes",
|
|
374
|
+
reasonCode: "sparse-tree",
|
|
375
|
+
effectiveDepth: nil,
|
|
376
|
+
collapsedLeafIndexes: nil
|
|
377
|
+
)
|
|
378
|
+
let message = Self.legacyQualityMessage(recovered)
|
|
379
|
+
XCTAssertTrue(message?.contains("queries accessibility backend") == true)
|
|
380
|
+
XCTAssertTrue(message?.contains("fixing the app's accessibility") == true)
|
|
381
|
+
XCTAssertTrue(message?.contains("screenshot as visual truth") == true)
|
|
382
|
+
XCTAssertNil(
|
|
383
|
+
Self.legacyQualityMessage(
|
|
384
|
+
SnapshotQuality(
|
|
385
|
+
state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil,
|
|
386
|
+
collapsedLeafIndexes: nil)
|
|
387
|
+
)
|
|
388
|
+
)
|
|
389
|
+
}
|
|
390
|
+
func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() {
|
|
391
|
+
// Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root
|
|
392
|
+
// "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard).
|
|
393
|
+
XCTAssertEqual(
|
|
394
|
+
Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true),
|
|
395
|
+
.failClosed
|
|
396
|
+
)
|
|
397
|
+
XCTAssertEqual(
|
|
398
|
+
Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false),
|
|
399
|
+
.sparseBest
|
|
400
|
+
)
|
|
401
|
+
XCTAssertEqual(
|
|
402
|
+
Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true),
|
|
403
|
+
.throwAxFailure
|
|
404
|
+
)
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() {
|
|
408
|
+
currentApp = app
|
|
409
|
+
currentBundleId = "com.example.app"
|
|
410
|
+
defer {
|
|
411
|
+
currentApp = nil
|
|
412
|
+
currentBundleId = nil
|
|
413
|
+
}
|
|
414
|
+
let payload = snapshotAccessibilityUnavailable(
|
|
415
|
+
failure: SnapshotCaptureFailure(
|
|
416
|
+
code: "IOS_AX_SNAPSHOT_FAILED",
|
|
417
|
+
message: "kAXErrorIllegalArgument",
|
|
418
|
+
hint: "use screenshot"
|
|
419
|
+
)
|
|
420
|
+
)
|
|
421
|
+
XCTAssertEqual(payload.runnerFatal, true)
|
|
422
|
+
XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
|
|
423
|
+
XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
|
|
424
|
+
}
|
|
425
|
+
}
|
|
@@ -29,12 +29,27 @@ extension RunnerTests {
|
|
|
29
29
|
}
|
|
30
30
|
var nodes: [SnapshotNode] = [modalNode]
|
|
31
31
|
|
|
32
|
+
for content in informativeElements(in: modal, excluding: actions) {
|
|
33
|
+
guard let contentNode = safeMakeSnapshotNode(
|
|
34
|
+
element: content,
|
|
35
|
+
index: nodes.count,
|
|
36
|
+
type: elementTypeName(content.elementType),
|
|
37
|
+
depth: 1,
|
|
38
|
+
parentIndex: 0,
|
|
39
|
+
hittableOverride: false
|
|
40
|
+
) else {
|
|
41
|
+
continue
|
|
42
|
+
}
|
|
43
|
+
nodes.append(contentNode)
|
|
44
|
+
}
|
|
45
|
+
|
|
32
46
|
for action in actions {
|
|
33
47
|
guard let actionNode = safeMakeSnapshotNode(
|
|
34
48
|
element: action,
|
|
35
49
|
index: nodes.count,
|
|
36
50
|
type: elementTypeName(action.elementType),
|
|
37
51
|
depth: 1,
|
|
52
|
+
parentIndex: 0,
|
|
38
53
|
hittableOverride: true
|
|
39
54
|
) else {
|
|
40
55
|
continue
|
|
@@ -101,9 +116,7 @@ extension RunnerTests {
|
|
|
101
116
|
func actionableElements(in element: XCUIElement) -> [XCUIElement] {
|
|
102
117
|
var seen = Set<String>()
|
|
103
118
|
var actions: [XCUIElement] = []
|
|
104
|
-
let descendants =
|
|
105
|
-
element.descendants(matching: .any).allElementsBoundByIndex
|
|
106
|
-
}
|
|
119
|
+
let descendants = actionableTypes.flatMap { modalDescendants(in: element, matching: $0) }
|
|
107
120
|
for candidate in descendants {
|
|
108
121
|
if !safeIsActionableCandidate(candidate, seen: &seen) { continue }
|
|
109
122
|
actions.append(candidate)
|
|
@@ -124,6 +137,61 @@ extension RunnerTests {
|
|
|
124
137
|
}
|
|
125
138
|
}
|
|
126
139
|
|
|
140
|
+
private func informativeElements(in element: XCUIElement, excluding actions: [XCUIElement]) -> [XCUIElement] {
|
|
141
|
+
let actionKeys = Set(actions.map(systemModalElementKey))
|
|
142
|
+
var seen = Set<String>()
|
|
143
|
+
var contents: [XCUIElement] = []
|
|
144
|
+
let descendants = readableSystemModalTypes.flatMap {
|
|
145
|
+
modalDescendants(in: element, matching: $0, limit: 2)
|
|
146
|
+
}
|
|
147
|
+
for candidate in descendants {
|
|
148
|
+
guard let key = safeInformativeElementKey(candidate, actionKeys: actionKeys) else {
|
|
149
|
+
continue
|
|
150
|
+
}
|
|
151
|
+
if seen.contains(key) { continue }
|
|
152
|
+
seen.insert(key)
|
|
153
|
+
contents.append(candidate)
|
|
154
|
+
}
|
|
155
|
+
return contents
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private var readableSystemModalTypes: [XCUIElement.ElementType] {
|
|
159
|
+
[.staticText, .textView]
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private func modalDescendants(
|
|
163
|
+
in element: XCUIElement,
|
|
164
|
+
matching type: XCUIElement.ElementType,
|
|
165
|
+
limit: Int? = nil
|
|
166
|
+
) -> [XCUIElement] {
|
|
167
|
+
let elements = safeElementsQuery {
|
|
168
|
+
element.descendants(matching: type).allElementsBoundByIndex
|
|
169
|
+
}
|
|
170
|
+
guard let limit else {
|
|
171
|
+
return elements
|
|
172
|
+
}
|
|
173
|
+
return Array(elements.prefix(limit))
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private func safeInformativeElementKey(_ candidate: XCUIElement, actionKeys: Set<String>) -> String? {
|
|
177
|
+
safely("MODAL_CONTENT") { () -> String? in
|
|
178
|
+
let key = systemModalElementKey(candidate)
|
|
179
|
+
if actionKeys.contains(key) { return nil }
|
|
180
|
+
if actionableTypes.contains(candidate.elementType) { return nil }
|
|
181
|
+
if !candidate.exists { return nil }
|
|
182
|
+
let frame = candidate.frame
|
|
183
|
+
if frame.isNull || frame.isEmpty { return nil }
|
|
184
|
+
let label = candidate.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
185
|
+
if label.isEmpty { return nil }
|
|
186
|
+
return key
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private func systemModalElementKey(_ element: XCUIElement) -> String {
|
|
191
|
+
let frame = element.frame
|
|
192
|
+
return "\(element.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(element.label)-\(element.identifier)"
|
|
193
|
+
}
|
|
194
|
+
|
|
127
195
|
private func preferredSystemModalTitle(_ element: XCUIElement) -> String {
|
|
128
196
|
let label = element.label
|
|
129
197
|
if !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
@@ -143,6 +211,7 @@ extension RunnerTests {
|
|
|
143
211
|
labelOverride: String? = nil,
|
|
144
212
|
identifierOverride: String? = nil,
|
|
145
213
|
depth: Int,
|
|
214
|
+
parentIndex: Int? = nil,
|
|
146
215
|
hittableOverride: Bool? = nil
|
|
147
216
|
) -> SnapshotNode {
|
|
148
217
|
let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
@@ -155,11 +224,11 @@ extension RunnerTests {
|
|
|
155
224
|
value: nil,
|
|
156
225
|
rect: snapshotRect(from: element.frame),
|
|
157
226
|
enabled: element.isEnabled,
|
|
158
|
-
focused:
|
|
159
|
-
selected:
|
|
227
|
+
focused: nil,
|
|
228
|
+
selected: nil,
|
|
160
229
|
hittable: hittableOverride ?? element.isHittable,
|
|
161
230
|
depth: depth,
|
|
162
|
-
parentIndex:
|
|
231
|
+
parentIndex: parentIndex,
|
|
163
232
|
hiddenContentAbove: nil,
|
|
164
233
|
hiddenContentBelow: nil
|
|
165
234
|
)
|
|
@@ -172,6 +241,7 @@ extension RunnerTests {
|
|
|
172
241
|
labelOverride: String? = nil,
|
|
173
242
|
identifierOverride: String? = nil,
|
|
174
243
|
depth: Int,
|
|
244
|
+
parentIndex: Int? = nil,
|
|
175
245
|
hittableOverride: Bool? = nil
|
|
176
246
|
) -> SnapshotNode? {
|
|
177
247
|
safely("MODAL_NODE") {
|
|
@@ -182,6 +252,7 @@ extension RunnerTests {
|
|
|
182
252
|
labelOverride: labelOverride,
|
|
183
253
|
identifierOverride: identifierOverride,
|
|
184
254
|
depth: depth,
|
|
255
|
+
parentIndex: parentIndex,
|
|
185
256
|
hittableOverride: hittableOverride
|
|
186
257
|
)
|
|
187
258
|
}
|
|
@@ -170,8 +170,17 @@ extension RunnerTests {
|
|
|
170
170
|
// MARK: - Response Encoding
|
|
171
171
|
|
|
172
172
|
private func jsonResponse(status: Int, response: Response) -> Data {
|
|
173
|
+
// Stamp the gesture-clock uptime at the END of command handling, just before the HTTP
|
|
174
|
+
// write, so the warm snapshot and recordStart responses carry the anchor for free. This
|
|
175
|
+
// runs AFTER commandJournal.finish, so journal-stored lifecycleResponseJson stays
|
|
176
|
+
// unstamped — recovered/status-replayed results carry no anchor and the daemon falls back
|
|
177
|
+
// rather than pairing a stale uptime with a much-later receipt time.
|
|
178
|
+
let stamped =
|
|
179
|
+
response.ok
|
|
180
|
+
? response.stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
|
|
181
|
+
: response
|
|
173
182
|
let encoder = JSONEncoder()
|
|
174
|
-
let body = (try? encoder.encode(
|
|
183
|
+
let body = (try? encoder.encode(stamped)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
|
|
175
184
|
return httpResponse(status: status, body: body)
|
|
176
185
|
}
|
|
177
186
|
|
package/ios-runner/README.md
CHANGED
|
@@ -35,16 +35,16 @@ Protocol and maintenance references:
|
|
|
35
35
|
|
|
36
36
|
## Snapshot Strategy
|
|
37
37
|
|
|
38
|
-
iOS snapshots have two explicit capture modes:
|
|
38
|
+
iOS snapshots have two explicit public capture modes:
|
|
39
39
|
|
|
40
40
|
- full/raw snapshots use recursive XCTest snapshots for rich hierarchy and diagnostics;
|
|
41
|
-
-
|
|
41
|
+
- interactive snapshots filter the same visible tree down to agent-facing refs.
|
|
42
42
|
|
|
43
43
|
Some simulator apps expose accessibility trees that lower-level AX services can inspect but XCTest
|
|
44
|
-
cannot serialize reliably. In those cases
|
|
45
|
-
|
|
44
|
+
cannot serialize reliably. In those cases interactive snapshots may return a sparse root quickly,
|
|
45
|
+
while full snapshots preserve the XCTest error. See
|
|
46
46
|
[`../docs/adr/0004-ios-snapshot-backend-strategy.md`](../docs/adr/0004-ios-snapshot-backend-strategy.md)
|
|
47
|
-
for the backend boundary and
|
|
47
|
+
for the backend boundary and future simulator AX-service direction.
|
|
48
48
|
|
|
49
49
|
## Protocol Notes
|
|
50
50
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-device",
|
|
3
|
-
"version": "0.17.
|
|
3
|
+
"version": "0.17.3",
|
|
4
4
|
"description": "Agent-native CLI for AI mobile testing and app automation across iOS, Android, tvOS, Android TV, macOS, and Linux.",
|
|
5
5
|
"mcpName": "io.github.callstackincubator/agent-device",
|
|
6
6
|
"license": "MIT",
|
|
@@ -76,6 +76,7 @@
|
|
|
76
76
|
"scripts": {
|
|
77
77
|
"build": "rslib build",
|
|
78
78
|
"clean:daemon": "node --experimental-strip-types scripts/clean-daemon.ts",
|
|
79
|
+
"daemon:state-dir": "node --experimental-strip-types scripts/print-daemon-state-dir.ts",
|
|
79
80
|
"clean:xcuitest": "node scripts/clean-xcuitest-derived.mjs",
|
|
80
81
|
"clean:xcuitest:ios": "node scripts/clean-xcuitest-derived.mjs ios",
|
|
81
82
|
"clean:xcuitest:macos": "node scripts/clean-xcuitest-derived.mjs macos",
|
|
@@ -103,7 +104,9 @@
|
|
|
103
104
|
"perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
|
|
104
105
|
"lint": "oxlint . --deny-warnings",
|
|
105
106
|
"format": "oxfmt --write src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
|
|
106
|
-
"
|
|
107
|
+
"format:check": "oxfmt --check src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
|
|
108
|
+
"fallow": "fallow audit --base origin/main",
|
|
109
|
+
"fallow:all": "fallow --summary",
|
|
107
110
|
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",
|
|
108
111
|
"check:fallow": "fallow audit",
|
|
109
112
|
"check:quick": "pnpm lint && pnpm typecheck",
|
|
@@ -129,8 +132,8 @@
|
|
|
129
132
|
"test:unit": "vitest run --project unit",
|
|
130
133
|
"test:coverage": "vitest run --coverage",
|
|
131
134
|
"test:integration:provider": "vitest run --project provider-integration",
|
|
132
|
-
"test:integration:progress": "node scripts/integration-progress.
|
|
133
|
-
"test:integration:progress:check": "node scripts/integration-progress.
|
|
135
|
+
"test:integration:progress": "node --experimental-strip-types scripts/integration-progress.ts",
|
|
136
|
+
"test:integration:progress:check": "node --experimental-strip-types scripts/integration-progress.ts --check",
|
|
134
137
|
"test:skillgym": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts",
|
|
135
138
|
"test:skillgym:case": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts --case",
|
|
136
139
|
"test:smoke": "node --test test/integration/smoke-*.test.ts",
|
|
@@ -212,7 +215,7 @@
|
|
|
212
215
|
"@rslib/core": "0.20.1",
|
|
213
216
|
"@types/node": "^22.0.0",
|
|
214
217
|
"@vitest/coverage-v8": "4.1.2",
|
|
215
|
-
"fallow": "^2.
|
|
218
|
+
"fallow": "^2.91.0",
|
|
216
219
|
"oxfmt": "^0.42.0",
|
|
217
220
|
"oxlint": "^1.57.0",
|
|
218
221
|
"skillgym": "^0.8.0",
|
package/server.json
CHANGED
|
@@ -7,12 +7,12 @@
|
|
|
7
7
|
"url": "https://github.com/callstackincubator/agent-device",
|
|
8
8
|
"source": "github"
|
|
9
9
|
},
|
|
10
|
-
"version": "0.17.
|
|
10
|
+
"version": "0.17.3",
|
|
11
11
|
"packages": [
|
|
12
12
|
{
|
|
13
13
|
"registryType": "npm",
|
|
14
14
|
"identifier": "agent-device",
|
|
15
|
-
"version": "0.17.
|
|
15
|
+
"version": "0.17.3",
|
|
16
16
|
"transport": {
|
|
17
17
|
"type": "stdio"
|
|
18
18
|
}
|
package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.1.apk.sha256
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
185df48455dc51982c340b9e98ec761214b31fb103c72dac6ed7ab8175c3372f agent-device-android-multitouch-helper-0.17.1.apk
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
b0c6a5572c4e7adca466d9cae744d5dc2c9b27d2b705e1a579e3fea47d9570e6 agent-device-android-snapshot-helper-0.17.1.apk
|