agent-device 0.17.2 → 0.17.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +2 -2
  2. package/android-multitouch-helper/dist/{agent-device-android-multitouch-helper-0.17.2.apk → agent-device-android-multitouch-helper-0.17.4.apk} +0 -0
  3. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.4.apk.sha256 +1 -0
  4. package/android-multitouch-helper/dist/{agent-device-android-multitouch-helper-0.17.2.manifest.json → agent-device-android-multitouch-helper-0.17.4.manifest.json} +4 -4
  5. package/android-snapshot-helper/README.md +3 -0
  6. package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.17.2.apk → agent-device-android-snapshot-helper-0.17.4.apk} +0 -0
  7. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.17.4.apk.sha256 +1 -0
  8. package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.17.2.manifest.json → agent-device-android-snapshot-helper-0.17.4.manifest.json} +6 -6
  9. package/dist/src/123.js +1 -1
  10. package/dist/src/1231.js +1 -1
  11. package/dist/src/1352.js +1 -1
  12. package/dist/src/221.js +2 -2
  13. package/dist/src/2403.js +3 -3
  14. package/dist/src/2415.js +37 -29
  15. package/dist/src/4012.js +4 -0
  16. package/dist/src/5611.js +1 -0
  17. package/dist/src/5827.js +1 -0
  18. package/dist/src/6088.js +1 -0
  19. package/dist/src/6133.js +1 -0
  20. package/dist/src/7847.js +1 -1
  21. package/dist/src/9471.js +1 -1
  22. package/dist/src/9542.js +6 -3
  23. package/dist/src/9616.js +2 -1
  24. package/dist/src/9818.js +1 -1
  25. package/dist/src/android-snapshot-helper.d.ts +0 -1
  26. package/dist/src/android.js +5 -5
  27. package/dist/src/apple.js +1 -1
  28. package/dist/src/apps.js +10 -10
  29. package/dist/src/args.js +53 -14
  30. package/dist/src/batch.js +1 -1
  31. package/dist/src/cli-output.js +30 -0
  32. package/dist/src/cli.js +15 -16
  33. package/dist/src/command-surface.js +1 -1
  34. package/dist/src/contracts.d.ts +52 -0
  35. package/dist/src/contracts.js +1 -1
  36. package/dist/src/find.js +1 -1
  37. package/dist/src/generic.js +5 -5
  38. package/dist/src/index.d.ts +127 -8
  39. package/dist/src/index.js +1 -1
  40. package/dist/src/input-actions.js +1 -1
  41. package/dist/src/interaction.js +1 -1
  42. package/dist/src/react-native.js +1 -1
  43. package/dist/src/record-trace-recording.js +3 -3
  44. package/dist/src/selector-runtime.js +1 -1
  45. package/dist/src/selectors.d.ts +14 -0
  46. package/dist/src/selectors.js +1 -1
  47. package/dist/src/server.js +2 -2
  48. package/dist/src/session.js +8 -10
  49. package/dist/src/snapshot.js +2 -2
  50. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  51. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +14 -0
  52. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +297 -0
  53. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +327 -0
  54. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +122 -173
  55. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +153 -5
  56. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +171 -0
  57. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +0 -10
  58. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +64 -18
  59. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +200 -0
  60. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +450 -0
  61. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +82 -178
  62. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +425 -0
  63. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +77 -6
  64. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +10 -1
  65. package/ios-runner/README.md +5 -5
  66. package/ios-runner/RUNNER_PROTOCOL.md +0 -1
  67. package/package.json +3 -3
  68. package/server.json +2 -2
  69. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.2.apk.sha256 +0 -1
  70. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.17.2.apk.sha256 +0 -1
  71. package/dist/src/1534.js +0 -1
  72. package/dist/src/1620.js +0 -1
  73. package/dist/src/1644.js +0 -2
  74. package/dist/src/2284.js +0 -1
  75. package/dist/src/2474.js +0 -1
  76. package/dist/src/2672.js +0 -1
  77. package/dist/src/3393.js +0 -1
  78. package/dist/src/5898.js +0 -1
  79. package/dist/src/695.js +0 -1
  80. package/dist/src/7455.js +0 -1
  81. package/dist/src/8173.js +0 -27
  82. package/dist/src/8407.js +0 -1
  83. package/dist/src/8699.js +0 -1
  84. package/dist/src/9238.js +0 -4
  85. package/dist/src/9673.js +0 -1
  86. package/dist/src/9974.js +0 -1
@@ -0,0 +1,425 @@
1
+ import XCTest
2
+
3
+ // MARK: - Snapshot capture plans (ADR 0004)
4
+ //
5
+ // Each snapshot strategy declares an ordered chain of capture backends. One runner walks the
6
+ // chain: capture, classify, accept the first payload the quality classifier calls usable, and
7
+ // stamp the outcome with a structured quality verdict so the daemon renders state instead of
8
+ // re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch.
9
+
10
+ /// Structured quality verdict shipped with every iOS snapshot payload.
11
+ struct SnapshotQuality: Codable {
12
+ /// healthy: first backend produced a usable tree. recovered: a later backend did.
13
+ /// sparse: no backend produced a usable tree; the best attempt is returned as-is.
14
+ let state: String
15
+ /// Backend that produced the returned payload: tree | queries | private-ax.
16
+ let backend: String
17
+ /// Why recovery ran (first failure) or why the payload is degraded.
18
+ let reason: String?
19
+ /// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes.
20
+ let reasonCode: String?
21
+ /// Private AX ladder cap when the accepted tree is shallower than requested.
22
+ let effectiveDepth: Int?
23
+ /// Leaves that merge many labels — a container marked accessible hides its descendants.
24
+ let collapsedLeafIndexes: [Int]?
25
+ }
26
+
27
+ enum SnapshotBackendKind: String, CaseIterable {
28
+ case recursiveTree = "tree"
29
+ case querySweep = "queries"
30
+ case privateAX = "private-ax"
31
+ }
32
+
33
+ /// What the plan runner does when every backend failed or stayed sparse.
34
+ enum SnapshotCaptureTerminalPolicy {
35
+ /// Return the best sparse payload; if the tree backend hit a real AX serialization failure
36
+ /// on an interactive request, fail closed: invalidate the cached target and mark runnerFatal
37
+ /// (AX-unavailable target invalidation, CONTEXT.md).
38
+ case sparseWithFatalOnAXFailure
39
+ /// Re-throw the tree backend's AX failure (raw diagnostics preserve errors, ADR 0004).
40
+ case throwOnAXFailure
41
+ }
42
+
43
+ struct SnapshotBackendCapture {
44
+ let payload: DataPayload
45
+ /// Set by the private AX backend when the ladder accepted a shallower depth than requested.
46
+ let effectiveDepth: Int?
47
+ }
48
+
49
+ extension RunnerTests {
50
+ static let sparseRecoveryTruncatedNodeThreshold = 8
51
+ /// Umbrella wall-clock budget for one capture plan. Individual backends bound themselves,
52
+ /// but chained recovery tiers must never stack past the 30s main-thread watchdog: when the
53
+ /// budget is spent, remaining tiers are skipped and the best payload so far is returned.
54
+ static let snapshotPlanBudget: TimeInterval = 20
55
+ static let collapsedLeafMinimumSegments = 10
56
+
57
+ static func payloadNodeCount(_ payload: DataPayload?) -> Int {
58
+ payload?.nodes?.count ?? 0
59
+ }
60
+
61
+ // MARK: Plan definitions
62
+
63
+ static let regularVisiblePlan: [SnapshotBackendKind] = [.recursiveTree, .querySweep, .privateAX]
64
+ static let rawDiagnosticPlan: [SnapshotBackendKind] = [.recursiveTree, .privateAX]
65
+
66
+ // MARK: Plan runner
67
+
68
+ func runSnapshotCapturePlan(
69
+ _ plan: [SnapshotBackendKind],
70
+ app: XCUIApplication,
71
+ options: SnapshotOptions,
72
+ terminal: SnapshotCaptureTerminalPolicy
73
+ ) throws -> DataPayload {
74
+ var best: (kind: SnapshotBackendKind, capture: SnapshotBackendCapture)?
75
+ var firstFailure: (reason: String, code: String)?
76
+ var axFailure: SnapshotCaptureFailure?
77
+ let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
78
+
79
+ for kind in plan {
80
+ if kind != plan.first && Date() >= deadline {
81
+ NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue)
82
+ if firstFailure == nil {
83
+ firstFailure = ("the capture plan ran out of its time budget", "budget")
84
+ }
85
+ break
86
+ }
87
+ let capture: SnapshotBackendCapture
88
+ do {
89
+ guard let result = try captureWithBackend(kind, app: app, options: options) else {
90
+ continue
91
+ }
92
+ capture = result
93
+ } catch let failure as SnapshotCaptureFailure {
94
+ if Self.isAxSnapshotFailure(failure) { axFailure = failure }
95
+ if firstFailure == nil {
96
+ firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed")
97
+ }
98
+ NSLog(
99
+ "AGENT_DEVICE_RUNNER_SNAPSHOT_BACKEND_FAILED backend=%@ error=%@",
100
+ kind.rawValue,
101
+ failure.message
102
+ )
103
+ continue
104
+ }
105
+
106
+ if let sparseReason = Self.sparsePayloadReason(capture.payload) {
107
+ if firstFailure == nil { firstFailure = sparseReason }
108
+ if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) {
109
+ best = (kind, capture)
110
+ }
111
+ continue
112
+ }
113
+
114
+ let recovered = kind != plan.first
115
+ if recovered {
116
+ NSLog(
117
+ "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@",
118
+ kind.rawValue,
119
+ firstFailure?.reason ?? "sparse tree"
120
+ )
121
+ }
122
+ return stampedSnapshotPayload(
123
+ capture,
124
+ backend: kind,
125
+ state: recovered ? "recovered" : "healthy",
126
+ reason: recovered ? firstFailure : nil
127
+ )
128
+ }
129
+
130
+ if let axFailure {
131
+ switch Self.resolveSnapshotPlanTerminal(
132
+ terminal: terminal,
133
+ interactiveOnly: options.interactiveOnly
134
+ ) {
135
+ case .throwAxFailure:
136
+ throw axFailure
137
+ case .failClosed:
138
+ // Fail closed on any interactive AX serialization failure that no backend recovered:
139
+ // invalidate the cached target so the next command reacquires it (AX-unavailable target
140
+ // invalidation, CONTEXT.md). A sparse `best` from a later tier (e.g. the query sweep's
141
+ // synthetic root) must NOT suppress this — reaching the terminal already means no backend
142
+ // produced a usable tree.
143
+ return snapshotAccessibilityUnavailable(failure: axFailure)
144
+ case .sparseBest:
145
+ break
146
+ }
147
+ }
148
+
149
+ let fallbackPayload =
150
+ best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) }
151
+ ?? stampedSnapshotPayload(
152
+ SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil),
153
+ backend: plan.last ?? .recursiveTree,
154
+ state: "sparse",
155
+ reason: firstFailure
156
+ )
157
+ return fallbackPayload
158
+ }
159
+
160
+ private func captureWithBackend(
161
+ _ kind: SnapshotBackendKind,
162
+ app: XCUIApplication,
163
+ options: SnapshotOptions
164
+ ) throws -> SnapshotBackendCapture? {
165
+ switch kind {
166
+ case .recursiveTree:
167
+ guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
168
+ return nil
169
+ }
170
+ let payload = options.raw
171
+ ? try rawTreeSnapshotPayload(context: context, options: options)
172
+ : recursiveTreeSnapshotPayload(context: context, options: options)
173
+ return SnapshotBackendCapture(payload: payload, effectiveDepth: nil)
174
+ case .querySweep:
175
+ return SnapshotBackendCapture(
176
+ payload: snapshotFlatInteractive(app: app, options: options),
177
+ effectiveDepth: nil
178
+ )
179
+ case .privateAX:
180
+ return privateAXSnapshotCapture(app: app, options: options)
181
+ }
182
+ }
183
+
184
+ // MARK: Quality classifier (the single source of "is this snapshot degraded")
185
+
186
+ /// Returns a degradation reason + machine code when the payload is too degraded to accept.
187
+ static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? {
188
+ guard let nodes = payload.nodes, !nodes.isEmpty else {
189
+ return ("snapshot returned no nodes", "no-nodes")
190
+ }
191
+ if isSparseApplicationWindowTree(nodes) {
192
+ return ("snapshot returned only structural application/window nodes", "sparse-tree")
193
+ }
194
+ if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold {
195
+ return ("snapshot was cut off by its budget with almost nothing collected", "budget")
196
+ }
197
+ return nil
198
+ }
199
+
200
+ /// Terminal action when a capture plan exhausted every backend with an AX serialization
201
+ /// failure still pending. Pure so the fail-closed-vs-sparse policy is unit-testable without
202
+ /// a live app (the ordering gap the architecture review flagged).
203
+ enum SnapshotPlanTerminalAction: Equatable {
204
+ case throwAxFailure
205
+ case failClosed
206
+ case sparseBest
207
+ }
208
+
209
+ static func resolveSnapshotPlanTerminal(
210
+ terminal: SnapshotCaptureTerminalPolicy,
211
+ interactiveOnly: Bool
212
+ ) -> SnapshotPlanTerminalAction {
213
+ switch terminal {
214
+ case .throwOnAXFailure:
215
+ return .throwAxFailure
216
+ case .sparseWithFatalOnAXFailure:
217
+ return interactiveOnly ? .failClosed : .sparseBest
218
+ }
219
+ }
220
+
221
+ static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
222
+ guard !nodes.isEmpty else { return false }
223
+ return nodes.allSatisfy { node in
224
+ // Application/Window labels are just the app/window name, and full-screen roots
225
+ // compute as hittable; neither says anything about tree health.
226
+ let isRootContainer = node.type == "Application" || node.type == "Window"
227
+ let hasContent = (!isRootContainer && node.label?.isEmpty == false)
228
+ || node.identifier?.isEmpty == false
229
+ || node.value?.isEmpty == false
230
+ return !hasContent
231
+ && (isRootContainer || !node.hittable)
232
+ && Self.structuralOnlyNodeTypes.contains(node.type)
233
+ }
234
+ }
235
+
236
+ /// A leaf whose label joins many short segments is a container marked as an accessibility
237
+ /// element: the platform folds every descendant into one merged node. Nothing below it can
238
+ /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
239
+ static func collapsedLeafIndexes(_ nodes: [SnapshotNode]) -> [Int]? {
240
+ let parents = Set(nodes.compactMap { $0.parentIndex })
241
+ let collapsed = nodes.filter { node in
242
+ guard !parents.contains(node.index) else { return false }
243
+ guard !(node.type.lowercased().contains("text")) else { return false }
244
+ let label = node.label ?? ""
245
+ return label.split(separator: ",").count > collapsedLeafMinimumSegments
246
+ }
247
+ return collapsed.isEmpty ? nil : collapsed.map(\.index)
248
+ }
249
+
250
+ // MARK: Outcome stamping
251
+
252
+ private func stampedSnapshotPayload(
253
+ _ capture: SnapshotBackendCapture,
254
+ backend: SnapshotBackendKind,
255
+ state: String,
256
+ reason: (reason: String, code: String)?
257
+ ) -> DataPayload {
258
+ let payload = capture.payload
259
+ let quality = SnapshotQuality(
260
+ state: state,
261
+ backend: backend.rawValue,
262
+ reason: reason?.reason,
263
+ reasonCode: reason?.code,
264
+ effectiveDepth: capture.effectiveDepth,
265
+ collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? [])
266
+ )
267
+ return DataPayload(
268
+ // Legacy human text for older daemons that read message instead of snapshotQuality.
269
+ message: Self.legacyQualityMessage(quality) ?? payload.message,
270
+ nodes: payload.nodes,
271
+ truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil,
272
+ snapshotQuality: quality,
273
+ runnerFatal: payload.runnerFatal,
274
+ runnerFatalReason: payload.runnerFatalReason
275
+ )
276
+ }
277
+
278
+ static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? {
279
+ guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil }
280
+ var parts: [String] = []
281
+ if quality.state == "recovered" {
282
+ let meaning = quality.reasonCode == "budget"
283
+ ? " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen."
284
+ : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears."
285
+ parts.append(
286
+ "Recovered this snapshot with the \(quality.backend) accessibility backend"
287
+ + (quality.reason.map { " after: \($0)." } ?? ".")
288
+ + meaning
289
+ )
290
+ }
291
+ if quality.state == "sparse" {
292
+ parts.append(
293
+ "No snapshot backend could read this screen"
294
+ + (quality.reason.map { " (\($0))" } ?? "")
295
+ + ". Use screenshot as visual truth and coordinate taps."
296
+ )
297
+ }
298
+ if let depth = quality.effectiveDepth {
299
+ parts.append(
300
+ "The accessibility server rejected deeper requests; this tree is capped at depth \(depth) — re-run with --depth \(depth) --scope <container> for deeper content."
301
+ )
302
+ }
303
+ return parts.isEmpty ? nil : parts.joined(separator: " ")
304
+ }
305
+ }
306
+
307
+ // MARK: - In-bundle unit tests
308
+
309
+ extension RunnerTests {
310
+ private func planTestNode(
311
+ index: Int,
312
+ type: String,
313
+ label: String? = nil,
314
+ identifier: String? = nil,
315
+ hittable: Bool = false,
316
+ parentIndex: Int? = nil
317
+ ) -> SnapshotNode {
318
+ SnapshotNode(
319
+ index: index,
320
+ type: type,
321
+ label: label,
322
+ identifier: identifier,
323
+ value: nil,
324
+ rect: snapshotRect(from: .zero),
325
+ enabled: true,
326
+ focused: nil,
327
+ selected: nil,
328
+ hittable: hittable,
329
+ depth: parentIndex == nil ? 0 : 1,
330
+ parentIndex: parentIndex,
331
+ hiddenContentAbove: nil,
332
+ hiddenContentBelow: nil
333
+ )
334
+ }
335
+
336
+ func testSparsePayloadReasonMatrix() {
337
+ let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true)
338
+ let window = planTestNode(index: 1, type: "Window", parentIndex: 0)
339
+ let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0)
340
+
341
+ // Labeled, hittable root over a bare window is still sparse.
342
+ XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false)))
343
+ // Deadline-truncated near-empty sweep needs recovery even with one real control.
344
+ XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true)))
345
+ // The same tiny tree from a completed sweep is a legitimately minimal screen.
346
+ XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false)))
347
+ // Empty payloads are degraded.
348
+ XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false)))
349
+ }
350
+
351
+ func testCollapsedLeafIndexesFlagsMergedContainersOnly() {
352
+ let root = planTestNode(index: 0, type: "Application", label: "App")
353
+ let merged = planTestNode(
354
+ index: 1,
355
+ type: "Other",
356
+ label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "),
357
+ parentIndex: 0
358
+ )
359
+ let prose = planTestNode(
360
+ index: 2,
361
+ type: "StaticText",
362
+ label: (0...30).map { "clause \($0)" }.joined(separator: ", "),
363
+ parentIndex: 0
364
+ )
365
+ XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1])
366
+ XCTAssertNil(Self.collapsedLeafIndexes([root, prose]))
367
+ }
368
+
369
+ func testLegacyQualityMessageStatesFallbackMeaning() {
370
+ let recovered = SnapshotQuality(
371
+ state: "recovered",
372
+ backend: "queries",
373
+ reason: "snapshot returned only structural application/window nodes",
374
+ reasonCode: "sparse-tree",
375
+ effectiveDepth: nil,
376
+ collapsedLeafIndexes: nil
377
+ )
378
+ let message = Self.legacyQualityMessage(recovered)
379
+ XCTAssertTrue(message?.contains("queries accessibility backend") == true)
380
+ XCTAssertTrue(message?.contains("fixing the app's accessibility") == true)
381
+ XCTAssertTrue(message?.contains("screenshot as visual truth") == true)
382
+ XCTAssertNil(
383
+ Self.legacyQualityMessage(
384
+ SnapshotQuality(
385
+ state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil,
386
+ collapsedLeafIndexes: nil)
387
+ )
388
+ )
389
+ }
390
+ func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() {
391
+ // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root
392
+ // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard).
393
+ XCTAssertEqual(
394
+ Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true),
395
+ .failClosed
396
+ )
397
+ XCTAssertEqual(
398
+ Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false),
399
+ .sparseBest
400
+ )
401
+ XCTAssertEqual(
402
+ Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true),
403
+ .throwAxFailure
404
+ )
405
+ }
406
+
407
+ func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() {
408
+ currentApp = app
409
+ currentBundleId = "com.example.app"
410
+ defer {
411
+ currentApp = nil
412
+ currentBundleId = nil
413
+ }
414
+ let payload = snapshotAccessibilityUnavailable(
415
+ failure: SnapshotCaptureFailure(
416
+ code: "IOS_AX_SNAPSHOT_FAILED",
417
+ message: "kAXErrorIllegalArgument",
418
+ hint: "use screenshot"
419
+ )
420
+ )
421
+ XCTAssertEqual(payload.runnerFatal, true)
422
+ XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
423
+ XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
424
+ }
425
+ }
@@ -29,12 +29,27 @@ extension RunnerTests {
29
29
  }
30
30
  var nodes: [SnapshotNode] = [modalNode]
31
31
 
32
+ for content in informativeElements(in: modal, excluding: actions) {
33
+ guard let contentNode = safeMakeSnapshotNode(
34
+ element: content,
35
+ index: nodes.count,
36
+ type: elementTypeName(content.elementType),
37
+ depth: 1,
38
+ parentIndex: 0,
39
+ hittableOverride: false
40
+ ) else {
41
+ continue
42
+ }
43
+ nodes.append(contentNode)
44
+ }
45
+
32
46
  for action in actions {
33
47
  guard let actionNode = safeMakeSnapshotNode(
34
48
  element: action,
35
49
  index: nodes.count,
36
50
  type: elementTypeName(action.elementType),
37
51
  depth: 1,
52
+ parentIndex: 0,
38
53
  hittableOverride: true
39
54
  ) else {
40
55
  continue
@@ -101,9 +116,7 @@ extension RunnerTests {
101
116
  func actionableElements(in element: XCUIElement) -> [XCUIElement] {
102
117
  var seen = Set<String>()
103
118
  var actions: [XCUIElement] = []
104
- let descendants = safeElementsQuery {
105
- element.descendants(matching: .any).allElementsBoundByIndex
106
- }
119
+ let descendants = actionableTypes.flatMap { modalDescendants(in: element, matching: $0) }
107
120
  for candidate in descendants {
108
121
  if !safeIsActionableCandidate(candidate, seen: &seen) { continue }
109
122
  actions.append(candidate)
@@ -124,6 +137,61 @@ extension RunnerTests {
124
137
  }
125
138
  }
126
139
 
140
+ private func informativeElements(in element: XCUIElement, excluding actions: [XCUIElement]) -> [XCUIElement] {
141
+ let actionKeys = Set(actions.map(systemModalElementKey))
142
+ var seen = Set<String>()
143
+ var contents: [XCUIElement] = []
144
+ let descendants = readableSystemModalTypes.flatMap {
145
+ modalDescendants(in: element, matching: $0, limit: 2)
146
+ }
147
+ for candidate in descendants {
148
+ guard let key = safeInformativeElementKey(candidate, actionKeys: actionKeys) else {
149
+ continue
150
+ }
151
+ if seen.contains(key) { continue }
152
+ seen.insert(key)
153
+ contents.append(candidate)
154
+ }
155
+ return contents
156
+ }
157
+
158
+ private var readableSystemModalTypes: [XCUIElement.ElementType] {
159
+ [.staticText, .textView]
160
+ }
161
+
162
+ private func modalDescendants(
163
+ in element: XCUIElement,
164
+ matching type: XCUIElement.ElementType,
165
+ limit: Int? = nil
166
+ ) -> [XCUIElement] {
167
+ let elements = safeElementsQuery {
168
+ element.descendants(matching: type).allElementsBoundByIndex
169
+ }
170
+ guard let limit else {
171
+ return elements
172
+ }
173
+ return Array(elements.prefix(limit))
174
+ }
175
+
176
+ private func safeInformativeElementKey(_ candidate: XCUIElement, actionKeys: Set<String>) -> String? {
177
+ safely("MODAL_CONTENT") { () -> String? in
178
+ let key = systemModalElementKey(candidate)
179
+ if actionKeys.contains(key) { return nil }
180
+ if actionableTypes.contains(candidate.elementType) { return nil }
181
+ if !candidate.exists { return nil }
182
+ let frame = candidate.frame
183
+ if frame.isNull || frame.isEmpty { return nil }
184
+ let label = candidate.label.trimmingCharacters(in: .whitespacesAndNewlines)
185
+ if label.isEmpty { return nil }
186
+ return key
187
+ }
188
+ }
189
+
190
+ private func systemModalElementKey(_ element: XCUIElement) -> String {
191
+ let frame = element.frame
192
+ return "\(element.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(element.label)-\(element.identifier)"
193
+ }
194
+
127
195
  private func preferredSystemModalTitle(_ element: XCUIElement) -> String {
128
196
  let label = element.label
129
197
  if !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -143,6 +211,7 @@ extension RunnerTests {
143
211
  labelOverride: String? = nil,
144
212
  identifierOverride: String? = nil,
145
213
  depth: Int,
214
+ parentIndex: Int? = nil,
146
215
  hittableOverride: Bool? = nil
147
216
  ) -> SnapshotNode {
148
217
  let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
@@ -155,11 +224,11 @@ extension RunnerTests {
155
224
  value: nil,
156
225
  rect: snapshotRect(from: element.frame),
157
226
  enabled: element.isEnabled,
158
- focused: elementHasFocus(element) ? true : nil,
159
- selected: element.isSelected ? true : nil,
227
+ focused: nil,
228
+ selected: nil,
160
229
  hittable: hittableOverride ?? element.isHittable,
161
230
  depth: depth,
162
- parentIndex: nil,
231
+ parentIndex: parentIndex,
163
232
  hiddenContentAbove: nil,
164
233
  hiddenContentBelow: nil
165
234
  )
@@ -172,6 +241,7 @@ extension RunnerTests {
172
241
  labelOverride: String? = nil,
173
242
  identifierOverride: String? = nil,
174
243
  depth: Int,
244
+ parentIndex: Int? = nil,
175
245
  hittableOverride: Bool? = nil
176
246
  ) -> SnapshotNode? {
177
247
  safely("MODAL_NODE") {
@@ -182,6 +252,7 @@ extension RunnerTests {
182
252
  labelOverride: labelOverride,
183
253
  identifierOverride: identifierOverride,
184
254
  depth: depth,
255
+ parentIndex: parentIndex,
185
256
  hittableOverride: hittableOverride
186
257
  )
187
258
  }
@@ -170,8 +170,17 @@ extension RunnerTests {
170
170
  // MARK: - Response Encoding
171
171
 
172
172
  private func jsonResponse(status: Int, response: Response) -> Data {
173
+ // Stamp the gesture-clock uptime at the END of command handling, just before the HTTP
174
+ // write, so the warm snapshot and recordStart responses carry the anchor for free. This
175
+ // runs AFTER commandJournal.finish, so journal-stored lifecycleResponseJson stays
176
+ // unstamped — recovered/status-replayed results carry no anchor and the daemon falls back
177
+ // rather than pairing a stale uptime with a much-later receipt time.
178
+ let stamped =
179
+ response.ok
180
+ ? response.stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
181
+ : response
173
182
  let encoder = JSONEncoder()
174
- let body = (try? encoder.encode(response)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
183
+ let body = (try? encoder.encode(stamped)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
175
184
  return httpResponse(status: status, body: body)
176
185
  }
177
186
 
@@ -35,16 +35,16 @@ Protocol and maintenance references:
35
35
 
36
36
  ## Snapshot Strategy
37
37
 
38
- iOS snapshots have two explicit capture modes:
38
+ iOS snapshots have two explicit public capture modes:
39
39
 
40
40
  - full/raw snapshots use recursive XCTest snapshots for rich hierarchy and diagnostics;
41
- - compact interactive snapshots use a bounded flat XCTest query path for fast agent-facing refs.
41
+ - interactive snapshots filter the same visible tree down to agent-facing refs.
42
42
 
43
43
  Some simulator apps expose accessibility trees that lower-level AX services can inspect but XCTest
44
- cannot serialize reliably. In those cases compact interactive snapshots may return a sparse root
45
- quickly, while full snapshots preserve the XCTest error. See
44
+ cannot serialize reliably. In those cases interactive snapshots may return a sparse root quickly,
45
+ while full snapshots preserve the XCTest error. See
46
46
  [`../docs/adr/0004-ios-snapshot-backend-strategy.md`](../docs/adr/0004-ios-snapshot-backend-strategy.md)
47
- for the backend boundary and the rationale for a future simulator AX-service backend.
47
+ for the backend boundary and future simulator AX-service direction.
48
48
 
49
49
  ## Protocol Notes
50
50
 
@@ -25,7 +25,6 @@ Examples:
25
25
  {
26
26
  "command": "snapshot",
27
27
  "interactiveOnly": true,
28
- "compact": true,
29
28
  "depth": 2,
30
29
  "scope": "app",
31
30
  "raw": false
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-device",
3
- "version": "0.17.2",
3
+ "version": "0.17.4",
4
4
  "description": "Agent-native CLI for AI mobile testing and app automation across iOS, Android, tvOS, Android TV, macOS, and Linux.",
5
5
  "mcpName": "io.github.callstackincubator/agent-device",
6
6
  "license": "MIT",
@@ -132,8 +132,8 @@
132
132
  "test:unit": "vitest run --project unit",
133
133
  "test:coverage": "vitest run --coverage",
134
134
  "test:integration:provider": "vitest run --project provider-integration",
135
- "test:integration:progress": "node scripts/integration-progress.mjs",
136
- "test:integration:progress:check": "node scripts/integration-progress.mjs --check",
135
+ "test:integration:progress": "node --experimental-strip-types scripts/integration-progress.ts",
136
+ "test:integration:progress:check": "node --experimental-strip-types scripts/integration-progress.ts --check",
137
137
  "test:skillgym": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts",
138
138
  "test:skillgym:case": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts --case",
139
139
  "test:smoke": "node --test test/integration/smoke-*.test.ts",
package/server.json CHANGED
@@ -7,12 +7,12 @@
7
7
  "url": "https://github.com/callstackincubator/agent-device",
8
8
  "source": "github"
9
9
  },
10
- "version": "0.17.2",
10
+ "version": "0.17.4",
11
11
  "packages": [
12
12
  {
13
13
  "registryType": "npm",
14
14
  "identifier": "agent-device",
15
- "version": "0.17.2",
15
+ "version": "0.17.4",
16
16
  "transport": {
17
17
  "type": "stdio"
18
18
  }
@@ -1 +0,0 @@
1
- e50fe5576ae7edbb9ae18414ae65d094de416ba3e0f6e418ac3cdc319e491a0d agent-device-android-multitouch-helper-0.17.2.apk
@@ -1 +0,0 @@
1
- 0f91b28c7374fbf1d12fa43c283d01354b3a4d68a44b0dc68ccc22959f02d13e agent-device-android-snapshot-helper-0.17.2.apk
package/dist/src/1534.js DELETED
@@ -1 +0,0 @@
1
- import{AppError as e}from"./9152.js";function t(e){return e?{message:e}:{}}function r(e,t){return t?{...e,message:t}:e}function a(e){return"string"==typeof e?.message&&e.message.length>0?e.message:null}let n=["portrait","portrait-upside-down","landscape-left","landscape-right"];function s(t){if(void 0===t)throw new e("INVALID_ARGS","rotate requires an orientation argument. Use portrait|portrait-upside-down|landscape-left|landscape-right.");switch(t?.trim().toLowerCase()){case"portrait":return"portrait";case"portrait-upside-down":case"upside-down":return"portrait-upside-down";case"landscape-left":case"left":return"landscape-left";case"landscape-right":case"right":return"landscape-right";default:throw new e("INVALID_ARGS",`Invalid rotation: ${t}. Use portrait|portrait-upside-down|landscape-left|landscape-right.`)}}export{n as DEVICE_ROTATIONS,s as parseDeviceRotation,a as readCommandMessage,t as successText,r as withSuccessText};