agent-device 0.19.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/README.md +61 -70
  2. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.0.manifest.json +10 -0
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.apk.sha256 +1 -0
  7. package/{android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.manifest.json → android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.0.manifest.json} +6 -6
  8. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +2 -0
  9. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +5 -0
  10. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +12 -10
  11. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +154 -185
  12. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +113 -19
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +82 -0
  14. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +275 -113
  15. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +2 -2
  16. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +150 -104
  17. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +56 -2
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +73 -0
  19. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +44 -14
  20. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +2 -2
  21. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +2 -106
  22. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +126 -11
  23. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +51 -8
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +24 -0
  25. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +1 -1
  26. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +43 -35
  27. package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +14 -0
  28. package/dist/src/agent-browser-tool.js +1 -1
  29. package/dist/src/agent-device-client.js +3 -0
  30. package/dist/src/android-adb.d.ts +1 -1
  31. package/dist/src/android-adb.js +1 -1
  32. package/dist/src/android-input-method-overlays.js +1 -0
  33. package/dist/src/android-input-ownership.js +1 -6
  34. package/dist/src/android-system-chrome.js +1 -0
  35. package/dist/src/android.js +1 -1
  36. package/dist/src/app-lifecycle.js +5 -5
  37. package/dist/src/app-log-request-scope.js +1 -0
  38. package/dist/src/app-resolution.js +1 -0
  39. package/dist/src/apple-multitouch-support.js +1 -0
  40. package/dist/src/apps.js +5 -5
  41. package/dist/src/audio-probe.js +1 -1
  42. package/dist/src/boot-diagnostics.js +2 -0
  43. package/dist/src/cli-config.js +1 -0
  44. package/dist/src/cli-help.js +64 -30
  45. package/dist/src/cli-runner.js +1 -1
  46. package/dist/src/cli.js +15 -11
  47. package/dist/src/command-schema.js +3 -3
  48. package/dist/src/config.js +1 -0
  49. package/dist/src/context.js +1 -0
  50. package/dist/src/contracts.d.ts +1 -1
  51. package/dist/src/contracts.js +1 -1
  52. package/dist/src/daemon-shutdown-report.js +1 -0
  53. package/dist/src/debug-symbols.js +1 -1
  54. package/dist/src/device-input-state.js +1 -1
  55. package/dist/src/devices2.js +1 -1
  56. package/dist/src/find.js +1 -1
  57. package/dist/src/finders.js +1 -1
  58. package/dist/src/generic.js +1 -1
  59. package/dist/src/helper.js +1 -1
  60. package/dist/src/index.d.ts +412 -289
  61. package/dist/src/index.js +1 -1
  62. package/dist/src/input-actions.js +4 -3
  63. package/dist/src/input-actions2.js +1 -1
  64. package/dist/src/install-artifact.js +1 -0
  65. package/dist/src/install-source.d.ts +5 -1
  66. package/dist/src/install-source.js +1 -1
  67. package/dist/src/install-source2.js +1 -1
  68. package/dist/src/interaction-snapshot.js +1 -1
  69. package/dist/src/interaction.d.ts +230 -0
  70. package/dist/src/interaction.js +1 -1
  71. package/dist/src/interactor.js +2 -2
  72. package/dist/src/internal/bin.js +1 -1
  73. package/dist/src/internal/daemon.js +43 -17
  74. package/dist/src/internal/png-worker.d.ts +22 -0
  75. package/dist/src/internal/png-worker.js +1 -1
  76. package/dist/src/launch-console.js +1 -0
  77. package/dist/src/lease-context.js +1 -0
  78. package/dist/src/lease.js +1 -1
  79. package/dist/src/linux.js +1 -1
  80. package/dist/src/logcat.js +5 -0
  81. package/dist/src/metro.d.ts +2 -2
  82. package/dist/src/notifications.js +1 -1
  83. package/dist/src/png-worker-contract.js +1 -1
  84. package/dist/src/process-lock.js +1 -0
  85. package/dist/src/react-native.js +1 -1
  86. package/dist/src/record-trace.js +1 -1
  87. package/dist/src/register-builtins.js +1 -0
  88. package/dist/src/registry.js +24 -24
  89. package/dist/src/reporting.js +8 -7
  90. package/dist/src/runner-client.js +1 -1
  91. package/dist/src/runner-disposal.js +2 -0
  92. package/dist/src/runner-lease.js +1 -0
  93. package/dist/src/runner-sequence.js +1 -1
  94. package/dist/src/runtime-transport.js +1 -1
  95. package/dist/src/runtime.js +3 -3
  96. package/dist/src/runtime2.js +1 -0
  97. package/dist/src/sdk-android-adb.d.ts +119 -33
  98. package/dist/src/sdk-android-adb.js +4 -4
  99. package/dist/src/sdk-app-inventory.js +1 -1
  100. package/dist/src/sdk-batch-runner.d.ts +2 -1
  101. package/dist/src/sdk-batch-runner.js +2 -2
  102. package/dist/src/sdk-batch.js +2 -2
  103. package/dist/src/sdk-contracts.d.ts +105 -98
  104. package/dist/src/sdk-contracts.js +2 -2
  105. package/dist/src/sdk-device.js +1 -0
  106. package/dist/src/sdk-finders.d.ts +3 -3
  107. package/dist/src/sdk-finders.js +1 -1
  108. package/dist/src/sdk-io.d.ts +1 -1
  109. package/dist/src/sdk-io.js +1 -1
  110. package/dist/src/sdk-remote-config.d.ts +9 -1
  111. package/dist/src/sdk-remote-config.js +1 -1
  112. package/dist/src/sdk-selectors.js +1 -1
  113. package/dist/src/selector-runtime.js +1 -1
  114. package/dist/src/selector-vocabulary.js +1 -0
  115. package/dist/src/selectors.d.ts +12 -3
  116. package/dist/src/selectors.js +1 -1
  117. package/dist/src/server.js +3 -3
  118. package/dist/src/session-target-evidence.js +1 -0
  119. package/dist/src/session.js +8 -10
  120. package/dist/src/setting-state.js +1 -1
  121. package/dist/src/simulator.js +1 -1
  122. package/dist/src/snapshot-diagnostics.js +5 -2
  123. package/dist/src/snapshot.js +1 -1
  124. package/dist/src/snapshot2.js +1 -2
  125. package/dist/src/tool-provider.js +2 -2
  126. package/dist/src/tool-provider2.js +1 -1
  127. package/dist/src/tv-remote.js +13 -12
  128. package/dist/src/url.js +1 -1
  129. package/dist/src/web.js +1 -1
  130. package/package.json +33 -20
  131. package/server.json +2 -2
  132. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.apk +0 -0
  133. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.apk.sha256 +0 -1
  134. package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.2.manifest.json +0 -10
  135. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.apk +0 -0
  136. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.2.apk.sha256 +0 -1
  137. package/dist/src/apple-runner-options.js +0 -1
  138. package/dist/src/cli-output.js +0 -1
  139. package/dist/src/client.js +0 -3
  140. package/dist/src/handler-utils.js +0 -1
  141. package/dist/src/output-path.js +0 -1
  142. package/dist/src/points.js +0 -4
  143. package/dist/src/provider.js +0 -1
  144. package/dist/src/providers.js +0 -1
  145. package/dist/src/record-trace-recording.js +0 -23
  146. package/dist/src/recording-provider.js +0 -1
  147. package/dist/src/selectors2.js +0 -1
  148. package/dist/src/simctl.js +0 -1
  149. package/dist/src/snapshot-helper.js +0 -5
  150. package/dist/src/ttl-memo.js +0 -1
  151. package/dist/src/xml.js +0 -2
  152. /package/{macos-helper → apple/macos-helper}/Package.swift +0 -0
  153. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +0 -0
  154. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +0 -0
  155. /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/main.swift +0 -0
  156. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +0 -0
  157. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +0 -0
  158. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -0
  159. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +0 -0
  160. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +0 -0
  161. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +0 -0
  162. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.h +0 -0
  163. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.m +0 -0
  164. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +0 -0
  165. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +0 -0
  166. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +0 -0
  167. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +0 -0
  168. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +0 -0
  169. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +0 -0
  170. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +0 -0
  171. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +0 -0
  172. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +0 -0
  173. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +0 -0
  174. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +0 -0
  175. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +0 -0
  176. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -0
  177. /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-trim.swift +0 -0
  178. /package/{src/platforms/linux → linux}/atspi-dump.py +0 -0
@@ -9,7 +9,7 @@ extension RunnerTests {
9
9
 
10
10
  /// Allowlisted step kinds. Validated on both sides so an unsupported kind is rejected with a
11
11
  /// clear INVALID_ARGS naming the step index, executing nothing.
12
- private var sequenceableStepKinds: Set<String> { ["tap", "doubleTap", "longPress", "drag"] }
12
+ private var sequenceableStepKinds: Set<String> { ["tap", "doubleTap", "longPress"] }
13
13
 
14
14
  /// Per-step outcome carried by `assembleSequenceExecution`. The timing is captured by the
15
15
  /// executor closure (via performGesture) so ordering/stop-on-failure stay device-free testable.
@@ -127,17 +127,12 @@ extension RunnerTests {
127
127
  private func validateSequenceStep(_ step: SequenceStep, index: Int) -> Response? {
128
128
  guard sequenceableStepKinds.contains(step.kind) else {
129
129
  return sequenceInvalidArgs(
130
- "sequence step \(index) has unsupported kind \"\(step.kind)\"; allowed: tap, doubleTap, longPress, drag"
130
+ "sequence step \(index) has unsupported kind \"\(step.kind)\"; allowed: tap, doubleTap, longPress"
131
131
  )
132
132
  }
133
133
  guard let x = step.x, let y = step.y, x.isFinite, y.isFinite else {
134
134
  return sequenceInvalidArgs("sequence step \(index) (\(step.kind)) requires finite x and y")
135
135
  }
136
- if step.kind == "drag" {
137
- guard let x2 = step.x2, let y2 = step.y2, x2.isFinite, y2.isFinite else {
138
- return sequenceInvalidArgs("sequence step \(index) (drag) requires finite x2 and y2")
139
- }
140
- }
141
136
  return nil
142
137
  }
143
138
 
@@ -194,91 +189,6 @@ extension RunnerTests {
194
189
  #endif
195
190
  // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
196
191
  }
197
- if step.kind == "drag", step.synthesized == true {
198
- let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
199
- let dragPoints: DragPoints
200
- let dragContext: SynthesizedCoordinateContext?
201
- #if os(iOS)
202
- guard let dragPlan = axFreeSynthesizedDragPlan(
203
- app: activeApp,
204
- x: x,
205
- y: y,
206
- x2: step.x2 ?? x,
207
- y2: step.y2 ?? y,
208
- context: synthesizedContext
209
- ) else {
210
- let nowMs = ProcessInfo.processInfo.systemUptime * 1000
211
- logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
212
- return SequenceStepOutcome(
213
- outcome: .unsupported(
214
- message: "synthesized coordinate drag could not resolve a finite coordinate frame",
215
- hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
216
- ),
217
- gestureStartUptimeMs: nowMs,
218
- gestureEndUptimeMs: nowMs
219
- )
220
- }
221
- dragPoints = dragPlan.points
222
- dragContext = dragPlan.context
223
- #else
224
- dragPoints = keyboardAvoidingDragPoints(
225
- app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
226
- dragContext = nil
227
- #endif
228
- let durationMs = min(max(step.durationMs ?? 250, 16), 10000)
229
- let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
230
- synthesizedDragAt(
231
- app: activeApp,
232
- x: dragPoints.x,
233
- y: dragPoints.y,
234
- x2: dragPoints.x2,
235
- y2: dragPoints.y2,
236
- durationMs: durationMs,
237
- context: dragContext
238
- )
239
- }
240
- if case .performed = outcome {
241
- logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
242
- if let pauseMs = step.pauseMs, pauseMs > 0 {
243
- sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
244
- }
245
- return SequenceStepOutcome(
246
- outcome: outcome,
247
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
248
- gestureEndUptimeMs: timing.gestureEndUptimeMs
249
- )
250
- }
251
- #if os(iOS)
252
- guard dragContext?.allowsXCTestCoordinateFallback == true else {
253
- logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
254
- return SequenceStepOutcome(
255
- outcome: outcome,
256
- gestureStartUptimeMs: timing.gestureStartUptimeMs,
257
- gestureEndUptimeMs: timing.gestureEndUptimeMs
258
- )
259
- }
260
- #endif
261
- logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: true)
262
- let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250)
263
- let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) {
264
- dragAt(
265
- app: activeApp,
266
- x: dragPoints.x,
267
- y: dragPoints.y,
268
- x2: dragPoints.x2,
269
- y2: dragPoints.y2,
270
- holdDuration: fallbackHoldDuration
271
- )
272
- }
273
- if case .performed = fallbackOutcome, let pauseMs = step.pauseMs, pauseMs > 0 {
274
- sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
275
- }
276
- return SequenceStepOutcome(
277
- outcome: fallbackOutcome,
278
- gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs,
279
- gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs
280
- )
281
- }
282
192
  let (timing, outcome) = performGesture(activeApp) {
283
193
  switch step.kind {
284
194
  case "doubleTap":
@@ -287,20 +197,6 @@ extension RunnerTests {
287
197
  case "longPress":
288
198
  let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0
289
199
  return longPressAt(app: activeApp, x: x, y: y, duration: duration)
290
- case "drag":
291
- // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command.
292
- // The non-synthesized coordinate-drag path ignores durationMs, matching that command's
293
- // non-synthesized branch.
294
- let dragPoints = keyboardAvoidingDragPoints(
295
- app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
296
- return dragAt(
297
- app: activeApp,
298
- x: dragPoints.x,
299
- y: dragPoints.y,
300
- x2: dragPoints.x2,
301
- y2: dragPoints.y2,
302
- holdDuration: coordinateDragHoldDuration()
303
- )
304
200
  default:
305
201
  return tapAt(app: activeApp, x: x, y: y)
306
202
  }
@@ -101,15 +101,22 @@ extension RunnerTests {
101
101
 
102
102
  static let flatInteractiveFallbackBudget: TimeInterval = 1.0
103
103
 
104
+ // The single production entry point -- always compiled, no unit-test overload. A unit test
105
+ // exercises this exact function; the only injectable seam lives inside
106
+ // `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting`
107
+ // in RunnerTests.swift), so reverting this entry point to bypass the bounded probe fails the
108
+ // regression test.
104
109
  func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
105
- if let blocking = blockingSystemAlertSnapshot() {
110
+ let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
111
+ if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) {
106
112
  return blocking
107
113
  }
108
114
  return try runSnapshotCapturePlan(
109
115
  Self.regularVisiblePlan,
110
116
  app: app,
111
117
  options: options,
112
- terminal: .sparseWithFatalOnAXFailure
118
+ terminal: .sparseWithFatalOnAXFailure,
119
+ deadline: deadline
113
120
  )
114
121
  }
115
122
 
@@ -189,7 +196,12 @@ extension RunnerTests {
189
196
  regularSnapshot: true
190
197
  )
191
198
 
192
- let key = "\(snapshot.elementType)-\(evaluation.label)-\(evaluation.identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
199
+ let key = Self.snapshotTraversalIdentity(
200
+ elementType: snapshot.elementType,
201
+ label: evaluation.label,
202
+ identifier: evaluation.identifier,
203
+ frame: snapshot.frame
204
+ )
193
205
  let isDuplicate = seen.contains(key)
194
206
  if !isDuplicate {
195
207
  seen.insert(key)
@@ -245,18 +257,98 @@ extension RunnerTests {
245
257
  )
246
258
  }
247
259
 
260
+ // See `snapshotFast` above: the single production entry point, no unit-test overload.
248
261
  func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
249
- if let blocking = blockingSystemAlertSnapshot() {
262
+ let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
263
+ if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) {
250
264
  return blocking
251
265
  }
252
266
  return try runSnapshotCapturePlan(
253
267
  Self.rawDiagnosticPlan,
254
268
  app: app,
255
269
  options: options,
256
- terminal: .throwOnAXFailure
270
+ terminal: .throwOnAXFailure,
271
+ deadline: deadline
257
272
  )
258
273
  }
259
274
 
275
+ /// Runs the pre-plan SpringBoard system-modal probe as a bounded capture tier sharing the plan
276
+ /// deadline, so a slow alert enumeration cannot bypass the snapshot timeout and stall (#1244).
277
+ func boundedBlockingSystemAlertSnapshot(deadline: Date) -> DataPayload? {
278
+ boundedBlockingSystemAlertSnapshotBody(deadline: deadline) { probeDeadline in
279
+ return self.blockingSystemAlertSnapshot(deadline: probeDeadline)
280
+ }
281
+ }
282
+
283
+ /// The real bounding/hook machinery used by `boundedBlockingSystemAlertSnapshot` above: the
284
+ /// probe closure it's given always calls `self.blockingSystemAlertSnapshot` in production, and
285
+ /// in unit-test builds may first consult `systemModalProbeOverrideForTesting`. Keeping this in
286
+ /// one place means the `runMainThreadWork` wrap and the `onAbandoned`/`onDrained` hooks can
287
+ /// never drift between what production runs and what the unit tests exercise.
288
+ private func boundedBlockingSystemAlertSnapshotBody(
289
+ deadline: Date,
290
+ probe: @escaping (Date) -> DataPayload?
291
+ ) -> DataPayload? {
292
+ #if os(macOS)
293
+ return nil
294
+ #else
295
+ let slice = Self.systemModalProbeSlice(
296
+ budget: systemModalProbeBudget,
297
+ deadlineRemaining: deadline.timeIntervalSinceNow
298
+ )
299
+ guard slice > 0 else {
300
+ NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_SKIPPED reason=budget_exhausted")
301
+ return nil
302
+ }
303
+ let probeDeadline = Date().addingTimeInterval(slice)
304
+ let startedAt = Date()
305
+ do {
306
+ return try runMainThreadWork(
307
+ command: nil,
308
+ timeout: slice,
309
+ timeoutError: {
310
+ SnapshotCaptureFailure(
311
+ code: Self.xCTestSnapshotTimeoutCode,
312
+ message: "the system-modal probe exceeded its \(slice)s time slice",
313
+ hint: "The capture plan recovers through non-XCTest snapshot tiers while the modal probe drains."
314
+ )
315
+ },
316
+ onAbandoned: {
317
+ self.retainAbandonedXCTestChannelWork()
318
+ NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_TIMEOUT slice=%.1f", slice)
319
+ self.penalizeSnapshotXCTestChannel(
320
+ bundleId: self.currentBundleId,
321
+ reason: "system_modal_probe_timeout"
322
+ )
323
+ },
324
+ onDrained: {
325
+ self.releaseAbandonedXCTestChannelWork()
326
+ NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_DRAINED")
327
+ }
328
+ ) {
329
+ probe(probeDeadline)
330
+ }
331
+ } catch {
332
+ NSLog(
333
+ "AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_ABORTED elapsedMs=%d error=%@",
334
+ Int(Date().timeIntervalSince(startedAt) * 1000),
335
+ String(describing: error)
336
+ )
337
+ return nil
338
+ }
339
+ #endif
340
+ }
341
+
342
+ /// The probe gets its own budget, clamped by whatever remains of the shared plan deadline, and
343
+ /// 0 (skip entirely) once that deadline is already spent.
344
+ static func systemModalProbeSlice(
345
+ budget: TimeInterval,
346
+ deadlineRemaining: TimeInterval
347
+ ) -> TimeInterval {
348
+ guard deadlineRemaining > 0 else { return 0 }
349
+ return min(budget, deadlineRemaining)
350
+ }
351
+
260
352
  func rawTreeSnapshotPayload(
261
353
  context: SnapshotTraversalContext,
262
354
  options: SnapshotOptions
@@ -566,6 +658,20 @@ extension RunnerTests {
566
658
  return abandonedTreeCaptureCount > 0
567
659
  }
568
660
 
661
+ /// The watchdog abandoned one unit of XCTest main-thread capture work that is still draining on
662
+ /// main; XCTest-backed snapshot tiers skip (`hasAbandonedTreeCapture`) until a matching release.
663
+ func retainAbandonedXCTestChannelWork() {
664
+ treeCaptureLock.lock()
665
+ abandonedTreeCaptureCount += 1
666
+ treeCaptureLock.unlock()
667
+ }
668
+
669
+ func releaseAbandonedXCTestChannelWork() {
670
+ treeCaptureLock.lock()
671
+ abandonedTreeCaptureCount -= 1
672
+ treeCaptureLock.unlock()
673
+ }
674
+
569
675
  /// Runs the blocking tree-snapshot XPC on the main thread bounded by `sliceSeconds`. On
570
676
  /// timeout the XPC keeps running on main (it cannot be cancelled); the capture is marked
571
677
  /// abandoned so plans avoid XCTest-backed tiers until it drains, the tree backend is penalized
@@ -583,9 +689,7 @@ extension RunnerTests {
583
689
  timeout: sliceSeconds,
584
690
  timeoutError: treeCaptureTimeoutError(sliceSeconds: sliceSeconds),
585
691
  onAbandoned: {
586
- self.treeCaptureLock.lock()
587
- self.abandonedTreeCaptureCount += 1
588
- self.treeCaptureLock.unlock()
692
+ self.retainAbandonedXCTestChannelWork()
589
693
  NSLog("AGENT_DEVICE_RUNNER_TREE_CAPTURE_SLICE_TIMEOUT slice=%.1f", sliceSeconds)
590
694
  self.penalizeSnapshotXCTestChannel(
591
695
  bundleId: self.currentBundleId,
@@ -593,9 +697,7 @@ extension RunnerTests {
593
697
  )
594
698
  },
595
699
  onDrained: {
596
- self.treeCaptureLock.lock()
597
- self.abandonedTreeCaptureCount -= 1
598
- self.treeCaptureLock.unlock()
700
+ self.releaseAbandonedXCTestChannelWork()
599
701
  NSLog("AGENT_DEVICE_RUNNER_TREE_CAPTURE_DRAINED")
600
702
  }
601
703
  ) {
@@ -789,6 +891,19 @@ extension RunnerTests {
789
891
  return .infinite
790
892
  }
791
893
 
894
+ static func snapshotTraversalIdentity(
895
+ elementType: XCUIElement.ElementType,
896
+ label: String,
897
+ identifier: String,
898
+ frame: CGRect
899
+ ) -> String {
900
+ #if os(iOS)
901
+ "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)-\(frame.width)-\(frame.height)"
902
+ #else
903
+ return "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)"
904
+ #endif
905
+ }
906
+
792
907
  private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
793
908
  if depth > 4 { return nil }
794
909
  let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -118,6 +118,17 @@ extension RunnerTests {
118
118
  )
119
119
  }
120
120
 
121
+ func clearSnapshotXCTestChannelPenalty(reason: String) {
122
+ snapshotXCTestChannelPenaltyLock.lock()
123
+ let hadActivePenalty = Date() < snapshotXCTestChannelPenaltyUntil
124
+ snapshotXCTestChannelPenaltyBundleId = nil
125
+ snapshotXCTestChannelPenaltyUntil = Date.distantPast
126
+ snapshotXCTestChannelPenaltyLock.unlock()
127
+ if hadActivePenalty {
128
+ NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason)
129
+ }
130
+ }
131
+
121
132
  func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool {
122
133
  snapshotXCTestChannelPenaltyLock.lock()
123
134
  defer { snapshotXCTestChannelPenaltyLock.unlock() }
@@ -127,6 +138,12 @@ extension RunnerTests {
127
138
  return penalized == bundleId
128
139
  }
129
140
 
141
+ func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool {
142
+ let pending = snapshotXCTestPenaltyWarmupExemptionPending
143
+ snapshotXCTestPenaltyWarmupExemptionPending = false
144
+ return pending
145
+ }
146
+
130
147
  /// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends
131
148
  /// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw
132
149
  /// diagnostic plan keeps tree-first errors, and unknown plans are left untouched.
@@ -168,12 +185,15 @@ extension RunnerTests {
168
185
  _ plan: [SnapshotBackendKind],
169
186
  app: XCUIApplication,
170
187
  options: SnapshotOptions,
171
- terminal: SnapshotCaptureTerminalPolicy
188
+ terminal: SnapshotCaptureTerminalPolicy,
189
+ deadline: Date? = nil
172
190
  ) throws -> DataPayload {
173
191
  var best: (kind: SnapshotBackendKind, capture: SnapshotBackendCapture)?
174
192
  var firstFailure: (reason: String, code: String)?
175
193
  var axFailure: SnapshotCaptureFailure?
176
- let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
194
+ // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244).
195
+ let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget)
196
+ let suppressXCTestPenalty = consumeSnapshotXCTestPenaltyWarmupExemption()
177
197
 
178
198
  // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while
179
199
  // the app remains visually responsive. Simulators can avoid that channel through private AX;
@@ -237,14 +257,30 @@ extension RunnerTests {
237
257
  treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride
238
258
  )
239
259
  else {
240
- recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt)
260
+ recordSlowXCTestSnapshotBackendIfNeeded(
261
+ kind,
262
+ startedAt: backendStartedAt,
263
+ penaltySuppressed: suppressXCTestPenalty
264
+ )
241
265
  continue
242
266
  }
243
267
  capture = result
244
- recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt)
268
+ recordSlowXCTestSnapshotBackendIfNeeded(
269
+ kind,
270
+ startedAt: backendStartedAt,
271
+ penaltySuppressed: suppressXCTestPenalty
272
+ )
245
273
  } catch let failure as SnapshotCaptureFailure {
246
- recordXCTestSnapshotBackendFailureIfNeeded(kind, failure: failure)
247
- recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt)
274
+ recordXCTestSnapshotBackendFailureIfNeeded(
275
+ kind,
276
+ failure: failure,
277
+ penaltySuppressed: suppressXCTestPenalty
278
+ )
279
+ recordSlowXCTestSnapshotBackendIfNeeded(
280
+ kind,
281
+ startedAt: backendStartedAt,
282
+ penaltySuppressed: suppressXCTestPenalty
283
+ )
248
284
  if Self.isAxSnapshotFailure(failure) { axFailure = failure }
249
285
  if firstFailure == nil {
250
286
  firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed")
@@ -313,7 +349,12 @@ extension RunnerTests {
313
349
 
314
350
  /// Marks XCTest-backed snapshot tiers as penalized when one attempt ground past the slow-capture
315
351
  /// threshold — even a successful one: the next capture of this screen must not re-grind.
316
- private func recordSlowXCTestSnapshotBackendIfNeeded(_ kind: SnapshotBackendKind, startedAt: Date) {
352
+ private func recordSlowXCTestSnapshotBackendIfNeeded(
353
+ _ kind: SnapshotBackendKind,
354
+ startedAt: Date,
355
+ penaltySuppressed: Bool
356
+ ) {
357
+ guard !penaltySuppressed else { return }
317
358
  guard kind.usesXCTestAccessibilityChannel else { return }
318
359
  let elapsed = Date().timeIntervalSince(startedAt)
319
360
  guard elapsed > snapshotXCTestSlowCaptureThreshold else { return }
@@ -325,8 +366,10 @@ extension RunnerTests {
325
366
 
326
367
  private func recordXCTestSnapshotBackendFailureIfNeeded(
327
368
  _ kind: SnapshotBackendKind,
328
- failure: SnapshotCaptureFailure
369
+ failure: SnapshotCaptureFailure,
370
+ penaltySuppressed: Bool
329
371
  ) {
372
+ guard !penaltySuppressed else { return }
330
373
  guard kind.usesXCTestAccessibilityChannel, failure.code == Self.xCTestSnapshotTimeoutCode else { return }
331
374
  penalizeSnapshotXCTestChannel(
332
375
  bundleId: currentBundleId,
@@ -0,0 +1,24 @@
1
+ import XCTest
2
+
3
+ extension RunnerTests {
4
+ func testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds() {
5
+ let wrapper = Self.snapshotTraversalIdentity(
6
+ elementType: .other,
7
+ label: "Article",
8
+ identifier: "",
9
+ frame: CGRect(x: 0, y: 97, width: 393, height: 48)
10
+ )
11
+ let leaf = Self.snapshotTraversalIdentity(
12
+ elementType: .other,
13
+ label: "Article",
14
+ identifier: "",
15
+ frame: CGRect(x: 0, y: 97, width: 131, height: 48)
16
+ )
17
+
18
+ #if os(iOS)
19
+ XCTAssertNotEqual(wrapper, leaf)
20
+ #else
21
+ XCTAssertEqual(wrapper, leaf)
22
+ #endif
23
+ }
24
+ }
@@ -103,7 +103,7 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz
103
103
 
104
104
  func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
105
105
  steps.contains { step in
106
- step.synthesized == true && (step.kind == "tap" || step.kind == "drag")
106
+ step.synthesized == true && step.kind == "tap"
107
107
  }
108
108
  }
109
109
 
@@ -3,17 +3,15 @@ import XCTest
3
3
  extension RunnerTests {
4
4
  // MARK: - Blocking System Modal Snapshot
5
5
 
6
- func blockingSystemAlertSnapshot() -> DataPayload? {
6
+ func blockingSystemAlertSnapshot(deadline: Date = .distantFuture) -> DataPayload? {
7
7
  #if os(macOS)
8
8
  return nil
9
9
  #else
10
- guard let modal = firstBlockingSystemModal(in: springboard) else {
11
- return nil
12
- }
13
- let actions = actionableElements(in: modal)
14
- guard !actions.isEmpty else {
10
+ guard case .resolved(let resolvedModal) = resolveBlockingSystemModal(deadline: deadline) else {
15
11
  return nil
16
12
  }
13
+ let modal = resolvedModal.root
14
+ let actions = resolvedModal.actions
17
15
 
18
16
  let title = preferredSystemModalTitle(modal)
19
17
  guard let modalNode = safeMakeSnapshotNode(
@@ -29,20 +27,6 @@ extension RunnerTests {
29
27
  }
30
28
  var nodes: [SnapshotNode] = [modalNode]
31
29
 
32
- for content in informativeElements(in: modal, excluding: actions) {
33
- guard let contentNode = safeMakeSnapshotNode(
34
- element: content,
35
- index: nodes.count,
36
- type: elementTypeName(content.elementType),
37
- depth: 1,
38
- parentIndex: 0,
39
- hittableOverride: false
40
- ) else {
41
- continue
42
- }
43
- nodes.append(contentNode)
44
- }
45
-
46
30
  for action in actions {
47
31
  guard let actionNode = safeMakeSnapshotNode(
48
32
  element: action,
@@ -57,11 +41,33 @@ extension RunnerTests {
57
41
  nodes.append(actionNode)
58
42
  }
59
43
 
44
+ // Action controls are the interaction contract. Informative text is useful context, but
45
+ // must not make a usable modal miss the bounded system-modal probe deadline.
46
+ let contentDeadline = deadline.addingTimeInterval(-0.5)
47
+ for content in informativeElements(in: modal, deadline: contentDeadline) {
48
+ guard Date() < contentDeadline,
49
+ let contentNode = safeMakeSnapshotNode(
50
+ element: content,
51
+ index: nodes.count,
52
+ type: elementTypeName(content.elementType),
53
+ depth: 1,
54
+ parentIndex: 0,
55
+ hittableOverride: false
56
+ )
57
+ else {
58
+ break
59
+ }
60
+ nodes.append(contentNode)
61
+ }
62
+
60
63
  return DataPayload(nodes: nodes, truncated: false)
61
64
  #endif
62
65
  }
63
66
 
64
- func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
67
+ func firstBlockingSystemModal(
68
+ in springboard: XCUIApplication,
69
+ deadline: Date = .distantFuture
70
+ ) -> XCUIElement? {
65
71
  let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
66
72
  let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
67
73
  if disableSafeProbe {
@@ -79,6 +85,12 @@ extension RunnerTests {
79
85
  }
80
86
  }
81
87
 
88
+ // Don't start the second (sheet) enumeration once the shared probe deadline is spent (#1244).
89
+ if Date() >= deadline {
90
+ NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_DEADLINE stage=sheets")
91
+ return nil
92
+ }
93
+
82
94
  let sheets = queryElements {
83
95
  springboard.sheets.allElementsBoundByIndex
84
96
  }
@@ -137,20 +149,18 @@ extension RunnerTests {
137
149
  }
138
150
  }
139
151
 
140
- private func informativeElements(in element: XCUIElement, excluding actions: [XCUIElement]) -> [XCUIElement] {
141
- let actionKeys = Set(actions.map(systemModalElementKey))
152
+ private func informativeElements(in element: XCUIElement, deadline: Date) -> [XCUIElement] {
142
153
  var seen = Set<String>()
143
154
  var contents: [XCUIElement] = []
144
- let descendants = readableSystemModalTypes.flatMap {
145
- modalDescendants(in: element, matching: $0, limit: 2)
146
- }
147
- for candidate in descendants {
148
- guard let key = safeInformativeElementKey(candidate, actionKeys: actionKeys) else {
149
- continue
155
+ for type in readableSystemModalTypes {
156
+ guard Date() < deadline else { break }
157
+ for candidate in modalDescendants(in: element, matching: type, limit: 2) {
158
+ guard Date() < deadline else { return contents }
159
+ guard let key = safeInformativeElementKey(candidate) else { continue }
160
+ if seen.contains(key) { continue }
161
+ seen.insert(key)
162
+ contents.append(candidate)
150
163
  }
151
- if seen.contains(key) { continue }
152
- seen.insert(key)
153
- contents.append(candidate)
154
164
  }
155
165
  return contents
156
166
  }
@@ -173,11 +183,9 @@ extension RunnerTests {
173
183
  return Array(elements.prefix(limit))
174
184
  }
175
185
 
176
- private func safeInformativeElementKey(_ candidate: XCUIElement, actionKeys: Set<String>) -> String? {
186
+ private func safeInformativeElementKey(_ candidate: XCUIElement) -> String? {
177
187
  safely("MODAL_CONTENT") { () -> String? in
178
188
  let key = systemModalElementKey(candidate)
179
- if actionKeys.contains(key) { return nil }
180
- if actionableTypes.contains(candidate.elementType) { return nil }
181
189
  if !candidate.exists { return nil }
182
190
  let frame = candidate.frame
183
191
  if frame.isNull || frame.isEmpty { return nil }
@@ -29,6 +29,15 @@ final class RunnerTests: XCTestCase {
29
29
  }
30
30
 
31
31
  static let springboardBundleId = "com.apple.springboard"
32
+ // SpringBoard hosts blocking system modals on iOS/visionOS; tvOS (PineBoard/HeadBoard)
33
+ // and macOS have no such host, so there is nothing to probe there.
34
+ static let hasSpringBoardSystemModalHost: Bool = {
35
+ #if os(tvOS) || os(macOS)
36
+ return false
37
+ #else
38
+ return true
39
+ #endif
40
+ }()
32
41
  static let defaultRecordingFps: Int32 = 15
33
42
  var listener: NWListener?
34
43
  var doneExpectation: XCTestExpectation?
@@ -38,6 +47,7 @@ final class RunnerTests: XCTestCase {
38
47
  lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId)
39
48
  var currentApp: XCUIApplication?
40
49
  var currentBundleId: String?
50
+ var currentAppProcessIdentifier: Int?
41
51
  let maxRequestBytes = 2 * 1024 * 1024
42
52
  let mainThreadExecutionTimeout: TimeInterval = 30
43
53
  let appExistenceTimeout: TimeInterval = 30
@@ -76,6 +86,7 @@ final class RunnerTests: XCTestCase {
76
86
  var snapshotXCTestChannelPenaltyBundleId: String?
77
87
  var snapshotXCTestChannelPenaltyUntil = Date.distantPast
78
88
  let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
89
+ var snapshotXCTestPenaltyWarmupExemptionPending = false
79
90
  // Bluesky-class screens can grind ~4-8s before an XCTest-backed snapshot tier fails; anything
80
91
  // past this threshold marks the screen hostile so the next capture uses non-XCTest recovery.
81
92
  let snapshotXCTestSlowCaptureThreshold: TimeInterval = 3
@@ -87,6 +98,9 @@ final class RunnerTests: XCTestCase {
87
98
  let treeCaptureLock = NSLock()
88
99
  var abandonedTreeCaptureCount = 0
89
100
  let treeCaptureSliceBudget: TimeInterval = 8
101
+ // Bounds the pre-plan SpringBoard system-modal probe, which can otherwise grind for tens of
102
+ // seconds on remote-hosted consent dialogs and bypass the plan budget (#1244).
103
+ let systemModalProbeBudget: TimeInterval = 4
90
104
  // Observability for the record(_:) suppression below: how many AX-broken-screen snapshot
91
105
  // issues this session muted, so wedge investigations see the volume without grepping logs.
92
106
  let suppressedIssueLock = NSLock()