agent-device 0.20.10 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (510) hide show
  1. package/README.md +2 -2
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.10.apk → agent-device-android-ime-helper-0.21.1.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.1.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.10.manifest.json → agent-device-android-ime-helper-0.21.1.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.1.apk +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.1.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.10.manifest.json → agent-device-android-snapshot-helper-0.21.1.manifest.json} +6 -6
  8. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +35 -33
  9. package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +6 -1
  10. package/apple/snapshot-bridge/LICENSE.idb +23 -0
  11. package/apple/snapshot-bridge/README.md +106 -0
  12. package/apple/snapshot-bridge/SnapshotBridge.m +286 -0
  13. package/apple/snapshot-bridge/SnapshotBridgeCapture.h +25 -0
  14. package/apple/snapshot-bridge/SnapshotBridgeCapture.m +138 -0
  15. package/apple/snapshot-bridge/SnapshotBridgeRuntime.h +34 -0
  16. package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +411 -0
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +58 -0
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +19 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +12 -0
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +20 -0
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertVerification.swift +36 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +15 -15
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerStringNormalization.swift +8 -0
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +12 -1
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +100 -6
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +72 -0
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +63 -14
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +677 -83
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +35 -92
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +124 -0
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +39 -4
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +1058 -213
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +296 -14
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +1 -1
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +10 -10
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +309 -149
  37. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +101 -42
  38. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +24 -17
  39. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +124 -199
  40. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +71 -4
  41. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXElementTypes.swift +12 -0
  42. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +211 -78
  43. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +30 -10
  44. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +254 -24
  45. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +206 -22
  46. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +465 -494
  47. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +168 -0
  48. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +802 -247
  49. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationInvariant.swift +19 -0
  50. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +185 -0
  51. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +206 -0
  52. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +83 -5
  53. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +156 -34
  54. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +5 -4
  55. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +35 -318
  56. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +206 -0
  57. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +158 -0
  58. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +28 -0
  59. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +101 -0
  60. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +56 -23
  61. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +56 -27
  62. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +30 -24
  63. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +126 -80
  64. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +111 -0
  65. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +47 -86
  66. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift +32 -79
  67. package/dist/apple/snapshot-presentation/Package.swift +21 -0
  68. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift +56 -0
  69. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +259 -0
  70. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentation.swift +153 -0
  71. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationFailure.swift +27 -0
  72. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift +91 -0
  73. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift +161 -0
  74. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotScopePolicy.swift +53 -0
  75. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift +242 -0
  76. package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift +97 -0
  77. package/dist/src/adapter.js +1 -0
  78. package/dist/src/adb-executor.js +2 -0
  79. package/dist/src/adb-failure.js +1 -0
  80. package/dist/src/adb-host.js +1 -0
  81. package/dist/src/adb.js +1 -0
  82. package/dist/src/agent-browser-lifecycle.js +1 -0
  83. package/dist/src/agent-browser-process-record.js +1 -0
  84. package/dist/src/agent-browser-provider.js +1 -0
  85. package/dist/src/agent-browser-tool.js +1 -1
  86. package/dist/src/agent-device-client.js +2 -2
  87. package/dist/src/ai-sdk.d.ts +2 -2
  88. package/dist/src/ai-sdk.js +1 -1
  89. package/dist/src/alert-contract.js +1 -0
  90. package/dist/src/alert-runtime.js +1 -0
  91. package/dist/src/android-adb.d.ts +27 -19
  92. package/dist/src/android-adb.js +1 -1
  93. package/dist/src/android-input-ownership.js +1 -0
  94. package/dist/src/android-system-chrome.js +1 -0
  95. package/dist/src/android.js +1 -1
  96. package/dist/src/api.js +11 -0
  97. package/dist/src/app-catalog.js +1 -0
  98. package/dist/src/app-deployment-resolution.js +2 -0
  99. package/dist/src/app-helpers.js +1 -1
  100. package/dist/src/app-inventory-contract.js +1 -0
  101. package/dist/src/app-inventory.js +1 -0
  102. package/dist/src/app-launch.js +2 -0
  103. package/dist/src/app-lifecycle-facade.js +1 -0
  104. package/dist/src/app-lifecycle.js +3 -6
  105. package/dist/src/app-lifecycle2.js +2 -2
  106. package/dist/src/app-lifecycle3.js +1 -0
  107. package/dist/src/app-log-files.js +1 -1
  108. package/dist/src/app-log-reconnect.js +1 -1
  109. package/dist/src/app-log-runtime.js +1 -4
  110. package/dist/src/app-log-runtime2.js +4 -0
  111. package/dist/src/app-parsers.js +4 -0
  112. package/dist/src/app-resolution-cache.js +1 -0
  113. package/dist/src/app-resolution-facade.js +1 -0
  114. package/dist/src/app-resolution.js +2 -2
  115. package/dist/src/app-state.js +1 -2
  116. package/dist/src/apple-runner-platform.js +1 -0
  117. package/dist/src/application-lifecycle-interaction.js +1 -0
  118. package/dist/src/application-lifecycle-runtime.js +1 -0
  119. package/dist/src/apps-simctl.js +1 -0
  120. package/dist/src/archive.js +4 -0
  121. package/dist/src/artifact-download.js +1 -0
  122. package/dist/src/artifacts.d.ts +6 -2
  123. package/dist/src/artifacts.js +1 -1
  124. package/dist/src/async-lifecycle.js +1 -0
  125. package/dist/src/audio-probe-result.js +1 -0
  126. package/dist/src/audio-probe-runtime.js +1 -0
  127. package/dist/src/audio-probe-support.js +1 -0
  128. package/dist/src/auth-session.js +1 -1
  129. package/dist/src/auth.js +1 -1
  130. package/dist/src/backend-snapshot-options.js +1 -1
  131. package/dist/src/{url.js → base-url.js} +1 -1
  132. package/dist/src/boot-diagnostics.js +2 -2
  133. package/dist/src/byte-limit-stream.js +1 -1
  134. package/dist/src/capabilities.js +1 -0
  135. package/dist/src/capture.js +1 -0
  136. package/dist/src/catalog.js +1 -0
  137. package/dist/src/cli-command-aliases.js +1 -0
  138. package/dist/src/cli-config.js +1 -1
  139. package/dist/src/cli-help.js +74 -51
  140. package/dist/src/cli-output.js +1 -1
  141. package/dist/src/cli-runner.js +1 -1
  142. package/dist/src/cli.js +4 -6
  143. package/dist/src/click-button.js +1 -0
  144. package/dist/src/client-metro-companion.js +1 -0
  145. package/dist/src/client-metro.js +1 -1
  146. package/dist/src/client-types.d.ts +280 -242
  147. package/dist/src/clipboard.js +1 -0
  148. package/dist/src/code-signature-cache.js +1 -0
  149. package/dist/src/code-signature.js +1 -0
  150. package/dist/src/color.js +1 -0
  151. package/dist/src/command-metadata.js +1 -1
  152. package/dist/src/command-schema.js +3 -3
  153. package/dist/src/command-surface.js +1 -1
  154. package/dist/src/command-tools.js +2 -2
  155. package/dist/src/command.js +1 -0
  156. package/dist/src/compatibility-policy.js +1 -0
  157. package/dist/src/config.js +1 -1
  158. package/dist/src/config2.js +1 -0
  159. package/dist/src/connection-runtime.js +1 -1
  160. package/dist/src/connection.js +5 -5
  161. package/dist/src/contracts.d.ts +1 -1
  162. package/dist/src/contracts.js +1 -1
  163. package/dist/src/daemon-client-lifecycle.js +1 -0
  164. package/dist/src/daemon-process.js +1 -0
  165. package/dist/src/daemon-registration.js +1 -0
  166. package/dist/src/daemon-shutdown-report.js +1 -1
  167. package/dist/src/daemon.js +2 -2
  168. package/dist/src/debug-symbols-facade.js +5 -0
  169. package/dist/src/deploy-result-target.js +1 -0
  170. package/dist/src/device-claim-inspection.js +1 -1
  171. package/dist/src/device-claim-rule.js +1 -0
  172. package/dist/src/device-input-state.d.ts +190 -3
  173. package/dist/src/device-inventory-context.js +1 -0
  174. package/dist/src/device-isolation.js +1 -0
  175. package/dist/src/device-release.js +1 -0
  176. package/dist/src/device-rotation.js +1 -0
  177. package/dist/src/device-selection-resolver.js +1 -0
  178. package/dist/src/device-session.js +1 -0
  179. package/dist/src/device.js +1 -1
  180. package/dist/src/device2.js +3 -2
  181. package/dist/src/diagnostics.js +3 -0
  182. package/dist/src/diagnostics2.js +1 -0
  183. package/dist/src/dispatch-resolve.js +1 -0
  184. package/dist/src/doctor-facade.js +1 -0
  185. package/dist/src/doctor-output.js +1 -0
  186. package/dist/src/doctor.js +1 -0
  187. package/dist/src/doctor2.js +1 -0
  188. package/dist/src/doctor3.js +1 -0
  189. package/dist/src/durable-resource.js +1 -0
  190. package/dist/src/entities.js +1 -1
  191. package/dist/src/error.js +2 -0
  192. package/dist/src/exec.js +1 -0
  193. package/dist/src/file.js +1 -0
  194. package/dist/src/find.js +1 -1
  195. package/dist/src/finders.js +1 -1
  196. package/dist/src/generic-settle.js +1 -1
  197. package/dist/src/generic.js +1 -1
  198. package/dist/src/gesture-admission.js +1 -0
  199. package/dist/src/gesture-plan-types.js +1 -0
  200. package/dist/src/gesture-plan.js +1 -0
  201. package/dist/src/gesture-runtime.js +1 -0
  202. package/dist/src/harmonyos.js +1 -1
  203. package/dist/src/hdc.js +1 -1
  204. package/dist/src/helper.js +2 -2
  205. package/dist/src/host-file.js +1 -0
  206. package/dist/src/human-control-contract.js +1 -0
  207. package/dist/src/human-control.js +1 -0
  208. package/dist/src/index.d.ts +1 -1
  209. package/dist/src/index.js +1 -1
  210. package/dist/src/input-actions.js +1 -4
  211. package/dist/src/input-actions2.js +2 -1
  212. package/dist/src/install-artifact-facade.js +1 -0
  213. package/dist/src/install-source-network-transport.js +1 -0
  214. package/dist/src/install-source.d.ts +3 -10
  215. package/dist/src/install-source2.js +1 -1
  216. package/dist/src/interaction-positionals.js +1 -0
  217. package/dist/src/interaction-ref-policy.js +1 -0
  218. package/dist/src/interaction.js +1 -1
  219. package/dist/src/interaction2.js +1 -1
  220. package/dist/src/interactor-operation-catalog.js +1 -0
  221. package/dist/src/interactor-types.js +1 -0
  222. package/dist/src/interactor.js +6 -2
  223. package/dist/src/interactor2.js +1 -1
  224. package/dist/src/interactors.js +1 -1
  225. package/dist/src/internal/bin.d.ts +1 -1
  226. package/dist/src/internal/bin.js +1 -2
  227. package/dist/src/internal/companion-tunnel.d.ts +1 -1
  228. package/dist/src/internal/companion-tunnel.js +1 -1
  229. package/dist/src/internal/daemon.d.ts +1 -1
  230. package/dist/src/internal/daemon.js +2 -4
  231. package/dist/src/internal/png-worker.d.ts +4 -4
  232. package/dist/src/internal/png-worker.js +1 -1
  233. package/dist/src/internal/run-script-http-child.d.ts +4 -0
  234. package/dist/src/internal/run-script-http-child.js +1 -0
  235. package/dist/src/internal/update-check-entry.d.ts +1 -1
  236. package/dist/src/inventory.js +4 -4
  237. package/dist/src/inventory2.js +1 -1
  238. package/dist/src/inventory3.js +1 -1
  239. package/dist/src/inventory5.js +1 -1
  240. package/dist/src/inventory6.js +1 -1
  241. package/dist/src/ios-client.js +16 -0
  242. package/dist/src/ios-snapshot-acquisition.js +1 -0
  243. package/dist/src/ios-snapshot-adapter.js +1 -0
  244. package/dist/src/ios-snapshot-planning.js +1 -0
  245. package/dist/src/ios-snapshot-runtime.js +1 -0
  246. package/dist/src/ios-system-surface.js +1 -0
  247. package/dist/src/json.js +1 -0
  248. package/dist/src/keyboard-actions.js +1 -0
  249. package/dist/src/launch-console.js +1 -0
  250. package/dist/src/lease-context.js +1 -1
  251. package/dist/src/lease-scope.js +1 -0
  252. package/dist/src/lease.js +1 -1
  253. package/dist/src/limrun-runtime-dependencies.js +1 -1
  254. package/dist/src/limrun.d.ts +1610 -7
  255. package/dist/src/limrun.js +1 -1
  256. package/dist/src/linux.js +1 -1
  257. package/dist/src/local-interactor-operation-set.js +1 -0
  258. package/dist/src/location-coordinates.js +1 -1
  259. package/dist/src/logcat.js +1 -1
  260. package/dist/src/macos-facade.js +1 -0
  261. package/dist/src/managed-device-scope.js +1 -0
  262. package/dist/src/mechanics.js +19 -0
  263. package/dist/src/metro.d.ts +4 -2
  264. package/dist/src/metro.js +1 -1
  265. package/dist/src/metro2.js +1 -1
  266. package/dist/src/mobile-snapshot-semantics.js +1 -1
  267. package/dist/src/multitouch-support.js +1 -0
  268. package/dist/src/numeric.js +1 -0
  269. package/dist/src/observability.js +1 -1
  270. package/dist/src/observation.js +2 -0
  271. package/dist/src/open-target-resolution.js +1 -0
  272. package/dist/src/output-path.js +1 -1
  273. package/dist/src/owner-identity.js +3 -1
  274. package/dist/src/parameterized-recorded-fill.js +1 -0
  275. package/dist/src/parser.js +1 -1
  276. package/dist/src/perf-facade.js +2 -0
  277. package/dist/src/perf-process-identity.js +1 -0
  278. package/dist/src/perf-runtime-operation-builder.js +1 -0
  279. package/dist/src/perf-runtime-plan.js +1 -0
  280. package/dist/src/perf-runtime.js +1 -0
  281. package/dist/src/perf.js +1 -4
  282. package/dist/src/physical-device-control.js +1 -1
  283. package/dist/src/physical-device-facade.js +1 -0
  284. package/dist/src/platform-request-scope.js +1 -0
  285. package/dist/src/platform-runtime-android-adb-host.js +1 -0
  286. package/dist/src/platform-runtime-android-mechanics.js +1 -0
  287. package/dist/src/platform-runtime-app-log-android-transport.js +1 -1
  288. package/dist/src/platform-runtime-app-log-process.js +1 -1
  289. package/dist/src/platform-runtime-host.js +1 -1
  290. package/dist/src/platform-runtime-managed-owner.js +1 -0
  291. package/dist/src/platform-runtime-network-web-transport.js +1 -1
  292. package/dist/src/platform-runtime-open-target.js +1 -1
  293. package/dist/src/platform-runtime-operation-host.js +4 -4
  294. package/dist/src/platform-runtime-runtime-hints.js +2 -2
  295. package/dist/src/platform-runtime-screen-recording-android-host.js +1 -1
  296. package/dist/src/platform-runtime-screen-recording-apple-runner-host.js +1 -1
  297. package/dist/src/platform-runtime-screen-recording-apple-runner-transport.js +1 -1
  298. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
  299. package/dist/src/platform-runtime-screen-recording-apple-transport.js +1 -1
  300. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
  301. package/dist/src/platform-runtime-screen-recording-harmony-host.js +1 -1
  302. package/dist/src/platform-runtime-screen-recording-process-host.js +1 -0
  303. package/dist/src/platform-runtime-screen-recording-web-host.js +1 -1
  304. package/dist/src/platform-runtime-unavailable.js +1 -0
  305. package/dist/src/platform-runtime.js +1 -1
  306. package/dist/src/platform-runtime2.js +1 -1
  307. package/dist/src/plist.js +1 -1
  308. package/dist/src/png-worker-contract.js +1 -1
  309. package/dist/src/prepare-kind.js +1 -0
  310. package/dist/src/process-exit.js +1 -1
  311. package/dist/src/process.js +1 -0
  312. package/dist/src/provider-device-runtime.js +1 -0
  313. package/dist/src/provider-session-ownership.js +2 -0
  314. package/dist/src/provider-webdriver.js +1 -1
  315. package/dist/src/provider.js +1 -1
  316. package/dist/src/providers.js +1 -0
  317. package/dist/src/proxy.js +2 -2
  318. package/dist/src/react-native-overlay.js +1 -0
  319. package/dist/src/react-native.js +1 -1
  320. package/dist/src/record-runtime.js +1 -1
  321. package/dist/src/record-trace.js +1 -1
  322. package/dist/src/rect-center.js +1 -0
  323. package/dist/src/register-builtins.js +1 -0
  324. package/dist/src/registry.js +40 -28
  325. package/dist/src/remote-config2.js +1 -1
  326. package/dist/src/replay-divergence.js +2 -0
  327. package/dist/src/replay.js +2 -2
  328. package/dist/src/reporting.js +8 -7
  329. package/dist/src/request-admission.js +1 -0
  330. package/dist/src/request-cancel.js +1 -0
  331. package/dist/src/request-cancellation.js +1 -0
  332. package/dist/src/request-progress-protocol.js +1 -0
  333. package/dist/src/request-providers.js +1 -0
  334. package/dist/src/request-runtime-binding.js +1 -0
  335. package/dist/src/request.js +1 -0
  336. package/dist/src/result-serialization.js +1 -0
  337. package/dist/src/retry.js +1 -0
  338. package/dist/src/rolldown-runtime.js +1 -1
  339. package/dist/src/runner-cache-metadata.js +2 -0
  340. package/dist/src/runner-client.js +1 -1
  341. package/dist/src/runner-disposal.js +1 -1
  342. package/dist/src/runner-operations-facade.js +1 -0
  343. package/dist/src/runner-owner-facade.js +1 -0
  344. package/dist/src/runner-owner-state.js +1 -0
  345. package/dist/src/runner-presentation.js +1 -0
  346. package/dist/src/runner.js +1 -0
  347. package/dist/src/{runtime-transport.js → runtime-transport-hints.js} +1 -1
  348. package/dist/src/runtime.js +3 -3
  349. package/dist/src/runtime2.js +4 -4
  350. package/dist/src/runtime3.js +1 -1
  351. package/dist/src/runtime4.js +5 -3
  352. package/dist/src/runtime5.js +1 -1
  353. package/dist/src/runtime6.js +1 -1
  354. package/dist/src/runtime7.js +1 -1
  355. package/dist/src/runtime8.js +1 -1
  356. package/dist/src/runtime9.js +1 -1
  357. package/dist/src/scoped-provider.js +1 -1
  358. package/dist/src/screen-recording-completion.js +1 -0
  359. package/dist/src/screen-recording-live-handle.js +1 -0
  360. package/dist/src/screen-recording-options.js +1 -0
  361. package/dist/src/screen-recording-resource-recovery.js +1 -0
  362. package/dist/src/screen-recording-runtime.js +1 -0
  363. package/dist/src/screenshot-density.js +1 -1
  364. package/dist/src/screenshot-result.js +1 -6
  365. package/dist/src/screenshot-runtime.js +1 -0
  366. package/dist/src/screenshot.js +1 -1
  367. package/dist/src/screenshot2.js +1 -0
  368. package/dist/src/screenshot3.js +5 -0
  369. package/dist/src/script.js +1 -0
  370. package/dist/src/scroll-command.js +1 -0
  371. package/dist/src/scroll-gesture.js +1 -0
  372. package/dist/src/scroll-indicator.js +1 -0
  373. package/dist/src/sdk-batch-runner.d.ts +23 -6
  374. package/dist/src/sdk-batch-runner.js +2 -3
  375. package/dist/src/sdk-batch.js +2 -0
  376. package/dist/src/sdk-contracts.d.ts +170 -41
  377. package/dist/src/sdk-contracts.js +2 -2
  378. package/dist/src/sdk-device.js +1 -1
  379. package/dist/src/sdk-finders.d.ts +1 -1
  380. package/dist/src/sdk-finders.js +1 -1
  381. package/dist/src/sdk-io.js +1 -1
  382. package/dist/src/sdk-remote-config.d.ts +1 -1
  383. package/dist/src/sdk-remote-config.js +1 -1
  384. package/dist/src/sdk-selectors.d.ts +524 -140
  385. package/dist/src/sdk-selectors.js +1 -1
  386. package/dist/src/selection.js +1 -0
  387. package/dist/src/selector-runtime.js +1 -1
  388. package/dist/src/selectors.js +1 -1
  389. package/dist/src/semantic-index.js +1 -0
  390. package/dist/src/server.js +4 -4
  391. package/dist/src/session-allocation.js +1 -0
  392. package/dist/src/session-artifact-paths.js +1 -0
  393. package/dist/src/session-observation.js +1 -0
  394. package/dist/src/session-paths.js +1 -0
  395. package/dist/src/session-repair-tombstone.js +1 -0
  396. package/dist/src/session-routing.js +1 -0
  397. package/dist/src/session-runtime-admission.js +1 -0
  398. package/dist/src/session-snapshot.js +1 -0
  399. package/dist/src/session-store.js +4 -0
  400. package/dist/src/session-target-evidence.js +1 -1
  401. package/dist/src/session2.js +5 -27
  402. package/dist/src/settings.js +1 -0
  403. package/dist/src/shared.js +2 -2
  404. package/dist/src/simctl-facade.js +1 -0
  405. package/dist/src/simctl.js +1 -1
  406. package/dist/src/simulator-facade.js +1 -0
  407. package/dist/src/simulator.js +1 -1
  408. package/dist/src/snapshot-capture-annotations.js +1 -0
  409. package/dist/src/snapshot-desktop-projection.js +1 -0
  410. package/dist/src/snapshot-diagnostics.js +1 -0
  411. package/dist/src/snapshot-process.js +1 -0
  412. package/dist/src/snapshot-quality-backend-capabilities.js +1 -0
  413. package/dist/src/snapshot-quality-verdict.js +1 -0
  414. package/dist/src/snapshot-runtime-binding.js +1 -0
  415. package/dist/src/snapshot-runtime.js +1 -1
  416. package/dist/src/snapshot-scope.js +1 -1
  417. package/dist/src/snapshot-state.js +1 -0
  418. package/dist/src/snapshot.js +1 -8
  419. package/dist/src/snapshot2.js +1 -1
  420. package/dist/src/snapshot3.js +1 -1
  421. package/dist/src/snapshot4.js +1 -0
  422. package/dist/src/src.js +1 -1
  423. package/dist/src/src2.js +1 -1
  424. package/dist/src/src3.js +1 -2
  425. package/dist/src/src4.js +1 -1
  426. package/dist/src/src5.js +1 -0
  427. package/dist/src/src6.js +1 -0
  428. package/dist/src/src7.js +2 -0
  429. package/dist/src/src8.js +115 -0
  430. package/dist/src/startup-recovery-fence.js +1 -0
  431. package/dist/src/status-markers.js +1 -0
  432. package/dist/src/string-enum.js +1 -1
  433. package/dist/src/success-text.js +1 -0
  434. package/dist/src/surface-snapshot.js +1 -0
  435. package/dist/src/surface-snapshot2.js +1 -0
  436. package/dist/src/takeover.js +5 -0
  437. package/dist/src/tool-provider-facade.js +1 -0
  438. package/dist/src/tool-provider.js +4 -3
  439. package/dist/src/tool-provider2.js +1 -1
  440. package/dist/src/tool-provider3.js +1 -1
  441. package/dist/src/toolchain-probe.js +2 -0
  442. package/dist/src/touch-reference-frame.js +1 -0
  443. package/dist/src/touch-runtime.js +1 -0
  444. package/dist/src/transport.js +3 -0
  445. package/dist/src/tree.js +1 -0
  446. package/dist/src/ttl-memo.js +1 -0
  447. package/dist/src/tv-remote.js +1 -0
  448. package/dist/src/type-text-runtime.js +1 -0
  449. package/dist/src/undici.js +99 -0
  450. package/dist/src/update-check.js +1 -1
  451. package/dist/src/verified-file.js +1 -1
  452. package/dist/src/version.js +1 -1
  453. package/dist/src/video.js +2 -2
  454. package/dist/src/web.js +2 -1
  455. package/dist/src/web2.js +1 -0
  456. package/dist/src/webdriver-ios-snapshot.js +1 -0
  457. package/dist/src/webdriver-source.js +1 -0
  458. package/dist/src/window-state.js +1 -0
  459. package/linux/atspi-dump.py +54 -9
  460. package/package.json +58 -35
  461. package/server.json +9 -3
  462. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.10.apk.sha256 +0 -1
  463. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.10.apk +0 -0
  464. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.10.apk.sha256 +0 -1
  465. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +0 -4
  466. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +0 -168
  467. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -6
  468. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+RecordingTests.swift +0 -4
  469. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SelectorMatchPolicyTests.swift +0 -4
  470. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift +0 -183
  471. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift +0 -0
  472. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -4
  473. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +0 -4
  474. package/dist/src/appearance.js +0 -1
  475. package/dist/src/apps.js +0 -5
  476. package/dist/src/audio-probe.js +0 -1
  477. package/dist/src/cli-grammar.js +0 -1
  478. package/dist/src/command-catalog.js +0 -1
  479. package/dist/src/debug-symbols.js +0 -5
  480. package/dist/src/device-input-state.js +0 -2
  481. package/dist/src/device-ready.js +0 -1
  482. package/dist/src/dispatch.js +0 -1
  483. package/dist/src/handler-utils.js +0 -1
  484. package/dist/src/helper-package-install.js +0 -1
  485. package/dist/src/host-process.js +0 -3
  486. package/dist/src/ime-lifecycle.js +0 -2
  487. package/dist/src/ime-recovery-marker.js +0 -1
  488. package/dist/src/input-actions3.js +0 -1
  489. package/dist/src/install-artifact.js +0 -1
  490. package/dist/src/install-artifact2.js +0 -1
  491. package/dist/src/interaction-common.js +0 -1
  492. package/dist/src/interaction-runtime.js +0 -1
  493. package/dist/src/interaction-snapshot.js +0 -1
  494. package/dist/src/manifest.js +0 -1
  495. package/dist/src/method-scope.js +0 -1
  496. package/dist/src/png-resize.js +0 -1
  497. package/dist/src/process-lock.js +0 -1
  498. package/dist/src/runner-lease.js +0 -1
  499. package/dist/src/runner-sequence.js +0 -1
  500. package/dist/src/script-source-bundle.js +0 -16
  501. package/dist/src/sdk-android-adb.d.ts +0 -245
  502. package/dist/src/sdk-android-adb.js +0 -4
  503. package/dist/src/sdk.js +0 -1
  504. package/dist/src/setting-state.js +0 -1
  505. package/dist/src/shell-quote.js +0 -1
  506. package/dist/src/snapshot-session.js +0 -1
  507. package/dist/src/timing-safe-equal.js +0 -1
  508. package/dist/src/unsupported-interactor.js +0 -1
  509. package/dist/src/verdict.js +0 -1
  510. /package/dist/src/{recording-telemetry.js → telemetry.js} +0 -0
package/README.md CHANGED
@@ -141,11 +141,11 @@ The same session and evidence model works at every step: the agent explores the
141
141
 
142
142
  ## How it works
143
143
 
144
- `agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux.
144
+ `agent-device` keeps device state in sessions. It uses a local accessibility bridge for iOS Simulator snapshots and XCTest for iOS interactions, physical iOS, and tvOS; ADB and the snapshot helper on Android; HDC and ArkUI `uitest` on HarmonyOS; Vega CLI/VDA on the Vega Virtual Device; a local helper on macOS; and AT-SPI on Linux.
145
145
 
146
146
  Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
147
147
 
148
- Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds).
148
+ Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Inspect ownership without a daemon via `agent-device device status`, and settle provably dead owners with `agent-device device release --stale`. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds).
149
149
 
150
150
  `agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system.
151
151
 
@@ -0,0 +1 @@
1
+ 65454a4663596adfc7fccd9a30e42f02b5ecde17eb1514fa115ef3036171b995 agent-device-android-ime-helper-0.21.1.apk
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "android-ime-helper",
3
- "version": "0.20.10",
4
- "assetName": "agent-device-android-ime-helper-0.20.10.apk",
5
- "sha256": "37cfdc9733315068659a6a4c1da8e65cd403e382401194f989f510fb0f320ba5",
3
+ "version": "0.21.1",
4
+ "assetName": "agent-device-android-ime-helper-0.21.1.apk",
5
+ "sha256": "65454a4663596adfc7fccd9a30e42f02b5ecde17eb1514fa115ef3036171b995",
6
6
  "packageName": "com.callstack.agentdevice.imehelper",
7
- "versionCode": 20010,
7
+ "versionCode": 21001,
8
8
  "serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
9
9
  "broadcastProtocol": "android-ime-helper-v1"
10
10
  }
@@ -0,0 +1 @@
1
+ 6ee75909ac499eca439d3663717d8fc1928ada3fec71bf2ae85fb567967a3793 agent-device-android-snapshot-helper-0.21.1.apk
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "android-snapshot-helper",
3
- "version": "0.20.10",
4
- "releaseTag": "v0.20.10",
5
- "assetName": "agent-device-android-snapshot-helper-0.20.10.apk",
3
+ "version": "0.21.1",
4
+ "releaseTag": "v0.21.1",
5
+ "assetName": "agent-device-android-snapshot-helper-0.21.1.apk",
6
6
  "apkUrl": null,
7
- "sha256": "a5e8c39ef8dc5ca6fc1959dfcd52b00bd7845ce584abc0930f8d4a517bd784cc",
8
- "checksumName": "agent-device-android-snapshot-helper-0.20.10.apk.sha256",
7
+ "sha256": "6ee75909ac499eca439d3663717d8fc1928ada3fec71bf2ae85fb567967a3793",
8
+ "checksumName": "agent-device-android-snapshot-helper-0.21.1.apk.sha256",
9
9
  "packageName": "com.callstack.agentdevice.snapshothelper",
10
- "versionCode": 20010,
10
+ "versionCode": 21001,
11
11
  "instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
12
12
  "minSdk": 23,
13
13
  "targetSdk": 36,
@@ -211,43 +211,28 @@ extension AudioProbeBucket {
211
211
  }
212
212
  }
213
213
 
214
+ // This one-shot helper keeps ScreenCaptureKit callbacks and bucket cadence synchronous.
214
215
  func runAudioProbe(durationMs: Int, bucketMs: Int, outPath: String) throws -> AudioProbeResponse {
215
216
  guard #available(macOS 13.0, *) else {
216
217
  throw HelperError.commandFailed("audio probe requires macOS 13 or newer")
217
218
  }
218
- let semaphore = DispatchSemaphore(value: 0)
219
- var response: AudioProbeResponse?
220
- var runError: Error?
221
- Task {
222
- do {
223
- response = try await runAudioProbeAsync(durationMs: durationMs, bucketMs: bucketMs, outPath: outPath)
224
- } catch {
225
- runError = error
226
- }
227
- semaphore.signal()
228
- }
229
- semaphore.wait()
230
- if let runError {
231
- throw runError
232
- }
233
- guard let response else {
234
- throw HelperError.commandFailed("audio probe failed")
235
- }
236
- return response
237
- }
238
219
 
239
- @available(macOS 13.0, *)
240
- private func runAudioProbeAsync(durationMs: Int, bucketMs: Int, outPath: String) async throws -> AudioProbeResponse {
241
- let content: SCShareableContent
242
- do {
243
- content = try await SCShareableContent.current
244
- } catch {
220
+ let contentSemaphore = DispatchSemaphore(value: 0)
221
+ var contentResult: SCShareableContent?
222
+ var contentError: Error?
223
+ SCShareableContent.getWithCompletionHandler { content, error in
224
+ contentResult = content
225
+ contentError = error
226
+ contentSemaphore.signal()
227
+ }
228
+ contentSemaphore.wait()
229
+ if let contentError {
245
230
  throw HelperError.commandFailed(
246
231
  "audio probe requires Screen Recording permission on macOS",
247
- details: ["permission": "screen-recording", "error": error.localizedDescription]
232
+ details: ["permission": "screen-recording", "error": contentError.localizedDescription]
248
233
  )
249
234
  }
250
- guard let display = content.displays.first else {
235
+ guard let display = contentResult?.displays.first else {
251
236
  throw HelperError.commandFailed("audio probe could not resolve a macOS display")
252
237
  }
253
238
 
@@ -267,18 +252,35 @@ private func runAudioProbeAsync(durationMs: Int, bucketMs: Int, outPath: String)
267
252
  let stream = SCStream(filter: filter, configuration: configuration, delegate: nil)
268
253
  let queue = DispatchQueue(label: "com.callstack.agent-device.audio-probe")
269
254
  try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: queue)
270
- try await stream.startCapture()
255
+
256
+ let startSemaphore = DispatchSemaphore(value: 0)
257
+ var startError: Error?
258
+ stream.startCapture { error in
259
+ startError = error
260
+ startSemaphore.signal()
261
+ }
262
+ startSemaphore.wait()
263
+ if let startError {
264
+ throw HelperError.commandFailed(
265
+ "audio probe could not start the ScreenCaptureKit stream",
266
+ details: ["error": startError.localizedDescription]
267
+ )
268
+ }
271
269
  try writer.flushRunning()
272
270
 
273
271
  let deadline = Date().addingTimeInterval(Double(durationMs) / 1000)
274
272
  while Date() < deadline {
275
- let remainingMs = Int(deadline.timeIntervalSinceNow * 1000)
276
- let sleepMs = max(1, min(bucketMs, remainingMs))
277
- try await Task.sleep(nanoseconds: UInt64(sleepMs) * 1_000_000)
273
+ let remainingMs = deadline.timeIntervalSinceNow * 1000
274
+ let sleepMs = max(1, min(Double(bucketMs), remainingMs))
275
+ Thread.sleep(forTimeInterval: sleepMs / 1000)
278
276
  try writer.flushRunning()
279
277
  }
280
278
 
281
- try await stream.stopCapture()
279
+ let stopSemaphore = DispatchSemaphore(value: 0)
280
+ stream.stopCapture { _ in
281
+ stopSemaphore.signal()
282
+ }
283
+ stopSemaphore.wait()
282
284
  try writer.finish(reason: "completed")
283
285
  let data = try Data(contentsOf: URL(fileURLWithPath: outPath))
284
286
  return try JSONDecoder().decode(AudioProbeResponse.self, from: data)
@@ -287,9 +287,14 @@ struct AgentDeviceMacOSHelper {
287
287
  let surface = optionValue(arguments: Array(arguments.dropFirst()), name: "--surface")
288
288
  let app = try resolveTargetApplication(bundleId: bundleId, surface: surface)
289
289
  guard let alertElement = findAlertElement(appElement: AXUIElementCreateApplication(app.processIdentifier)) else {
290
+ // `reason` is the typed channel the host retries on; the message is for humans only.
290
291
  throw HelperError.commandFailed(
291
292
  "alert not found",
292
- details: ["bundleId": app.bundleIdentifier ?? "", "appName": app.localizedName ?? ""]
293
+ details: [
294
+ "reason": "alert-not-found",
295
+ "bundleId": app.bundleIdentifier ?? "",
296
+ "appName": app.localizedName ?? "",
297
+ ]
293
298
  )
294
299
  }
295
300
  let buttons = collectButtons(root: alertElement)
@@ -0,0 +1,23 @@
1
+ The Simulator AX bridge contains code adapted from Meta Platforms, Inc. idb
2
+ v1.5.2, specifically SimulatorFrameworkBridge/AccessibilityService.m and
3
+ SimulatorFrameworkBridge/AccessibilityRuntime.m.
4
+
5
+ Copyright (c) Meta Platforms, Inc. and affiliates.
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining a copy
8
+ of this software and associated documentation files (the "Software"), to deal
9
+ in the Software without restriction, including without limitation the rights
10
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11
+ copies of the Software, and to permit persons to whom the Software is
12
+ furnished to do so, subject to the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be included in all
15
+ copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23
+ SOFTWARE.
@@ -0,0 +1,106 @@
1
+ # Simulator AX bridge
2
+
3
+ This directory contains the small private accessibility reader used by the
4
+ Apple platform acquisition facet. The framed server and request validation
5
+ live in `SnapshotBridge.m`; private runtime binding lives in
6
+ `SnapshotBridgeRuntime.m`. It is compiled for the iOS Simulator on first use
7
+ and is never downloaded, pre-signed, or built by npm installation.
8
+
9
+ The guest process uses the `XCTAccessibilityFramework` remote-access client
10
+ from the simulator runtime and the `userTestingSnapshotForElement:options:error:`
11
+ snapshot API. Requests and responses are length-prefixed JSON frames:
12
+
13
+ ```text
14
+ uint32 big-endian byte length
15
+ UTF-8 JSON object
16
+ ```
17
+
18
+ The host owns all target identity, bounds, deadlines, and lifecycle decisions.
19
+ The guest returns only a bounded raw tree, the target pid, truncation, and
20
+ protocol/source versions. It does not expose an HTTP route or a public CLI
21
+ surface.
22
+
23
+ The private API is intentionally pinned to the idb v1.5.2-compatible shape.
24
+ See `LICENSE.idb` for attribution.
25
+
26
+ ## Why Objective-C
27
+
28
+ The selected #2192 mechanism was idb v1.5.2's Objective-C
29
+ `SimulatorFrameworkBridge`; the Python used during the spike was only a client
30
+ for exercising that guest reader. This bridge keeps the proven native boundary
31
+ and removes the Python/idb client dependency.
32
+
33
+ Objective-C is the narrowest implementation for this private runtime adapter:
34
+ it resolves unavailable classes and functions with `dlopen`, `dlsym`, and the
35
+ Objective-C runtime, invokes dynamically discovered selectors, and contains
36
+ `NSException` failures. A Swift implementation would still require an
37
+ Objective-C shim for those operations, adding another native boundary. Keeping
38
+ the guest in Objective-C also allows direct lazy compilation with `clang`
39
+ without an Xcode project or Swift module for private headers.
40
+
41
+ ## Foreground ownership
42
+
43
+ The reader checks AXRuntime's primary foreground application before and after
44
+ acquisition. If the target is covered by system UI, ownership is unavailable,
45
+ or the owner changes during capture, it returns a typed failure without the
46
+ app tree. The existing route then uses XCTest, which owns system-modal
47
+ resolution. Secondary owners such as the return-to-app status-bar control do
48
+ not replace the native primary owner. The route's generation circuit remains
49
+ disabled after fallback until that app relaunches.
50
+
51
+ ## Remote content
52
+
53
+ The reader snapshots one process. A WebKit page — Safari's or a `WKWebView`'s —
54
+ lives in a WebContent process and appears in that tree as an `AXRemoteElement`
55
+ leaf under the web view. The guest returns the leaf as delivered; the host
56
+ refuses a tree in which such a leaf sits under a web view and reaches the
57
+ viewport, or reports no frame (`remote-content-boundary`), and routes that app
58
+ generation to XCTest, which resolves remote elements (#2484). A zero-area or
59
+ off-screen leaf is published: it hosts nothing the capture can miss.
60
+
61
+ ## Bounded depth recovery
62
+
63
+ A healthy capture uses one native request. If native acquisition rejects it,
64
+ `SnapshotBridgeCapture.m` retries supported native failure codes at lower depths
65
+ and fetches withheld children from their accessibility elements. The completed
66
+ tree keeps the original depth and node limits; partial trees disclose truncation.
67
+ The traversal depth counts edges below the root; native requests count the root
68
+ as one level. Each acquisition allows two lower-depth retries, and recovery
69
+ allows at most 32 native requests within the existing capture deadline,
70
+ checks foreground ownership on every request, and returns a failure when it
71
+ cannot complete a continuation. Budget exhaustion and malformed continuations
72
+ use non-launch failure codes, so the route falls back without launch re-polling.
73
+ At each native fragment boundary, an absent or invalid child count means unknown
74
+ completeness and fails closed. Natural leaves above that boundary need no
75
+ continuation evidence. Unchanged native dictionaries and child arrays are reused.
76
+
77
+ ## Accepted-depth hints
78
+
79
+ The host source (`packages/platform-apple/src/snapshot-source/depth-hints.ts`)
80
+ remembers the native levels a finished recovery accepted, keyed by the resolved
81
+ target id, its app generation, and this producer. The next capture of that
82
+ generation sends `nativeLevelsHint`, so the guest's first request asks for the
83
+ accepted levels instead of re-paying the known rejection. A hint changes the
84
+ request strategy only: the delivered depth, node bounds, and completeness rules
85
+ are unchanged. Hints are learned only from a recovery that observed a rejection
86
+ and then finished (a tree bounded by the requested depth or node budget still
87
+ teaches), never cross apps, generations, or producers, and expire after eight
88
+ hinted captures so the next capture probes the full depth again; a capture that
89
+ merely succeeds at the hinted depth does not renew it. Explicit raw-depth
90
+ requests neither use nor teach hints. Every response carries `recovery`
91
+ (`requests`, `rejected`, `continuations`, `acceptedLevels`), which the host
92
+ emits as the `ios_snapshot_source_recovery` diagnostic.
93
+
94
+ ## Recovery conformance
95
+
96
+ `contracts/fixtures/ios-ax-recovery-conformance.json` is the shared, executable
97
+ recovery contract for this bridge and the XCTest runner's private AX bridge.
98
+ `packages/platform-apple/src/snapshot-source/fixtures/recovery-conformance.m`
99
+ replays each case through `captureSnapshotTree`; the runner replays the same
100
+ cases through its own bridge in
101
+ `RunnerTests+AXRecoveryConformanceTests.swift`. Each expectation names the
102
+ outcome, the native request accounting, and the delivered tree as a canonical
103
+ preorder signature with its retained node count, so a producer that drops,
104
+ duplicates, reorders, or re-parents nodes cannot pass as complete. The fixture
105
+ records each producer's expectation and documents the intentional differences
106
+ (depth vocabulary, ladders, completeness evidence, budgets, hint lifetime).
@@ -0,0 +1,286 @@
1
+ /*
2
+ * The framed server and request validation for the private Simulator AX reader.
3
+ * The runtime binding is isolated in SnapshotBridgeRuntime.m.
4
+ */
5
+
6
+ #import "SnapshotBridgeRuntime.h"
7
+
8
+ #import <Foundation/Foundation.h>
9
+
10
+ #import <arpa/inet.h>
11
+ #import <errno.h>
12
+ #import <limits.h>
13
+ #import <math.h>
14
+ #import <poll.h>
15
+ #import <sys/socket.h>
16
+ #import <sys/stat.h>
17
+ #import <sys/types.h>
18
+ #import <sys/un.h>
19
+ #import <unistd.h>
20
+
21
+ static const int kDefaultIdleTimeoutSeconds = 60;
22
+
23
+ static void bridgeLog(NSString *message)
24
+ {
25
+ fprintf(stderr, "[agent-device-snapshot-bridge] %s\n", message.UTF8String ?: "(no message)");
26
+ fflush(stderr);
27
+ }
28
+
29
+ NSDictionary *failureResponse(NSString *requestId,
30
+ NSString *kind,
31
+ NSString *code,
32
+ NSString *message)
33
+ {
34
+ return @{
35
+ kProtocolVersionKey : @(kProtocolVersion),
36
+ kSourceVersionKey : kSourceVersion,
37
+ kRequestIdKey : requestId ?: @"",
38
+ @"ok" : @NO,
39
+ @"error_kind" : kind ?: @"reader_unavailable",
40
+ @"error_code" : code ?: @"unknown",
41
+ @"error" : message ?: @"snapshot bridge request failed",
42
+ };
43
+ }
44
+
45
+ static BOOL validBoundInteger(id value, NSUInteger minimum, NSUInteger maximum, NSUInteger *output)
46
+ {
47
+ if (![value isKindOfClass:NSNumber.class]) return NO;
48
+ NSNumber *number = value;
49
+ if (number.doubleValue != floor(number.doubleValue)) return NO;
50
+ if (number.unsignedIntegerValue < minimum || number.unsignedIntegerValue > maximum) return NO;
51
+ if (output) *output = number.unsignedIntegerValue;
52
+ return YES;
53
+ }
54
+
55
+ static NSDictionary *handleRequest(NSDictionary *request)
56
+ {
57
+ NSString *requestId = [request[kRequestIdKey] isKindOfClass:NSString.class] ? request[kRequestIdKey] : @"";
58
+ id verb = request[@"verb"];
59
+ if (![verb isKindOfClass:NSString.class] || ![verb isEqualToString:@"describe"]) {
60
+ return failureResponse(requestId, @"bad_request", @"verb-not-supported", @"snapshot bridge accepts describe requests only");
61
+ }
62
+ NSNumber *pidValue = request[@"pid"];
63
+ if (!validBoundInteger(pidValue, 1, INT_MAX, NULL)) {
64
+ return failureResponse(requestId, @"bad_request", @"pid-required", @"describe requires a positive target pid");
65
+ }
66
+ NSString *generation = [request[@"generation"] isKindOfClass:NSString.class] ? request[@"generation"] : @"";
67
+ if (generation.length == 0) {
68
+ return failureResponse(requestId, @"bad_request", @"generation-required", @"describe requires an opaque target generation");
69
+ }
70
+ id snapshotTree = request[@"snapshotTree"];
71
+ if (![snapshotTree isKindOfClass:NSNumber.class] || ![snapshotTree boolValue]) {
72
+ return failureResponse(requestId, @"bad_request", @"snapshot-tree-required", @"snapshotTree must be enabled");
73
+ }
74
+ id automationMode = request[@"automationMode"];
75
+ if (![automationMode isKindOfClass:NSNumber.class] || ![automationMode boolValue]) {
76
+ return failureResponse(requestId, @"bad_request", @"automation-mode-required", @"automationMode must be enabled");
77
+ }
78
+ NSUInteger maxDepth = 0;
79
+ NSUInteger maxNodes = 0;
80
+ NSUInteger maxDurationMs = 0;
81
+ NSUInteger maxResponseBytes = 0;
82
+ if (!validBoundInteger(request[@"maxDepth"], 0, kMaximumDepth, &maxDepth) ||
83
+ !validBoundInteger(request[@"maxNodes"], 1, kMaximumNodes, &maxNodes) ||
84
+ !validBoundInteger(request[@"maxDurationMs"], 1, kMaximumDurationMs, &maxDurationMs) ||
85
+ !validBoundInteger(request[@"maxResponseBytes"], 1024, kMaximumFrameBytes, &maxResponseBytes)) {
86
+ return failureResponse(requestId, @"bad_request", @"bounds-invalid", @"snapshot bridge request bounds are outside the bridge limits");
87
+ }
88
+ NSUInteger nativeLevelsHint = 0;
89
+ if (request[@"nativeLevelsHint"] != nil &&
90
+ !validBoundInteger(request[@"nativeLevelsHint"], 1, kMaximumDepth + 1, &nativeLevelsHint)) {
91
+ return failureResponse(requestId, @"bad_request", @"bounds-invalid", @"nativeLevelsHint is outside the bridge depth limits");
92
+ }
93
+
94
+ NSString *setupError = nil;
95
+ BridgeRuntime *runtime = sharedRuntime(&setupError);
96
+ if (!runtime) {
97
+ NSMutableDictionary *unavailable = [failureResponse(requestId, @"unsupported", @"runtime-unavailable", setupError) mutableCopy];
98
+ unavailable[@"pid"] = pidValue;
99
+ unavailable[@"generation"] = generation;
100
+ return unavailable;
101
+ }
102
+ NSDictionary *error = nil;
103
+ NSDictionary *response = [runtime snapshotForProcess:pidValue.intValue
104
+ maxDepth:maxDepth
105
+ maxNodes:maxNodes
106
+ nativeLevelsHint:nativeLevelsHint
107
+ requestId:requestId
108
+ generation:generation
109
+ maxDurationMs:maxDurationMs
110
+ error:&error];
111
+ if (response) return response;
112
+ if (error) {
113
+ NSMutableDictionary *annotated = [error mutableCopy];
114
+ annotated[@"pid"] = pidValue;
115
+ annotated[@"generation"] = generation;
116
+ return annotated;
117
+ }
118
+ return failureResponse(requestId, @"reader_unavailable", @"empty-response", @"AX bridge returned no response");
119
+ }
120
+
121
+ static BOOL readFully(int fd, void *buffer, size_t length)
122
+ {
123
+ size_t offset = 0;
124
+ while (offset < length) {
125
+ ssize_t count = recv(fd, (char *)buffer + offset, length - offset, 0);
126
+ if (count > 0) {
127
+ offset += (size_t)count;
128
+ continue;
129
+ }
130
+ if (count < 0 && errno == EINTR) continue;
131
+ return NO;
132
+ }
133
+ return YES;
134
+ }
135
+
136
+ static BOOL writeFully(int fd, const void *buffer, size_t length)
137
+ {
138
+ size_t offset = 0;
139
+ while (offset < length) {
140
+ ssize_t count = send(fd, (const char *)buffer + offset, length - offset, MSG_NOSIGNAL);
141
+ if (count > 0) {
142
+ offset += (size_t)count;
143
+ continue;
144
+ }
145
+ if (count < 0 && errno == EINTR) continue;
146
+ return NO;
147
+ }
148
+ return YES;
149
+ }
150
+
151
+ static NSData *serializedResponse(NSDictionary *response, NSUInteger maxResponseBytes)
152
+ {
153
+ NSError *error = nil;
154
+ NSData *data = nil;
155
+ @try {
156
+ data = [NSJSONSerialization dataWithJSONObject:response options:0 error:&error];
157
+ if (data && data.length + sizeof(uint32_t) <= maxResponseBytes) return data;
158
+ } @catch (NSException *exception) {
159
+ bridgeLog(exception.reason ?: @"response serialization raised an exception");
160
+ }
161
+ NSMutableDictionary *fallback = [failureResponse(
162
+ response[kRequestIdKey],
163
+ data ? @"response_limit_exceeded" : @"malformed_tree",
164
+ data ? @"response-too-large" : @"response-not-json-safe",
165
+ data ? @"snapshot response exceeds the per-request response bound" : (error.localizedDescription ?: @"response was not JSON serializable")) mutableCopy];
166
+ if (response[@"pid"] != nil) fallback[@"pid"] = response[@"pid"];
167
+ if (response[@"generation"] != nil) fallback[@"generation"] = response[@"generation"];
168
+ return [NSJSONSerialization dataWithJSONObject:fallback options:0 error:NULL];
169
+ }
170
+
171
+ static NSUInteger responseLimitForRequest(id request)
172
+ {
173
+ if (![request isKindOfClass:NSDictionary.class]) return kMaximumFrameBytes;
174
+ NSNumber *value = request[@"maxResponseBytes"];
175
+ if (![value isKindOfClass:NSNumber.class]) return kMaximumFrameBytes;
176
+ NSUInteger result = value.unsignedIntegerValue;
177
+ return result >= 1024 && result <= kMaximumFrameBytes ? result : kMaximumFrameBytes;
178
+ }
179
+
180
+ static int serve(NSString *socketPath, int idleTimeoutSeconds, BOOL exitOnDisconnect)
181
+ {
182
+ if (socketPath.length == 0 || socketPath.length >= sizeof(((struct sockaddr_un *)0)->sun_path)) {
183
+ bridgeLog(@"socket path is empty or too long");
184
+ return 1;
185
+ }
186
+
187
+ int listener = socket(AF_UNIX, SOCK_STREAM, 0);
188
+ if (listener < 0) {
189
+ bridgeLog([NSString stringWithFormat:@"socket failed: %s", strerror(errno)]);
190
+ return 1;
191
+ }
192
+ struct sockaddr_un address = {0};
193
+ address.sun_family = AF_UNIX;
194
+ strlcpy(address.sun_path, socketPath.fileSystemRepresentation, sizeof(address.sun_path));
195
+ unlink(address.sun_path);
196
+ if (bind(listener, (struct sockaddr *)&address, sizeof(address)) != 0 || listen(listener, 4) != 0) {
197
+ bridgeLog([NSString stringWithFormat:@"bind/listen failed for %@: %s", socketPath, strerror(errno)]);
198
+ close(listener);
199
+ return 1;
200
+ }
201
+ chmod(address.sun_path, S_IRUSR | S_IWUSR);
202
+ bridgeLog([NSString stringWithFormat:@"serving protocol %lu on %@", (unsigned long)kProtocolVersion, socketPath]);
203
+
204
+ BOOL done = NO;
205
+ while (!done) {
206
+ struct pollfd waitForClient = {.fd = listener, .events = POLLIN, .revents = 0};
207
+ int ready = poll(&waitForClient, 1, idleTimeoutSeconds * 1000);
208
+ if (ready == 0) break;
209
+ if (ready < 0) {
210
+ if (errno == EINTR) continue;
211
+ break;
212
+ }
213
+ int connection = accept(listener, NULL, NULL);
214
+ if (connection < 0) {
215
+ if (errno == EINTR) continue;
216
+ break;
217
+ }
218
+ struct timeval timeout = {.tv_sec = idleTimeoutSeconds, .tv_usec = 0};
219
+ setsockopt(connection, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
220
+ setsockopt(connection, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
221
+ while (YES) {
222
+ @autoreleasepool {
223
+ uint32_t networkLength = 0;
224
+ if (!readFully(connection, &networkLength, sizeof(networkLength))) break;
225
+ uint32_t length = ntohl(networkLength);
226
+ if (length == 0 || length > kMaximumFrameBytes) break;
227
+ NSMutableData *body = [NSMutableData dataWithLength:length];
228
+ if (!readFully(connection, body.mutableBytes, length)) break;
229
+ id parsed = [NSJSONSerialization JSONObjectWithData:body options:0 error:NULL];
230
+ NSDictionary *response = [parsed isKindOfClass:NSDictionary.class]
231
+ ? handleRequest(parsed)
232
+ : failureResponse(@"", @"bad_request", @"json-object-required", @"request frame must be a JSON object");
233
+ NSData *encoded = serializedResponse(response, responseLimitForRequest(parsed));
234
+ if (encoded.length > kMaximumFrameBytes) break;
235
+ uint32_t responseLength = htonl((uint32_t)encoded.length);
236
+ if (!writeFully(connection, &responseLength, sizeof(responseLength)) ||
237
+ !writeFully(connection, encoded.bytes, encoded.length)) break;
238
+ }
239
+ }
240
+ close(connection);
241
+ if (exitOnDisconnect) done = YES;
242
+ }
243
+
244
+ close(listener);
245
+ unlink(address.sun_path);
246
+ return 0;
247
+ }
248
+
249
+ static int integerArgument(NSArray<NSString *> *arguments, NSString *flag, int fallback)
250
+ {
251
+ for (NSUInteger index = 0; index + 1 < arguments.count; index += 1) {
252
+ if (![arguments[index] isEqualToString:flag]) continue;
253
+ NSInteger value = arguments[index + 1].integerValue;
254
+ if (value > 0 && value <= INT_MAX) return (int)value;
255
+ }
256
+ return fallback;
257
+ }
258
+
259
+ static BOOL boolArgument(NSArray<NSString *> *arguments, NSString *flag, BOOL fallback)
260
+ {
261
+ for (NSUInteger index = 0; index + 1 < arguments.count; index += 1) {
262
+ if ([arguments[index] isEqualToString:flag]) return [arguments[index + 1] boolValue];
263
+ }
264
+ return fallback;
265
+ }
266
+
267
+ int main(int argc, const char *argv[])
268
+ {
269
+ @autoreleasepool {
270
+ if (argc < 3 || strcmp(argv[1], "serve") != 0) {
271
+ fprintf(stderr, "Usage: %s serve <socket> [--idle-timeout <seconds>] [--exit-on-disconnect <bool>]\n", argv[0]);
272
+ return 2;
273
+ }
274
+ NSMutableArray<NSString *> *arguments = [NSMutableArray array];
275
+ for (int index = 2; index < argc; index += 1) {
276
+ NSString *value = [NSString stringWithUTF8String:argv[index]];
277
+ if (value) [arguments addObject:value];
278
+ }
279
+ NSString *socketPath = arguments.firstObject;
280
+ if (socketPath.length == 0) return 2;
281
+ NSArray<NSString *> *flags = [arguments subarrayWithRange:NSMakeRange(1, arguments.count - 1)];
282
+ return serve(socketPath,
283
+ integerArgument(flags, @"--idle-timeout", kDefaultIdleTimeoutSeconds),
284
+ boolArgument(flags, @"--exit-on-disconnect", YES));
285
+ }
286
+ }
@@ -0,0 +1,25 @@
1
+ #import <Foundation/Foundation.h>
2
+
3
+ NS_ASSUME_NONNULL_BEGIN
4
+
5
+ typedef id _Nullable (^SnapshotElementReader)(id element, NSUInteger depth, NSUInteger nodes, NSError **error);
6
+
7
+ /// Native request accounting for one acquisition: requests issued, requests the accessibility
8
+ /// server rejected at their depth, continuations (requests that reached the reader beyond the
9
+ /// root's accepted one), and the native levels of the last accepted request.
10
+ typedef struct {
11
+ NSUInteger requests;
12
+ NSUInteger rejected;
13
+ NSUInteger continuations;
14
+ NSUInteger acceptedLevels;
15
+ } SnapshotCaptureRecovery;
16
+
17
+ /// Materializes one bounded tree; retries bounded native acquisition failures and re-roots withheld
18
+ /// children. `nativeLevelsHint` (0 for none) caps only the first request's native levels; the
19
+ /// delivered depth, node bounds, and completeness rules are unchanged.
20
+ NSDictionary *_Nullable captureSnapshotTree(id element, NSUInteger maxDepth, NSUInteger maxNodes,
21
+ NSUInteger nativeLevelsHint, SnapshotElementReader reader,
22
+ BOOL *truncated, SnapshotCaptureRecovery *_Nullable recovery,
23
+ NSError **error);
24
+
25
+ NS_ASSUME_NONNULL_END