agent-device 0.20.7 → 0.20.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/README.md +94 -31
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.7.apk → agent-device-android-ime-helper-0.20.9.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.9.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.7.manifest.json → agent-device-android-ime-helper-0.20.9.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.7.apk → agent-device-android-snapshot-helper-0.20.9.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.9.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.7.manifest.json → agent-device-android-snapshot-helper-0.20.9.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +3 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +13 -7
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +4 -112
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +5 -1
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +9 -1
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +90 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+RecordingTests.swift +4 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +46 -10
  16. package/dist/src/agent-browser-tool.js +1 -1
  17. package/dist/src/agent-device-client.js +2 -2
  18. package/dist/src/android-adb.d.ts +5 -5
  19. package/dist/src/android-adb.js +1 -1
  20. package/dist/src/android.js +1 -1
  21. package/dist/src/app-helpers.js +1 -1
  22. package/dist/src/app-lifecycle.js +6 -7
  23. package/dist/src/app-lifecycle2.js +2 -2
  24. package/dist/src/app-log-reconnect.js +1 -1
  25. package/dist/src/app-log-runtime.js +3 -3
  26. package/dist/src/app-resolution.js +2 -2
  27. package/dist/src/app-state.js +2 -0
  28. package/dist/src/appearance.js +1 -0
  29. package/dist/src/apps.js +5 -5
  30. package/dist/src/artifacts.d.ts +1 -1
  31. package/dist/src/audio-probe.js +1 -1
  32. package/dist/src/backend-snapshot-options.js +1 -0
  33. package/dist/src/boot-diagnostics.js +2 -2
  34. package/dist/src/bounds.js +1 -0
  35. package/dist/src/byte-limit-stream.js +1 -0
  36. package/dist/src/cancel.js +1 -0
  37. package/dist/src/cli-config.js +1 -1
  38. package/dist/src/cli-help.js +25 -20
  39. package/dist/src/cli.js +6 -6
  40. package/dist/src/client-metro.js +1 -1
  41. package/dist/src/command-catalog.js +1 -0
  42. package/dist/src/command-metadata.js +1 -1
  43. package/dist/src/command-schema.js +1 -1
  44. package/dist/src/command-surface.js +1 -1
  45. package/dist/src/config.js +1 -0
  46. package/dist/src/connection-runtime.js +1 -1
  47. package/dist/src/connection.js +3 -3
  48. package/dist/src/contracts.js +1 -1
  49. package/dist/src/debug-symbols.js +1 -1
  50. package/dist/src/device-claim-inspection.js +1 -1
  51. package/dist/src/device-input-state.js +1 -1
  52. package/dist/src/device-ready.js +1 -0
  53. package/dist/src/device2.js +2 -2
  54. package/dist/src/dispatch.js +1 -0
  55. package/dist/src/entities.js +1 -1
  56. package/dist/src/find.js +1 -1
  57. package/dist/src/finders.js +1 -1
  58. package/dist/src/generic-settle.js +1 -1
  59. package/dist/src/generic.js +1 -1
  60. package/dist/src/handler-utils.js +1 -0
  61. package/dist/src/harmonyos.js +1 -1
  62. package/dist/src/hdc.js +1 -0
  63. package/dist/src/helper-package-install.js +1 -0
  64. package/dist/src/helper.js +2 -2
  65. package/dist/src/host-process.js +3 -2
  66. package/dist/src/ime-lifecycle.js +2 -9
  67. package/dist/src/ime-recovery-marker.js +1 -0
  68. package/dist/src/index.d.ts +35 -26
  69. package/dist/src/index.js +1 -1
  70. package/dist/src/input-actions.js +3 -3
  71. package/dist/src/input-actions2.js +1 -1
  72. package/dist/src/input-actions3.js +1 -1
  73. package/dist/src/install-artifact.js +1 -1
  74. package/dist/src/install-artifact2.js +1 -1
  75. package/dist/src/install-source2.js +1 -1
  76. package/dist/src/interaction-common.js +1 -0
  77. package/dist/src/interaction-runtime.js +1 -1
  78. package/dist/src/interaction-snapshot.js +1 -1
  79. package/dist/src/interaction.js +1 -1
  80. package/dist/src/interaction2.js +1 -0
  81. package/dist/src/interactor.js +2 -2
  82. package/dist/src/interactor2.js +1 -1
  83. package/dist/src/interactors.js +1 -0
  84. package/dist/src/internal/bin.js +1 -1
  85. package/dist/src/internal/daemon.js +4 -29
  86. package/dist/src/internal/png-worker.js +1 -1
  87. package/dist/src/inventory.js +1 -1
  88. package/dist/src/inventory2.js +1 -1
  89. package/dist/src/keyed-lock.js +1 -0
  90. package/dist/src/lease-context.js +1 -1
  91. package/dist/src/lease.js +1 -1
  92. package/dist/src/limrun-runtime-dependencies.js +1 -1
  93. package/dist/src/limrun.d.ts +3 -1
  94. package/dist/src/limrun.js +1 -1
  95. package/dist/src/linux.js +1 -1
  96. package/dist/src/location-coordinates.js +1 -0
  97. package/dist/src/logcat.js +1 -1
  98. package/dist/src/manifest.js +1 -1
  99. package/dist/src/observability.js +1 -0
  100. package/dist/src/output-path.js +1 -0
  101. package/dist/src/owner-identity.js +1 -1
  102. package/dist/src/perf.js +4 -0
  103. package/dist/src/physical-device-control.js +1 -1
  104. package/dist/src/platform-runtime-app-log-process.js +1 -1
  105. package/dist/src/platform-runtime-host.js +1 -1
  106. package/dist/src/platform-runtime-open-target.js +1 -0
  107. package/dist/src/platform-runtime-operation-host.js +4 -4
  108. package/dist/src/platform-runtime-runtime-hints.js +2 -0
  109. package/dist/src/platform-runtime-screen-recording-android-host.js +1 -0
  110. package/dist/src/platform-runtime-screen-recording-apple-host.js +1 -0
  111. package/dist/src/platform-runtime-screen-recording-apple-runner-host.js +1 -0
  112. package/dist/src/platform-runtime-screen-recording-apple-runner-transport.js +1 -0
  113. package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -0
  114. package/dist/src/platform-runtime-screen-recording-apple-transport.js +1 -0
  115. package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -0
  116. package/dist/src/platform-runtime-screen-recording-harmony-host.js +1 -0
  117. package/dist/src/platform-runtime-screen-recording-output-host.js +1 -0
  118. package/dist/src/platform-runtime-screen-recording-web-host.js +1 -0
  119. package/dist/src/platform-runtime.js +1 -1
  120. package/dist/src/platform-runtime2.js +1 -0
  121. package/dist/src/plist.js +1 -1
  122. package/dist/src/png-resize.js +1 -0
  123. package/dist/src/png-worker-contract.js +1 -1
  124. package/dist/src/prepare-kind.js +13 -13
  125. package/dist/src/process-lock.js +1 -1
  126. package/dist/src/project-runtime.js +1 -0
  127. package/dist/src/provider-webdriver.js +1 -1
  128. package/dist/src/provider.js +1 -1
  129. package/dist/src/proxy.js +1 -1
  130. package/dist/src/react-native.js +1 -1
  131. package/dist/src/record-runtime.js +1 -0
  132. package/dist/src/record-trace.js +1 -1
  133. package/dist/src/recording-export-quality.js +1 -0
  134. package/dist/src/recording-telemetry.js +1 -0
  135. package/dist/src/registry.js +29 -26
  136. package/dist/src/replay.js +2 -2
  137. package/dist/src/reporting.js +3 -3
  138. package/dist/src/rolldown-runtime.js +1 -0
  139. package/dist/src/runner-client.js +1 -1
  140. package/dist/src/runner-disposal.js +1 -1
  141. package/dist/src/runner-lease.js +1 -1
  142. package/dist/src/runner-sequence.js +1 -1
  143. package/dist/src/runtime-transport.js +1 -1
  144. package/dist/src/runtime.js +3 -3
  145. package/dist/src/runtime2.js +4 -3
  146. package/dist/src/runtime3.js +1 -3
  147. package/dist/src/runtime4.js +3 -1
  148. package/dist/src/runtime5.js +1 -1
  149. package/dist/src/runtime6.js +1 -1
  150. package/dist/src/runtime7.js +1 -1
  151. package/dist/src/runtime8.js +1 -0
  152. package/dist/src/runtime9.js +1 -0
  153. package/dist/src/scoped-provider.js +1 -0
  154. package/dist/src/screenshot-density.js +1 -0
  155. package/dist/src/screenshot-geometry.js +1 -0
  156. package/dist/src/screenshot-result.js +4 -4
  157. package/dist/src/screenshot.js +1 -1
  158. package/dist/src/sdk-android-adb.d.ts +20 -2
  159. package/dist/src/sdk-android-adb.js +4 -4
  160. package/dist/src/sdk-batch-runner.js +3 -3
  161. package/dist/src/sdk-contracts.d.ts +3 -3
  162. package/dist/src/sdk-contracts.js +2 -2
  163. package/dist/src/sdk-device.js +1 -1
  164. package/dist/src/sdk-finders.js +1 -1
  165. package/dist/src/sdk-io.js +1 -1
  166. package/dist/src/sdk-selectors.d.ts +24 -20
  167. package/dist/src/sdk-selectors.js +1 -1
  168. package/dist/src/sdk.js +1 -0
  169. package/dist/src/selector-runtime.js +1 -1
  170. package/dist/src/selectors.d.ts +1 -1
  171. package/dist/src/selectors.js +1 -1
  172. package/dist/src/server.js +2 -2
  173. package/dist/src/session-target-evidence.js +1 -1
  174. package/dist/src/session.js +1 -30
  175. package/dist/src/session2.js +28 -0
  176. package/dist/src/setting-state.js +1 -1
  177. package/dist/src/shared.js +1 -1
  178. package/dist/src/simctl.js +1 -0
  179. package/dist/src/simulator-state.js +1 -0
  180. package/dist/src/simulator.js +1 -1
  181. package/dist/src/snapshot-runtime.js +1 -1
  182. package/dist/src/snapshot-session.js +1 -1
  183. package/dist/src/snapshot.js +8 -1
  184. package/dist/src/snapshot2.js +1 -1
  185. package/dist/src/snapshot3.js +1 -0
  186. package/dist/src/src.js +1 -1
  187. package/dist/src/src2.js +1 -2
  188. package/dist/src/src3.js +2 -1
  189. package/dist/src/string-enum.js +1 -0
  190. package/dist/src/tool-provider.js +3 -3
  191. package/dist/src/tool-provider2.js +1 -1
  192. package/dist/src/update-check.js +1 -1
  193. package/dist/src/verdict.js +1 -0
  194. package/dist/src/verified-file.js +1 -0
  195. package/dist/src/version.js +1 -1
  196. package/dist/src/video.js +23 -0
  197. package/package.json +8 -6
  198. package/server.json +3 -3
  199. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +0 -1
  200. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +0 -1
  201. package/dist/src/context.js +0 -1
  202. package/dist/src/interaction-touch-reference-frame.js +0 -1
  203. package/dist/src/notifications.js +0 -1
  204. package/dist/src/platform-runtime-toolchain-host.js +0 -1
  205. package/dist/src/runner-provider.js +0 -1
  206. package/dist/src/text-surface.js +0 -1
  207. package/dist/src/tool-provider3.js +0 -1
  208. package/skills/agent-device/SKILL.md +0 -20
  209. package/skills/dogfood/SKILL.md +0 -27
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  <a href="https://www.callstack.com/open-source?utm_campaign=generic&utm_source=github&utm_medium=referral&utm_content=agent-device" align="center">
2
2
  <picture>
3
- <img alt="agent-device: device automation CLI for AI agents" src="website/docs/public/agent-device-banner.jpg">
3
+ <img alt="agent-device: mobile app automation and verification for AI coding agents" src="website/docs/public/agent-device-banner.jpg">
4
4
  </picture>
5
5
  </a>
6
6
 
@@ -13,13 +13,13 @@
13
13
  [![License: MIT](https://img.shields.io/badge/license-MIT-black.svg)](LICENSE)
14
14
  [![Glama MCP server](https://glama.ai/mcp/servers/callstack/agent-device/badges/score.svg)](https://glama.ai/mcp/servers/callstack/agent-device)
15
15
 
16
- Let your coding agent verify its changes in the running app.
16
+ **Mobile app automation and verification for AI coding agents.** Give coding agents a live app feedback loop through a CLI, built-in MCP server, or typed Node.js API.
17
17
 
18
- `agent-device` lets coding agents inspect, control, and verify apps on iOS, Android, HarmonyOS, tvOS, Android TV, Amazon Vega OS TV through the Vega Virtual Device (VVD), web, macOS, and Linux. Agents can read token-efficient accessibility snapshots where supported, find elements by ref or selector, run device actions, and save evidence for review. HarmonyOS support uses HDC and ArkUI `uitest` on connected devices and DevEco emulators; use `capabilities --platform harmonyos` to inspect its evidence-backed command subset. Initial Vega OS support is VVD-only and covers discovery, app lifecycle, and complete TV-remote control; physical Fire TV, capture, and selector backends remain unsupported.
18
+ <!-- Intro rule: category line, job pitch with the platform list, works-with/proof line, nothing else. Per-platform transports, tool names, vendor lists, and support caveats belong in "How it works" and website/docs, never here. Every name in the proof line must link to public evidence. -->
19
19
 
20
- Your coding agent or QA tool reads each result and chooses the next command. `agent-device` runs the command and saves evidence when asked.
20
+ Let your coding agent verify its changes in the running app. `agent-device` lets agents inspect, control, debug, and verify apps on iOS, Android, and HarmonyOS (simulators, emulators, and physical devices), plus tvOS, Android TV, Amazon Vega OS TV (Vega Virtual Device), web, macOS, and Linux. Agents read token-efficient accessibility snapshots instead of reasoning over screenshots alone, act through refs and selectors, and save evidence for review. It also coordinates device access across parallel agent worktrees and connects to remote device clouds.
21
21
 
22
- `agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system.
22
+ Works with Claude Code, Codex, Cursor, Windsurf, Cline, Goose, and any agent that can run a CLI or connect over MCP, or as the runtime under agents you build with the [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk) or [Eve](https://oss.callstack.com/agent-device/docs/eve). Developers at [Expensify](https://www.callstack.com/blog/how-expensify-uses-agent-device-for-mobile-bug-evidence-and-profiling), [Shopify](https://x.com/mustafa01ali/status/2036577353178943826), and [others](#who-uses-agent-device) use it to verify their apps.
23
23
 
24
24
  ## Quick start
25
25
 
@@ -31,7 +31,9 @@ agent-device doctor
31
31
  agent-device help workflow
32
32
  ```
33
33
 
34
- Run `agent-device doctor` yourself before handing the CLI to an agent. The installed CLI help defines current behavior. `agent-device help workflow` links to guides for debugging, replay, React Native profiling, and other tasks.
34
+ Run `doctor` yourself before handing the CLI to an agent; `help workflow` links to the guides for debugging, replay, and profiling, and the installed help always matches the installed version.
35
+
36
+ ### Drive an app from the CLI
35
37
 
36
38
  Add a contact in the built-in iOS Contacts app:
37
39
 
@@ -59,13 +61,46 @@ agent-device screenshot ./contact-form.png
59
61
  agent-device close
60
62
  ```
61
63
 
62
- Use refs only from the latest output. Do not assume an earlier `@eN` still identifies the same element. After a command with `--settle`, use the refs in its diff. Take another snapshot only if the diff omits what you need.
64
+ Refs are only valid from the latest output: after a `--settle` command, use the refs in its diff, and take a new snapshot only if the diff omits what you need. Snapshots come from the app's accessibility tree, so clear labels, roles, and test IDs make agent runs more reliable; use screenshots and video as evidence or when accessibility data is poor.
63
65
 
64
- `--settle` works the same way on `scroll` and `back`, so scroll-then-observe and back-then-observe are one call too.
66
+ ![agent-device demo showing Codex using agent-device to create a new contact in the iOS Contacts app from a simple prompt](./website/docs/public/agent-device-contacts.gif)
65
67
 
66
- Snapshots use the app's accessibility tree. Clear labels, roles, and test IDs make agent runs more reliable. Use screenshots and videos as evidence or when accessibility data is poor. Use refs and selectors for actions and assertions when you can.
68
+ ### Add MCP tools to your agent
67
69
 
68
- ![agent-device demo showing Codex using agent-device to create a new contact in the iOS Contacts app from a simple prompt](./website/docs/public/agent-device-contacts.gif)
70
+ `agent-device mcp` starts the official stdio MCP server, exposing the installed commands as structured tools over the same execution path as the CLI:
71
+
72
+ ```json
73
+ {
74
+ "mcpServers": {
75
+ "agent-device": {
76
+ "command": "agent-device",
77
+ "args": ["mcp"]
78
+ }
79
+ }
80
+ }
81
+ ```
82
+
83
+ See [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) for per-client setup and when to prefer plain CLI over MCP.
84
+
85
+ ### Script it from Node.js
86
+
87
+ `createAgentDeviceClient()` gives Node.js code typed access to the same commands, as model tools in your own agent or from orchestration code:
88
+
89
+ ```ts
90
+ import { createAgentDeviceClient } from 'agent-device';
91
+
92
+ const client = createAgentDeviceClient({ session: 'qa-run' });
93
+ try {
94
+ await client.apps.open({ app: 'com.apple.Preferences', platform: 'ios' });
95
+ const snapshot = await client.capture.snapshot({ interactiveOnly: true });
96
+ const button = snapshot.nodes.find((node) => node.role === 'button');
97
+ if (button) await client.interactions.press({ ref: button.ref });
98
+ } finally {
99
+ await client.sessions.close();
100
+ }
101
+ ```
102
+
103
+ See the [Node.js API](https://oss.callstack.com/agent-device/docs/client-api), the [runnable examples](https://github.com/callstack/agent-device/tree/main/examples/sdk), and the [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk) and [Eve](https://oss.callstack.com/agent-device/docs/eve) integration guides.
69
104
 
70
105
  ## What agents can do
71
106
 
@@ -78,58 +113,86 @@ See [Commands](https://oss.callstack.com/agent-device/docs/commands) for the com
78
113
 
79
114
  ![Diagram of the agentic development loop: humans assign tasks, agents write and review code, agent-device verifies mobile apps, pull requests receive evidence, and bugs or performance issues lead to fixes](./website/docs/public/agentic-development-loop.svg)
80
115
 
81
- ## Next steps
82
-
83
- - **Set up your agent**: run the CLI from Cursor, Codex, Claude Code, Windsurf, or another agent terminal. See [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) for skills, rules, MCP tools, and setup for each client.
84
- - **Try the sample app**: clone the repo and run the bundled Expo test app. [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start) covers a guided run with screenshots, replay, and performance data.
85
- - **Build repeatable tests**: use [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) to repeat tests. Use [Debugging & Profiling](https://oss.callstack.com/agent-device/docs/debugging-profiling) to find bugs.
116
+ ## What to ask your agent
86
117
 
87
- ## Articles and videos
118
+ With the CLI installed, prompts like these work end to end:
88
119
 
89
- ### Articles
120
+ - "Implement the onboarding screen, run it on the iOS simulator and Android emulator, and attach screenshots."
121
+ - "Reproduce this crash and capture the logs that lead up to it."
122
+ - "Check whether this change causes unnecessary React Native re-renders."
123
+ - "Explore the checkout flow once, save it as a replay script, and run it in CI."
124
+ - "Verify this pull request on a physical device and attach reviewable evidence."
90
125
 
91
- - [Build an AI QA agent for Expo apps with EAS Workflows](https://expo.dev/blog/build-an-ai-qa-agent-for-expo-apps-with-eas-workflows-in-minutes-today)
92
- - [Agent Device: iOS & Android automation for AI agents](https://www.callstack.com/blog/agent-device-ai-native-mobile-automation-for-ios-android)
93
- - [Building mobile QA agents with Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve)
94
- - [How we optimized Agent Device for mobile app automation](https://www.callstack.com/blog/how-we-optimized-agent-device-for-mobile-app-automation)
95
-
96
- ### Videos
126
+ ## Next steps
97
127
 
98
- - [Verifying mobile apps with agent-device](https://youtu.be/kZDU-k5r9kE)
99
- - [Using agent-device in an AI coding workflow](https://youtu.be/dfVG_aNPkW4)
100
- - [Cloud agents that test mobile apps on real devices](https://youtu.be/r5P0detC4bs?is=_KB6SZbLFRB1au_z)
128
+ - [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup): skills, project rules, and per-client setup for Cursor, Codex, Claude Code, Windsurf, and others.
129
+ - [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start): a guided run on the bundled Expo test app with screenshots, replay, and performance data.
130
+ - [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) and [Debugging & Profiling](https://oss.callstack.com/agent-device/docs/debugging-profiling): repeatable tests and bug hunting.
101
131
 
102
132
  ## Where to run agent-device
103
133
 
134
+ The same session and evidence model works at every step: the agent explores the app, captures evidence, saves a replay, runs it in CI, and moves onto remote devices.
135
+
104
136
  | Path | Best for | Start with |
105
137
  | --- | --- | --- |
106
138
  | Local | Trying commands and debugging apps on simulators, emulators, physical devices, macOS, and Linux. | Follow the Quick Start. |
107
- | CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). GitHub Actions template coming soon. |
108
- | Cloud / remote | Linux runners, managed devices, and remote jobs. | Use [Agent Device Cloud](https://agent-device.dev/cloud), set a remote profile with [Commands](https://oss.callstack.com/agent-device/docs/commands), or [contact Callstack](mailto:hello@callstack.com) for team QA. |
139
+ | CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). |
140
+ | Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. |
109
141
 
110
142
  ## How it works
111
143
 
112
144
  `agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux.
113
145
 
114
- Node.js apps can use the typed client or public subpaths. `agent-device/android-adb` provides the Android ADB provider interface, helpers for logcat, the clipboard, the keyboard, and apps, and port reverse management.
146
+ Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
147
+
148
+ Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds).
149
+
150
+ `agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system.
115
151
 
116
152
  ## FAQ
117
153
 
118
154
  ### What is agent-device?
119
155
 
120
- `agent-device` is a command-line tool that lets coding agents inspect, control, and verify apps and save evidence for review. It supports iOS, Android, HarmonyOS, TV, web, macOS, and Linux.
156
+ `agent-device` is a command-line tool and MCP server that lets AI coding agents inspect, control, and verify mobile apps and save evidence for review. It supports iOS, Android, HarmonyOS, TV, web, macOS, and Linux.
157
+
158
+ ### Is there an MCP server for mobile app automation?
159
+
160
+ Yes. `agent-device mcp` starts the official stdio MCP server. The Quick start above has the client config, and [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) covers per-client details.
121
161
 
122
162
  ### Does it work with React Native, Expo, Flutter, and native apps?
123
163
 
124
164
  Yes. `agent-device` supports native iOS and Android apps, plus React Native, Expo, and Flutter apps on supported targets. The commands and evidence vary by target.
125
165
 
166
+ ### How is it different from mobile MCP servers?
167
+
168
+ The MCP server is one entry point to the same runtime used by the CLI and typed Node.js API. Sessions, device ownership, selectors, evidence, replay, CI workflows, and cloud routing stay consistent across all three.
169
+
170
+ ### Can I build my own agent or QA product on agent-device?
171
+
172
+ Yes. The typed Node.js client is a public surface over that same runtime, so an agent you build inherits everything above. Start from the [Node.js API](https://oss.callstack.com/agent-device/docs/client-api), [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk), or [Eve](https://oss.callstack.com/agent-device/docs/eve) guides.
173
+
126
174
  ### How is it different from Appium, Detox, or Maestro?
127
175
 
128
176
  With `agent-device`, an agent reads app state and chooses each command at run time. Teams use Appium, Detox, and Maestro to write and maintain test suites. `agent-device` can complement them by saving its runs as `.ad` scripts or exporting them as strict Maestro YAML.
129
177
 
130
178
  ### Can agent-device run in CI?
131
179
 
132
- Yes. Record a run as an `.ad` script, replay it locally or in CI, and save screenshots, logs, and other artifacts for review. See [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) or start with the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml).
180
+ Yes. Record a run as an `.ad` script, replay it in CI, and keep the screenshots and logs as artifacts; the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml) is a working example.
181
+
182
+ ## Articles and videos
183
+
184
+ ### Articles
185
+
186
+ - [Build an AI QA agent for Expo apps with EAS Workflows](https://expo.dev/blog/build-an-ai-qa-agent-for-expo-apps-with-eas-workflows-in-minutes-today)
187
+ - [Agent Device: iOS & Android automation for AI agents](https://www.callstack.com/blog/agent-device-ai-native-mobile-automation-for-ios-android)
188
+ - [Building mobile QA agents with Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve)
189
+ - [How we optimized Agent Device for mobile app automation](https://www.callstack.com/blog/how-we-optimized-agent-device-for-mobile-app-automation)
190
+
191
+ ### Videos
192
+
193
+ - [Verifying mobile apps with agent-device](https://youtu.be/kZDU-k5r9kE)
194
+ - [Using agent-device in an AI coding workflow](https://youtu.be/dfVG_aNPkW4)
195
+ - [Cloud agents that test mobile apps on real devices](https://youtu.be/r5P0detC4bs?is=_KB6SZbLFRB1au_z)
133
196
 
134
197
  ## Who uses agent-device?
135
198
 
@@ -0,0 +1 @@
1
+ f98ded51f104c10268a75b7eb77f2e479461b844e49f238593d44b821d13e294 agent-device-android-ime-helper-0.20.9.apk
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "android-ime-helper",
3
- "version": "0.20.7",
4
- "assetName": "agent-device-android-ime-helper-0.20.7.apk",
5
- "sha256": "7311d6a527bdf8be33b752e59e73f4a527585654f8d530d3680ab15ccf271cb0",
3
+ "version": "0.20.9",
4
+ "assetName": "agent-device-android-ime-helper-0.20.9.apk",
5
+ "sha256": "f98ded51f104c10268a75b7eb77f2e479461b844e49f238593d44b821d13e294",
6
6
  "packageName": "com.callstack.agentdevice.imehelper",
7
- "versionCode": 20007,
7
+ "versionCode": 20009,
8
8
  "serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
9
9
  "broadcastProtocol": "android-ime-helper-v1"
10
10
  }
@@ -0,0 +1 @@
1
+ a5169f62fb38bb1980cf7c7963542f2bc5bbaafd4d9dc7b5b23d3e41ee782725 agent-device-android-snapshot-helper-0.20.9.apk
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "android-snapshot-helper",
3
- "version": "0.20.7",
4
- "releaseTag": "v0.20.7",
5
- "assetName": "agent-device-android-snapshot-helper-0.20.7.apk",
3
+ "version": "0.20.9",
4
+ "releaseTag": "v0.20.9",
5
+ "assetName": "agent-device-android-snapshot-helper-0.20.9.apk",
6
6
  "apkUrl": null,
7
- "sha256": "c45f72eea93c19b4a17fedb4b25f880d5e225c9ab41be4017db8613fdd276e09",
8
- "checksumName": "agent-device-android-snapshot-helper-0.20.7.apk.sha256",
7
+ "sha256": "a5169f62fb38bb1980cf7c7963542f2bc5bbaafd4d9dc7b5b23d3e41ee782725",
8
+ "checksumName": "agent-device-android-snapshot-helper-0.20.9.apk.sha256",
9
9
  "packageName": "com.callstack.agentdevice.snapshothelper",
10
- "versionCode": 20007,
10
+ "versionCode": 20009,
11
11
  "instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
12
12
  "minSdk": 23,
13
13
  "targetSdk": 36,
@@ -97,6 +97,9 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey;
97
97
  /// adds no work", which is only observable as this counter standing still.
98
98
  + (NSInteger)customActionReadDispatchCount;
99
99
 
100
+ /// Total reads refused by single-flight admission.
101
+ + (NSInteger)customActionReadBlockedCount;
102
+
100
103
  /// The shared AX client (`XCUIDevice.accessibilityInterface`), or nil when the
101
104
  /// private interface is unavailable.
102
105
  + (nullable id)accessibilityClient;
@@ -63,6 +63,9 @@ static atomic_int RunnerAXCustomActionReadsInFlight = 0;
63
63
  /// staying put, so it is recorded rather than inferred.
64
64
  static atomic_long RunnerAXCustomActionReadDispatches = 0;
65
65
 
66
+ /// Total single-flight admission refusals.
67
+ static atomic_long RunnerAXCustomActionReadBlocked = 0;
68
+
66
69
  /// Output caps. The element budget bounds how many elements we read; these
67
70
  /// bound what any ONE element can put in the response, so a pathological app
68
71
  /// cannot spend the whole snapshot on one node's action list.
@@ -514,17 +517,15 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
514
517
  if (![axClient respondsToSelector:selector]) {
515
518
  return nil;
516
519
  }
517
- // Single flight. An abandoned read still owns the serial queue, so
518
- // dispatching here would only queue work behind it — and repeating the
519
- // capture would keep queueing more. Refuse without dispatching instead.
520
- if (atomic_load(&RunnerAXCustomActionReadsInFlight) > 0) {
521
- NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_CUSTOM_ACTIONS_READ_BLOCKED");
520
+ int expectedReadsInFlight = 0;
521
+ if (!atomic_compare_exchange_strong(
522
+ &RunnerAXCustomActionReadsInFlight, &expectedReadsInFlight, 1)) {
523
+ atomic_fetch_add(&RunnerAXCustomActionReadBlocked, 1);
522
524
  return nil;
523
525
  }
524
526
  // The AX call is a synchronous XPC round trip with no timeout of its own, so
525
527
  // it runs off-thread behind a bounded wait. On timeout the result box is
526
528
  // never read, so a late-returning wedged call cannot race this thread.
527
- atomic_fetch_add(&RunnerAXCustomActionReadsInFlight, 1);
528
529
  atomic_fetch_add(&RunnerAXCustomActionReadDispatches, 1);
529
530
  NSMutableArray *box = [NSMutableArray array];
530
531
  dispatch_semaphore_t finished = dispatch_semaphore_create(0);
@@ -543,7 +544,7 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
543
544
  }
544
545
  // Clear in-flight BEFORE signalling, so a waiter that wakes on this signal
545
546
  // never observes its own completed read as still outstanding.
546
- atomic_fetch_sub(&RunnerAXCustomActionReadsInFlight, 1);
547
+ atomic_store(&RunnerAXCustomActionReadsInFlight, 0);
547
548
  dispatch_semaphore_signal(finished);
548
549
  });
549
550
  long waited = dispatch_semaphore_wait(
@@ -622,6 +623,11 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
622
623
  return (NSInteger)atomic_load(&RunnerAXCustomActionReadDispatches);
623
624
  }
624
625
 
626
+ + (NSInteger)customActionReadBlockedCount
627
+ {
628
+ return (NSInteger)atomic_load(&RunnerAXCustomActionReadBlocked);
629
+ }
630
+
625
631
  + (nullable id)accessibilityClient
626
632
  {
627
633
  return [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"];
@@ -175,15 +175,10 @@ extension RunnerTests {
175
175
 
176
176
  let rootFrame = privateAXRect(root["frame"])
177
177
  let viewport = privateAXSnapshotViewport(app: app, rootFrame: rootFrame)
178
- var nodes: [SnapshotNode] = []
179
- appendPrivateAXNode(
180
- root,
181
- to: &nodes,
178
+ let nodes = privateAXPresentation(
179
+ rawRoot: root,
182
180
  options: options,
183
- viewport: viewport,
184
- depth: 0,
185
- parentIndex: nil,
186
- insideMatchedScope: false
181
+ viewport: viewport
187
182
  )
188
183
  if nodes.count <= 1 {
189
184
  NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count)
@@ -250,110 +245,7 @@ extension RunnerTests {
250
245
  }
251
246
  }
252
247
 
253
- private func appendPrivateAXNode(
254
- _ rawNode: [String: Any],
255
- to nodes: inout [SnapshotNode],
256
- options: SnapshotOptions,
257
- viewport: CGRect,
258
- depth: Int,
259
- parentIndex: Int?,
260
- insideMatchedScope: Bool
261
- ) {
262
- if let limit = options.depth, depth > limit { return }
263
-
264
- let rect = privateAXRect(rawNode["frame"])
265
- let label = privateAXString(rawNode["label"])
266
- let identifier = privateAXString(rawNode["identifier"])
267
- let value = privateAXString(rawNode["value"])
268
- let rawType = privateAXInt(rawNode["type"]) ?? 0
269
- let typeName = elementTypeName(rawElementType: rawType)
270
- let enabled = privateAXBool(rawNode["enabled"]) ?? true
271
- let visible = isVisibleInViewport(rect, viewport)
272
- let interactiveCandidate = privateAXInteractiveCandidate(rawElementType: rawType)
273
- let filterDecision = flatSnapshotFilterDecision(
274
- FlatSnapshotFilterNode(
275
- isRoot: parentIndex == nil,
276
- label: label,
277
- identifier: identifier,
278
- valueText: value.isEmpty ? nil : value,
279
- visible: visible
280
- ),
281
- options: options,
282
- insideMatchedScope: insideMatchedScope
283
- )
284
- let include = filterDecision.include
285
- let nowInsideScope = filterDecision.insideMatchedScope
286
-
287
- let currentIndex: Int?
288
- if include {
289
- currentIndex = nodes.count
290
- nodes.append(
291
- SnapshotNode(
292
- index: nodes.count,
293
- type: typeName,
294
- label: label.isEmpty ? nil : label,
295
- identifier: identifier.isEmpty ? nil : identifier,
296
- value: value.isEmpty ? nil : value,
297
- rect: snapshotRect(from: rect),
298
- enabled: enabled,
299
- focused: privateAXBool(rawNode["focused"]) == true ? true : nil,
300
- selected: privateAXBool(rawNode["selected"]) == true ? true : nil,
301
- hittable: visible && enabled && interactiveCandidate,
302
- depth: depth,
303
- parentIndex: parentIndex,
304
- hiddenContentAbove: nil,
305
- hiddenContentBelow: nil,
306
- actions: rawNode["actions"] as? [String]
307
- )
308
- )
309
- } else {
310
- currentIndex = parentIndex
311
- }
312
-
313
- guard let children = rawNode["children"] as? [[String: Any]] else {
314
- return
315
- }
316
- for child in children {
317
- appendPrivateAXNode(
318
- child,
319
- to: &nodes,
320
- options: options,
321
- viewport: viewport,
322
- depth: depth + 1,
323
- parentIndex: currentIndex,
324
- insideMatchedScope: nowInsideScope
325
- )
326
- }
327
- }
328
-
329
- private func elementTypeName(rawElementType: Int) -> String {
330
- if let type = flatSnapshotElementType(rawElementType: rawElementType) {
331
- return elementTypeName(type)
332
- }
333
- return "Element(\(rawElementType))"
334
- }
335
-
336
- private func privateAXString(_ value: Any?) -> String {
337
- guard let value else { return "" }
338
- if let string = value as? String {
339
- return string.trimmingCharacters(in: .whitespacesAndNewlines)
340
- }
341
- return String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
342
- }
343
-
344
- private func privateAXInt(_ value: Any?) -> Int? {
345
- if let value = value as? Int { return value }
346
- if let value = value as? NSNumber { return value.intValue }
347
- return nil
348
- }
349
-
350
- private func privateAXBool(_ value: Any?) -> Bool? {
351
- if let value = value as? Bool { return value }
352
- if let value = value as? NSNumber { return value.boolValue }
353
- return nil
354
- }
355
-
356
- private func privateAXRect(_ value: Any?) -> CGRect {
248
+ func privateAXRect(_ value: Any?) -> CGRect {
357
249
  guard let frame = value as? [String: Any] else {
358
250
  return .zero
359
251
  }
@@ -761,7 +761,11 @@ extension RunnerTests {
761
761
  }
762
762
  case .recordStop:
763
763
  guard let recorder = activeRecording else {
764
- return Response(ok: false, error: ErrorPayload(message: "no active recording"))
764
+ // The runner protocol is the durable cleanup primitive. A daemon may crash after the
765
+ // native stop succeeds but before it commits the resource transition, so exact-owner
766
+ // recovery must be able to repeat this command safely. Public `record stop` still owns
767
+ // its user-facing no-active validation through the daemon session manifest.
768
+ return Response(ok: true, data: DataPayload(message: "recording already stopped"))
765
769
  }
766
770
  do {
767
771
  try recorder.stop()
@@ -18,10 +18,16 @@ struct FlatSnapshotFilterDecision {
18
18
  let insideMatchedScope: Bool
19
19
  }
20
20
 
21
+ enum FlatSnapshotVisibilityPolicy {
22
+ case interactiveOnly
23
+ case viewportProjected
24
+ }
25
+
21
26
  extension RunnerTests {
22
27
  func flatSnapshotFilterDecision(
23
28
  _ node: FlatSnapshotFilterNode,
24
29
  options: SnapshotOptions,
30
+ visibilityPolicy: FlatSnapshotVisibilityPolicy,
25
31
  insideMatchedScope: Bool
26
32
  ) -> FlatSnapshotFilterDecision {
27
33
  let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -39,7 +45,9 @@ extension RunnerTests {
39
45
  include = true
40
46
  } else if scopeActive && !nowInsideScope {
41
47
  include = false
42
- } else if options.interactiveOnly && !node.visible {
48
+ } else if !node.visible
49
+ && (options.interactiveOnly || visibilityPolicy == .viewportProjected)
50
+ {
43
51
  include = false
44
52
  } else {
45
53
  include = true
@@ -0,0 +1,90 @@
1
+ import XCTest
2
+
3
+ extension RunnerTests {
4
+ func privateAXPresentation(rawRoot: [String: Any], options: SnapshotOptions, viewport: CGRect)
5
+ -> [SnapshotNode]
6
+ {
7
+ var nodes: [SnapshotNode] = []
8
+ var hints: [Int: (above: Bool, below: Bool)] = [:]
9
+ appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
10
+ depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil)
11
+ return applyHiddenContentHints(hints, to: nodes)
12
+ }
13
+
14
+ private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [SnapshotNode],
15
+ hints: inout [Int: (above: Bool, below: Bool)], options: SnapshotOptions, viewport: CGRect,
16
+ depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
17
+ scrollContext: (index: Int, rect: CGRect)?)
18
+ {
19
+ if let limit = options.depth, depth > limit { return }
20
+ let rect = privateAXRect(raw["frame"])
21
+ let label = privateAXPresentationString(raw["label"])
22
+ let identifier = privateAXPresentationString(raw["identifier"])
23
+ let value = privateAXPresentationString(raw["value"])
24
+ let rawType = privateAXPresentationInt(raw["type"]) ?? 0
25
+ let enabled = privateAXPresentationBool(raw["enabled"]) ?? true
26
+ let children = raw["children"] as? [[String: Any]] ?? []
27
+ let elementType = flatSnapshotElementType(rawElementType: rawType)
28
+ let hasFrame = !rect.isNull && !rect.isEmpty
29
+ let onScreen = hasFrame
30
+ && isVisibleInRegularSnapshot(
31
+ rect,
32
+ viewport: viewport,
33
+ scrollContainerAnchor: scrollContext
34
+ )
35
+ let presentationVisible = !hasFrame || onScreen
36
+ let decision = flatSnapshotFilterDecision(
37
+ FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier,
38
+ valueText: value.isEmpty ? nil : value, visible: presentationVisible),
39
+ options: options, visibilityPolicy: .viewportProjected,
40
+ insideMatchedScope: insideMatchedScope)
41
+ let include = decision.include
42
+
43
+ if !presentationVisible, let scrollContext {
44
+ rememberHiddenContentHint(for: rect, relativeTo: scrollContext, hints: &hints)
45
+ }
46
+
47
+ let currentIndex: Int?
48
+ if include {
49
+ currentIndex = nodes.count
50
+ let typeName = elementType.map(elementTypeName) ?? "Element(\(rawType))"
51
+ nodes.append(SnapshotNode(index: nodes.count, type: typeName,
52
+ label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
53
+ value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
54
+ focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
55
+ selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
56
+ hittable: onScreen && enabled && privateAXInteractiveCandidate(rawElementType: rawType),
57
+ depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
58
+ actions: raw["actions"] as? [String]))
59
+ } else { currentIndex = parentIndex }
60
+
61
+ let nextScrollContext: (index: Int, rect: CGRect)?
62
+ if include, let elementType, let currentIndex {
63
+ nextScrollContext = scrollContainerAnchor(
64
+ for: elementType,
65
+ hasChildren: !children.isEmpty,
66
+ visible: onScreen,
67
+ frame: rect,
68
+ nodeIndex: currentIndex
69
+ ) ?? scrollContext
70
+ } else { nextScrollContext = scrollContext }
71
+ for child in children {
72
+ appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport,
73
+ depth: depth + 1, parentIndex: currentIndex,
74
+ insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext)
75
+ }
76
+ }
77
+
78
+ private func privateAXPresentationString(_ value: Any?) -> String {
79
+ guard let value else { return "" }
80
+ return (value as? String ?? String(describing: value))
81
+ .trimmingCharacters(in: .whitespacesAndNewlines)
82
+ }
83
+ private func privateAXPresentationInt(_ value: Any?) -> Int? {
84
+ (value as? Int) ?? (value as? NSNumber)?.intValue
85
+ }
86
+ private func privateAXPresentationBool(_ value: Any?) -> Bool? {
87
+ (value as? Bool) ?? (value as? NSNumber)?.boolValue
88
+ }
89
+ }
90
+