agent-device 0.20.7 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -31
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.7.apk → agent-device-android-ime-helper-0.20.9.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.9.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.7.manifest.json → agent-device-android-ime-helper-0.20.9.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.7.apk → agent-device-android-snapshot-helper-0.20.9.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.9.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.7.manifest.json → agent-device-android-snapshot-helper-0.20.9.manifest.json} +6 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +3 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +13 -7
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +4 -112
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +5 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +9 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +90 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+RecordingTests.swift +4 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +46 -10
- package/dist/src/agent-browser-tool.js +1 -1
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/android-adb.d.ts +5 -5
- package/dist/src/android-adb.js +1 -1
- package/dist/src/android.js +1 -1
- package/dist/src/app-helpers.js +1 -1
- package/dist/src/app-lifecycle.js +6 -7
- package/dist/src/app-lifecycle2.js +2 -2
- package/dist/src/app-log-reconnect.js +1 -1
- package/dist/src/app-log-runtime.js +3 -3
- package/dist/src/app-resolution.js +2 -2
- package/dist/src/app-state.js +2 -0
- package/dist/src/appearance.js +1 -0
- package/dist/src/apps.js +5 -5
- package/dist/src/artifacts.d.ts +1 -1
- package/dist/src/audio-probe.js +1 -1
- package/dist/src/backend-snapshot-options.js +1 -0
- package/dist/src/boot-diagnostics.js +2 -2
- package/dist/src/bounds.js +1 -0
- package/dist/src/byte-limit-stream.js +1 -0
- package/dist/src/cancel.js +1 -0
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +25 -20
- package/dist/src/cli.js +6 -6
- package/dist/src/client-metro.js +1 -1
- package/dist/src/command-catalog.js +1 -0
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +1 -1
- package/dist/src/command-surface.js +1 -1
- package/dist/src/config.js +1 -0
- package/dist/src/connection-runtime.js +1 -1
- package/dist/src/connection.js +3 -3
- package/dist/src/contracts.js +1 -1
- package/dist/src/debug-symbols.js +1 -1
- package/dist/src/device-claim-inspection.js +1 -1
- package/dist/src/device-input-state.js +1 -1
- package/dist/src/device-ready.js +1 -0
- package/dist/src/device2.js +2 -2
- package/dist/src/dispatch.js +1 -0
- package/dist/src/entities.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/generic-settle.js +1 -1
- package/dist/src/generic.js +1 -1
- package/dist/src/handler-utils.js +1 -0
- package/dist/src/harmonyos.js +1 -1
- package/dist/src/hdc.js +1 -0
- package/dist/src/helper-package-install.js +1 -0
- package/dist/src/helper.js +2 -2
- package/dist/src/host-process.js +3 -2
- package/dist/src/ime-lifecycle.js +2 -9
- package/dist/src/ime-recovery-marker.js +1 -0
- package/dist/src/index.d.ts +35 -26
- package/dist/src/index.js +1 -1
- package/dist/src/input-actions.js +3 -3
- package/dist/src/input-actions2.js +1 -1
- package/dist/src/input-actions3.js +1 -1
- package/dist/src/install-artifact.js +1 -1
- package/dist/src/install-artifact2.js +1 -1
- package/dist/src/install-source2.js +1 -1
- package/dist/src/interaction-common.js +1 -0
- package/dist/src/interaction-runtime.js +1 -1
- package/dist/src/interaction-snapshot.js +1 -1
- package/dist/src/interaction.js +1 -1
- package/dist/src/interaction2.js +1 -0
- package/dist/src/interactor.js +2 -2
- package/dist/src/interactor2.js +1 -1
- package/dist/src/interactors.js +1 -0
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/internal/daemon.js +4 -29
- package/dist/src/internal/png-worker.js +1 -1
- package/dist/src/inventory.js +1 -1
- package/dist/src/inventory2.js +1 -1
- package/dist/src/keyed-lock.js +1 -0
- package/dist/src/lease-context.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun-runtime-dependencies.js +1 -1
- package/dist/src/limrun.d.ts +3 -1
- package/dist/src/limrun.js +1 -1
- package/dist/src/linux.js +1 -1
- package/dist/src/location-coordinates.js +1 -0
- package/dist/src/logcat.js +1 -1
- package/dist/src/manifest.js +1 -1
- package/dist/src/observability.js +1 -0
- package/dist/src/output-path.js +1 -0
- package/dist/src/owner-identity.js +1 -1
- package/dist/src/perf.js +4 -0
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/platform-runtime-app-log-process.js +1 -1
- package/dist/src/platform-runtime-host.js +1 -1
- package/dist/src/platform-runtime-open-target.js +1 -0
- package/dist/src/platform-runtime-operation-host.js +4 -4
- package/dist/src/platform-runtime-runtime-hints.js +2 -0
- package/dist/src/platform-runtime-screen-recording-android-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-apple-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-apple-runner-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-apple-runner-transport.js +1 -0
- package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-apple-transport.js +1 -0
- package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-harmony-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-output-host.js +1 -0
- package/dist/src/platform-runtime-screen-recording-web-host.js +1 -0
- package/dist/src/platform-runtime.js +1 -1
- package/dist/src/platform-runtime2.js +1 -0
- package/dist/src/plist.js +1 -1
- package/dist/src/png-resize.js +1 -0
- package/dist/src/png-worker-contract.js +1 -1
- package/dist/src/prepare-kind.js +13 -13
- package/dist/src/process-lock.js +1 -1
- package/dist/src/project-runtime.js +1 -0
- package/dist/src/provider-webdriver.js +1 -1
- package/dist/src/provider.js +1 -1
- package/dist/src/proxy.js +1 -1
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-runtime.js +1 -0
- package/dist/src/record-trace.js +1 -1
- package/dist/src/recording-export-quality.js +1 -0
- package/dist/src/recording-telemetry.js +1 -0
- package/dist/src/registry.js +29 -26
- package/dist/src/replay.js +2 -2
- package/dist/src/reporting.js +3 -3
- package/dist/src/rolldown-runtime.js +1 -0
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-lease.js +1 -1
- package/dist/src/runner-sequence.js +1 -1
- package/dist/src/runtime-transport.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/runtime2.js +4 -3
- package/dist/src/runtime3.js +1 -3
- package/dist/src/runtime4.js +3 -1
- package/dist/src/runtime5.js +1 -1
- package/dist/src/runtime6.js +1 -1
- package/dist/src/runtime7.js +1 -1
- package/dist/src/runtime8.js +1 -0
- package/dist/src/runtime9.js +1 -0
- package/dist/src/scoped-provider.js +1 -0
- package/dist/src/screenshot-density.js +1 -0
- package/dist/src/screenshot-geometry.js +1 -0
- package/dist/src/screenshot-result.js +4 -4
- package/dist/src/screenshot.js +1 -1
- package/dist/src/sdk-android-adb.d.ts +20 -2
- package/dist/src/sdk-android-adb.js +4 -4
- package/dist/src/sdk-batch-runner.js +3 -3
- package/dist/src/sdk-contracts.d.ts +3 -3
- package/dist/src/sdk-contracts.js +2 -2
- package/dist/src/sdk-device.js +1 -1
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/sdk-io.js +1 -1
- package/dist/src/sdk-selectors.d.ts +24 -20
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/sdk.js +1 -0
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selectors.d.ts +1 -1
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +2 -2
- package/dist/src/session-target-evidence.js +1 -1
- package/dist/src/session.js +1 -30
- package/dist/src/session2.js +28 -0
- package/dist/src/setting-state.js +1 -1
- package/dist/src/shared.js +1 -1
- package/dist/src/simctl.js +1 -0
- package/dist/src/simulator-state.js +1 -0
- package/dist/src/simulator.js +1 -1
- package/dist/src/snapshot-runtime.js +1 -1
- package/dist/src/snapshot-session.js +1 -1
- package/dist/src/snapshot.js +8 -1
- package/dist/src/snapshot2.js +1 -1
- package/dist/src/snapshot3.js +1 -0
- package/dist/src/src.js +1 -1
- package/dist/src/src2.js +1 -2
- package/dist/src/src3.js +2 -1
- package/dist/src/string-enum.js +1 -0
- package/dist/src/tool-provider.js +3 -3
- package/dist/src/tool-provider2.js +1 -1
- package/dist/src/update-check.js +1 -1
- package/dist/src/verdict.js +1 -0
- package/dist/src/verified-file.js +1 -0
- package/dist/src/version.js +1 -1
- package/dist/src/video.js +23 -0
- package/package.json +8 -6
- package/server.json +3 -3
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +0 -1
- package/dist/src/context.js +0 -1
- package/dist/src/interaction-touch-reference-frame.js +0 -1
- package/dist/src/notifications.js +0 -1
- package/dist/src/platform-runtime-toolchain-host.js +0 -1
- package/dist/src/runner-provider.js +0 -1
- package/dist/src/text-surface.js +0 -1
- package/dist/src/tool-provider3.js +0 -1
- package/skills/agent-device/SKILL.md +0 -20
- package/skills/dogfood/SKILL.md +0 -27
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
<a href="https://www.callstack.com/open-source?utm_campaign=generic&utm_source=github&utm_medium=referral&utm_content=agent-device" align="center">
|
|
2
2
|
<picture>
|
|
3
|
-
<img alt="agent-device:
|
|
3
|
+
<img alt="agent-device: mobile app automation and verification for AI coding agents" src="website/docs/public/agent-device-banner.jpg">
|
|
4
4
|
</picture>
|
|
5
5
|
</a>
|
|
6
6
|
|
|
@@ -13,13 +13,13 @@
|
|
|
13
13
|
[](LICENSE)
|
|
14
14
|
[](https://glama.ai/mcp/servers/callstack/agent-device)
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
**Mobile app automation and verification for AI coding agents.** Give coding agents a live app feedback loop through a CLI, built-in MCP server, or typed Node.js API.
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
<!-- Intro rule: category line, job pitch with the platform list, works-with/proof line, nothing else. Per-platform transports, tool names, vendor lists, and support caveats belong in "How it works" and website/docs, never here. Every name in the proof line must link to public evidence. -->
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
Let your coding agent verify its changes in the running app. `agent-device` lets agents inspect, control, debug, and verify apps on iOS, Android, and HarmonyOS (simulators, emulators, and physical devices), plus tvOS, Android TV, Amazon Vega OS TV (Vega Virtual Device), web, macOS, and Linux. Agents read token-efficient accessibility snapshots instead of reasoning over screenshots alone, act through refs and selectors, and save evidence for review. It also coordinates device access across parallel agent worktrees and connects to remote device clouds.
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
Works with Claude Code, Codex, Cursor, Windsurf, Cline, Goose, and any agent that can run a CLI or connect over MCP, or as the runtime under agents you build with the [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk) or [Eve](https://oss.callstack.com/agent-device/docs/eve). Developers at [Expensify](https://www.callstack.com/blog/how-expensify-uses-agent-device-for-mobile-bug-evidence-and-profiling), [Shopify](https://x.com/mustafa01ali/status/2036577353178943826), and [others](#who-uses-agent-device) use it to verify their apps.
|
|
23
23
|
|
|
24
24
|
## Quick start
|
|
25
25
|
|
|
@@ -31,7 +31,9 @@ agent-device doctor
|
|
|
31
31
|
agent-device help workflow
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
Run `
|
|
34
|
+
Run `doctor` yourself before handing the CLI to an agent; `help workflow` links to the guides for debugging, replay, and profiling, and the installed help always matches the installed version.
|
|
35
|
+
|
|
36
|
+
### Drive an app from the CLI
|
|
35
37
|
|
|
36
38
|
Add a contact in the built-in iOS Contacts app:
|
|
37
39
|
|
|
@@ -59,13 +61,46 @@ agent-device screenshot ./contact-form.png
|
|
|
59
61
|
agent-device close
|
|
60
62
|
```
|
|
61
63
|
|
|
62
|
-
|
|
64
|
+
Refs are only valid from the latest output: after a `--settle` command, use the refs in its diff, and take a new snapshot only if the diff omits what you need. Snapshots come from the app's accessibility tree, so clear labels, roles, and test IDs make agent runs more reliable; use screenshots and video as evidence or when accessibility data is poor.
|
|
63
65
|
|
|
64
|
-
|
|
66
|
+

|
|
65
67
|
|
|
66
|
-
|
|
68
|
+
### Add MCP tools to your agent
|
|
67
69
|
|
|
68
|
-
|
|
70
|
+
`agent-device mcp` starts the official stdio MCP server, exposing the installed commands as structured tools over the same execution path as the CLI:
|
|
71
|
+
|
|
72
|
+
```json
|
|
73
|
+
{
|
|
74
|
+
"mcpServers": {
|
|
75
|
+
"agent-device": {
|
|
76
|
+
"command": "agent-device",
|
|
77
|
+
"args": ["mcp"]
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
See [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) for per-client setup and when to prefer plain CLI over MCP.
|
|
84
|
+
|
|
85
|
+
### Script it from Node.js
|
|
86
|
+
|
|
87
|
+
`createAgentDeviceClient()` gives Node.js code typed access to the same commands, as model tools in your own agent or from orchestration code:
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
import { createAgentDeviceClient } from 'agent-device';
|
|
91
|
+
|
|
92
|
+
const client = createAgentDeviceClient({ session: 'qa-run' });
|
|
93
|
+
try {
|
|
94
|
+
await client.apps.open({ app: 'com.apple.Preferences', platform: 'ios' });
|
|
95
|
+
const snapshot = await client.capture.snapshot({ interactiveOnly: true });
|
|
96
|
+
const button = snapshot.nodes.find((node) => node.role === 'button');
|
|
97
|
+
if (button) await client.interactions.press({ ref: button.ref });
|
|
98
|
+
} finally {
|
|
99
|
+
await client.sessions.close();
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
See the [Node.js API](https://oss.callstack.com/agent-device/docs/client-api), the [runnable examples](https://github.com/callstack/agent-device/tree/main/examples/sdk), and the [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk) and [Eve](https://oss.callstack.com/agent-device/docs/eve) integration guides.
|
|
69
104
|
|
|
70
105
|
## What agents can do
|
|
71
106
|
|
|
@@ -78,58 +113,86 @@ See [Commands](https://oss.callstack.com/agent-device/docs/commands) for the com
|
|
|
78
113
|
|
|
79
114
|

|
|
80
115
|
|
|
81
|
-
##
|
|
82
|
-
|
|
83
|
-
- **Set up your agent**: run the CLI from Cursor, Codex, Claude Code, Windsurf, or another agent terminal. See [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) for skills, rules, MCP tools, and setup for each client.
|
|
84
|
-
- **Try the sample app**: clone the repo and run the bundled Expo test app. [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start) covers a guided run with screenshots, replay, and performance data.
|
|
85
|
-
- **Build repeatable tests**: use [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) to repeat tests. Use [Debugging & Profiling](https://oss.callstack.com/agent-device/docs/debugging-profiling) to find bugs.
|
|
116
|
+
## What to ask your agent
|
|
86
117
|
|
|
87
|
-
|
|
118
|
+
With the CLI installed, prompts like these work end to end:
|
|
88
119
|
|
|
89
|
-
|
|
120
|
+
- "Implement the onboarding screen, run it on the iOS simulator and Android emulator, and attach screenshots."
|
|
121
|
+
- "Reproduce this crash and capture the logs that lead up to it."
|
|
122
|
+
- "Check whether this change causes unnecessary React Native re-renders."
|
|
123
|
+
- "Explore the checkout flow once, save it as a replay script, and run it in CI."
|
|
124
|
+
- "Verify this pull request on a physical device and attach reviewable evidence."
|
|
90
125
|
|
|
91
|
-
|
|
92
|
-
- [Agent Device: iOS & Android automation for AI agents](https://www.callstack.com/blog/agent-device-ai-native-mobile-automation-for-ios-android)
|
|
93
|
-
- [Building mobile QA agents with Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve)
|
|
94
|
-
- [How we optimized Agent Device for mobile app automation](https://www.callstack.com/blog/how-we-optimized-agent-device-for-mobile-app-automation)
|
|
95
|
-
|
|
96
|
-
### Videos
|
|
126
|
+
## Next steps
|
|
97
127
|
|
|
98
|
-
- [
|
|
99
|
-
- [
|
|
100
|
-
- [
|
|
128
|
+
- [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup): skills, project rules, and per-client setup for Cursor, Codex, Claude Code, Windsurf, and others.
|
|
129
|
+
- [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start): a guided run on the bundled Expo test app with screenshots, replay, and performance data.
|
|
130
|
+
- [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) and [Debugging & Profiling](https://oss.callstack.com/agent-device/docs/debugging-profiling): repeatable tests and bug hunting.
|
|
101
131
|
|
|
102
132
|
## Where to run agent-device
|
|
103
133
|
|
|
134
|
+
The same session and evidence model works at every step: the agent explores the app, captures evidence, saves a replay, runs it in CI, and moves onto remote devices.
|
|
135
|
+
|
|
104
136
|
| Path | Best for | Start with |
|
|
105
137
|
| --- | --- | --- |
|
|
106
138
|
| Local | Trying commands and debugging apps on simulators, emulators, physical devices, macOS, and Linux. | Follow the Quick Start. |
|
|
107
|
-
| CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml).
|
|
108
|
-
| Cloud / remote | Linux runners, managed devices, and remote jobs. |
|
|
139
|
+
| CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). |
|
|
140
|
+
| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. |
|
|
109
141
|
|
|
110
142
|
## How it works
|
|
111
143
|
|
|
112
144
|
`agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux.
|
|
113
145
|
|
|
114
|
-
|
|
146
|
+
Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
|
|
147
|
+
|
|
148
|
+
Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds).
|
|
149
|
+
|
|
150
|
+
`agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system.
|
|
115
151
|
|
|
116
152
|
## FAQ
|
|
117
153
|
|
|
118
154
|
### What is agent-device?
|
|
119
155
|
|
|
120
|
-
`agent-device` is a command-line tool that lets coding agents inspect, control, and verify apps and save evidence for review. It supports iOS, Android, HarmonyOS, TV, web, macOS, and Linux.
|
|
156
|
+
`agent-device` is a command-line tool and MCP server that lets AI coding agents inspect, control, and verify mobile apps and save evidence for review. It supports iOS, Android, HarmonyOS, TV, web, macOS, and Linux.
|
|
157
|
+
|
|
158
|
+
### Is there an MCP server for mobile app automation?
|
|
159
|
+
|
|
160
|
+
Yes. `agent-device mcp` starts the official stdio MCP server. The Quick start above has the client config, and [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup) covers per-client details.
|
|
121
161
|
|
|
122
162
|
### Does it work with React Native, Expo, Flutter, and native apps?
|
|
123
163
|
|
|
124
164
|
Yes. `agent-device` supports native iOS and Android apps, plus React Native, Expo, and Flutter apps on supported targets. The commands and evidence vary by target.
|
|
125
165
|
|
|
166
|
+
### How is it different from mobile MCP servers?
|
|
167
|
+
|
|
168
|
+
The MCP server is one entry point to the same runtime used by the CLI and typed Node.js API. Sessions, device ownership, selectors, evidence, replay, CI workflows, and cloud routing stay consistent across all three.
|
|
169
|
+
|
|
170
|
+
### Can I build my own agent or QA product on agent-device?
|
|
171
|
+
|
|
172
|
+
Yes. The typed Node.js client is a public surface over that same runtime, so an agent you build inherits everything above. Start from the [Node.js API](https://oss.callstack.com/agent-device/docs/client-api), [AI SDK](https://oss.callstack.com/agent-device/docs/ai-sdk), or [Eve](https://oss.callstack.com/agent-device/docs/eve) guides.
|
|
173
|
+
|
|
126
174
|
### How is it different from Appium, Detox, or Maestro?
|
|
127
175
|
|
|
128
176
|
With `agent-device`, an agent reads app state and chooses each command at run time. Teams use Appium, Detox, and Maestro to write and maintain test suites. `agent-device` can complement them by saving its runs as `.ad` scripts or exporting them as strict Maestro YAML.
|
|
129
177
|
|
|
130
178
|
### Can agent-device run in CI?
|
|
131
179
|
|
|
132
|
-
Yes. Record a run as an `.ad` script, replay it
|
|
180
|
+
Yes. Record a run as an `.ad` script, replay it in CI, and keep the screenshots and logs as artifacts; the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml) is a working example.
|
|
181
|
+
|
|
182
|
+
## Articles and videos
|
|
183
|
+
|
|
184
|
+
### Articles
|
|
185
|
+
|
|
186
|
+
- [Build an AI QA agent for Expo apps with EAS Workflows](https://expo.dev/blog/build-an-ai-qa-agent-for-expo-apps-with-eas-workflows-in-minutes-today)
|
|
187
|
+
- [Agent Device: iOS & Android automation for AI agents](https://www.callstack.com/blog/agent-device-ai-native-mobile-automation-for-ios-android)
|
|
188
|
+
- [Building mobile QA agents with Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve)
|
|
189
|
+
- [How we optimized Agent Device for mobile app automation](https://www.callstack.com/blog/how-we-optimized-agent-device-for-mobile-app-automation)
|
|
190
|
+
|
|
191
|
+
### Videos
|
|
192
|
+
|
|
193
|
+
- [Verifying mobile apps with agent-device](https://youtu.be/kZDU-k5r9kE)
|
|
194
|
+
- [Using agent-device in an AI coding workflow](https://youtu.be/dfVG_aNPkW4)
|
|
195
|
+
- [Cloud agents that test mobile apps on real devices](https://youtu.be/r5P0detC4bs?is=_KB6SZbLFRB1au_z)
|
|
133
196
|
|
|
134
197
|
## Who uses agent-device?
|
|
135
198
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
f98ded51f104c10268a75b7eb77f2e479461b844e49f238593d44b821d13e294 agent-device-android-ime-helper-0.20.9.apk
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-ime-helper",
|
|
3
|
-
"version": "0.20.
|
|
4
|
-
"assetName": "agent-device-android-ime-helper-0.20.
|
|
5
|
-
"sha256": "
|
|
3
|
+
"version": "0.20.9",
|
|
4
|
+
"assetName": "agent-device-android-ime-helper-0.20.9.apk",
|
|
5
|
+
"sha256": "f98ded51f104c10268a75b7eb77f2e479461b844e49f238593d44b821d13e294",
|
|
6
6
|
"packageName": "com.callstack.agentdevice.imehelper",
|
|
7
|
-
"versionCode":
|
|
7
|
+
"versionCode": 20009,
|
|
8
8
|
"serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
|
|
9
9
|
"broadcastProtocol": "android-ime-helper-v1"
|
|
10
10
|
}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
a5169f62fb38bb1980cf7c7963542f2bc5bbaafd4d9dc7b5b23d3e41ee782725 agent-device-android-snapshot-helper-0.20.9.apk
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-snapshot-helper",
|
|
3
|
-
"version": "0.20.
|
|
4
|
-
"releaseTag": "v0.20.
|
|
5
|
-
"assetName": "agent-device-android-snapshot-helper-0.20.
|
|
3
|
+
"version": "0.20.9",
|
|
4
|
+
"releaseTag": "v0.20.9",
|
|
5
|
+
"assetName": "agent-device-android-snapshot-helper-0.20.9.apk",
|
|
6
6
|
"apkUrl": null,
|
|
7
|
-
"sha256": "
|
|
8
|
-
"checksumName": "agent-device-android-snapshot-helper-0.20.
|
|
7
|
+
"sha256": "a5169f62fb38bb1980cf7c7963542f2bc5bbaafd4d9dc7b5b23d3e41ee782725",
|
|
8
|
+
"checksumName": "agent-device-android-snapshot-helper-0.20.9.apk.sha256",
|
|
9
9
|
"packageName": "com.callstack.agentdevice.snapshothelper",
|
|
10
|
-
"versionCode":
|
|
10
|
+
"versionCode": 20009,
|
|
11
11
|
"instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
|
|
12
12
|
"minSdk": 23,
|
|
13
13
|
"targetSdk": 36,
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h
CHANGED
|
@@ -97,6 +97,9 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey;
|
|
|
97
97
|
/// adds no work", which is only observable as this counter standing still.
|
|
98
98
|
+ (NSInteger)customActionReadDispatchCount;
|
|
99
99
|
|
|
100
|
+
/// Total reads refused by single-flight admission.
|
|
101
|
+
+ (NSInteger)customActionReadBlockedCount;
|
|
102
|
+
|
|
100
103
|
/// The shared AX client (`XCUIDevice.accessibilityInterface`), or nil when the
|
|
101
104
|
/// private interface is unavailable.
|
|
102
105
|
+ (nullable id)accessibilityClient;
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m
CHANGED
|
@@ -63,6 +63,9 @@ static atomic_int RunnerAXCustomActionReadsInFlight = 0;
|
|
|
63
63
|
/// staying put, so it is recorded rather than inferred.
|
|
64
64
|
static atomic_long RunnerAXCustomActionReadDispatches = 0;
|
|
65
65
|
|
|
66
|
+
/// Total single-flight admission refusals.
|
|
67
|
+
static atomic_long RunnerAXCustomActionReadBlocked = 0;
|
|
68
|
+
|
|
66
69
|
/// Output caps. The element budget bounds how many elements we read; these
|
|
67
70
|
/// bound what any ONE element can put in the response, so a pathological app
|
|
68
71
|
/// cannot spend the whole snapshot on one node's action list.
|
|
@@ -514,17 +517,15 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
514
517
|
if (![axClient respondsToSelector:selector]) {
|
|
515
518
|
return nil;
|
|
516
519
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_CUSTOM_ACTIONS_READ_BLOCKED");
|
|
520
|
+
int expectedReadsInFlight = 0;
|
|
521
|
+
if (!atomic_compare_exchange_strong(
|
|
522
|
+
&RunnerAXCustomActionReadsInFlight, &expectedReadsInFlight, 1)) {
|
|
523
|
+
atomic_fetch_add(&RunnerAXCustomActionReadBlocked, 1);
|
|
522
524
|
return nil;
|
|
523
525
|
}
|
|
524
526
|
// The AX call is a synchronous XPC round trip with no timeout of its own, so
|
|
525
527
|
// it runs off-thread behind a bounded wait. On timeout the result box is
|
|
526
528
|
// never read, so a late-returning wedged call cannot race this thread.
|
|
527
|
-
atomic_fetch_add(&RunnerAXCustomActionReadsInFlight, 1);
|
|
528
529
|
atomic_fetch_add(&RunnerAXCustomActionReadDispatches, 1);
|
|
529
530
|
NSMutableArray *box = [NSMutableArray array];
|
|
530
531
|
dispatch_semaphore_t finished = dispatch_semaphore_create(0);
|
|
@@ -543,7 +544,7 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
543
544
|
}
|
|
544
545
|
// Clear in-flight BEFORE signalling, so a waiter that wakes on this signal
|
|
545
546
|
// never observes its own completed read as still outstanding.
|
|
546
|
-
|
|
547
|
+
atomic_store(&RunnerAXCustomActionReadsInFlight, 0);
|
|
547
548
|
dispatch_semaphore_signal(finished);
|
|
548
549
|
});
|
|
549
550
|
long waited = dispatch_semaphore_wait(
|
|
@@ -622,6 +623,11 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
622
623
|
return (NSInteger)atomic_load(&RunnerAXCustomActionReadDispatches);
|
|
623
624
|
}
|
|
624
625
|
|
|
626
|
+
+ (NSInteger)customActionReadBlockedCount
|
|
627
|
+
{
|
|
628
|
+
return (NSInteger)atomic_load(&RunnerAXCustomActionReadBlocked);
|
|
629
|
+
}
|
|
630
|
+
|
|
625
631
|
+ (nullable id)accessibilityClient
|
|
626
632
|
{
|
|
627
633
|
return [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"];
|
|
@@ -175,15 +175,10 @@ extension RunnerTests {
|
|
|
175
175
|
|
|
176
176
|
let rootFrame = privateAXRect(root["frame"])
|
|
177
177
|
let viewport = privateAXSnapshotViewport(app: app, rootFrame: rootFrame)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
root,
|
|
181
|
-
to: &nodes,
|
|
178
|
+
let nodes = privateAXPresentation(
|
|
179
|
+
rawRoot: root,
|
|
182
180
|
options: options,
|
|
183
|
-
viewport: viewport
|
|
184
|
-
depth: 0,
|
|
185
|
-
parentIndex: nil,
|
|
186
|
-
insideMatchedScope: false
|
|
181
|
+
viewport: viewport
|
|
187
182
|
)
|
|
188
183
|
if nodes.count <= 1 {
|
|
189
184
|
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count)
|
|
@@ -250,110 +245,7 @@ extension RunnerTests {
|
|
|
250
245
|
}
|
|
251
246
|
}
|
|
252
247
|
|
|
253
|
-
|
|
254
|
-
_ rawNode: [String: Any],
|
|
255
|
-
to nodes: inout [SnapshotNode],
|
|
256
|
-
options: SnapshotOptions,
|
|
257
|
-
viewport: CGRect,
|
|
258
|
-
depth: Int,
|
|
259
|
-
parentIndex: Int?,
|
|
260
|
-
insideMatchedScope: Bool
|
|
261
|
-
) {
|
|
262
|
-
if let limit = options.depth, depth > limit { return }
|
|
263
|
-
|
|
264
|
-
let rect = privateAXRect(rawNode["frame"])
|
|
265
|
-
let label = privateAXString(rawNode["label"])
|
|
266
|
-
let identifier = privateAXString(rawNode["identifier"])
|
|
267
|
-
let value = privateAXString(rawNode["value"])
|
|
268
|
-
let rawType = privateAXInt(rawNode["type"]) ?? 0
|
|
269
|
-
let typeName = elementTypeName(rawElementType: rawType)
|
|
270
|
-
let enabled = privateAXBool(rawNode["enabled"]) ?? true
|
|
271
|
-
let visible = isVisibleInViewport(rect, viewport)
|
|
272
|
-
let interactiveCandidate = privateAXInteractiveCandidate(rawElementType: rawType)
|
|
273
|
-
let filterDecision = flatSnapshotFilterDecision(
|
|
274
|
-
FlatSnapshotFilterNode(
|
|
275
|
-
isRoot: parentIndex == nil,
|
|
276
|
-
label: label,
|
|
277
|
-
identifier: identifier,
|
|
278
|
-
valueText: value.isEmpty ? nil : value,
|
|
279
|
-
visible: visible
|
|
280
|
-
),
|
|
281
|
-
options: options,
|
|
282
|
-
insideMatchedScope: insideMatchedScope
|
|
283
|
-
)
|
|
284
|
-
let include = filterDecision.include
|
|
285
|
-
let nowInsideScope = filterDecision.insideMatchedScope
|
|
286
|
-
|
|
287
|
-
let currentIndex: Int?
|
|
288
|
-
if include {
|
|
289
|
-
currentIndex = nodes.count
|
|
290
|
-
nodes.append(
|
|
291
|
-
SnapshotNode(
|
|
292
|
-
index: nodes.count,
|
|
293
|
-
type: typeName,
|
|
294
|
-
label: label.isEmpty ? nil : label,
|
|
295
|
-
identifier: identifier.isEmpty ? nil : identifier,
|
|
296
|
-
value: value.isEmpty ? nil : value,
|
|
297
|
-
rect: snapshotRect(from: rect),
|
|
298
|
-
enabled: enabled,
|
|
299
|
-
focused: privateAXBool(rawNode["focused"]) == true ? true : nil,
|
|
300
|
-
selected: privateAXBool(rawNode["selected"]) == true ? true : nil,
|
|
301
|
-
hittable: visible && enabled && interactiveCandidate,
|
|
302
|
-
depth: depth,
|
|
303
|
-
parentIndex: parentIndex,
|
|
304
|
-
hiddenContentAbove: nil,
|
|
305
|
-
hiddenContentBelow: nil,
|
|
306
|
-
actions: rawNode["actions"] as? [String]
|
|
307
|
-
)
|
|
308
|
-
)
|
|
309
|
-
} else {
|
|
310
|
-
currentIndex = parentIndex
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
guard let children = rawNode["children"] as? [[String: Any]] else {
|
|
314
|
-
return
|
|
315
|
-
}
|
|
316
|
-
for child in children {
|
|
317
|
-
appendPrivateAXNode(
|
|
318
|
-
child,
|
|
319
|
-
to: &nodes,
|
|
320
|
-
options: options,
|
|
321
|
-
viewport: viewport,
|
|
322
|
-
depth: depth + 1,
|
|
323
|
-
parentIndex: currentIndex,
|
|
324
|
-
insideMatchedScope: nowInsideScope
|
|
325
|
-
)
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
private func elementTypeName(rawElementType: Int) -> String {
|
|
330
|
-
if let type = flatSnapshotElementType(rawElementType: rawElementType) {
|
|
331
|
-
return elementTypeName(type)
|
|
332
|
-
}
|
|
333
|
-
return "Element(\(rawElementType))"
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
private func privateAXString(_ value: Any?) -> String {
|
|
337
|
-
guard let value else { return "" }
|
|
338
|
-
if let string = value as? String {
|
|
339
|
-
return string.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
340
|
-
}
|
|
341
|
-
return String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
private func privateAXInt(_ value: Any?) -> Int? {
|
|
345
|
-
if let value = value as? Int { return value }
|
|
346
|
-
if let value = value as? NSNumber { return value.intValue }
|
|
347
|
-
return nil
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
private func privateAXBool(_ value: Any?) -> Bool? {
|
|
351
|
-
if let value = value as? Bool { return value }
|
|
352
|
-
if let value = value as? NSNumber { return value.boolValue }
|
|
353
|
-
return nil
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
private func privateAXRect(_ value: Any?) -> CGRect {
|
|
248
|
+
func privateAXRect(_ value: Any?) -> CGRect {
|
|
357
249
|
guard let frame = value as? [String: Any] else {
|
|
358
250
|
return .zero
|
|
359
251
|
}
|
|
@@ -761,7 +761,11 @@ extension RunnerTests {
|
|
|
761
761
|
}
|
|
762
762
|
case .recordStop:
|
|
763
763
|
guard let recorder = activeRecording else {
|
|
764
|
-
|
|
764
|
+
// The runner protocol is the durable cleanup primitive. A daemon may crash after the
|
|
765
|
+
// native stop succeeds but before it commits the resource transition, so exact-owner
|
|
766
|
+
// recovery must be able to repeat this command safely. Public `record stop` still owns
|
|
767
|
+
// its user-facing no-active validation through the daemon session manifest.
|
|
768
|
+
return Response(ok: true, data: DataPayload(message: "recording already stopped"))
|
|
765
769
|
}
|
|
766
770
|
do {
|
|
767
771
|
try recorder.stop()
|
|
@@ -18,10 +18,16 @@ struct FlatSnapshotFilterDecision {
|
|
|
18
18
|
let insideMatchedScope: Bool
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
enum FlatSnapshotVisibilityPolicy {
|
|
22
|
+
case interactiveOnly
|
|
23
|
+
case viewportProjected
|
|
24
|
+
}
|
|
25
|
+
|
|
21
26
|
extension RunnerTests {
|
|
22
27
|
func flatSnapshotFilterDecision(
|
|
23
28
|
_ node: FlatSnapshotFilterNode,
|
|
24
29
|
options: SnapshotOptions,
|
|
30
|
+
visibilityPolicy: FlatSnapshotVisibilityPolicy,
|
|
25
31
|
insideMatchedScope: Bool
|
|
26
32
|
) -> FlatSnapshotFilterDecision {
|
|
27
33
|
let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
@@ -39,7 +45,9 @@ extension RunnerTests {
|
|
|
39
45
|
include = true
|
|
40
46
|
} else if scopeActive && !nowInsideScope {
|
|
41
47
|
include = false
|
|
42
|
-
} else if
|
|
48
|
+
} else if !node.visible
|
|
49
|
+
&& (options.interactiveOnly || visibilityPolicy == .viewportProjected)
|
|
50
|
+
{
|
|
43
51
|
include = false
|
|
44
52
|
} else {
|
|
45
53
|
include = true
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
extension RunnerTests {
|
|
4
|
+
func privateAXPresentation(rawRoot: [String: Any], options: SnapshotOptions, viewport: CGRect)
|
|
5
|
+
-> [SnapshotNode]
|
|
6
|
+
{
|
|
7
|
+
var nodes: [SnapshotNode] = []
|
|
8
|
+
var hints: [Int: (above: Bool, below: Bool)] = [:]
|
|
9
|
+
appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
|
|
10
|
+
depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil)
|
|
11
|
+
return applyHiddenContentHints(hints, to: nodes)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [SnapshotNode],
|
|
15
|
+
hints: inout [Int: (above: Bool, below: Bool)], options: SnapshotOptions, viewport: CGRect,
|
|
16
|
+
depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
|
|
17
|
+
scrollContext: (index: Int, rect: CGRect)?)
|
|
18
|
+
{
|
|
19
|
+
if let limit = options.depth, depth > limit { return }
|
|
20
|
+
let rect = privateAXRect(raw["frame"])
|
|
21
|
+
let label = privateAXPresentationString(raw["label"])
|
|
22
|
+
let identifier = privateAXPresentationString(raw["identifier"])
|
|
23
|
+
let value = privateAXPresentationString(raw["value"])
|
|
24
|
+
let rawType = privateAXPresentationInt(raw["type"]) ?? 0
|
|
25
|
+
let enabled = privateAXPresentationBool(raw["enabled"]) ?? true
|
|
26
|
+
let children = raw["children"] as? [[String: Any]] ?? []
|
|
27
|
+
let elementType = flatSnapshotElementType(rawElementType: rawType)
|
|
28
|
+
let hasFrame = !rect.isNull && !rect.isEmpty
|
|
29
|
+
let onScreen = hasFrame
|
|
30
|
+
&& isVisibleInRegularSnapshot(
|
|
31
|
+
rect,
|
|
32
|
+
viewport: viewport,
|
|
33
|
+
scrollContainerAnchor: scrollContext
|
|
34
|
+
)
|
|
35
|
+
let presentationVisible = !hasFrame || onScreen
|
|
36
|
+
let decision = flatSnapshotFilterDecision(
|
|
37
|
+
FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier,
|
|
38
|
+
valueText: value.isEmpty ? nil : value, visible: presentationVisible),
|
|
39
|
+
options: options, visibilityPolicy: .viewportProjected,
|
|
40
|
+
insideMatchedScope: insideMatchedScope)
|
|
41
|
+
let include = decision.include
|
|
42
|
+
|
|
43
|
+
if !presentationVisible, let scrollContext {
|
|
44
|
+
rememberHiddenContentHint(for: rect, relativeTo: scrollContext, hints: &hints)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
let currentIndex: Int?
|
|
48
|
+
if include {
|
|
49
|
+
currentIndex = nodes.count
|
|
50
|
+
let typeName = elementType.map(elementTypeName) ?? "Element(\(rawType))"
|
|
51
|
+
nodes.append(SnapshotNode(index: nodes.count, type: typeName,
|
|
52
|
+
label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
|
|
53
|
+
value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
|
|
54
|
+
focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
|
|
55
|
+
selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
|
|
56
|
+
hittable: onScreen && enabled && privateAXInteractiveCandidate(rawElementType: rawType),
|
|
57
|
+
depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
|
|
58
|
+
actions: raw["actions"] as? [String]))
|
|
59
|
+
} else { currentIndex = parentIndex }
|
|
60
|
+
|
|
61
|
+
let nextScrollContext: (index: Int, rect: CGRect)?
|
|
62
|
+
if include, let elementType, let currentIndex {
|
|
63
|
+
nextScrollContext = scrollContainerAnchor(
|
|
64
|
+
for: elementType,
|
|
65
|
+
hasChildren: !children.isEmpty,
|
|
66
|
+
visible: onScreen,
|
|
67
|
+
frame: rect,
|
|
68
|
+
nodeIndex: currentIndex
|
|
69
|
+
) ?? scrollContext
|
|
70
|
+
} else { nextScrollContext = scrollContext }
|
|
71
|
+
for child in children {
|
|
72
|
+
appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport,
|
|
73
|
+
depth: depth + 1, parentIndex: currentIndex,
|
|
74
|
+
insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private func privateAXPresentationString(_ value: Any?) -> String {
|
|
79
|
+
guard let value else { return "" }
|
|
80
|
+
return (value as? String ?? String(describing: value))
|
|
81
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
82
|
+
}
|
|
83
|
+
private func privateAXPresentationInt(_ value: Any?) -> Int? {
|
|
84
|
+
(value as? Int) ?? (value as? NSNumber)?.intValue
|
|
85
|
+
}
|
|
86
|
+
private func privateAXPresentationBool(_ value: Any?) -> Bool? {
|
|
87
|
+
(value as? Bool) ?? (value as? NSNumber)?.boolValue
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|