agent-device 0.21.22 → 0.21.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.22.apk → agent-device-android-ime-helper-0.21.24.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.24.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.22.manifest.json → agent-device-android-ime-helper-0.21.24.manifest.json} +4 -4
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.24.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.24.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.22.manifest.json → agent-device-android-snapshot-helper-0.21.24.manifest.json} +6 -6
- package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +12 -6
- package/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/RectResponseTests.swift +21 -0
- package/apple/snapshot-bridge/README.md +9 -7
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +56 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +14 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +8 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +12 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +1 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +29 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +34 -10
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +19 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +3 -3
- package/dist/src/adapter.js +1 -1
- package/dist/src/adb.js +1 -1
- package/dist/src/agent-browser-lifecycle.js +1 -1
- package/dist/src/agent-browser-process-record.js +1 -1
- package/dist/src/agent-browser-provider.js +1 -1
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/alert-runtime.js +1 -1
- package/dist/src/alert.js +1 -1
- package/dist/src/android-adb.d.ts +3 -2
- package/dist/src/android-input-ownership.js +1 -1
- package/dist/src/app-backend.js +1 -1
- package/dist/src/app-inventory.d.ts +4 -0
- package/dist/src/app-launch.js +1 -1
- package/dist/src/app-lifecycle.js +1 -1
- package/dist/src/app-log-reconnect.js +1 -1
- package/dist/src/app-log-runtime2.js +3 -3
- package/dist/src/app-resolution.js +2 -2
- package/dist/src/app-state-runtime.d.ts +15 -0
- package/dist/src/apple-runner-platform.js +1 -1
- package/dist/src/application-lifecycle-interaction.js +1 -1
- package/dist/src/aws-device-farm-connection.js +1 -0
- package/dist/src/back-mode.js +1 -1
- package/dist/src/backend-snapshot-options.js +1 -1
- package/dist/src/browserstack-connection.js +1 -0
- package/dist/src/browserstack-device-features.js +1 -0
- package/dist/src/browserstack.js +1 -0
- package/dist/src/cache.js +1 -1
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +35 -19
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +3 -4
- package/dist/src/client-companion-tunnel.js +1 -0
- package/dist/src/client-metro-companion.js +1 -1
- package/dist/src/client-metro.js +1 -1
- package/dist/src/client-options.js +3 -3
- package/dist/src/client-types.d.ts +7 -5
- package/dist/src/cloud-artifacts.d.ts +9 -2
- package/dist/src/code-signature-cache.js +1 -1
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +1 -1
- package/dist/src/command-tools.js +2 -2
- package/dist/src/connection-runtime.js +1 -1
- package/dist/src/connection-verification.js +1 -0
- package/dist/src/connection.js +5 -5
- package/dist/src/contracts.d.ts +1 -1
- package/dist/src/daemon-client-lease-beat.js +1 -1
- package/dist/src/daemon-client-lifecycle.js +1 -1
- package/dist/src/daemon-client-transport.js +1 -1
- package/dist/src/daemon-http.js +1 -1
- package/dist/src/daemon-policy-file.js +1 -1
- package/dist/src/daemon-registration-owner.js +1 -1
- package/dist/src/device-claim-inspection.js +1 -1
- package/dist/src/device-input-state.d.ts +2 -15
- package/dist/src/device-readiness-runtime.js +1 -1
- package/dist/src/device-selection-resolver.js +1 -1
- package/dist/src/device-session.js +3 -1
- package/dist/src/device.js +1 -1
- package/dist/src/device2.js +1 -1
- package/dist/src/device3.js +1 -1
- package/dist/src/dispatch-resolve.js +3 -3
- package/dist/src/doctor-facade.js +1 -1
- package/dist/src/durable-capture-resource.js +1 -1
- package/dist/src/durable-resource.js +1 -1
- package/dist/src/exec.js +1 -1
- package/dist/src/file.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/format.js +2 -1
- package/dist/src/gesture-admission.js +1 -1
- package/dist/src/gesture-plan-types.d.ts +2 -5
- package/dist/src/gesture-runtime.js +1 -1
- package/dist/src/helper.js +2 -2
- package/dist/src/human-control.js +1 -1
- package/dist/src/ime-rebind.js +1 -0
- package/dist/src/ime-settings-record.js +1 -0
- package/dist/src/ime-state.js +1 -0
- package/dist/src/index.d.ts +3 -3
- package/dist/src/input-actions.js +1 -1
- package/dist/src/input-actions2.js +1 -1
- package/dist/src/install-source2.js +1 -1
- package/dist/src/instance-client.js +1 -1
- package/dist/src/interaction-positionals.js +1 -1
- package/dist/src/interaction-ref-policy.js +1 -1
- package/dist/src/interaction.js +1 -1
- package/dist/src/interaction2.js +1 -1
- package/dist/src/interactor.js +5 -5
- package/dist/src/interactors.js +1 -1
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/internal/daemon.js +1 -1
- package/dist/src/inventory2.js +1 -1
- package/dist/src/ios-client.js +4 -4
- package/dist/src/ios-snapshot-acquisition.js +1 -1
- package/dist/src/ios-snapshot-engine.js +1 -1
- package/dist/src/ios-snapshot-runtime.js +1 -1
- package/dist/src/launch-confirmation.js +1 -1
- package/dist/src/lease-scope.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun.d.ts +4 -555
- package/dist/src/load.js +1 -1
- package/dist/src/local-interactor-operation-set.js +1 -1
- package/dist/src/log-stream.js +33 -0
- package/dist/src/managed-device-scope.js +1 -1
- package/dist/src/manifest.js +1 -0
- package/dist/src/mechanics.js +16 -16
- package/dist/src/mobile-snapshot-semantics.js +1 -1
- package/dist/src/node-http.js +1 -0
- package/dist/src/observation.js +1 -2
- package/dist/src/overlay.js +3 -3
- package/dist/src/owner-identity.js +3 -3
- package/dist/src/perf-facade.js +1 -1
- package/dist/src/perf-runtime-plan.js +1 -1
- package/dist/src/perf-target.js +1 -1
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/platform-request-scope.js +1 -1
- package/dist/src/platform-runtime-app-log-process.js +1 -1
- package/dist/src/platform-runtime-open-target.js +1 -1
- package/dist/src/platform-runtime-operation-host.js +4 -4
- package/dist/src/platform-runtime-runtime-hints.js +1 -1
- package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
- package/dist/src/platform-runtime-screen-recording-process-host.js +1 -1
- package/dist/src/platform-runtime-unavailable.js +1 -1
- package/dist/src/platform-runtime.js +1 -1
- package/dist/src/platform-runtime2.js +1 -1
- package/dist/src/plugin.js +1 -0
- package/dist/src/plugins/webdriver.d.ts +111 -0
- package/dist/src/plugins/webdriver.js +1 -0
- package/dist/src/plugins.d.ts +2 -1
- package/dist/src/plugins2.js +1 -1
- package/dist/src/process-exit.js +1 -1
- package/dist/src/process.js +1 -1
- package/dist/src/provider-credential-fingerprint.js +1 -1
- package/dist/src/provider-device-admission.js +1 -1
- package/dist/src/provider-device-runtime.d.ts +556 -0
- package/dist/src/provider-policy.js +1 -0
- package/dist/src/provider-profile-fields.js +1 -1
- package/dist/src/provider-webdriver.js +1 -2
- package/dist/src/providers.js +1 -1
- package/dist/src/proxy.js +2 -2
- package/dist/src/react-devtools.js +2 -0
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-runtime.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/recording.js +1 -1
- package/dist/src/registry.js +24 -24
- package/dist/src/remote-bridge.js +1 -0
- package/dist/src/remote-config-core.js +1 -0
- package/dist/src/remote-config.js +1 -1
- package/dist/src/remote-config2.js +1 -1
- package/dist/src/remote-connection-state.js +1 -0
- package/dist/src/replay.js +2 -2
- package/dist/src/request-admission.js +1 -1
- package/dist/src/request-runtime-binding.js +1 -1
- package/dist/src/result-serialization.js +1 -1
- package/dist/src/retry.js +1 -1
- package/dist/src/retry2.js +1 -0
- package/dist/src/runner-artifact-manifest.js +1 -1
- package/dist/src/runner-cache-metadata.js +2 -3
- package/dist/src/runner-cache-retention.js +1 -0
- package/dist/src/runner-cache-trim.js +1 -0
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-dev-tools-security.js +1 -1
- package/dist/src/runner-device-readiness.js +1 -1
- package/dist/src/runner-device-set.js +1 -0
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-error-classification.js +1 -1
- package/dist/src/runner-exchange.js +1 -1
- package/dist/src/runner-lease.js +1 -0
- package/dist/src/runner-operations-facade.js +1 -1
- package/dist/src/runner-session-types.js +1 -1
- package/dist/src/runner-start-budget.js +1 -1
- package/dist/src/runner-toolchain-probe.js +2 -0
- package/dist/src/runner.js +1 -1
- package/dist/src/runtime-deployment.js +1 -0
- package/dist/src/runtime.js +1 -1
- package/dist/src/runtime10.js +1 -2
- package/dist/src/runtime11.js +2 -0
- package/dist/src/runtime2.js +2 -5
- package/dist/src/runtime3.js +5 -1
- package/dist/src/runtime4.js +1 -5
- package/dist/src/runtime5.js +5 -1
- package/dist/src/runtime6.js +1 -1
- package/dist/src/runtime7.js +1 -1
- package/dist/src/runtime8.js +1 -1
- package/dist/src/runtime9.js +1 -1
- package/dist/src/screen-recording-transport.js +1 -1
- package/dist/src/screenshot-density.js +1 -1
- package/dist/src/screenshot-overlay-draw.js +1 -1
- package/dist/src/screenshot3.js +2 -2
- package/dist/src/script.js +1 -1
- package/dist/src/scroll-gesture.js +1 -1
- package/dist/src/sdk-batch.js +2 -2
- package/dist/src/sdk-contracts.d.ts +25 -33
- package/dist/src/sdk-contracts.js +2 -2
- package/dist/src/sdk-device.js +1 -1
- package/dist/src/sdk-finders.d.ts +1 -1
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/sdk-remote-config.d.ts +4 -0
- package/dist/src/sdk-remote-config.js +1 -1
- package/dist/src/sdk-selectors.d.ts +1 -1
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selection.js +0 -0
- package/dist/src/selector-pipeline-policy.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +2 -2
- package/dist/src/session-snapshot.js +1 -1
- package/dist/src/session2.js +5 -5
- package/dist/src/simctl.js +1 -1
- package/dist/src/simulator-boot.js +1 -1
- package/dist/src/simulator-state.js +1 -1
- package/dist/src/simulator.js +1 -1
- package/dist/src/snapshot-capture-annotations.js +1 -1
- package/dist/src/snapshot-desktop-projection.js +1 -1
- package/dist/src/snapshot-private-evidence.js +1 -0
- package/dist/src/snapshot-runtime-binding.js +1 -1
- package/dist/src/snapshot-runtime.js +1 -1
- package/dist/src/snapshot-state.js +1 -1
- package/dist/src/snapshot4.js +1 -1
- package/dist/src/src2.js +1 -1
- package/dist/src/src3.js +1 -1
- package/dist/src/src4.js +1 -1
- package/dist/src/src5.js +1 -1
- package/dist/src/src6.js +1 -1
- package/dist/src/src7.js +1 -1
- package/dist/src/src8.js +20 -20
- package/dist/src/store.js +1 -1
- package/dist/src/target-evidence.js +1 -1
- package/dist/src/text-entry-focus.js +1 -1
- package/dist/src/tool-provider2.js +1 -1
- package/dist/src/touch-runtime2.js +1 -1
- package/dist/src/tree.js +1 -1
- package/dist/src/type-text-runtime.js +1 -1
- package/dist/src/video.js +1 -1
- package/package.json +24 -16
- package/server.json +2 -2
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.22.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.22.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.22.apk.sha256 +0 -1
- package/dist/src/capabilities.js +0 -1
- package/dist/src/compatibility-policy.js +0 -1
- package/dist/src/daemon-artifacts.js +0 -1
- package/dist/src/device-claim-rule.js +0 -1
- package/dist/src/human-control-contract.js +0 -1
- package/dist/src/prepare-kind.js +0 -1
- package/dist/src/provider-device-runtimes.js +0 -1
- package/dist/src/screen-recording-admission-ledger.js +0 -1
- package/dist/src/screenshot-runtime.js +0 -1
- package/dist/src/session-routing.js +0 -1
- package/dist/src/session-runtime-admission.js +0 -1
- package/dist/src/session-stream.js +0 -27
- package/dist/src/timeouts.js +0 -1
- package/dist/src/touch-reference-frame.js +0 -1
- package/dist/src/vars.js +0 -2
- package/dist/src/viewport.js +0 -1
package/README.md
CHANGED
|
@@ -31,7 +31,7 @@ agent-device doctor
|
|
|
31
31
|
agent-device help workflow
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
Run `doctor` yourself before
|
|
34
|
+
Run `doctor` yourself before you hand the CLI to an agent. `help workflow` links to the guides for debugging, replay, and profiling.
|
|
35
35
|
|
|
36
36
|
### Drive an app from the CLI
|
|
37
37
|
|
|
@@ -61,7 +61,7 @@ agent-device screenshot ./contact-form.png
|
|
|
61
61
|
agent-device close
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
Refs are only
|
|
64
|
+
Refs are valid only in the latest output: after a `--settle` command, use the refs in its diff, and take a new snapshot only if the diff omits what you need. Snapshots come from the app's accessibility tree, so clear labels, roles, and test IDs make agent runs more reliable; use screenshots and video as evidence or when accessibility data is poor.
|
|
65
65
|
|
|
66
66
|

|
|
67
67
|
|
|
@@ -126,7 +126,7 @@ With the CLI installed, prompts like these work end to end:
|
|
|
126
126
|
## Next steps
|
|
127
127
|
|
|
128
128
|
- [AI Agent Setup](https://oss.callstack.com/agent-device/docs/agent-setup): skills, project rules, and per-client setup for Cursor, Codex, Claude Code, Windsurf, and others.
|
|
129
|
-
- [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start):
|
|
129
|
+
- [Quick Start](https://oss.callstack.com/agent-device/docs/quick-start): open an app, read its UI, and interact with it from the command line.
|
|
130
130
|
- [Replay & E2E](https://oss.callstack.com/agent-device/docs/replay-e2e) and [Debugging & Profiling](https://oss.callstack.com/agent-device/docs/debugging-profiling): repeatable tests and bug hunting.
|
|
131
131
|
|
|
132
132
|
## Where to run agent-device
|
|
@@ -137,15 +137,15 @@ The same session and evidence model works at every step: the agent explores the
|
|
|
137
137
|
| --- | --- | --- |
|
|
138
138
|
| Local | Trying commands and debugging apps on simulators, emulators, physical devices, macOS, and Linux. | Follow the Quick Start. |
|
|
139
139
|
| CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). |
|
|
140
|
-
| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. |
|
|
140
|
+
| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, TestMu AI, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. |
|
|
141
141
|
|
|
142
142
|
## How it works
|
|
143
143
|
|
|
144
144
|
`agent-device` keeps device state in sessions. It uses a local accessibility bridge for iOS Simulator snapshots and XCTest for iOS interactions, physical iOS, and tvOS; ADB and the snapshot helper on Android; HDC and ArkUI `uitest` on HarmonyOS; Vega CLI/VDA on the Vega Virtual Device; a local helper on macOS; and AT-SPI on Linux.
|
|
145
145
|
|
|
146
|
-
Support depth varies by target.
|
|
146
|
+
Support depth varies by target. Some backends, such as HarmonyOS and Vega OS, cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
|
|
147
147
|
|
|
148
|
-
Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators.
|
|
148
|
+
Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Run `agent-device device status` to inspect ownership without a daemon, and `agent-device device release --stale` to release claims whose owner is provably no longer running. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, TestMu AI, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds).
|
|
149
149
|
|
|
150
150
|
`agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system.
|
|
151
151
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ca5c3b48c9da40f1fcddd785f1129906125e4880549cfacb1b63ff70848aed1e agent-device-android-ime-helper-0.21.24.apk
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-ime-helper",
|
|
3
|
-
"version": "0.21.
|
|
4
|
-
"assetName": "agent-device-android-ime-helper-0.21.
|
|
5
|
-
"sha256": "
|
|
3
|
+
"version": "0.21.24",
|
|
4
|
+
"assetName": "agent-device-android-ime-helper-0.21.24.apk",
|
|
5
|
+
"sha256": "ca5c3b48c9da40f1fcddd785f1129906125e4880549cfacb1b63ff70848aed1e",
|
|
6
6
|
"packageName": "com.callstack.agentdevice.imehelper",
|
|
7
|
-
"versionCode":
|
|
7
|
+
"versionCode": 21024,
|
|
8
8
|
"serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
|
|
9
9
|
"broadcastProtocol": "android-ime-helper-v1"
|
|
10
10
|
}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
d3a6ab00e617da9fb019d21eb5a894e1f4699f263044f0ce387a5ea16e56ca47 agent-device-android-snapshot-helper-0.21.24.apk
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-snapshot-helper",
|
|
3
|
-
"version": "0.21.
|
|
4
|
-
"releaseTag": "v0.21.
|
|
5
|
-
"assetName": "agent-device-android-snapshot-helper-0.21.
|
|
3
|
+
"version": "0.21.24",
|
|
4
|
+
"releaseTag": "v0.21.24",
|
|
5
|
+
"assetName": "agent-device-android-snapshot-helper-0.21.24.apk",
|
|
6
6
|
"apkUrl": null,
|
|
7
|
-
"sha256": "
|
|
8
|
-
"checksumName": "agent-device-android-snapshot-helper-0.21.
|
|
7
|
+
"sha256": "d3a6ab00e617da9fb019d21eb5a894e1f4699f263044f0ce387a5ea16e56ca47",
|
|
8
|
+
"checksumName": "agent-device-android-snapshot-helper-0.21.24.apk.sha256",
|
|
9
9
|
"packageName": "com.callstack.agentdevice.snapshothelper",
|
|
10
|
-
"versionCode":
|
|
10
|
+
"versionCode": 21024,
|
|
11
11
|
"instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
|
|
12
12
|
"minSdk": 23,
|
|
13
13
|
"targetSdk": 36,
|
|
@@ -795,12 +795,18 @@ func rectAttribute(_ element: AXUIElement) -> RectResponse? {
|
|
|
795
795
|
return nil
|
|
796
796
|
}
|
|
797
797
|
|
|
798
|
-
return
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
798
|
+
return finiteRectResponse(position: position, size: size)
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/// JSONEncoder throws on inf/NaN, which fails the whole snapshot. SwiftUI reports lazy
|
|
802
|
+
/// containers scrolled out of view as an AXOpaqueProviderGroup at (inf, inf, 0, 0).
|
|
803
|
+
func finiteRectResponse(position: CGPoint, size: CGSize) -> RectResponse? {
|
|
804
|
+
let x = Double(position.x)
|
|
805
|
+
let y = Double(position.y)
|
|
806
|
+
let width = Double(size.width)
|
|
807
|
+
let height = Double(size.height)
|
|
808
|
+
guard x.isFinite, y.isFinite, width.isFinite, height.isFinite else { return nil }
|
|
809
|
+
return RectResponse(x: x, y: y, width: width, height: height)
|
|
804
810
|
}
|
|
805
811
|
|
|
806
812
|
private func accessibilityAxValue(_ value: CFTypeRef?) -> AXValue? {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
@testable import AgentDeviceMacOSHelper
|
|
4
|
+
|
|
5
|
+
final class RectResponseTests: XCTestCase {
|
|
6
|
+
func testFiniteRectKeepsItsValues() throws {
|
|
7
|
+
let rect = try XCTUnwrap(
|
|
8
|
+
finiteRectResponse(position: CGPoint(x: 1, y: 2), size: CGSize(width: 3, height: 4)))
|
|
9
|
+
XCTAssertEqual([rect.x, rect.y, rect.width, rect.height], [1, 2, 3, 4])
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
func testNonFiniteComponentYieldsNoRect() {
|
|
13
|
+
let nonFinite: [CGFloat] = [.infinity, -.infinity, .nan]
|
|
14
|
+
for value in nonFinite {
|
|
15
|
+
XCTAssertNil(finiteRectResponse(position: CGPoint(x: value, y: 0), size: .zero))
|
|
16
|
+
XCTAssertNil(finiteRectResponse(position: CGPoint(x: 0, y: value), size: .zero))
|
|
17
|
+
XCTAssertNil(finiteRectResponse(position: .zero, size: CGSize(width: value, height: 0)))
|
|
18
|
+
XCTAssertNil(finiteRectResponse(position: .zero, size: CGSize(width: 0, height: value)))
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -50,13 +50,15 @@ disabled after fallback until that app relaunches.
|
|
|
50
50
|
|
|
51
51
|
## Remote content
|
|
52
52
|
|
|
53
|
-
The reader snapshots one process.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
53
|
+
The reader snapshots one process. Two hosts put another process's UI into that
|
|
54
|
+
tree as an `AXRemoteElement` leaf: a WebKit page — Safari's or a `WKWebView`'s —
|
|
55
|
+
lives in a WebContent process under the web view, and a scene-hosted remote view
|
|
56
|
+
controller, such as a share extension presented over the app, lives in the
|
|
57
|
+
extension's process under a `_UISceneHostingView`. The guest returns the leaf as
|
|
58
|
+
delivered; the host refuses a tree in which such a leaf sits under either host
|
|
59
|
+
and reaches the viewport, or reports no frame (`remote-content-boundary`), and
|
|
60
|
+
routes that app generation to XCTest, which resolves remote elements (#2484). A
|
|
61
|
+
zero-area or off-screen leaf is published: it hosts nothing the capture can miss.
|
|
60
62
|
|
|
61
63
|
## Bounded depth recovery
|
|
62
64
|
|
|
@@ -114,6 +114,7 @@ extension RunnerTests {
|
|
|
114
114
|
let app: XCUIApplication
|
|
115
115
|
|
|
116
116
|
var systemSurface: SystemSurfaceHost? = nil
|
|
117
|
+
var observation: SnapshotObservationPayload? = nil
|
|
117
118
|
}
|
|
118
119
|
|
|
119
120
|
enum ActiveCommandPreparation {
|
|
@@ -494,6 +495,26 @@ extension RunnerTests {
|
|
|
494
495
|
command: Command,
|
|
495
496
|
routeToSpringboard: Bool = false
|
|
496
497
|
) -> ActiveCommandPreparation {
|
|
498
|
+
if command.command == .snapshot, command.observeOnly == true {
|
|
499
|
+
#if os(iOS)
|
|
500
|
+
guard command.appBundleId?.trimmedNonEmpty != nil else {
|
|
501
|
+
return .response(observeOnlyUnavailableResponse())
|
|
502
|
+
}
|
|
503
|
+
let observedApp = resolveAppWithoutActivation(command: command)
|
|
504
|
+
let reportedState = observedApp.state
|
|
505
|
+
guard reportedState == .runningForeground else {
|
|
506
|
+
return .response(observeOnlyUnavailableResponse())
|
|
507
|
+
}
|
|
508
|
+
return .context(ActiveCommandContext(
|
|
509
|
+
app: observedApp,
|
|
510
|
+
observation: SnapshotObservationPayload(appState: Self.applicationStateName(reportedState))
|
|
511
|
+
))
|
|
512
|
+
#else
|
|
513
|
+
return .response(Response(ok: false, error: ErrorPayload(
|
|
514
|
+
code: "UNSUPPORTED_OPERATION", message: "observe-only snapshot is supported on iOS and iPadOS only"
|
|
515
|
+
)))
|
|
516
|
+
#endif
|
|
517
|
+
}
|
|
497
518
|
if routeToSpringboard {
|
|
498
519
|
return .context(ActiveCommandContext(app: springboard))
|
|
499
520
|
}
|
|
@@ -516,11 +537,26 @@ extension RunnerTests {
|
|
|
516
537
|
|
|
517
538
|
|
|
518
539
|
|
|
540
|
+
|
|
541
|
+
|
|
519
542
|
return prepareActivatedTarget(command: command)
|
|
520
543
|
#endif
|
|
521
544
|
case .existingApp:
|
|
522
545
|
|
|
523
546
|
|
|
547
|
+
#if os(macOS)
|
|
548
|
+
if let bundleId = command.appBundleId?.trimmedNonEmpty,
|
|
549
|
+
macReadMayBeServedInBackground(
|
|
550
|
+
command: command,
|
|
551
|
+
targetState: XCUIApplication(bundleIdentifier: bundleId).state
|
|
552
|
+
)
|
|
553
|
+
{
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
refreshCachedTargetIfProcessChanged(bundleId: bundleId)
|
|
557
|
+
return .context(ActiveCommandContext(app: resolveAppWithoutActivation(command: command)))
|
|
558
|
+
}
|
|
559
|
+
#endif
|
|
524
560
|
return prepareActivatedTarget(command: command)
|
|
525
561
|
case .mayLaunch:
|
|
526
562
|
if shouldSkipAppActivationPreflight(command) {
|
|
@@ -644,6 +680,14 @@ extension RunnerTests {
|
|
|
644
680
|
return XCUIApplication(bundleIdentifier: host.bundleId).state
|
|
645
681
|
}
|
|
646
682
|
|
|
683
|
+
func observeOnlyUnavailableResponse() -> Response {
|
|
684
|
+
Response(ok: false, error: ErrorPayload(
|
|
685
|
+
code: "OBSERVATION_UNAVAILABLE",
|
|
686
|
+
message: "The session app cannot be observed without activation.",
|
|
687
|
+
hint: "XCUIApplication must report runningForeground before and after capture. This state report does not prove screen ownership (#2696)."
|
|
688
|
+
))
|
|
689
|
+
}
|
|
690
|
+
|
|
647
691
|
func currentXCTestFailureCount() -> Int {
|
|
648
692
|
return testRun?.failureCount ?? 0
|
|
649
693
|
}
|
|
@@ -746,4 +790,16 @@ extension RunnerTests {
|
|
|
746
790
|
}
|
|
747
791
|
return XCUIApplication(bundleIdentifier: bundleId)
|
|
748
792
|
}
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
func macReadMayBeServedInBackground(command: Command, targetState: XCUIApplication.State) -> Bool {
|
|
803
|
+
command.appBundleId?.trimmedNonEmpty != nil && targetState == .runningBackground
|
|
804
|
+
}
|
|
749
805
|
}
|
|
@@ -429,17 +429,26 @@ extension RunnerTests {
|
|
|
429
429
|
case .screenshot:
|
|
430
430
|
#if os(macOS)
|
|
431
431
|
|
|
432
|
-
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
let screenshotBundleId = command.appBundleId?.trimmedNonEmpty
|
|
436
|
+
if let bundleId = screenshotBundleId {
|
|
433
437
|
let targetApp = XCUIApplication(bundleIdentifier: bundleId)
|
|
434
|
-
|
|
435
|
-
|
|
438
|
+
if macAppCaptureNeedsRaise(
|
|
439
|
+
fullscreen: command.fullscreen,
|
|
440
|
+
targetState: targetApp.state
|
|
441
|
+
) {
|
|
442
|
+
targetApp.activate()
|
|
436
443
|
|
|
437
|
-
|
|
444
|
+
sleepFor(0.5)
|
|
445
|
+
}
|
|
446
|
+
activeApp = targetApp
|
|
438
447
|
}
|
|
439
448
|
let screenshot: XCUIScreenshot
|
|
440
449
|
if command.fullscreen == true {
|
|
441
450
|
screenshot = XCUIScreen.main.screenshot()
|
|
442
|
-
} else if
|
|
451
|
+
} else if screenshotBundleId != nil {
|
|
443
452
|
screenshot = screenshotRoot(app: activeApp).screenshot()
|
|
444
453
|
} else {
|
|
445
454
|
screenshot = XCUIScreen.main.screenshot()
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift
CHANGED
|
@@ -284,6 +284,14 @@ extension RunnerTests {
|
|
|
284
284
|
return false
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
func macAppCaptureNeedsRaise(fullscreen: Bool?, targetState: XCUIApplication.State) -> Bool {
|
|
291
|
+
guard targetState != .runningForeground else { return false }
|
|
292
|
+
return fullscreen != true || targetState == .notRunning
|
|
293
|
+
}
|
|
294
|
+
|
|
287
295
|
@MainActor
|
|
288
296
|
func canUseFastForegroundAppGuard(
|
|
289
297
|
activeApp: XCUIApplication,
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift
CHANGED
|
@@ -66,6 +66,9 @@ enum CommandLaunchPolicy: Equatable {
|
|
|
66
66
|
|
|
67
67
|
|
|
68
68
|
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
69
72
|
case existingApp
|
|
70
73
|
|
|
71
74
|
case mayLaunch
|
|
@@ -239,6 +242,7 @@ struct Command: Codable {
|
|
|
239
242
|
let interactiveOnly: Bool?
|
|
240
243
|
let preferredBackend: String?
|
|
241
244
|
let customActions: Bool?
|
|
245
|
+
let observeOnly: Bool?
|
|
242
246
|
let depth: Int?
|
|
243
247
|
let scope: String?
|
|
244
248
|
let raw: Bool?
|
|
@@ -488,6 +492,7 @@ struct DataPayload: Codable {
|
|
|
488
492
|
var failedStepIndex: Int?
|
|
489
493
|
var sequenceResults: [SequenceStepResult]?
|
|
490
494
|
var targetActivation: TargetActivationFactPayload?
|
|
495
|
+
var observation: SnapshotObservationPayload?
|
|
491
496
|
|
|
492
497
|
|
|
493
498
|
var screenshotMetadata: ScreenshotMetadataPayload?
|
|
@@ -529,6 +534,13 @@ struct SystemSurfaceProvenancePayload: Codable {
|
|
|
529
534
|
let kind: String
|
|
530
535
|
}
|
|
531
536
|
|
|
537
|
+
struct SnapshotObservationPayload: Codable {
|
|
538
|
+
var mode = "observe-only"
|
|
539
|
+
var activationPerformed = false
|
|
540
|
+
let appState: String
|
|
541
|
+
var appStateSource = "xcuiapplication-state"
|
|
542
|
+
}
|
|
543
|
+
|
|
532
544
|
struct SnapshotQualityPayload: Codable {
|
|
533
545
|
let nodes: [PresentedNode]
|
|
534
546
|
let truncated: Bool
|
|
@@ -17,7 +17,7 @@ struct SnapshotCaptureTarget {
|
|
|
17
17
|
|
|
18
18
|
enum SnapshotCommandPreparation {
|
|
19
19
|
case response(Response)
|
|
20
|
-
case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?)
|
|
20
|
+
case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?, observation: SnapshotObservationPayload?)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
|
|
@@ -19,19 +19,27 @@ extension RunnerTests {
|
|
|
19
19
|
return .response(response)
|
|
20
20
|
case .context(let context):
|
|
21
21
|
return .capture(
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
context.observation == nil
|
|
23
|
+
? self.takeSnapshotCaptureTarget(app: context.app)
|
|
24
|
+
: SnapshotCaptureTarget(
|
|
25
|
+
app: context.app,
|
|
26
|
+
bundleId: command.appBundleId?.trimmedNonEmpty,
|
|
27
|
+
processIdentifier: Self.processIdentifier(of: context.app)
|
|
28
|
+
),
|
|
29
|
+
systemSurface: context.systemSurface,
|
|
30
|
+
observation: context.observation
|
|
24
31
|
)
|
|
25
32
|
}
|
|
26
33
|
}
|
|
27
34
|
switch preparation {
|
|
28
35
|
case .response(let response):
|
|
29
36
|
return response
|
|
30
|
-
case .capture(let target, let systemSurface):
|
|
37
|
+
case .capture(let target, let systemSurface, let observation):
|
|
31
38
|
return try executeSnapshotPrepared(
|
|
32
39
|
command: command,
|
|
33
40
|
target: target,
|
|
34
|
-
systemSurface: systemSurface
|
|
41
|
+
systemSurface: systemSurface,
|
|
42
|
+
observation: observation
|
|
35
43
|
)
|
|
36
44
|
}
|
|
37
45
|
}
|
|
@@ -86,7 +94,8 @@ extension RunnerTests {
|
|
|
86
94
|
private func executeSnapshotPrepared(
|
|
87
95
|
command: Command,
|
|
88
96
|
target: SnapshotCaptureTarget,
|
|
89
|
-
systemSurface: SystemSurfaceHost
|
|
97
|
+
systemSurface: SystemSurfaceHost?,
|
|
98
|
+
observation: SnapshotObservationPayload?
|
|
90
99
|
) throws -> Response {
|
|
91
100
|
let options = Self.presentationOptions(from: command)
|
|
92
101
|
do {
|
|
@@ -102,6 +111,21 @@ extension RunnerTests {
|
|
|
102
111
|
kind: systemSurface.kind.rawValue
|
|
103
112
|
)
|
|
104
113
|
}
|
|
114
|
+
if let observation {
|
|
115
|
+
guard let capturedProcessIdentifier = target.processIdentifier else {
|
|
116
|
+
return observeOnlyUnavailableResponse()
|
|
117
|
+
}
|
|
118
|
+
let unchanged = try runMainThreadWork(
|
|
119
|
+
"observe_only_state",
|
|
120
|
+
timeout: Self.mainThreadExecutionTimeout,
|
|
121
|
+
timeoutError: Self.mainThreadExecutionTimeoutError
|
|
122
|
+
) {
|
|
123
|
+
target.app.state == .runningForeground
|
|
124
|
+
&& Self.processIdentifier(of: target.app) == capturedProcessIdentifier
|
|
125
|
+
}
|
|
126
|
+
guard unchanged else { return observeOnlyUnavailableResponse() }
|
|
127
|
+
payload.observation = observation
|
|
128
|
+
}
|
|
105
129
|
setNeedsPostSnapshotInteractionDelay()
|
|
106
130
|
return Response(ok: true, data: payload)
|
|
107
131
|
} catch let failure as SnapshotCaptureFailure {
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift
CHANGED
|
@@ -243,7 +243,7 @@ extension RunnerTests {
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
|
|
246
|
-
func clearTextInput(_ element: XCUIElement) {
|
|
246
|
+
func clearTextInput(app: XCUIApplication, _ element: XCUIElement) {
|
|
247
247
|
|
|
248
248
|
|
|
249
249
|
|
|
@@ -257,7 +257,7 @@ extension RunnerTests {
|
|
|
257
257
|
return
|
|
258
258
|
}
|
|
259
259
|
#if !os(tvOS)
|
|
260
|
-
moveCaretToEnd(element: element)
|
|
260
|
+
moveCaretToEnd(app: app, element: element)
|
|
261
261
|
#endif
|
|
262
262
|
let count = estimatedDeleteCount(for: element)
|
|
263
263
|
let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
|
|
@@ -358,6 +358,21 @@ extension RunnerTests {
|
|
|
358
358
|
}
|
|
359
359
|
|
|
360
360
|
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
func textEntrySnapshotFrame(_ element: XCUIElement) -> CGRect? {
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
safely("TEXT_ENTRY_SNAPSHOT_FRAME", { element.exists ? try? element.snapshot() : nil })?.frame
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
|
|
361
376
|
func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
|
|
362
377
|
var probe = TextEntryInputProbe.unavailable
|
|
363
378
|
let (_, exception) = catchingObjCException(fallback: ()) {
|
|
@@ -395,20 +410,29 @@ extension RunnerTests {
|
|
|
395
410
|
return nil
|
|
396
411
|
}
|
|
397
412
|
|
|
398
|
-
private func moveCaretToEnd(element: XCUIElement) {
|
|
413
|
+
private func moveCaretToEnd(app: XCUIApplication, element: XCUIElement) {
|
|
399
414
|
#if os(tvOS)
|
|
400
415
|
return
|
|
401
416
|
#else
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
guard let frame = textEntrySnapshotFrame(element), !frame.isEmpty else {
|
|
405
429
|
return
|
|
406
430
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
431
|
+
_ = tapAt(
|
|
432
|
+
app: app,
|
|
433
|
+
x: frame.minX + max(2, frame.width - 4),
|
|
434
|
+
y: frame.minY + max(2, frame.height / 2)
|
|
410
435
|
)
|
|
411
|
-
target.tap()
|
|
412
436
|
#endif
|
|
413
437
|
}
|
|
414
438
|
|
|
@@ -137,7 +137,7 @@ extension RunnerTests {
|
|
|
137
137
|
if keyboardVisibleBeforeTap, let target {
|
|
138
138
|
return TextEntryTarget(
|
|
139
139
|
element: target,
|
|
140
|
-
refreshPoint: textEntryRefreshPoint(for: target
|
|
140
|
+
refreshPoint: textEntryRefreshPoint(for: target, fallbackTo: requestedPoint),
|
|
141
141
|
prefersFocusedElement: false
|
|
142
142
|
)
|
|
143
143
|
}
|
|
@@ -157,18 +157,32 @@ extension RunnerTests {
|
|
|
157
157
|
: (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target)
|
|
158
158
|
return TextEntryTarget(
|
|
159
159
|
element: element,
|
|
160
|
-
refreshPoint: textEntryRefreshPoint(for: element
|
|
160
|
+
refreshPoint: textEntryRefreshPoint(for: element, fallbackTo: requestedPoint),
|
|
161
161
|
prefersFocusedElement: false
|
|
162
162
|
)
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
-
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
private func textEntryRefreshPoint(
|
|
175
|
+
for element: XCUIElement?,
|
|
176
|
+
fallbackTo requestedPoint: CGPoint?
|
|
177
|
+
) -> CGPoint? {
|
|
166
178
|
guard let element else {
|
|
179
|
+
return requestedPoint
|
|
180
|
+
}
|
|
181
|
+
guard let frame = textEntrySnapshotFrame(element) else {
|
|
167
182
|
return nil
|
|
168
183
|
}
|
|
169
|
-
let frame = element.frame
|
|
170
184
|
guard !frame.isEmpty else {
|
|
171
|
-
return
|
|
185
|
+
return requestedPoint
|
|
172
186
|
}
|
|
173
187
|
return CGPoint(x: frame.midX, y: frame.midY)
|
|
174
188
|
}
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift
CHANGED
|
@@ -34,7 +34,7 @@ extension RunnerTests {
|
|
|
34
34
|
failure: .notFocused
|
|
35
35
|
)
|
|
36
36
|
}
|
|
37
|
-
clearTextInput(clearTarget)
|
|
37
|
+
clearTextInput(app: app, clearTarget)
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
let observed = editableTextValue(for: clearTarget, treatingPlaceholderAsEmpty: true)
|
|
@@ -125,7 +125,7 @@ extension RunnerTests {
|
|
|
125
125
|
}
|
|
126
126
|
if currentText == nil || currentText?.isEmpty == false {
|
|
127
127
|
let clearStartedAt = Date()
|
|
128
|
-
clearTextInput(replacementTarget)
|
|
128
|
+
clearTextInput(app: app, replacementTarget)
|
|
129
129
|
activeTarget = activeTarget.withElement(replacementTarget)
|
|
130
130
|
logTextEntryPhase(
|
|
131
131
|
commandId: commandId,
|
|
@@ -465,7 +465,7 @@ extension RunnerTests {
|
|
|
465
465
|
expectedText.count,
|
|
466
466
|
observedText.count
|
|
467
467
|
)
|
|
468
|
-
clearTextInput(repairTarget)
|
|
468
|
+
clearTextInput(app: app, repairTarget)
|
|
469
469
|
repairTarget.typeText(expectedText)
|
|
470
470
|
let repairedTarget = target.withElement(repairTarget)
|
|
471
471
|
let repairedResult = verifyTextEntry(
|