agent-device 0.19.3 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -70
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.1.apk +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.1.apk.sha256 +1 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.1.manifest.json +10 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.1.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.1.apk.sha256 +1 -0
- package/{android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.manifest.json → android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.1.manifest.json} +6 -6
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +2 -0
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +5 -0
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +5 -10
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +74 -174
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +113 -19
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +82 -0
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +325 -123
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +2 -2
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +149 -105
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +56 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +73 -0
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +51 -26
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +2 -2
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +2 -107
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +130 -13
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +51 -8
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +24 -0
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +1 -1
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +43 -35
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +1 -1
- package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +16 -1
- package/dist/src/agent-browser-tool.js +1 -1
- package/dist/src/agent-device-client.js +3 -0
- package/dist/src/android-adb.d.ts +1 -1
- package/dist/src/android-adb.js +1 -1
- package/dist/src/android-input-method-overlays.js +1 -0
- package/dist/src/android-input-ownership.js +1 -6
- package/dist/src/android-system-chrome.js +1 -0
- package/dist/src/android.js +1 -1
- package/dist/src/app-lifecycle.js +5 -5
- package/dist/src/app-log-request-scope.js +1 -0
- package/dist/src/app-resolution.js +1 -0
- package/dist/src/apple-multitouch-support.js +1 -0
- package/dist/src/apps.js +5 -5
- package/dist/src/boot-diagnostics.js +2 -0
- package/dist/src/cli-config.js +1 -0
- package/dist/src/cli-help.js +131 -40
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +15 -11
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +3 -3
- package/dist/src/compatibility-policy.js +1 -0
- package/dist/src/context.js +1 -0
- package/dist/src/contracts.d.ts +2 -2
- package/dist/src/debug-symbols.js +1 -1
- package/dist/src/device-input-state.js +1 -1
- package/dist/src/devices2.js +1 -1
- package/dist/src/devices4.js +1 -0
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/generic.js +1 -1
- package/dist/src/helper.js +1 -1
- package/dist/src/index.d.ts +533 -279
- package/dist/src/index.js +1 -1
- package/dist/src/input-actions.js +4 -3
- package/dist/src/install-artifact.js +1 -0
- package/dist/src/install-source.d.ts +5 -1
- package/dist/src/install-source.js +1 -1
- package/dist/src/install-source2.js +1 -1
- package/dist/src/interaction-snapshot.js +1 -1
- package/dist/src/interaction.d.ts +230 -0
- package/dist/src/interaction.js +1 -1
- package/dist/src/interactor.js +2 -2
- package/dist/src/interactor2.js +1 -0
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/internal/daemon.js +42 -20
- package/dist/src/internal/png-worker.d.ts +22 -0
- package/dist/src/internal/png-worker.js +1 -1
- package/dist/src/launch-console.js +1 -0
- package/dist/src/lease-context.js +1 -0
- package/dist/src/lease.js +1 -1
- package/dist/src/linux.js +1 -1
- package/dist/src/logcat.js +5 -0
- package/dist/src/method-scope.js +1 -0
- package/dist/src/metro-types.d.ts +52 -0
- package/dist/src/metro.d.ts +3 -3
- package/dist/src/notifications.js +1 -1
- package/dist/src/png-worker-contract.js +1 -1
- package/dist/src/process-lock.js +1 -0
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/recording-export-quality.js +15 -0
- package/dist/src/register-builtins.js +1 -0
- package/dist/src/registry.js +24 -24
- package/dist/src/remote-config.d.ts +1 -1
- package/dist/src/reporting.js +8 -7
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-disposal.js +2 -0
- package/dist/src/runner-lease.js +1 -0
- package/dist/src/runner-provider.js +1 -1
- package/dist/src/runner-sequence.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/runtime2.js +1 -0
- package/dist/src/sdk-android-adb.d.ts +97 -33
- package/dist/src/sdk-android-adb.js +4 -4
- package/dist/src/sdk-batch-runner.d.ts +2 -1
- package/dist/src/sdk-batch-runner.js +3 -2
- package/dist/src/sdk-contracts.d.ts +70 -149
- package/dist/src/sdk-contracts.js +1 -1
- package/dist/src/sdk-device.js +1 -1
- package/dist/src/sdk-finders.d.ts +5 -4
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/sdk-io.d.ts +1 -1
- package/dist/src/sdk-remote-config.d.ts +19 -96
- package/dist/src/sdk-remote-config.js +1 -1
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selector-vocabulary.js +1 -0
- package/dist/src/selectors.d.ts +12 -3
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +3 -3
- package/dist/src/session-target-evidence.js +1 -0
- package/dist/src/session.js +8 -10
- package/dist/src/simctl.js +1 -1
- package/dist/src/simulator.js +1 -1
- package/dist/src/snapshot-diagnostics.js +5 -2
- package/dist/src/snapshot-quality.js +1 -1
- package/dist/src/snapshot.js +1 -1
- package/dist/src/snapshot2.js +1 -2
- package/dist/src/tool-provider3.js +1 -0
- package/dist/src/unsupported-interactor.js +1 -0
- package/dist/src/url.js +1 -1
- package/dist/src/web.js +1 -1
- package/package.json +34 -27
- package/server.json +2 -2
- package/skills/agent-device/SKILL.md +5 -3
- package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.apk +0 -0
- package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.apk.sha256 +0 -1
- package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.19.3.manifest.json +0 -10
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.apk +0 -0
- package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.19.3.apk.sha256 +0 -1
- package/dist/src/apple-runner-options.js +0 -1
- package/dist/src/cli-output.js +0 -1
- package/dist/src/client.js +0 -3
- package/dist/src/handler-utils.js +0 -1
- package/dist/src/output-path.js +0 -1
- package/dist/src/points.js +0 -4
- package/dist/src/provider.js +0 -1
- package/dist/src/providers.js +0 -1
- package/dist/src/record-trace-recording.js +0 -23
- package/dist/src/recording-provider.js +0 -1
- package/dist/src/sdk-batch.js +0 -2
- package/dist/src/selectors2.js +0 -1
- package/dist/src/ttl-memo.js +0 -1
- package/dist/src/tv-remote.js +0 -14
- package/dist/src/xml.js +0 -2
- /package/{macos-helper → apple/macos-helper}/Package.swift +0 -0
- /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +0 -0
- /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +0 -0
- /package/{macos-helper → apple/macos-helper}/Sources/AgentDeviceMacOSHelper/main.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests.entitlements +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.h +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerObjCExceptionCatcher.m +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -0
- /package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/RecordingScripts/recording-trim.swift +0 -0
- /package/{src/platforms/linux → linux}/atspi-dump.py +0 -0
package/dist/src/cli-help.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{ur as e}from"./recording-export-quality.js";import{f as t,v as n}from"./registry.js";import{n as r,r as i}from"./command-schema.js";const a=[`Flows: launchApp; runFlow file/inline with platform, visibility, and limited boolean conditions; onFlowStart/onFlowComplete; repeat.times and retry.`,`Interactions: tapOn, doubleTapOn, longPressOn, inputText, eraseText, openLink, hideKeyboard, basic pressKey, and back; targets support index, childOf, label, points, and optional.`,`Assertions and navigation: assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage/target swipe, takeScreenshot, waitForAnimationToEnd, and stopApp.`,`Scripts: ordered runScript file/env scripts with http.post, json, and output variables.`],o=[`Runtime: iOS and Android only; launchApp.clearState supports Android and iOS simulators, launch arguments are Apple-only, and standalone device utility/state commands are unsupported.`,`Expressions: when.true supports boolean literals and maestro.platform comparisons; repeat.while, evalScript, and broader JavaScript expressions are unsupported.`,`Environment: flow env is the default, AD_VAR_* overrides it, and CLI -e KEY=VALUE wins over both.`,`Trust: runScript executes trusted scripts, may make http.post network requests, and is not a security sandbox; output keys cannot contain a dot.`,`Errors and tracking: unsupported commands and fields fail with source context when available; open a focused issue only when implementation work is planned.`];function s(){return[`Supported subset:`,...a.map(e=>` - ${e}`),``,`Boundaries:`,...o.map(e=>` - ${e}`)].join(`
|
|
2
|
+
`)}function c(e){let t=e.endsWith(`?`),n=t?e.slice(0,-1):e;return t?`[${n}]`:`<${n}>`}function l(e){return n().filter(t=>t.key===e)}function u(e,t){if(t.usageOverride)return t.usageOverride;let n=(t.positionalArgs??[]).map(c),r=(t.allowedFlags??[]).flatMap(e=>l(e).map(e=>e.usageLabel??e.names[0])).map(e=>`[${e}]`);return[e,...n,...r].join(` `)}const d=[{label:`agent-device help manual-qa`,description:`Follow a manual test script with exact interactions and verification`},{label:`agent-device help dogfood`,description:`Explore an app and report issues with evidence`},{label:`agent-device help validate`,description:`Validate code changes, perf, visuals, logs, and cleanup`},{label:`agent-device help workflow`,description:`Full app automation reference for commands, refs, selectors, and waits`},{label:`agent-device help debugging`,description:`Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter`},{label:`agent-device help tv`,description:`Use when navigating Android TV, tvOS, or Vega VVD focus-first surfaces`},{label:`agent-device help react-native`,description:`Use when the target app is React Native, Expo, or a dev client`},{label:`agent-device help react-devtools`,description:`Use when inspecting components, props/state/hooks, renders, or profiles`},{label:`agent-device help cdp`,description:`Use when investigating JS heap growth, heap snapshots, or retainers`},{label:`agent-device help physical-device`,description:`Use when using a connected phone/tablet or iOS signing setup`},{label:`agent-device help remote`,description:`Use when working through cloud config, tenants, leases, or local tunnels`},{label:`agent-device help web`,description:`Use when automating a browser through agent-device sessions`},{label:`agent-device help macos`,description:`Use when targeting desktop, frontmost app, or menu bar surfaces`}],f=[`Write full command lines starting with agent-device; do not output pseudo commands, helper prose, pipes, grep, jq, or hidden stderr.`,`Default app loop: agent-device open <app> -> agent-device snapshot -i -> mutate a current target with --settle -> continue from that settled diff -> agent-device close.`,`Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close.`,`Follow structured command hints before choosing a recovery action.`,`Targets are concrete refs or selectors: @e12, label="Query", role=button label="Submit".`,`Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.`,`A literal @ handle is a label such as label="@account.example", never a bare @ref.`,`Boundary shapes: agent-device fill 'label="Query"' "text" --settle; agent-device press 'role=button label="Submit"' --settle.`,`fill takes a target then text. press targets an accessible element; key=Enter is not a supported target.`,`Pick the help mode below when the task is manual QA, dogfooding, engineering validation, or debugging.`],p=[`Planning output contract: when asked to plan commands, output command lines only: no prose, numbering, Markdown fences, pipes, or shell helpers.`,`If you did not use --settle, verify a mutation with diff snapshot (or diff snapshot -i), not a full snapshot: it prints only the added/removed/changed lines since the last snapshot in this session.`,`Network-backed or debounced results may arrive after the --settle quiet window; follow the settled action with wait text "Expected result" or wait <selector> instead of polling full snapshots.`,`Pin a raw CLI ref to the response that minted it with ~s<n> (n = refsGeneration): press @e12~s4. On iOS, stale refs are rejected for mutations; refresh refs or use a stable selector.`,`Plain snapshot reads state; snapshot -i refreshes current interactive refs only.`,`Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions.`,`Read-only visible/state question: use snapshot/get/is/find; use snapshot -i only when refs are needed.`,`Truncated text/input preview: expand first with snapshot -s @e12, not get text.`,`React Native apps: read help react-native.`,`Text fields: use fill <target> <text> --settle to replace a field value. Use type <text> only to append after focusing a field with press.`,`Clearing text: do not use fill <target> ""; use a visible clear/reset control or report that clearing is unsupported.`,`Implicit default sessions are scoped to the current worktree; if a prompt names a Session, include --session <name> on every command in that flow.`,`Run mutating commands serially within one session; parallelize only read-only commands or separate sessions/devices.`,`After mutation: refs are stale. If the next target is known, use its selector directly; otherwise refresh with snapshot -i, scoped with -s when a stable container is known. Use press/click for taps.`,`macOS context menus use click <ref> --button secondary, then snapshot -i. Longpress is for mobile hold gestures, not macOS secondary-click menus.`,`Remote lifecycle: use connect, then open, commands, close, and disconnect. Read help remote for proxy, cloud, and device-cloud provider flows.`,`TV/D-pad targets: read help tv. Web browser sessions: read help web.`,`Debug evidence: Session state contains request diagnostics and runner.log; use logs clear --restart/mark/path, trace, and network dump --include headers for app evidence.`,`Routine QA loop with concrete command shapes: agent-device help manual-qa. Full operating guide: agent-device help workflow. Exploratory QA: agent-device help dogfood.`],m=[`Default config files: ~/.agent-device/config.json, ./agent-device.json`,`Use --config <path> or AGENT_DEVICE_CONFIG to load one explicit config file.`],h=[{label:`AGENT_DEVICE_SESSION`,description:`Explicit session name`},{label:`AGENT_DEVICE_PLATFORM`,description:`Default platform binding`},{label:`AGENT_DEVICE_SESSION_LOCK`,description:`Bound-session conflict mode`},{label:`AGENT_DEVICE_DAEMON_BASE_URL`,description:`Connect to remote daemon`},{label:`AGENT_DEVICE_DAEMON_AUTH_TOKEN`,description:`Remote daemon service/API token`},{label:`AGENT_DEVICE_CLOUD_BASE_URL`,description:`Bridge/control-plane API origin for cloud auth and /api-keys`}],g=[`agent-device open Settings --platform ios`,`agent-device open https://example.com --platform web`,`agent-device snapshot -i`,`agent-device fill @e3 "test@example.com"`,`agent-device replay ./session.ad`,`agent-device test ./suite --platform android`],_={"manual-qa":{summary:`Follow manual test scripts with exact interactions and verification`,body:`agent-device help manual-qa
|
|
2
3
|
|
|
3
4
|
Use this when asked to follow a manual QA script, test case, checklist, acceptance flow, or user-provided instructions.
|
|
4
5
|
|
|
@@ -36,7 +37,14 @@ Recovery:
|
|
|
36
37
|
Network/typeahead result missing: wait text "Expected result" or wait <selector>.
|
|
37
38
|
Keyboard visible over the next target: the on-screen keyboard usually does not block presses, so press the target directly instead of dismissing. If the press fails or reports no visible effect, scroll the target into view or use keyboard enter when submission is wanted.
|
|
38
39
|
Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i.
|
|
39
|
-
Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change.`},
|
|
40
|
+
Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change.`},maestro:{summary:`Supported Maestro YAML commands, grammar, and runtime boundaries`,body:`agent-device help maestro
|
|
41
|
+
|
|
42
|
+
Run Maestro compatibility flows with replay <flow.yaml> --maestro or test <path> --maestro. Bind an iOS or Android target with --platform or an existing session.
|
|
43
|
+
|
|
44
|
+
${s()}
|
|
45
|
+
|
|
46
|
+
Unsupported syntax fails loudly rather than being skipped. Architecture, performance tradeoffs, and declared conformance divergences: https://github.com/callstack/agent-device/blob/main/docs/adr/0015-direct-maestro-engine.md
|
|
47
|
+
Focused compatibility request: https://github.com/callstack/agent-device/issues/new`},workflow:{summary:`Normal agent-device bootstrap, exploration, and validation loop`,body:`agent-device help workflow
|
|
40
48
|
|
|
41
49
|
Version-matched operating guide for normal agent-device work.
|
|
42
50
|
|
|
@@ -54,7 +62,7 @@ Command shape:
|
|
|
54
62
|
Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output.
|
|
55
63
|
If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @<ref>, @ref, @Label_Name, or @eN placeholders.
|
|
56
64
|
Close means agent-device close. App-owned back means back; system back means back --system.
|
|
57
|
-
Taps are press or click; tap is an alias for press. On Android TV and
|
|
65
|
+
Taps are press or click; tap is an alias for press. On Android TV, tvOS, and Vega OS, read help tv and use tv-remote press up|down|left|right|select to move D-pad/remote focus before activating controls; use tv-remote longpress <button> for a held remote button. Gestures use swipe, longpress, or gesture <pan|fling|swipe|pinch|rotate|transform>. Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Android swipe and multi-touch gestures use provider-native touch injection when available, then the bundled touch helper. iOS simulator multi-touch uses private XCTest synthesis for a continuous two-pointer path; otherwise it reports UNSUPPORTED_OPERATION.
|
|
58
66
|
|
|
59
67
|
Bootstrap:
|
|
60
68
|
agent-device devices --platform ios
|
|
@@ -73,6 +81,19 @@ Bootstrap:
|
|
|
73
81
|
CI may cache ~/.agent-device/apple-runner/derived with an exact key that includes the agent-device package and Xcode version. Avoid broad restore-key fallbacks; prepare ios-runner already recovers bad restored runner artifacts and one retryable non-connecting runner launch. Runner build/start output is written to the session's runner.log; daemon.log is for daemon lifecycle/startup issues.
|
|
74
82
|
Do not open artifact paths or invent package ids. If apps lookup misses the target and no URL/artifact is provided, ask or stop.
|
|
75
83
|
|
|
84
|
+
Reusable open-to-destination scripts:
|
|
85
|
+
Arm recording on the first open, perform the full journey, verify the ready destination with a selective selector-targeted wait, then publish without closing:
|
|
86
|
+
agent-device open com.example.app --relaunch --save-script=screen-x.ad
|
|
87
|
+
agent-device press 'id="continue"' --settle
|
|
88
|
+
agent-device wait 'role="heading" label="Screen X"'
|
|
89
|
+
agent-device session save-script
|
|
90
|
+
session save-script [path] [--force] publishes the sole recorded open through the destination guard, omits close, and leaves the session active. The guard is a selector wait on a labeled or id-bearing landmark: its recorded identity is captured while armed, and replay verifies that identity after the wait's selector resolves, so a reshuffled screen with the same label elsewhere fails closed instead of false-passing. A duration wait, wait stable, wait @ref, or a selector wait on an unlabeled element is not a destination guard. A second successful open aborts publication; start a fresh session to author again.
|
|
91
|
+
Unparameterized fill/type inputs are literal .ad script content. For a sensitive fill, arm recording first, keep the live value in an environment variable, and name its replay placeholder explicitly:
|
|
92
|
+
export AD_VAR_PASSWORD='<secret>'
|
|
93
|
+
agent-device fill 'id="password"' "$AD_VAR_PASSWORD" --record-as PASSWORD
|
|
94
|
+
The live app receives the value, while recording state and the published script contain only \${PASSWORD}. Reuse the same name for repeated values and choose distinct names for distinct inputs. --record-as accepts uppercase replay variable names, is fill-only, requires an armed recording, and cannot be combined with --no-record.
|
|
95
|
+
Publish with session save-script, then replay with AD_VAR_PASSWORD still set or pass --env PASSWORD=<value>. A missing value fails before that fill runs. Do not record passwords, tokens, or other secrets without --record-as; their literal text will be written to the .ad target.
|
|
96
|
+
|
|
76
97
|
Snapshots and refs:
|
|
77
98
|
snapshot reads visible state. snapshot -i gets current interactive refs only; it is the fast path when the next step is an interaction.
|
|
78
99
|
Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions; use --raw or --json only when you need the full provider tree.
|
|
@@ -84,7 +105,7 @@ Snapshots and refs:
|
|
|
84
105
|
For press/fill/click/longpress, prefer --settle and continue from its settled diff when it exposes the next target or evidence. Refresh with snapshot -i only when you did not settle, settle printed not settled, or the settle output lacks what you need.
|
|
85
106
|
Anti-pattern: snapshot -i followed by snapshot -i | grep ..., or adding 2>/dev/null | jq ... before reading the raw command output.
|
|
86
107
|
Refs from the first snapshot remain valid until you press, click, fill, type, scroll, go back, wait for async UI, or otherwise change app state.
|
|
87
|
-
Pinned refs (@e12~s4, generation from refsGeneration or settle.refsGeneration)
|
|
108
|
+
Pinned refs (@e12~s4, generation from refsGeneration or settle.refsGeneration) identify their source tree. On iOS, stale refs are rejected for press/fill/click/longpress before dispatch; refresh with snapshot -i or use a stable selector. Read-only commands report a staleness warning, and other platforms retain their existing warning behavior.
|
|
88
109
|
After a mutation, prefer a known selector/label directly (for example press 'label="Send"') because interaction commands refresh interactive state internally. A settled diff with no added refs (for example a modal dismiss) also lists an "unchanged interactive" tail of still-present refs, so check that before falling back. If you need to discover a new control not shown by settle or its tail, use snapshot -i, or snapshot -i -s "Composer" when a stable container label/id can scope the refresh.
|
|
89
110
|
If typing/fill opened the keyboard or changed layout and the next target has no stable selector, run snapshot -i, use the fresh ref, then verify with wait/find or diff snapshot -i.
|
|
90
111
|
For a targeted query, use find/get/is. If you truly need the full tree again, pass --force-full.
|
|
@@ -102,7 +123,7 @@ Selectors:
|
|
|
102
123
|
agent-device press 'id="submit-order"'
|
|
103
124
|
agent-device is visible 'label="Online"'
|
|
104
125
|
agent-device get text 'id="quantity-value"'
|
|
105
|
-
Ambiguous selector disambiguation: a selector on an interactive command (press/click/fill/focus/longpress/scroll/swipe/pinch) that matches multiple elements does not fail by default. It auto-resolves deepest node first (largest depth in the tree), then smallest on-screen area; only an exact tie on both depth and area fails with "Selector did not resolve uniquely". replay
|
|
126
|
+
Ambiguous selector disambiguation: a selector on an interactive command (press/click/fill/focus/longpress/scroll/swipe/pinch) that matches multiple elements does not fail by default. It auto-resolves deepest node first (largest depth in the tree), then smallest on-screen area; only an exact tie on both depth and area fails with "Selector did not resolve uniquely". replay's suggestion re-resolution (in a divergence report) applies the same depth-then-area policy for touch/fill/get-text, so recorded flows and live commands pick the same candidate. This exists because short/reused labels (tab + header + button with the same text, or a duplicated list-row label) are common in real apps; add id="..." or a longer/more specific text to force a different match instead of assuming ambiguous selectors always fail.
|
|
106
127
|
Selector match can still land on a non-interactive node (for example an off-screen map annotation that exact-matches text= while the real control has a longer label). Success responses for press/fill/click/ref targets carry targetHittable: false and a hint when the resolved element reports hittable: false, since the tap may have had no visible effect; treat that as a signal to verify with a snapshot or re-target by @ref/longer text, not as a command failure.
|
|
107
128
|
|
|
108
129
|
Text entry:
|
|
@@ -113,11 +134,13 @@ Text entry:
|
|
|
113
134
|
agent-device type "Handle with care" --delay-ms 80
|
|
114
135
|
Empty replacement is not a supported clear-field command: do not plan fill <target> "" or fill <target> ''. Prefer a visible clear/reset control; if the app exposes none, report the tool gap instead of inventing a clear command.
|
|
115
136
|
Debounced field with no result selector: agent-device wait 1000. Keyboard read-only: keyboard status/get. The on-screen keyboard usually does not block agent-device interactions; press the next target directly instead of dismissing. If that press fails or reports no visible effect, scroll the target into view or use keyboard enter when submission is wanted.
|
|
116
|
-
Only dismiss the keyboard when hiding it is the actual goal. To hide the keyboard, use keyboard dismiss. It taps safe controls like Done when available and verifies the keyboard closed.
|
|
137
|
+
Only dismiss the keyboard when hiding it is the actual goal. To hide the keyboard, use keyboard dismiss. It taps safe controls like Done when available and verifies the keyboard closed.
|
|
138
|
+
On iOS, if it returns UNSUPPORTED_OPERATION, there is no generic app-agnostic blur fallback: do not assume a static text or heading is safe to press, because it can belong to a tappable parent. Use an app-provided dismiss control only when its action is explicitly intended; otherwise report that keyboard dismissal is unavailable.
|
|
139
|
+
On Android, keyboard dismiss first avoids navigation. If it returns UNSUPPORTED_OPERATION because the current IME needs back navigation, use back only when normal back behavior is acceptable; otherwise report that keyboard dismissal is unavailable.
|
|
117
140
|
Use plain fill/type first for ordinary login and form fields. If an iOS debounced or search-as-you-type field actually drops characters, or must receive incremental updates, retry with --delay-ms before trying clipboard paste; --delay-ms intentionally paces character entry.
|
|
118
141
|
iOS Allow Paste prompt cannot be exercised under XCUITest. To test paste-driven app behavior, prefill first with agent-device clipboard write "some text"; test the system prompt manually.
|
|
119
142
|
Android Gboard handwriting/stylus UI can capture text in an IME-owned input instead of the app field. If fill reports that input was captured by the keyboard/IME, use the diagnostic targetInput/actualInput details, inspect keyboard status/get if needed, and switch or disable handwriting outside the command plan before retrying. Do not keep retrying fill/type against the same field while the IME owns focus.
|
|
120
|
-
Android text entry is owned by agent-device: provider-native text injection when available,
|
|
143
|
+
Android text entry is owned by agent-device: provider-native text injection when available, else the bundled test IME helper (emulators activate it automatically; real devices need open --test-ime), else chunk-safe ASCII shell input. Do not switch to raw adb, clipboard, or paste as an agent fallback. If non-ASCII text still fails, report the tool/device gap.
|
|
121
144
|
|
|
122
145
|
Session ordering:
|
|
123
146
|
Stateful commands within one session must run serially. Do not run open/press/fill/type/scroll/back/alert/replay/batch/close commands in parallel against the same session.
|
|
@@ -141,20 +164,21 @@ Read-only and waits:
|
|
|
141
164
|
Ambiguous find: add --first or --last. If info is not visible/exposed, report that gap instead of typing/searching/navigating to reveal it.
|
|
142
165
|
|
|
143
166
|
Navigation and gestures:
|
|
144
|
-
Use scroll for lists; swipe for coordinate gestures/carousels; gesture pan for deliberate drags; gesture fling for fast directional throws.
|
|
167
|
+
Use scroll for lists; swipe for quick coordinate gestures/carousels; gesture pan for deliberate timed drags; gesture fling for fast directional throws.
|
|
145
168
|
For fast macOS desktop list traversal, prefer fixed pixel wheel steps and batch them when no snapshot is needed between passes:
|
|
146
169
|
agent-device scroll down --pixels 200 --duration-ms 50 --platform macos
|
|
147
170
|
agent-device batch --steps '[{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}},{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}}]' --platform macos
|
|
148
171
|
For raw coordinate gestures, run snapshot -i first and choose a point near the center of the intended app-owned target. Avoid screen edges, tab bars, navigation bars, and home indicators because those areas can trigger system or app navigation instead of the gesture under test.
|
|
149
172
|
If app-owned back is ambiguous or has just misrouted, prefer a visible nav/back button ref, tab-bar ref, or deep link over repeated back/system back.
|
|
150
173
|
App-owned action sheets, menus, and camera/scan screens are normal UI. After opening one, run snapshot -i or wait for the option, press by label/ref, handle visible permission sheets through UI or platform-supported native alerts, then wait for a concrete result before returning to chat/form state.
|
|
151
|
-
Keep count/pause/pattern on one swipe; flags are --count, --pause-ms, --pattern ping-pong.
|
|
174
|
+
Keep count/pause/pattern on one swipe; flags are --count, --pause-ms, --pattern ping-pong. Count is capped at 200, pause at 10000ms, and the combined swipe/pause schedule at 60000ms.
|
|
152
175
|
For repeated iOS gesture smoke checks, use press <x> <y> --count <n> --jitter-px <n> for tap series and swipe <x1> <y1> <x2> <y2> --count <n> for drag series.
|
|
153
176
|
longpress accepts coordinates, @refs, or selectors. Prefer @ref/selector from snapshot -i; use coordinates only as a fallback when accessibility refs miss the exact target. Duration and gesture scale/center are positional:
|
|
154
177
|
agent-device longpress 300 500 800
|
|
155
178
|
agent-device longpress @e12 800
|
|
156
179
|
agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong
|
|
157
180
|
agent-device gesture pan 200 420 0 -80 500
|
|
181
|
+
agent-device gesture pan 200 420 80 -40 700 --pointer-count 2
|
|
158
182
|
agent-device gesture fling right 200 420 180
|
|
159
183
|
agent-device gesture pinch 0.5 200 400
|
|
160
184
|
agent-device gesture rotate 35 200 420
|
|
@@ -165,7 +189,9 @@ Navigation and gestures:
|
|
|
165
189
|
agent-device wait text "pan changed yes" 3000 --platform android
|
|
166
190
|
agent-device wait text "pinch changed yes" 3000 --platform android
|
|
167
191
|
agent-device wait text "rotate changed yes" 3000 --platform android
|
|
168
|
-
If Android needs exact app-state values, prefer isolated gesture pan, gesture pinch, or gesture rotate commands over one combined transform.
|
|
192
|
+
If Android needs exact app-state values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate commands over one combined transform.
|
|
193
|
+
Gesture planning prefers the active-app frame. A backend without a gesture viewport resolver falls back to the visible snapshot union, which can be less accurate near edges.
|
|
194
|
+
tvOS coordinate pan and fling preserve only the dominant direction as a remote swipe; authored endpoints and duration are not preserved.
|
|
169
195
|
macOS context menus are secondary clicks, not long presses:
|
|
170
196
|
agent-device click @e66 --button secondary --platform macos
|
|
171
197
|
agent-device snapshot -i --platform macos
|
|
@@ -181,7 +207,7 @@ Validation and evidence:
|
|
|
181
207
|
agent-device screenshot
|
|
182
208
|
agent-device press 124 817
|
|
183
209
|
agent-device snapshot -i
|
|
184
|
-
Startup/CPU/memory/frame first pass: perf metrics --json (bare perf and metrics are aliases). Focused frame/jank health: perf frames --json. Memory-only sample: perf memory sample --json returns compact JSON with bounded top offenders. Heap/memgraph artifact escalation: perf memory snapshot --out heap.artifact; use --kind android-hprof on Android or --kind memgraph on supported Apple simulator/macOS app sessions. Android native profiling: perf cpu profile start|stop|report --kind simpleperf --out <path>; Android native traces: perf trace start|stop --kind perfetto --out <path>. Artifact collectors return compact state/path/size metadata only; raw heap/profile/trace files stay on disk. Treat native perf output as the agent evidence: for example, a Perfetto stop can return state=stopped, outPath=/tmp/app.perfetto-trace, sizeBytes=5392410, and method=adb-shell-perfetto while the 5.3 MB raw trace stays in the artifact. This is better than raw dumps for agents because it is stable, bounded, and keeps large artifacts out of context. heapprofd is deferred until Perfetto plumbing is available. Replay
|
|
210
|
+
Startup/CPU/memory/frame first pass: perf metrics --json (bare perf and metrics are aliases). Focused frame/jank health: perf frames --json. Memory-only sample: perf memory sample --json returns compact JSON with bounded top offenders. Heap/memgraph artifact escalation: perf memory snapshot --out heap.artifact; use --kind android-hprof on Android or --kind memgraph on supported Apple simulator/macOS app sessions. Android native profiling: perf cpu profile start|stop|report --kind simpleperf --out <path>; Android native traces: perf trace start|stop --kind perfetto --out <path>. Artifact collectors return compact state/path/size metadata only; raw heap/profile/trace files stay on disk. Treat native perf output as the agent evidence: for example, a Perfetto stop can return state=stopped, outPath=/tmp/app.perfetto-trace, sizeBytes=5392410, and method=adb-shell-perfetto while the 5.3 MB raw trace stays in the artifact. This is better than raw dumps for agents because it is stable, bounded, and keeps large artifacts out of context. heapprofd is deferred until Perfetto plumbing is available. Replay divergence and resume: a failing replay/test step returns REPLAY_DIVERGENCE with a bounded report (screen digest, ranked selector suggestions, resume). Repair app state, then resume with replay --from <n> --plan-digest <sha256> (both from the report's resume field) to continue from the failed step without re-running earlier ones; resume never re-executes skipped steps, so app state is the caller's responsibility, and it is rejected with INVALID_ARGS when the plan digest is stale, --from is out of range, or the skipped range/target touches runtime control flow. The plan digest binds the script, its includes, the effective --platform/--target, and per-action runtime/identity. Native .ad interpolation is late-bound after planning, so changing only its values keeps the digest; Maestro environment substitution occurs during compatibility parsing and can change action inputs, includes, or control expansion, so it can change the digest. --from is replay-only; test rejects it. --update/-u no longer rewrites the script (ADR 0012) — it is a no-op kept for compatibility; every divergence already carries the same ranked suggestions. Agent-supervised repair (heal-by-doing): arm replay <file>.ad --save-script[=<out>] before step 1 (armed once; --from continuations do not need it again). Every divergence carries a repairHint: record-and-heal means press the correct control via a blessed @ref from the divergence's screen.refs, recorded (no --no-record), then continue with replay --from <n+1> --plan-digest <sha256>; state-repair means the script is correct but app state is not, so fix state with --no-record actions, then replay --from <n> --plan-digest <sha256> to re-run the unchanged step; caution means something already matches the recorded selector, so a blind re-press may repeat the mistake; manual means no safe automated repair could be proven. (If close --save-script is run before a required resume, the repair is aborted and no script is written). While armed, read-only inspection YOU run to locate the repair target (snapshot -i, get attrs, find, is) is excluded from the healed script by default — no --no-record needed; the script's own authored get/is/find steps are unaffected and stay in the heal. If the step you are repairing is itself a read, pass --record on that one command so it lands in the heal (--record is accepted only on snapshot/get/is and a read-only find; it is mutually exclusive with --no-record). Ending the repair with close --save-script[=<out>] writes only the steps recorded since the arming replay as <out>, defaulting to the <file> sibling <stem>.healed.ad — review its diff before promoting it over the original.
|
|
185
211
|
Recording: record start/stop. The default scope is app and expects an active session created by open <app>; this keeps app proof videos tied to the intended app session. Use record start --scope device/system to explicitly request whole-screen capture where the selected backend supports it, such as recordings that intentionally span multiple apps, home screen, settings, or app transitions. Use --max-size to cap the longest edge and --quality medium|high to choose output quality across Android and Apple targets. By default, stop burns touch overlays into the video; use record start --hide-touches for the fastest raw recording. Android record start publishes a durable device manifest. Android adb screenrecord has a 180s platform limit, so longer Android recordings are returned as multiple MP4 chunks while the daemon stays alive; after daemon restart, record stop recovers only manifest-owned chunks and warns when gesture overlays are unavailable. For gesture-heavy iOS simulator proof videos, prefer --hide-touches because overlay timing depends on a stable runner session while gestures are executing. Tracing: trace start ./trace.log, trace stop ./trace.log. Paths are positional.
|
|
186
212
|
Stable known flow: batch ./steps.json, not workflow batch.
|
|
187
213
|
Inline batch JSON example:
|
|
@@ -243,7 +269,7 @@ React Native dev loop:
|
|
|
243
269
|
|
|
244
270
|
Guarantees:
|
|
245
271
|
Statements of fact for agents to reason from without probing them via trial commands. Each is backed by source in the agent-device repo; behavior changes land with an updated statement here.
|
|
246
|
-
Selector ambiguity: a selector on an interactive command that matches multiple elements does not fail by default. Resolution auto-disambiguates deepest node first, then smallest on-screen area; only an exact tie on both fails with "Selector did not resolve uniquely (...)". replay
|
|
272
|
+
Selector ambiguity: a selector on an interactive command that matches multiple elements does not fail by default. Resolution auto-disambiguates deepest node first, then smallest on-screen area; only an exact tie on both fails with "Selector did not resolve uniquely (...)". replay's suggestion re-resolution applies the same depth-then-area policy, so recorded and live commands pick the same candidate.
|
|
247
273
|
Hittability: iOS AX hittable:false on a resolved node does not block resolution or fail the command; non-hittable resolution is allowed by design because iOS AX hittable flags are unreliable on deep React Native trees. press/fill/click success responses carry targetHittable: false plus a hint when the resolved ref or selector target reports hittable: false, so treat that as a signal to verify with a snapshot or re-target, not as a failure.
|
|
248
274
|
Open: on iOS, open <app> without --relaunch dispatches a plain simctl launch, which is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators (not real devices or macOS) this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch, so relaunch is a single step there.
|
|
249
275
|
Close and runner retention: close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that device skips the runner build, unless --shutdown was requested, the session was recording, the session held a device lease, or the device used a scoped (non-default) simulator set — any of those tear the runner down on close. A retained runner auto-stops after an idle window (default 5 minutes) to release the device's runner lease for other daemons; set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override the window, or 0 to disable idle stop and retain until daemon exit.
|
|
@@ -257,13 +283,14 @@ Escalate:
|
|
|
257
283
|
help dogfood exploratory QA report workflow
|
|
258
284
|
help validate engineering self-validation loops
|
|
259
285
|
help debugging logs, network, alerts, traces, flaky runtime failures
|
|
260
|
-
help tv Android TV and
|
|
286
|
+
help tv Android TV, tvOS, and Vega VVD focus-first remote navigation
|
|
261
287
|
help react-devtools React Native performance, profiling, props/state/hooks, slow renders, rerenders
|
|
262
288
|
help react-native React Native app automation hazards, overlays, Metro/Re.Pack, and routing
|
|
263
289
|
help remote remote/cloud config, tenant, lease, local service tunnels
|
|
264
|
-
help macos desktop, frontmost-app, menu bar surfaces
|
|
290
|
+
help macos desktop, frontmost-app, menu bar surfaces
|
|
291
|
+
help ios-system-ui iOS SpringBoard, widget add/edit/remove, system-UI surfaces`},tv:{summary:`Android TV, tvOS, and Vega VVD focus-first remote navigation`,body:`agent-device help tv
|
|
265
292
|
|
|
266
|
-
Use this when the target is Android TV, Apple TV, or
|
|
293
|
+
Use this when the target is Android TV, Apple TV/tvOS, or an Amazon Vega OS TV app running in the Vega Virtual Device (VVD). TV surfaces are focus-first: move focus with remote/D-pad buttons, then activate the focused control.
|
|
267
294
|
|
|
268
295
|
Core loop:
|
|
269
296
|
agent-device open Settings --platform android --target tv --session tv
|
|
@@ -273,13 +300,30 @@ Core loop:
|
|
|
273
300
|
agent-device tv-remote press select --platform android --target tv --session tv
|
|
274
301
|
agent-device screenshot ./tv-focus.png --overlay-refs --platform android --target tv --session tv
|
|
275
302
|
|
|
303
|
+
Vega OS:
|
|
304
|
+
Vega OS is driven through the SDK-matched Vega CLI and VDA, not ADB.
|
|
305
|
+
Initial support is VVD-only. Physical Fire TV devices remain unsupported until their discovery, lifecycle, and remote controls have durable hardware evidence.
|
|
306
|
+
Use --platform vega --target tv for the running Vega Virtual Device.
|
|
307
|
+
vega virtual-device start
|
|
308
|
+
agent-device devices --platform vega --target tv
|
|
309
|
+
agent-device open <component-id> --platform vega --target tv --session vega-tv
|
|
310
|
+
agent-device tv-remote press down --platform vega --target tv --session vega-tv
|
|
311
|
+
agent-device tv-remote press select --platform vega --target tv --session vega-tv
|
|
312
|
+
agent-device close <component-id> --session vega-tv
|
|
313
|
+
vega virtual-device stop
|
|
314
|
+
Use a component ID from the app package or Vega SDK tooling; agent-device app inventory is not yet supported.
|
|
315
|
+
Use --serial VirtualDevice for explicit VVD selection.
|
|
316
|
+
agent-device does not boot the VVD implicitly.
|
|
317
|
+
Snapshot, screenshot, selectors, install, touch/text/gesture, logs, and performance commands remain unsupported until their Vega backends are implemented.
|
|
318
|
+
|
|
276
319
|
Buttons:
|
|
277
320
|
tv-remote press up|down|left|right|select|menu|home|back
|
|
278
321
|
tv-remote longpress select
|
|
279
322
|
tv-remote press select --duration-ms 500
|
|
280
323
|
ok, center, and enter are input aliases for select; command output still reports button: "select".
|
|
281
324
|
longpress is CLI sugar for --duration-ms 500. --duration-ms overrides that preset.
|
|
282
|
-
--duration-ms holds a tvOS remote button for that duration. On Android TV, any positive duration maps to the ADB longpress form because Android input keyevent has no exact hold duration.
|
|
325
|
+
--duration-ms holds a tvOS or Vega OS remote button for that exact duration. On Android TV, any positive duration maps to the ADB longpress form because Android input keyevent has no exact hold duration.
|
|
326
|
+
Vega OS uses the exact hold duration through inputd-cli in the VVD.
|
|
283
327
|
|
|
284
328
|
Android TV:
|
|
285
329
|
Android TV uses ADB keyevents behind agent-device tv-remote. Keep command plans on agent-device; do not switch to raw adb keyevent.
|
|
@@ -291,10 +335,10 @@ tvOS:
|
|
|
291
335
|
Use --platform ios --target tv for Apple TV simulators and devices.
|
|
292
336
|
|
|
293
337
|
Focus and visual truth:
|
|
294
|
-
|
|
338
|
+
On Android TV and tvOS, if snapshot -i exposes a focused node, verify it with is focused <selector>.
|
|
295
339
|
Use wait focused=true only when repeated snapshots preserve focus metadata for the app.
|
|
296
|
-
If the app exposes only a surface view, or focus metadata is transient, use screenshot --overlay-refs, screenshot, or diff snapshot as visual truth and keep moving focus with tv-remote.
|
|
297
|
-
Do not assume press/click @ref works on Android TV or
|
|
340
|
+
If the app exposes only a surface view, or focus metadata is transient, use screenshot --overlay-refs, screenshot, or diff snapshot as visual truth and keep moving focus with tv-remote. On Vega OS, use the VVD display as visual truth until capture support lands.
|
|
341
|
+
Do not assume press/click @ref works on Android TV, tvOS, or Vega OS until the desired element is focused.`},debugging:{summary:`Targeted failure evidence without dumping stale context`,body:`agent-device help debugging
|
|
298
342
|
|
|
299
343
|
Use this when behavior fails, hangs, times out, throws alerts, or needs runtime evidence.
|
|
300
344
|
|
|
@@ -316,7 +360,7 @@ Events:
|
|
|
316
360
|
Use events for a compact session timeline without dumping full app logs.
|
|
317
361
|
agent-device events
|
|
318
362
|
agent-device events 50 100
|
|
319
|
-
Events preserve command names, status, durations,
|
|
363
|
+
Events preserve command names, status, durations, bounded device/app inventory previews, lifecycle outcomes, artifact basenames, and structural action details such as scroll distance/direction, safe refs, and coordinates. User-entered text, clipboard contents, push/event payloads, selector values, free-form flags/messages/paths, and raw unknown command arguments are omitted or replaced with content-free placeholders. --no-record suppresses action.recorded entries, but request start/finish entries still record command, status, and timing.
|
|
320
364
|
|
|
321
365
|
Network:
|
|
322
366
|
Use network dump for recent session HTTP traffic parsed from app logs.
|
|
@@ -606,8 +650,11 @@ Discovery:
|
|
|
606
650
|
Use --device <name-or-udid> only when multiple devices are present.
|
|
607
651
|
|
|
608
652
|
iOS physical-device prerequisites:
|
|
609
|
-
Xcode and xcrun
|
|
653
|
+
Xcode, xcrun xcdevice, and xcrun xctrace must be available from the selected Xcode.
|
|
610
654
|
The device must be paired/trusted, connected, unlocked when needed, and have Developer Mode enabled.
|
|
655
|
+
Modern devices visible to devicectl use CoreDevice. Older devices visible only to xctrace use the XCTest backend automatically.
|
|
656
|
+
XCTest-backed devices must already have the target app installed and should be opened by bundle ID; app inventory, install/reinstall, logs, performance sampling, recording, deep links, and launch arguments require CoreDevice.
|
|
657
|
+
XCTest-backed runner commands travel through macOS usbmuxd; keep the trusted device connected by cable.
|
|
611
658
|
The AgentDeviceRunner XCTest host must be signed before commands can run on a physical device.
|
|
612
659
|
Start with Automatic Signing and only these env vars:
|
|
613
660
|
AGENT_DEVICE_IOS_TEAM_ID=ABCDE12345
|
|
@@ -621,7 +668,35 @@ iOS physical-device prerequisites:
|
|
|
621
668
|
|
|
622
669
|
Android physical-device prerequisites:
|
|
623
670
|
Enable USB debugging and confirm the device appears in agent-device devices --platform android.
|
|
624
|
-
Android does not need the iOS runner signing setup. For React Native/Expo Metro reachability, read help react-native
|
|
671
|
+
Android does not need the iOS runner signing setup. For React Native/Expo Metro reachability, read help react-native.
|
|
672
|
+
|
|
673
|
+
For iOS SpringBoard, widget, or other system-UI surfaces, read agent-device help ios-system-ui.`},"ios-system-ui":{summary:`iOS SpringBoard, widget, and system-surface workflow`,body:`agent-device help ios-system-ui
|
|
674
|
+
|
|
675
|
+
Use this when a task needs iOS SpringBoard (home screen), widget add/edit/remove, or other system-UI surfaces instead of the app under test.
|
|
676
|
+
|
|
677
|
+
This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings shown below. This workflow is verified on iOS simulator; physical-iPhone SpringBoard support is not yet verified.
|
|
678
|
+
|
|
679
|
+
Core loop:
|
|
680
|
+
1. Reach the app state you want to prepare (for example, arrange the widget/Live Activity data the app should show) with normal app automation, then agent-device open com.apple.springboard --platform ios. From an existing app session, agent-device home first also lands on the home screen, but open com.apple.springboard is what actually binds the session to SpringBoard for selector-driven commands.
|
|
681
|
+
2. agent-device snapshot -i to read the current localized SpringBoard controls.
|
|
682
|
+
3. agent-device longpress <x> <y> on an empty area of the home screen to enter edit mode. This is the one deliberate coordinate step; there is no reliable non-coordinate way to trigger it.
|
|
683
|
+
4. Re-snapshot and use selectors from the fresh tree to drive the Edit menu -> widget gallery -> search -> size picker -> Add Widget, for example:
|
|
684
|
+
agent-device snapshot -i
|
|
685
|
+
agent-device press 'label="Add Widget"'
|
|
686
|
+
agent-device fill 'label="Search"' "Calendar"
|
|
687
|
+
5. The widget-gallery search-result rows currently fall back to unlabeled nodes (a known capture gap), so tap the result by coordinates read from a screenshot until that is fixed:
|
|
688
|
+
agent-device screenshot
|
|
689
|
+
agent-device press <x> <y>
|
|
690
|
+
6. Continue with the semantic size picker and agent-device press 'label="Add Widget"' to place it.
|
|
691
|
+
7. To edit or remove an installed widget, longpress it, then re-snapshot and use the fresh context-menu selectors (Edit Widget / Remove Widget).
|
|
692
|
+
8. Use screenshots for visual assertions, and as the fallback wherever a system surface exposes sparse accessibility, not only in the gallery step.
|
|
693
|
+
9. Reopen the app bundle under test (agent-device open <app-id> --platform ios) to return to normal app automation; leaving SpringBoard bound does not resume the app session on its own.
|
|
694
|
+
|
|
695
|
+
Rules:
|
|
696
|
+
Do not hard-code Edit/Done/Add Widget or other SpringBoard label text into a plan as a fixed assumption; take them from the latest snapshot -i so the plan survives iOS version/locale differences.
|
|
697
|
+
A real system permission alert can appear mid-flow; it composes with this workflow normally, so handle it with alert wait/accept/dismiss or by pressing the visible label like any other step.
|
|
698
|
+
Prefer refs/selectors from the fresh snapshot for every step except the two documented coordinate fallbacks (empty-space long-press to enter edit mode, and the gallery search-result tap).
|
|
699
|
+
This topic covers what already works by opening SpringBoard as the session app. It does not yet cover keeping an app session open while alternating individual commands against SpringBoard, or Live Activity/Dynamic Island semantics; those land separately.`},remote:{summary:`Direct proxy, cloud profiles, and remote config`,body:`agent-device help remote
|
|
625
700
|
|
|
626
701
|
Remote connection providers use the same lifecycle:
|
|
627
702
|
connect -> open -> commands -> close -> disconnect
|
|
@@ -632,9 +707,10 @@ Providers:
|
|
|
632
707
|
Direct proxy: agent-device connect proxy --daemon-base-url <proxy-agent-device-url> stores the shared proxy profile and client identity.
|
|
633
708
|
BrowserStack: agent-device connect browserstack stores a local provider profile and creates the App Automate session on first open.
|
|
634
709
|
AWS Device Farm: agent-device connect aws-device-farm stores a local provider profile and creates the remote access session on first open.
|
|
710
|
+
Limrun: agent-device connect limrun stores a local provider profile and creates a direct iOS or Android instance on first open.
|
|
635
711
|
|
|
636
712
|
Device cloud interfaces:
|
|
637
|
-
CLI is the canonical bootstrap path: connect browserstack/aws-device-farm, then use normal open/snapshot/click/close/artifacts/disconnect commands.
|
|
713
|
+
CLI is the canonical bootstrap path: connect limrun/browserstack/aws-device-farm, then use normal open/snapshot/click/close/artifacts/disconnect commands.
|
|
638
714
|
JavaScript can skip persisted connect state by passing leaseProvider plus provider fields to createAgentDeviceClient or per-command options.
|
|
639
715
|
MCP exposes operational tools such as open, snapshot, click, close, and artifacts. It does not expose connect/disconnect; run CLI connect first in the same state dir before relying on MCP tools.
|
|
640
716
|
|
|
@@ -674,6 +750,17 @@ AWS Device Farm hosted-device flow:
|
|
|
674
750
|
agent-device close
|
|
675
751
|
agent-device artifacts --json
|
|
676
752
|
agent-device disconnect
|
|
753
|
+
AWS Device Farm currently supports Android and iOS WebDriver sessions only; Vega OS and Vega Fire TV ARNs are not routed.
|
|
754
|
+
|
|
755
|
+
Limrun direct-device flow:
|
|
756
|
+
LIMRUN_API_KEY=...
|
|
757
|
+
agent-device connect limrun --platform android
|
|
758
|
+
|
|
759
|
+
Limrun creates remote iOS simulators and Android emulators only. Do not pass local device selectors such as --udid, --serial, or --device.
|
|
760
|
+
agent-device open com.example.app
|
|
761
|
+
agent-device snapshot -i
|
|
762
|
+
agent-device close
|
|
763
|
+
agent-device disconnect
|
|
677
764
|
|
|
678
765
|
Local profile flow:
|
|
679
766
|
agent-device connect --remote-config ./remote-config.json
|
|
@@ -691,9 +778,10 @@ Rules:
|
|
|
691
778
|
Use connect without --remote-config when the cloud control plane owns the connection profile.
|
|
692
779
|
Prefer connect --remote-config over --daemon-base-url, --tenant, --run-id, and --lease-id when using a local profile.
|
|
693
780
|
Use agent-device proxy for direct tunnel access to a Mac you control. Expose the printed proxy URL through cloudflared/ngrok, then run agent-device connect proxy with the tunnel URL and printed token before normal commands.
|
|
694
|
-
Use BrowserStack and AWS Device Farm through local provider profiles; they do not accept a remote agent-device daemon URL.
|
|
695
|
-
Device cloud credentials must be available before the command starts. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables.
|
|
696
|
-
Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not BrowserStack access keys or AWS credentials.
|
|
781
|
+
Use Limrun, BrowserStack, and AWS Device Farm through local provider profiles; they do not accept a remote agent-device daemon URL.
|
|
782
|
+
Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables.
|
|
783
|
+
Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not Limrun API keys, BrowserStack access keys, or AWS credentials.
|
|
784
|
+
Limrun Android supports direct ADB port reverse for local Metro. Limrun iOS requires a public Metro/React DevTools URL because it cannot reach local host ports directly.
|
|
697
785
|
After closing a device cloud session, run agent-device artifacts --json to retrieve provider video/log/dashboard URLs when the provider has made them available.
|
|
698
786
|
connect proxy stores the connection profile and client identity. Device leases are acquired on open and expire after five minutes without commands.
|
|
699
787
|
Multiple agents can share one proxy when each uses connect proxy, open, commands, close, and disconnect.
|
|
@@ -832,6 +920,8 @@ Report shape:
|
|
|
832
920
|
Rules:
|
|
833
921
|
Findings must come from observed runtime behavior, not source reads.
|
|
834
922
|
After each mutation, use the --settle diff as evidence when available; otherwise re-snapshot.
|
|
923
|
+
Wait timeouts are integer milliseconds in the trailing positional: agent-device wait 'role=tab' 10000. Do not write duration suffixes such as 10s.
|
|
924
|
+
scroll takes direction then amount and does not support a selector or --settle: agent-device scroll down 3.
|
|
835
925
|
Keep commands in the report reproducible; use selectors or refs from fresh snapshots, not guessed coordinates.
|
|
836
926
|
Prefer refs for exploration and selectors for deterministic replay.
|
|
837
927
|
Use logs, network, screenshot --overlay-refs, trace, perf metrics, perf frames, or react-devtools only when they add evidence to a specific issue.
|
|
@@ -844,9 +934,10 @@ Contract:
|
|
|
844
934
|
Prove the changed behavior through public agent-device surfaces. Do not validate against stale dist output, a retained stale daemon, or a runner built before the change.
|
|
845
935
|
Keep evidence reproducible: exact commands, target device/app, observed output, artifact paths, and cleanup status.
|
|
846
936
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
937
|
+
Required freshness gate before device verification:
|
|
938
|
+
For a TypeScript runtime or CLI output change, start with pnpm build. For non-Android device verification, run pnpm clean:daemon next.
|
|
939
|
+
Before local Android verification, run pnpm build:android before pnpm clean:daemon so the bundled helpers match current source.
|
|
940
|
+
For an Apple runner change, run pnpm build:xcuitest and avoid inherited retained runners from older source. Do not build the Apple runner for TypeScript-only changes.
|
|
850
941
|
Use open --relaunch when startup state matters. Use a purpose-specific --session for multi-step validation.
|
|
851
942
|
|
|
852
943
|
Loop:
|
|
@@ -868,13 +959,13 @@ Evidence:
|
|
|
868
959
|
|
|
869
960
|
Report:
|
|
870
961
|
Summarize what changed, exact validation commands, pass/fail observations, artifact paths, and residual risk.
|
|
871
|
-
If live validation is blocked, state the blocker, device/session, and exact next command needed.`}};function
|
|
962
|
+
If live validation is blocked, state the blocker, device/session, and exact next command needed.`}};function v(e,t,n){let r=n.endsWith(`?`),i=r?n.slice(0,-1):n,a=/^[a-z-]+(?:\|[a-z-]+)+$/i.test(i),o=a||t.usageOverride!==void 0&&t.usageOverride.startsWith(`${e} ${i}`);return r?a?`[${i}]`:o?i:`[${i}]`:o?i:`<${i}>`}function y(e,t){return t.listUsageOverride?t.listUsageOverride:[e,...(t.positionalArgs??[]).map(n=>v(e,t,n))].join(` `)}function b(){let n=E(e().map(e=>{let t=r(e);return{name:e,schema:t,usage:y(e,t)}})),i=C(`Global Flags:`,S(t)),a=T(`Agent Starting Point:`,f),o=T(`Agent Quickstart:`,p);return`agent-device <command> [args] [--json]
|
|
872
963
|
|
|
873
964
|
CLI to automate supported app, device, desktop, and web targets for AI agents.
|
|
874
965
|
|
|
875
966
|
${a}
|
|
876
967
|
|
|
877
|
-
${
|
|
968
|
+
${w(`Agent Workflows:`,d)}
|
|
878
969
|
|
|
879
970
|
${n}
|
|
880
971
|
|
|
@@ -882,26 +973,26 @@ ${i}
|
|
|
882
973
|
|
|
883
974
|
${o}
|
|
884
975
|
|
|
885
|
-
${
|
|
976
|
+
${T(`Configuration:`,m)}
|
|
886
977
|
|
|
887
|
-
${
|
|
978
|
+
${w(`Environment:`,h)}
|
|
888
979
|
|
|
889
|
-
${
|
|
890
|
-
`}function
|
|
891
|
-
`)}function
|
|
892
|
-
`)}function
|
|
980
|
+
${T(`Examples:`,g)}
|
|
981
|
+
`}function x(){return b()}function S(e){return n().filter(t=>e.has(t.key)&&t.usageLabel!==void 0&&t.usageDescription!==void 0)}function C(e,t){return w(e,t.map(e=>({label:e.usageLabel??``,description:e.usageDescription??``})))}function w(e,t){if(t.length===0)return`${e}\n (none)`;let n=Math.max(...t.map(e=>Math.min(e.label.length,26)))+2,r=[e];for(let e of t){let t=e.label.length<=26?e.label.padEnd(n):`${e.label} `;r.push(` ${t}${e.description}`)}return r.join(`
|
|
982
|
+
`)}function T(e,t){return t.length===0?`${e}\n (none)`:[e,...t.map(e=>` ${e}`)].join(`
|
|
983
|
+
`)}function E(e){return w(`Commands:`,e.map(e=>({label:e.usage,description:e.schema.summary??e.schema.helpDescription})))}function D(e){let t=O(e);if(t)return t;let n=i(e);if(!n)return null;let r=u(e,n),a=S(new Set(n.allowedFlags??[])),o=[];a.length>0&&o.push(C(`Command flags:`,a));let s=o.length>0?`\n\n${o.join(`
|
|
893
984
|
|
|
894
985
|
`)}`:``;return`agent-device ${r}
|
|
895
986
|
|
|
896
987
|
${n.helpDescription}
|
|
897
988
|
|
|
898
989
|
Usage:
|
|
899
|
-
agent-device ${r}${
|
|
900
|
-
`}function
|
|
990
|
+
agent-device ${r}${s}
|
|
991
|
+
`}function O(e){let t=_[e];return t?`${t.body}
|
|
901
992
|
|
|
902
993
|
Related:
|
|
903
994
|
agent-device help command list and global flags
|
|
904
995
|
agent-device help <command> command-specific flags
|
|
905
996
|
agent-device help manual-qa routine QA loop with concrete command shapes
|
|
906
997
|
agent-device help workflow full app automation reference
|
|
907
|
-
`:null}export{
|
|
998
|
+
`:null}export{D as buildCommandUsageText,x as buildUsageText};
|
package/dist/src/cli-runner.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{fn as e}from"./recording-export-quality.js";import{n as t}from"./registry.js";import{n}from"./cli-config.js";import{n as r}from"./command-surface.js";function i(t,n,r){if(t.json){e({success:!0,data:n});return}let i=r?.();i&&a(i)}function a(e){process.stdout.write(e.endsWith(`
|
|
2
2
|
`)?e:`${e}\n`)}const o=t();function s(e,t,n){let r=o[e];if(!r)throw Error(`Missing CLI reader for command: ${e}`);return r(t,n)}async function c(e){return(await l(e)).result}async function l(e){let t=s(e.command,e.positionals,e.flags),i=await r(e.client,e.command,t);return{result:i,cliOutput:n({name:e.command,input:t,result:i})}}export{i,l as n,s as r,c as t};
|