@loadmill/droid-cua 2.23.0 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/build/index.js +12 -0
- package/build/src/cli/headless-runner.js +2 -0
- package/build/src/cli/skill-command.js +240 -0
- package/build/src/commands/help.js +2 -1
- package/build/src/commands/run.js +5 -3
- package/build/src/core/cua-models.js +29 -9
- package/build/src/core/droid-cua-run-usage.js +16 -7
- package/build/src/core/execution-engine.js +7 -1
- package/build/src/core/runtime-config.js +2 -5
- package/build/src/core/session.js +2 -2
- package/build/src/core/tool-handlers/close-keyboard.js +103 -0
- package/build/src/device/actions.js +7 -0
- package/build/src/device/android/actions.js +16 -0
- package/build/src/device/cloud/actions.js +27 -0
- package/build/src/device/cloud/cloud-appium-client.js +15 -0
- package/build/src/device/connection.js +5 -2
- package/build/src/device/factory.js +3 -0
- package/build/src/device/interface.js +4 -0
- package/build/src/device/ios/actions.js +13 -0
- package/build/src/device/ios/appium-client.js +16 -0
- package/build/src/device/keyboard.js +119 -0
- package/build/src/device/openai.js +19 -1
- package/build/src/modes/design-mode-ink.js +1 -0
- package/build/src/modes/design-mode.js +1 -0
- package/build/src/modes/execution-mode.js +88 -16
- package/build/src/prompts/base.js +29 -4
- package/build/src/prompts/execution.js +14 -3
- package/package.json +2 -1
- package/skill/droid-cua/SKILL.md +39 -0
- package/skill/droid-cua/agents/openai.yaml +6 -0
- package/skill/droid-cua/assets/droid-cua-app-icon.png +0 -0
- package/skill/droid-cua/references/authoring-tests.md +39 -0
- package/skill/droid-cua/references/onboarding-and-troubleshooting.md +74 -0
- package/skill/droid-cua/references/project-and-auth.md +41 -0
- package/skill/droid-cua/references/running-and-healing.md +110 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
2
|
+
import { CUA_CONVERSATION_POLICIES, resolveConversationPolicy } from "../core/cua-models.js";
|
|
2
3
|
import { normalizeSecretKeys } from "../core/secrets.js";
|
|
3
4
|
import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
|
|
4
5
|
export function buildAppContextSection(text) {
|
|
@@ -43,10 +44,14 @@ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, ap
|
|
|
43
44
|
const testDataContextSection = buildTestDataContextSection(options.testDataContext);
|
|
44
45
|
const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
|
|
45
46
|
const browserTarget = isBrowserTarget(deviceInfo);
|
|
47
|
+
const conversationPolicy = resolveConversationPolicy(options.cuaModel);
|
|
48
|
+
const instructionContextRule = conversationPolicy === CUA_CONVERSATION_POLICIES.ISOLATED
|
|
49
|
+
? "This model starts each instruction without prior model history. Execute only the current instruction and stop when it is complete."
|
|
50
|
+
: "Instructions are sequential parts of one scenario. Use prior instructions and outcomes to resolve references in the current instruction, but execute only the current instruction. Preserve expected values established earlier; never replace them with the observed state merely to produce a pass.";
|
|
46
51
|
const prompt = `${basePrompt}
|
|
47
52
|
|
|
48
53
|
EXECUTION MODE - Critical Behavior:
|
|
49
|
-
You are executing test script commands one at a time. This is
|
|
54
|
+
You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
|
|
50
55
|
|
|
51
56
|
CRITICAL RULES:
|
|
52
57
|
- DO NOT generate conversational text or narration
|
|
@@ -70,14 +75,20 @@ Your process:
|
|
|
70
75
|
- Before calling \`assert\`, make sure the current screen contains the evidence needed to validate the condition. If needed, perform only minimal preparation first, such as taking a screenshot, waiting briefly, closing the keyboard, scrolling within the relevant view, or opening the relevant menu.
|
|
71
76
|
- Once you call \`assert\`, validation is read-only. The assertion tool will not click, type, wait, scroll, request screenshots, or change the device state.
|
|
72
77
|
- Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
|
|
78
|
+
${browserTarget ? "" : ` - Every \`type\` action must end its computer-action batch so you can inspect the post-type screenshot
|
|
79
|
+
- Call \`close_keyboard\` next when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden
|
|
80
|
+
- Do not complete the current instruction while that required \`close_keyboard\` action is pending
|
|
81
|
+
- Otherwise, including when only a slim hardware-keyboard toolbar or suggestions are visible, continue without \`close_keyboard\`
|
|
82
|
+
- Apply the same dismissal decision after \`type_sensitive_text\`
|
|
83
|
+
`}
|
|
73
84
|
- Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
|
|
74
85
|
3. Execute the required actions
|
|
75
|
-
4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
|
|
86
|
+
4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
|
|
76
87
|
5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
|
|
77
88
|
6. Stop as soon as success is visible
|
|
78
89
|
7. Stop immediately - no commentary, no questions
|
|
79
90
|
|
|
80
|
-
|
|
91
|
+
${instructionContextRule}
|
|
81
92
|
${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
|
|
82
93
|
return appendCustomSections(prompt, [
|
|
83
94
|
{ title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loadmill/droid-cua",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.25.0",
|
|
4
4
|
"description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"build/**/*",
|
|
12
12
|
"bin/**/*",
|
|
13
|
+
"skill/**/*",
|
|
13
14
|
"README.md",
|
|
14
15
|
"LICENSE"
|
|
15
16
|
],
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: droid-cua
|
|
3
|
+
description: Operate Droid CUA `.dcua` test projects. Use when the user explicitly names Droid CUA or @loadmill/droid-cua, or asks to author, configure, run, or diagnose `.dcua` tests with the Droid CUA CLI or desktop app. Do not use for general web browsing, website development, browser automation, generic mobile work, or non-Droid test tasks.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Droid CUA
|
|
7
|
+
|
|
8
|
+
Use the existing `droid-cua` CLI and repository files. Do not invent commands, flags, or product capabilities.
|
|
9
|
+
|
|
10
|
+
## Workflow
|
|
11
|
+
|
|
12
|
+
1. Inspect the active test project before creating files. Find existing `.dcua` tests, `context.md`, `test-data.md`, `.secrets`, `.env`, and `ci/droid-cua.json` without reading secret values. Do not search other projects or the user's home directory for credentials.
|
|
13
|
+
2. Determine the requested platform and target from the task or project. Ask the user only when the choice is unclear.
|
|
14
|
+
3. Check authentication and environment readiness before the first run. Read [project-and-auth.md](references/project-and-auth.md) for project boundaries and token handling. Read [onboarding-and-troubleshooting.md](references/onboarding-and-troubleshooting.md) when setup, dependency, device, simulator, browser, or connection work is needed.
|
|
15
|
+
4. Author or update the smallest useful test. Read [authoring-tests.md](references/authoring-tests.md) before changing `.dcua`, context, test data, secrets, or config.
|
|
16
|
+
5. Before execution, present the proposed tests and target to the user, resolve any questions, and get their confirmation. Read [running-and-healing.md](references/running-and-healing.md) for the required pre-run check.
|
|
17
|
+
6. Run the confirmed, narrowest relevant test and monitor it to completion. Keep the user oriented during the run by surfacing meaningful live screenshots, not just terminal output.
|
|
18
|
+
7. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
|
|
19
|
+
|
|
20
|
+
## Safety
|
|
21
|
+
|
|
22
|
+
- Never print, open, or echo secret values. Check only whether required keys are configured.
|
|
23
|
+
- Never search for credentials outside the active test project or source another project's `.env` file.
|
|
24
|
+
- Use Loadmill for every run unless the user explicitly asks to use their own OpenAI API key. A discovered OpenAI key is never permission to switch providers.
|
|
25
|
+
- Never use credentials from another project, a skill, an agent configuration, or the user's home directory. Configure credentials only in the active test project's `.env` (or an explicitly supplied CI secret).
|
|
26
|
+
- Get approval before installing dependencies, running `sudo`, accepting licenses, or changing system configuration.
|
|
27
|
+
- During setup, do not create or edit user tests. Rerun only the failed readiness check, then resume the original task.
|
|
28
|
+
- During healing, edit only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never modify application code under the guise of healing a test.
|
|
29
|
+
- Never weaken or remove an assertion merely to make a failing run pass.
|
|
30
|
+
- Make one minimal change and run the affected test again. Stop after two edit/rerun cycles and present the evidence.
|
|
31
|
+
- Stop immediately when evidence indicates a likely application regression; do not rewrite the test to match broken behavior.
|
|
32
|
+
- Gracefully allow cloud runs to finish cleanup. Avoid terminating a metered device session unless necessary.
|
|
33
|
+
|
|
34
|
+
## Operating principles
|
|
35
|
+
|
|
36
|
+
- Prefer an existing project convention over creating a parallel structure.
|
|
37
|
+
- Prefer headless runs for agent work because they are observable and return deterministic exit codes.
|
|
38
|
+
- Use explicit, user-observable outcomes in tests rather than implementation details or coordinates.
|
|
39
|
+
- Diagnose the first divergence between the instruction and observed behavior before changing anything.
|
|
Binary file
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Authoring tests
|
|
2
|
+
|
|
3
|
+
## Inspect first
|
|
4
|
+
|
|
5
|
+
Find nearby `.dcua` tests and project files before writing. Match their directory layout, naming, vocabulary, starting state, and config conventions. Do not open `.env` or `.secrets`.
|
|
6
|
+
|
|
7
|
+
## Write `.dcua` instructions
|
|
8
|
+
|
|
9
|
+
- Put one clear instruction or assertion on each non-empty line.
|
|
10
|
+
- Describe visible UI intent, not coordinates, selectors, implementation details, or model strategy.
|
|
11
|
+
- Prefer a high-level business intent when `context.md` makes the flow unambiguous. Do not translate every tap and keystroke into a brittle micro-script.
|
|
12
|
+
- Break an instruction down only when the agent demonstrably needs help finding a control or completing part of the flow.
|
|
13
|
+
- State the starting condition when it is not guaranteed by the environment.
|
|
14
|
+
- Use explicit assertions for important outcomes. Prefer stable visible facts such as a screen title, confirmation message, changed value, or navigation result.
|
|
15
|
+
- Use exact visible labels when naming controls or outcomes.
|
|
16
|
+
- Keep each test focused on one behavior. Split unrelated flows.
|
|
17
|
+
- Use `//` comments sparingly for human context; comments are not executed.
|
|
18
|
+
- Never include secret values in the test.
|
|
19
|
+
|
|
20
|
+
Example:
|
|
21
|
+
|
|
22
|
+
```text
|
|
23
|
+
Sign in using the USER_EMAIL and USER_PASSWORD secrets.
|
|
24
|
+
Assert that the Home screen is visible and shows the signed-in account.
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Use the secret key names that exist in `.secrets`; do not inspect their values. If key discovery is necessary, filter the file so only keys reach output, for example `sed -n 's/^\([^#][^=]*\)=.*/\1/p' .secrets`. If a required key is absent, ask the user to add it without sharing its value.
|
|
28
|
+
|
|
29
|
+
## Put knowledge in the right place
|
|
30
|
+
|
|
31
|
+
- Put durable product behavior and terminology in `context.md`, not repeated in every test.
|
|
32
|
+
- Put non-secret reusable people, accounts, products, and scenario combinations in `test-data.md`.
|
|
33
|
+
- Put credentials and sensitive values in `.secrets`.
|
|
34
|
+
- Put reproducible headless settings in `ci/droid-cua.json`.
|
|
35
|
+
- Keep environment and provider credentials in `.env` or the shell/CI secret store.
|
|
36
|
+
|
|
37
|
+
Use `--scenario "<goal>" --test-data <path>` only when the user wants generated permutations or cases. A normal `.dcua` run does not automatically consume `test-data.md`.
|
|
38
|
+
|
|
39
|
+
After editing, show the user the test instructions and explain any new context, data keys, or secret keys required. Never show secret values.
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Onboarding and troubleshooting
|
|
2
|
+
|
|
3
|
+
Follow the Desktop Device Setup Wizard sequence: choose a platform, run checks, discover/connect a target, verify runtime access during the requested run, then continue the original task. Run authentication as the separate first preflight described in `project-and-auth.md`.
|
|
4
|
+
|
|
5
|
+
Do not create or modify user test files merely to perform setup. Get approval before installing software, changing system configuration, running `sudo`, or accepting a license. After remediation, rerun only the failed check.
|
|
6
|
+
|
|
7
|
+
## Android
|
|
8
|
+
|
|
9
|
+
Resolve `adb` and `emulator` from `ANDROID_HOME`, `ANDROID_SDK_ROOT`, the macOS default `~/Library/Android/sdk`, the Windows default `%LOCALAPPDATA%\Android\Sdk`, then `PATH`. Respect explicit `DROID_CUA_ANDROID_ADB_PATH` and `DROID_CUA_ANDROID_EMULATOR_PATH` overrides.
|
|
10
|
+
|
|
11
|
+
Run these checks:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
adb version
|
|
15
|
+
emulator -list-avds
|
|
16
|
+
adb devices
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
- `adb` is blocking.
|
|
20
|
+
- The emulator CLI and existence of an AVD are individually non-blocking.
|
|
21
|
+
- Target availability is blocking and passes when `adb devices` contains a `device` entry or `emulator -list-avds` contains a launchable AVD.
|
|
22
|
+
- For `unauthorized`, ask the user to accept the computer authorization prompt on the device.
|
|
23
|
+
- For `offline`, reconnect the device or restart ADB, then retry.
|
|
24
|
+
- If nothing is available, guide the user to connect a device or create an AVD in Android Studio Device Manager.
|
|
25
|
+
|
|
26
|
+
Use `--avd adb:<serial>` for an attached target and `--avd avd:<name>` to launch an AVD.
|
|
27
|
+
|
|
28
|
+
## iOS
|
|
29
|
+
|
|
30
|
+
Local iOS execution requires macOS. Run checks in this order:
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
xcrun --version
|
|
34
|
+
xcode-select -p
|
|
35
|
+
xcodebuild -license check
|
|
36
|
+
xcodebuild -checkFirstLaunchStatus
|
|
37
|
+
node --version
|
|
38
|
+
npm --version
|
|
39
|
+
appium --version
|
|
40
|
+
appium driver list --installed
|
|
41
|
+
xcrun simctl list devices --json
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Require `xcode-select -p` to point to an `/Xcode*.app/Contents/Developer` path, an installed XCUITest driver, and at least one available iPhone simulator. A booted simulator is helpful but not blocking.
|
|
45
|
+
|
|
46
|
+
Offer only the relevant remediation, with approval:
|
|
47
|
+
|
|
48
|
+
```sh
|
|
49
|
+
sudo xcode-select -s /Applications/Xcode.app/Contents/Developer
|
|
50
|
+
sudo xcodebuild -license accept
|
|
51
|
+
sudo xcodebuild -runFirstLaunch
|
|
52
|
+
npm install -g appium
|
|
53
|
+
appium driver install xcuitest
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Use `--platform ios --avd "<simulator name>"` for the run.
|
|
57
|
+
|
|
58
|
+
## Web
|
|
59
|
+
|
|
60
|
+
Droid CUA automatically discovers Chrome or Edge on macOS and Windows. Check the standard application locations, then run the requested test with `--target web --browser <chrome|edge> --session-mode fresh`. A successful Playwright launch is the runtime verification; do not create a persistent setup-only test.
|
|
61
|
+
|
|
62
|
+
On Linux, automatic installed-browser discovery is not currently supported. Do not claim that a PATH-installed browser will be found.
|
|
63
|
+
|
|
64
|
+
## Cloud targets
|
|
65
|
+
|
|
66
|
+
For LambdaTest, require `LAMBDATEST_USERNAME`, `LAMBDATEST_ACCESS_KEY`, a device name, OS version, and `.apk` or `.ipa`. The CLI uploads the app for each run.
|
|
67
|
+
|
|
68
|
+
For AWS Device Farm, require `AWS_DEVICE_FARM_ACCESS_KEY_ID`, `AWS_DEVICE_FARM_SECRET_ACCESS_KEY`, and `AWS_DEVICE_FARM_PROJECT_ARN`. Discover targets with:
|
|
69
|
+
|
|
70
|
+
```sh
|
|
71
|
+
droid-cua --device-source aws-device-farm --list-devices
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Do not document BrowserStack CLI runs; BrowserStack is not consistently exposed by the current CLI.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Project files and authentication
|
|
2
|
+
|
|
3
|
+
## Project files
|
|
4
|
+
|
|
5
|
+
- `.dcua`: executable test instructions. Each non-empty line is one instruction; `//` begins a comment.
|
|
6
|
+
- `context.md`: durable knowledge about product terminology, screens, navigation, and behavior. Pass it with `--context`, or reference it from `ci/droid-cua.json`.
|
|
7
|
+
- `test-data.md`: non-secret entities and scenario inputs. It is used with `--scenario` and `--test-data`; it is not automatically loaded by every normal test run.
|
|
8
|
+
- `.secrets`: dotenv-style values that Droid CUA may type into the tested application. The model receives the keys, not the resolved values. Headless execution loads this file from the current working directory; `--secrets` overrides it.
|
|
9
|
+
- `.env`: infrastructure credentials and runtime environment variables loaded from the current working directory, including `LOADMILL_API_TOKEN`.
|
|
10
|
+
- `ci/droid-cua.json`: optional checked-in headless config. Paths inside it are relative to the config file.
|
|
11
|
+
- `logs/`: structured JSONL and screenshot artifacts created by `--debug`.
|
|
12
|
+
- HTML reports: self-contained evidence produced by `--report`.
|
|
13
|
+
|
|
14
|
+
Keep `.env` and `.secrets` out of version control. Use `git check-ignore -q .env` and `git check-ignore -q .secrets` to check ignore coverage without opening either file.
|
|
15
|
+
|
|
16
|
+
## Loadmill authentication
|
|
17
|
+
|
|
18
|
+
Loadmill is the required default LLM provider. Use it unless the user explicitly asks to use their own OpenAI API key. Never switch to OpenAI because a Loadmill token is missing or because an OpenAI key happens to be available.
|
|
19
|
+
|
|
20
|
+
Before a run, check only for the presence of a Loadmill token in the active test project's `.env` or explicitly supplied CI secret:
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
grep -Eq '^LOADMILL_API_TOKEN=.+$' .env 2>/dev/null
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Run this check from the active test project's working directory. Do not print the variable, run `env`, open `.env`, or inspect inherited environment variables. Do not search parent directories, sibling repositories, the user's home directory, agent skills/configuration, or the Droid CUA source repository for credentials.
|
|
27
|
+
|
|
28
|
+
If it is missing, explain that Droid CUA uses a Loadmill token by default and guide the user to:
|
|
29
|
+
|
|
30
|
+
1. Sign in, then open [Loadmill Settings → Security](https://app.loadmill.com/app/user/settings/security).
|
|
31
|
+
2. Create an API token with a description such as `Droid CUA CLI`.
|
|
32
|
+
3. Copy it when shown; Loadmill shows it once.
|
|
33
|
+
|
|
34
|
+
Offer exactly these two choices, then wait for the user's selection:
|
|
35
|
+
|
|
36
|
+
1. **Use it once:** The user provides the token through the agent's secret-input mechanism and asks to use it for this run only. Pass it only to that command through a secret-safe mechanism; do not write it to `.env`, a shell profile, or any other file.
|
|
37
|
+
2. **Save it for this project:** The user provides the token through the agent's secret-input mechanism and approves saving it. Create the active project's `.env` if needed or add/update only its `LOADMILL_API_TOKEN` entry. First ensure `.env` is ignored by version control; add `.env` to the active project's `.gitignore` when needed.
|
|
38
|
+
|
|
39
|
+
Use a secret-safe mechanism that does not print or expose the value in commands, logs, or responses. Verify only that the key is present. Never copy a token from another file, a shell profile, another project, an agent skill/configuration, or any other location. Do not inspect inherited environment variables to discover a token. When the active project's `.env` contains the key, run `droid-cua` normally from that project; the CLI loads that file automatically.
|
|
40
|
+
|
|
41
|
+
Only when the user explicitly selects direct OpenAI access, use `--llm-provider openai` with one of the supported OpenAI CUA models: `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Check only the active project's `.env` for `OPENAI_API_KEY`; do not use one found elsewhere. If it is missing, guide the user to create and configure their own key in the same project using the same secret-safe process. Do not silently switch providers to bypass missing Loadmill authentication.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Running, monitoring, and guarded healing
|
|
2
|
+
|
|
3
|
+
## Construct the run
|
|
4
|
+
|
|
5
|
+
Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Beacon when the user prioritizes faster, lower-cost execution:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
# Recommended default
|
|
9
|
+
--llm-provider loadmill --cua-model loadmill-smart
|
|
10
|
+
|
|
11
|
+
# Faster, lower-cost option
|
|
12
|
+
--llm-provider loadmill --cua-model loadmill-beacon
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Both Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart and Beacon silently; preserve the project's model or explain the speed/cost tradeoff and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
|
|
16
|
+
|
|
17
|
+
Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` for every agent-observed run: it writes the screenshots needed for live progress updates as well as diagnosis.
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
# Attached Android device or running emulator
|
|
21
|
+
droid-cua --llm-provider loadmill --cua-model loadmill-smart --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
|
|
22
|
+
|
|
23
|
+
# Launchable Android AVD
|
|
24
|
+
droid-cua --avd avd:<avd-name> --instructions tests/example.dcua --config ci/droid-cua.json --debug --report reports/example.html
|
|
25
|
+
|
|
26
|
+
# iOS Simulator
|
|
27
|
+
droid-cua --llm-provider loadmill --cua-model loadmill-smart --platform ios --avd "<simulator-name>" --instructions tests/example.dcua --debug --report reports/example.html
|
|
28
|
+
|
|
29
|
+
# Local web
|
|
30
|
+
droid-cua --llm-provider loadmill --cua-model loadmill-smart --target web --browser chrome --session-mode fresh --instructions tests/example.dcua --debug --report reports/example.html
|
|
31
|
+
|
|
32
|
+
# LambdaTest
|
|
33
|
+
droid-cua --llm-provider loadmill --cua-model loadmill-smart --device-source lambdatest --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
|
|
34
|
+
|
|
35
|
+
# AWS Device Farm
|
|
36
|
+
droid-cua --llm-provider loadmill --cua-model loadmill-smart --device-source aws-device-farm --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
|
|
37
|
+
|
|
38
|
+
# Direct OpenAI alternative
|
|
39
|
+
droid-cua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Use `.ipa` and `--platform ios` for supported cloud iOS runs. Preserve existing config flags rather than replacing them with defaults.
|
|
43
|
+
|
|
44
|
+
## Confirm the run with the user
|
|
45
|
+
|
|
46
|
+
Before starting any test command, send the user a short, plain-language run plan and wait for explicit confirmation. Treat this as a product conversation, not a command preview. Do not infer that a request to run a test also confirms an automatically selected device, browser, or batch of tests.
|
|
47
|
+
|
|
48
|
+
The run plan should briefly state:
|
|
49
|
+
|
|
50
|
+
- What behavior or flow will be checked, and whether one test or several will run. Mention the test filename only when it helps identify the test.
|
|
51
|
+
- Where it will run: the chosen device, simulator, or browser. If more than one viable target is available, present the human-readable choices and ask the user to choose.
|
|
52
|
+
- Any user-visible consequence, such as launching an emulator, resetting a fresh browser session, or starting a metered cloud-device session.
|
|
53
|
+
- That the agent will share progress screenshots and provide a report afterward.
|
|
54
|
+
|
|
55
|
+
Do not include CLI flags, commands, provider or model names, credential/key names, config/context filenames, or internal log/report paths unless the user asks or one changes a decision they need to make. Do not paste the test source into the confirmation.
|
|
56
|
+
|
|
57
|
+
Ask the user about every unresolved or ambiguous choice before running. If the plan is complete, ask for a clear go-ahead (for example, `Ready to run this?`). Start only after they confirm. If they change the tests, target, or other material run detail, show the revised plan and confirm it again.
|
|
58
|
+
|
|
59
|
+
For example: `I’m ready to run the Chrome smoke test in a fresh local Chrome window. It will check that Chrome opens and then closes. I’ll share screenshots as it progresses and provide a report at the end. Ready to run this?`
|
|
60
|
+
|
|
61
|
+
## Monitor and collect evidence
|
|
62
|
+
|
|
63
|
+
Remain attached until the command exits and cloud cleanup finishes. Run the command through an interactive/pollable terminal mechanism so progress and new artifacts can be observed before it exits. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
|
|
64
|
+
|
|
65
|
+
## Show live visual progress
|
|
66
|
+
|
|
67
|
+
Use the screenshots already captured by `--debug` to make the run understandable while it is happening. This is a required user-facing behavior, not merely an artifact to inspect after the run. After the CLI prints `Debug logging enabled: <logs/execution-...>.jsonl`, new PNGs appear in that log file's sibling directory. Keep the CLI running in a pollable/background session and poll that directory while the test is running.
|
|
68
|
+
|
|
69
|
+
For agents whose chat UI renders local Markdown images (including Cursor and Codex), send a separate progress message containing an absolute-path image, for example:
|
|
70
|
+
|
|
71
|
+
```md
|
|
72
|
+
Step 2/4: add the item to the cart
|
|
73
|
+
|
|
74
|
+

|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Do not put this Markdown only in a terminal command, tool argument, or hidden reasoning: it must be an actual user-visible chat update. Do not replace it by opening the final report.
|
|
78
|
+
|
|
79
|
+
- After every completed executable instruction, send exactly one short progress update with that instruction and its screenshot (for example, `Step 2/4 complete: add the item to the cart`). An image path alone is not a visual update.
|
|
80
|
+
- Screenshot filenames include `instruction-001`, `instruction-002`, and so on. Select the latest safe PNG for the completed instruction; prefer an end-of-turn or assertion screenshot over an input or click-preview frame. This connects each user-visible update to the matching script step even when the model took several actions to finish it.
|
|
81
|
+
- Also show a frame immediately when the run is retrying, reconnecting, or fails. Do not stream every model action or repeat visually identical frames.
|
|
82
|
+
- Do not show `preview_click` / aim-assist annotations unless they help explain an issue.
|
|
83
|
+
- If the latest frame visibly exposes a secret or other sensitive user data, do not render it; provide a text-only progress update and wait for a safe frame.
|
|
84
|
+
- Do not delay the test to collect screenshots. If no new frame exists during a long model call, report the current stage in text and keep polling.
|
|
85
|
+
|
|
86
|
+
When the command completes, show the final safe screenshot in addition to the result summary, then provide a clickable link to the HTML report (or its absolute path when local links are unavailable). The report is the final artifact; the interim frames are progress updates, not a substitute for it.
|
|
87
|
+
|
|
88
|
+
- Exit `0`: all requested tests passed.
|
|
89
|
+
- Exit `1`: configuration, connection, execution, or assertion failure.
|
|
90
|
+
- Exit `130`: interrupted run.
|
|
91
|
+
- `--debug`: writes structured execution JSONL and adjacent screenshots under `logs/`.
|
|
92
|
+
- `--report`: writes a self-contained HTML report. Prefer an explicit `.html` file path such as `reports/example.html` so the evidence location is unambiguous.
|
|
93
|
+
|
|
94
|
+
After the run, verify that the report exists and resolve its absolute path. Open it for the user when the agent environment supports local-file opening and policy permits; otherwise provide a clickable local-file link or the absolute path so the user can open it directly.
|
|
95
|
+
|
|
96
|
+
Report the outcome, failing instruction and first divergence when applicable, relevant error text, screenshot/debug/report paths, target, provider/model, and command with secret values omitted.
|
|
97
|
+
|
|
98
|
+
## Classify the first divergence
|
|
99
|
+
|
|
100
|
+
1. **Setup or authentication**: missing/invalid provider credential, dependency, license, browser, or executable. Return to onboarding and rerun only the failed check.
|
|
101
|
+
2. **Target connectivity**: no device, unauthorized/offline ADB state, unavailable simulator, cloud credential/device/app problem, or lost session. Restore the same target, then rerun.
|
|
102
|
+
3. **Test ambiguity or stale test knowledge**: instruction has multiple interpretations, relies on an unstable label, lacks a starting condition, or context contradicts the visible product. A minimal test/context correction is allowed.
|
|
103
|
+
4. **Application regression**: the app crashes, displays an error, violates an explicit requirement, or consistently fails a stable assertion after correct setup. Stop and report a bug with evidence.
|
|
104
|
+
5. **Transient execution**: temporary network, service, model, or timing failure without evidence of a deterministic product/test issue. Retry the unchanged targeted test once.
|
|
105
|
+
|
|
106
|
+
## Guarded healing
|
|
107
|
+
|
|
108
|
+
Before editing, state the classification and evidence. Change only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never edit app code, delete coverage, loosen the expected outcome, or add broad waits simply to obtain green output.
|
|
109
|
+
|
|
110
|
+
Make one minimal change, explain why it corrects the test rather than hides a product defect, and rerun only the affected test. Allow at most two edit/rerun cycles. If it still fails, stop with the original and final evidence, changes attempted, and recommended owner/action.
|