@loadmill/droid-cua 2.24.0 → 2.26.0

package/build/src/prompts/execution.js

@@ -1,4 +1,5 @@
 import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
+import { CUA_CONVERSATION_POLICIES, resolveConversationPolicy } from "../core/cua-models.js";
 import { normalizeSecretKeys } from "../core/secrets.js";
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
 export function buildAppContextSection(text) {
@@ -43,10 +44,14 @@ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, ap
     const testDataContextSection = buildTestDataContextSection(options.testDataContext);
     const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
+    const conversationPolicy = resolveConversationPolicy(options.cuaModel);
+    const instructionContextRule = conversationPolicy === CUA_CONVERSATION_POLICIES.ISOLATED
+        ? "This model starts each instruction without prior model history. Execute only the current instruction and stop when it is complete."
+        : "Instructions are sequential parts of one scenario. Use prior instructions and outcomes to resolve references in the current instruction, but execute only the current instruction. Preserve expected values established earlier; never replace them with the observed state merely to produce a pass.";
     const prompt = `${basePrompt}
 
 EXECUTION MODE - Critical Behavior:
-You are executing test script commands one at a time. This is NOT a conversation.
+You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
 
 CRITICAL RULES:
 - DO NOT generate conversational text or narration
@@ -70,14 +75,20 @@ Your process:
    - Before calling \`assert\`, make sure the current screen contains the evidence needed to validate the condition. If needed, perform only minimal preparation first, such as taking a screenshot, waiting briefly, closing the keyboard, scrolling within the relevant view, or opening the relevant menu.
    - Once you call \`assert\`, validation is read-only. The assertion tool will not click, type, wait, scroll, request screenshots, or change the device state.
    - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
+${browserTarget ? "" : `   - Every \`type\` action must end its computer-action batch so you can inspect the post-type screenshot
+   - Call \`close_keyboard\` next when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden
+   - Do not complete the current instruction while that required \`close_keyboard\` action is pending
+   - Otherwise, including when only a slim hardware-keyboard toolbar or suggestions are visible, continue without \`close_keyboard\`
+   - Apply the same dismissal decision after \`type_sensitive_text\`
+`}
    - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
 3. Execute the required actions
-4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
+4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
 6. Stop as soon as success is visible
 7. Stop immediately - no commentary, no questions
 
-Each instruction is independent. Do not reference previous instructions or ask about next steps.
+${instructionContextRule}
 ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.24.0",
+  "version": "2.26.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",

package/skill/droid-cua/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: droid-cua
-description: Write, run, monitor, and troubleshoot Droid CUA mobile and web tests. Use when an agent needs to onboard @loadmill/droid-cua, configure Loadmill authentication, discover Android devices or AVDs, prepare iOS simulators or local browsers, author or revise .dcua tests and project context, run tests locally or on supported cloud devices, analyze reports and debug artifacts, distinguish test failures from product bugs, or perform guarded test healing.
+description: Operate Droid CUA `.dcua` test projects. Use when the user explicitly names Droid CUA or @loadmill/droid-cua, or asks to author, configure, run, or diagnose `.dcua` tests with the Droid CUA CLI or desktop app. Do not use for general web browsing, website development, browser automation, generic mobile work, or non-Droid test tasks.
 ---
 
 # Droid CUA
@@ -13,13 +13,16 @@ Use the existing `droid-cua` CLI and repository files. Do not invent commands, f
 2. Determine the requested platform and target from the task or project. Ask the user only when the choice is unclear.
 3. Check authentication and environment readiness before the first run. Read [project-and-auth.md](references/project-and-auth.md) for project boundaries and token handling. Read [onboarding-and-troubleshooting.md](references/onboarding-and-troubleshooting.md) when setup, dependency, device, simulator, browser, or connection work is needed.
 4. Author or update the smallest useful test. Read [authoring-tests.md](references/authoring-tests.md) before changing `.dcua`, context, test data, secrets, or config.
-5. Run the narrowest relevant test and monitor it to completion. Read [running-and-healing.md](references/running-and-healing.md) for commands, targets, artifacts, failure classification, and guarded healing.
-6. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
+5. Before execution, present the proposed tests and target to the user, resolve any questions, and get their confirmation. Read [running-and-healing.md](references/running-and-healing.md) for the required pre-run check.
+6. Run the confirmed, narrowest relevant test and monitor it to completion. Keep the user oriented during the run by surfacing meaningful live screenshots, not just terminal output.
+7. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
 
 ## Safety
 
 - Never print, open, or echo secret values. Check only whether required keys are configured.
 - Never search for credentials outside the active test project or source another project's `.env` file.
+- Use Loadmill for every run unless the user explicitly asks to use their own OpenAI API key. A discovered OpenAI key is never permission to switch providers.
+- Never use credentials from another project, a skill, an agent configuration, or the user's home directory. Configure credentials only in the active test project's `.env` (or an explicitly supplied CI secret).
 - Get approval before installing dependencies, running `sudo`, accepting licenses, or changing system configuration.
 - During setup, do not create or edit user tests. Rerun only the failed readiness check, then resume the original task.
 - During healing, edit only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never modify application code under the guise of healing a test.

package/skill/droid-cua/agents/openai.yaml

@@ -1,6 +1,6 @@
 interface:
   display_name: "Droid Cua"
-  short_description: "Write, run, monitor, and troubleshoot Droid CUA tests"
+  short_description: "Operate Droid CUA .dcua test projects"
   icon_small: "./assets/droid-cua-app-icon.png"
   icon_large: "./assets/droid-cua-app-icon.png"
   brand_color: "#1895F7"

package/skill/droid-cua/references/project-and-auth.md

@@ -15,24 +15,27 @@ Keep `.env` and `.secrets` out of version control. Use `git check-ignore -q .env
 
 ## Loadmill authentication
 
-Loadmill is the default LLM provider. Before a run, check only for presence:
+Loadmill is the required default LLM provider. Use it unless the user explicitly asks to use their own OpenAI API key. Never switch to OpenAI because a Loadmill token is missing or because an OpenAI key happens to be available.
+
+Before a run, check only for the presence of a Loadmill token in the active test project's `.env` or explicitly supplied CI secret:
 
 ```sh
-test -n "${LOADMILL_API_TOKEN:-}" || grep -Eq '^LOADMILL_API_TOKEN=.+$' .env 2>/dev/null
+grep -Eq '^LOADMILL_API_TOKEN=.+$' .env 2>/dev/null
 ```
 
-Run this check from the active test project's working directory. Do not print the variable, run `env`, open `.env`, or ask the user to paste the token. Do not search parent directories, sibling repositories, the user's home directory, or the Droid CUA source repository for credentials.
+Run this check from the active test project's working directory. Do not print the variable, run `env`, open `.env`, or inspect inherited environment variables. Do not search parent directories, sibling repositories, the user's home directory, agent skills/configuration, or the Droid CUA source repository for credentials.
+
+If it is missing, explain that Droid CUA uses a Loadmill token by default and guide the user to:
+
+1. Sign in, then open [Loadmill Settings → Security](https://app.loadmill.com/app/user/settings/security).
+2. Create an API token with a description such as `Droid CUA CLI`.
+3. Copy it when shown; Loadmill shows it once.
 
-If it is missing, instruct the user to:
+Offer exactly these two choices, then wait for the user's selection:
 
-1. Sign in at <https://app.loadmill.com>.
-2. Open **Settings → Security**.
-3. Create an API token with a description such as `Droid CUA CLI`.
-4. Copy it when shown; Loadmill shows it once.
-5. Ensure `.env` is ignored by version control before storing a project token; add `.env` to `.gitignore` if needed.
-6. Save it as `LOADMILL_API_TOKEN=...` in the active test project's `.env`, a shell environment variable, or a CI secret.
-7. Confirm when configuration is complete, without sharing the value.
+1. **Use it once:** The user provides the token through the agent's secret-input mechanism and asks to use it for this run only. Pass it only to that command through a secret-safe mechanism; do not write it to `.env`, a shell profile, or any other file.
+2. **Save it for this project:** The user provides the token through the agent's secret-input mechanism and approves saving it. Create the active project's `.env` if needed or add/update only its `LOADMILL_API_TOKEN` entry. First ensure `.env` is ignored by version control; add `.env` to the active project's `.gitignore` when needed.
 
-Do not create or populate the token value for the user. Do not copy a token from another file or source another project's `.env`. Sourcing an unrelated file can import development endpoints, proxy settings, and unrelated secrets. When the active project's `.env` contains the key, run `droid-cua` normally from that project; the CLI loads that file automatically.
+Use a secret-safe mechanism that does not print or expose the value in commands, logs, or responses. Verify only that the key is present. Never copy a token from another file, a shell profile, another project, an agent skill/configuration, or any other location. Do not inspect inherited environment variables to discover a token. When the active project's `.env` contains the key, run `droid-cua` normally from that project; the CLI loads that file automatically.
 
-If the user explicitly selects direct OpenAI access, use `--llm-provider openai` with one of the supported OpenAI CUA models: `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Check only for the presence of `OPENAI_API_KEY`. If it is missing, instruct the user to create an API key in their OpenAI account and save it in the active project's gitignored `.env`, their shell environment, or a CI secret. Apply the same secret-handling rules: never request, print, inspect, copy, or source the key. Do not silently switch providers to bypass missing Loadmill authentication.
+Only when the user explicitly selects direct OpenAI access, use `--llm-provider openai` with one of the supported OpenAI CUA models: `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Check only the active project's `.env` for `OPENAI_API_KEY`; do not use one found elsewhere. If it is missing, guide the user to create and configure their own key in the same project using the same secret-safe process. Do not silently switch providers to bypass missing Loadmill authentication.

package/skill/droid-cua/references/running-and-healing.md

@@ -2,19 +2,19 @@
 
 ## Construct the run
 
-Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Beacon when the user prioritizes faster, lower-cost execution:
+Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Pulse when the user prioritizes faster, lower-cost execution:
 
 ```sh
 # Recommended default
 --llm-provider loadmill --cua-model loadmill-smart
 
 # Faster, lower-cost option
---llm-provider loadmill --cua-model loadmill-beacon
+--llm-provider loadmill --cua-model loadmill-pulse
 ```
 
-Both Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart and Beacon silently; preserve the project's model or explain the speed/cost tradeoff and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
+All Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart, Pulse, and Beacon silently; preserve the project's model or explain the capability, speed, and experimental tradeoffs and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
 
-Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` when diagnosis may be needed.
+Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` for every agent-observed run: it writes the screenshots needed for live progress updates as well as diagnosis.
 
 ```sh
 # Attached Android device or running emulator
@@ -41,9 +41,49 @@ droid-cua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --instruc
 
 Use `.ipa` and `--platform ios` for supported cloud iOS runs. Preserve existing config flags rather than replacing them with defaults.
 
+## Confirm the run with the user
+
+Before starting any test command, send the user a short, plain-language run plan and wait for explicit confirmation. Treat this as a product conversation, not a command preview. Do not infer that a request to run a test also confirms an automatically selected device, browser, or batch of tests.
+
+The run plan should briefly state:
+
+- What behavior or flow will be checked, and whether one test or several will run. Mention the test filename only when it helps identify the test.
+- Where it will run: the chosen device, simulator, or browser. If more than one viable target is available, present the human-readable choices and ask the user to choose.
+- Any user-visible consequence, such as launching an emulator, resetting a fresh browser session, or starting a metered cloud-device session.
+- That the agent will share progress screenshots and provide a report afterward.
+
+Do not include CLI flags, commands, provider or model names, credential/key names, config/context filenames, or internal log/report paths unless the user asks or one changes a decision they need to make. Do not paste the test source into the confirmation.
+
+Ask the user about every unresolved or ambiguous choice before running. If the plan is complete, ask for a clear go-ahead (for example, `Ready to run this?`). Start only after they confirm. If they change the tests, target, or other material run detail, show the revised plan and confirm it again.
+
+For example: `I’m ready to run the Chrome smoke test in a fresh local Chrome window. It will check that Chrome opens and then closes. I’ll share screenshots as it progresses and provide a report at the end. Ready to run this?`
+
 ## Monitor and collect evidence
 
-Remain attached until the command exits and cloud cleanup finishes. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
+Remain attached until the command exits and cloud cleanup finishes. Run the command through an interactive/pollable terminal mechanism so progress and new artifacts can be observed before it exits. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
+
+## Show live visual progress
+
+Use the screenshots already captured by `--debug` to make the run understandable while it is happening. This is a required user-facing behavior, not merely an artifact to inspect after the run. After the CLI prints `Debug logging enabled: <logs/execution-...>.jsonl`, new PNGs appear in that log file's sibling directory. Keep the CLI running in a pollable/background session and poll that directory while the test is running.
+
+For agents whose chat UI renders local Markdown images (including Cursor and Codex), send a separate progress message containing an absolute-path image, for example:
+
+```md
+Step 2/4: add the item to the cart
+
+![Live test progress](/absolute/path/to/logs/execution-.../0004_....png)
+```
+
+Do not put this Markdown only in a terminal command, tool argument, or hidden reasoning: it must be an actual user-visible chat update. Do not replace it by opening the final report.
+
+- After every completed executable instruction, send exactly one short progress update with that instruction and its screenshot (for example, `Step 2/4 complete: add the item to the cart`). An image path alone is not a visual update.
+- Screenshot filenames include `instruction-001`, `instruction-002`, and so on. Select the latest safe PNG for the completed instruction; prefer an end-of-turn or assertion screenshot over an input or click-preview frame. This connects each user-visible update to the matching script step even when the model took several actions to finish it.
+- Also show a frame immediately when the run is retrying, reconnecting, or fails. Do not stream every model action or repeat visually identical frames.
+- Do not show `preview_click` / aim-assist annotations unless they help explain an issue.
+- If the latest frame visibly exposes a secret or other sensitive user data, do not render it; provide a text-only progress update and wait for a safe frame.
+- Do not delay the test to collect screenshots. If no new frame exists during a long model call, report the current stage in text and keep polling.
+
+When the command completes, show the final safe screenshot in addition to the result summary, then provide a clickable link to the HTML report (or its absolute path when local links are unavailable). The report is the final artifact; the interim frames are progress updates, not a substitute for it.
 
 - Exit `0`: all requested tests passed.
 - Exit `1`: configuration, connection, execution, or assertion failure.