@loadmill/droid-cua 2.29.0 → 2.31.0

package/build/src/device/cloud/loadmill-cloud/run-pool.js

@@ -0,0 +1,41 @@
+import { LoadmillCloudControlPlaneClient } from "./control-plane-client.js";
+import { isLoadmillCloudNotEntitledError } from "./errors.js";
+export const LOADMILL_CLOUD_POOLS = ["web", "mobile"];
+function isOtherPoolResponse(error) {
+    return error?.status === 404 || isLoadmillCloudNotEntitledError(error);
+}
+/**
+ * Resolves which pool a Loadmill Cloud session belongs to by asking each pool's
+ * router for the run: the server 404s a run-scoped call whose pool does not
+ * match the mount, and the run state itself carries no pool field. A pool the
+ * team is not entitled to answers 403 before the run is ever looked up, which
+ * is equally a "not this pool" answer.
+ *
+ * @param {{ credentials: { bearerToken: string, baseUrl?: string }, sessionId: string }} args
+ * @returns {Promise<{ pool: "mobile" | "web", client: LoadmillCloudControlPlaneClient, state: object }>}
+ */
+export async function resolveRunPool({ credentials, sessionId }) {
+    let firstDenial = null;
+    for (const pool of LOADMILL_CLOUD_POOLS) {
+        const client = new LoadmillCloudControlPlaneClient({
+            bearerToken: credentials.bearerToken,
+            baseUrl: credentials.baseUrl,
+            pool,
+        });
+        try {
+            return { pool, client, state: await client.getRunState(sessionId) };
+        }
+        catch (error) {
+            if (!isOtherPoolResponse(error)) {
+                throw error;
+            }
+            if (!firstDenial && isLoadmillCloudNotEntitledError(error)) {
+                firstDenial = error;
+            }
+        }
+    }
+    if (firstDenial) {
+        throw firstDenial;
+    }
+    throw new Error(`Loadmill Cloud session ${sessionId} was not found in either the web or the mobile pool.`);
+}

package/build/src/device/cloud/loadmill-cloud/run-status.js

@@ -0,0 +1,4 @@
+export const TERMINAL_STATUSES = new Set(["COMPLETED", "CANCELLED", "ERROR"]);
+// Artifacts are readable while a run is still tearing down (STOPPING), not just
+// once it reaches a true terminal state. Mirrors the server's ARTIFACT_READY_STATES.
+export const ARTIFACT_READY_STATUSES = new Set(["STOPPING", ...TERMINAL_STATUSES]);

package/build/src/integrations/loadmill/run-reporter.js

@@ -13,6 +13,9 @@ function nonNegativeIntegerOrNull(value) {
     const normalized = nonNegativeNumberOrNull(value);
     return normalized == null ? null : Math.floor(normalized);
 }
+function booleanOrNull(value) {
+    return typeof value === "boolean" ? value : null;
+}
 function normalizeSource(value) {
     return value === "ci" ? "ci" : "desktop";
 }
@@ -47,6 +50,39 @@ export function normalizeDroidRunStatus({ status = null, outcome = null, success
     }
     return "error";
 }
+/**
+ * @param {{
+ *   droidVersion?: string | null,
+ *   runtimeConfig?: {
+ *     cuaModel?: string | null,
+ *     llmProviderMode?: string | null,
+ *     stepDelayMs?: number | null,
+ *     contextOptimizationEnabled?: boolean | null,
+ *     contextOptimizationThreshold?: number | null,
+ *   } | null,
+ *   aimAssistMode?: string | null,
+ * }} input
+ * @returns {{
+ *   droidVersion: string | null,
+ *   model: string | null | undefined,
+ *   llmProviderMode: string | null | undefined,
+ *   stepDelayMs: number | null | undefined,
+ *   aimAssistMode: string | null,
+ *   contextOptimizationEnabled: boolean | null | undefined,
+ *   contextOptimizationThreshold: number | null | undefined,
+ * }}
+ */
+export function buildDroidRunConfiguration({ droidVersion = null, runtimeConfig = null, aimAssistMode = null, } = {}) {
+    return {
+        droidVersion,
+        model: runtimeConfig?.cuaModel,
+        llmProviderMode: runtimeConfig?.llmProviderMode,
+        stepDelayMs: runtimeConfig?.stepDelayMs,
+        aimAssistMode,
+        contextOptimizationEnabled: runtimeConfig?.contextOptimizationEnabled,
+        contextOptimizationThreshold: runtimeConfig?.contextOptimizationThreshold,
+    };
+}
 export function buildDroidRunReportPayload(input = {}) {
     const completedAt = trimToNull(input.completedAt) || new Date().toISOString();
     const testCases = nonNegativeIntegerOrNull(input.testCases);
@@ -58,6 +94,13 @@ export function buildDroidRunReportPayload(input = {}) {
         deviceSource: trimToNull(input.deviceSource),
         deviceName: trimToNull(input.deviceName),
         osVersion: trimToNull(input.osVersion),
+        droidVersion: trimToNull(input.droidVersion),
+        model: trimToNull(input.model),
+        llmProviderMode: trimToNull(input.llmProviderMode),
+        stepDelayMs: nonNegativeIntegerOrNull(input.stepDelayMs),
+        aimAssistMode: trimToNull(input.aimAssistMode),
+        contextOptimizationEnabled: booleanOrNull(input.contextOptimizationEnabled),
+        contextOptimizationThreshold: nonNegativeIntegerOrNull(input.contextOptimizationThreshold),
         instructionsTotal: nonNegativeIntegerOrNull(input.instructionsTotal),
         instructionsCompleted: nonNegativeIntegerOrNull(input.instructionsCompleted),
         actionsTotal: nonNegativeIntegerOrNull(input.actionsTotal),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.29.0",
+  "version": "2.31.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",

package/skill/droid-cua/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: droid-cua
-description: Operate Droid CUA `.dcua` test projects. Use when the user explicitly names Droid CUA or @loadmill/droid-cua, or asks to author, configure, run, or diagnose `.dcua` tests with the Droid CUA CLI or desktop app. Do not use for general web browsing, website development, browser automation, generic mobile work, or non-Droid test tasks.
+description: Operate and improve Droid CUA `.dcua` test projects. Use when the user explicitly names Droid CUA or @loadmill/droid-cua, or asks to author, configure, run, diagnose, heal, stabilize, or speed up `.dcua` tests with the Droid CUA CLI or desktop app. Do not use for general web browsing, website development, browser automation, generic mobile work, or non-Droid test tasks.
 ---
 
 # Droid CUA
@@ -15,7 +15,8 @@ Use the existing `droid-cua` CLI and repository files. Do not invent commands, f
 4. Author or update the smallest useful test. Read [authoring-tests.md](references/authoring-tests.md) before changing `.dcua`, context, test data, secrets, or config.
 5. Before execution, present the proposed tests and target to the user, resolve any questions, and get their confirmation. Read [running-and-healing.md](references/running-and-healing.md) for the required pre-run check.
 6. Run the confirmed, narrowest relevant test and monitor it to completion. Keep the user oriented during the run by surfacing meaningful live screenshots, not just terminal output.
-7. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
+7. For a new, changed, flaky, slow, or explicitly optimized test, continue with the evidence-driven author-run-improve loop in [iterative-improvement.md](references/iterative-improvement.md). A passing run is a baseline, not proof of readiness.
+8. End with a short user-facing improvement summary: what was tested, the main problems found, what changed and why, measured timing or stability gains, final readiness, and any remaining risk. Include evidence paths separately without making the user reconstruct the story from reports or the full run history.
 
 ## Safety
 
@@ -27,7 +28,10 @@ Use the existing `droid-cua` CLI and repository files. Do not invent commands, f
 - During setup, do not create or edit user tests. Rerun only the failed readiness check, then resume the original task.
 - During healing, edit only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never modify application code under the guise of healing a test.
 - Never weaken or remove an assertion merely to make a failing run pass.
-- Make one minimal change and run the affected test again. Stop after two edit/rerun cycles and present the evidence.
+- Keep one canonical context file and one canonical Droid CUA config file per test project. Reuse and update those shared files; never create test-specific context or config copies for a new or improved test.
+- When improving an existing test, keep its complete flow in one independently runnable `.dcua` file. Do not split it into ordered or state-dependent tests as an optimization; Droid CUA does not currently guarantee that separate tests will run consecutively or share state.
+- Before rerunning a long test, apply the smallest evidence-supported batch that addresses its known bottlenecks. The batch may include multiple `.dcua`, context, test-data, or config improvements when each has a clear rationale and preserves coverage. Record the complete change set; isolate or revert individual changes only if the combined result regresses or remains ambiguous.
+- When the user asks to create, improve, heal, stabilize, or optimize a test, continue the evidence-driven edit/rerun loop through a final representative run until the test is ready or a genuine blocker remains. Do not stop because an arbitrary number of reruns has been reached.
 - Stop immediately when evidence indicates a likely application regression; do not rewrite the test to match broken behavior.
 - Gracefully allow cloud runs to finish cleanup. Avoid terminating a metered device session unless necessary.
 
@@ -35,5 +39,8 @@ Use the existing `droid-cua` CLI and repository files. Do not invent commands, f
 
 - Prefer an existing project convention over creating a parallel structure.
 - Prefer headless runs for agent work because they are observable and return deterministic exit codes.
+- Write tests as concise, human-readable user journeys. Prefer one business-level instruction per user goal over one instruction per tap, menu, dialog, or other UI gesture; add lower-level guidance only when run evidence shows it is necessary.
 - Use explicit, user-observable outcomes in tests rather than implementation details or coordinates.
 - Diagnose the first divergence between the instruction and observed behavior before changing anything.
+- Compare runs on the same target and app build, and clearly identify every test, context, data, model, or config change included in the optimization batch.
+- Treat internal recovery loops, repeated screenshots, previews, compactions, and retries as stability signals even when the run passes.

package/skill/droid-cua/agents/openai.yaml

@@ -1,6 +1,6 @@
 interface:
-  display_name: "Droid Cua"
-  short_description: "Operate Droid CUA .dcua test projects"
+  display_name: "Droid CUA"
+  short_description: "Author, run, diagnose, and stabilize .dcua tests"
   icon_small: "./assets/droid-cua-app-icon.png"
   icon_large: "./assets/droid-cua-app-icon.png"
   brand_color: "#1895F7"

package/skill/droid-cua/references/authoring-tests.md

@@ -6,14 +6,15 @@ Find nearby `.dcua` tests and project files before writing. Match their director
 
 ## Write `.dcua` instructions
 
-- Put one clear instruction or assertion on each non-empty line.
-- Describe visible UI intent, not coordinates, selectors, implementation details, or model strategy.
-- Prefer a high-level business intent when `context.md` makes the flow unambiguous. Do not translate every tap and keystroke into a brittle micro-script.
-- Break an instruction down only when the agent demonstrably needs help finding a control or completing part of the flow.
+- Put one clear user goal or assertion on each non-empty line. One instruction does not need to equal one UI action.
+- Write concise, human-readable business or journey instructions that describe what the user wants to accomplish. Prefer `Select the desired date` or `Log out` over separate instructions for opening a picker or menu, clicking controls, handling an expected confirmation, and submitting.
+- Do not prescribe coordinates, selectors, implementation details, model strategy, or every tap and keystroke. These micro-steps couple the test to incidental UI structure, add model rounds, and make harmless UI changes more likely to break it.
+- Break a business-level instruction into smaller actions only when run evidence shows that the agent cannot complete it reliably or the intermediate state is itself behavior the test must verify.
 - State the starting condition when it is not guaranteed by the environment.
 - Use explicit assertions for important outcomes. Prefer stable visible facts such as a screen title, confirmation message, changed value, or navigation result.
 - Use exact visible labels when naming controls or outcomes.
-- Keep each test focused on one behavior. Split unrelated flows.
+- Keep each new test focused on one behavior. Separate unrelated flows only when each resulting test is fully independent and does not rely on execution order or state created by another test.
+- When updating or optimizing an existing test, preserve its complete flow in one `.dcua` file. Do not split it into sequential or dependent tests unless the user explicitly requests that structure and the project has a verified orchestration mechanism that guarantees ordering and shared state.
 - Use `//` comments sparingly for human context; comments are not executed.
 - Never include secret values in the test.
 
@@ -22,18 +23,47 @@ Example:
 ```text
 Sign in using the USER_EMAIL and USER_PASSWORD secrets.
 Assert that the Home screen is visible and shows the signed-in account.
+Select December 15 as the requested date.
+Assert that December 15 is shown as the requested date.
+Log out.
+Assert that the sign-in screen is visible.
 ```
 
+Keep meaningful outcomes explicit even when actions are high-level. For example, follow `Log out` with an assertion that the sign-in screen is visible when logout behavior is under test.
+
 Use the secret key names that exist in `.secrets`; do not inspect their values. If key discovery is necessary, filter the file so only keys reach output, for example `sed -n 's/^\([^#][^=]*\)=.*/\1/p' .secrets`. If a required key is absent, ask the user to add it without sharing its value.
 
+## Invoke a saved Loadmill flow
+
+When a test should run an existing Loadmill flow, invoke it either with the explicit `loadmill: <request>` form or with a natural-language instruction that clearly mentions Loadmill, such as `Run the Loadmill <flow name> flow with username=name.` Describe the flow with its exact or distinctive name. If several flows could match, ask the user for identifying terms before writing the test.
+
+The request is interpreted as natural language. When passing runtime parameter overrides, prefer `name=value`, because that is the explicitly supported extraction format; include it naturally in the request rather than treating the whole instruction as rigid syntax.
+
+```text
+loadmill: create premium customer with country=US
+Sign in as the customer created by the Loadmill flow.
+Assert that the Premium dashboard is visible.
+```
+
+Droid searches for the best matching Loadmill flow, starts it, and waits for it to finish before continuing. On success, relevant returned parameters can be added to the agent context for later instructions. Refer to those values semantically, as in the example; do not invent placeholder syntax such as `{email}` or include returned values in the `.dcua` file.
+
 ## Put knowledge in the right place
 
-- Put durable product behavior and terminology in `context.md`, not repeated in every test.
+- Put durable product behavior, terminology, stable navigation knowledge, and recurring product-specific confirmations in `context.md`, not repeated in every test. Use context to make business intent unambiguous, not to hide a scenario-specific micro-script.
 - Put non-secret reusable people, accounts, products, and scenario combinations in `test-data.md`.
 - Put credentials and sensitive values in `.secrets`.
 - Put reproducible headless settings in `ci/droid-cua.json`.
 - Keep environment and provider credentials in `.env` or the shell/CI secret store.
 
+Use one shared context file and one shared Droid CUA config file for the project:
+
+- Find the canonical files from nearby tests, existing commands, and config references before creating anything. If they exist, update them in place; do not create files such as `<test-name>-context.md` or `<test-name>-config.json`.
+- Add only durable, project-wide knowledge to the shared context. Keep scenario-specific steps, values, and expected outcomes in the `.dcua` test or test data.
+- Treat the shared config as affecting every project test. Review its existing consumers and likely impact before changing it. Use a CLI override for a temporary experiment, then persist the setting in the canonical config only when it is intended to become the project-wide behavior.
+- If the project has no context or config, create at most one canonical file of each kind using its established directory and naming conventions.
+
 Use `--scenario "<goal>" --test-data <path>` only when the user wants generated permutations or cases. A normal `.dcua` run does not automatically consume `test-data.md`.
 
 After editing, show the user the test instructions and explain any new context, data keys, or secret keys required. Never show secret values.
+
+Do not call a new or materially changed test complete after static validation. Continue with the confirmed baseline, evidence analysis, and stability workflow in [iterative-improvement.md](iterative-improvement.md).

package/skill/droid-cua/references/iterative-improvement.md

@@ -0,0 +1,143 @@
+# Iterative test improvement
+
+Use this workflow after creating or changing a test, when a test is flaky or slow, or when the user asks to stabilize, heal, or optimize it. Do not wait for a failure: a green run can still contain expensive recovery loops and recurring instability.
+
+## 1. Agree on the iterative run
+
+Before the first run, describe the test behavior, target, user-visible side effects, and metered-resource use as required by `running-and-healing.md`. When iteration is requested, explain that the workflow includes a baseline followed by the evidence-driven edits, targeted reruns, and final representative runs needed to reach a readiness decision on the same target.
+
+One confirmation may cover that iterative workflow. Continue without asking again merely because another rerun is needed. Confirm again before changing the app build, target, test scope, provider/model, or another user-visible side effect.
+
+## 2. Establish a reproducible baseline
+
+Run the narrowest test with its existing project model and config. Keep these fixed for the baseline, then change any of them only as part of a documented, evidence-supported optimization batch:
+
+- App build and initial account/data state.
+- Device, OS, device source, or browser session mode.
+- Test, context, model/provider, Aim Assist, compaction, and Explain Mode settings.
+- Report, debug, and cloud artifact settings.
+
+Use `--debug` and an explicit `--report` path. Preserve the project's Explain Mode setting for the performance baseline. When reasoning is unclear, run a separate diagnostic comparison with Explain Mode enabled, then repeat final confidence runs with the intended CI setting.
+
+On Loadmill Cloud, use `--artifacts video`, `--artifacts deviceLog,appiumLog`, or `--artifacts all` to download selected artifacts after the run reaches terminal status. `--artifacts` is Loadmill-Cloud-only. Use video when motion or timing matters and device/Appium logs when platform or automation behavior is in question. An existing cloud run can fetch artifacts retroactively with `--cloud-session-id <id> --artifacts <kinds>` and no instructions.
+
+## 3. Build an evidence timeline
+
+Inspect the HTML report, structured debug JSONL, safe screenshots, terminal output, and available video or platform logs. Separate cloud provisioning and artifact-download time from test execution time.
+
+Record at least:
+
+- Pass/fail, completed instructions, assertions, framework retries, action count, and total execution time.
+- Per-instruction duration and the first divergence from expected visible behavior.
+- Model rounds, screenshot-only turns, previews versus real actions, repeated coordinates, waits, compactions, and reconnects around each hotspot.
+- In-step recoveries that do not increment the CLI retry counter, such as repeated typing, refocusing, reopening the app, or resubmitting.
+- Whether the UI changed after each action and whether the same anomaly recurs in another run.
+
+Rank hotspots by time, excess rounds/actions, and risk. Do not assume the slowest step is an app delay: repeated observe-decide-act cycles often dominate execution time.
+
+## 4. Classify before editing
+
+Use the first-divergence categories in `running-and-healing.md`, with two additional distinctions:
+
+- **Droid/provider behavior**: aim-preview loops, compaction state loss, unsafe automation-tool behavior, model confusion despite clear instructions, or provider/network retries. Do not disguise these as app context.
+- **Performance-only opportunity**: the flow is correct, but an unnecessary scenario step, redundant instruction, repeated action, or overly conservative setting adds time without coverage value.
+
+Stop healing when evidence indicates an application regression. Preserve the failing assertion and report the product bug. Retry an unchanged test once for a plausible transient failure before editing.
+
+## 5. Choose the smallest useful batch
+
+For long-running tests, fix all well-supported bottlenecks that can be addressed safely before paying for another full run. Use this order to build the batch unless evidence points elsewhere:
+
+1. **`.dcua` wording and scope**
+   - Express one human-readable business goal per instruction, not one click, menu, field, dialog, or submit action per line.
+   - Collapse consecutive micro-steps into the user outcome they jointly accomplish, such as `Select the desired date` or `Log out`, while keeping meaningful assertions explicit.
+   - Use exact visible labels only when they disambiguate the goal or identify an outcome the test must verify.
+   - Add the missing starting condition or assertion at the first ambiguous transition.
+   - Remove scenario steps that do not protect the behavior under test.
+   - Add lower-level actions only when the evidence shows the business-level instruction is unreliable or an intermediate state is part of the coverage.
+   - Keep the existing scenario in one independently runnable `.dcua` file. Splitting a long test into ordered or state-dependent tests is not an available optimization because separate execution order and shared state are not guaranteed.
+   - Never remove or weaken a meaningful assertion to obtain a pass.
+
+2. **`context.md`**
+   - Update the project's existing canonical context file; never create a test-specific context file for the optimization.
+   - Add durable product navigation, terminology, stable interruptions, and narrowly observed recovery behavior.
+   - Move reusable mechanics out of the test, but keep scenario-specific values and expectations in the test.
+   - Remove redundant, speculative, or conflicting instructions. Do not restate Droid's base behavior or encode a workaround for a Droid/product bug as product knowledge.
+
+3. **Test data or reproducible config**
+   - Use stable, rerunnable data and prevent values from growing or colliding across runs.
+   - Use the project's existing canonical config rather than creating a config for the affected test.
+   - Evaluate how a setting would affect the other project tests. Use a CLI override for a temporary experiment; persist it in the canonical config only when the evidence supports making it project-wide.
+
+4. **Execution configuration**
+   - Include only settings supported by the evidence. Keep the prior run as the comparison baseline and record every override.
+
+For every included change, record the evidence, rationale, expected effect, and preserved coverage. Bundle multiple `.dcua`, context, test-data, and configuration improvements when they address distinct observed hotspots and do not conflict. Prefer low-risk wording and context fixes; include model or execution-setting changes when evidence supports them and the saved run time outweighs the loss of perfect attribution. After editing, rerun only the affected test on the same target and compare each hotspot plus the overall result against the baseline.
+
+## Configuration decision guide
+
+### CUA model
+
+- Preserve the project's model first. For a project without an established choice, use `loadmill-pulse`; it is the recommended model for daily authoring, local execution, and CI.
+- Use `loadmill-smart` when stability is a real, model-related issue and comparison evidence shows that switching from Pulse improves the affected flow. Do not escalate to Smart based only on a failure, slow app transition, ambiguous test, or target problem.
+- `loadmill-beacon` is opt-in and starts a fresh model conversation for every instruction. Cross-step facts come from Droid's client-owned transcript rather than a scenario-long model chain, so interpret late-run context and compaction differently and verify that cross-step references remain reliable.
+- Treat a model change as a higher-risk part of the batch. Prefer addressing obvious test/context ambiguity first. Keep Smart in the final configuration only when the available evidence supports its stability benefit.
+- Do not silently switch provider or model. Keep provider/model combinations valid.
+
+### App context and prompt customizations
+
+- Keep app context enabled for normal runs. `context.md` should describe stable product behavior, not the current scenario transcript or model strategy.
+- Use `--no-context` only as a diagnostic comparison when evidence suggests stale or conflicting context; it is not a normal healing step.
+- Treat base/execution prompt customizations as high-blast-radius settings because they affect every instruction. Inspect them for rules that cause repeated screenshots, waits, keyboard actions, or conflicts with Droid's base prompt.
+- Prefer fixing the narrow `.dcua` instruction or `context.md` entry first. Change a shared prompt customization only when the same behavior affects multiple tests and the evidence supports the broader rule.
+
+### Aim Assist
+
+- `always` is the default and previews every click. It maximizes coordinate verification but adds preview rounds and latency.
+- `smart` may skip preview for broad, isolated, stable targets while retaining it for small, dense, ambiguous, keyboard-adjacent, or recently changed UI. Use it when `always` adds repeated safe previews or material time.
+- `off` clicks directly. Use it only for a controlled comparison or a flow whose targets are consistently large and unambiguous; watch for misclicks and neighboring controls.
+- A preview followed by another screenshot instead of a real click is a diagnostic signal. Check compaction, screen changes, and prompt/context conflicts before merely changing Aim Assist.
+
+### Context Optimization
+
+- Context Optimization is enabled by default with a 60,000-input-token threshold. The threshold triggers compaction; it is not a guaranteed post-compaction size.
+- Repeated compaction near a late-run hotspot can lose short-lived action state or add latency. Raising the threshold may reduce compaction frequency but retains more history and can increase request size, latency, and usage.
+- A lower threshold reduces the working history sooner but can cause more frequent compaction and loss of relevant state in long flows.
+- Disable optimization only for a short diagnostic comparison when the model/provider supports the accumulated context. Do not make disabling it the default fix for a long test.
+- Prefer removing unrelated steps from the scenario or reducing unnecessary rounds before relying on a very high threshold. Do not turn one existing scenario into dependent test files as a compaction workaround.
+
+### Explain Mode and artifacts
+
+- Enable Explain Mode for a diagnostic comparison when the baseline does not reveal why the agent acted or stopped. It changes the execution prompt and may add tokens and latency, so do not compare its performance directly with a normal-mode baseline or silently keep it in routine CI.
+- `--debug` is required for agent-observed runs because it provides structured JSONL and instruction-linked screenshots.
+- Use an HTML report for every normal run. Use Loadmill Cloud video for motion/timing ambiguity and device/Appium logs for platform or automation-tool causality.
+
+## 6. Compare and iterate
+
+After each rerun, compare:
+
+- Did the first divergence disappear for the expected reason?
+- Did the hotspot's duration, rounds, actions, previews, retries, or recoveries improve?
+- Did another step regress or did coverage change?
+- Did the combined change set improve the intended hotspots without obscuring a regression?
+
+Keep the batch when the evidence supports the overall result. If it regresses or remains ambiguous, use the timeline to isolate, revise, or revert only the suspect subset instead of automatically rerunning every change separately. Continue while the evidence supports another useful test, context, data, or configuration improvement. Stop and report the strongest remaining explanation only when a genuine blocker remains or no evidence-supported improvement is left to try.
+
+## 7. Prove readiness
+
+After the optimized test passes, decide whether the accumulated evidence is strong enough for the test's risk, prior flakiness, state sensitivity, and intended CI use. A clean representative pass may be sufficient when the behavior is straightforward and earlier iterations already provide consistent evidence. Run the final test unchanged again when meaningful uncertainty remains—for example after a major last edit, intermittent failures, state-dependent behavior, or a configuration change. Prefer a fresh cloud device or equivalently reset local state when repeatability across sessions matters.
+
+Call the test ship-ready only when:
+
+- All intended instructions and meaningful assertions pass in the final representative evidence.
+- No unexplained recovery, repeated-action loop, or recurring high-risk hotspot remains.
+- Timing and action counts are reasonably consistent, with transient provider retries identified separately.
+- The test is rerunnable and leaves account/app state in the expected condition.
+- Test, context, data, secrets references, and config are reproducible in CI.
+- Reports and debug evidence exist, while secrets and generated artifacts remain uncommitted unless the project explicitly tracks them.
+
+Report before/after metrics, the final configuration, the evidence supporting the readiness judgment, remaining product/Droid/provider risks, and exact artifact paths. State why the available run history is sufficient or why more evidence is still needed. A stable test may still reveal a separate platform or product issue; record that risk instead of encoding it into the test.
+
+## 8. Summarize the improvement
+
+End the work with the concise retrospective required by `running-and-healing.md`. Lead with the final result, then highlight the important problems found, the changes and their rationale, and the measured timing or stability improvement. Distill a long iteration history into the few facts the user needs to understand what became faster or more reliable and why. Separate artifact links and lower-level diagnostics from this summary.

package/skill/droid-cua/references/onboarding-and-troubleshooting.md

@@ -4,6 +4,25 @@ Follow the Desktop Device Setup Wizard sequence: choose a platform, run checks,
 
 Do not create or modify user test files merely to perform setup. Get approval before installing software, changing system configuration, running `sudo`, or accepting a license. After remediation, rerun only the failed check.
 
+## Droid CUA CLI compatibility
+
+If the CLI rejects a model, option, command, or capability that this skill explicitly documents, treat an outdated global `@loadmill/droid-cua` package as a likely setup issue. Do not suggest an update for an ordinary test, target, provider, or application failure.
+
+Check the installed and latest published versions:
+
+```sh
+npm list -g @loadmill/droid-cua --depth=0
+npm view @loadmill/droid-cua version
+```
+
+If the global package is missing or behind the published version, explain the mismatch and get approval before updating it:
+
+```sh
+npm install -g @loadmill/droid-cua@latest
+```
+
+After the update, rerun only the rejected command or failed readiness check, then resume the original task.
+
 ## Android
 
 Resolve `adb` and `emulator` from `ANDROID_HOME`, `ANDROID_SDK_ROOT`, the macOS default `~/Library/Android/sdk`, the Windows default `%LOCALAPPDATA%\Android\Sdk`, then `PATH`. Respect explicit `DROID_CUA_ANDROID_ADB_PATH` and `DROID_CUA_ANDROID_EMULATOR_PATH` overrides.

package/skill/droid-cua/references/project-and-auth.md

@@ -11,6 +11,8 @@
 - `logs/`: structured JSONL and screenshot artifacts created by `--debug`.
 - HTML reports: self-contained evidence produced by `--report`.
 
+Treat `context.md` and `ci/droid-cua.json` as canonical project-level resources. Discover and reuse the paths established by the project; do not create one context or config per `.dcua` test. If the project has neither file, create at most one canonical context and one canonical config using the repository's existing layout conventions.
+
 Keep `.env` and `.secrets` out of version control. Use `git check-ignore -q .env` and `git check-ignore -q .secrets` to check ignore coverage without opening either file.
 
 ## Loadmill authentication

package/skill/droid-cua/references/running-and-healing.md

@@ -2,38 +2,38 @@
 
 ## Construct the run
 
-Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Pulse when the user prioritizes faster, lower-cost execution:
+Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider with Loadmill Pulse as the recommended model for daily authoring, local runs, and CI. Escalate to Loadmill Smart when stability is a demonstrated problem and run evidence shows Smart improves it:
 
 ```sh
-# Recommended default
---llm-provider loadmill --cua-model loadmill-smart
-
-# Faster, lower-cost option
+# Recommended daily model
 --llm-provider loadmill --cua-model loadmill-pulse
+
+# Evidence-driven stability escalation
+--llm-provider loadmill --cua-model loadmill-smart
 ```
 
-All Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart, Pulse, and Beacon silently; preserve the project's model or explain the capability, speed, and experimental tradeoffs and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
+All Loadmill models require `LOADMILL_API_TOKEN`. Do not switch from Pulse to Smart merely because a test failed or was slow; first identify a model-related stability issue, then keep Smart only when comparison evidence shows an improvement. Do not switch between Smart, Pulse, and Beacon silently; preserve the project's model or explain the capability, speed, and experimental tradeoffs and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
 
 Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` for every agent-observed run: it writes the screenshots needed for live progress updates as well as diagnosis.
 
 ```sh
 # Attached Android device or running emulator
-droid-cua --llm-provider loadmill --cua-model loadmill-smart --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
+droid-cua --llm-provider loadmill --cua-model loadmill-pulse --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
 
 # Launchable Android AVD
 droid-cua --avd avd:<avd-name> --instructions tests/example.dcua --config ci/droid-cua.json --debug --report reports/example.html
 
 # iOS Simulator
-droid-cua --llm-provider loadmill --cua-model loadmill-smart --platform ios --avd "<simulator-name>" --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua --llm-provider loadmill --cua-model loadmill-pulse --platform ios --avd "<simulator-name>" --instructions tests/example.dcua --debug --report reports/example.html
 
 # Local web
-droid-cua --llm-provider loadmill --cua-model loadmill-smart --target web --browser chrome --session-mode fresh --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua --llm-provider loadmill --cua-model loadmill-pulse --target web --browser chrome --session-mode fresh --instructions tests/example.dcua --debug --report reports/example.html
 
 # LambdaTest
-droid-cua --llm-provider loadmill --cua-model loadmill-smart --device-source lambdatest --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua --llm-provider loadmill --cua-model loadmill-pulse --device-source lambdatest --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
 
 # AWS Device Farm
-droid-cua --llm-provider loadmill --cua-model loadmill-smart --device-source aws-device-farm --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua --llm-provider loadmill --cua-model loadmill-pulse --device-source aws-device-farm --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
 
 # Direct OpenAI alternative
 droid-cua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
@@ -43,7 +43,7 @@ Use `.ipa` and `--platform ios` for supported cloud iOS runs. Preserve existing
 
 ## Confirm the run with the user
 
-Before starting any test command, send the user a short, plain-language run plan and wait for explicit confirmation. Treat this as a product conversation, not a command preview. Do not infer that a request to run a test also confirms an automatically selected device, browser, or batch of tests.
+Before starting any test command, send the user a short, plain-language run plan and wait for explicit confirmation. Treat this as a product conversation, not a command preview. Do not infer that a request to run a test also confirms an automatically selected device, browser, or batch of tests. For an explicitly requested author-run-improve workflow, the plan may ask approval for the baseline and the evidence-driven edit/rerun and confidence runs needed to reach a readiness decision; follow [iterative-improvement.md](iterative-improvement.md).
 
 The run plan should briefly state:
 
@@ -54,7 +54,7 @@ The run plan should briefly state:
 
 Do not include CLI flags, commands, provider or model names, credential/key names, config/context filenames, or internal log/report paths unless the user asks or one changes a decision they need to make. Do not paste the test source into the confirmation.
 
-Ask the user about every unresolved or ambiguous choice before running. If the plan is complete, ask for a clear go-ahead (for example, `Ready to run this?`). Start only after they confirm. If they change the tests, target, or other material run detail, show the revised plan and confirm it again.
+Ask the user about every unresolved or ambiguous choice before running. If the plan is complete, ask for a clear go-ahead (for example, `Ready to run this?`). Start only after they confirm. If they change the target, app build, test scope, provider/model, or another user-visible consequence, show the revised plan and confirm it again. Wording, context, config experiments, targeted reruns, and final confidence runs within an approved iterative workflow do not require separate confirmation when the target and side effects stay unchanged.
 
 For example: `I’m ready to run the Chrome smoke test in a fresh local Chrome window. It will check that Chrome opens and then closes. I’ll share screenshots as it progresses and provide a report at the end. Ready to run this?`
 
@@ -62,6 +62,14 @@ For example: `I’m ready to run the Chrome smoke test in a fresh local Chrome w
 
 Remain attached until the command exits and cloud cleanup finishes. Run the command through an interactive/pollable terminal mechanism so progress and new artifacts can be observed before it exits. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
 
+## Diagnose Loadmill flow failures
+
+When an instruction that invokes a saved Loadmill flow fails, inspect the `[Loadmill] FAILED` block first. Capture the command, the `Error:` text, and the suite run ID. Droid asks Loadmill for a failure explanation automatically, so the `Error:` text contains Loadmill's reason when one is available.
+
+If that reason is insufficient, open the suite run from the Loadmill Flow card in the desktop app and inspect the failed flow step and its available request, response, and assertion evidence. For a headless run, use the reported suite run ID and Loadmill site base URL to locate the same run at `/app/api-tests/test-suite-runs/<suite-run-id>`. Do not expose secrets from Loadmill evidence in chat, logs, or reports.
+
+Classify a flow that started and then failed inside Loadmill as a **Loadmill flow failure**, not a Droid UI-test failure. Do not rewrite later UI instructions or assertions to hide it. Correct the `.dcua` instruction only when the evidence shows Droid selected the wrong flow or passed the wrong parameters because the request was ambiguous. Retry the unchanged instruction once only for a transient Loadmill service or network failure; otherwise stop and report the Loadmill explanation, failing flow step, and suite-run link or ID.
+
 ## Show live visual progress
 
 Use the screenshots already captured by `--debug` to make the run understandable while it is happening. This is a required user-facing behavior, not merely an artifact to inspect after the run. After the CLI prints `Debug logging enabled: <logs/execution-...>.jsonl`, new PNGs appear in that log file's sibling directory. Keep the CLI running in a pollable/background session and poll that directory while the test is running.
@@ -95,6 +103,16 @@ After the run, verify that the report exists and resolve its absolute path. Open
 
 Report the outcome, failing instruction and first divergence when applicable, relevant error text, screenshot/debug/report paths, target, provider/model, and command with secret values omitted.
 
+After all requested running and healing work is complete, end with a short plain-language retrospective. Summarize the overall journey rather than listing every action or rerun:
+
+- What behavior was tested and whether it is ready.
+- The main ambiguity, failure, slow step, recovery loop, or other bottleneck found.
+- What changed in the test, context, data, or configuration and why it addressed that evidence.
+- The most meaningful before/after timing and stability results, such as total or hotspot duration, retries, recoveries, repeated actions, or consistency. Use exact numbers only when the artifacts support them; otherwise describe the observed improvement without inventing precision.
+- Any remaining risk or blocker, followed by links to the final report and relevant evidence.
+
+Keep this retrospective concise and lead with the result and improvement. Do not make the user reconstruct what happened from terminal output, screenshots, reports, or a chronological transcript.
+
 ## Classify the first divergence
 
 1. **Setup or authentication**: missing/invalid provider credential, dependency, license, browser, or executable. Return to onboarding and rerun only the failed check.
@@ -107,4 +125,4 @@ Report the outcome, failing instruction and first divergence when applicable, re
 
 Before editing, state the classification and evidence. Change only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never edit app code, delete coverage, loosen the expected outcome, or add broad waits simply to obtain green output.
 
-Make one minimal change, explain why it corrects the test rather than hides a product defect, and rerun only the affected test. Allow at most two edit/rerun cycles. If it still fails, stop with the original and final evidence, changes attempted, and recommended owner/action.
+Apply the smallest evidence-supported batch, explain why each change corrects the test rather than hides a product defect, and rerun only the affected test. Continue analyzing, editing, and rerunning until the final representative evidence supports readiness or a genuine blocker remains. Do not stop based on a fixed rerun count. Stop when the evidence indicates an application regression, an environment or product dependency prevents progress, or no evidence-supported test, context, data, or configuration improvement remains; report the original and final evidence, changes attempted, and recommended owner/action.