@loadmill/droid-cua 2.44.0 → 2.45.0

package/README.md

@@ -58,10 +58,16 @@ models use your own OpenAI API key.
 - Android: connect a device or select an emulator
 - iOS: choose a simulator on macOS
 
+No app of your own yet? The **Apps** page includes a built-in **Demo Shop**
+Android app. Install it on a connected Android device or emulator, or pick it
+for a cloud Android session, to try the agent right away.
+
 **5. Create or run a test**
 
 Use the desktop app to create a new test, edit an existing one, or run a saved script with live execution logs.
 
+Use the Back and Forward buttons next to the sidebar toggle to revisit pages and selections. Pending editor changes are saved before Back or Forward leaves the page. Automatic batch-test transitions do not add history entries. Report summaries remain visible while switching between saved runs. Navigation history lasts for the current app session; opening a new destination after going back clears forward history.
+
 You can also keep project run history in a results folder and review past runs from desktop app reports.
 
 ---
@@ -99,7 +105,7 @@ Desktop app support:
 - **Desktop-first workflow** - Create, run, and manage tests from one app
 - **Setup guidance** - Configure API access and platform prerequisites in the app
 - **Device and simulator connection** - Connect Android targets and iOS simulators
-- **Network certificate preparation** - Manage one active root CA and install it on a connected local simulator or supported emulator, or get manual setup steps for a physical iPhone or Android phone, from the desktop app's Tools menu
+- **Network certificate preparation** - Manage one active root CA and install it on a connected local simulator or supported emulator, or get manual setup steps for a physical iPhone or Android phone, from the desktop app's Settings → Network capture page
 - **Natural-language test creation** - Describe flows in plain English
 - **Test management** - Create, edit, save, and rerun reusable scripts, including tests organized in nested project folders
 - **Live execution logs** - Watch actions and progress as tests run
@@ -108,7 +114,7 @@ Desktop app support:
 - **Run video export** - Generate and download a polished MP4 replay from a desktop run's screenshots and instruction timeline
 - **Headless support** - Reuse scripts in CLI and automation workflows
 
-**Network certificate preparation:** Use **Tools → Manage Certificates** to see the active CA, upload a single `.pem` containing a CA certificate and its matching unencrypted private key, generate a new Droid CA, or remove the active CA. Droid stores one active pair in `~/.droid-cua/network/`; upload copies the PEM material without changing the source file, and removal does not change device trust. Connect a local iOS Simulator or a rootable Android Emulator, then choose **Tools → Install Network Certificate**. This installs the active CA, generating a Droid CA if none exists. Devices prepared with a previous CA may need installation again. Droid may reapply Android system trust when a prepared emulator reconnects after a cold boot. Restart an already running Android app after installation so it loads the updated trust store. With a local iPhone connected by USB, **Tools → Install Network Certificate** does not change the phone; it shows the manual steps instead. **Send to iPhone** opens the macOS share menu with a DER `.cer` copy of the active CA named after its common name (for example `Droid CUA Root fbd877ab.cer`, written to `~/.droid-cua/network/iphone/`); choose AirDrop and your iPhone, install it from **Settings → Profile Downloaded**, then turn on full trust for that common name under **Settings → General → About → Certificate Trust Settings**. Android Emulator certificate setup supports capable rootable images through Android 16; Google Play images without root access and Android 17 are not supported. With a physical Android phone connected over ADB, **Tools → Install Network Certificate** writes a PEM `.crt` copy of the active CA named after its common name (for example `Droid CUA Root fbd877ab.crt`, in `~/.droid-cua/network/android/`), copies it to the phone's `/sdcard/Download/` folder with `adb push`, and shows the manual steps; it changes no phone settings. Install it from **Settings → Security → Encryption & credentials → Install a certificate → CA certificate** (names vary by vendor), then restart the tested app. If the copy fails, Droid shows the local file path so you can copy it to the phone yourself.
+**Network certificate preparation:** Use **Settings → Network capture → Manage certificates** to see the active CA, upload a single `.pem` containing a CA certificate and its matching unencrypted private key, generate a new Droid CA, or remove the active CA. Droid stores one active pair in `~/.droid-cua/network/`; upload copies the PEM material without changing the source file, and removal does not change device trust. Connect a local iOS Simulator or a rootable Android Emulator, then choose **Settings → Network capture → Install on connected device**. This installs the active CA, generating a Droid CA if none exists. Devices prepared with a previous CA may need installation again. Droid may reapply Android system trust when a prepared emulator reconnects after a cold boot. Restart an already running Android app after installation so it loads the updated trust store. With a local iPhone connected by USB, **Settings → Network capture → Install on connected device** does not change the phone; it shows the manual steps instead. **Send to iPhone** opens the macOS share menu with a DER `.cer` copy of the active CA named after its common name (for example `Droid CUA Root fbd877ab.cer`, written to `~/.droid-cua/network/iphone/`); choose AirDrop and your iPhone, install it from **Settings → Profile Downloaded**, then turn on full trust for that common name under **Settings → General → About → Certificate Trust Settings**. Android Emulator certificate setup supports capable rootable images through Android 16; Google Play images without root access and Android 17 are not supported. With a physical Android phone connected over ADB, **Settings → Network capture → Install on connected device** writes a PEM `.crt` copy of the active CA named after its common name (for example `Droid CUA Root fbd877ab.crt`, in `~/.droid-cua/network/android/`), copies it to the phone's `/sdcard/Download/` folder with `adb push`, and shows the manual steps; it changes no phone settings. Install it from **Settings → Security → Encryption & credentials → Install a certificate → CA certificate** (names vary by vendor), then restart the tested app. If the copy fails, Droid shows the local file path so you can copy it to the phone yourself.
 
 **Network capture:** Enable **Network Capture** in desktop Settings or pass `--capture-network` for CLI tests. Local Chrome and Edge capture through Playwright with no certificate or proxy setup, using the same network queries and report cards as mobile. Capture covers the test browser context, including frames and popups, from run start. Previously loaded traffic is not recovered and existing pages are not automatically reloaded. CLI `--url` navigation happens after capture starts. Cloud browser capture, WebSocket messages and HAR export are not supported.
 
@@ -319,6 +325,7 @@ Supported CLI options include:
 - `--context-optimization-threshold`
 - `--no-context-optimization`
 - `--explain`
+- `--step-watchdog <off|on>`
 - `--base-prompt`
 - `--execution-prompt`
 - `--base-prompt-file`
@@ -359,6 +366,7 @@ Config and precedence rules:
 - `--no-context-optimization` disables Context Optimization.
 - Explain Mode is enabled by default and asks the agent for concise explanations of non-obvious decisions, skipped actions, and completion evidence; explanations appear in terminal output and reports.
 - Set `"explainMode": false` in the JSON config to disable it. The `--explain` flag remains available to explicitly enable it for a run.
+- Step Watchdog is experimental and defaults to `off`. Use `--step-watchdog on` to check for failed operations or blocked execution, or `--step-watchdog off` to disable checks. The JSON field is `"stepWatchdogMode": "on"`; the CLI flag overrides it. Desktop users can toggle **Settings → Agent → Verify Step Results** on or off. Successful checks run silently. Failures use an assertion-style **Step Failed** card with the expected instruction, actor explanation, and screenshot. Interactive runs offer **Skip** to continue or **Stop** to fail; headless runs fail automatically. Evaluator errors end execution. Enable `--debug` or desktop **Debug Mode** to inspect `step_watchdog.result` entries; probabilities and evaluator diagnostics remain out of normal output and reports. See [Step Watchdog](docs/step-watchdog.md) for provider requirements, logging, and calibration.
 - `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
 - `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
 - If both inline and file-based prompt overrides are provided, the inline prompt flags win.
@@ -560,7 +568,7 @@ droid-cua \
 
 Headless debug artifacts:
 - `--debug` writes desktop-style structured JSONL artifacts under `logs/`.
-- Each run creates `logs/execution-<runId>-<timestamp>.jsonl`.
+- Each run creates `logs/execution-<runId>-<timestamp>.jsonl`. New run IDs are UUIDs; a batch shares one ID across its tests and aggregate report, and a rerun gets a new ID.
 - Each run also creates a sibling screenshot folder next to that JSONL file.
 - Shared device events are written to `logs/device-events.jsonl`.
 - `--debug` no longer creates the legacy `logs/debug-*.log` file for headless runs.

package/build/index.js

@@ -1,3 +1,5 @@
+import { randomUUID } from "node:crypto";
+import { withExecutionRunId } from "./src/core/execution-context.js";
 const processStartTime = Date.now();
 import minimist from "minimist";
 import dotenv from "dotenv";
@@ -850,6 +852,7 @@ async function main() {
     session.setSystemPrompt(initialSystemText);
     // If tests were selected, run in headless mode.
     if (isHeadlessInstructionsRun) {
+        const runId = randomUUID();
         let exitCode = null;
         let succeeded = false;
         try {
@@ -860,6 +863,7 @@ async function main() {
                 llmProviderMode: headlessConfig.llmProviderMode,
                 cuaModel: headlessConfig.cuaModel,
                 explainMode: headlessConfig.explainMode,
+                stepWatchdogMode: headlessConfig.stepWatchdogMode,
                 promptCustomizations: headlessConfig.promptCustomizations,
                 contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
                 contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
@@ -875,7 +879,7 @@ async function main() {
             console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
             if (headlessConfig.scenarioPrompt) {
                 const candidateRuns = await loadScenarioCandidateRuns(headlessInstructionRuns);
-                const scenarioCases = await planScenarioRuns({
+                const scenarioCases = await withExecutionRunId(runId, () => planScenarioRuns({
                     client: runtimeContext.openAiClient,
                     runtimeConfig: runtimeContext.config,
                     scenarioPrompt: headlessConfig.scenarioPrompt,
@@ -885,7 +889,7 @@ async function main() {
                     })),
                     appContextText,
                     testDataText: preloadedTestData?.text || "",
-                });
+                }));
                 printScenarioCasePreview(scenarioCases);
                 headlessInstructionRuns = buildScenarioRuns({
                     candidateRuns,
@@ -894,6 +898,7 @@ async function main() {
                 preloadedHeadlessTarget = buildScenarioInstructionTarget(preloadedHeadlessTarget, headlessConfig.scenarioPrompt);
             }
             const result = await runHeadlessInstructionRuns({
+                runId,
                 runs: headlessInstructionRuns,
                 instructionsTarget: preloadedHeadlessTarget,
                 session,

package/build/src/cli/headless-execution-config.js

@@ -1,4 +1,5 @@
 import path from "path";
+import { parseStepWatchdogMode } from "../core/step-watchdog.js";
 import { readFile } from "fs/promises";
 import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
 import { AIM_ASSIST_MODES, DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
@@ -131,6 +132,7 @@ async function loadConfigFromFile(configPath) {
         cuaModel: undefined,
         aimAssistMode: undefined,
         explainMode: undefined,
+        stepWatchdogMode: undefined,
         promptCustomizations: createEmptyPromptCustomizations(),
         appContextEnabled: undefined,
         appContextPath: undefined,
@@ -152,6 +154,9 @@ async function loadConfigFromFile(configPath) {
         }
         normalized.explainMode = rawConfig.explainMode;
     }
+    if ("stepWatchdogMode" in rawConfig) {
+        normalized.stepWatchdogMode = parseStepWatchdogMode(rawConfig.stepWatchdogMode);
+    }
     if ("promptCustomizations" in rawConfig) {
         normalized.promptCustomizations = normalizePromptCustomizations(rawConfig.promptCustomizations, "config.promptCustomizations");
     }
@@ -226,6 +231,7 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
         cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
         aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
         explainMode: fileConfig?.explainMode ?? true,
+        stepWatchdogMode: fileConfig?.stepWatchdogMode ?? "off",
         promptCustomizations,
         appContextEnabled: fileConfig?.appContextEnabled ?? true,
         appContextPath: fileConfig?.appContextPath || null,
@@ -247,6 +253,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     if (args.explain === true) {
         resolved.explainMode = true;
     }
+    if (args["step-watchdog"] !== undefined) {
+        resolved.stepWatchdogMode = parseStepWatchdogMode(args["step-watchdog"]);
+    }
     if (hasContextOptimizationThresholdFlag) {
         resolved.contextOptimizationEnabled = true;
         resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");

package/build/src/cli/headless-report.js

@@ -16,7 +16,7 @@ export function createCapturingAddOutput(underlying, sink, { cloudSink = null, u
     return (item) => {
         if (item && typeof item === "object") {
             const event = {
-                ts: typeof item.ts === "string" ? item.ts : null,
+                ts: typeof item.ts === "string" && item.ts.trim() ? item.ts : new Date().toISOString(),
                 kind: typeof item.type === "string" ? item.type : "info",
                 text: typeof item.text === "string" ? item.text : "",
                 eventType: item.eventType,
@@ -45,26 +45,52 @@ export function createCapturingAddOutput(underlying, sink, { cloudSink = null, u
 // boolean/error/event signals. Exactly one of {stopped, failures, errors} is ever set, so
 // the shared precedence (error > fail > stopped > skipped > pass) yields the same verdict
 // the CLI produced before — see R1.
-function deriveCounts(success, error, events) {
+function deriveCounts(success, error, events, failureKind) {
     if (success)
         return { failures: 0, errors: 0, stopped: false };
     if (String(error || "").toLowerCase().includes("stopped by user")) {
         return { failures: 0, errors: 0, stopped: true };
     }
+    if (failureKind === "step_watchdog_error")
+        return { failures: 0, errors: 1, stopped: false };
+    if (failureKind === "step_watchdog")
+        return { failures: 1, errors: 0, stopped: false };
     const failedAssertion = events.some((event) => event.eventType === "assertion_result" && event.payload?.passed === false) ||
         String(error || "").startsWith("Assertion failed:");
     return failedAssertion ? { failures: 1, errors: 0, stopped: false } : { failures: 0, errors: 1, stopped: false };
 }
+function deviceReportEvents(deviceInfo, startedAtIso, testName) {
+    if (!deviceInfo)
+        return [];
+    return [{
+            ts: startedAtIso ?? null,
+            kind: "system",
+            text: `Loading test: ${testName ?? "Test run"}`,
+            eventType: "run_started",
+            payload: {
+                deviceName: deviceInfo.device_name,
+                platform: deviceInfo.platform,
+                deviceSource: deviceInfo.source,
+                osVersion: deviceInfo.os_version,
+            },
+        }];
+}
 // Builds a RunReportModel from captured CLI events plus the run's final stats. This is the
 // CLI-acquisition adapter: it knows how to capture live events + run stats, then delegates
 // ALL model assembly to the shared run-report-model.js (same code path as the XML/Desktop
 // parser). A `run_finished` event is synthesized when the captured output stream omits one.
-export function buildCliRunReportModel({ suiteName, testName, startedAtIso, durationMs, success, error, instructionsTotal, stats = {}, events, }) {
+export function buildCliRunReportModel({ suiteName, testName, startedAtIso, durationMs, success, error, failureKind = undefined, stepWatchdogMode = undefined, stepWatchdogUsage = undefined, instructionsTotal, stats = {}, events, deviceInfo = undefined, }) {
     const warnings = [];
     const normalizedEvents = normalizeEvents(events, { fallbackTs: startedAtIso ?? null, warnings });
+    const terminalCheck = normalizedEvents.findLast((event) => event.eventType === "step_watchdog_result" && event.payload?.mode === "on");
+    failureKind ??= terminalCheck?.payload?.outcome === "failure" ? "step_watchdog"
+        : terminalCheck?.payload?.outcome === "error" ? "step_watchdog_error" : undefined;
     const summaryPayload = buildRunFinishedPayload({
         success,
         error,
+        failureKind,
+        stepWatchdogMode,
+        stepWatchdogUsage,
         durationMs,
         instructionsTotal,
         instructionsCompleted: stats.instructionsCompleted,
@@ -74,15 +100,18 @@ export function buildCliRunReportModel({ suiteName, testName, startedAtIso, dura
         retries: stats.retryCount,
     });
     const hasRunFinished = normalizedEvents.some((event) => event.eventType === "run_finished");
-    // The synthetic run_finished is already in canonical shape; append it as-is (ts stays
-    // null, matching the historical CLI output) only when the producer did not emit one.
+    // Use execution time, not report generation/upload time, for the final step boundary.
+    const startedAtMs = Date.parse(startedAtIso);
+    const finishedAt = Number.isFinite(startedAtMs) && Number.isFinite(durationMs) && durationMs >= 0
+        ? new Date(startedAtMs + durationMs).toISOString()
+        : null;
     const allEvents = hasRunFinished
         ? normalizedEvents
         : [
             ...normalizedEvents,
-            { ts: null, kind: "system", text: "Execution finished", eventType: "run_finished", payload: summaryPayload },
+            { ts: finishedAt, kind: "system", text: "Execution finished", eventType: "run_finished", payload: summaryPayload },
         ];
-    const { failures, errors, stopped } = deriveCounts(success, error, allEvents);
+    const { failures, errors, stopped } = deriveCounts(success, error, allEvents, failureKind);
     const status = deriveRunStatus({ tests: 1, failures, errors, stopped });
     const successRate = computeSuccessRate({ tests: 1, failures, errors, status });
     return assembleRunReportModel({
@@ -96,7 +125,7 @@ export function buildCliRunReportModel({ suiteName, testName, startedAtIso, dura
         failureOrError: success
             ? undefined
             : makeFailureOrError({ kind: status === "error" ? "error" : "failure", message: error ?? undefined }),
-        events: allEvents,
+        events: [...deviceReportEvents(deviceInfo, startedAtIso, testName), ...allEvents],
         warnings,
     });
 }
@@ -127,7 +156,7 @@ export function createCliRunReport(modelInput) {
         }),
     };
 }
-export function createCliAggregateRunReport({ suiteName, testName, startedAtIso, durationMs, childModels, }) {
+export function createCliAggregateRunReport({ suiteName, testName, startedAtIso, durationMs, childModels, deviceInfo = undefined, }) {
     const childTests = childModels.map((model) => ({
         testName: model.testName ?? "Test",
         durationSeconds: model.durationSeconds,
@@ -162,7 +191,7 @@ export function createCliAggregateRunReport({ suiteName, testName, startedAtIso,
             skipped,
             status,
         }),
-        events: [],
+        events: deviceReportEvents(deviceInfo, startedAtIso, testName),
         childTests,
         warnings: [],
     });
@@ -183,7 +212,7 @@ export function renderCliCloudReport(model, savedRunId) {
     for (const child of cloudModel.childTests ?? []) {
         replaceScreenshots(child.events);
     }
-    return renderHtmlReport(cloudModel, { includeScripts: false });
+    return renderHtmlReport(cloudModel, { includeScripts: false, showRunDetails: false });
 }
 export async function writeCliRunReportArtifact(artifact, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
     const outputPath = resolveReportOutputPath({

package/build/src/cli/headless-runner.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import path from "path";
 import { mkdir, writeFile } from "fs/promises";
 import { getNetworkCaptureContext, navigateToUrl as navigateBrowserToUrl, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "../browser/connection.js";
@@ -9,6 +10,7 @@ import { buildExecutionModePrompt } from "../core/prompts.js";
 import { addOpenAiResponseToDroidCuaRunUsage, createDroidCuaRunUsageTotals, estimateDroidCuaRunCostUsd, setDroidCuaRunUsageModel, } from "../core/droid-cua-run-usage.js";
 import { getDroidVersion } from "../core/droid-version.js";
 import { ExecutionMode } from "../modes/execution-mode.js";
+import { STEP_WATCHDOG_MODEL } from "../core/step-watchdog.js";
 import { buildDroidRunConfiguration, reportDroidRunCompletion, } from "../integrations/loadmill/run-reporter.js";
 import { droidReportUploadManager } from "../integrations/loadmill/report-upload-manager.js";
 import { reportDroidCuaCiTestRunCompleted } from "../integrations/segment/droid-cua-segment.js";
@@ -28,10 +30,6 @@ function sanitizePathSegment(value) {
         .replace(/-+/g, "-")
         .replace(/^[-_.]+|[-_.]+$/g, "") || "test";
 }
-function createHeadlessRunId(index = 0) {
-    const suffix = index > 0 ? `-${index + 1}` : "";
-    return `run-${Date.now()}${suffix}`;
-}
 function networkLogPath(runId, report) {
     if (!report?.enabled)
         return path.resolve("logs", `network-${runId}.jsonl`);
@@ -73,14 +71,14 @@ function getRecordScreenshotDirForTest(baseScreenshotDir, testName, isBatchRun)
     }
     return path.join(baseScreenshotDir, sanitizePathSegment(testName));
 }
-function toDroidCuaSegmentRunStatus(success, error = null) {
+function toDroidCuaSegmentRunStatus(success, error = null, failureKind = null) {
     if (success) {
         return "pass";
     }
     if (error === "Stopped by user") {
         return "skipped";
     }
-    if (typeof error === "string" && error.startsWith("Assertion failed:")) {
+    if (failureKind === "step_watchdog" || (typeof error === "string" && error.startsWith("Assertion failed:"))) {
         return "failure";
     }
     return "error";
@@ -179,7 +177,7 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
     }
     return engineOptions;
 }
-async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, networkCapture = null, initialBrowserUrl = null, }) {
+async function runHeadlessTest({ runId, captureIndex = 0, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, networkCapture = null, initialBrowserUrl = null, }) {
     let executionMode = null;
     let captureStarted = false;
     const finishCapture = async () => {
@@ -201,12 +199,13 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         }
     };
     let loadmillSessionUser = null;
+    let stepWatchdogUsage = createDroidCuaRunUsageTotals({ model: STEP_WATCHDOG_MODEL });
     let runUsageTotals = setDroidCuaRunUsageModel(createDroidCuaRunUsageTotals(), runtimeContext.config.cuaModel);
     const buildSegmentUsageProperties = () => ({
         model: runUsageTotals.model,
         llmProviderMode: runtimeContext.config.llmProviderMode,
-        estimatedCostUsd: estimateDroidCuaRunCostUsd(runUsageTotals),
-        tokensUsed: runUsageTotals.totalTokens,
+        estimatedCostUsd: estimateDroidCuaRunCostUsd(runUsageTotals, stepWatchdogUsage),
+        tokensUsed: runUsageTotals.totalTokens + stepWatchdogUsage.totalTokens,
     });
     const reportSegmentRunCompleted = async (input) => {
         await reportDroidCuaCiTestRunCompleted({
@@ -288,7 +287,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         if (networkCapture) {
             try {
                 const browserContext = isBrowserTarget ? getNetworkCaptureContext(session.deviceId) : null;
-                const capturePath = await networkCapture.startCapture(networkLogPath(runId, report), { browserContext });
+                const capturePath = await networkCapture.startCapture(networkLogPath(isBatchRun ? `${runId}-${captureIndex + 1}` : runId, report), { browserContext });
                 console.log(`Network log: ${capturePath}`);
                 captureStarted = true;
                 engine.networkQuery = networkCapture.query;
@@ -309,6 +308,10 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             loadmillClient: runtimeContext.loadmillClient,
             runtimeConfig: runtimeContext.config,
             runtimeCredentials: runtimeContext.credentials,
+            testDataContext,
+            onStepWatchdogResponse: (response) => {
+                stepWatchdogUsage = addOpenAiResponseToDroidCuaRunUsage(stepWatchdogUsage, response);
+            },
             // Inline screenshots as data URLs so the HTML report is self-contained.
             ...(reportEnabled ? { includeScreenshotDataUrl: true } : {}),
         });
@@ -318,6 +321,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         await headlessDebug.endExecutionSession(runId, {
             success: Boolean(result.success),
             error: result.error ?? null,
+            failureKind: result.failureKind,
             durationMs,
             instructionsTotal: instructions.length,
             instructionsCompleted: stats.instructionsCompleted ?? 0,
@@ -327,7 +331,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             retries: stats.retryCount ?? 0
         });
         await reportSegmentRunCompleted({
-            status: toDroidCuaSegmentRunStatus(Boolean(result.success), result.error ?? null),
+            status: toDroidCuaSegmentRunStatus(Boolean(result.success), result.error ?? null, result.failureKind),
             durationMs,
             numOfSteps: instructions.length,
         });
@@ -338,6 +342,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
                 loadmillClient: runtimeContext.loadmillClient,
                 source: "ci",
                 runId,
+                outcome: toDroidCuaSegmentRunStatus(Boolean(result.success), result.error ?? null, result.failureKind),
                 projectName: path.basename(path.dirname(testPath)) || null,
                 testName,
                 runType: "test",
@@ -363,12 +368,17 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             console.error(`\nTest failed: ${result.error}`);
         }
         const reportInput = {
+            deviceInfo,
             suiteName: path.basename(path.dirname(testPath)) || null,
             testName,
             startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
             durationMs,
             success: Boolean(result.success),
             error: result.error ?? null,
+            failureKind: result.failureKind,
+            ...(runtimeContext.config.stepWatchdogMode !== "off" && runtimeContext.config.stepWatchdogMode
+                ? { stepWatchdogMode: runtimeContext.config.stepWatchdogMode, stepWatchdogUsage }
+                : {}),
             instructionsTotal: instructions.length,
             stats,
         };
@@ -465,6 +475,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             });
         }
         const failedReportInput = {
+            deviceInfo,
             suiteName: path.basename(path.dirname(testPath)) || null,
             testName,
             startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
@@ -518,7 +529,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         throw error;
     }
 }
-export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget = false, desktopController = null, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, networkCapture = null, initialBrowserUrl = null, }) {
+export async function runHeadlessInstructionRuns({ runId = randomUUID(), runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget = false, desktopController = null, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, networkCapture = null, initialBrowserUrl = null, }) {
     await clearLoadmillReportMetadata(report?.metadataPath);
     const resolvedRunTarget = runTarget ?? { isBrowserTarget, isDesktopTarget, isCloudTarget };
     const runConfiguration = buildDroidRunConfiguration({
@@ -560,7 +571,6 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
     const aggregateLocalReportModels = [];
     const aggregateReportModels = [];
     for (const [index, run] of runs.entries()) {
-        const runId = createHeadlessRunId(index);
         const testScreenshotDir = getRecordScreenshotDirForTest(screenshotDir, run.testName, isBatchRun);
         if (isBatchRun) {
             console.log(`\n[${index + 1}/${runs.length}] ${run.testName}`);
@@ -572,6 +582,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
             }
             const result = await runHeadlessTestImpl({
                 runId,
+                captureIndex: index,
                 testPath: run.testPath,
                 testName: run.testName,
                 instructions,
@@ -625,6 +636,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
             const message = error instanceof Error ? error.message : String(error);
             if (instructionsTarget?.type === "selection" && (report?.enabled || cloudReportEnabled)) {
                 const failedReportModel = buildCliRunReportModel({
+                    deviceInfo,
                     suiteName: path.basename(path.dirname(run.testPath)) || null,
                     testName: run.testName,
                     startedAtIso: new Date().toISOString(),
@@ -670,6 +682,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
                 : path.basename(instructionsTarget.path);
         if (isSelection && report?.enabled && aggregateLocalReportModels.length > 0) {
             const aggregateReport = createCliAggregateRunReport({
+                deviceInfo,
                 suiteName: projectName,
                 testName,
                 startedAtIso: new Date(batchStartedAt).toISOString(),
@@ -691,7 +704,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
             ...runConfiguration,
             loadmillClient: runtimeContext.loadmillClient,
             source: "ci",
-            runId: createHeadlessRunId(),
+            runId,
             projectName,
             testName,
             runType: "project",
@@ -715,6 +728,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
         });
         if (cloudUploadSession && reportedRun.savedRunId) {
             const aggregateReport = createCliAggregateRunReport({
+                deviceInfo,
                 suiteName: projectName,
                 testName,
                 startedAtIso: new Date(batchStartedAt).toISOString(),

package/build/src/commands/help.js

@@ -56,6 +56,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --execution-prompt-file Headless execution prompt customization file' });
     addOutput({ type: 'info', text: '  --record               Record screenshots during execution' });
     addOutput({ type: 'info', text: '  --explain              Explain non-obvious decisions and completion evidence' });
+    addOutput({ type: 'info', text: '  --step-watchdog <off|on>  Check for failures after each instruction: off (default), on' });
     addOutput({ type: 'info', text: '  --report [path]        Save a self-contained HTML report locally' });
     addOutput({ type: 'info', text: '  --network-proxy        Forward local mobile traffic through port 8765 without capture (all Mac addresses for iPhones and Android phones)' });
     addOutput({ type: 'info', text: '  --capture-network      Capture local mobile or browser test traffic to one JSONL log per test' });

package/build/src/commands/stop.js

@@ -23,6 +23,7 @@ export async function handleStop(args, session, context) {
         // Set flag to stop execution
         if (context.activeExecutionMode) {
             context.activeExecutionMode.shouldStop = true;
+            context.activeExecutionMode.requestStop?.();
         }
         return true; // Continue loop
     }

package/build/src/core/capture-sources.js

@@ -20,15 +20,19 @@ export const CAPTURE_SOURCES = Object.freeze({
     CALL_OUTPUT_EXPLICIT_ACTION: "call-output-explicit-action",
     INSTRUCTION_INPUT: "instruction-input",
     INSTRUCTION_INPUT_ASSERTION: "instruction-input-assertion",
+    STEP_WATCHDOG: "step-watchdog",
+    STEP_FAILURE: "step-failure",
     DESIGN_INPUT: "design-input",
 });
 const HIDDEN_DEBUG_MEDIA_KINDS = new Set([
     CAPTURE_SOURCES.PREVIEW_CLICK,
+    CAPTURE_SOURCES.STEP_WATCHDOG,
 ]);
 const CLICK_ANNOTATION_MEDIA_KINDS = new Set([
     CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
 ]);
 const SCREENSHOT_MEDIA_KINDS = new Set([
+    CAPTURE_SOURCES.STEP_FAILURE,
     CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
     CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION,
     CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,

package/build/src/core/droid-cua-run-usage.js

@@ -61,6 +61,7 @@ export function normalizeDroidCuaCostModel(model) {
     }
     return null;
 }
+/** @param {{ model?: string | null }} [options] */
 export function createDroidCuaRunUsageTotals({ model = null } = {}) {
     const normalizedModel = trimToNull(model);
     return {
@@ -110,7 +111,11 @@ export function addOpenAiResponseToDroidCuaRunUsage(totals, response) {
         totalTokens: totals.totalTokens + totalTokens,
     };
 }
-export function estimateDroidCuaRunCostUsd(totals) {
+/**
+ * @param {ReturnType<typeof createDroidCuaRunUsageTotals>} totals
+ * @param {{ inputTokens: number } | null} [stepWatchdogUsage]
+ */
+export function estimateDroidCuaRunCostUsd(totals, stepWatchdogUsage = null) {
     const costModel = totals.costModel ?? normalizeDroidCuaCostModel(totals.model);
     if (!costModel) {
         return null;
@@ -119,6 +124,8 @@ export function estimateDroidCuaRunCostUsd(totals) {
     if (!pricing) {
         return null;
     }
-    const cost = (totals.inputTokens / TOKENS_PER_MILLION) * pricing.inputPerMillionUsd;
+    // Decisions charges input tokens only; keep its usage separate from the actor model.
+    const cost = (totals.inputTokens / TOKENS_PER_MILLION) * pricing.inputPerMillionUsd
+        + (finiteNumber(stepWatchdogUsage?.inputTokens) / TOKENS_PER_MILLION) * 0.1;
     return Number(cost.toFixed(8));
 }

package/build/src/core/execution-context.js

@@ -0,0 +1,8 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+const executionRunId = new AsyncLocalStorage();
+export function withExecutionRunId(runId, callback) {
+    return executionRunId.run(runId, callback);
+}
+export function getExecutionRunId() {
+    return executionRunId.getStore();
+}