@loadmill/droid-cua 2.26.0 → 2.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -101,6 +101,7 @@ Desktop app support:
101
101
  - **Live execution logs** - Watch actions and progress as tests run
102
102
  - **Reports and history** - Review past runs from a project results folder inside the desktop app
103
103
  - **JUnit XML output** - Write standard test reports for CI systems and external tooling
104
+ - **Run video export** - Generate and download a polished MP4 replay from a desktop run's screenshots and instruction timeline
104
105
  - **Headless support** - Reuse scripts in CLI and automation workflows
105
106
 
106
107
  ---
@@ -250,6 +251,7 @@ Supported CLI options include:
250
251
  - `--no-context`
251
252
  - `--context-optimization-threshold`
252
253
  - `--no-context-optimization`
254
+ - `--explain`
253
255
  - `--base-prompt`
254
256
  - `--execution-prompt`
255
257
  - `--base-prompt-file`
@@ -257,6 +259,7 @@ Supported CLI options include:
257
259
  - `--secrets`
258
260
  - `--record`
259
261
  - `--report`
262
+ - `--no-cloud-report`
260
263
  - `--debug`
261
264
 
262
265
  `--cua-model` accepts `loadmill-smart`, `loadmill-pulse`, `loadmill-beacon`,
@@ -271,6 +274,8 @@ Config and precedence rules:
271
274
  - Context Optimization is enabled by default with a `30000` input-token threshold.
272
275
  - `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
273
276
  - `--no-context-optimization` disables Context Optimization.
277
+ - `--explain` enables Explain Mode for the run. Explain Mode asks the agent for concise explanations of non-obvious decisions, skipped actions, and completion evidence; explanations appear in terminal output and reports.
278
+ - Set `"explainMode": true` in the JSON config to enable the same behavior without a flag. Explain Mode is off by default, and `--explain` enables it when the config leaves it off.
274
279
  - `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
275
280
  - `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
276
281
  - If both inline and file-based prompt overrides are provided, the inline prompt flags win.
@@ -286,6 +291,8 @@ HTML report (`--report`):
286
291
  - Writes a self-contained HTML report of the run — the same Run Details view the desktop app shows, with screenshots inlined — that you can share and open in any browser.
287
292
  - With no value it saves `<test>--<date>--<time>.html` (e.g. `login--2026-06-03--16-09-20.html`) in the current directory; pass `--report <file>` to pick a filename or `--report <dir>` to pick a folder.
288
293
  - In the desktop app the same report is available from a run's **Share → Download HTML report** button.
294
+ - Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 browser replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename.
295
+ - Authenticated headless runs upload their HTML report and screenshots to Loadmill by default. Use `--no-cloud-report` to disable that artifact upload. The basic Droid run row is still reported to Loadmill.
289
296
 
290
297
  ```sh
291
298
  droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua --report ./reports
package/build/index.js CHANGED
@@ -31,6 +31,7 @@ import { formatCloudUsageReport } from "./src/cli/loadmill-cloud-usage.js";
31
31
  import { LoadmillCloudControlPlaneClient } from "./src/device/cloud/loadmill-cloud/control-plane-client.js";
32
32
  import { createStatusLine } from "./src/cli/cloud-status-line.js";
33
33
  import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
34
+ import { droidReportUploadManager } from "./src/integrations/loadmill/report-upload-manager.js";
34
35
  import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
35
36
  import { setCloudProgressListener } from "./src/device/cloud/progress.js";
36
37
  import { runSkillCommand } from "./src/cli/skill-command.js";
@@ -78,6 +79,7 @@ const debugMode = args["debug"] || false;
78
79
  const reportConfig = {
79
80
  enabled: args.report === true || (typeof args.report === "string" && args.report.trim().length > 0),
80
81
  path: typeof args.report === "string" && args.report.trim().length > 0 ? args.report.trim() : null,
82
+ cloudEnabled: args["no-cloud-report"] !== true,
81
83
  };
82
84
  const screenshotDir = path.join("droid-cua-recording-" + Date.now());
83
85
  if (recordScreenshots)
@@ -618,6 +620,7 @@ async function main() {
618
620
  const runtimeContext = createCliRuntimeContext({
619
621
  llmProviderMode: headlessConfig.llmProviderMode,
620
622
  cuaModel: headlessConfig.cuaModel,
623
+ explainMode: headlessConfig.explainMode,
621
624
  promptCustomizations: headlessConfig.promptCustomizations,
622
625
  contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
623
626
  contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
@@ -671,6 +674,7 @@ async function main() {
671
674
  succeeded = true;
672
675
  }
673
676
  finally {
677
+ await droidReportUploadManager.drain();
674
678
  if (lambdaTestSessionConnected) {
675
679
  console.log("Ending LambdaTest session...");
676
680
  await disconnectDevice();
@@ -129,6 +129,7 @@ async function loadConfigFromFile(configPath) {
129
129
  llmProviderMode: undefined,
130
130
  cuaModel: undefined,
131
131
  aimAssistMode: undefined,
132
+ explainMode: undefined,
132
133
  promptCustomizations: createEmptyPromptCustomizations(),
133
134
  appContextEnabled: undefined,
134
135
  appContextPath: undefined,
@@ -144,6 +145,12 @@ async function loadConfigFromFile(configPath) {
144
145
  if ("aimAssistMode" in rawConfig) {
145
146
  normalized.aimAssistMode = validateAimAssistMode(rawConfig.aimAssistMode, "config.aimAssistMode");
146
147
  }
148
+ if ("explainMode" in rawConfig) {
149
+ if (typeof rawConfig.explainMode !== "boolean") {
150
+ throw new Error("config.explainMode must be a boolean.");
151
+ }
152
+ normalized.explainMode = rawConfig.explainMode;
153
+ }
147
154
  if ("promptCustomizations" in rawConfig) {
148
155
  normalized.promptCustomizations = normalizePromptCustomizations(rawConfig.promptCustomizations, "config.promptCustomizations");
149
156
  }
@@ -217,6 +224,7 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
217
224
  llmProviderMode: fileConfig?.llmProviderMode || readEnvDefaultLlmProviderMode(),
218
225
  cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
219
226
  aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
227
+ explainMode: fileConfig?.explainMode ?? false,
220
228
  promptCustomizations,
221
229
  appContextEnabled: fileConfig?.appContextEnabled ?? true,
222
230
  appContextPath: fileConfig?.appContextPath || null,
@@ -235,6 +243,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
235
243
  if (typeof args["aim-assist"] === "string") {
236
244
  resolved.aimAssistMode = validateAimAssistMode(args["aim-assist"], "--aim-assist");
237
245
  }
246
+ if (args.explain === true) {
247
+ resolved.explainMode = true;
248
+ }
238
249
  if (hasContextOptimizationThresholdFlag) {
239
250
  resolved.contextOptimizationEnabled = true;
240
251
  resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
@@ -8,20 +8,35 @@ import { mkdir, writeFile } from "node:fs/promises";
8
8
  import { renderHtmlReport } from "../core/report/render-html-report.js";
9
9
  import { buildReportFileName } from "../core/report/report-file-name.js";
10
10
  import { normalizeEvents, deriveRunStatus, computeSuccessRate, makeFailureOrError, buildRunFinishedPayload, assembleRunReportModel, } from "../core/report/run-report-model.js";
11
+ import { droidReportScreenshotUrl } from "../integrations/loadmill/report-upload-manager.js";
11
12
  // Captures the emitted events into `sink` and forwards them to the underlying output
12
13
  // function (so console behaviour is unchanged). Maps the engine's `type` to the canonical
13
14
  // `kind`; `stepId` is intentionally not collected (the shared normalizer drops it — see R2).
14
- export function createCapturingAddOutput(underlying, sink) {
15
+ export function createCapturingAddOutput(underlying, sink, { cloudSink = null, uploadManager = null, uploadSession = null } = {}) {
15
16
  return (item) => {
16
17
  if (item && typeof item === "object") {
17
- sink.push({
18
+ const event = {
18
19
  ts: typeof item.ts === "string" ? item.ts : null,
19
20
  kind: typeof item.type === "string" ? item.type : "info",
20
21
  text: typeof item.text === "string" ? item.text : "",
21
22
  eventType: item.eventType,
22
23
  actionType: item.actionType,
23
24
  payload: item.payload,
24
- });
25
+ };
26
+ sink?.push(event);
27
+ if (cloudSink) {
28
+ const payload = event.payload && typeof event.payload === "object"
29
+ ? { ...event.payload }
30
+ : event.payload;
31
+ if (typeof payload?.screenshotDataUrl === "string") {
32
+ const screenshotObjectName = uploadManager?.enqueueScreenshot(uploadSession, payload.screenshotDataUrl);
33
+ delete payload.screenshotDataUrl;
34
+ if (screenshotObjectName) {
35
+ payload.screenshotObjectName = screenshotObjectName;
36
+ }
37
+ }
38
+ cloudSink.push({ ...event, payload });
39
+ }
25
40
  }
26
41
  return underlying(item);
27
42
  };
@@ -43,7 +58,7 @@ function deriveCounts(success, error, events) {
43
58
  // Builds a RunReportModel from captured CLI events plus the run's final stats. This is the
44
59
  // CLI-acquisition adapter: it knows how to capture live events + run stats, then delegates
45
60
  // ALL model assembly to the shared run-report-model.js (same code path as the XML/Desktop
46
- // parser). A `run_finished` event is synthesized because the CLI output stream omits one.
61
+ // parser). A `run_finished` event is synthesized when the captured output stream omits one.
47
62
  export function buildCliRunReportModel({ suiteName, testName, startedAtIso, durationMs, success, error, instructionsTotal, stats = {}, events, }) {
48
63
  const warnings = [];
49
64
  const normalizedEvents = normalizeEvents(events, { fallbackTs: startedAtIso ?? null, warnings });
@@ -58,12 +73,15 @@ export function buildCliRunReportModel({ suiteName, testName, startedAtIso, dura
58
73
  assertionsFailed: stats.assertionsFailed,
59
74
  retries: stats.retryCount,
60
75
  });
61
- // The synthetic run_finished is already in canonical shape; appended as-is (ts stays null,
62
- // matching the historical CLI output) rather than through the normalizer.
63
- const allEvents = [
64
- ...normalizedEvents,
65
- { ts: null, kind: "system", text: "Execution finished", eventType: "run_finished", payload: summaryPayload },
66
- ];
76
+ const hasRunFinished = normalizedEvents.some((event) => event.eventType === "run_finished");
77
+ // The synthetic run_finished is already in canonical shape; append it as-is (ts stays
78
+ // null, matching the historical CLI output) only when the producer did not emit one.
79
+ const allEvents = hasRunFinished
80
+ ? normalizedEvents
81
+ : [
82
+ ...normalizedEvents,
83
+ { ts: null, kind: "system", text: "Execution finished", eventType: "run_finished", payload: summaryPayload },
84
+ ];
67
85
  const { failures, errors, stopped } = deriveCounts(success, error, allEvents);
68
86
  const status = deriveRunStatus({ tests: 1, failures, errors, stopped });
69
87
  const successRate = computeSuccessRate({ tests: 1, failures, errors, status });
@@ -97,17 +115,89 @@ export function resolveReportOutputPath({ reportPath, testName, startedAtIso, cw
97
115
  }
98
116
  return path.join(resolved, fileName);
99
117
  }
100
- // Renders and writes the HTML report for one run. Returns the absolute output path.
101
- export async function writeCliRunReport(modelInput, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
118
+ export function createCliRunReport(modelInput) {
102
119
  const model = buildCliRunReportModel(modelInput);
120
+ return {
121
+ model,
122
+ html: renderHtmlReport(model),
123
+ fileName: buildReportFileName({
124
+ testName: modelInput.testName,
125
+ suiteName: modelInput.suiteName,
126
+ timestamp: modelInput.startedAtIso,
127
+ }),
128
+ };
129
+ }
130
+ export function createCliAggregateRunReport({ suiteName, testName, startedAtIso, durationMs, childModels, }) {
131
+ const childTests = childModels.map((model) => ({
132
+ testName: model.testName ?? "Test",
133
+ durationSeconds: model.durationSeconds,
134
+ status: model.status,
135
+ successRate: model.successRate,
136
+ failureOrError: model.failureOrError,
137
+ events: model.events,
138
+ warnings: model.warnings,
139
+ }));
140
+ const failures = childTests.filter((child) => child.status === "fail").length;
141
+ const errors = childTests.filter((child) => child.status === "error").length;
142
+ const skipped = childTests.filter((child) => child.status === "skipped").length;
143
+ const stopped = childTests.some((child) => child.status === "stopped");
144
+ const status = deriveRunStatus({
145
+ tests: childTests.length,
146
+ failures,
147
+ errors,
148
+ skipped,
149
+ stopped,
150
+ });
151
+ const model = assembleRunReportModel({
152
+ suiteName,
153
+ testName,
154
+ timestamp: startedAtIso,
155
+ durationSeconds: Number.isFinite(durationMs) ? durationMs / 1000 : null,
156
+ status,
157
+ testCases: childTests.length,
158
+ successRate: computeSuccessRate({
159
+ tests: childTests.length,
160
+ failures,
161
+ errors,
162
+ skipped,
163
+ status,
164
+ }),
165
+ events: [],
166
+ childTests,
167
+ warnings: [],
168
+ });
169
+ return { model, html: renderHtmlReport(model) };
170
+ }
171
+ export function renderCliCloudReport(model, savedRunId) {
172
+ const cloudModel = structuredClone(model);
173
+ const replaceScreenshots = (events = []) => {
174
+ for (const event of events) {
175
+ const name = event.payload?.screenshotObjectName;
176
+ if (typeof name === "string") {
177
+ event.payload.screenshotUrl = droidReportScreenshotUrl(savedRunId, name);
178
+ delete event.payload.screenshotObjectName;
179
+ }
180
+ }
181
+ };
182
+ replaceScreenshots(cloudModel.events);
183
+ for (const child of cloudModel.childTests ?? []) {
184
+ replaceScreenshots(child.events);
185
+ }
186
+ return renderHtmlReport(cloudModel, { includeScripts: false });
187
+ }
188
+ export async function writeCliRunReportArtifact(artifact, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
103
189
  const outputPath = resolveReportOutputPath({
104
190
  reportPath,
105
- testName: modelInput.testName,
106
- startedAtIso: modelInput.startedAtIso,
191
+ testName: artifact.model.testName,
192
+ startedAtIso: artifact.model.timestamp,
107
193
  cwd,
108
194
  isBatch,
109
195
  });
110
196
  await mkdir(path.dirname(outputPath), { recursive: true });
111
- await writeFile(outputPath, renderHtmlReport(model), "utf-8");
197
+ await writeFile(outputPath, artifact.html, "utf-8");
112
198
  return outputPath;
113
199
  }
200
+ // Renders and writes the HTML report for one run. Returns the absolute output path.
201
+ export async function writeCliRunReport(modelInput, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
202
+ return await writeCliRunReportArtifact(createCliRunReport(modelInput), { reportPath, cwd, isBatch });
203
+ }
@@ -7,11 +7,12 @@ import { buildExecutionModePrompt } from "../core/prompts.js";
7
7
  import { addOpenAiResponseToDroidCuaRunUsage, createDroidCuaRunUsageTotals, estimateDroidCuaRunCostUsd, setDroidCuaRunUsageModel, } from "../core/droid-cua-run-usage.js";
8
8
  import { ExecutionMode } from "../modes/execution-mode.js";
9
9
  import { reportDroidRunCompletion } from "../integrations/loadmill/run-reporter.js";
10
+ import { droidReportUploadManager } from "../integrations/loadmill/report-upload-manager.js";
10
11
  import { reportDroidCuaCiTestRunCompleted } from "../integrations/segment/droid-cua-segment.js";
11
12
  import { printCliOutput } from "../utils/console-output.js";
12
13
  import { emitDesktopDebug } from "../utils/desktop-debug.js";
13
14
  import { loadHeadlessInstructions } from "./headless-instructions.js";
14
- import { createCapturingAddOutput, writeCliRunReport } from "./headless-report.js";
15
+ import { buildCliRunReportModel, createCapturingAddOutput, createCliAggregateRunReport, createCliRunReport, renderCliCloudReport, writeCliRunReportArtifact, } from "./headless-report.js";
15
16
  function sanitizePathSegment(value) {
16
17
  return String(value || "test")
17
18
  .replace(/\.dcua$/i, "")
@@ -24,6 +25,14 @@ function createHeadlessRunId(index = 0) {
24
25
  const suffix = index > 0 ? `-${index + 1}` : "";
25
26
  return `run-${Date.now()}${suffix}`;
26
27
  }
28
+ function logPendingScreenshotUploads(uploadManager, uploadSession, subject) {
29
+ console.log('Verifying no pending upload exists');
30
+ const pendingCount = uploadManager.getPendingCount?.(uploadSession) ?? 0;
31
+ if (pendingCount > 0) {
32
+ const uploads = pendingCount === 1 ? "upload" : "uploads";
33
+ console.log(`${subject} finished. Waiting for ${pendingCount} screenshot ${uploads} to finish...`);
34
+ }
35
+ }
27
36
  function getRecordScreenshotDirForTest(baseScreenshotDir, testName, isBatchRun) {
28
37
  if (!baseScreenshotDir || !isBatchRun) {
29
38
  return baseScreenshotDir;
@@ -122,7 +131,7 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
122
131
  }
123
132
  return engineOptions;
124
133
  }
125
- async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true }) {
134
+ async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
126
135
  let executionMode = null;
127
136
  let loadmillSessionUser = null;
128
137
  let runUsageTotals = setDroidCuaRunUsageModel(createDroidCuaRunUsageTotals(), runtimeContext.config.cuaModel);
@@ -142,10 +151,17 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
142
151
  ...buildSegmentUsageProperties(),
143
152
  });
144
153
  };
145
- const reportEnabled = Boolean(report?.enabled);
146
- const collectedEvents = [];
154
+ const localReportEnabled = Boolean(report?.enabled);
155
+ const cloudReportEnabled = Boolean(cloudUploadSession);
156
+ const reportEnabled = localReportEnabled || cloudReportEnabled;
157
+ const localEvents = [];
158
+ const cloudEvents = [];
147
159
  const addOutput = reportEnabled
148
- ? createCapturingAddOutput(printCliOutput, collectedEvents)
160
+ ? createCapturingAddOutput(printCliOutput, localReportEnabled ? localEvents : null, {
161
+ cloudSink: cloudReportEnabled ? cloudEvents : null,
162
+ uploadManager,
163
+ uploadSession: cloudUploadSession,
164
+ })
149
165
  : printCliOutput;
150
166
  try {
151
167
  try {
@@ -175,6 +191,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
175
191
  testDataContext,
176
192
  availableSecretKeys: secrets?.availableSecretKeys || [],
177
193
  cuaModel: runtimeContext.config.cuaModel,
194
+ explainMode: runtimeContext.config.explainMode,
178
195
  });
179
196
  session.setSystemPrompt(executionPrompt);
180
197
  const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
@@ -226,8 +243,9 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
226
243
  durationMs,
227
244
  numOfSteps: instructions.length,
228
245
  });
246
+ let reportedRun = null;
229
247
  if (reportToLoadmill) {
230
- await reportDroidRunCompletion({
248
+ reportedRun = await reportDroidRunCompletion({
231
249
  loadmillClient: runtimeContext.loadmillClient,
232
250
  source: "ci",
233
251
  runId,
@@ -255,19 +273,19 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
255
273
  if (!result.success) {
256
274
  console.error(`\nTest failed: ${result.error}`);
257
275
  }
258
- if (reportEnabled) {
276
+ const reportInput = {
277
+ suiteName: path.basename(path.dirname(testPath)) || null,
278
+ testName,
279
+ startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
280
+ durationMs,
281
+ success: Boolean(result.success),
282
+ error: result.error ?? null,
283
+ instructionsTotal: instructions.length,
284
+ stats,
285
+ };
286
+ if (localReportEnabled) {
259
287
  try {
260
- const outputPath = await writeCliRunReport({
261
- suiteName: path.basename(path.dirname(testPath)) || null,
262
- testName,
263
- startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
264
- durationMs,
265
- success: Boolean(result.success),
266
- error: result.error ?? null,
267
- instructionsTotal: instructions.length,
268
- stats,
269
- events: collectedEvents,
270
- }, { reportPath: report?.path ?? null, isBatch: isBatchRun });
288
+ const outputPath = await writeCliRunReportArtifact(createCliRunReport({ ...reportInput, events: localEvents }), { reportPath: report?.path ?? null, isBatch: isBatchRun });
271
289
  console.log(`HTML report saved: ${outputPath}`);
272
290
  }
273
291
  catch (reportError) {
@@ -275,6 +293,16 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
275
293
  console.warn(`Warning: failed to write HTML report: ${message}`);
276
294
  }
277
295
  }
296
+ const cloudModel = cloudReportEnabled
297
+ ? buildCliRunReportModel({ ...reportInput, events: cloudEvents })
298
+ : null;
299
+ if (cloudModel && reportToLoadmill && reportedRun?.savedRunId) {
300
+ logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Test");
301
+ await uploadManager.finish(cloudUploadSession, {
302
+ savedRunId: reportedRun.savedRunId,
303
+ html: renderCliCloudReport(cloudModel, reportedRun.savedRunId),
304
+ });
305
+ }
278
306
  return {
279
307
  success: Boolean(result.success),
280
308
  error: result.error ?? null,
@@ -284,7 +312,8 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
284
312
  actionsTotal: stats.actionCount ?? 0,
285
313
  assertionsPassed: stats.assertionsPassed ?? 0,
286
314
  assertionsFailed: stats.assertionsFailed ?? 0,
287
- retries: stats.retryCount ?? 0
315
+ retries: stats.retryCount ?? 0,
316
+ reportModel: cloudModel,
288
317
  };
289
318
  }
290
319
  catch (error) {
@@ -311,8 +340,9 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
311
340
  durationMs,
312
341
  numOfSteps: instructions.length,
313
342
  });
343
+ let reportedRun = null;
314
344
  if (reportToLoadmill) {
315
- await reportDroidRunCompletion({
345
+ reportedRun = await reportDroidRunCompletion({
316
346
  loadmillClient: runtimeContext.loadmillClient,
317
347
  source: "ci",
318
348
  runId,
@@ -337,11 +367,63 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
337
367
  onWarning: (warningMessage) => console.warn(`Warning: ${warningMessage}`),
338
368
  });
339
369
  }
370
+ const failedReportInput = {
371
+ suiteName: path.basename(path.dirname(testPath)) || null,
372
+ testName,
373
+ startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
374
+ durationMs,
375
+ success: false,
376
+ error: message,
377
+ instructionsTotal: instructions.length,
378
+ stats,
379
+ };
380
+ if (localReportEnabled) {
381
+ try {
382
+ const outputPath = await writeCliRunReportArtifact(createCliRunReport({ ...failedReportInput, events: localEvents }), { reportPath: report?.path ?? null, isBatch: isBatchRun });
383
+ console.log(`HTML report saved: ${outputPath}`);
384
+ }
385
+ catch (reportError) {
386
+ console.warn(`Warning: failed to write HTML report: ${reportError instanceof Error ? reportError.message : String(reportError)}`);
387
+ }
388
+ }
389
+ const cloudModel = cloudReportEnabled
390
+ ? buildCliRunReportModel({ ...failedReportInput, events: cloudEvents })
391
+ : null;
392
+ if (cloudModel && reportToLoadmill && reportedRun?.savedRunId) {
393
+ logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Test");
394
+ await uploadManager.finish(cloudUploadSession, {
395
+ savedRunId: reportedRun.savedRunId,
396
+ html: renderCliCloudReport(cloudModel, reportedRun.savedRunId),
397
+ });
398
+ }
399
+ if (!reportToLoadmill) {
400
+ console.error(`\nTest failed: ${testName}: ${message}`);
401
+ return {
402
+ success: false,
403
+ error: message,
404
+ durationMs,
405
+ instructionsTotal: instructions.length,
406
+ instructionsCompleted: stats.instructionsCompleted ?? 0,
407
+ actionsTotal: stats.actionCount ?? 0,
408
+ assertionsPassed: stats.assertionsPassed ?? 0,
409
+ assertionsFailed: stats.assertionsFailed ?? 0,
410
+ retries: stats.retryCount ?? 0,
411
+ reportModel: cloudModel,
412
+ };
413
+ }
340
414
  throw error;
341
415
  }
342
416
  }
343
- export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, report = null, runHeadlessTestImpl = runHeadlessTest }) {
417
+ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, report = null, runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
344
418
  const isBatchRun = instructionsTarget?.type === "directory" || instructionsTarget?.type === "scenario";
419
+ const cloudReportEnabled = report?.cloudEnabled !== false &&
420
+ Boolean(runtimeContext.loadmillClient?.getApiToken?.());
421
+ const cloudUploadSession = cloudReportEnabled
422
+ ? uploadManager.startSession({
423
+ loadmillClient: runtimeContext.loadmillClient,
424
+ onWarning: (message) => console.warn(`Warning: ${message}`),
425
+ })
426
+ : null;
345
427
  if (instructionsTarget?.type === "scenario") {
346
428
  console.log(`\nRunning ${runs.length} scenario case${runs.length === 1 ? "" : "s"}\n`);
347
429
  }
@@ -359,6 +441,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
359
441
  assertionsFailed: 0,
360
442
  retries: 0
361
443
  };
444
+ const aggregateReportModels = [];
362
445
  for (const [index, run] of runs.entries()) {
363
446
  const runId = createHeadlessRunId(index);
364
447
  const testScreenshotDir = getRecordScreenshotDirForTest(screenshotDir, run.testName, isBatchRun);
@@ -389,7 +472,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
389
472
  isCloudTarget,
390
473
  report,
391
474
  isBatchRun,
392
- reportToLoadmill: !isBatchRun
475
+ reportToLoadmill: !isBatchRun,
476
+ cloudUploadSession,
477
+ uploadManager,
393
478
  });
394
479
  aggregateStats.durationMs += result.durationMs ?? 0;
395
480
  aggregateStats.instructionsTotal += result.instructionsTotal ?? 0;
@@ -398,6 +483,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
398
483
  aggregateStats.assertionsPassed += result.assertionsPassed ?? 0;
399
484
  aggregateStats.assertionsFailed += result.assertionsFailed ?? 0;
400
485
  aggregateStats.retries += result.retries ?? 0;
486
+ if (result.reportModel) {
487
+ aggregateReportModels.push(result.reportModel);
488
+ }
401
489
  if (!result.success) {
402
490
  failures += 1;
403
491
  }
@@ -422,7 +510,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
422
510
  if (isBatchRun) {
423
511
  const isScenario = instructionsTarget?.type === "scenario";
424
512
  const testName = isScenario ? "Scenario run" : path.basename(instructionsTarget.path);
425
- await reportDroidRunCompletion({
513
+ const reportedRun = await reportDroidRunCompletion({
426
514
  loadmillClient: runtimeContext.loadmillClient,
427
515
  source: "ci",
428
516
  runId: createHeadlessRunId(),
@@ -447,6 +535,20 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
447
535
  retries: aggregateStats.retries,
448
536
  onWarning: (message) => console.warn(`Warning: ${message}`),
449
537
  });
538
+ if (cloudUploadSession && reportedRun.savedRunId) {
539
+ const aggregateReport = createCliAggregateRunReport({
540
+ suiteName: isScenario ? path.basename(process.cwd()) : path.basename(instructionsTarget.path),
541
+ testName,
542
+ startedAtIso: new Date(batchStartedAt).toISOString(),
543
+ durationMs: Math.max(aggregateStats.durationMs, Date.now() - batchStartedAt),
544
+ childModels: aggregateReportModels,
545
+ });
546
+ logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Run");
547
+ await uploadManager.finish(cloudUploadSession, {
548
+ savedRunId: reportedRun.savedRunId,
549
+ html: renderCliCloudReport(aggregateReport.model, reportedRun.savedRunId),
550
+ });
551
+ }
450
552
  }
451
553
  return {
452
554
  exitCode: failures === 0 ? 0 : 1,
@@ -46,6 +46,9 @@ export async function handleHelp(args, session, context) {
46
46
  addOutput({ type: 'info', text: ' --base-prompt-file Headless base prompt customization file' });
47
47
  addOutput({ type: 'info', text: ' --execution-prompt-file Headless execution prompt customization file' });
48
48
  addOutput({ type: 'info', text: ' --record Record screenshots during execution' });
49
+ addOutput({ type: 'info', text: ' --explain Explain non-obvious decisions and completion evidence' });
50
+ addOutput({ type: 'info', text: ' --report [path] Save a self-contained HTML report locally' });
51
+ addOutput({ type: 'info', text: ' --no-cloud-report Skip uploading the HTML report and screenshots to Loadmill' });
49
52
  addOutput({ type: 'info', text: ' --debug Enable structured JSONL debug artifacts' });
50
53
  addOutput({ type: 'info', text: '' });
51
54
  addOutput({ type: 'info', text: 'Interactive commands:' });
@@ -399,7 +399,7 @@ export class ExecutionEngine {
399
399
  eventId: actionEventId,
400
400
  captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
401
401
  });
402
- // Inline the green click marker as a data URL so self-contained reports show it.
402
+ // Inline the accent-colored click marker as a data URL so self-contained reports show it.
403
403
  // The Desktop recorder returns one from disk; for runs without a recorder (e.g. a
404
404
  // plain CLI `--report`) render the same marker in-memory from the raw screenshot.
405
405
  const annotatedDataUrl = await this.resolveAnnotatedDataUrl(annotatedScreenshot, currentScreenshot, action, context);
@@ -526,7 +526,7 @@ export class ExecutionEngine {
526
526
  type: "assistant",
527
527
  text: textPart.text,
528
528
  eventType: "assistant_message",
529
- ...eventMeta(),
529
+ ...eventMeta(options.explainMode ? { payload: { explainMode: true } } : {}),
530
530
  });
531
531
  }
532
532
  this.session.addToTranscript(`[Assistant] ${textPart.text}`);
@@ -689,7 +689,10 @@ export class ExecutionEngine {
689
689
  const items = response.output || [];
690
690
  const actionableItems = extractActionableOutputItems(items);
691
691
  const pendingSafetyChecks = collectPendingSafetyChecks(items);
692
- this.emitResponseOutputItems(items, addOutput, eventMeta, options);
692
+ this.emitResponseOutputItems(items, addOutput, eventMeta, {
693
+ ...options,
694
+ explainMode: options.explainMode === true || context?.runtimeConfig?.explainMode === true,
695
+ });
693
696
  if (actionableItems.length === 0) {
694
697
  // No actions = turn complete
695
698
  break;
@@ -83,13 +83,60 @@ function buildVisibleEvents(events) {
83
83
  // full-size lightbox overlay can be rendered. The href links to the CSS `:target`
84
84
  // overlay, so click-to-enlarge works even with JavaScript disabled.
85
85
  function screenshotImg(event, shots) {
86
- const url = event.payload?.screenshotDataUrl;
87
- if (typeof url !== "string" || !url.startsWith("data:"))
86
+ const url = event.payload?.screenshotDataUrl ?? event.payload?.screenshotUrl;
87
+ const isAllowedUrl = typeof url === "string" &&
88
+ (url.startsWith("data:") || url.startsWith("/api/droid-cua/runs/"));
89
+ if (!isAllowedUrl)
88
90
  return "";
89
91
  const index = shots.length + 1;
90
92
  shots.push({ index, url });
91
93
  return `<a class="dc-shot-link" href="#dc-shot-${index}" aria-label="Open screenshot ${index}"><img class="dc-shot" src="${escapeHtml(url)}" alt="Screenshot ${index}" /></a>`;
92
94
  }
95
+ function icon(name, shapes) {
96
+ return `<svg class="dc-icon" data-icon="${name}" aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${shapes}</svg>`;
97
+ }
98
+ const ICONS = {
99
+ arrowUpDown: icon("arrow-up-down", `<path d="m21 16-4 4-4-4"/><path d="M17 20V4"/><path d="m3 8 4-4 4 4"/><path d="M7 4v16"/>`),
100
+ botMessageSquare: icon("bot-message-square", `<path d="M12 6V2H8"/><path d="m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z"/><path d="M2 12h2"/><path d="M9 11v2"/><path d="M15 11v2"/><path d="M20 12h2"/>`),
101
+ camera: icon("camera", `<path d="M14.5 4h-5L7 7H4a2 2 0 0 0-2 2v9a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-3l-2.5-3z"/><circle cx="12" cy="13" r="3"/>`),
102
+ compass: icon("compass", `<path d="m16.24 7.76-1.804 5.411a2 2 0 0 1-1.265 1.265L7.76 16.24l1.804-5.411a2 2 0 0 1 1.265-1.265z"/><circle cx="12" cy="12" r="10"/>`),
103
+ flag: icon("flag", `<path d="M4 15s1-1 4-1 5 2 8 2 4-1 4-1V3s-1 1-4 1-5-2-8-2-4 1-4 1z"/><line x1="4" x2="4" y1="22" y2="15"/>`),
104
+ keyboard: icon("keyboard", `<path d="M10 8h.01"/><path d="M12 12h.01"/><path d="M14 8h.01"/><path d="M16 12h.01"/><path d="M18 8h.01"/><path d="M6 8h.01"/><path d="M7 16h10"/><path d="M8 12h.01"/><rect width="20" height="16" x="2" y="4" rx="2"/>`),
105
+ keyboardOff: icon("keyboard-off", `<path d="M20 4a2 2 0 0 1 2 2"/><path d="M22 6v10.41"/><path d="M7 16h9"/><path d="M9.69 4H20"/><path d="M14 8h.01"/><path d="M18 8h.01"/><path d="m2 2 20 20"/><path d="M20 20H4a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2"/><path d="M6 8h.01"/><path d="M8 12h.01"/>`),
106
+ keyRound: icon("key-round", `<path d="M2.586 17.414A2 2 0 0 0 2 18.828V21a1 1 0 0 0 1 1h3a1 1 0 0 0 1-1v-1a1 1 0 0 1 1-1h1a1 1 0 0 0 1-1v-1a1 1 0 0 1 1-1h.172a2 2 0 0 0 1.414-.586l.814-.814a6.5 6.5 0 1 0-4-4z"/><circle cx="16.5" cy="7.5" r=".5" fill="currentColor"/>`),
107
+ mousePointer: icon("mouse-pointer-2", `<path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/>`),
108
+ moveDiagonal: icon("move-diagonal-2", `<path d="M19 13v6h-6"/><path d="M5 11V5h6"/><path d="m5 5 14 14"/>`),
109
+ rotateCcw: icon("rotate-ccw", `<path d="M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8"/><path d="M3 3v5h5"/>`),
110
+ squareX: icon("square-x", `<rect width="18" height="18" x="3" y="3" rx="2"/><path d="m15 9-6 6"/><path d="m9 9 6 6"/>`),
111
+ timer: icon("timer", `<line x1="10" x2="14" y1="2" y2="2"/><line x1="12" x2="15" y1="14" y2="11"/><circle cx="12" cy="14" r="8"/>`),
112
+ };
113
+ function actionIcon(event) {
114
+ const lowerText = event.text.toLowerCase();
115
+ const toolName = typeof event.payload?.toolName === "string" ? event.payload.toolName : null;
116
+ const isSensitiveTyping = Boolean(event.payload?.sensitive) || lowerText.includes("typing secret");
117
+ if (toolName === "navigate")
118
+ return ICONS.compass;
119
+ if (toolName === "close_current_tab")
120
+ return ICONS.squareX;
121
+ if (toolName === "close_keyboard")
122
+ return ICONS.keyboardOff;
123
+ if (isSensitiveTyping)
124
+ return ICONS.keyRound;
125
+ if (event.actionType === "type" || event.actionType === "keypress" || lowerText.includes("typing"))
126
+ return ICONS.keyboard;
127
+ if (event.actionType === "scroll" || lowerText.includes("scroll"))
128
+ return ICONS.arrowUpDown;
129
+ if (event.actionType === "drag" || lowerText.includes("drag"))
130
+ return ICONS.moveDiagonal;
131
+ if (event.actionType === "screenshot" || lowerText.includes("capturing screen"))
132
+ return ICONS.camera;
133
+ if (event.actionType === "wait" || lowerText.includes("waiting"))
134
+ return ICONS.timer;
135
+ return ICONS.mousePointer;
136
+ }
137
+ function renderAgentText(text) {
138
+ return `<div class="dc-line dc-agent-line">${ICONS.botMessageSquare}<span>${escapeHtml(text)}</span></div>`;
139
+ }
93
140
  // ---------- per-event timeline rendering (mirrors ExecutionLogItem.tsx) ----------
94
141
  function renderEvent(event, shots) {
95
142
  if (isHiddenExecutionLogEvent(event))
@@ -123,10 +170,16 @@ function renderEvent(event, shots) {
123
170
  if (isAssistant) {
124
171
  const stepCompletion = stepCompletionData(event);
125
172
  if (stepCompletion) {
173
+ if (event.payload?.explainMode === true) {
174
+ return `<div class="dc-line">${ICONS.flag}<span>Step complete</span></div>\n${renderAgentText(stepCompletion.reason)}`;
175
+ }
126
176
  const tooltip = stepCompletionReasonTooltip(stepCompletion);
127
- return `<div class="dc-line"${tooltip ? ` title="${escapeHtml(tooltip)}"` : ""}><span aria-hidden="true">&#9873;</span><span>Step complete</span></div>`;
177
+ return `<div class="dc-line"${tooltip ? ` title="${escapeHtml(tooltip)}"` : ""}>${ICONS.flag}<span>Step complete</span></div>`;
128
178
  }
129
- return `<div class="dc-assistant">${escapeHtml(event.text)}</div>`;
179
+ return renderAgentText(event.text);
180
+ }
181
+ if (event.eventType === "reasoning" || event.kind === "reasoning") {
182
+ return renderAgentText(event.text);
130
183
  }
131
184
  const assertion = assertionData(event);
132
185
  if (assertion) {
@@ -138,13 +191,13 @@ function renderEvent(event, shots) {
138
191
  </div>`;
139
192
  }
140
193
  if (event.eventType === "retry") {
141
- return `<div class="dc-line dc-retry">${escapeHtml(event.text)}</div>`;
194
+ return `<div class="dc-line dc-retry">${ICONS.rotateCcw}<span>${escapeHtml(event.text)}</span></div>`;
142
195
  }
143
196
  if (event.kind === "action") {
144
- return `<div class="dc-line">${escapeHtml(event.text)}${screenshotImg(event, shots)}</div>`;
197
+ return `<div class="dc-line">${actionIcon(event)}<span>${escapeHtml(event.text)}</span>${screenshotImg(event, shots)}</div>`;
145
198
  }
146
199
  if (event.eventType === "screenshot_captured") {
147
- return `<div class="dc-line">${escapeHtml(event.text)}${screenshotImg(event, shots)}</div>`;
200
+ return `<div class="dc-line">${ICONS.camera}<span>${escapeHtml(event.text)}</span>${screenshotImg(event, shots)}</div>`;
148
201
  }
149
202
  return `<div class="dc-line dc-${escapeHtml(event.kind)}">${escapeHtml(event.text)}</div>`;
150
203
  }
@@ -243,13 +296,16 @@ body { margin: 0; background: #f1f5f9; color: #0f172a; font-family: -apple-syste
243
296
  .dc-row { display: flex; }
244
297
  .dc-row-right { justify-content: flex-end; }
245
298
  .dc-bubble { display: inline-block; max-width: 70%; background: #f1f5f9; border: 1px solid #e2e8f0; border-radius: 16px; border-bottom-right-radius: 6px; padding: 8px 14px; color: #0f172a; }
246
- .dc-assistant { max-width: 82%; color: #334155; }
247
299
  .dc-line { display: flex; align-items: center; gap: 8px; color: #475569; }
300
+ .dc-agent-line { align-items: flex-start; }
301
+ .dc-agent-line .dc-icon { margin-top: 4px; }
302
+ .dc-icon { width: 14px; height: 14px; flex: 0 0 auto; color: #94a3b8; }
248
303
  .dc-retry { color: #b45309; }
249
304
  .dc-error { color: #b91c1c; }
250
305
  .dc-warning { color: #b45309; }
251
- .dc-shot-link { display: inline-flex; margin-left: 8px; vertical-align: middle; }
252
- .dc-shot { height: 34px; width: auto; border: 1px solid #e2e8f0; border-radius: 4px; cursor: zoom-in; transition: border-color .12s; }
306
+ .dc-shot-link { display: inline-flex; align-items: center; justify-content: center; position: relative; min-width: 34px; min-height: 36px; margin-left: 8px; vertical-align: middle; }
307
+ .dc-shot-link::before, .dc-lightbox:target::before { content: ""; position: absolute; width: 14px; height: 14px; border: 2px solid #cbd5e1; border-top-color: #475569; border-radius: 50%; animation: dc-spin .7s linear infinite; }
308
+ .dc-shot { position: relative; z-index: 1; height: 34px; width: auto; border: 1px solid #e2e8f0; border-radius: 4px; cursor: zoom-in; transition: border-color .12s; }
253
309
  .dc-shot-link:hover .dc-shot, .dc-shot-link:focus-visible .dc-shot { border-color: #94a3b8; }
254
310
  .dc-lightbox { display: none; position: fixed; inset: 0; z-index: 1000; align-items: center; justify-content: center; padding: 32px; background: rgba(15, 23, 42, .82); }
255
311
  .dc-lightbox:target { display: flex; }
@@ -275,9 +331,10 @@ body { margin: 0; background: #f1f5f9; color: #0f172a; font-family: -apple-syste
275
331
  .dc-pill-error, .dc-pill-errored, .dc-pill-amber { border-color: #fcd34d; background: #fef3c7; color: #b45309; }
276
332
  .dc-pill-stopped { border-color: #fdba74; background: #ffedd5; color: #c2410c; }
277
333
  .dc-pill-skipped { border-color: #cbd5e1; background: #f1f5f9; color: #334155; }
334
+ @keyframes dc-spin { to { transform: rotate(360deg); } }
278
335
  `.trim();
279
336
  // ---------- entry point ----------
280
- export function renderHtmlReport(model, { title } = {}) {
337
+ export function renderHtmlReport(model, { title, includeScripts = true } = {}) {
281
338
  const docTitle = title ?? `Run Report: ${model.testName ?? model.suiteName ?? "Test run"}`;
282
339
  const heading = model.testName ?? model.suiteName ?? "Test run";
283
340
  // Screenshots are collected across the whole document (top-level + child timelines)
@@ -301,7 +358,7 @@ ${renderTimeline(model.events, shots)}
301
358
  </div>`;
302
359
  }
303
360
  const lightboxes = renderLightboxes(shots);
304
- const script = renderLightboxScript(shots.length);
361
+ const script = includeScripts ? renderLightboxScript(shots.length) : "";
305
362
  return `<!DOCTYPE html>
306
363
  <html lang="en">
307
364
  <head>
@@ -123,7 +123,7 @@ export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToke
123
123
  loadmillBaseUrl: normalizeLoadmillSiteBaseUrl(loadmillBaseUrl),
124
124
  };
125
125
  }
126
- export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
126
+ export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null, explainMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
127
127
  const normalizedCuaModel = typeof cuaModel === "string" && LOADMILL_MODEL_ALIAS_SET.has(cuaModel)
128
128
  ? cuaModel
129
129
  : resolveCuaModel(cuaModel);
@@ -133,6 +133,7 @@ export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null,
133
133
  ? "loadmill"
134
134
  : normalizedProviderMode,
135
135
  cuaModel: normalizedCuaModel,
136
+ explainMode: explainMode === true,
136
137
  stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
137
138
  contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
138
139
  contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),
@@ -1,4 +1,29 @@
1
1
  import { buildTextToolOutputItem, parseStringToolArgument } from "./utils.js";
2
+ import { emitDesktopDebug } from "../../utils/desktop-debug.js";
3
+ function emitLoadmillToolCall({ callId, argumentsText, command, context, stepContext }) {
4
+ const scope = context?.sessionId ? "design" : "execution";
5
+ const ids = {
6
+ ...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
7
+ ...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
8
+ ...(Number.isInteger(stepContext?.instructionIndex) ? { instructionIndex: stepContext.instructionIndex } : {}),
9
+ };
10
+ const payload = {
11
+ toolName: "loadmill",
12
+ callId,
13
+ command,
14
+ arguments: argumentsText,
15
+ };
16
+ emitDesktopDebug("tool_call", scope, ids, payload);
17
+ context?.addOutput?.({
18
+ type: "muted",
19
+ text: `Running Loadmill flow: ${command}`,
20
+ eventType: "tool_call",
21
+ runId: context?.runId,
22
+ stepId: stepContext?.stepId,
23
+ instructionIndex: stepContext?.instructionIndex,
24
+ payload,
25
+ });
26
+ }
2
27
  export async function handleLoadmillToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
3
28
  const session = engine.session;
4
29
  const parsed = parseStringToolArgument(argumentsText, "command", "loadmill");
@@ -9,6 +34,13 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
9
34
  };
10
35
  }
11
36
  const loadmillCommand = parsed.value;
37
+ emitLoadmillToolCall({
38
+ callId,
39
+ argumentsText,
40
+ command: loadmillCommand,
41
+ context,
42
+ stepContext,
43
+ });
12
44
  session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
13
45
  const result = await engine.executeLoadmillInstruction(loadmillCommand, engine.isHeadlessMode, {
14
46
  ...context,
@@ -2,6 +2,7 @@
2
2
  * Assertion handling for script validation
3
3
  */
4
4
  import { printCliOutput } from "../utils/console-output.js";
5
+ import { CAPTURE_SOURCES } from "../core/capture-sources.js";
5
6
  export { buildAssertionSystemPrompt } from "../prompts/execution.js";
6
7
  const ASSERTION_PREFIXES = ["assert", "verify", "validate", "ensure"];
7
8
  function hasAssertionPrefix(lower) {
@@ -54,6 +55,7 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
54
55
  assertion: assertionPrompt,
55
56
  passed: false,
56
57
  details,
58
+ captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
57
59
  ...eventPayload
58
60
  }
59
61
  };
@@ -77,6 +79,7 @@ export function handleAssertionSuccess(assertionPrompt, context = null, stepCont
77
79
  payload: {
78
80
  assertion: assertionPrompt,
79
81
  passed: true,
82
+ captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
80
83
  ...eventPayload
81
84
  }
82
85
  });
@@ -138,6 +138,18 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
138
138
  body: JSON.stringify(payload),
139
139
  });
140
140
  }
141
+ async function requestDroidRunReportUploadPolicy(uploadId, payload) {
142
+ return await apiRequest(`/droid-cua/report-uploads/${uploadId}/policy`, {
143
+ method: "POST",
144
+ body: JSON.stringify(payload),
145
+ });
146
+ }
147
+ async function associateDroidRunReport(runId, payload) {
148
+ return await apiRequest(`/droid-cua/runs/${runId}/report`, {
149
+ method: "POST",
150
+ body: JSON.stringify(payload),
151
+ });
152
+ }
141
153
  async function validateApiToken() {
142
154
  const resolvedToken = getApiToken();
143
155
  if (!resolvedToken) {
@@ -187,6 +199,8 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
187
199
  getTestSuiteFlowRun,
188
200
  explainFailure,
189
201
  reportDroidRun,
202
+ requestDroidRunReportUploadPolicy,
203
+ associateDroidRunReport,
190
204
  validateApiToken,
191
205
  getCurrentUser,
192
206
  };
@@ -0,0 +1,221 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ export const MAX_SCREENSHOTS_PER_REPORT = 1000;
3
+ export const MAX_REPORT_OBJECT_BYTES = 10 * 1024 * 1024;
4
+ function warn(session, message) {
5
+ session.failed = true;
6
+ session.onWarning?.(message);
7
+ }
8
+ function getHttpStatus(error) {
9
+ const message = error instanceof Error ? error.message : String(error);
10
+ return /\((\d{3})\)/.exec(message)?.[1] ?? null;
11
+ }
12
+ function warnAboutUploadFailure(session) {
13
+ if (!session.uploadFailed) {
14
+ return;
15
+ }
16
+ const status = session.uploadFailureStatus ? ` (HTTP ${session.uploadFailureStatus})` : "";
17
+ session.onWarning?.(`Cloud report upload failed${status}; further uploads skipped.`);
18
+ }
19
+ function decodeScreenshot(dataUrl) {
20
+ const match = /^data:image\/png;base64,([a-zA-Z0-9+/=\s]+)$/.exec(dataUrl);
21
+ return match ? Buffer.from(match[1], "base64") : null;
22
+ }
23
+ async function uploadObject(uploadPolicy, body, name, contentType, fetchImpl) {
24
+ const form = new FormData();
25
+ for (const [key, value] of Object.entries(uploadPolicy.data ?? {})) {
26
+ form.append(key, String(value));
27
+ }
28
+ form.append("file", new Blob([body], { type: contentType }), name);
29
+ const response = await fetchImpl(uploadPolicy.url, { method: "POST", body: form });
30
+ if (!response.ok) {
31
+ throw new Error(`S3 upload failed (${response.status}): ${await response.text()}`);
32
+ }
33
+ }
34
+ export class DroidReportUploadManager {
35
+ constructor({ workers = 4, fetchImpl = fetch } = {}) {
36
+ this.workers = workers;
37
+ this.fetchImpl = fetchImpl;
38
+ this.queue = [];
39
+ this.activeWorkers = 0;
40
+ this.sessions = new Set();
41
+ }
42
+ /**
43
+ * @param {{
44
+ * loadmillClient?: {
45
+ * requestDroidRunReportUploadPolicy: (uploadId: string, payload: { name: string, size: number }) => Promise<unknown>,
46
+ * associateDroidRunReport: (runId: string, payload: object) => Promise<unknown>,
47
+ * },
48
+ * onWarning?: ((message: string) => void) | null,
49
+ * }} options
50
+ */
51
+ startSession({ loadmillClient, onWarning = null } = {}) {
52
+ const session = {
53
+ uploadId: randomUUID(),
54
+ loadmillClient,
55
+ onWarning,
56
+ sequence: 0,
57
+ screenshotCount: 0,
58
+ deduplication: new Map(),
59
+ pending: new Set(),
60
+ failed: false,
61
+ uploadFailed: false,
62
+ uploadFailureStatus: null,
63
+ finishing: false,
64
+ finishPromise: null,
65
+ };
66
+ this.sessions.add(session);
67
+ return session;
68
+ }
69
+ enqueueScreenshot(session, dataUrl) {
70
+ if (!session || session.finishing || session.uploadFailed || typeof dataUrl !== "string") {
71
+ return null;
72
+ }
73
+ const hash = createHash("sha256").update(dataUrl).digest("hex");
74
+ const existing = session.deduplication.get(hash);
75
+ if (existing) {
76
+ return existing;
77
+ }
78
+ const body = decodeScreenshot(dataUrl);
79
+ if (!body) {
80
+ warn(session, "Failed to upload Droid report screenshot: unsupported image data.");
81
+ return null;
82
+ }
83
+ if (body.length > MAX_REPORT_OBJECT_BYTES) {
84
+ warn(session, `Droid report screenshot exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
85
+ return null;
86
+ }
87
+ if (session.screenshotCount >= MAX_SCREENSHOTS_PER_REPORT) {
88
+ warn(session, `Droid report exceeds ${MAX_SCREENSHOTS_PER_REPORT} screenshots.`);
89
+ return null;
90
+ }
91
+ const name = `screenshots/${String(++session.sequence).padStart(4, "0")}.png`;
92
+ session.screenshotCount += 1;
93
+ session.deduplication.set(hash, name);
94
+ this.#enqueue(session, { name, body, contentType: "image/png" });
95
+ return name;
96
+ }
97
+ finish(session, { savedRunId, html } = {}) {
98
+ if (!session) {
99
+ return Promise.resolve({ published: false, reason: "missing_session" });
100
+ }
101
+ if (session.finishPromise) {
102
+ return session.finishPromise;
103
+ }
104
+ session.finishing = true;
105
+ session.finishPromise = this.#finish(session, { savedRunId, html }).catch((error) => {
106
+ const status = getHttpStatus(error);
107
+ warn(session, `Failed to publish Droid run report${status ? ` (HTTP ${status})` : ""}.`);
108
+ return { published: false, reason: "publish_failed" };
109
+ }).finally(() => {
110
+ this.sessions.delete(session);
111
+ });
112
+ return session.finishPromise;
113
+ }
114
+ async #finish(session, { savedRunId, html }) {
115
+ await this.waitForSession(session);
116
+ if (session.uploadFailed) {
117
+ warnAboutUploadFailure(session);
118
+ return { published: false, reason: "upload_failed" };
119
+ }
120
+ if (!savedRunId || typeof html !== "string") {
121
+ return { published: false, reason: "missing_report" };
122
+ }
123
+ const body = Buffer.from(html, "utf8");
124
+ if (body.length > MAX_REPORT_OBJECT_BYTES) {
125
+ warn(session, `Droid HTML report exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
126
+ return { published: false, reason: "too_large" };
127
+ }
128
+ const htmlUpload = await this.#enqueue(session, {
129
+ name: "report.html",
130
+ body,
131
+ contentType: "text/html; charset=utf-8",
132
+ });
133
+ if (session.failed) {
134
+ warnAboutUploadFailure(session);
135
+ return { published: false, reason: "upload_failed" };
136
+ }
137
+ const key = htmlUpload.key;
138
+ await session.loadmillClient.associateDroidRunReport(savedRunId, {
139
+ key,
140
+ screenshotCount: session.screenshotCount,
141
+ });
142
+ return { published: true, key };
143
+ }
144
+ async waitForSession(session) {
145
+ while (session?.pending?.size) {
146
+ await Promise.all([...session.pending]);
147
+ }
148
+ return { failed: Boolean(session?.failed) };
149
+ }
150
+ getPendingCount(session) {
151
+ return session?.pending?.size ?? 0;
152
+ }
153
+ /**
154
+ * @param {{ timeoutMs?: number | null }} options
155
+ */
156
+ async drain({ timeoutMs = null } = {}) {
157
+ const pending = Promise.all([...this.sessions].map((session) => session.finishPromise ?? this.waitForSession(session)));
158
+ if (!Number.isFinite(timeoutMs)) {
159
+ await pending;
160
+ return { timedOut: false };
161
+ }
162
+ let timer;
163
+ const result = await Promise.race([
164
+ pending.then(() => false),
165
+ new Promise((resolve) => {
166
+ timer = setTimeout(() => resolve(true), timeoutMs);
167
+ }),
168
+ ]);
169
+ clearTimeout(timer);
170
+ return { timedOut: result };
171
+ }
172
+ #enqueue(session, object) {
173
+ let resolveTask;
174
+ const promise = new Promise((resolve) => {
175
+ resolveTask = resolve;
176
+ });
177
+ session.pending.add(promise);
178
+ this.queue.push({ session, object, resolveTask, promise });
179
+ this.#pump();
180
+ return promise;
181
+ }
182
+ #pump() {
183
+ while (this.activeWorkers < this.workers && this.queue.length > 0) {
184
+ const task = this.queue.shift();
185
+ this.activeWorkers += 1;
186
+ this.#runTask(task).finally(() => {
187
+ this.activeWorkers -= 1;
188
+ this.#pump();
189
+ });
190
+ }
191
+ }
192
+ async #runTask(task) {
193
+ let result = { uploaded: false, key: null };
194
+ try {
195
+ if (task.session.uploadFailed) {
196
+ return;
197
+ }
198
+ const policyResult = await task.session.loadmillClient.requestDroidRunReportUploadPolicy(task.session.uploadId, { name: task.object.name, size: task.object.body.length });
199
+ const uploadPolicy = policyResult?.uploadPolicy;
200
+ if (!uploadPolicy?.key) {
201
+ throw new Error("Loadmill did not return an upload policy.");
202
+ }
203
+ await uploadObject(uploadPolicy, task.object.body, task.object.name.split("/").at(-1), task.object.contentType, this.fetchImpl);
204
+ result = { uploaded: true, key: uploadPolicy.key };
205
+ }
206
+ catch (error) {
207
+ task.session.failed = true;
208
+ task.session.uploadFailed = true;
209
+ task.session.uploadFailureStatus ??= getHttpStatus(error);
210
+ }
211
+ finally {
212
+ task.session.pending.delete(task.promise);
213
+ task.resolveTask(result);
214
+ }
215
+ }
216
+ }
217
+ export const droidReportUploadManager = new DroidReportUploadManager();
218
+ export function droidReportScreenshotUrl(savedRunId, objectName) {
219
+ const fileName = objectName.split("/").at(-1);
220
+ return `/api/droid-cua/runs/${encodeURIComponent(savedRunId)}/report/screenshots/${encodeURIComponent(fileName)}`;
221
+ }
@@ -100,8 +100,12 @@ export async function reportDroidRunCompletion({ loadmillClient, onWarning = nul
100
100
  return { reported: false, reason: "missing_run_id" };
101
101
  }
102
102
  try {
103
- await loadmillClient.reportDroidRun(payload);
104
- return { reported: true };
103
+ const response = await loadmillClient.reportDroidRun(payload);
104
+ return {
105
+ reported: true,
106
+ savedRunId: response?.run?.id ?? null,
107
+ run: response?.run ?? null,
108
+ };
105
109
  }
106
110
  catch (error) {
107
111
  const message = error instanceof Error ? error.message : String(error);
@@ -592,7 +592,8 @@ export class ExecutionMode {
592
592
  // clearMessages() restores the base system prompt, but we need to add context
593
593
  const recoverySystemPrompt = buildExecutionRecoveryPrompt({
594
594
  basePrompt: this.initialSystemText,
595
- transcript: transcriptContext
595
+ transcript: transcriptContext,
596
+ explainMode: context?.runtimeConfig?.explainMode === true,
596
597
  });
597
598
  // Replace the system message with the enhanced one
598
599
  this.session.installRecoverySystemPrompt(recoverySystemPrompt);
@@ -2,6 +2,7 @@ import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./
2
2
  import { buildAppContextSection, buildSecretKeysContextSection } from "./execution.js";
3
3
  export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
4
4
  const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
5
+ const explainMode = options.explainMode === true;
5
6
  const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
6
7
  const appContextSection = buildAppContextSection(appContextText);
7
8
  const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
@@ -94,7 +95,14 @@ WRONG Example (DON'T DO THIS):
94
95
  3. etc...
95
96
  \`\`\`
96
97
 
97
- Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
98
+ ${explainMode ? `EXPLAIN MODE:
99
+ - Briefly explain non-obvious decisions, including why you choose one path, skip an expected action, or consider the goal complete
100
+ - Name only the visible evidence that matters to the decision
101
+ - Do not narrate routine clicks, typing, scrolling, or other obvious actions
102
+ - Before a non-obvious action, briefly explain the decision in the same response as that action
103
+ - Include the evidence that proves the requested flow is complete in the same response as the \`test_design_complete\` call
104
+
105
+ ` : ""}Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
98
106
  `;
99
107
  return appendCustomSections(prompt, [
100
108
  { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
@@ -38,6 +38,7 @@ ${normalized.map((key) => `- ${key}`).join("\n")}`;
38
38
  }
39
39
  export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
40
40
  const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
41
+ const explainMode = options.explainMode === true;
41
42
  const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
42
43
  const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
43
44
  const appContextSection = buildAppContextSection(appContextText);
@@ -54,12 +55,12 @@ EXECUTION MODE - Critical Behavior:
54
55
  You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
55
56
 
56
57
  CRITICAL RULES:
57
- - DO NOT generate conversational text or narration
58
+ - ${explainMode ? "Explain non-obvious decisions briefly, including why you skip an expected action or consider the current instruction complete" : "DO NOT generate conversational text or narration"}
58
59
  - DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
59
- - DO NOT describe what you see on screen
60
+ - ${explainMode ? "When a decision depends on visible state, name only the evidence that matters to that decision" : "DO NOT describe what you see on screen"}
60
61
  - DO NOT say "Let me know if you need help" or similar phrases
61
- - Just execute the action silently and stop immediately
62
- - Only generate text if the action FAILED or cannot be completed
62
+ - ${explainMode ? "Do not narrate routine clicks, typing, scrolling, or other obvious actions" : "Just execute the action silently and stop immediately"}
63
+ - ${explainMode ? "Before a non-obvious action, briefly explain the decision in the same response as that action\n- Explain completion evidence using assistant text; do not call `assert` merely to confirm that an ordinary action succeeded\n- Call `assert` only when the current instruction explicitly contains assertion or verification intent\n- If an action FAILED or the instruction cannot be completed, state that clearly and explain why" : "Only generate text if the action FAILED or cannot be completed"}
63
64
  - ${browserTarget ? "Use browser-appropriate keyboard input only when it is the most reliable path" : "Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses"}
64
65
  - Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
65
66
  - If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
@@ -86,7 +87,7 @@ ${browserTarget ? "" : ` - Every \`type\` action must end its computer-action
86
87
  4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
87
88
  5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
88
89
  6. Stop as soon as success is visible
89
- 7. Stop immediately - no commentary, no questions
90
+ 7. ${explainMode ? "If the completion decision is not obvious, briefly state the visible evidence, then stop - no questions" : "Stop immediately - no commentary, no questions"}
90
91
 
91
92
  ${instructionContextRule}
92
93
  ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
@@ -95,7 +96,7 @@ ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ?
95
96
  { title: "Execution Mode Instructions", text: executionCustomText }
96
97
  ]);
97
98
  }
98
- export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
99
+ export function buildExecutionRecoveryPrompt({ basePrompt, transcript, explainMode = false }) {
99
100
  if (!transcript) {
100
101
  return basePrompt;
101
102
  }
@@ -104,7 +105,9 @@ export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
104
105
  [SESSION RECOVERY - Connection was lost. Previous actions completed before the error:]
105
106
  ${transcript}
106
107
 
107
- [IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]`;
108
+ ${explainMode
109
+ ? "[IMPORTANT: Resume execution. Continue using concise decision explanations without narrating routine actions.]"
110
+ : "[IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]"}`;
108
111
  }
109
112
  export function buildExecutionNavigationRecoveryPrompt({ basePrompt, transcript, navigatedUrl }) {
110
113
  const normalizedTranscript = typeof transcript === "string" ? transcript.trim() : "";
@@ -27,6 +27,9 @@ export function formatCliOutput(item) {
27
27
  }
28
28
  const stepCompletion = stepCompletionData(text);
29
29
  if (stepCompletion) {
30
+ if (item.payload?.explainMode === true) {
31
+ return `assistant: Step complete — ${stepCompletion.instruction}\nassistant: Reason: ${stepCompletion.reason}`;
32
+ }
30
33
  return `assistant: Step complete — ${stepCompletion.instruction}`;
31
34
  }
32
35
  return text
@@ -84,9 +84,9 @@ function buildMarkerPlacement(marker = {}, width, height) {
84
84
  }
85
85
  const MARKER_APPEARANCES = {
86
86
  click: {
87
- haloFill: "#39ff14",
87
+ haloFill: "#2f7df6",
88
88
  haloOpacity: 0.28,
89
- ringStroke: "#39ff14",
89
+ ringStroke: "#2f7df6",
90
90
  centerDotFill: "#07131f",
91
91
  },
92
92
  preview: {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loadmill/droid-cua",
3
- "version": "2.26.0",
3
+ "version": "2.28.0",
4
4
  "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
5
5
  "main": "build/index.js",
6
6
  "type": "module",