@loadmill/droid-cua 2.26.0 → 2.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/build/index.js +4 -0
- package/build/src/cli/headless-execution-config.js +11 -0
- package/build/src/cli/headless-report.js +105 -15
- package/build/src/cli/headless-runner.js +125 -23
- package/build/src/commands/help.js +3 -0
- package/build/src/core/execution-engine.js +6 -3
- package/build/src/core/report/render-html-report.js +69 -12
- package/build/src/core/runtime-config.js +2 -1
- package/build/src/core/tool-handlers/loadmill.js +32 -0
- package/build/src/device/assertions.js +3 -0
- package/build/src/integrations/loadmill/client.js +14 -0
- package/build/src/integrations/loadmill/report-upload-manager.js +221 -0
- package/build/src/integrations/loadmill/run-reporter.js +6 -2
- package/build/src/modes/execution-mode.js +2 -1
- package/build/src/prompts/design.js +9 -1
- package/build/src/prompts/execution.js +10 -7
- package/build/src/utils/console-output.js +3 -0
- package/build/src/utils/run-screenshot-recorder.js +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -101,6 +101,7 @@ Desktop app support:
|
|
|
101
101
|
- **Live execution logs** - Watch actions and progress as tests run
|
|
102
102
|
- **Reports and history** - Review past runs from a project results folder inside the desktop app
|
|
103
103
|
- **JUnit XML output** - Write standard test reports for CI systems and external tooling
|
|
104
|
+
- **Run video export** - Generate and download a polished MP4 replay from a desktop run's screenshots and instruction timeline
|
|
104
105
|
- **Headless support** - Reuse scripts in CLI and automation workflows
|
|
105
106
|
|
|
106
107
|
---
|
|
@@ -250,6 +251,7 @@ Supported CLI options include:
|
|
|
250
251
|
- `--no-context`
|
|
251
252
|
- `--context-optimization-threshold`
|
|
252
253
|
- `--no-context-optimization`
|
|
254
|
+
- `--explain`
|
|
253
255
|
- `--base-prompt`
|
|
254
256
|
- `--execution-prompt`
|
|
255
257
|
- `--base-prompt-file`
|
|
@@ -257,6 +259,7 @@ Supported CLI options include:
|
|
|
257
259
|
- `--secrets`
|
|
258
260
|
- `--record`
|
|
259
261
|
- `--report`
|
|
262
|
+
- `--no-cloud-report`
|
|
260
263
|
- `--debug`
|
|
261
264
|
|
|
262
265
|
`--cua-model` accepts `loadmill-smart`, `loadmill-pulse`, `loadmill-beacon`,
|
|
@@ -271,6 +274,8 @@ Config and precedence rules:
|
|
|
271
274
|
- Context Optimization is enabled by default with a `30000` input-token threshold.
|
|
272
275
|
- `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
|
|
273
276
|
- `--no-context-optimization` disables Context Optimization.
|
|
277
|
+
- `--explain` enables Explain Mode for the run. Explain Mode asks the agent for concise explanations of non-obvious decisions, skipped actions, and completion evidence; explanations appear in terminal output and reports.
|
|
278
|
+
- Set `"explainMode": true` in the JSON config to enable the same behavior without a flag. Explain Mode is off by default, and `--explain` enables it when the config leaves it off.
|
|
274
279
|
- `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
|
|
275
280
|
- `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
|
|
276
281
|
- If both inline and file-based prompt overrides are provided, the inline prompt flags win.
|
|
@@ -286,6 +291,8 @@ HTML report (`--report`):
|
|
|
286
291
|
- Writes a self-contained HTML report of the run — the same Run Details view the desktop app shows, with screenshots inlined — that you can share and open in any browser.
|
|
287
292
|
- With no value it saves `<test>--<date>--<time>.html` (e.g. `login--2026-06-03--16-09-20.html`) in the current directory; pass `--report <file>` to pick a filename or `--report <dir>` to pick a folder.
|
|
288
293
|
- In the desktop app the same report is available from a run's **Share → Download HTML report** button.
|
|
294
|
+
- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 browser replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename.
|
|
295
|
+
- Authenticated headless runs upload their HTML report and screenshots to Loadmill by default. Use `--no-cloud-report` to disable that artifact upload. The basic Droid run row is still reported to Loadmill.
|
|
289
296
|
|
|
290
297
|
```sh
|
|
291
298
|
droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua --report ./reports
|
package/build/index.js
CHANGED
|
@@ -31,6 +31,7 @@ import { formatCloudUsageReport } from "./src/cli/loadmill-cloud-usage.js";
|
|
|
31
31
|
import { LoadmillCloudControlPlaneClient } from "./src/device/cloud/loadmill-cloud/control-plane-client.js";
|
|
32
32
|
import { createStatusLine } from "./src/cli/cloud-status-line.js";
|
|
33
33
|
import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
|
|
34
|
+
import { droidReportUploadManager } from "./src/integrations/loadmill/report-upload-manager.js";
|
|
34
35
|
import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
|
|
35
36
|
import { setCloudProgressListener } from "./src/device/cloud/progress.js";
|
|
36
37
|
import { runSkillCommand } from "./src/cli/skill-command.js";
|
|
@@ -78,6 +79,7 @@ const debugMode = args["debug"] || false;
|
|
|
78
79
|
const reportConfig = {
|
|
79
80
|
enabled: args.report === true || (typeof args.report === "string" && args.report.trim().length > 0),
|
|
80
81
|
path: typeof args.report === "string" && args.report.trim().length > 0 ? args.report.trim() : null,
|
|
82
|
+
cloudEnabled: args["no-cloud-report"] !== true,
|
|
81
83
|
};
|
|
82
84
|
const screenshotDir = path.join("droid-cua-recording-" + Date.now());
|
|
83
85
|
if (recordScreenshots)
|
|
@@ -618,6 +620,7 @@ async function main() {
|
|
|
618
620
|
const runtimeContext = createCliRuntimeContext({
|
|
619
621
|
llmProviderMode: headlessConfig.llmProviderMode,
|
|
620
622
|
cuaModel: headlessConfig.cuaModel,
|
|
623
|
+
explainMode: headlessConfig.explainMode,
|
|
621
624
|
promptCustomizations: headlessConfig.promptCustomizations,
|
|
622
625
|
contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
|
|
623
626
|
contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
|
|
@@ -671,6 +674,7 @@ async function main() {
|
|
|
671
674
|
succeeded = true;
|
|
672
675
|
}
|
|
673
676
|
finally {
|
|
677
|
+
await droidReportUploadManager.drain();
|
|
674
678
|
if (lambdaTestSessionConnected) {
|
|
675
679
|
console.log("Ending LambdaTest session...");
|
|
676
680
|
await disconnectDevice();
|
|
@@ -129,6 +129,7 @@ async function loadConfigFromFile(configPath) {
|
|
|
129
129
|
llmProviderMode: undefined,
|
|
130
130
|
cuaModel: undefined,
|
|
131
131
|
aimAssistMode: undefined,
|
|
132
|
+
explainMode: undefined,
|
|
132
133
|
promptCustomizations: createEmptyPromptCustomizations(),
|
|
133
134
|
appContextEnabled: undefined,
|
|
134
135
|
appContextPath: undefined,
|
|
@@ -144,6 +145,12 @@ async function loadConfigFromFile(configPath) {
|
|
|
144
145
|
if ("aimAssistMode" in rawConfig) {
|
|
145
146
|
normalized.aimAssistMode = validateAimAssistMode(rawConfig.aimAssistMode, "config.aimAssistMode");
|
|
146
147
|
}
|
|
148
|
+
if ("explainMode" in rawConfig) {
|
|
149
|
+
if (typeof rawConfig.explainMode !== "boolean") {
|
|
150
|
+
throw new Error("config.explainMode must be a boolean.");
|
|
151
|
+
}
|
|
152
|
+
normalized.explainMode = rawConfig.explainMode;
|
|
153
|
+
}
|
|
147
154
|
if ("promptCustomizations" in rawConfig) {
|
|
148
155
|
normalized.promptCustomizations = normalizePromptCustomizations(rawConfig.promptCustomizations, "config.promptCustomizations");
|
|
149
156
|
}
|
|
@@ -217,6 +224,7 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
217
224
|
llmProviderMode: fileConfig?.llmProviderMode || readEnvDefaultLlmProviderMode(),
|
|
218
225
|
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
219
226
|
aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
|
|
227
|
+
explainMode: fileConfig?.explainMode ?? false,
|
|
220
228
|
promptCustomizations,
|
|
221
229
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
222
230
|
appContextPath: fileConfig?.appContextPath || null,
|
|
@@ -235,6 +243,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
235
243
|
if (typeof args["aim-assist"] === "string") {
|
|
236
244
|
resolved.aimAssistMode = validateAimAssistMode(args["aim-assist"], "--aim-assist");
|
|
237
245
|
}
|
|
246
|
+
if (args.explain === true) {
|
|
247
|
+
resolved.explainMode = true;
|
|
248
|
+
}
|
|
238
249
|
if (hasContextOptimizationThresholdFlag) {
|
|
239
250
|
resolved.contextOptimizationEnabled = true;
|
|
240
251
|
resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
|
|
@@ -8,20 +8,35 @@ import { mkdir, writeFile } from "node:fs/promises";
|
|
|
8
8
|
import { renderHtmlReport } from "../core/report/render-html-report.js";
|
|
9
9
|
import { buildReportFileName } from "../core/report/report-file-name.js";
|
|
10
10
|
import { normalizeEvents, deriveRunStatus, computeSuccessRate, makeFailureOrError, buildRunFinishedPayload, assembleRunReportModel, } from "../core/report/run-report-model.js";
|
|
11
|
+
import { droidReportScreenshotUrl } from "../integrations/loadmill/report-upload-manager.js";
|
|
11
12
|
// Captures the emitted events into `sink` and forwards them to the underlying output
|
|
12
13
|
// function (so console behaviour is unchanged). Maps the engine's `type` to the canonical
|
|
13
14
|
// `kind`; `stepId` is intentionally not collected (the shared normalizer drops it — see R2).
|
|
14
|
-
export function createCapturingAddOutput(underlying, sink) {
|
|
15
|
+
export function createCapturingAddOutput(underlying, sink, { cloudSink = null, uploadManager = null, uploadSession = null } = {}) {
|
|
15
16
|
return (item) => {
|
|
16
17
|
if (item && typeof item === "object") {
|
|
17
|
-
|
|
18
|
+
const event = {
|
|
18
19
|
ts: typeof item.ts === "string" ? item.ts : null,
|
|
19
20
|
kind: typeof item.type === "string" ? item.type : "info",
|
|
20
21
|
text: typeof item.text === "string" ? item.text : "",
|
|
21
22
|
eventType: item.eventType,
|
|
22
23
|
actionType: item.actionType,
|
|
23
24
|
payload: item.payload,
|
|
24
|
-
}
|
|
25
|
+
};
|
|
26
|
+
sink?.push(event);
|
|
27
|
+
if (cloudSink) {
|
|
28
|
+
const payload = event.payload && typeof event.payload === "object"
|
|
29
|
+
? { ...event.payload }
|
|
30
|
+
: event.payload;
|
|
31
|
+
if (typeof payload?.screenshotDataUrl === "string") {
|
|
32
|
+
const screenshotObjectName = uploadManager?.enqueueScreenshot(uploadSession, payload.screenshotDataUrl);
|
|
33
|
+
delete payload.screenshotDataUrl;
|
|
34
|
+
if (screenshotObjectName) {
|
|
35
|
+
payload.screenshotObjectName = screenshotObjectName;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
cloudSink.push({ ...event, payload });
|
|
39
|
+
}
|
|
25
40
|
}
|
|
26
41
|
return underlying(item);
|
|
27
42
|
};
|
|
@@ -43,7 +58,7 @@ function deriveCounts(success, error, events) {
|
|
|
43
58
|
// Builds a RunReportModel from captured CLI events plus the run's final stats. This is the
|
|
44
59
|
// CLI-acquisition adapter: it knows how to capture live events + run stats, then delegates
|
|
45
60
|
// ALL model assembly to the shared run-report-model.js (same code path as the XML/Desktop
|
|
46
|
-
// parser). A `run_finished` event is synthesized
|
|
61
|
+
// parser). A `run_finished` event is synthesized when the captured output stream omits one.
|
|
47
62
|
export function buildCliRunReportModel({ suiteName, testName, startedAtIso, durationMs, success, error, instructionsTotal, stats = {}, events, }) {
|
|
48
63
|
const warnings = [];
|
|
49
64
|
const normalizedEvents = normalizeEvents(events, { fallbackTs: startedAtIso ?? null, warnings });
|
|
@@ -58,12 +73,15 @@ export function buildCliRunReportModel({ suiteName, testName, startedAtIso, dura
|
|
|
58
73
|
assertionsFailed: stats.assertionsFailed,
|
|
59
74
|
retries: stats.retryCount,
|
|
60
75
|
});
|
|
61
|
-
|
|
62
|
-
//
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
76
|
+
const hasRunFinished = normalizedEvents.some((event) => event.eventType === "run_finished");
|
|
77
|
+
// The synthetic run_finished is already in canonical shape; append it as-is (ts stays
|
|
78
|
+
// null, matching the historical CLI output) only when the producer did not emit one.
|
|
79
|
+
const allEvents = hasRunFinished
|
|
80
|
+
? normalizedEvents
|
|
81
|
+
: [
|
|
82
|
+
...normalizedEvents,
|
|
83
|
+
{ ts: null, kind: "system", text: "Execution finished", eventType: "run_finished", payload: summaryPayload },
|
|
84
|
+
];
|
|
67
85
|
const { failures, errors, stopped } = deriveCounts(success, error, allEvents);
|
|
68
86
|
const status = deriveRunStatus({ tests: 1, failures, errors, stopped });
|
|
69
87
|
const successRate = computeSuccessRate({ tests: 1, failures, errors, status });
|
|
@@ -97,17 +115,89 @@ export function resolveReportOutputPath({ reportPath, testName, startedAtIso, cw
|
|
|
97
115
|
}
|
|
98
116
|
return path.join(resolved, fileName);
|
|
99
117
|
}
|
|
100
|
-
|
|
101
|
-
export async function writeCliRunReport(modelInput, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
|
|
118
|
+
export function createCliRunReport(modelInput) {
|
|
102
119
|
const model = buildCliRunReportModel(modelInput);
|
|
120
|
+
return {
|
|
121
|
+
model,
|
|
122
|
+
html: renderHtmlReport(model),
|
|
123
|
+
fileName: buildReportFileName({
|
|
124
|
+
testName: modelInput.testName,
|
|
125
|
+
suiteName: modelInput.suiteName,
|
|
126
|
+
timestamp: modelInput.startedAtIso,
|
|
127
|
+
}),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
export function createCliAggregateRunReport({ suiteName, testName, startedAtIso, durationMs, childModels, }) {
|
|
131
|
+
const childTests = childModels.map((model) => ({
|
|
132
|
+
testName: model.testName ?? "Test",
|
|
133
|
+
durationSeconds: model.durationSeconds,
|
|
134
|
+
status: model.status,
|
|
135
|
+
successRate: model.successRate,
|
|
136
|
+
failureOrError: model.failureOrError,
|
|
137
|
+
events: model.events,
|
|
138
|
+
warnings: model.warnings,
|
|
139
|
+
}));
|
|
140
|
+
const failures = childTests.filter((child) => child.status === "fail").length;
|
|
141
|
+
const errors = childTests.filter((child) => child.status === "error").length;
|
|
142
|
+
const skipped = childTests.filter((child) => child.status === "skipped").length;
|
|
143
|
+
const stopped = childTests.some((child) => child.status === "stopped");
|
|
144
|
+
const status = deriveRunStatus({
|
|
145
|
+
tests: childTests.length,
|
|
146
|
+
failures,
|
|
147
|
+
errors,
|
|
148
|
+
skipped,
|
|
149
|
+
stopped,
|
|
150
|
+
});
|
|
151
|
+
const model = assembleRunReportModel({
|
|
152
|
+
suiteName,
|
|
153
|
+
testName,
|
|
154
|
+
timestamp: startedAtIso,
|
|
155
|
+
durationSeconds: Number.isFinite(durationMs) ? durationMs / 1000 : null,
|
|
156
|
+
status,
|
|
157
|
+
testCases: childTests.length,
|
|
158
|
+
successRate: computeSuccessRate({
|
|
159
|
+
tests: childTests.length,
|
|
160
|
+
failures,
|
|
161
|
+
errors,
|
|
162
|
+
skipped,
|
|
163
|
+
status,
|
|
164
|
+
}),
|
|
165
|
+
events: [],
|
|
166
|
+
childTests,
|
|
167
|
+
warnings: [],
|
|
168
|
+
});
|
|
169
|
+
return { model, html: renderHtmlReport(model) };
|
|
170
|
+
}
|
|
171
|
+
export function renderCliCloudReport(model, savedRunId) {
|
|
172
|
+
const cloudModel = structuredClone(model);
|
|
173
|
+
const replaceScreenshots = (events = []) => {
|
|
174
|
+
for (const event of events) {
|
|
175
|
+
const name = event.payload?.screenshotObjectName;
|
|
176
|
+
if (typeof name === "string") {
|
|
177
|
+
event.payload.screenshotUrl = droidReportScreenshotUrl(savedRunId, name);
|
|
178
|
+
delete event.payload.screenshotObjectName;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
replaceScreenshots(cloudModel.events);
|
|
183
|
+
for (const child of cloudModel.childTests ?? []) {
|
|
184
|
+
replaceScreenshots(child.events);
|
|
185
|
+
}
|
|
186
|
+
return renderHtmlReport(cloudModel, { includeScripts: false });
|
|
187
|
+
}
|
|
188
|
+
export async function writeCliRunReportArtifact(artifact, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
|
|
103
189
|
const outputPath = resolveReportOutputPath({
|
|
104
190
|
reportPath,
|
|
105
|
-
testName:
|
|
106
|
-
startedAtIso:
|
|
191
|
+
testName: artifact.model.testName,
|
|
192
|
+
startedAtIso: artifact.model.timestamp,
|
|
107
193
|
cwd,
|
|
108
194
|
isBatch,
|
|
109
195
|
});
|
|
110
196
|
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
111
|
-
await writeFile(outputPath,
|
|
197
|
+
await writeFile(outputPath, artifact.html, "utf-8");
|
|
112
198
|
return outputPath;
|
|
113
199
|
}
|
|
200
|
+
// Renders and writes the HTML report for one run. Returns the absolute output path.
|
|
201
|
+
export async function writeCliRunReport(modelInput, { reportPath, cwd = process.cwd(), isBatch = false } = {}) {
|
|
202
|
+
return await writeCliRunReportArtifact(createCliRunReport(modelInput), { reportPath, cwd, isBatch });
|
|
203
|
+
}
|
|
@@ -7,11 +7,12 @@ import { buildExecutionModePrompt } from "../core/prompts.js";
|
|
|
7
7
|
import { addOpenAiResponseToDroidCuaRunUsage, createDroidCuaRunUsageTotals, estimateDroidCuaRunCostUsd, setDroidCuaRunUsageModel, } from "../core/droid-cua-run-usage.js";
|
|
8
8
|
import { ExecutionMode } from "../modes/execution-mode.js";
|
|
9
9
|
import { reportDroidRunCompletion } from "../integrations/loadmill/run-reporter.js";
|
|
10
|
+
import { droidReportUploadManager } from "../integrations/loadmill/report-upload-manager.js";
|
|
10
11
|
import { reportDroidCuaCiTestRunCompleted } from "../integrations/segment/droid-cua-segment.js";
|
|
11
12
|
import { printCliOutput } from "../utils/console-output.js";
|
|
12
13
|
import { emitDesktopDebug } from "../utils/desktop-debug.js";
|
|
13
14
|
import { loadHeadlessInstructions } from "./headless-instructions.js";
|
|
14
|
-
import { createCapturingAddOutput,
|
|
15
|
+
import { buildCliRunReportModel, createCapturingAddOutput, createCliAggregateRunReport, createCliRunReport, renderCliCloudReport, writeCliRunReportArtifact, } from "./headless-report.js";
|
|
15
16
|
function sanitizePathSegment(value) {
|
|
16
17
|
return String(value || "test")
|
|
17
18
|
.replace(/\.dcua$/i, "")
|
|
@@ -24,6 +25,14 @@ function createHeadlessRunId(index = 0) {
|
|
|
24
25
|
const suffix = index > 0 ? `-${index + 1}` : "";
|
|
25
26
|
return `run-${Date.now()}${suffix}`;
|
|
26
27
|
}
|
|
28
|
+
function logPendingScreenshotUploads(uploadManager, uploadSession, subject) {
|
|
29
|
+
console.log('Verifying no pending upload exists');
|
|
30
|
+
const pendingCount = uploadManager.getPendingCount?.(uploadSession) ?? 0;
|
|
31
|
+
if (pendingCount > 0) {
|
|
32
|
+
const uploads = pendingCount === 1 ? "upload" : "uploads";
|
|
33
|
+
console.log(`${subject} finished. Waiting for ${pendingCount} screenshot ${uploads} to finish...`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
27
36
|
function getRecordScreenshotDirForTest(baseScreenshotDir, testName, isBatchRun) {
|
|
28
37
|
if (!baseScreenshotDir || !isBatchRun) {
|
|
29
38
|
return baseScreenshotDir;
|
|
@@ -122,7 +131,7 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
|
|
|
122
131
|
}
|
|
123
132
|
return engineOptions;
|
|
124
133
|
}
|
|
125
|
-
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true }) {
|
|
134
|
+
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
|
|
126
135
|
let executionMode = null;
|
|
127
136
|
let loadmillSessionUser = null;
|
|
128
137
|
let runUsageTotals = setDroidCuaRunUsageModel(createDroidCuaRunUsageTotals(), runtimeContext.config.cuaModel);
|
|
@@ -142,10 +151,17 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
142
151
|
...buildSegmentUsageProperties(),
|
|
143
152
|
});
|
|
144
153
|
};
|
|
145
|
-
const
|
|
146
|
-
const
|
|
154
|
+
const localReportEnabled = Boolean(report?.enabled);
|
|
155
|
+
const cloudReportEnabled = Boolean(cloudUploadSession);
|
|
156
|
+
const reportEnabled = localReportEnabled || cloudReportEnabled;
|
|
157
|
+
const localEvents = [];
|
|
158
|
+
const cloudEvents = [];
|
|
147
159
|
const addOutput = reportEnabled
|
|
148
|
-
? createCapturingAddOutput(printCliOutput,
|
|
160
|
+
? createCapturingAddOutput(printCliOutput, localReportEnabled ? localEvents : null, {
|
|
161
|
+
cloudSink: cloudReportEnabled ? cloudEvents : null,
|
|
162
|
+
uploadManager,
|
|
163
|
+
uploadSession: cloudUploadSession,
|
|
164
|
+
})
|
|
149
165
|
: printCliOutput;
|
|
150
166
|
try {
|
|
151
167
|
try {
|
|
@@ -175,6 +191,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
175
191
|
testDataContext,
|
|
176
192
|
availableSecretKeys: secrets?.availableSecretKeys || [],
|
|
177
193
|
cuaModel: runtimeContext.config.cuaModel,
|
|
194
|
+
explainMode: runtimeContext.config.explainMode,
|
|
178
195
|
});
|
|
179
196
|
session.setSystemPrompt(executionPrompt);
|
|
180
197
|
const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
|
|
@@ -226,8 +243,9 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
226
243
|
durationMs,
|
|
227
244
|
numOfSteps: instructions.length,
|
|
228
245
|
});
|
|
246
|
+
let reportedRun = null;
|
|
229
247
|
if (reportToLoadmill) {
|
|
230
|
-
await reportDroidRunCompletion({
|
|
248
|
+
reportedRun = await reportDroidRunCompletion({
|
|
231
249
|
loadmillClient: runtimeContext.loadmillClient,
|
|
232
250
|
source: "ci",
|
|
233
251
|
runId,
|
|
@@ -255,19 +273,19 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
255
273
|
if (!result.success) {
|
|
256
274
|
console.error(`\nTest failed: ${result.error}`);
|
|
257
275
|
}
|
|
258
|
-
|
|
276
|
+
const reportInput = {
|
|
277
|
+
suiteName: path.basename(path.dirname(testPath)) || null,
|
|
278
|
+
testName,
|
|
279
|
+
startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
|
|
280
|
+
durationMs,
|
|
281
|
+
success: Boolean(result.success),
|
|
282
|
+
error: result.error ?? null,
|
|
283
|
+
instructionsTotal: instructions.length,
|
|
284
|
+
stats,
|
|
285
|
+
};
|
|
286
|
+
if (localReportEnabled) {
|
|
259
287
|
try {
|
|
260
|
-
const outputPath = await
|
|
261
|
-
suiteName: path.basename(path.dirname(testPath)) || null,
|
|
262
|
-
testName,
|
|
263
|
-
startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
|
|
264
|
-
durationMs,
|
|
265
|
-
success: Boolean(result.success),
|
|
266
|
-
error: result.error ?? null,
|
|
267
|
-
instructionsTotal: instructions.length,
|
|
268
|
-
stats,
|
|
269
|
-
events: collectedEvents,
|
|
270
|
-
}, { reportPath: report?.path ?? null, isBatch: isBatchRun });
|
|
288
|
+
const outputPath = await writeCliRunReportArtifact(createCliRunReport({ ...reportInput, events: localEvents }), { reportPath: report?.path ?? null, isBatch: isBatchRun });
|
|
271
289
|
console.log(`HTML report saved: ${outputPath}`);
|
|
272
290
|
}
|
|
273
291
|
catch (reportError) {
|
|
@@ -275,6 +293,16 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
275
293
|
console.warn(`Warning: failed to write HTML report: ${message}`);
|
|
276
294
|
}
|
|
277
295
|
}
|
|
296
|
+
const cloudModel = cloudReportEnabled
|
|
297
|
+
? buildCliRunReportModel({ ...reportInput, events: cloudEvents })
|
|
298
|
+
: null;
|
|
299
|
+
if (cloudModel && reportToLoadmill && reportedRun?.savedRunId) {
|
|
300
|
+
logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Test");
|
|
301
|
+
await uploadManager.finish(cloudUploadSession, {
|
|
302
|
+
savedRunId: reportedRun.savedRunId,
|
|
303
|
+
html: renderCliCloudReport(cloudModel, reportedRun.savedRunId),
|
|
304
|
+
});
|
|
305
|
+
}
|
|
278
306
|
return {
|
|
279
307
|
success: Boolean(result.success),
|
|
280
308
|
error: result.error ?? null,
|
|
@@ -284,7 +312,8 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
284
312
|
actionsTotal: stats.actionCount ?? 0,
|
|
285
313
|
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
286
314
|
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
287
|
-
retries: stats.retryCount ?? 0
|
|
315
|
+
retries: stats.retryCount ?? 0,
|
|
316
|
+
reportModel: cloudModel,
|
|
288
317
|
};
|
|
289
318
|
}
|
|
290
319
|
catch (error) {
|
|
@@ -311,8 +340,9 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
311
340
|
durationMs,
|
|
312
341
|
numOfSteps: instructions.length,
|
|
313
342
|
});
|
|
343
|
+
let reportedRun = null;
|
|
314
344
|
if (reportToLoadmill) {
|
|
315
|
-
await reportDroidRunCompletion({
|
|
345
|
+
reportedRun = await reportDroidRunCompletion({
|
|
316
346
|
loadmillClient: runtimeContext.loadmillClient,
|
|
317
347
|
source: "ci",
|
|
318
348
|
runId,
|
|
@@ -337,11 +367,63 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
337
367
|
onWarning: (warningMessage) => console.warn(`Warning: ${warningMessage}`),
|
|
338
368
|
});
|
|
339
369
|
}
|
|
370
|
+
const failedReportInput = {
|
|
371
|
+
suiteName: path.basename(path.dirname(testPath)) || null,
|
|
372
|
+
testName,
|
|
373
|
+
startedAtIso: stats.startTime ? new Date(stats.startTime).toISOString() : null,
|
|
374
|
+
durationMs,
|
|
375
|
+
success: false,
|
|
376
|
+
error: message,
|
|
377
|
+
instructionsTotal: instructions.length,
|
|
378
|
+
stats,
|
|
379
|
+
};
|
|
380
|
+
if (localReportEnabled) {
|
|
381
|
+
try {
|
|
382
|
+
const outputPath = await writeCliRunReportArtifact(createCliRunReport({ ...failedReportInput, events: localEvents }), { reportPath: report?.path ?? null, isBatch: isBatchRun });
|
|
383
|
+
console.log(`HTML report saved: ${outputPath}`);
|
|
384
|
+
}
|
|
385
|
+
catch (reportError) {
|
|
386
|
+
console.warn(`Warning: failed to write HTML report: ${reportError instanceof Error ? reportError.message : String(reportError)}`);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
const cloudModel = cloudReportEnabled
|
|
390
|
+
? buildCliRunReportModel({ ...failedReportInput, events: cloudEvents })
|
|
391
|
+
: null;
|
|
392
|
+
if (cloudModel && reportToLoadmill && reportedRun?.savedRunId) {
|
|
393
|
+
logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Test");
|
|
394
|
+
await uploadManager.finish(cloudUploadSession, {
|
|
395
|
+
savedRunId: reportedRun.savedRunId,
|
|
396
|
+
html: renderCliCloudReport(cloudModel, reportedRun.savedRunId),
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
if (!reportToLoadmill) {
|
|
400
|
+
console.error(`\nTest failed: ${testName}: ${message}`);
|
|
401
|
+
return {
|
|
402
|
+
success: false,
|
|
403
|
+
error: message,
|
|
404
|
+
durationMs,
|
|
405
|
+
instructionsTotal: instructions.length,
|
|
406
|
+
instructionsCompleted: stats.instructionsCompleted ?? 0,
|
|
407
|
+
actionsTotal: stats.actionCount ?? 0,
|
|
408
|
+
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
409
|
+
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
410
|
+
retries: stats.retryCount ?? 0,
|
|
411
|
+
reportModel: cloudModel,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
340
414
|
throw error;
|
|
341
415
|
}
|
|
342
416
|
}
|
|
343
|
-
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, report = null, runHeadlessTestImpl = runHeadlessTest }) {
|
|
417
|
+
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, report = null, runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
|
|
344
418
|
const isBatchRun = instructionsTarget?.type === "directory" || instructionsTarget?.type === "scenario";
|
|
419
|
+
const cloudReportEnabled = report?.cloudEnabled !== false &&
|
|
420
|
+
Boolean(runtimeContext.loadmillClient?.getApiToken?.());
|
|
421
|
+
const cloudUploadSession = cloudReportEnabled
|
|
422
|
+
? uploadManager.startSession({
|
|
423
|
+
loadmillClient: runtimeContext.loadmillClient,
|
|
424
|
+
onWarning: (message) => console.warn(`Warning: ${message}`),
|
|
425
|
+
})
|
|
426
|
+
: null;
|
|
345
427
|
if (instructionsTarget?.type === "scenario") {
|
|
346
428
|
console.log(`\nRunning ${runs.length} scenario case${runs.length === 1 ? "" : "s"}\n`);
|
|
347
429
|
}
|
|
@@ -359,6 +441,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
359
441
|
assertionsFailed: 0,
|
|
360
442
|
retries: 0
|
|
361
443
|
};
|
|
444
|
+
const aggregateReportModels = [];
|
|
362
445
|
for (const [index, run] of runs.entries()) {
|
|
363
446
|
const runId = createHeadlessRunId(index);
|
|
364
447
|
const testScreenshotDir = getRecordScreenshotDirForTest(screenshotDir, run.testName, isBatchRun);
|
|
@@ -389,7 +472,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
389
472
|
isCloudTarget,
|
|
390
473
|
report,
|
|
391
474
|
isBatchRun,
|
|
392
|
-
reportToLoadmill: !isBatchRun
|
|
475
|
+
reportToLoadmill: !isBatchRun,
|
|
476
|
+
cloudUploadSession,
|
|
477
|
+
uploadManager,
|
|
393
478
|
});
|
|
394
479
|
aggregateStats.durationMs += result.durationMs ?? 0;
|
|
395
480
|
aggregateStats.instructionsTotal += result.instructionsTotal ?? 0;
|
|
@@ -398,6 +483,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
398
483
|
aggregateStats.assertionsPassed += result.assertionsPassed ?? 0;
|
|
399
484
|
aggregateStats.assertionsFailed += result.assertionsFailed ?? 0;
|
|
400
485
|
aggregateStats.retries += result.retries ?? 0;
|
|
486
|
+
if (result.reportModel) {
|
|
487
|
+
aggregateReportModels.push(result.reportModel);
|
|
488
|
+
}
|
|
401
489
|
if (!result.success) {
|
|
402
490
|
failures += 1;
|
|
403
491
|
}
|
|
@@ -422,7 +510,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
422
510
|
if (isBatchRun) {
|
|
423
511
|
const isScenario = instructionsTarget?.type === "scenario";
|
|
424
512
|
const testName = isScenario ? "Scenario run" : path.basename(instructionsTarget.path);
|
|
425
|
-
await reportDroidRunCompletion({
|
|
513
|
+
const reportedRun = await reportDroidRunCompletion({
|
|
426
514
|
loadmillClient: runtimeContext.loadmillClient,
|
|
427
515
|
source: "ci",
|
|
428
516
|
runId: createHeadlessRunId(),
|
|
@@ -447,6 +535,20 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
447
535
|
retries: aggregateStats.retries,
|
|
448
536
|
onWarning: (message) => console.warn(`Warning: ${message}`),
|
|
449
537
|
});
|
|
538
|
+
if (cloudUploadSession && reportedRun.savedRunId) {
|
|
539
|
+
const aggregateReport = createCliAggregateRunReport({
|
|
540
|
+
suiteName: isScenario ? path.basename(process.cwd()) : path.basename(instructionsTarget.path),
|
|
541
|
+
testName,
|
|
542
|
+
startedAtIso: new Date(batchStartedAt).toISOString(),
|
|
543
|
+
durationMs: Math.max(aggregateStats.durationMs, Date.now() - batchStartedAt),
|
|
544
|
+
childModels: aggregateReportModels,
|
|
545
|
+
});
|
|
546
|
+
logPendingScreenshotUploads(uploadManager, cloudUploadSession, "Run");
|
|
547
|
+
await uploadManager.finish(cloudUploadSession, {
|
|
548
|
+
savedRunId: reportedRun.savedRunId,
|
|
549
|
+
html: renderCliCloudReport(aggregateReport.model, reportedRun.savedRunId),
|
|
550
|
+
});
|
|
551
|
+
}
|
|
450
552
|
}
|
|
451
553
|
return {
|
|
452
554
|
exitCode: failures === 0 ? 0 : 1,
|
|
@@ -46,6 +46,9 @@ export async function handleHelp(args, session, context) {
|
|
|
46
46
|
addOutput({ type: 'info', text: ' --base-prompt-file Headless base prompt customization file' });
|
|
47
47
|
addOutput({ type: 'info', text: ' --execution-prompt-file Headless execution prompt customization file' });
|
|
48
48
|
addOutput({ type: 'info', text: ' --record Record screenshots during execution' });
|
|
49
|
+
addOutput({ type: 'info', text: ' --explain Explain non-obvious decisions and completion evidence' });
|
|
50
|
+
addOutput({ type: 'info', text: ' --report [path] Save a self-contained HTML report locally' });
|
|
51
|
+
addOutput({ type: 'info', text: ' --no-cloud-report Skip uploading the HTML report and screenshots to Loadmill' });
|
|
49
52
|
addOutput({ type: 'info', text: ' --debug Enable structured JSONL debug artifacts' });
|
|
50
53
|
addOutput({ type: 'info', text: '' });
|
|
51
54
|
addOutput({ type: 'info', text: 'Interactive commands:' });
|
|
@@ -399,7 +399,7 @@ export class ExecutionEngine {
|
|
|
399
399
|
eventId: actionEventId,
|
|
400
400
|
captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
|
|
401
401
|
});
|
|
402
|
-
// Inline the
|
|
402
|
+
// Inline the accent-colored click marker as a data URL so self-contained reports show it.
|
|
403
403
|
// The Desktop recorder returns one from disk; for runs without a recorder (e.g. a
|
|
404
404
|
// plain CLI `--report`) render the same marker in-memory from the raw screenshot.
|
|
405
405
|
const annotatedDataUrl = await this.resolveAnnotatedDataUrl(annotatedScreenshot, currentScreenshot, action, context);
|
|
@@ -526,7 +526,7 @@ export class ExecutionEngine {
|
|
|
526
526
|
type: "assistant",
|
|
527
527
|
text: textPart.text,
|
|
528
528
|
eventType: "assistant_message",
|
|
529
|
-
...eventMeta(),
|
|
529
|
+
...eventMeta(options.explainMode ? { payload: { explainMode: true } } : {}),
|
|
530
530
|
});
|
|
531
531
|
}
|
|
532
532
|
this.session.addToTranscript(`[Assistant] ${textPart.text}`);
|
|
@@ -689,7 +689,10 @@ export class ExecutionEngine {
|
|
|
689
689
|
const items = response.output || [];
|
|
690
690
|
const actionableItems = extractActionableOutputItems(items);
|
|
691
691
|
const pendingSafetyChecks = collectPendingSafetyChecks(items);
|
|
692
|
-
this.emitResponseOutputItems(items, addOutput, eventMeta,
|
|
692
|
+
this.emitResponseOutputItems(items, addOutput, eventMeta, {
|
|
693
|
+
...options,
|
|
694
|
+
explainMode: options.explainMode === true || context?.runtimeConfig?.explainMode === true,
|
|
695
|
+
});
|
|
693
696
|
if (actionableItems.length === 0) {
|
|
694
697
|
// No actions = turn complete
|
|
695
698
|
break;
|
|
@@ -83,13 +83,60 @@ function buildVisibleEvents(events) {
|
|
|
83
83
|
// full-size lightbox overlay can be rendered. The href links to the CSS `:target`
|
|
84
84
|
// overlay, so click-to-enlarge works even with JavaScript disabled.
|
|
85
85
|
function screenshotImg(event, shots) {
|
|
86
|
-
const url = event.payload?.screenshotDataUrl;
|
|
87
|
-
|
|
86
|
+
const url = event.payload?.screenshotDataUrl ?? event.payload?.screenshotUrl;
|
|
87
|
+
const isAllowedUrl = typeof url === "string" &&
|
|
88
|
+
(url.startsWith("data:") || url.startsWith("/api/droid-cua/runs/"));
|
|
89
|
+
if (!isAllowedUrl)
|
|
88
90
|
return "";
|
|
89
91
|
const index = shots.length + 1;
|
|
90
92
|
shots.push({ index, url });
|
|
91
93
|
return `<a class="dc-shot-link" href="#dc-shot-${index}" aria-label="Open screenshot ${index}"><img class="dc-shot" src="${escapeHtml(url)}" alt="Screenshot ${index}" /></a>`;
|
|
92
94
|
}
|
|
95
|
+
function icon(name, shapes) {
|
|
96
|
+
return `<svg class="dc-icon" data-icon="${name}" aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${shapes}</svg>`;
|
|
97
|
+
}
|
|
98
|
+
const ICONS = {
|
|
99
|
+
arrowUpDown: icon("arrow-up-down", `<path d="m21 16-4 4-4-4"/><path d="M17 20V4"/><path d="m3 8 4-4 4 4"/><path d="M7 4v16"/>`),
|
|
100
|
+
botMessageSquare: icon("bot-message-square", `<path d="M12 6V2H8"/><path d="m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z"/><path d="M2 12h2"/><path d="M9 11v2"/><path d="M15 11v2"/><path d="M20 12h2"/>`),
|
|
101
|
+
camera: icon("camera", `<path d="M14.5 4h-5L7 7H4a2 2 0 0 0-2 2v9a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-3l-2.5-3z"/><circle cx="12" cy="13" r="3"/>`),
|
|
102
|
+
compass: icon("compass", `<path d="m16.24 7.76-1.804 5.411a2 2 0 0 1-1.265 1.265L7.76 16.24l1.804-5.411a2 2 0 0 1 1.265-1.265z"/><circle cx="12" cy="12" r="10"/>`),
|
|
103
|
+
flag: icon("flag", `<path d="M4 15s1-1 4-1 5 2 8 2 4-1 4-1V3s-1 1-4 1-5-2-8-2-4 1-4 1z"/><line x1="4" x2="4" y1="22" y2="15"/>`),
|
|
104
|
+
keyboard: icon("keyboard", `<path d="M10 8h.01"/><path d="M12 12h.01"/><path d="M14 8h.01"/><path d="M16 12h.01"/><path d="M18 8h.01"/><path d="M6 8h.01"/><path d="M7 16h10"/><path d="M8 12h.01"/><rect width="20" height="16" x="2" y="4" rx="2"/>`),
|
|
105
|
+
keyboardOff: icon("keyboard-off", `<path d="M20 4a2 2 0 0 1 2 2"/><path d="M22 6v10.41"/><path d="M7 16h9"/><path d="M9.69 4H20"/><path d="M14 8h.01"/><path d="M18 8h.01"/><path d="m2 2 20 20"/><path d="M20 20H4a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2"/><path d="M6 8h.01"/><path d="M8 12h.01"/>`),
|
|
106
|
+
keyRound: icon("key-round", `<path d="M2.586 17.414A2 2 0 0 0 2 18.828V21a1 1 0 0 0 1 1h3a1 1 0 0 0 1-1v-1a1 1 0 0 1 1-1h1a1 1 0 0 0 1-1v-1a1 1 0 0 1 1-1h.172a2 2 0 0 0 1.414-.586l.814-.814a6.5 6.5 0 1 0-4-4z"/><circle cx="16.5" cy="7.5" r=".5" fill="currentColor"/>`),
|
|
107
|
+
mousePointer: icon("mouse-pointer-2", `<path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/>`),
|
|
108
|
+
moveDiagonal: icon("move-diagonal-2", `<path d="M19 13v6h-6"/><path d="M5 11V5h6"/><path d="m5 5 14 14"/>`),
|
|
109
|
+
rotateCcw: icon("rotate-ccw", `<path d="M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8"/><path d="M3 3v5h5"/>`),
|
|
110
|
+
squareX: icon("square-x", `<rect width="18" height="18" x="3" y="3" rx="2"/><path d="m15 9-6 6"/><path d="m9 9 6 6"/>`),
|
|
111
|
+
timer: icon("timer", `<line x1="10" x2="14" y1="2" y2="2"/><line x1="12" x2="15" y1="14" y2="11"/><circle cx="12" cy="14" r="8"/>`),
|
|
112
|
+
};
|
|
113
|
+
function actionIcon(event) {
|
|
114
|
+
const lowerText = event.text.toLowerCase();
|
|
115
|
+
const toolName = typeof event.payload?.toolName === "string" ? event.payload.toolName : null;
|
|
116
|
+
const isSensitiveTyping = Boolean(event.payload?.sensitive) || lowerText.includes("typing secret");
|
|
117
|
+
if (toolName === "navigate")
|
|
118
|
+
return ICONS.compass;
|
|
119
|
+
if (toolName === "close_current_tab")
|
|
120
|
+
return ICONS.squareX;
|
|
121
|
+
if (toolName === "close_keyboard")
|
|
122
|
+
return ICONS.keyboardOff;
|
|
123
|
+
if (isSensitiveTyping)
|
|
124
|
+
return ICONS.keyRound;
|
|
125
|
+
if (event.actionType === "type" || event.actionType === "keypress" || lowerText.includes("typing"))
|
|
126
|
+
return ICONS.keyboard;
|
|
127
|
+
if (event.actionType === "scroll" || lowerText.includes("scroll"))
|
|
128
|
+
return ICONS.arrowUpDown;
|
|
129
|
+
if (event.actionType === "drag" || lowerText.includes("drag"))
|
|
130
|
+
return ICONS.moveDiagonal;
|
|
131
|
+
if (event.actionType === "screenshot" || lowerText.includes("capturing screen"))
|
|
132
|
+
return ICONS.camera;
|
|
133
|
+
if (event.actionType === "wait" || lowerText.includes("waiting"))
|
|
134
|
+
return ICONS.timer;
|
|
135
|
+
return ICONS.mousePointer;
|
|
136
|
+
}
|
|
137
|
+
function renderAgentText(text) {
|
|
138
|
+
return `<div class="dc-line dc-agent-line">${ICONS.botMessageSquare}<span>${escapeHtml(text)}</span></div>`;
|
|
139
|
+
}
|
|
93
140
|
// ---------- per-event timeline rendering (mirrors ExecutionLogItem.tsx) ----------
|
|
94
141
|
function renderEvent(event, shots) {
|
|
95
142
|
if (isHiddenExecutionLogEvent(event))
|
|
@@ -123,10 +170,16 @@ function renderEvent(event, shots) {
|
|
|
123
170
|
if (isAssistant) {
|
|
124
171
|
const stepCompletion = stepCompletionData(event);
|
|
125
172
|
if (stepCompletion) {
|
|
173
|
+
if (event.payload?.explainMode === true) {
|
|
174
|
+
return `<div class="dc-line">${ICONS.flag}<span>Step complete</span></div>\n${renderAgentText(stepCompletion.reason)}`;
|
|
175
|
+
}
|
|
126
176
|
const tooltip = stepCompletionReasonTooltip(stepCompletion);
|
|
127
|
-
return `<div class="dc-line"${tooltip ? ` title="${escapeHtml(tooltip)}"` : ""}
|
|
177
|
+
return `<div class="dc-line"${tooltip ? ` title="${escapeHtml(tooltip)}"` : ""}>${ICONS.flag}<span>Step complete</span></div>`;
|
|
128
178
|
}
|
|
129
|
-
return
|
|
179
|
+
return renderAgentText(event.text);
|
|
180
|
+
}
|
|
181
|
+
if (event.eventType === "reasoning" || event.kind === "reasoning") {
|
|
182
|
+
return renderAgentText(event.text);
|
|
130
183
|
}
|
|
131
184
|
const assertion = assertionData(event);
|
|
132
185
|
if (assertion) {
|
|
@@ -138,13 +191,13 @@ function renderEvent(event, shots) {
|
|
|
138
191
|
</div>`;
|
|
139
192
|
}
|
|
140
193
|
if (event.eventType === "retry") {
|
|
141
|
-
return `<div class="dc-line dc-retry">${escapeHtml(event.text)}</div>`;
|
|
194
|
+
return `<div class="dc-line dc-retry">${ICONS.rotateCcw}<span>${escapeHtml(event.text)}</span></div>`;
|
|
142
195
|
}
|
|
143
196
|
if (event.kind === "action") {
|
|
144
|
-
return `<div class="dc-line">${escapeHtml(event.text)}
|
|
197
|
+
return `<div class="dc-line">${actionIcon(event)}<span>${escapeHtml(event.text)}</span>${screenshotImg(event, shots)}</div>`;
|
|
145
198
|
}
|
|
146
199
|
if (event.eventType === "screenshot_captured") {
|
|
147
|
-
return `<div class="dc-line">${escapeHtml(event.text)}
|
|
200
|
+
return `<div class="dc-line">${ICONS.camera}<span>${escapeHtml(event.text)}</span>${screenshotImg(event, shots)}</div>`;
|
|
148
201
|
}
|
|
149
202
|
return `<div class="dc-line dc-${escapeHtml(event.kind)}">${escapeHtml(event.text)}</div>`;
|
|
150
203
|
}
|
|
@@ -243,13 +296,16 @@ body { margin: 0; background: #f1f5f9; color: #0f172a; font-family: -apple-syste
|
|
|
243
296
|
.dc-row { display: flex; }
|
|
244
297
|
.dc-row-right { justify-content: flex-end; }
|
|
245
298
|
.dc-bubble { display: inline-block; max-width: 70%; background: #f1f5f9; border: 1px solid #e2e8f0; border-radius: 16px; border-bottom-right-radius: 6px; padding: 8px 14px; color: #0f172a; }
|
|
246
|
-
.dc-assistant { max-width: 82%; color: #334155; }
|
|
247
299
|
.dc-line { display: flex; align-items: center; gap: 8px; color: #475569; }
|
|
300
|
+
.dc-agent-line { align-items: flex-start; }
|
|
301
|
+
.dc-agent-line .dc-icon { margin-top: 4px; }
|
|
302
|
+
.dc-icon { width: 14px; height: 14px; flex: 0 0 auto; color: #94a3b8; }
|
|
248
303
|
.dc-retry { color: #b45309; }
|
|
249
304
|
.dc-error { color: #b91c1c; }
|
|
250
305
|
.dc-warning { color: #b45309; }
|
|
251
|
-
.dc-shot-link { display: inline-flex; margin-left: 8px; vertical-align: middle; }
|
|
252
|
-
.dc-shot {
|
|
306
|
+
.dc-shot-link { display: inline-flex; align-items: center; justify-content: center; position: relative; min-width: 34px; min-height: 36px; margin-left: 8px; vertical-align: middle; }
|
|
307
|
+
.dc-shot-link::before, .dc-lightbox:target::before { content: ""; position: absolute; width: 14px; height: 14px; border: 2px solid #cbd5e1; border-top-color: #475569; border-radius: 50%; animation: dc-spin .7s linear infinite; }
|
|
308
|
+
.dc-shot { position: relative; z-index: 1; height: 34px; width: auto; border: 1px solid #e2e8f0; border-radius: 4px; cursor: zoom-in; transition: border-color .12s; }
|
|
253
309
|
.dc-shot-link:hover .dc-shot, .dc-shot-link:focus-visible .dc-shot { border-color: #94a3b8; }
|
|
254
310
|
.dc-lightbox { display: none; position: fixed; inset: 0; z-index: 1000; align-items: center; justify-content: center; padding: 32px; background: rgba(15, 23, 42, .82); }
|
|
255
311
|
.dc-lightbox:target { display: flex; }
|
|
@@ -275,9 +331,10 @@ body { margin: 0; background: #f1f5f9; color: #0f172a; font-family: -apple-syste
|
|
|
275
331
|
.dc-pill-error, .dc-pill-errored, .dc-pill-amber { border-color: #fcd34d; background: #fef3c7; color: #b45309; }
|
|
276
332
|
.dc-pill-stopped { border-color: #fdba74; background: #ffedd5; color: #c2410c; }
|
|
277
333
|
.dc-pill-skipped { border-color: #cbd5e1; background: #f1f5f9; color: #334155; }
|
|
334
|
+
@keyframes dc-spin { to { transform: rotate(360deg); } }
|
|
278
335
|
`.trim();
|
|
279
336
|
// ---------- entry point ----------
|
|
280
|
-
export function renderHtmlReport(model, { title } = {}) {
|
|
337
|
+
export function renderHtmlReport(model, { title, includeScripts = true } = {}) {
|
|
281
338
|
const docTitle = title ?? `Run Report: ${model.testName ?? model.suiteName ?? "Test run"}`;
|
|
282
339
|
const heading = model.testName ?? model.suiteName ?? "Test run";
|
|
283
340
|
// Screenshots are collected across the whole document (top-level + child timelines)
|
|
@@ -301,7 +358,7 @@ ${renderTimeline(model.events, shots)}
|
|
|
301
358
|
</div>`;
|
|
302
359
|
}
|
|
303
360
|
const lightboxes = renderLightboxes(shots);
|
|
304
|
-
const script = renderLightboxScript(shots.length);
|
|
361
|
+
const script = includeScripts ? renderLightboxScript(shots.length) : "";
|
|
305
362
|
return `<!DOCTYPE html>
|
|
306
363
|
<html lang="en">
|
|
307
364
|
<head>
|
|
@@ -123,7 +123,7 @@ export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToke
|
|
|
123
123
|
loadmillBaseUrl: normalizeLoadmillSiteBaseUrl(loadmillBaseUrl),
|
|
124
124
|
};
|
|
125
125
|
}
|
|
126
|
-
export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
|
|
126
|
+
export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null, explainMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
|
|
127
127
|
const normalizedCuaModel = typeof cuaModel === "string" && LOADMILL_MODEL_ALIAS_SET.has(cuaModel)
|
|
128
128
|
? cuaModel
|
|
129
129
|
: resolveCuaModel(cuaModel);
|
|
@@ -133,6 +133,7 @@ export function resolveRuntimeConfig({ llmProviderMode = null, cuaModel = null,
|
|
|
133
133
|
? "loadmill"
|
|
134
134
|
: normalizedProviderMode,
|
|
135
135
|
cuaModel: normalizedCuaModel,
|
|
136
|
+
explainMode: explainMode === true,
|
|
136
137
|
stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
|
|
137
138
|
contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
|
|
138
139
|
contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),
|
|
@@ -1,4 +1,29 @@
|
|
|
1
1
|
import { buildTextToolOutputItem, parseStringToolArgument } from "./utils.js";
|
|
2
|
+
import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
3
|
+
function emitLoadmillToolCall({ callId, argumentsText, command, context, stepContext }) {
|
|
4
|
+
const scope = context?.sessionId ? "design" : "execution";
|
|
5
|
+
const ids = {
|
|
6
|
+
...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
|
|
7
|
+
...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
|
|
8
|
+
...(Number.isInteger(stepContext?.instructionIndex) ? { instructionIndex: stepContext.instructionIndex } : {}),
|
|
9
|
+
};
|
|
10
|
+
const payload = {
|
|
11
|
+
toolName: "loadmill",
|
|
12
|
+
callId,
|
|
13
|
+
command,
|
|
14
|
+
arguments: argumentsText,
|
|
15
|
+
};
|
|
16
|
+
emitDesktopDebug("tool_call", scope, ids, payload);
|
|
17
|
+
context?.addOutput?.({
|
|
18
|
+
type: "muted",
|
|
19
|
+
text: `Running Loadmill flow: ${command}`,
|
|
20
|
+
eventType: "tool_call",
|
|
21
|
+
runId: context?.runId,
|
|
22
|
+
stepId: stepContext?.stepId,
|
|
23
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
24
|
+
payload,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
2
27
|
export async function handleLoadmillToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
|
|
3
28
|
const session = engine.session;
|
|
4
29
|
const parsed = parseStringToolArgument(argumentsText, "command", "loadmill");
|
|
@@ -9,6 +34,13 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
|
|
|
9
34
|
};
|
|
10
35
|
}
|
|
11
36
|
const loadmillCommand = parsed.value;
|
|
37
|
+
emitLoadmillToolCall({
|
|
38
|
+
callId,
|
|
39
|
+
argumentsText,
|
|
40
|
+
command: loadmillCommand,
|
|
41
|
+
context,
|
|
42
|
+
stepContext,
|
|
43
|
+
});
|
|
12
44
|
session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
|
|
13
45
|
const result = await engine.executeLoadmillInstruction(loadmillCommand, engine.isHeadlessMode, {
|
|
14
46
|
...context,
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Assertion handling for script validation
|
|
3
3
|
*/
|
|
4
4
|
import { printCliOutput } from "../utils/console-output.js";
|
|
5
|
+
import { CAPTURE_SOURCES } from "../core/capture-sources.js";
|
|
5
6
|
export { buildAssertionSystemPrompt } from "../prompts/execution.js";
|
|
6
7
|
const ASSERTION_PREFIXES = ["assert", "verify", "validate", "ensure"];
|
|
7
8
|
function hasAssertionPrefix(lower) {
|
|
@@ -54,6 +55,7 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
|
|
|
54
55
|
assertion: assertionPrompt,
|
|
55
56
|
passed: false,
|
|
56
57
|
details,
|
|
58
|
+
captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
|
|
57
59
|
...eventPayload
|
|
58
60
|
}
|
|
59
61
|
};
|
|
@@ -77,6 +79,7 @@ export function handleAssertionSuccess(assertionPrompt, context = null, stepCont
|
|
|
77
79
|
payload: {
|
|
78
80
|
assertion: assertionPrompt,
|
|
79
81
|
passed: true,
|
|
82
|
+
captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
|
|
80
83
|
...eventPayload
|
|
81
84
|
}
|
|
82
85
|
});
|
|
@@ -138,6 +138,18 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
138
138
|
body: JSON.stringify(payload),
|
|
139
139
|
});
|
|
140
140
|
}
|
|
141
|
+
async function requestDroidRunReportUploadPolicy(uploadId, payload) {
|
|
142
|
+
return await apiRequest(`/droid-cua/report-uploads/${uploadId}/policy`, {
|
|
143
|
+
method: "POST",
|
|
144
|
+
body: JSON.stringify(payload),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
async function associateDroidRunReport(runId, payload) {
|
|
148
|
+
return await apiRequest(`/droid-cua/runs/${runId}/report`, {
|
|
149
|
+
method: "POST",
|
|
150
|
+
body: JSON.stringify(payload),
|
|
151
|
+
});
|
|
152
|
+
}
|
|
141
153
|
async function validateApiToken() {
|
|
142
154
|
const resolvedToken = getApiToken();
|
|
143
155
|
if (!resolvedToken) {
|
|
@@ -187,6 +199,8 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
187
199
|
getTestSuiteFlowRun,
|
|
188
200
|
explainFailure,
|
|
189
201
|
reportDroidRun,
|
|
202
|
+
requestDroidRunReportUploadPolicy,
|
|
203
|
+
associateDroidRunReport,
|
|
190
204
|
validateApiToken,
|
|
191
205
|
getCurrentUser,
|
|
192
206
|
};
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
export const MAX_SCREENSHOTS_PER_REPORT = 1000;
|
|
3
|
+
export const MAX_REPORT_OBJECT_BYTES = 10 * 1024 * 1024;
|
|
4
|
+
function warn(session, message) {
|
|
5
|
+
session.failed = true;
|
|
6
|
+
session.onWarning?.(message);
|
|
7
|
+
}
|
|
8
|
+
function getHttpStatus(error) {
|
|
9
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
10
|
+
return /\((\d{3})\)/.exec(message)?.[1] ?? null;
|
|
11
|
+
}
|
|
12
|
+
function warnAboutUploadFailure(session) {
|
|
13
|
+
if (!session.uploadFailed) {
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
const status = session.uploadFailureStatus ? ` (HTTP ${session.uploadFailureStatus})` : "";
|
|
17
|
+
session.onWarning?.(`Cloud report upload failed${status}; further uploads skipped.`);
|
|
18
|
+
}
|
|
19
|
+
function decodeScreenshot(dataUrl) {
|
|
20
|
+
const match = /^data:image\/png;base64,([a-zA-Z0-9+/=\s]+)$/.exec(dataUrl);
|
|
21
|
+
return match ? Buffer.from(match[1], "base64") : null;
|
|
22
|
+
}
|
|
23
|
+
async function uploadObject(uploadPolicy, body, name, contentType, fetchImpl) {
|
|
24
|
+
const form = new FormData();
|
|
25
|
+
for (const [key, value] of Object.entries(uploadPolicy.data ?? {})) {
|
|
26
|
+
form.append(key, String(value));
|
|
27
|
+
}
|
|
28
|
+
form.append("file", new Blob([body], { type: contentType }), name);
|
|
29
|
+
const response = await fetchImpl(uploadPolicy.url, { method: "POST", body: form });
|
|
30
|
+
if (!response.ok) {
|
|
31
|
+
throw new Error(`S3 upload failed (${response.status}): ${await response.text()}`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
export class DroidReportUploadManager {
|
|
35
|
+
constructor({ workers = 4, fetchImpl = fetch } = {}) {
|
|
36
|
+
this.workers = workers;
|
|
37
|
+
this.fetchImpl = fetchImpl;
|
|
38
|
+
this.queue = [];
|
|
39
|
+
this.activeWorkers = 0;
|
|
40
|
+
this.sessions = new Set();
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* @param {{
|
|
44
|
+
* loadmillClient?: {
|
|
45
|
+
* requestDroidRunReportUploadPolicy: (uploadId: string, payload: { name: string, size: number }) => Promise<unknown>,
|
|
46
|
+
* associateDroidRunReport: (runId: string, payload: object) => Promise<unknown>,
|
|
47
|
+
* },
|
|
48
|
+
* onWarning?: ((message: string) => void) | null,
|
|
49
|
+
* }} options
|
|
50
|
+
*/
|
|
51
|
+
startSession({ loadmillClient, onWarning = null } = {}) {
|
|
52
|
+
const session = {
|
|
53
|
+
uploadId: randomUUID(),
|
|
54
|
+
loadmillClient,
|
|
55
|
+
onWarning,
|
|
56
|
+
sequence: 0,
|
|
57
|
+
screenshotCount: 0,
|
|
58
|
+
deduplication: new Map(),
|
|
59
|
+
pending: new Set(),
|
|
60
|
+
failed: false,
|
|
61
|
+
uploadFailed: false,
|
|
62
|
+
uploadFailureStatus: null,
|
|
63
|
+
finishing: false,
|
|
64
|
+
finishPromise: null,
|
|
65
|
+
};
|
|
66
|
+
this.sessions.add(session);
|
|
67
|
+
return session;
|
|
68
|
+
}
|
|
69
|
+
enqueueScreenshot(session, dataUrl) {
|
|
70
|
+
if (!session || session.finishing || session.uploadFailed || typeof dataUrl !== "string") {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const hash = createHash("sha256").update(dataUrl).digest("hex");
|
|
74
|
+
const existing = session.deduplication.get(hash);
|
|
75
|
+
if (existing) {
|
|
76
|
+
return existing;
|
|
77
|
+
}
|
|
78
|
+
const body = decodeScreenshot(dataUrl);
|
|
79
|
+
if (!body) {
|
|
80
|
+
warn(session, "Failed to upload Droid report screenshot: unsupported image data.");
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
if (body.length > MAX_REPORT_OBJECT_BYTES) {
|
|
84
|
+
warn(session, `Droid report screenshot exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
if (session.screenshotCount >= MAX_SCREENSHOTS_PER_REPORT) {
|
|
88
|
+
warn(session, `Droid report exceeds ${MAX_SCREENSHOTS_PER_REPORT} screenshots.`);
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
const name = `screenshots/${String(++session.sequence).padStart(4, "0")}.png`;
|
|
92
|
+
session.screenshotCount += 1;
|
|
93
|
+
session.deduplication.set(hash, name);
|
|
94
|
+
this.#enqueue(session, { name, body, contentType: "image/png" });
|
|
95
|
+
return name;
|
|
96
|
+
}
|
|
97
|
+
finish(session, { savedRunId, html } = {}) {
|
|
98
|
+
if (!session) {
|
|
99
|
+
return Promise.resolve({ published: false, reason: "missing_session" });
|
|
100
|
+
}
|
|
101
|
+
if (session.finishPromise) {
|
|
102
|
+
return session.finishPromise;
|
|
103
|
+
}
|
|
104
|
+
session.finishing = true;
|
|
105
|
+
session.finishPromise = this.#finish(session, { savedRunId, html }).catch((error) => {
|
|
106
|
+
const status = getHttpStatus(error);
|
|
107
|
+
warn(session, `Failed to publish Droid run report${status ? ` (HTTP ${status})` : ""}.`);
|
|
108
|
+
return { published: false, reason: "publish_failed" };
|
|
109
|
+
}).finally(() => {
|
|
110
|
+
this.sessions.delete(session);
|
|
111
|
+
});
|
|
112
|
+
return session.finishPromise;
|
|
113
|
+
}
|
|
114
|
+
async #finish(session, { savedRunId, html }) {
|
|
115
|
+
await this.waitForSession(session);
|
|
116
|
+
if (session.uploadFailed) {
|
|
117
|
+
warnAboutUploadFailure(session);
|
|
118
|
+
return { published: false, reason: "upload_failed" };
|
|
119
|
+
}
|
|
120
|
+
if (!savedRunId || typeof html !== "string") {
|
|
121
|
+
return { published: false, reason: "missing_report" };
|
|
122
|
+
}
|
|
123
|
+
const body = Buffer.from(html, "utf8");
|
|
124
|
+
if (body.length > MAX_REPORT_OBJECT_BYTES) {
|
|
125
|
+
warn(session, `Droid HTML report exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
|
|
126
|
+
return { published: false, reason: "too_large" };
|
|
127
|
+
}
|
|
128
|
+
const htmlUpload = await this.#enqueue(session, {
|
|
129
|
+
name: "report.html",
|
|
130
|
+
body,
|
|
131
|
+
contentType: "text/html; charset=utf-8",
|
|
132
|
+
});
|
|
133
|
+
if (session.failed) {
|
|
134
|
+
warnAboutUploadFailure(session);
|
|
135
|
+
return { published: false, reason: "upload_failed" };
|
|
136
|
+
}
|
|
137
|
+
const key = htmlUpload.key;
|
|
138
|
+
await session.loadmillClient.associateDroidRunReport(savedRunId, {
|
|
139
|
+
key,
|
|
140
|
+
screenshotCount: session.screenshotCount,
|
|
141
|
+
});
|
|
142
|
+
return { published: true, key };
|
|
143
|
+
}
|
|
144
|
+
async waitForSession(session) {
|
|
145
|
+
while (session?.pending?.size) {
|
|
146
|
+
await Promise.all([...session.pending]);
|
|
147
|
+
}
|
|
148
|
+
return { failed: Boolean(session?.failed) };
|
|
149
|
+
}
|
|
150
|
+
getPendingCount(session) {
|
|
151
|
+
return session?.pending?.size ?? 0;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* @param {{ timeoutMs?: number | null }} options
|
|
155
|
+
*/
|
|
156
|
+
async drain({ timeoutMs = null } = {}) {
|
|
157
|
+
const pending = Promise.all([...this.sessions].map((session) => session.finishPromise ?? this.waitForSession(session)));
|
|
158
|
+
if (!Number.isFinite(timeoutMs)) {
|
|
159
|
+
await pending;
|
|
160
|
+
return { timedOut: false };
|
|
161
|
+
}
|
|
162
|
+
let timer;
|
|
163
|
+
const result = await Promise.race([
|
|
164
|
+
pending.then(() => false),
|
|
165
|
+
new Promise((resolve) => {
|
|
166
|
+
timer = setTimeout(() => resolve(true), timeoutMs);
|
|
167
|
+
}),
|
|
168
|
+
]);
|
|
169
|
+
clearTimeout(timer);
|
|
170
|
+
return { timedOut: result };
|
|
171
|
+
}
|
|
172
|
+
#enqueue(session, object) {
|
|
173
|
+
let resolveTask;
|
|
174
|
+
const promise = new Promise((resolve) => {
|
|
175
|
+
resolveTask = resolve;
|
|
176
|
+
});
|
|
177
|
+
session.pending.add(promise);
|
|
178
|
+
this.queue.push({ session, object, resolveTask, promise });
|
|
179
|
+
this.#pump();
|
|
180
|
+
return promise;
|
|
181
|
+
}
|
|
182
|
+
#pump() {
|
|
183
|
+
while (this.activeWorkers < this.workers && this.queue.length > 0) {
|
|
184
|
+
const task = this.queue.shift();
|
|
185
|
+
this.activeWorkers += 1;
|
|
186
|
+
this.#runTask(task).finally(() => {
|
|
187
|
+
this.activeWorkers -= 1;
|
|
188
|
+
this.#pump();
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async #runTask(task) {
|
|
193
|
+
let result = { uploaded: false, key: null };
|
|
194
|
+
try {
|
|
195
|
+
if (task.session.uploadFailed) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const policyResult = await task.session.loadmillClient.requestDroidRunReportUploadPolicy(task.session.uploadId, { name: task.object.name, size: task.object.body.length });
|
|
199
|
+
const uploadPolicy = policyResult?.uploadPolicy;
|
|
200
|
+
if (!uploadPolicy?.key) {
|
|
201
|
+
throw new Error("Loadmill did not return an upload policy.");
|
|
202
|
+
}
|
|
203
|
+
await uploadObject(uploadPolicy, task.object.body, task.object.name.split("/").at(-1), task.object.contentType, this.fetchImpl);
|
|
204
|
+
result = { uploaded: true, key: uploadPolicy.key };
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
task.session.failed = true;
|
|
208
|
+
task.session.uploadFailed = true;
|
|
209
|
+
task.session.uploadFailureStatus ??= getHttpStatus(error);
|
|
210
|
+
}
|
|
211
|
+
finally {
|
|
212
|
+
task.session.pending.delete(task.promise);
|
|
213
|
+
task.resolveTask(result);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
export const droidReportUploadManager = new DroidReportUploadManager();
|
|
218
|
+
export function droidReportScreenshotUrl(savedRunId, objectName) {
|
|
219
|
+
const fileName = objectName.split("/").at(-1);
|
|
220
|
+
return `/api/droid-cua/runs/${encodeURIComponent(savedRunId)}/report/screenshots/${encodeURIComponent(fileName)}`;
|
|
221
|
+
}
|
|
@@ -100,8 +100,12 @@ export async function reportDroidRunCompletion({ loadmillClient, onWarning = nul
|
|
|
100
100
|
return { reported: false, reason: "missing_run_id" };
|
|
101
101
|
}
|
|
102
102
|
try {
|
|
103
|
-
await loadmillClient.reportDroidRun(payload);
|
|
104
|
-
return {
|
|
103
|
+
const response = await loadmillClient.reportDroidRun(payload);
|
|
104
|
+
return {
|
|
105
|
+
reported: true,
|
|
106
|
+
savedRunId: response?.run?.id ?? null,
|
|
107
|
+
run: response?.run ?? null,
|
|
108
|
+
};
|
|
105
109
|
}
|
|
106
110
|
catch (error) {
|
|
107
111
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -592,7 +592,8 @@ export class ExecutionMode {
|
|
|
592
592
|
// clearMessages() restores the base system prompt, but we need to add context
|
|
593
593
|
const recoverySystemPrompt = buildExecutionRecoveryPrompt({
|
|
594
594
|
basePrompt: this.initialSystemText,
|
|
595
|
-
transcript: transcriptContext
|
|
595
|
+
transcript: transcriptContext,
|
|
596
|
+
explainMode: context?.runtimeConfig?.explainMode === true,
|
|
596
597
|
});
|
|
597
598
|
// Replace the system message with the enhanced one
|
|
598
599
|
this.session.installRecoverySystemPrompt(recoverySystemPrompt);
|
|
@@ -2,6 +2,7 @@ import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./
|
|
|
2
2
|
import { buildAppContextSection, buildSecretKeysContextSection } from "./execution.js";
|
|
3
3
|
export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
|
|
4
4
|
const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
|
|
5
|
+
const explainMode = options.explainMode === true;
|
|
5
6
|
const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
|
|
6
7
|
const appContextSection = buildAppContextSection(appContextText);
|
|
7
8
|
const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
|
|
@@ -94,7 +95,14 @@ WRONG Example (DON'T DO THIS):
|
|
|
94
95
|
3. etc...
|
|
95
96
|
\`\`\`
|
|
96
97
|
|
|
97
|
-
|
|
98
|
+
${explainMode ? `EXPLAIN MODE:
|
|
99
|
+
- Briefly explain non-obvious decisions, including why you choose one path, skip an expected action, or consider the goal complete
|
|
100
|
+
- Name only the visible evidence that matters to the decision
|
|
101
|
+
- Do not narrate routine clicks, typing, scrolling, or other obvious actions
|
|
102
|
+
- Before a non-obvious action, briefly explain the decision in the same response as that action
|
|
103
|
+
- Include the evidence that proves the requested flow is complete in the same response as the \`test_design_complete\` call
|
|
104
|
+
|
|
105
|
+
` : ""}Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
|
|
98
106
|
`;
|
|
99
107
|
return appendCustomSections(prompt, [
|
|
100
108
|
{ title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
|
|
@@ -38,6 +38,7 @@ ${normalized.map((key) => `- ${key}`).join("\n")}`;
|
|
|
38
38
|
}
|
|
39
39
|
export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
|
|
40
40
|
const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
|
|
41
|
+
const explainMode = options.explainMode === true;
|
|
41
42
|
const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
|
|
42
43
|
const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
|
|
43
44
|
const appContextSection = buildAppContextSection(appContextText);
|
|
@@ -54,12 +55,12 @@ EXECUTION MODE - Critical Behavior:
|
|
|
54
55
|
You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
|
|
55
56
|
|
|
56
57
|
CRITICAL RULES:
|
|
57
|
-
- DO NOT generate conversational text or narration
|
|
58
|
+
- ${explainMode ? "Explain non-obvious decisions briefly, including why you skip an expected action or consider the current instruction complete" : "DO NOT generate conversational text or narration"}
|
|
58
59
|
- DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
|
|
59
|
-
- DO NOT describe what you see on screen
|
|
60
|
+
- ${explainMode ? "When a decision depends on visible state, name only the evidence that matters to that decision" : "DO NOT describe what you see on screen"}
|
|
60
61
|
- DO NOT say "Let me know if you need help" or similar phrases
|
|
61
|
-
- Just execute the action silently and stop immediately
|
|
62
|
-
- Only generate text if the action FAILED or cannot be completed
|
|
62
|
+
- ${explainMode ? "Do not narrate routine clicks, typing, scrolling, or other obvious actions" : "Just execute the action silently and stop immediately"}
|
|
63
|
+
- ${explainMode ? "Before a non-obvious action, briefly explain the decision in the same response as that action\n- Explain completion evidence using assistant text; do not call `assert` merely to confirm that an ordinary action succeeded\n- Call `assert` only when the current instruction explicitly contains assertion or verification intent\n- If an action FAILED or the instruction cannot be completed, state that clearly and explain why" : "Only generate text if the action FAILED or cannot be completed"}
|
|
63
64
|
- ${browserTarget ? "Use browser-appropriate keyboard input only when it is the most reliable path" : "Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses"}
|
|
64
65
|
- Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
|
|
65
66
|
- If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
|
|
@@ -86,7 +87,7 @@ ${browserTarget ? "" : ` - Every \`type\` action must end its computer-action
|
|
|
86
87
|
4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
|
|
87
88
|
5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
|
|
88
89
|
6. Stop as soon as success is visible
|
|
89
|
-
7. Stop immediately - no commentary, no questions
|
|
90
|
+
7. ${explainMode ? "If the completion decision is not obvious, briefly state the visible evidence, then stop - no questions" : "Stop immediately - no commentary, no questions"}
|
|
90
91
|
|
|
91
92
|
${instructionContextRule}
|
|
92
93
|
${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
|
|
@@ -95,7 +96,7 @@ ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ?
|
|
|
95
96
|
{ title: "Execution Mode Instructions", text: executionCustomText }
|
|
96
97
|
]);
|
|
97
98
|
}
|
|
98
|
-
export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
|
|
99
|
+
export function buildExecutionRecoveryPrompt({ basePrompt, transcript, explainMode = false }) {
|
|
99
100
|
if (!transcript) {
|
|
100
101
|
return basePrompt;
|
|
101
102
|
}
|
|
@@ -104,7 +105,9 @@ export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
|
|
|
104
105
|
[SESSION RECOVERY - Connection was lost. Previous actions completed before the error:]
|
|
105
106
|
${transcript}
|
|
106
107
|
|
|
107
|
-
|
|
108
|
+
${explainMode
|
|
109
|
+
? "[IMPORTANT: Resume execution. Continue using concise decision explanations without narrating routine actions.]"
|
|
110
|
+
: "[IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]"}`;
|
|
108
111
|
}
|
|
109
112
|
export function buildExecutionNavigationRecoveryPrompt({ basePrompt, transcript, navigatedUrl }) {
|
|
110
113
|
const normalizedTranscript = typeof transcript === "string" ? transcript.trim() : "";
|
|
@@ -27,6 +27,9 @@ export function formatCliOutput(item) {
|
|
|
27
27
|
}
|
|
28
28
|
const stepCompletion = stepCompletionData(text);
|
|
29
29
|
if (stepCompletion) {
|
|
30
|
+
if (item.payload?.explainMode === true) {
|
|
31
|
+
return `assistant: Step complete — ${stepCompletion.instruction}\nassistant: Reason: ${stepCompletion.reason}`;
|
|
32
|
+
}
|
|
30
33
|
return `assistant: Step complete — ${stepCompletion.instruction}`;
|
|
31
34
|
}
|
|
32
35
|
return text
|
|
@@ -84,9 +84,9 @@ function buildMarkerPlacement(marker = {}, width, height) {
|
|
|
84
84
|
}
|
|
85
85
|
const MARKER_APPEARANCES = {
|
|
86
86
|
click: {
|
|
87
|
-
haloFill: "#
|
|
87
|
+
haloFill: "#2f7df6",
|
|
88
88
|
haloOpacity: 0.28,
|
|
89
|
-
ringStroke: "#
|
|
89
|
+
ringStroke: "#2f7df6",
|
|
90
90
|
centerDotFill: "#07131f",
|
|
91
91
|
},
|
|
92
92
|
preview: {
|