@loadmill/droid-cua 2.34.0 → 2.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -5
- package/THIRD_PARTY_NOTICES.md +5 -0
- package/build/index.js +73 -33
- package/build/src/cli/headless-runner.js +89 -14
- package/build/src/cli/headless-selection.js +80 -0
- package/build/src/cli/headless-target.js +50 -3
- package/build/src/commands/help.js +8 -2
- package/build/src/core/session.js +5 -1
- package/build/src/core/test-data-import.js +2 -2
- package/build/src/desktop/computer-use.js +432 -0
- package/build/src/integrations/loadmill/client.js +25 -5
- package/build/src/modes/execution-mode.js +25 -0
- package/build/src/prompts/base.js +38 -4
- package/build/src/prompts/test-data-import.js +2 -2
- package/build/src/utils/cua-debug-tracer.js +2 -12
- package/build/src/utils/error-details.js +56 -0
- package/package.json +4 -2
- package/skill/droid-cua/references/running-and-healing.md +15 -8
package/README.md
CHANGED
|
@@ -79,6 +79,7 @@ Desktop app support:
|
|
|
79
79
|
|
|
80
80
|
**All platforms:**
|
|
81
81
|
- Loadmill sign-in for the Loadmill models, or an OpenAI API key for the OpenAI models
|
|
82
|
+
- Node.js 20 or newer for CLI use
|
|
82
83
|
|
|
83
84
|
**Android:**
|
|
84
85
|
- Android Debug Bridge (ADB)
|
|
@@ -172,13 +173,26 @@ Examples:
|
|
|
172
173
|
droid-cua
|
|
173
174
|
|
|
174
175
|
# Headless Android run
|
|
175
|
-
droid-cua --avd adb:emulator-5554
|
|
176
|
+
droid-cua run tests/login.dcua --avd adb:emulator-5554
|
|
176
177
|
|
|
177
178
|
# Headless Android folder run
|
|
178
|
-
droid-cua --avd adb:emulator-5554
|
|
179
|
+
droid-cua run tests --avd adb:emulator-5554
|
|
180
|
+
|
|
181
|
+
# Select tests from different folders
|
|
182
|
+
droid-cua run tests/auth/login.dcua tests/checkout/purchase.dcua admin/refund.dcua \
|
|
183
|
+
--avd adb:emulator-5554
|
|
179
184
|
|
|
180
185
|
# Headless iOS simulator run
|
|
181
|
-
droid-cua --platform ios --avd "iPhone 16"
|
|
186
|
+
droid-cua run tests/login.dcua --platform ios --avd "iPhone 16"
|
|
187
|
+
|
|
188
|
+
# List running desktop applications and their window IDs (macOS or Windows)
|
|
189
|
+
droid-cua --target desktop --list-desktop-apps
|
|
190
|
+
|
|
191
|
+
# Control one already-running desktop application window
|
|
192
|
+
droid-cua --target desktop --desktop-app com.apple.TextEdit --desktop-window 123 --instructions tests/editor.dcua
|
|
193
|
+
|
|
194
|
+
# Explicitly use cursor-free accessibility input instead of the default physical input
|
|
195
|
+
droid-cua --target desktop --desktop-app com.apple.TextEdit --desktop-input accessibility --instructions tests/editor.dcua
|
|
182
196
|
|
|
183
197
|
# Headless run with prompt-parity config
|
|
184
198
|
droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua --config ci/droid-cua.json
|
|
@@ -212,6 +226,16 @@ droid-cua \
|
|
|
212
226
|
--instructions tests/login.dcua
|
|
213
227
|
```
|
|
214
228
|
|
|
229
|
+
`droid-cua run` accepts one or more `.dcua` files or directories. Inputs are
|
|
230
|
+
expanded in the order provided, files inside each directory retain their natural
|
|
231
|
+
name ordering, and overlapping paths are run only once. A single config, target,
|
|
232
|
+
context, test-data file, and secrets source applies to the entire invocation.
|
|
233
|
+
The `run` command must be the first argument. Keep all selected paths contiguous
|
|
234
|
+
and place options after the complete path list. Arbitrary option/path
|
|
235
|
+
interleaving is not supported.
|
|
236
|
+
The existing `--instructions <file-or-directory>` form remains supported for
|
|
237
|
+
backward compatibility.
|
|
238
|
+
|
|
215
239
|
Example headless config:
|
|
216
240
|
```json
|
|
217
241
|
{
|
|
@@ -246,6 +270,11 @@ Supported CLI options include:
|
|
|
246
270
|
- `--app`
|
|
247
271
|
- `--list-devices`
|
|
248
272
|
- `--browser`
|
|
273
|
+
- `--target` (`mobile`, `web`, or `desktop`)
|
|
274
|
+
- `--desktop-app` (desktop application ID or exact name)
|
|
275
|
+
- `--desktop-window` (required when the application has multiple visible windows)
|
|
276
|
+
- `--desktop-input` (`physical` by default; use `accessibility` for cursor-free input)
|
|
277
|
+
- `--list-desktop-apps`
|
|
249
278
|
- `--instructions`
|
|
250
279
|
- `--config`
|
|
251
280
|
- `--cua-model`
|
|
@@ -268,6 +297,8 @@ Supported CLI options include:
|
|
|
268
297
|
`gpt-5.6-terra`, `gpt-5.6-luna`, and `gpt-5.4`. OpenAI models use direct OpenAI
|
|
269
298
|
mode and require `OPENAI_API_KEY`.
|
|
270
299
|
|
|
300
|
+
Desktop control supports macOS and Windows. In the desktop app, enable **Experimental desktop control** at the bottom of Settings to show Desktop in the platform picker. Desktop runs use visual input by default: they capture the selected window and send physical pointer and keyboard input at the model's coordinates. Use `--desktop-input accessibility` for cursor-free CLI input instead. macOS requires Screen & System Audio Recording and Accessibility permissions for Droid or the terminal running the CLI. Windows targets must run at the same integrity level as Droid; a non-elevated Droid process cannot inject input into an elevated application.
|
|
301
|
+
|
|
271
302
|
Config and precedence rules:
|
|
272
303
|
- Use `--config <file>` to supply prompt-affecting settings for headless runs.
|
|
273
304
|
- CLI flags override config file values.
|
|
@@ -292,12 +323,13 @@ Secret typing:
|
|
|
292
323
|
HTML report (`--report`):
|
|
293
324
|
- Writes a self-contained HTML report of the run — the same Run Details view the desktop app shows, with screenshots inlined — that you can share and open in any browser.
|
|
294
325
|
- With no value it saves `<test>--<date>--<time>.html` (e.g. `login--2026-06-03--16-09-20.html`) in the current directory; pass `--report <file>` to pick a filename or `--report <dir>` to pick a folder.
|
|
326
|
+
- A multi-path selection writes the individual test reports plus one aggregate `Selected tests` report with a child result and timeline for every selected test. When `--report <file.html>` is used, that exact path is used for the aggregate report.
|
|
295
327
|
- In the desktop app the same report is available from a run's **Share → Download HTML report** button.
|
|
296
|
-
- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9
|
|
328
|
+
- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 web and desktop replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename. While rendering, a compact **Background tasks** card shows progress for **Generating test replay**; it can be minimized to a task-count card or canceled without opening the save dialog.
|
|
297
329
|
- Authenticated headless runs upload their HTML report and screenshots to Loadmill by default. The Loadmill run record also includes the resolved Droid version, model, provider, Agent Pace, Aim Assist, Context Optimization, and target environment. Use `--no-cloud-report` to disable artifact upload; the Droid run record is still reported.
|
|
298
330
|
|
|
299
331
|
```sh
|
|
300
|
-
droid-cua --avd adb:emulator-5554 --
|
|
332
|
+
droid-cua run tests/login.dcua --avd adb:emulator-5554 --report ./reports
|
|
301
333
|
```
|
|
302
334
|
|
|
303
335
|
LambdaTest CLI runs:
|
package/build/index.js
CHANGED
|
@@ -22,7 +22,7 @@ import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-con
|
|
|
22
22
|
import { artifactKindsForPool, resolveHeadlessTarget, scopeArtifactKindsToPool, shouldWaitForLoadmillCloudCompletion, } from "./src/cli/headless-target.js";
|
|
23
23
|
import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
|
|
24
24
|
import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
|
|
25
|
-
import {
|
|
25
|
+
import { assertRunCommandPosition, getHeadlessInstructionSelectors, resolveHeadlessInstructionSelection, } from "./src/cli/headless-selection.js";
|
|
26
26
|
import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
|
|
27
27
|
import { buildLoadmillCloudDeviceConnectSummaryLines } from "./src/cli/loadmill-cloud-device-connect-summary.js";
|
|
28
28
|
import { planScenarioRuns } from "./src/cli/scenario-planner.js";
|
|
@@ -43,6 +43,7 @@ import { resolveRunPool } from "./src/device/cloud/loadmill-cloud/run-pool.js";
|
|
|
43
43
|
import { ARTIFACT_READY_STATUSES } from "./src/device/cloud/loadmill-cloud/run-status.js";
|
|
44
44
|
import { formatDuration } from "./src/core/report/run-log-model.js";
|
|
45
45
|
import { createBrowserSessionTeardown, withLoadmillCloudSession } from "./src/cli/loadmill-cloud-session.js";
|
|
46
|
+
import { createDesktopController, listDesktopApplications } from "./src/desktop/computer-use.js";
|
|
46
47
|
dotenv.config();
|
|
47
48
|
process.once("SIGTERM", async () => {
|
|
48
49
|
process.exitCode = 143;
|
|
@@ -58,7 +59,8 @@ process.once("SIGTERM", async () => {
|
|
|
58
59
|
process.once("beforeExit", () => {
|
|
59
60
|
loadmillCloudAdapter.releaseActiveSession();
|
|
60
61
|
});
|
|
61
|
-
const
|
|
62
|
+
const cliArgs = process.argv.slice(2);
|
|
63
|
+
const args = minimist(cliArgs);
|
|
62
64
|
if (args._[0] === "skill") {
|
|
63
65
|
try {
|
|
64
66
|
await runSkillCommand({ args });
|
|
@@ -75,7 +77,18 @@ let avdName = targetConfig.avdName;
|
|
|
75
77
|
let platform = targetConfig.platform || null; // 'ios' or 'android'
|
|
76
78
|
const recordScreenshots = args["record"] || false;
|
|
77
79
|
const startupUrl = typeof args.url === "string" && args.url.trim().length > 0 ? args.url.trim() : null;
|
|
78
|
-
|
|
80
|
+
let instructionSelectors;
|
|
81
|
+
try {
|
|
82
|
+
if (args.instructions == null && args.i == null) {
|
|
83
|
+
assertRunCommandPosition(cliArgs);
|
|
84
|
+
}
|
|
85
|
+
instructionSelectors = getHeadlessInstructionSelectors(args);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
89
|
+
console.error(`\nTest failed: ${message}`);
|
|
90
|
+
process.exit(1);
|
|
91
|
+
}
|
|
79
92
|
const appContextPath = typeof args.context === "string" ? args.context : null;
|
|
80
93
|
const debugMode = args["debug"] || false;
|
|
81
94
|
// --report [path]: generate a self-contained HTML report at the end of the run.
|
|
@@ -230,7 +243,7 @@ async function main() {
|
|
|
230
243
|
targetConfig.cloudSessionId &&
|
|
231
244
|
Array.isArray(targetConfig.artifacts) &&
|
|
232
245
|
targetConfig.artifacts.length > 0 &&
|
|
233
|
-
|
|
246
|
+
instructionSelectors.length === 0) {
|
|
234
247
|
const credentials = resolveLoadmillCloudCredentials(process.env);
|
|
235
248
|
// Share the exact same rendering the normal run command uses — install
|
|
236
249
|
// the progress listener before touching the adapter, since prepareSession
|
|
@@ -259,7 +272,7 @@ async function main() {
|
|
|
259
272
|
if (pool === "web") {
|
|
260
273
|
if (!ARTIFACT_READY_STATUSES.has(state.status)) {
|
|
261
274
|
retroactiveStatusLine.stop();
|
|
262
|
-
console.error(`Loadmill Cloud session ${targetConfig.cloudSessionId} is not terminal yet; add --instructions to drive it, or wait until it finishes and re-run this same command to fetch artifacts.`);
|
|
275
|
+
console.error(`Loadmill Cloud session ${targetConfig.cloudSessionId} is not terminal yet; use \`droid-cua run <path>\` or add --instructions to drive it, or wait until it finishes and re-run this same command to fetch artifacts.`);
|
|
263
276
|
process.exit(1);
|
|
264
277
|
}
|
|
265
278
|
await fetchAndDownloadArtifacts({ client, sessionId: targetConfig.cloudSessionId, kinds });
|
|
@@ -284,7 +297,7 @@ async function main() {
|
|
|
284
297
|
// exiting via the terminal/artifacts branch above — there's nothing to
|
|
285
298
|
// drive without --instructions, so release the slot and say so.
|
|
286
299
|
await loadmillCloudAdapter.releaseSession(credentials, { waitForCompletion: false });
|
|
287
|
-
console.error(`Loadmill Cloud session ${targetConfig.cloudSessionId} is not terminal yet; add --instructions to drive it, or wait until it finishes and re-run this same command to fetch artifacts.`);
|
|
300
|
+
console.error(`Loadmill Cloud session ${targetConfig.cloudSessionId} is not terminal yet; use \`droid-cua run <path>\` or add --instructions to drive it, or wait until it finishes and re-run this same command to fetch artifacts.`);
|
|
288
301
|
process.exit(1);
|
|
289
302
|
}
|
|
290
303
|
catch (error) {
|
|
@@ -310,8 +323,20 @@ async function main() {
|
|
|
310
323
|
console.log(`Run with: --device-name "<name>" --os-version "<osVersion>"`);
|
|
311
324
|
process.exit(0);
|
|
312
325
|
}
|
|
313
|
-
|
|
326
|
+
if (targetConfig.listDesktopApps) {
|
|
327
|
+
const applications = await listDesktopApplications({ excludePids: [process.pid] });
|
|
328
|
+
for (const application of applications) {
|
|
329
|
+
console.log(`${application.displayName} — ${application.bundleId} (pid ${application.pid})`);
|
|
330
|
+
for (const window of application.windows) {
|
|
331
|
+
console.log(` ${window.windowId}: ${window.title || "Untitled"}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
console.log(`\n${applications.length} desktop applications with visible windows.`);
|
|
335
|
+
process.exit(0);
|
|
336
|
+
}
|
|
337
|
+
const isHeadlessInstructionsRun = instructionSelectors.length > 0;
|
|
314
338
|
const isBrowserTarget = targetConfig.target === "web";
|
|
339
|
+
const isDesktopTarget = targetConfig.target === "desktop";
|
|
315
340
|
const isLambdaTestTarget = targetConfig.deviceSource === "lambdatest";
|
|
316
341
|
const isAwsDeviceFarmTarget = targetConfig.deviceSource === "aws-device-farm";
|
|
317
342
|
const isLoadmillCloudTarget = targetConfig.deviceSource === "loadmill-cloud";
|
|
@@ -326,6 +351,7 @@ async function main() {
|
|
|
326
351
|
let loadmillCloudBrowserSessionConnected = false;
|
|
327
352
|
let loadmillCloudBrowserTeardown = null;
|
|
328
353
|
let loadmillCloudStatusLine = null;
|
|
354
|
+
let desktopController = null;
|
|
329
355
|
const headlessDebug = createHeadlessDebugArtifacts({
|
|
330
356
|
cwd: process.cwd(),
|
|
331
357
|
enabled: isHeadlessInstructionsRun && debugMode
|
|
@@ -336,16 +362,19 @@ async function main() {
|
|
|
336
362
|
// Initialize legacy plain-text debug logging only for non-headless flows.
|
|
337
363
|
await logger.init(debugMode && !isHeadlessInstructionsRun);
|
|
338
364
|
if (isBrowserTarget && !isHeadlessInstructionsRun) {
|
|
339
|
-
throw new Error("--target web
|
|
365
|
+
throw new Error("--target web requires `droid-cua run <path>` or --instructions headless execution.");
|
|
366
|
+
}
|
|
367
|
+
if (isDesktopTarget && !isHeadlessInstructionsRun) {
|
|
368
|
+
throw new Error("--target desktop is only supported with --instructions headless execution.");
|
|
340
369
|
}
|
|
341
370
|
if (isCloudTarget && !isHeadlessInstructionsRun) {
|
|
342
|
-
throw new Error(`--device-source ${targetConfig.deviceSource}
|
|
371
|
+
throw new Error(`--device-source ${targetConfig.deviceSource} requires \`droid-cua run <path>\` or --instructions headless execution.`);
|
|
343
372
|
}
|
|
344
373
|
if (!isHeadlessInstructionsRun && args.scenario != null) {
|
|
345
|
-
throw new Error("--scenario
|
|
374
|
+
throw new Error("--scenario requires `droid-cua run <path>` or --instructions headless execution.");
|
|
346
375
|
}
|
|
347
376
|
if (!isHeadlessInstructionsRun && args["test-data"] != null) {
|
|
348
|
-
throw new Error("--test-data
|
|
377
|
+
throw new Error("--test-data requires `droid-cua run <path>` or --instructions headless execution.");
|
|
349
378
|
}
|
|
350
379
|
let preloadedHeadlessConfig = null;
|
|
351
380
|
let preloadedHeadlessTarget = null;
|
|
@@ -358,28 +387,15 @@ async function main() {
|
|
|
358
387
|
await assertCliRuntimeCredentials({
|
|
359
388
|
llmProviderMode: preloadedHeadlessConfig.llmProviderMode,
|
|
360
389
|
});
|
|
361
|
-
|
|
390
|
+
const selection = await resolveHeadlessInstructionSelection(instructionSelectors);
|
|
391
|
+
preloadedHeadlessTarget = selection.target;
|
|
392
|
+
headlessInstructionRuns = selection.runs;
|
|
362
393
|
preloadedSecrets = await resolveExecutionSecrets({
|
|
363
394
|
secretsPath: getDefaultSecretsPath(process.cwd()),
|
|
364
395
|
cliSecrets: preloadedHeadlessConfig.secretsOverride,
|
|
365
396
|
});
|
|
366
|
-
if (preloadedHeadlessTarget.type === "
|
|
367
|
-
|
|
368
|
-
if (files.length === 0) {
|
|
369
|
-
throw new Error(`No .dcua tests found in instructions directory ${preloadedHeadlessTarget.path}.`);
|
|
370
|
-
}
|
|
371
|
-
headlessInstructionRuns = files.map((file) => ({
|
|
372
|
-
testPath: file.path,
|
|
373
|
-
testName: file.filename,
|
|
374
|
-
instructions: null
|
|
375
|
-
}));
|
|
376
|
-
}
|
|
377
|
-
else {
|
|
378
|
-
headlessInstructionRuns = [{
|
|
379
|
-
testPath: preloadedHeadlessTarget.path,
|
|
380
|
-
testName: path.basename(preloadedHeadlessTarget.path),
|
|
381
|
-
instructions: await loadHeadlessInstructions(preloadedHeadlessTarget.path)
|
|
382
|
-
}];
|
|
397
|
+
if (preloadedHeadlessTarget.type === "file") {
|
|
398
|
+
headlessInstructionRuns[0].instructions = await loadHeadlessInstructions(headlessInstructionRuns[0].testPath);
|
|
383
399
|
}
|
|
384
400
|
preloadedAppContext = await resolveExecutionAppContext({
|
|
385
401
|
enabled: preloadedHeadlessConfig.appContextEnabled,
|
|
@@ -398,7 +414,7 @@ async function main() {
|
|
|
398
414
|
}
|
|
399
415
|
}
|
|
400
416
|
// If no device specified, show interactive selection menu
|
|
401
|
-
if (!isBrowserTarget && !isCloudTarget && !avdName && !platform) {
|
|
417
|
+
if (!isBrowserTarget && !isDesktopTarget && !isCloudTarget && !avdName && !platform) {
|
|
402
418
|
const selection = await selectDevice();
|
|
403
419
|
platform = selection.platform;
|
|
404
420
|
avdName = selection.deviceName;
|
|
@@ -506,6 +522,24 @@ async function main() {
|
|
|
506
522
|
console.log(`Starting URL: ${deviceInfo.currentUrl}`);
|
|
507
523
|
}
|
|
508
524
|
}
|
|
525
|
+
else if (isDesktopTarget) {
|
|
526
|
+
desktopController = await createDesktopController({
|
|
527
|
+
app: targetConfig.desktopApp,
|
|
528
|
+
windowId: targetConfig.desktopWindow,
|
|
529
|
+
inputMode: targetConfig.desktopInput,
|
|
530
|
+
excludePids: [process.pid],
|
|
531
|
+
});
|
|
532
|
+
targetId = desktopController.deviceId;
|
|
533
|
+
deviceInfo = desktopController.deviceInfo;
|
|
534
|
+
console.log(`Attached to desktop window: ${deviceInfo.device_name}`);
|
|
535
|
+
console.log(`Model sees window: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
|
|
536
|
+
console.log(`Desktop input: ${deviceInfo.desktopInputMode === "physical" ? "physical OS pointer" : "cursor-free accessibility"}`);
|
|
537
|
+
process.once("SIGINT", async () => {
|
|
538
|
+
console.log("\nStopping desktop run...");
|
|
539
|
+
await desktopController?.close().catch(() => { });
|
|
540
|
+
process.exit(130);
|
|
541
|
+
});
|
|
542
|
+
}
|
|
509
543
|
else if (isLambdaTestTarget) {
|
|
510
544
|
lambdaTestCredentials = resolveLambdaTestCredentials(process.env);
|
|
511
545
|
const lambdaTestBuildName = createLambdaTestCliBuildName();
|
|
@@ -729,8 +763,8 @@ async function main() {
|
|
|
729
763
|
const session = new Session(targetId, deviceInfo);
|
|
730
764
|
const initialSystemText = buildBaseSystemPrompt(deviceInfo);
|
|
731
765
|
session.setSystemPrompt(initialSystemText);
|
|
732
|
-
// If
|
|
733
|
-
if (
|
|
766
|
+
// If tests were selected, run in headless mode.
|
|
767
|
+
if (isHeadlessInstructionsRun) {
|
|
734
768
|
let exitCode = null;
|
|
735
769
|
let succeeded = false;
|
|
736
770
|
try {
|
|
@@ -787,8 +821,10 @@ async function main() {
|
|
|
787
821
|
recordScreenshots,
|
|
788
822
|
screenshotDir,
|
|
789
823
|
isBrowserTarget,
|
|
824
|
+
isDesktopTarget,
|
|
825
|
+
desktopController,
|
|
790
826
|
isCloudTarget,
|
|
791
|
-
runTarget: { isBrowserTarget, isCloudTarget },
|
|
827
|
+
runTarget: { isBrowserTarget, isDesktopTarget, isCloudTarget },
|
|
792
828
|
report: reportConfig
|
|
793
829
|
});
|
|
794
830
|
exitCode = result.exitCode;
|
|
@@ -845,6 +881,10 @@ async function main() {
|
|
|
845
881
|
console.log(`Done in ${formatDuration(Date.now() - processStartTime)}.`);
|
|
846
882
|
}
|
|
847
883
|
}
|
|
884
|
+
if (desktopController) {
|
|
885
|
+
await desktopController.close();
|
|
886
|
+
desktopController = null;
|
|
887
|
+
}
|
|
848
888
|
}
|
|
849
889
|
if (exitCode !== null) {
|
|
850
890
|
process.exit(exitCode);
|
|
@@ -83,9 +83,10 @@ function emitCliAppContextDebug(runId, appContext, appContextText) {
|
|
|
83
83
|
});
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
|
-
export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, onCuaResponse = null, secrets = null }) {
|
|
87
|
-
const resolvedRunTarget = runTarget ?? { isBrowserTarget, isCloudTarget };
|
|
86
|
+
export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, onCuaResponse = null, secrets = null }) {
|
|
87
|
+
const resolvedRunTarget = runTarget ?? { isBrowserTarget, isDesktopTarget, isCloudTarget };
|
|
88
88
|
const isBrowserRunTarget = Boolean(resolvedRunTarget.isBrowserTarget);
|
|
89
|
+
const isDesktopRunTarget = Boolean(resolvedRunTarget.isDesktopTarget);
|
|
89
90
|
const isCloudRunTarget = Boolean(resolvedRunTarget.isCloudTarget);
|
|
90
91
|
const sendCUARequest = async (requestArgs) => {
|
|
91
92
|
const response = await runtimeContext.openAi.sendCUARequest({
|
|
@@ -101,7 +102,7 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
|
|
|
101
102
|
toolCapabilities: {
|
|
102
103
|
assert: true,
|
|
103
104
|
navigate: isBrowserRunTarget,
|
|
104
|
-
closeKeyboard: !isBrowserRunTarget,
|
|
105
|
+
closeKeyboard: !isBrowserRunTarget && !isDesktopRunTarget,
|
|
105
106
|
loadmill: true,
|
|
106
107
|
},
|
|
107
108
|
secretsByKey: secrets?.secretsByKey || {},
|
|
@@ -130,6 +131,11 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
|
|
|
130
131
|
engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
|
|
131
132
|
engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
|
|
132
133
|
}
|
|
134
|
+
else if (isDesktopRunTarget) {
|
|
135
|
+
engineOptions.getScreenshotAsBase64 = desktopController.getScreenshotAsBase64;
|
|
136
|
+
engineOptions.handleModelAction = desktopController.handleModelAction;
|
|
137
|
+
engineOptions.getCurrentPlatform = () => "desktop";
|
|
138
|
+
}
|
|
133
139
|
if (isCloudRunTarget) {
|
|
134
140
|
if (isBrowserRunTarget) {
|
|
135
141
|
engineOptions.connectToBrowser = createRefuseReconnectConnectToBrowser();
|
|
@@ -142,7 +148,7 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
|
|
|
142
148
|
}
|
|
143
149
|
return engineOptions;
|
|
144
150
|
}
|
|
145
|
-
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
|
|
151
|
+
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
|
|
146
152
|
let executionMode = null;
|
|
147
153
|
let loadmillSessionUser = null;
|
|
148
154
|
let runUsageTotals = setDroidCuaRunUsageModel(createDroidCuaRunUsageTotals(), runtimeContext.config.cuaModel);
|
|
@@ -218,6 +224,8 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
218
224
|
recordScreenshots,
|
|
219
225
|
screenshotDir,
|
|
220
226
|
isBrowserTarget,
|
|
227
|
+
isDesktopTarget,
|
|
228
|
+
desktopController,
|
|
221
229
|
isCloudTarget,
|
|
222
230
|
runTarget,
|
|
223
231
|
onCuaResponse: (response) => {
|
|
@@ -296,9 +304,12 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
296
304
|
instructionsTotal: instructions.length,
|
|
297
305
|
stats,
|
|
298
306
|
};
|
|
299
|
-
|
|
307
|
+
const localReport = localReportEnabled
|
|
308
|
+
? createCliRunReport({ ...reportInput, events: localEvents })
|
|
309
|
+
: null;
|
|
310
|
+
if (localReport) {
|
|
300
311
|
try {
|
|
301
|
-
const outputPath = await writeCliRunReportArtifact(
|
|
312
|
+
const outputPath = await writeCliRunReportArtifact(localReport, { reportPath: report?.path ?? null, isBatch: isBatchRun });
|
|
302
313
|
console.log(`HTML report saved: ${outputPath}`);
|
|
303
314
|
}
|
|
304
315
|
catch (reportError) {
|
|
@@ -326,6 +337,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
326
337
|
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
327
338
|
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
328
339
|
retries: stats.retryCount ?? 0,
|
|
340
|
+
localReportModel: localReport?.model ?? null,
|
|
329
341
|
reportModel: cloudModel,
|
|
330
342
|
};
|
|
331
343
|
}
|
|
@@ -391,9 +403,12 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
391
403
|
instructionsTotal: instructions.length,
|
|
392
404
|
stats,
|
|
393
405
|
};
|
|
394
|
-
|
|
406
|
+
const localReport = localReportEnabled
|
|
407
|
+
? createCliRunReport({ ...failedReportInput, events: localEvents })
|
|
408
|
+
: null;
|
|
409
|
+
if (localReport) {
|
|
395
410
|
try {
|
|
396
|
-
const outputPath = await writeCliRunReportArtifact(
|
|
411
|
+
const outputPath = await writeCliRunReportArtifact(localReport, { reportPath: report?.path ?? null, isBatch: isBatchRun });
|
|
397
412
|
console.log(`HTML report saved: ${outputPath}`);
|
|
398
413
|
}
|
|
399
414
|
catch (reportError) {
|
|
@@ -422,20 +437,21 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
422
437
|
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
423
438
|
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
424
439
|
retries: stats.retryCount ?? 0,
|
|
440
|
+
localReportModel: localReport?.model ?? null,
|
|
425
441
|
reportModel: cloudModel,
|
|
426
442
|
};
|
|
427
443
|
}
|
|
428
444
|
throw error;
|
|
429
445
|
}
|
|
430
446
|
}
|
|
431
|
-
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
|
|
432
|
-
const resolvedRunTarget = runTarget ?? { isBrowserTarget, isCloudTarget };
|
|
447
|
+
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget = false, desktopController = null, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
|
|
448
|
+
const resolvedRunTarget = runTarget ?? { isBrowserTarget, isDesktopTarget, isCloudTarget };
|
|
433
449
|
const runConfiguration = buildDroidRunConfiguration({
|
|
434
450
|
droidVersion,
|
|
435
451
|
runtimeConfig: runtimeContext.config,
|
|
436
452
|
aimAssistMode: headlessConfig.aimAssistMode,
|
|
437
453
|
});
|
|
438
|
-
const isBatchRun =
|
|
454
|
+
const isBatchRun = ["directory", "selection", "scenario"].includes(instructionsTarget?.type);
|
|
439
455
|
const cloudReportEnabled = report?.cloudEnabled !== false &&
|
|
440
456
|
Boolean(runtimeContext.loadmillClient?.getApiToken?.());
|
|
441
457
|
const cloudUploadSession = cloudReportEnabled
|
|
@@ -447,6 +463,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
447
463
|
if (instructionsTarget?.type === "scenario") {
|
|
448
464
|
console.log(`\nRunning ${runs.length} scenario case${runs.length === 1 ? "" : "s"}\n`);
|
|
449
465
|
}
|
|
466
|
+
else if (instructionsTarget?.type === "selection") {
|
|
467
|
+
console.log(`\nRunning ${runs.length} selected test${runs.length === 1 ? "" : "s"}\n`);
|
|
468
|
+
}
|
|
450
469
|
else if (isBatchRun) {
|
|
451
470
|
console.log(`\nRunning ${runs.length} tests from: ${instructionsTarget.path}\n`);
|
|
452
471
|
}
|
|
@@ -461,6 +480,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
461
480
|
assertionsFailed: 0,
|
|
462
481
|
retries: 0
|
|
463
482
|
};
|
|
483
|
+
const aggregateLocalReportModels = [];
|
|
464
484
|
const aggregateReportModels = [];
|
|
465
485
|
for (const [index, run] of runs.entries()) {
|
|
466
486
|
const runId = createHeadlessRunId(index);
|
|
@@ -489,6 +509,8 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
489
509
|
recordScreenshots,
|
|
490
510
|
screenshotDir: testScreenshotDir,
|
|
491
511
|
isBrowserTarget: resolvedRunTarget.isBrowserTarget,
|
|
512
|
+
isDesktopTarget: resolvedRunTarget.isDesktopTarget,
|
|
513
|
+
desktopController,
|
|
492
514
|
isCloudTarget: resolvedRunTarget.isCloudTarget,
|
|
493
515
|
runTarget: resolvedRunTarget,
|
|
494
516
|
report,
|
|
@@ -508,6 +530,9 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
508
530
|
if (result.reportModel) {
|
|
509
531
|
aggregateReportModels.push(result.reportModel);
|
|
510
532
|
}
|
|
533
|
+
if (result.localReportModel) {
|
|
534
|
+
aggregateLocalReportModels.push(result.localReportModel);
|
|
535
|
+
}
|
|
511
536
|
if (!result.success) {
|
|
512
537
|
failures += 1;
|
|
513
538
|
}
|
|
@@ -518,6 +543,25 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
518
543
|
throw error;
|
|
519
544
|
}
|
|
520
545
|
const message = error instanceof Error ? error.message : String(error);
|
|
546
|
+
if (instructionsTarget?.type === "selection" && (report?.enabled || cloudReportEnabled)) {
|
|
547
|
+
const failedReportModel = buildCliRunReportModel({
|
|
548
|
+
suiteName: path.basename(path.dirname(run.testPath)) || null,
|
|
549
|
+
testName: run.testName,
|
|
550
|
+
startedAtIso: new Date().toISOString(),
|
|
551
|
+
durationMs: 0,
|
|
552
|
+
success: false,
|
|
553
|
+
error: message,
|
|
554
|
+
instructionsTotal: 0,
|
|
555
|
+
stats: {},
|
|
556
|
+
events: [],
|
|
557
|
+
});
|
|
558
|
+
if (report?.enabled) {
|
|
559
|
+
aggregateLocalReportModels.push(failedReportModel);
|
|
560
|
+
}
|
|
561
|
+
if (cloudReportEnabled) {
|
|
562
|
+
aggregateReportModels.push(failedReportModel);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
521
565
|
console.error(`\nTest failed: ${run.testName}: ${message}`);
|
|
522
566
|
}
|
|
523
567
|
}
|
|
@@ -525,19 +569,50 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
525
569
|
const passed = runs.length - failures;
|
|
526
570
|
console.log(`\nScenario run complete: ${passed}/${runs.length} passed`);
|
|
527
571
|
}
|
|
572
|
+
else if (instructionsTarget?.type === "selection") {
|
|
573
|
+
const passed = runs.length - failures;
|
|
574
|
+
console.log(`\nSelected tests run complete: ${passed}/${runs.length} passed`);
|
|
575
|
+
}
|
|
528
576
|
else if (isBatchRun) {
|
|
529
577
|
const passed = runs.length - failures;
|
|
530
578
|
console.log(`\nFolder run complete: ${passed}/${runs.length} passed`);
|
|
531
579
|
}
|
|
532
580
|
if (isBatchRun) {
|
|
533
581
|
const isScenario = instructionsTarget?.type === "scenario";
|
|
534
|
-
const
|
|
582
|
+
const isSelection = instructionsTarget?.type === "selection";
|
|
583
|
+
const projectName = isScenario || isSelection
|
|
584
|
+
? path.basename(process.cwd())
|
|
585
|
+
: path.basename(instructionsTarget.path);
|
|
586
|
+
const testName = isScenario
|
|
587
|
+
? "Scenario run"
|
|
588
|
+
: isSelection
|
|
589
|
+
? instructionsTarget.label
|
|
590
|
+
: path.basename(instructionsTarget.path);
|
|
591
|
+
if (isSelection && report?.enabled && aggregateLocalReportModels.length > 0) {
|
|
592
|
+
const aggregateReport = createCliAggregateRunReport({
|
|
593
|
+
suiteName: projectName,
|
|
594
|
+
testName,
|
|
595
|
+
startedAtIso: new Date(batchStartedAt).toISOString(),
|
|
596
|
+
durationMs: Math.max(aggregateStats.durationMs, Date.now() - batchStartedAt),
|
|
597
|
+
childModels: aggregateLocalReportModels,
|
|
598
|
+
});
|
|
599
|
+
try {
|
|
600
|
+
const outputPath = await writeCliRunReportArtifact(aggregateReport, {
|
|
601
|
+
reportPath: report?.path ?? null,
|
|
602
|
+
});
|
|
603
|
+
console.log(`Aggregate HTML report saved: ${outputPath}`);
|
|
604
|
+
}
|
|
605
|
+
catch (reportError) {
|
|
606
|
+
const message = reportError instanceof Error ? reportError.message : String(reportError);
|
|
607
|
+
console.warn(`Warning: failed to write aggregate HTML report: ${message}`);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
535
610
|
const reportedRun = await reportDroidRunCompletion({
|
|
536
611
|
...runConfiguration,
|
|
537
612
|
loadmillClient: runtimeContext.loadmillClient,
|
|
538
613
|
source: "ci",
|
|
539
614
|
runId: createHeadlessRunId(),
|
|
540
|
-
projectName
|
|
615
|
+
projectName,
|
|
541
616
|
testName,
|
|
542
617
|
runType: "project",
|
|
543
618
|
testCases: runs.length,
|
|
@@ -560,7 +635,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
560
635
|
});
|
|
561
636
|
if (cloudUploadSession && reportedRun.savedRunId) {
|
|
562
637
|
const aggregateReport = createCliAggregateRunReport({
|
|
563
|
-
suiteName:
|
|
638
|
+
suiteName: projectName,
|
|
564
639
|
testName,
|
|
565
640
|
startedAtIso: new Date(batchStartedAt).toISOString(),
|
|
566
641
|
durationMs: Math.max(aggregateStats.durationMs, Date.now() - batchStartedAt),
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget, } from "./headless-instruction-target.js";
|
|
3
|
+
function normalizeOptionValues(value, optionName) {
|
|
4
|
+
if (value == null) {
|
|
5
|
+
return [];
|
|
6
|
+
}
|
|
7
|
+
const values = Array.isArray(value) ? value : [value];
|
|
8
|
+
if (values.some((entry) => typeof entry !== "string" || !entry.trim())) {
|
|
9
|
+
throw new Error(`${optionName} requires a file or directory path.`);
|
|
10
|
+
}
|
|
11
|
+
return values.map((entry) => entry.trim());
|
|
12
|
+
}
|
|
13
|
+
export function assertRunCommandPosition(argv) {
|
|
14
|
+
const runIndex = Array.isArray(argv) ? argv.indexOf("run") : -1;
|
|
15
|
+
if (runIndex > 0) {
|
|
16
|
+
throw new Error("`run` must be the first argument. Use `droid-cua run <paths...> [options]`.");
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function getHeadlessInstructionSelectors(args) {
|
|
20
|
+
const positional = Array.isArray(args?._) ? args._ : [];
|
|
21
|
+
const instructionSelectors = [
|
|
22
|
+
...normalizeOptionValues(args?.instructions, "--instructions"),
|
|
23
|
+
...normalizeOptionValues(args?.i, "-i"),
|
|
24
|
+
];
|
|
25
|
+
if (positional[0] !== "run") {
|
|
26
|
+
return instructionSelectors;
|
|
27
|
+
}
|
|
28
|
+
if (instructionSelectors.length > 0) {
|
|
29
|
+
throw new Error("Do not combine `droid-cua run <paths...>` with --instructions or -i.");
|
|
30
|
+
}
|
|
31
|
+
const runSelectors = positional.slice(1);
|
|
32
|
+
if (runSelectors.length === 0) {
|
|
33
|
+
throw new Error("`droid-cua run` requires at least one test file or directory.");
|
|
34
|
+
}
|
|
35
|
+
return runSelectors;
|
|
36
|
+
}
|
|
37
|
+
function displayPath(filePath, cwd) {
|
|
38
|
+
const relativePath = path.relative(cwd, path.resolve(cwd, filePath));
|
|
39
|
+
return relativePath || path.basename(filePath);
|
|
40
|
+
}
|
|
41
|
+
export async function resolveHeadlessInstructionSelection(selectors, { cwd = process.cwd() } = {}) {
|
|
42
|
+
if (!Array.isArray(selectors) || selectors.length === 0) {
|
|
43
|
+
throw new Error("At least one test file or directory is required.");
|
|
44
|
+
}
|
|
45
|
+
const targets = [];
|
|
46
|
+
const selectedFiles = [];
|
|
47
|
+
const seenPaths = new Set();
|
|
48
|
+
for (const selector of selectors) {
|
|
49
|
+
const target = await resolveHeadlessInstructionsTarget(selector);
|
|
50
|
+
targets.push(target);
|
|
51
|
+
const files = target.type === "directory"
|
|
52
|
+
? await listHeadlessInstructionFiles(target.path)
|
|
53
|
+
: [{ filename: path.basename(target.path), path: target.path }];
|
|
54
|
+
if (target.type === "directory" && files.length === 0) {
|
|
55
|
+
throw new Error(`No .dcua tests found in instructions directory ${target.path}.`);
|
|
56
|
+
}
|
|
57
|
+
for (const file of files) {
|
|
58
|
+
const dedupeKey = path.resolve(cwd, file.path);
|
|
59
|
+
if (seenPaths.has(dedupeKey)) {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
seenPaths.add(dedupeKey);
|
|
63
|
+
selectedFiles.push(file);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const isSelection = selectors.length > 1;
|
|
67
|
+
const target = isSelection
|
|
68
|
+
? {
|
|
69
|
+
type: "selection",
|
|
70
|
+
paths: targets.map((entry) => entry.path),
|
|
71
|
+
label: "Selected tests",
|
|
72
|
+
}
|
|
73
|
+
: targets[0];
|
|
74
|
+
const runs = selectedFiles.map((file) => ({
|
|
75
|
+
testPath: file.path,
|
|
76
|
+
testName: isSelection ? displayPath(file.path, cwd) : file.filename,
|
|
77
|
+
instructions: null,
|
|
78
|
+
}));
|
|
79
|
+
return { target, runs };
|
|
80
|
+
}
|