@loadmill/droid-cua 2.9.0 → 2.10.1-rc.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -2
- package/build/index.js +24 -18
- package/build/src/brand/loadmill-mark.js +20 -0
- package/build/src/browser/playwright/connection.js +148 -11
- package/build/src/browser/playwright/welcome-page.js +161 -0
- package/build/src/cli/headless-execution-config.js +16 -24
- package/build/src/cli/headless-target.js +0 -9
- package/build/src/commands/help.js +3 -5
- package/build/src/commands/run.js +1 -1
- package/build/src/core/aim-assist.js +5 -0
- package/build/src/core/execution-engine.js +266 -505
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/runtime-config.js +1 -2
- package/build/src/core/screenshot-store.js +153 -0
- package/build/src/core/tool-handlers/assert.js +155 -0
- package/build/src/core/tool-handlers/loadmill.js +29 -0
- package/build/src/core/tool-handlers/navigate.js +110 -0
- package/build/src/core/tool-handlers/preview-click.js +112 -0
- package/build/src/core/tool-handlers/test-design-complete.js +93 -0
- package/build/src/core/tool-handlers/utils.js +80 -0
- package/build/src/device/openai.js +103 -4
- package/build/src/modes/design-mode-ink.js +17 -5
- package/build/src/modes/design-mode.js +17 -5
- package/build/src/modes/execution-mode.js +42 -234
- package/build/src/prompts/base.js +44 -31
- package/build/src/prompts/design.js +55 -15
- package/build/src/prompts/editor.js +1 -1
- package/build/src/prompts/execution.js +15 -10
- package/build/src/utils/cua-debug-tracer.js +3 -1
- package/package.json +3 -4
- package/build/src/device/screenshot-resolution.js +0 -33
package/README.md
CHANGED
package/build/index.js
CHANGED
|
@@ -32,11 +32,6 @@ const recordScreenshots = args["record"] || false;
|
|
|
32
32
|
const instructionsFile = args.instructions || args.i || null;
|
|
33
33
|
const appContextPath = typeof args.context === "string" ? args.context : null;
|
|
34
34
|
const debugMode = args["debug"] || false;
|
|
35
|
-
const strictFlag = args.strict === true;
|
|
36
|
-
const noStrictFlag = args["no-strict"] === true || args.strict === false;
|
|
37
|
-
if (strictFlag && noStrictFlag) {
|
|
38
|
-
throw new Error("--strict and --no-strict cannot be used together.");
|
|
39
|
-
}
|
|
40
35
|
const screenshotDir = path.join("droid-cua-recording-" + Date.now());
|
|
41
36
|
if (recordScreenshots)
|
|
42
37
|
await mkdir(screenshotDir, { recursive: true });
|
|
@@ -87,7 +82,6 @@ async function main() {
|
|
|
87
82
|
let deviceInfo;
|
|
88
83
|
if (isBrowserTarget) {
|
|
89
84
|
targetId = await connectToBrowser({
|
|
90
|
-
url: targetConfig.url,
|
|
91
85
|
headless: true,
|
|
92
86
|
sessionMode: targetConfig.sessionMode
|
|
93
87
|
});
|
|
@@ -115,10 +109,12 @@ async function main() {
|
|
|
115
109
|
let instructions = [];
|
|
116
110
|
let executionMode = null;
|
|
117
111
|
try {
|
|
112
|
+
const headlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
118
113
|
await headlessDebug.startExecutionSession(runId, {
|
|
119
114
|
testName,
|
|
120
115
|
platform: deviceInfo.platform,
|
|
121
|
-
deviceName: deviceInfo.device_name
|
|
116
|
+
deviceName: deviceInfo.device_name,
|
|
117
|
+
aimAssistMode: headlessConfig.aimAssistMode
|
|
122
118
|
});
|
|
123
119
|
const currentLogFilePath = await headlessDebug.getCurrentLogFilePath();
|
|
124
120
|
if (currentLogFilePath) {
|
|
@@ -126,7 +122,6 @@ async function main() {
|
|
|
126
122
|
}
|
|
127
123
|
console.log(`\nRunning test from: ${instructionsFile}\n`);
|
|
128
124
|
instructions = await loadHeadlessInstructions(instructionsFile);
|
|
129
|
-
const headlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
130
125
|
const appContext = await resolveExecutionAppContext({
|
|
131
126
|
enabled: headlessConfig.appContextEnabled,
|
|
132
127
|
contextPath: headlessConfig.appContextPath,
|
|
@@ -134,7 +129,6 @@ async function main() {
|
|
|
134
129
|
const appContextText = appContext.text;
|
|
135
130
|
const runtimeContext = createCliRuntimeContext({
|
|
136
131
|
cuaModel: headlessConfig.cuaModel,
|
|
137
|
-
strictMode: headlessConfig.strictMode,
|
|
138
132
|
promptCustomizations: headlessConfig.promptCustomizations,
|
|
139
133
|
contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
|
|
140
134
|
contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
|
|
@@ -174,29 +168,44 @@ async function main() {
|
|
|
174
168
|
status: appContext.status
|
|
175
169
|
});
|
|
176
170
|
}
|
|
177
|
-
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, {
|
|
171
|
+
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
|
|
178
172
|
session.setSystemPrompt(executionPrompt);
|
|
179
173
|
const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
|
|
180
174
|
runId,
|
|
181
175
|
recordScreenshots,
|
|
182
176
|
screenshotDir
|
|
183
177
|
});
|
|
178
|
+
const executionEngineOptions = {
|
|
179
|
+
toolCapabilities: {
|
|
180
|
+
assert: true,
|
|
181
|
+
navigate: isBrowserTarget,
|
|
182
|
+
loadmill: true,
|
|
183
|
+
},
|
|
184
|
+
baseSystemPrompt: executionPrompt,
|
|
185
|
+
isHeadlessMode: true,
|
|
186
|
+
};
|
|
184
187
|
const engineOptions = screenshotRecorder
|
|
185
188
|
? {
|
|
186
189
|
recordScreenshots: true,
|
|
187
190
|
screenshotRecorder,
|
|
188
|
-
strictMode: runtimeContext.config.strictMode,
|
|
189
191
|
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
190
|
-
sendCUARequest: runtimeContext.openAi.sendCUARequest
|
|
192
|
+
sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
|
|
193
|
+
...requestArgs,
|
|
194
|
+
aimAssistMode: headlessConfig.aimAssistMode,
|
|
195
|
+
}),
|
|
191
196
|
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
197
|
+
...executionEngineOptions,
|
|
192
198
|
}
|
|
193
199
|
: {
|
|
194
200
|
recordScreenshots,
|
|
195
201
|
screenshotDir,
|
|
196
|
-
strictMode: runtimeContext.config.strictMode,
|
|
197
202
|
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
198
|
-
sendCUARequest: runtimeContext.openAi.sendCUARequest
|
|
203
|
+
sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
|
|
204
|
+
...requestArgs,
|
|
205
|
+
aimAssistMode: headlessConfig.aimAssistMode,
|
|
206
|
+
}),
|
|
199
207
|
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
208
|
+
...executionEngineOptions,
|
|
200
209
|
};
|
|
201
210
|
if (isBrowserTarget) {
|
|
202
211
|
engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
|
|
@@ -255,13 +264,10 @@ async function main() {
|
|
|
255
264
|
throw error;
|
|
256
265
|
}
|
|
257
266
|
}
|
|
258
|
-
const runtimeContext = createCliRuntimeContext(
|
|
259
|
-
strictMode: strictFlag,
|
|
260
|
-
});
|
|
267
|
+
const runtimeContext = createCliRuntimeContext();
|
|
261
268
|
const engine = new ExecutionEngine(session, {
|
|
262
269
|
recordScreenshots,
|
|
263
270
|
screenshotDir,
|
|
264
|
-
strictMode: runtimeContext.config.strictMode,
|
|
265
271
|
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
266
272
|
sendCUARequest: runtimeContext.openAi.sendCUARequest,
|
|
267
273
|
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export const LOADMILL_MARK_PATHS = [
|
|
2
|
+
{
|
|
3
|
+
d: `M808.982 297.016 644.14 29.3c-1.484-2.012-5.587-8.153-11.234-8.488-7.747-.46-9.872 3.935-11.428
|
|
4
|
+
7.151l-.029.06-111.354 457.775c-1.141 5.191.769 9.126 4.5 12.835 3.731 3.709 10.086 1.879
|
|
5
|
+
12.796.5l275.123-180.566c2.845-1.478 8.763-5.472 9.675-9.625.912-4.153-1.758-9.681-3.207-11.926ZM221.403
|
|
6
|
+
695.361l265.91-167.74c2.219-1.152 8.607-4.858 13.825-2.672 7.156 3 7.129 7.881 7.109 11.453v.067L406.583
|
|
7
|
+
996.493c-1.262 5.167-4.71 7.857-9.694 9.537-4.983 1.69-9.882-2.75-11.708-5.18L217.718 717.557c-1.903-2.58-5.457-8.772-4.447-12.902
|
|
8
|
+
1.01-4.131 5.843-7.917 8.132-9.294Z`,
|
|
9
|
+
fill: "#52BAD3"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
d: `M282.079 171.043 8.876 355.76c-1.948 1.568-7.91 5.926-8.008 11.582-.133 7.758 4.346 9.697 7.625
|
|
13
|
+
11.116l.062.027 452.431 99.593c5.234.922 9.086-1.152 12.634-5.036 3.549-3.884 1.453-10.156-.039-12.806L303.882
|
|
14
|
+
176.598c-1.596-2.78-5.835-8.525-10.023-9.261-4.188-.737-9.598 2.163-11.78 3.706ZM769.31 811.501
|
|
15
|
+
554.939 562.32c-1.503-1.998-6.214-7.686-4.919-13.193 1.775-7.554 6.594-8.334 10.12-8.904l.066-.011
|
|
16
|
+
462.134 22.113c5.3.392 8.52 3.347 11.01 7.984 2.49 4.636-1.08 10.201-3.18 12.405L791.81 811.468c-2.23
|
|
17
|
+
2.303-7.75 6.832-11.99 6.518-4.241-.314-8.774-4.454-10.51-6.485Z`,
|
|
18
|
+
fill: "#5691E2"
|
|
19
|
+
}
|
|
20
|
+
];
|
|
@@ -6,8 +6,10 @@ import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
|
6
6
|
import { requireInstalledBrowser } from "./installed-browser.js";
|
|
7
7
|
import { clearActiveBrowserSession, getActiveBrowserSession, setActiveBrowserSession } from "./session-state.js";
|
|
8
8
|
import { normalizeAutomationUrl, normalizeOptionalAutomationUrl } from "./url.js";
|
|
9
|
+
import { getWelcomePageUrl, isWelcomePageUrl } from "./welcome-page.js";
|
|
9
10
|
const DEFAULT_VIEWPORT = { width: 1440, height: 900 };
|
|
10
11
|
let browserConnectionTestHooks = null;
|
|
12
|
+
let pendingBrowserSessionShutdown = null;
|
|
11
13
|
async function gotoWithTimeoutTolerance(page, url) {
|
|
12
14
|
try {
|
|
13
15
|
await page.goto(url, { waitUntil: "load" });
|
|
@@ -20,7 +22,7 @@ async function gotoWithTimeoutTolerance(page, url) {
|
|
|
20
22
|
}
|
|
21
23
|
}
|
|
22
24
|
function normalizeKnownPageUrl(url) {
|
|
23
|
-
if (typeof url !== "string" || url.trim().length === 0 || url.trim() === "about:blank") {
|
|
25
|
+
if (typeof url !== "string" || url.trim().length === 0 || url.trim() === "about:blank" || isWelcomePageUrl(url.trim())) {
|
|
24
26
|
return null;
|
|
25
27
|
}
|
|
26
28
|
try {
|
|
@@ -30,6 +32,125 @@ function normalizeKnownPageUrl(url) {
|
|
|
30
32
|
return null;
|
|
31
33
|
}
|
|
32
34
|
}
|
|
35
|
+
async function closeExtraPages(context, primaryPage) {
|
|
36
|
+
for (const page of context.pages()) {
|
|
37
|
+
if (page === primaryPage) {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
await page.close();
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Best-effort cleanup for restored tabs.
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async function cleanupClosedSession(session) {
|
|
49
|
+
if (!session.cleanupOnDisconnect) {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
await rm(session.userDataDir, { recursive: true, force: true });
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// Best-effort cleanup when the browser disappears outside normal disconnect flow.
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function isBenignCloseError(error) {
|
|
60
|
+
if (!(error instanceof Error)) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
const message = `${error.name} ${error.message}`.toLowerCase();
|
|
64
|
+
return (message.includes("target page, context or browser has been closed") ||
|
|
65
|
+
message.includes("browser has been closed") ||
|
|
66
|
+
message.includes("context closed") ||
|
|
67
|
+
message.includes("target closed") ||
|
|
68
|
+
message.includes("session closed") ||
|
|
69
|
+
message.includes("closed") ||
|
|
70
|
+
message.includes("disconnected"));
|
|
71
|
+
}
|
|
72
|
+
function isCurrentSession(session) {
|
|
73
|
+
try {
|
|
74
|
+
return getActiveBrowserSession().sessionId === session.sessionId;
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
async function waitForPendingBrowserSessionShutdown() {
|
|
81
|
+
if (!pendingBrowserSessionShutdown) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
try {
|
|
85
|
+
await pendingBrowserSessionShutdown;
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// Ignore teardown failures here. A follow-up launch attempt can surface the next actionable error.
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async function closeBrowserSession(session, options = {}) {
|
|
92
|
+
if (session.shutdownPromise) {
|
|
93
|
+
return await session.shutdownPromise;
|
|
94
|
+
}
|
|
95
|
+
const { externalReason = null, clearActive = true } = options;
|
|
96
|
+
session.disconnecting = true;
|
|
97
|
+
const shutdownPromise = (async () => {
|
|
98
|
+
try {
|
|
99
|
+
await session.context.close();
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (!isBenignCloseError(error)) {
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
if (clearActive && isCurrentSession(session)) {
|
|
108
|
+
clearActiveBrowserSession();
|
|
109
|
+
}
|
|
110
|
+
await cleanupClosedSession(session);
|
|
111
|
+
}
|
|
112
|
+
if (externalReason && !session.externalDisconnectNotified && typeof session.onDisconnected === "function") {
|
|
113
|
+
session.externalDisconnectNotified = true;
|
|
114
|
+
await session.onDisconnected({
|
|
115
|
+
sessionId: session.sessionId,
|
|
116
|
+
browserName: session.installedBrowser?.browserName || "Browser",
|
|
117
|
+
browserFamily: session.installedBrowser?.browserFamily || null,
|
|
118
|
+
reason: externalReason,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
})();
|
|
122
|
+
session.shutdownPromise = shutdownPromise;
|
|
123
|
+
pendingBrowserSessionShutdown = shutdownPromise.finally(() => {
|
|
124
|
+
if (pendingBrowserSessionShutdown === session.shutdownPromise) {
|
|
125
|
+
pendingBrowserSessionShutdown = null;
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
return await session.shutdownPromise;
|
|
129
|
+
}
|
|
130
|
+
function handleExternalSessionClosed(session, reason) {
|
|
131
|
+
if (session.disconnecting) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
void closeBrowserSession(session, {
|
|
135
|
+
externalReason: reason,
|
|
136
|
+
clearActive: true,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
async function launchPersistentContextWithRetry(chromium, userDataDir, launchOptions, sessionMode) {
|
|
140
|
+
try {
|
|
141
|
+
return await chromium.launchPersistentContext(userDataDir, launchOptions);
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
145
|
+
const isProfileLockError = sessionMode === "persistent" && message.includes("ProcessSingleton");
|
|
146
|
+
if (!isProfileLockError) {
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
await waitForPendingBrowserSessionShutdown();
|
|
150
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
151
|
+
return await chromium.launchPersistentContext(userDataDir, launchOptions);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
33
154
|
function buildSessionBrowserInfo(url, installedBrowser) {
|
|
34
155
|
const currentUrl = normalizeKnownPageUrl(url);
|
|
35
156
|
const browserInfo = {
|
|
@@ -63,6 +184,7 @@ function normalizeConnectOptions(targetOrOptions) {
|
|
|
63
184
|
url: normalizeOptionalAutomationUrl(options.url),
|
|
64
185
|
headless: options.headless === true,
|
|
65
186
|
sessionMode: options.sessionMode === "persistent" ? "persistent" : "fresh",
|
|
187
|
+
onDisconnected: typeof options.onDisconnected === "function" ? options.onDisconnected : null,
|
|
66
188
|
persistentProfileDir: typeof options.persistentProfileDir === "string" && options.persistentProfileDir.trim().length > 0
|
|
67
189
|
? path.resolve(options.persistentProfileDir)
|
|
68
190
|
: null,
|
|
@@ -115,6 +237,7 @@ export async function connectToBrowser(targetOrOptions) {
|
|
|
115
237
|
const options = normalizeConnectOptions(targetOrOptions);
|
|
116
238
|
const installedBrowser = await resolveInstalledBrowserForLaunch();
|
|
117
239
|
const chromium = await loadChromium();
|
|
240
|
+
await waitForPendingBrowserSessionShutdown();
|
|
118
241
|
try {
|
|
119
242
|
getActiveBrowserSession();
|
|
120
243
|
await disconnect();
|
|
@@ -123,16 +246,20 @@ export async function connectToBrowser(targetOrOptions) {
|
|
|
123
246
|
// no active session to close
|
|
124
247
|
}
|
|
125
248
|
const { userDataDir, cleanupOnDisconnect } = await buildUserDataDir(options);
|
|
126
|
-
const
|
|
249
|
+
const launchOptions = {
|
|
127
250
|
executablePath: installedBrowser.executablePath,
|
|
128
251
|
headless: options.headless,
|
|
129
252
|
viewport: DEFAULT_VIEWPORT,
|
|
130
253
|
screen: DEFAULT_VIEWPORT,
|
|
131
254
|
acceptDownloads: false,
|
|
132
|
-
}
|
|
255
|
+
};
|
|
256
|
+
const context = await launchPersistentContextWithRetry(chromium, userDataDir, launchOptions, options.sessionMode);
|
|
133
257
|
const primaryPage = context.pages()[0] ?? (await context.newPage());
|
|
134
258
|
const sessionId = `browser-${randomUUID()}`;
|
|
135
259
|
const normalizedUrl = options.url;
|
|
260
|
+
const welcomePageUrl = await getWelcomePageUrl();
|
|
261
|
+
const initialUrl = normalizedUrl || welcomePageUrl;
|
|
262
|
+
await closeExtraPages(context, primaryPage);
|
|
136
263
|
context.on("page", (page) => {
|
|
137
264
|
if (page === primaryPage) {
|
|
138
265
|
return;
|
|
@@ -143,18 +270,28 @@ export async function connectToBrowser(targetOrOptions) {
|
|
|
143
270
|
url: page.url() || null,
|
|
144
271
|
});
|
|
145
272
|
});
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
149
|
-
setActiveBrowserSession({
|
|
273
|
+
await gotoWithTimeoutTolerance(primaryPage, initialUrl);
|
|
274
|
+
const session = {
|
|
150
275
|
sessionId,
|
|
151
276
|
context,
|
|
152
277
|
page: primaryPage,
|
|
153
278
|
url: normalizedUrl,
|
|
279
|
+
welcomePageUrl,
|
|
154
280
|
sessionMode: options.sessionMode,
|
|
155
281
|
userDataDir,
|
|
156
282
|
cleanupOnDisconnect,
|
|
157
283
|
installedBrowser,
|
|
284
|
+
onDisconnected: options.onDisconnected,
|
|
285
|
+
disconnecting: false,
|
|
286
|
+
externalDisconnectNotified: false,
|
|
287
|
+
shutdownPromise: null,
|
|
288
|
+
};
|
|
289
|
+
setActiveBrowserSession(session);
|
|
290
|
+
primaryPage.on?.("close", () => {
|
|
291
|
+
handleExternalSessionClosed(session, "page_closed");
|
|
292
|
+
});
|
|
293
|
+
context.on?.("close", () => {
|
|
294
|
+
handleExternalSessionClosed(session, "context_closed");
|
|
158
295
|
});
|
|
159
296
|
emitDesktopDebug("browser.connect", "device", {}, {
|
|
160
297
|
platform: "browser",
|
|
@@ -194,13 +331,13 @@ export async function disconnect() {
|
|
|
194
331
|
session = getActiveBrowserSession();
|
|
195
332
|
}
|
|
196
333
|
catch {
|
|
334
|
+
await waitForPendingBrowserSessionShutdown();
|
|
197
335
|
return;
|
|
198
336
|
}
|
|
199
337
|
clearActiveBrowserSession();
|
|
200
|
-
await session
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
}
|
|
338
|
+
await closeBrowserSession(session, {
|
|
339
|
+
clearActive: false,
|
|
340
|
+
});
|
|
204
341
|
}
|
|
205
342
|
export function __setBrowserConnectionTestHooks(hooks = null) {
|
|
206
343
|
browserConnectionTestHooks = hooks;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
|
+
import { pathToFileURL } from "node:url";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import { LOADMILL_MARK_PATHS } from "../../brand/loadmill-mark.js";
|
|
6
|
+
const WELCOME_PAGE_DIR = path.join(os.homedir(), ".droid-cua", "browser-assets");
|
|
7
|
+
const WELCOME_PAGE_PATH = path.join(WELCOME_PAGE_DIR, "loadmill-droid-cua-welcome.html");
|
|
8
|
+
const WELCOME_PAGE_URL = pathToFileURL(WELCOME_PAGE_PATH).toString();
|
|
9
|
+
let ensureWelcomePagePromise = null;
|
|
10
|
+
let welcomePageTestHooks = null;
|
|
11
|
+
function renderLogoSvg() {
|
|
12
|
+
const paths = LOADMILL_MARK_PATHS.map(({ d, fill }) => `<path d="${d}" fill="${fill}" />`).join("");
|
|
13
|
+
return `<svg viewBox="0 0 1035 1035" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">${paths}</svg>`;
|
|
14
|
+
}
|
|
15
|
+
function buildWelcomePageHtml() {
|
|
16
|
+
return `<!doctype html>
|
|
17
|
+
<html lang="en">
|
|
18
|
+
<head>
|
|
19
|
+
<meta charset="utf-8" />
|
|
20
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
21
|
+
<title>Loadmill Droid CUA</title>
|
|
22
|
+
<style>
|
|
23
|
+
* { box-sizing: border-box; }
|
|
24
|
+
html, body { height: 100%; margin: 0; }
|
|
25
|
+
|
|
26
|
+
body {
|
|
27
|
+
font-family: "SF Pro Text", "Inter Tight", "Segoe UI", sans-serif;
|
|
28
|
+
background: linear-gradient(135deg, #f3f7fd 0%, #d4e2f3 55%, #b8cce8 100%);
|
|
29
|
+
color: #475569;
|
|
30
|
+
position: relative;
|
|
31
|
+
overflow-x: hidden;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
body::before,
|
|
35
|
+
body::after {
|
|
36
|
+
content: "";
|
|
37
|
+
position: absolute;
|
|
38
|
+
width: 168px;
|
|
39
|
+
height: 132px;
|
|
40
|
+
background-image: radial-gradient(rgba(91, 142, 226, 0.36) 1.4px, transparent 1.6px);
|
|
41
|
+
background-size: 18px 18px;
|
|
42
|
+
pointer-events: none;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
body::before { bottom: 7vh; left: 5vw; }
|
|
46
|
+
body::after { top: 7vh; right: 5vw; }
|
|
47
|
+
|
|
48
|
+
main {
|
|
49
|
+
position: relative;
|
|
50
|
+
min-height: 100vh;
|
|
51
|
+
display: flex;
|
|
52
|
+
align-items: flex-start;
|
|
53
|
+
justify-content: center;
|
|
54
|
+
padding: clamp(10vh, 13vh, 18vh) 32px 32px;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
.content { width: min(100%, 720px); text-align: center; }
|
|
58
|
+
|
|
59
|
+
.brand { margin: 0 auto 32px; }
|
|
60
|
+
.logo { width: 72px; height: 72px; margin: 0 auto 14px; }
|
|
61
|
+
.wordmark {
|
|
62
|
+
font-size: 17px;
|
|
63
|
+
letter-spacing: 0.32em;
|
|
64
|
+
font-weight: 600;
|
|
65
|
+
color: #0f172a;
|
|
66
|
+
text-indent: 0.32em;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
h1 {
|
|
70
|
+
margin: 0;
|
|
71
|
+
font-size: clamp(40px, 5.2vw, 56px);
|
|
72
|
+
line-height: 1.05;
|
|
73
|
+
letter-spacing: -0.035em;
|
|
74
|
+
color: #0f172a;
|
|
75
|
+
font-weight: 700;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
p { margin: 0; }
|
|
79
|
+
|
|
80
|
+
.subhead { margin-top: 22px; font-size: 18px; line-height: 1.65; }
|
|
81
|
+
|
|
82
|
+
.divider {
|
|
83
|
+
width: 48px;
|
|
84
|
+
height: 2px;
|
|
85
|
+
margin: 28px auto;
|
|
86
|
+
background: #c7d3e5;
|
|
87
|
+
border-radius: 999px;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
.pointer { font-size: 16px; line-height: 1.65; color: #64748b; }
|
|
91
|
+
|
|
92
|
+
.tip {
|
|
93
|
+
margin-top: 36px;
|
|
94
|
+
font-size: 13px;
|
|
95
|
+
line-height: 1.5;
|
|
96
|
+
color: #7c8aa0;
|
|
97
|
+
display: inline-flex;
|
|
98
|
+
align-items: center;
|
|
99
|
+
gap: 8px;
|
|
100
|
+
}
|
|
101
|
+
.tip svg { width: 14px; height: 14px; flex-shrink: 0; opacity: 0.8; }
|
|
102
|
+
|
|
103
|
+
@media (max-width: 720px) {
|
|
104
|
+
main { padding: 10vh 24px 24px; }
|
|
105
|
+
body::before, body::after { width: 120px; height: 96px; }
|
|
106
|
+
body::before { bottom: 4vh; left: 4vw; }
|
|
107
|
+
body::after { top: 4vh; right: 4vw; }
|
|
108
|
+
.logo { width: 60px; height: 60px; }
|
|
109
|
+
.wordmark { font-size: 15px; }
|
|
110
|
+
.subhead { font-size: 16px; }
|
|
111
|
+
.pointer { font-size: 15px; }
|
|
112
|
+
}
|
|
113
|
+
</style>
|
|
114
|
+
</head>
|
|
115
|
+
<body>
|
|
116
|
+
<main>
|
|
117
|
+
<div class="content">
|
|
118
|
+
<div class="brand">
|
|
119
|
+
<div class="logo">${renderLogoSvg()}</div>
|
|
120
|
+
<div class="wordmark">LOADMILL</div>
|
|
121
|
+
</div>
|
|
122
|
+
<h1>Test Browser Ready</h1>
|
|
123
|
+
<p class="subhead">This is the browser the Loadmill Droid CUA agent will drive when you start a test.</p>
|
|
124
|
+
<div class="divider" role="presentation"></div>
|
|
125
|
+
<p class="pointer">Switch back to Loadmill Droid CUA to design or run a test — you should keep this window in the background.</p>
|
|
126
|
+
<p class="tip">
|
|
127
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
|
128
|
+
<circle cx="12" cy="12" r="10"></circle>
|
|
129
|
+
<line x1="12" y1="16" x2="12" y2="12"></line>
|
|
130
|
+
<line x1="12" y1="8" x2="12.01" y2="8"></line>
|
|
131
|
+
</svg>
|
|
132
|
+
Want to explore manually first? Use the address bar above.
|
|
133
|
+
</p>
|
|
134
|
+
</div>
|
|
135
|
+
</main>
|
|
136
|
+
</body>
|
|
137
|
+
</html>`;
|
|
138
|
+
}
|
|
139
|
+
export async function getWelcomePageUrl() {
|
|
140
|
+
if (typeof welcomePageTestHooks?.getWelcomePageUrl === "function") {
|
|
141
|
+
return await welcomePageTestHooks.getWelcomePageUrl();
|
|
142
|
+
}
|
|
143
|
+
if (!ensureWelcomePagePromise) {
|
|
144
|
+
ensureWelcomePagePromise = (async () => {
|
|
145
|
+
await mkdir(WELCOME_PAGE_DIR, { recursive: true });
|
|
146
|
+
await writeFile(WELCOME_PAGE_PATH, buildWelcomePageHtml(), "utf8");
|
|
147
|
+
return WELCOME_PAGE_URL;
|
|
148
|
+
})();
|
|
149
|
+
}
|
|
150
|
+
return await ensureWelcomePagePromise;
|
|
151
|
+
}
|
|
152
|
+
export function isWelcomePageUrl(url) {
|
|
153
|
+
if (typeof welcomePageTestHooks?.isWelcomePageUrl === "function") {
|
|
154
|
+
return welcomePageTestHooks.isWelcomePageUrl(url);
|
|
155
|
+
}
|
|
156
|
+
return typeof url === "string" && url === WELCOME_PAGE_URL;
|
|
157
|
+
}
|
|
158
|
+
export function __setWelcomePageTestHooks(hooks = null) {
|
|
159
|
+
welcomePageTestHooks = hooks;
|
|
160
|
+
ensureWelcomePagePromise = null;
|
|
161
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
3
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
|
|
4
|
+
import { AIM_ASSIST_MODES, DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
4
5
|
import { CUA_MODELS, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
|
|
5
6
|
function createEmptyPromptCustomizations() {
|
|
6
7
|
return {
|
|
@@ -23,15 +24,15 @@ function validateCuaModel(value, label) {
|
|
|
23
24
|
}
|
|
24
25
|
return value;
|
|
25
26
|
}
|
|
27
|
+
function validateAimAssistMode(value, label) {
|
|
28
|
+
if (typeof value !== "string" || !AIM_ASSIST_MODES.includes(value)) {
|
|
29
|
+
throw new Error(`${label} must be one of: ${AIM_ASSIST_MODES.join(", ")}.`);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
26
33
|
function readEnvDefaultCuaModel() {
|
|
27
34
|
return resolveCuaModel(process.env.OPENAI_CUA_MODEL);
|
|
28
35
|
}
|
|
29
|
-
function parseBooleanFlagState(args, key) {
|
|
30
|
-
return {
|
|
31
|
-
enabled: args[key] === true,
|
|
32
|
-
disabled: args[`no-${key}`] === true || args[key] === false,
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
36
|
function parseContextOptimizationThreshold(rawValue, label) {
|
|
36
37
|
const numericValue = (() => {
|
|
37
38
|
if (typeof rawValue === "number") {
|
|
@@ -107,16 +108,19 @@ async function loadConfigFromFile(configPath) {
|
|
|
107
108
|
const normalized = {
|
|
108
109
|
configPath: absoluteConfigPath,
|
|
109
110
|
cuaModel: undefined,
|
|
111
|
+
aimAssistMode: undefined,
|
|
110
112
|
promptCustomizations: createEmptyPromptCustomizations(),
|
|
111
113
|
appContextEnabled: undefined,
|
|
112
114
|
appContextPath: undefined,
|
|
113
115
|
contextOptimizationEnabled: undefined,
|
|
114
116
|
contextOptimizationThreshold: undefined,
|
|
115
|
-
strictMode: undefined,
|
|
116
117
|
};
|
|
117
118
|
if ("cuaModel" in rawConfig) {
|
|
118
119
|
normalized.cuaModel = validateCuaModel(rawConfig.cuaModel, "config.cuaModel");
|
|
119
120
|
}
|
|
121
|
+
if ("aimAssistMode" in rawConfig) {
|
|
122
|
+
normalized.aimAssistMode = validateAimAssistMode(rawConfig.aimAssistMode, "config.aimAssistMode");
|
|
123
|
+
}
|
|
120
124
|
if ("promptCustomizations" in rawConfig) {
|
|
121
125
|
normalized.promptCustomizations = normalizePromptCustomizations(rawConfig.promptCustomizations, "config.promptCustomizations");
|
|
122
126
|
}
|
|
@@ -139,12 +143,7 @@ async function loadConfigFromFile(configPath) {
|
|
|
139
143
|
if ("contextOptimizationThreshold" in rawConfig) {
|
|
140
144
|
normalized.contextOptimizationThreshold = parseContextOptimizationThreshold(rawConfig.contextOptimizationThreshold, "config.contextOptimizationThreshold");
|
|
141
145
|
}
|
|
142
|
-
|
|
143
|
-
if (typeof rawConfig.strictMode !== "boolean") {
|
|
144
|
-
throw new Error("config.strictMode must be a boolean.");
|
|
145
|
-
}
|
|
146
|
-
normalized.strictMode = rawConfig.strictMode;
|
|
147
|
-
}
|
|
146
|
+
// Legacy strictMode is silently accepted and ignored.
|
|
148
147
|
return normalized;
|
|
149
148
|
}
|
|
150
149
|
export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
@@ -155,7 +154,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
155
154
|
const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
|
|
156
155
|
const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
|
|
157
156
|
const hasContextOptimizationThresholdFlag = args["context-optimization-threshold"] != null;
|
|
158
|
-
const { enabled: strictEnabledByFlag, disabled: strictDisabledByFlag } = parseBooleanFlagState(args, "strict");
|
|
159
157
|
const basePromptText = typeof args["base-prompt"] === "string" ? args["base-prompt"] : null;
|
|
160
158
|
const executionPromptText = typeof args["execution-prompt"] === "string" ? args["execution-prompt"] : null;
|
|
161
159
|
const basePromptFilePath = typeof args["base-prompt-file"] === "string" ? path.resolve(cwd, args["base-prompt-file"]) : null;
|
|
@@ -163,9 +161,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
163
161
|
if (explicitContextPath && noContext) {
|
|
164
162
|
throw new Error("--context and --no-context cannot be used together.");
|
|
165
163
|
}
|
|
166
|
-
if (strictEnabledByFlag && strictDisabledByFlag) {
|
|
167
|
-
throw new Error("--strict and --no-strict cannot be used together.");
|
|
168
|
-
}
|
|
169
164
|
if (contextOptimizationEnabledByFlag) {
|
|
170
165
|
throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
|
|
171
166
|
}
|
|
@@ -180,16 +175,19 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
180
175
|
const resolved = {
|
|
181
176
|
configPath: fileConfig?.configPath || null,
|
|
182
177
|
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
178
|
+
aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
|
|
183
179
|
promptCustomizations,
|
|
184
180
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
185
181
|
appContextPath: fileConfig?.appContextPath || null,
|
|
186
182
|
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
187
183
|
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
188
|
-
strictMode: fileConfig?.strictMode ?? false,
|
|
189
184
|
};
|
|
190
185
|
if (typeof args["cua-model"] === "string") {
|
|
191
186
|
resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
|
|
192
187
|
}
|
|
188
|
+
if (typeof args["aim-assist"] === "string") {
|
|
189
|
+
resolved.aimAssistMode = validateAimAssistMode(args["aim-assist"], "--aim-assist");
|
|
190
|
+
}
|
|
193
191
|
if (hasContextOptimizationThresholdFlag) {
|
|
194
192
|
resolved.contextOptimizationEnabled = true;
|
|
195
193
|
resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
|
|
@@ -217,11 +215,5 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
217
215
|
if (noContextOptimization) {
|
|
218
216
|
resolved.contextOptimizationEnabled = false;
|
|
219
217
|
}
|
|
220
|
-
if (strictEnabledByFlag) {
|
|
221
|
-
resolved.strictMode = true;
|
|
222
|
-
}
|
|
223
|
-
else if (strictDisabledByFlag) {
|
|
224
|
-
resolved.strictMode = false;
|
|
225
|
-
}
|
|
226
218
|
return resolved;
|
|
227
219
|
}
|