@loadmill/droid-cua 2.29.0 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +4 -4
  2. package/build/index.js +142 -10
  3. package/build/src/browser/cloud-browser-catalog.js +27 -0
  4. package/build/src/browser/factory.js +33 -10
  5. package/build/src/browser/gesture-dispatch.js +281 -0
  6. package/build/src/browser/interface.js +3 -0
  7. package/build/src/browser/normalize.js +3 -1
  8. package/build/src/browser/playwright/actions.js +5 -271
  9. package/build/src/browser/webdriver/actions.js +10 -0
  10. package/build/src/browser/webdriver/client.js +83 -0
  11. package/build/src/browser/webdriver/connection.js +300 -0
  12. package/build/src/browser/webdriver/page-facade.js +123 -0
  13. package/build/src/browser/webdriver/session-state.js +35 -0
  14. package/build/src/cli/headless-execution-config.js +1 -1
  15. package/build/src/cli/headless-runner.js +26 -13
  16. package/build/src/cli/headless-target.js +87 -16
  17. package/build/src/cli/headless-test-source-label.js +4 -0
  18. package/build/src/cli/loadmill-cloud-device-connect-summary.js +6 -0
  19. package/build/src/cli/loadmill-cloud-session.js +26 -0
  20. package/build/src/core/context-optimization.js +1 -1
  21. package/build/src/core/execution-engine.js +9 -0
  22. package/build/src/core/runtime-config.js +2 -2
  23. package/build/src/desktop-shared/connection-predicates.js +52 -0
  24. package/build/src/device/cloud/loadmill-cloud/adapter.js +17 -12
  25. package/build/src/device/cloud/loadmill-cloud/artifact-downloader.js +13 -7
  26. package/build/src/device/cloud/loadmill-cloud/control-plane-client.js +22 -5
  27. package/build/src/device/cloud/loadmill-cloud/errors.js +25 -0
  28. package/build/src/device/cloud/loadmill-cloud/run-pool.js +41 -0
  29. package/build/src/device/cloud/loadmill-cloud/run-status.js +4 -0
  30. package/package.json +1 -1
  31. package/skill/droid-cua/SKILL.md +10 -3
  32. package/skill/droid-cua/agents/openai.yaml +2 -2
  33. package/skill/droid-cua/references/authoring-tests.md +36 -6
  34. package/skill/droid-cua/references/iterative-improvement.md +143 -0
  35. package/skill/droid-cua/references/onboarding-and-troubleshooting.md +19 -0
  36. package/skill/droid-cua/references/project-and-auth.md +2 -0
  37. package/skill/droid-cua/references/running-and-healing.md +32 -14
@@ -1,272 +1,6 @@
1
- import { logger } from "../../utils/logger.js";
2
- import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
3
- import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
4
- import { buildActionEventMeta } from "../../device/action-event-meta.js";
5
- import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../../device/sensitive-action.js";
6
- import { normalizeAutomationUrl } from "./url.js";
1
+ import { createGestureDispatch } from "../gesture-dispatch.js";
7
2
  import { getActiveBrowserSession, getActiveBrowserPage } from "./session-state.js";
8
- const KEY_ALIASES = {
9
- CTRL: "Control",
10
- CONTROL: "Control",
11
- CMD: "Meta",
12
- COMMAND: "Meta",
13
- META: "Meta",
14
- ALT: "Alt",
15
- OPTION: "Alt",
16
- SHIFT: "Shift",
17
- ENTER: "Enter",
18
- RETURN: "Enter",
19
- ESC: "Escape",
20
- ESCAPE: "Escape",
21
- BACKSPACE: "Backspace",
22
- DELETE: "Delete",
23
- TAB: "Tab",
24
- SPACE: "Space",
25
- HOME: "Home",
26
- END: "End",
27
- PAGEUP: "PageUp",
28
- PAGEDOWN: "PageDown",
29
- ARROWUP: "ArrowUp",
30
- ARROWDOWN: "ArrowDown",
31
- ARROWLEFT: "ArrowLeft",
32
- ARROWRIGHT: "ArrowRight",
33
- };
34
- function normalizeKeyEntry(key) {
35
- const raw = String(key ?? "").trim();
36
- if (!raw) {
37
- throw new Error("Keypress action is missing keys.");
38
- }
39
- const normalized = KEY_ALIASES[raw.toUpperCase()];
40
- if (normalized) {
41
- return normalized;
42
- }
43
- if (raw.length === 1) {
44
- return raw;
45
- }
46
- return raw[0].toUpperCase() + raw.slice(1);
47
- }
48
- function normalizeShortcut(keys = []) {
49
- if (!Array.isArray(keys) || keys.length === 0) {
50
- throw new Error("Keypress action is missing keys.");
51
- }
52
- return keys.map(normalizeKeyEntry).join("+");
53
- }
54
- function buildMeta(context, action, payload = {}) {
55
- return buildActionEventMeta({
56
- context,
57
- action,
58
- platform: "browser",
59
- payload,
60
- });
61
- }
62
- function handleNavigationTimeout(error, description) {
63
- if (!(error instanceof Error) || error.name !== "TimeoutError") {
64
- throw error;
65
- }
66
- console.warn(`Page load did not finish within the timeout for ${description}. Continuing with the current page state.`);
67
- }
68
- async function gotoWithTimeoutTolerance(page, url) {
69
- try {
70
- await page.goto(url, { waitUntil: "load" });
71
- }
72
- catch (error) {
73
- handleNavigationTimeout(error, url);
74
- }
75
- }
76
- async function goBackWithTimeoutTolerance(page) {
77
- try {
78
- await page.goBack({ waitUntil: "load" });
79
- }
80
- catch (error) {
81
- handleNavigationTimeout(error, "browser history back");
82
- }
83
- }
84
- async function goForwardWithTimeoutTolerance(page) {
85
- try {
86
- await page.goForward({ waitUntil: "load" });
87
- }
88
- catch (error) {
89
- handleNavigationTimeout(error, "browser history forward");
90
- }
91
- }
92
- function updateSessionUrlFromPage(session, page) {
93
- const currentUrl = typeof page.url === "function" ? page.url() : null;
94
- if (!currentUrl) {
95
- return;
96
- }
97
- session.pageUrls?.set?.(page, currentUrl);
98
- session.url = currentUrl;
99
- }
100
- export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
101
- const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
102
- const session = getActiveBrowserSession(browserSessionId);
103
- const page = getActiveBrowserPage(browserSessionId);
104
- try {
105
- emitDesktopDebug("browser.action.execute", "device", {
106
- runId: context?.runId,
107
- sessionId: context?.sessionId,
108
- stepId: context?.stepId,
109
- instructionIndex: context?.instructionIndex,
110
- }, {
111
- platform: "browser",
112
- browserSessionId,
113
- actionType: action?.type,
114
- scale,
115
- x: action?.x,
116
- y: action?.y,
117
- x1: action?.x1,
118
- y1: action?.y1,
119
- x2: action?.x2,
120
- y2: action?.y2,
121
- text: buildDebugTextField(action, truncateForDebug),
122
- ...buildSensitiveDebugFields(action),
123
- keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
124
- pathPoints: Array.isArray(action?.path) ? action.path.length : 0,
125
- });
126
- switch (action.type) {
127
- case "click": {
128
- const x = Math.round(action.x / scale);
129
- const y = Math.round(action.y / scale);
130
- addOutput({ type: "action", text: "Click", ...buildMeta(context, action, { x, y, unit: "px" }) });
131
- await page.mouse.click(x, y);
132
- break;
133
- }
134
- case "double_click": {
135
- const x = Math.round(action.x / scale);
136
- const y = Math.round(action.y / scale);
137
- addOutput({ type: "action", text: `Double-clicking at (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
138
- await page.mouse.dblclick(x, y);
139
- break;
140
- }
141
- case "move": {
142
- const x = Math.round(action.x / scale);
143
- const y = Math.round(action.y / scale);
144
- addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
145
- await page.mouse.move(x, y);
146
- break;
147
- }
148
- case "scroll": {
149
- const scrollX = Math.round((action.scroll_x ?? 0) / scale);
150
- const scrollY = Math.round((action.scroll_y ?? 0) / scale);
151
- addOutput({
152
- type: "action",
153
- text: `Scrolling by (${scrollX}, ${scrollY})`,
154
- ...buildMeta(context, action, { scrollX, scrollY, unit: "px" }),
155
- });
156
- await page.mouse.wheel(scrollX, scrollY);
157
- break;
158
- }
159
- case "drag": {
160
- const dragPath = Array.isArray(action.path) ? action.path : [];
161
- if (dragPath.length < 2) {
162
- throw new Error("Drag action requires a path with at least two points.");
163
- }
164
- const [start] = dragPath;
165
- const end = dragPath[dragPath.length - 1];
166
- const scaledStart = { x: Math.round(start.x / scale), y: Math.round(start.y / scale) };
167
- const scaledEnd = { x: Math.round(end.x / scale), y: Math.round(end.y / scale) };
168
- addOutput({
169
- type: "action",
170
- text: `Dragging from (${scaledStart.x}, ${scaledStart.y}) to (${scaledEnd.x}, ${scaledEnd.y})`,
171
- ...buildMeta(context, action, { pathStart: scaledStart, pathEnd: scaledEnd, unit: "px" }),
172
- });
173
- await page.mouse.move(scaledStart.x, scaledStart.y);
174
- await page.mouse.down();
175
- for (const point of dragPath.slice(1)) {
176
- await page.mouse.move(Math.round(point.x / scale), Math.round(point.y / scale));
177
- }
178
- await page.mouse.up();
179
- break;
180
- }
181
- case "type": {
182
- const text = typeof action.text === "string" ? action.text : "";
183
- addOutput({
184
- type: "action",
185
- text: buildTypeActionText(action, text),
186
- ...buildMeta(context, action, buildTypeActionPayload(action, text))
187
- });
188
- await page.keyboard.type(text);
189
- break;
190
- }
191
- case "keypress": {
192
- const shortcut = normalizeShortcut(action.keys);
193
- addOutput({
194
- type: "action",
195
- text: `Pressing key: ${shortcut}`,
196
- ...buildMeta(context, action, { keys: action.keys, shortcut }),
197
- });
198
- await page.keyboard.press(shortcut);
199
- break;
200
- }
201
- case "wait": {
202
- addOutput({ type: "action", text: "Waiting...", ...buildMeta(context, action) });
203
- const timeoutMs = typeof action.ms === "number" && Number.isFinite(action.ms) && action.ms >= 0
204
- ? Math.round(action.ms)
205
- : getConfiguredStepDelayMs();
206
- await page.waitForTimeout(timeoutMs);
207
- break;
208
- }
209
- case "goto": {
210
- const normalizedUrl = normalizeAutomationUrl(action.url);
211
- addOutput({
212
- type: "action",
213
- text: `Navigating to ${normalizedUrl}`,
214
- ...buildMeta(context, action, { url: normalizedUrl }),
215
- });
216
- await gotoWithTimeoutTolerance(page, normalizedUrl);
217
- session.pageUrls?.set?.(page, normalizedUrl);
218
- session.url = normalizedUrl;
219
- break;
220
- }
221
- case "back": {
222
- addOutput({ type: "action", text: "Going back", ...buildMeta(context, action) });
223
- await goBackWithTimeoutTolerance(page);
224
- updateSessionUrlFromPage(session, page);
225
- break;
226
- }
227
- case "forward": {
228
- addOutput({ type: "action", text: "Going forward", ...buildMeta(context, action) });
229
- await goForwardWithTimeoutTolerance(page);
230
- updateSessionUrlFromPage(session, page);
231
- break;
232
- }
233
- default:
234
- addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
235
- }
236
- return { success: true };
237
- }
238
- catch (error) {
239
- const outputError = buildActionErrorOutput(action, error);
240
- logger.error("Action execution error", {
241
- action: actionForErrorLog(action),
242
- message: outputError.message,
243
- stack: isSensitiveAction(action) ? undefined : error.stack,
244
- });
245
- emitDesktopDebug("browser.error", "device", {
246
- runId: context?.runId,
247
- sessionId: context?.sessionId,
248
- stepId: context?.stepId,
249
- instructionIndex: context?.instructionIndex,
250
- }, {
251
- platform: "browser",
252
- operation: "action.execute",
253
- actionType: action?.type,
254
- message: outputError.message,
255
- });
256
- addOutput({
257
- type: "error",
258
- text: outputError.text,
259
- eventType: "error",
260
- actionType: action?.type,
261
- runId: context?.runId,
262
- stepId: context?.stepId,
263
- instructionIndex: context?.instructionIndex,
264
- payload: {
265
- message: outputError.message,
266
- platform: "browser",
267
- },
268
- });
269
- addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
270
- return { success: false, error: outputError.message };
271
- }
272
- }
3
+ export const handleModelAction = createGestureDispatch({
4
+ getSession: getActiveBrowserSession,
5
+ getPage: getActiveBrowserPage,
6
+ });
@@ -0,0 +1,10 @@
1
+ import { createGestureDispatch } from "../gesture-dispatch.js";
2
+ import { getActiveWebDriverPage, getActiveWebDriverSession, reportRunningOnFirstUse, } from "./session-state.js";
3
+ const dispatchModelAction = createGestureDispatch({
4
+ getSession: getActiveWebDriverSession,
5
+ getPage: getActiveWebDriverPage,
6
+ });
7
+ export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
8
+ reportRunningOnFirstUse(browserSessionId);
9
+ return dispatchModelAction(browserSessionId, action, scale, context);
10
+ }
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Thin fetch wrapper over an already-attached W3C WebDriver session. The
3
+ * backend built on this client never issues `POST /session` or
4
+ * `DELETE /session/:id` — it only ever drives a session the Loadmill grid
5
+ * already created.
6
+ */
7
+ const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
8
+ export class WebDriverClient {
9
+ constructor({ url, sessionId, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS }) {
10
+ this.url = url;
11
+ this.sessionId = sessionId;
12
+ this.requestTimeoutMs = requestTimeoutMs;
13
+ }
14
+ async request(method, path, body = null) {
15
+ let response;
16
+ try {
17
+ response = await fetch(`${this.url}${path}`, {
18
+ method,
19
+ headers: body !== null ? { "Content-Type": "application/json" } : {},
20
+ body: body !== null ? JSON.stringify(body) : undefined,
21
+ signal: AbortSignal.timeout(this.requestTimeoutMs),
22
+ });
23
+ }
24
+ catch (error) {
25
+ if (error?.name === "AbortError" || error?.name === "TimeoutError") {
26
+ throw new Error(`WebDriver request to session ${this.sessionId} timed out after ${this.requestTimeoutMs}ms; the grid is not responding.`);
27
+ }
28
+ throw error;
29
+ }
30
+ const text = await response.text();
31
+ const payload = text ? JSON.parse(text) : {};
32
+ if (!response.ok) {
33
+ const message = payload?.value?.message ||
34
+ payload?.message ||
35
+ `WebDriver request failed with status ${response.status}.`;
36
+ throw new Error(message);
37
+ }
38
+ return payload;
39
+ }
40
+ async navigateTo(url) {
41
+ await this.request("POST", `/session/${this.sessionId}/url`, { url });
42
+ }
43
+ async goBack() {
44
+ await this.request("POST", `/session/${this.sessionId}/back`, {});
45
+ }
46
+ async goForward() {
47
+ await this.request("POST", `/session/${this.sessionId}/forward`, {});
48
+ }
49
+ async getScreenshot() {
50
+ const response = await this.request("GET", `/session/${this.sessionId}/screenshot`);
51
+ return response.value;
52
+ }
53
+ async setWindowRect({ width, height }) {
54
+ await this.request("POST", `/session/${this.sessionId}/window/rect`, {
55
+ x: 0,
56
+ y: 0,
57
+ width,
58
+ height,
59
+ });
60
+ }
61
+ async getWindowHandle() {
62
+ const response = await this.request("GET", `/session/${this.sessionId}/window`);
63
+ return response.value;
64
+ }
65
+ async getWindowHandles() {
66
+ const response = await this.request("GET", `/session/${this.sessionId}/window/handles`);
67
+ return response.value;
68
+ }
69
+ async switchToWindow(handle) {
70
+ await this.request("POST", `/session/${this.sessionId}/window`, { handle });
71
+ }
72
+ async closeWindow() {
73
+ const response = await this.request("DELETE", `/session/${this.sessionId}/window`);
74
+ return response.value;
75
+ }
76
+ async performActions(actions) {
77
+ await this.request("POST", `/session/${this.sessionId}/actions`, { actions });
78
+ }
79
+ async executeScript(script, args = []) {
80
+ const response = await this.request("POST", `/session/${this.sessionId}/execute/sync`, { script, args });
81
+ return response.value;
82
+ }
83
+ }
@@ -0,0 +1,300 @@
1
+ import sharp from "sharp";
2
+ import { normalizeAutomationUrl } from "../playwright/url.js";
3
+ import { normalizeBrowserInfoForSession } from "../normalize.js";
4
+ import { LoadmillCloudControlPlaneClient } from "../../device/cloud/loadmill-cloud/control-plane-client.js";
5
+ import { fetchAndDownloadArtifacts } from "../../device/cloud/loadmill-cloud/adapter.js";
6
+ import { ARTIFACT_READY_STATUSES, TERMINAL_STATUSES } from "../../device/cloud/loadmill-cloud/run-status.js";
7
+ import { reportCloudPhase, reportCloudProgress } from "../../device/cloud/progress.js";
8
+ import { emitDesktopDebug } from "../../utils/desktop-debug.js";
9
+ import { WebDriverClient } from "./client.js";
10
+ import { createWebDriverPageFacade } from "./page-facade.js";
11
+ import { clearActiveWebDriverSession, getActiveWebDriverSession, reportRunningOnFirstUse, setActiveWebDriverSession, } from "./session-state.js";
12
+ export { handleModelAction } from "./actions.js";
13
+ const DEFAULT_VIEWPORT_WIDTH = 1440;
14
+ const DEFAULT_VIEWPORT_HEIGHT = 900;
15
+ const LOADMILL_CLOUD_VIEWPORT = { width: DEFAULT_VIEWPORT_WIDTH, height: DEFAULT_VIEWPORT_HEIGHT };
16
+ const VIEWPORT_METRICS_SCRIPT = "return { innerWidth: window.innerWidth, innerHeight: window.innerHeight, devicePixelRatio: window.devicePixelRatio };";
17
+ const SESSION_PROBE_SCRIPT = "return true;";
18
+ const TEARDOWN_STOP_BUDGET_MS = 1_500;
19
+ async function setCloudViewport(client) {
20
+ await client.setWindowRect(LOADMILL_CLOUD_VIEWPORT);
21
+ const metrics = await client.executeScript(VIEWPORT_METRICS_SCRIPT, []);
22
+ const viewportWidth = Number(metrics?.innerWidth) || LOADMILL_CLOUD_VIEWPORT.width;
23
+ const viewportHeight = Number(metrics?.innerHeight) || LOADMILL_CLOUD_VIEWPORT.height;
24
+ const adjustedWindowRect = {
25
+ width: LOADMILL_CLOUD_VIEWPORT.width + (LOADMILL_CLOUD_VIEWPORT.width - viewportWidth),
26
+ height: LOADMILL_CLOUD_VIEWPORT.height + (LOADMILL_CLOUD_VIEWPORT.height - viewportHeight),
27
+ };
28
+ if (adjustedWindowRect.width !== LOADMILL_CLOUD_VIEWPORT.width ||
29
+ adjustedWindowRect.height !== LOADMILL_CLOUD_VIEWPORT.height) {
30
+ await client.setWindowRect(adjustedWindowRect);
31
+ }
32
+ }
33
+ async function attachWebDriverSession({ webdriverEndpoint, webdriverSessionId, cloud = null }) {
34
+ const client = new WebDriverClient({ url: webdriverEndpoint, sessionId: webdriverSessionId });
35
+ if (cloud) {
36
+ await setCloudViewport(client);
37
+ }
38
+ const windowHandle = await client.getWindowHandle();
39
+ const page = createWebDriverPageFacade({ client });
40
+ const session = {
41
+ sessionId: webdriverSessionId,
42
+ client,
43
+ page,
44
+ pageUrls: new Map(),
45
+ url: null,
46
+ cloud,
47
+ windowStack: [windowHandle],
48
+ activeWindowHandle: windowHandle,
49
+ };
50
+ setActiveWebDriverSession(session);
51
+ return webdriverSessionId;
52
+ }
53
+ async function syncActiveWindow(session) {
54
+ const handles = await session.client.getWindowHandles();
55
+ const openHandles = new Set(handles);
56
+ session.windowStack = session.windowStack.filter((handle) => openHandles.has(handle));
57
+ const knownHandles = new Set(session.windowStack);
58
+ const newHandles = handles.filter((handle) => !knownHandles.has(handle));
59
+ session.windowStack.push(...newHandles);
60
+ const nextHandle = newHandles.at(-1) ?? (openHandles.has(session.activeWindowHandle) ? null : session.windowStack.at(-1));
61
+ if (nextHandle) {
62
+ await session.client.switchToWindow(nextHandle);
63
+ session.activeWindowHandle = nextHandle;
64
+ }
65
+ }
66
+ async function probeAttachedWebDriverSession({ webdriverEndpoint, webdriverSessionId }, cloudSessionId) {
67
+ const probeClient = new WebDriverClient({ url: webdriverEndpoint, sessionId: webdriverSessionId });
68
+ try {
69
+ await probeClient.executeScript(SESSION_PROBE_SCRIPT, []);
70
+ }
71
+ catch (error) {
72
+ throw new Error(`Loadmill Cloud web session expired: session ${cloudSessionId}'s attached browser is no longer reachable — ${error.message}`);
73
+ }
74
+ }
75
+ function formatQueueMessage(state) {
76
+ const parts = [];
77
+ if (typeof state.queuePosition === "number") {
78
+ parts.push(`queue position ${state.queuePosition}`);
79
+ }
80
+ if (typeof state.etaSeconds === "number") {
81
+ parts.push(`up to ${Math.max(1, Math.round(state.etaSeconds / 60))}m`);
82
+ }
83
+ return parts.length > 0 ? parts.join(", ") : "waiting for a browser slot";
84
+ }
85
+ function waitForProvisionedWebDriverSession(client, sessionId, { signal } = {}) {
86
+ return new Promise((resolve, reject) => {
87
+ const watchAbort = new AbortController();
88
+ let lastStatus = null;
89
+ let settled = false;
90
+ const settle = (fn, value) => {
91
+ if (settled) {
92
+ return;
93
+ }
94
+ settled = true;
95
+ watchAbort.abort();
96
+ fn(value);
97
+ };
98
+ if (signal?.aborted) {
99
+ settle(reject, signal.reason ?? new Error("Provisioning cancelled."));
100
+ return;
101
+ }
102
+ signal?.addEventListener("abort", () => settle(reject, signal.reason ?? new Error("Provisioning cancelled.")), { once: true });
103
+ client
104
+ .watchRun(sessionId, (state) => {
105
+ if (!state || typeof state !== "object") {
106
+ return;
107
+ }
108
+ if (state.status && state.status !== lastStatus) {
109
+ lastStatus = state.status;
110
+ reportCloudPhase(state.status.toLowerCase());
111
+ }
112
+ if (state.status === "PENDING") {
113
+ reportCloudProgress(formatQueueMessage(state));
114
+ }
115
+ if (state.status === "PROVISIONED" && state.webdriverEndpoint) {
116
+ settle(resolve, {
117
+ webdriverEndpoint: state.webdriverEndpoint,
118
+ webdriverSessionId: state.webdriverSessionId,
119
+ });
120
+ return;
121
+ }
122
+ if (state.status === "ERROR") {
123
+ settle(reject, new Error(state.error?.message || "Loadmill Cloud web run failed while provisioning."));
124
+ return;
125
+ }
126
+ if (state.status === "CANCELLED") {
127
+ settle(reject, new Error("Loadmill Cloud web run was cancelled."));
128
+ }
129
+ }, { signal: watchAbort.signal })
130
+ .catch(() => { });
131
+ });
132
+ }
133
+ async function connectToBrowserViaLoadmillCloud({ credentials, browserFamily, browserVersion, cloudSessionId, artifacts, signal, }) {
134
+ const client = new LoadmillCloudControlPlaneClient({
135
+ bearerToken: credentials?.bearerToken,
136
+ baseUrl: credentials?.baseUrl,
137
+ pool: "web",
138
+ });
139
+ if (cloudSessionId) {
140
+ const state = await client.getRunState(cloudSessionId);
141
+ if (ARTIFACT_READY_STATUSES.has(state.status) && Array.isArray(artifacts) && artifacts.length > 0) {
142
+ await fetchAndDownloadArtifacts({ client, sessionId: cloudSessionId, kinds: artifacts });
143
+ const exitError = new Error("Loadmill Cloud artifact fetch complete.");
144
+ exitError.artifactsOnlyExit = true;
145
+ throw exitError;
146
+ }
147
+ if (TERMINAL_STATUSES.has(state.status)) {
148
+ throw new Error(`Loadmill Cloud web session ${cloudSessionId} is already terminal (${state.status}). ` +
149
+ `Fetch its recordings/logs instead with: droid-cua --target web --cloud-session-id ${cloudSessionId} --artifacts all`);
150
+ }
151
+ if (!state.webdriverEndpoint || !state.webdriverSessionId) {
152
+ throw new Error(`Loadmill Cloud web session ${cloudSessionId} is not yet ready to reconnect (status: ${state.status}).`);
153
+ }
154
+ await probeAttachedWebDriverSession(state, cloudSessionId);
155
+ if (state.status === "PROVISIONED") {
156
+ await client.reportStatus(cloudSessionId, "CONNECTED");
157
+ }
158
+ client.startHeartbeat(cloudSessionId);
159
+ try {
160
+ return await attachWebDriverSession({
161
+ webdriverEndpoint: state.webdriverEndpoint,
162
+ webdriverSessionId: state.webdriverSessionId,
163
+ cloud: { client, sessionId: cloudSessionId, runningReported: state.status === "RUNNING", artifacts },
164
+ });
165
+ }
166
+ catch (error) {
167
+ client.stopHeartbeat();
168
+ throw error;
169
+ }
170
+ }
171
+ reportCloudPhase("request browser");
172
+ const initState = await client.initRun({ platform: "web", deviceName: browserFamily, osVersion: browserVersion });
173
+ const sessionId = initState.sessionId;
174
+ // Server reaper kills runs that only heartbeat after CONNECTED, so start now.
175
+ client.startHeartbeat(sessionId);
176
+ try {
177
+ const { webdriverEndpoint, webdriverSessionId } = await waitForProvisionedWebDriverSession(client, sessionId, {
178
+ signal,
179
+ });
180
+ await client.reportStatus(sessionId, "CONNECTED");
181
+ return await attachWebDriverSession({
182
+ webdriverEndpoint,
183
+ webdriverSessionId,
184
+ cloud: { client, sessionId, runningReported: false, artifacts },
185
+ });
186
+ }
187
+ catch (error) {
188
+ client.stopHeartbeat();
189
+ await client.stopRun(sessionId, { signal: AbortSignal.timeout(TEARDOWN_STOP_BUDGET_MS) }).catch(() => { });
190
+ throw error;
191
+ }
192
+ }
193
+ export async function connectToBrowser(options = {}) {
194
+ if (options.source === "loadmill-cloud") {
195
+ return connectToBrowserViaLoadmillCloud(options);
196
+ }
197
+ const { url, sessionId } = options;
198
+ if (!url || !sessionId) {
199
+ throw new Error("Attaching to a WebDriver session requires a url and a sessionId.");
200
+ }
201
+ return attachWebDriverSession({ webdriverEndpoint: url, webdriverSessionId: sessionId, cloud: null });
202
+ }
203
+ export async function getBrowserInfo(browserSessionId) {
204
+ const session = getActiveWebDriverSession(browserSessionId);
205
+ reportRunningOnFirstUse(browserSessionId);
206
+ await syncActiveWindow(session);
207
+ const metrics = await session.client.executeScript(VIEWPORT_METRICS_SCRIPT, []);
208
+ const viewportWidth = Number(metrics?.innerWidth) || DEFAULT_VIEWPORT_WIDTH;
209
+ const viewportHeight = Number(metrics?.innerHeight) || DEFAULT_VIEWPORT_HEIGHT;
210
+ const devicePixelRatio = Number(metrics?.devicePixelRatio) || 1;
211
+ return normalizeBrowserInfoForSession({
212
+ viewportWidth,
213
+ viewportHeight,
214
+ scaledWidth: viewportWidth,
215
+ scaledHeight: viewportHeight,
216
+ scale: 1,
217
+ devicePixelRatio,
218
+ currentUrl: session.url,
219
+ });
220
+ }
221
+ export async function getScreenshotAsBase64(browserSessionId, browserInfo = {}) {
222
+ const session = getActiveWebDriverSession(browserSessionId);
223
+ const currentBrowserInfo = await getBrowserInfo(browserSessionId);
224
+ Object.assign(browserInfo, currentBrowserInfo);
225
+ const rawBase64 = await session.client.getScreenshot();
226
+ let buffer = Buffer.from(rawBase64, "base64");
227
+ if (currentBrowserInfo.scaledWidth && currentBrowserInfo.scaledHeight) {
228
+ buffer = await sharp(buffer)
229
+ .resize({
230
+ width: currentBrowserInfo.scaledWidth,
231
+ height: currentBrowserInfo.scaledHeight,
232
+ fit: "fill",
233
+ })
234
+ .png()
235
+ .toBuffer();
236
+ }
237
+ return buffer.toString("base64");
238
+ }
239
+ export async function navigateToUrl(browserSessionId, url) {
240
+ const session = getActiveWebDriverSession(browserSessionId);
241
+ reportRunningOnFirstUse(browserSessionId);
242
+ const normalizedUrl = normalizeAutomationUrl(url);
243
+ await session.page.goto(normalizedUrl, { waitUntil: "load" });
244
+ session.pageUrls.set(session.page, normalizedUrl);
245
+ session.url = normalizedUrl;
246
+ return normalizedUrl;
247
+ }
248
+ export async function closeCurrentTab(browserSessionId) {
249
+ const session = getActiveWebDriverSession(browserSessionId);
250
+ await syncActiveWindow(session);
251
+ if (session.windowStack.length <= 1) {
252
+ return {
253
+ closed: false,
254
+ restoredUrl: null,
255
+ message: "The primary browser tab cannot be closed.",
256
+ };
257
+ }
258
+ await session.client.closeWindow();
259
+ session.windowStack.pop();
260
+ session.activeWindowHandle = session.windowStack.at(-1);
261
+ await session.client.switchToWindow(session.activeWindowHandle);
262
+ const restoredUrl = await session.page.url();
263
+ session.url = restoredUrl;
264
+ const message = restoredUrl
265
+ ? `Closed current tab. Returned to ${restoredUrl}.`
266
+ : "Closed current tab. Returned to the previous browser tab.";
267
+ emitDesktopDebug("browser.tab.close_current", "device", { browserSessionId }, {
268
+ platform: "browser",
269
+ closed: true,
270
+ restoredUrl,
271
+ stackDepth: session.windowStack.length,
272
+ });
273
+ return {
274
+ closed: true,
275
+ restoredUrl,
276
+ message,
277
+ };
278
+ }
279
+ export async function disconnect() {
280
+ let session = null;
281
+ try {
282
+ session = getActiveWebDriverSession();
283
+ }
284
+ catch {
285
+ session = null;
286
+ }
287
+ clearActiveWebDriverSession();
288
+ if (session?.cloud) {
289
+ const { client, sessionId, artifacts } = session.cloud;
290
+ try {
291
+ await client.stopRun(sessionId);
292
+ if (Array.isArray(artifacts) && artifacts.length > 0) {
293
+ await fetchAndDownloadArtifacts({ client, sessionId, kinds: artifacts });
294
+ }
295
+ }
296
+ finally {
297
+ client.stopHeartbeat();
298
+ }
299
+ }
300
+ }