@loadmill/droid-cua 2.13.0 → 2.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/build/index.js +39 -149
- package/build/src/browser/connection.js +6 -0
- package/build/src/browser/factory.js +1 -0
- package/build/src/browser/playwright/connection.js +117 -18
- package/build/src/browser/playwright/session-state.js +8 -1
- package/build/src/cli/headless-instruction-target.js +64 -0
- package/build/src/cli/headless-runner.js +245 -0
- package/build/src/commands/loadmill.js +5 -21
- package/build/src/core/execution-engine.js +5 -1
- package/build/src/core/tool-handlers/browser-session.js +7 -0
- package/build/src/core/tool-handlers/close-current-tab.js +53 -0
- package/build/src/core/tool-handlers/navigate.js +2 -5
- package/build/src/device/android/connection.js +24 -13
- package/build/src/device/cloud/connection.js +4 -3
- package/build/src/device/connection.js +9 -7
- package/build/src/device/ios/appium-server.js +5 -4
- package/build/src/device/ios/connection.js +19 -15
- package/build/src/device/loadmill.js +41 -78
- package/build/src/device/openai.js +16 -1
- package/build/src/integrations/loadmill/executor.js +36 -21
- package/build/src/integrations/loadmill/interpreter.js +1 -2
- package/build/src/modes/design-mode-ink.js +1 -0
- package/build/src/modes/design-mode.js +1 -0
- package/build/src/modes/execution-mode.js +1 -0
- package/build/src/prompts/base.js +1 -0
- package/build/src/prompts/execution.js +1 -1
- package/build/src/prompts/loadmill.js +4 -8
- package/package.json +4 -1
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { mkdir } from "fs/promises";
|
|
3
|
+
import { getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "../browser/connection.js";
|
|
4
|
+
import { handleModelAction as handleBrowserModelAction } from "../browser/actions.js";
|
|
5
|
+
import { ExecutionEngine } from "../core/execution-engine.js";
|
|
6
|
+
import { buildExecutionModePrompt } from "../core/prompts.js";
|
|
7
|
+
import { ExecutionMode } from "../modes/execution-mode.js";
|
|
8
|
+
import { printCliOutput } from "../utils/console-output.js";
|
|
9
|
+
import { emitDesktopDebug } from "../utils/desktop-debug.js";
|
|
10
|
+
import { loadHeadlessInstructions } from "./headless-instructions.js";
|
|
11
|
+
function sanitizePathSegment(value) {
|
|
12
|
+
return String(value || "test")
|
|
13
|
+
.replace(/\.dcua$/i, "")
|
|
14
|
+
.trim()
|
|
15
|
+
.replace(/[^a-zA-Z0-9._-]+/g, "-")
|
|
16
|
+
.replace(/-+/g, "-")
|
|
17
|
+
.replace(/^[-_.]+|[-_.]+$/g, "") || "test";
|
|
18
|
+
}
|
|
19
|
+
function createHeadlessRunId(index = 0) {
|
|
20
|
+
const suffix = index > 0 ? `-${index + 1}` : "";
|
|
21
|
+
return `run-${Date.now()}${suffix}`;
|
|
22
|
+
}
|
|
23
|
+
function getRecordScreenshotDirForTest(baseScreenshotDir, testName, isBatchRun) {
|
|
24
|
+
if (!baseScreenshotDir || !isBatchRun) {
|
|
25
|
+
return baseScreenshotDir;
|
|
26
|
+
}
|
|
27
|
+
return path.join(baseScreenshotDir, sanitizePathSegment(testName));
|
|
28
|
+
}
|
|
29
|
+
function emitCliAppContextDebug(runId, appContext, appContextText) {
|
|
30
|
+
if (appContext.status === "loaded" || appContext.status === "empty") {
|
|
31
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
32
|
+
source: "cli_context_flag",
|
|
33
|
+
contextPath: appContext.contextPath,
|
|
34
|
+
status: appContext.status,
|
|
35
|
+
text: appContextText
|
|
36
|
+
});
|
|
37
|
+
if (appContextText) {
|
|
38
|
+
console.log(`Using app context from: ${appContext.contextPath}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else if (appContext.status === "failed") {
|
|
42
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
43
|
+
source: "cli_context_flag",
|
|
44
|
+
contextPath: appContext.contextPath,
|
|
45
|
+
status: "failed",
|
|
46
|
+
message: appContext.message
|
|
47
|
+
});
|
|
48
|
+
console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
52
|
+
source: "cli_context_flag",
|
|
53
|
+
contextPath: appContext.contextPath,
|
|
54
|
+
status: appContext.status
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
59
|
+
const executionEngineOptions = {
|
|
60
|
+
toolCapabilities: {
|
|
61
|
+
assert: true,
|
|
62
|
+
navigate: isBrowserTarget,
|
|
63
|
+
loadmill: true,
|
|
64
|
+
},
|
|
65
|
+
baseSystemPrompt: executionPrompt,
|
|
66
|
+
isHeadlessMode: true,
|
|
67
|
+
};
|
|
68
|
+
const engineOptions = screenshotRecorder
|
|
69
|
+
? {
|
|
70
|
+
recordScreenshots: true,
|
|
71
|
+
screenshotRecorder,
|
|
72
|
+
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
73
|
+
sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
|
|
74
|
+
...requestArgs,
|
|
75
|
+
aimAssistMode: headlessConfig.aimAssistMode,
|
|
76
|
+
}),
|
|
77
|
+
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
78
|
+
...executionEngineOptions,
|
|
79
|
+
}
|
|
80
|
+
: {
|
|
81
|
+
recordScreenshots,
|
|
82
|
+
screenshotDir,
|
|
83
|
+
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
84
|
+
sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
|
|
85
|
+
...requestArgs,
|
|
86
|
+
aimAssistMode: headlessConfig.aimAssistMode,
|
|
87
|
+
}),
|
|
88
|
+
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
89
|
+
...executionEngineOptions,
|
|
90
|
+
};
|
|
91
|
+
if (isBrowserTarget) {
|
|
92
|
+
engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
|
|
93
|
+
engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
|
|
94
|
+
engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
|
|
95
|
+
}
|
|
96
|
+
else if (isLambdaTestTarget) {
|
|
97
|
+
engineOptions.connectToDevice = async () => {
|
|
98
|
+
throw new Error("LambdaTest cloud session disconnected; automatic CLI reconnect is not supported yet.");
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return engineOptions;
|
|
102
|
+
}
|
|
103
|
+
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
104
|
+
let executionMode = null;
|
|
105
|
+
try {
|
|
106
|
+
await headlessDebug.startExecutionSession(runId, {
|
|
107
|
+
testName,
|
|
108
|
+
platform: deviceInfo.platform,
|
|
109
|
+
deviceName: deviceInfo.device_name,
|
|
110
|
+
aimAssistMode: headlessConfig.aimAssistMode
|
|
111
|
+
});
|
|
112
|
+
const currentLogFilePath = await headlessDebug.getCurrentLogFilePath();
|
|
113
|
+
if (currentLogFilePath) {
|
|
114
|
+
console.log(`Debug logging enabled: ${currentLogFilePath}`);
|
|
115
|
+
}
|
|
116
|
+
console.log(`\nRunning test from: ${testPath}\n`);
|
|
117
|
+
console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
|
|
118
|
+
const appContextText = appContext.text;
|
|
119
|
+
emitCliAppContextDebug(runId, appContext, appContextText);
|
|
120
|
+
session.updateResponseId(undefined);
|
|
121
|
+
session.clearMessages();
|
|
122
|
+
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
|
|
123
|
+
session.setSystemPrompt(executionPrompt);
|
|
124
|
+
const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
|
|
125
|
+
runId,
|
|
126
|
+
recordScreenshots,
|
|
127
|
+
screenshotDir
|
|
128
|
+
});
|
|
129
|
+
const engineOptions = buildHeadlessEngineOptions({
|
|
130
|
+
runtimeContext,
|
|
131
|
+
headlessConfig,
|
|
132
|
+
executionPrompt,
|
|
133
|
+
screenshotRecorder,
|
|
134
|
+
recordScreenshots,
|
|
135
|
+
screenshotDir,
|
|
136
|
+
isBrowserTarget,
|
|
137
|
+
isLambdaTestTarget
|
|
138
|
+
});
|
|
139
|
+
const engine = new ExecutionEngine(session, engineOptions);
|
|
140
|
+
executionMode = new ExecutionMode(session, engine, instructions, true);
|
|
141
|
+
const result = await executionMode.execute({
|
|
142
|
+
runId,
|
|
143
|
+
addOutput: printCliOutput,
|
|
144
|
+
openAiClient: runtimeContext.openAiClient,
|
|
145
|
+
loadmillClient: runtimeContext.loadmillClient,
|
|
146
|
+
runtimeCredentials: runtimeContext.credentials,
|
|
147
|
+
});
|
|
148
|
+
const stats = executionMode.stats || {};
|
|
149
|
+
const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
|
|
150
|
+
await headlessDebug.endExecutionSession(runId, {
|
|
151
|
+
success: Boolean(result.success),
|
|
152
|
+
error: result.error ?? null,
|
|
153
|
+
durationMs,
|
|
154
|
+
instructionsTotal: instructions.length,
|
|
155
|
+
instructionsCompleted: stats.instructionsCompleted ?? 0,
|
|
156
|
+
actionsTotal: stats.actionCount ?? 0,
|
|
157
|
+
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
158
|
+
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
159
|
+
retries: stats.retryCount ?? 0
|
|
160
|
+
});
|
|
161
|
+
if (!result.success) {
|
|
162
|
+
console.error(`\nTest failed: ${result.error}`);
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
success: Boolean(result.success),
|
|
166
|
+
error: result.error ?? null
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
const message = error instanceof Error ? error.message : "Failed to start execution.";
|
|
171
|
+
const stats = executionMode?.stats || {};
|
|
172
|
+
const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
|
|
173
|
+
await headlessDebug.endExecutionSession(runId, {
|
|
174
|
+
success: false,
|
|
175
|
+
error: message,
|
|
176
|
+
...(stats.startTime
|
|
177
|
+
? {
|
|
178
|
+
durationMs,
|
|
179
|
+
instructionsTotal: instructions.length,
|
|
180
|
+
instructionsCompleted: stats.instructionsCompleted ?? 0,
|
|
181
|
+
actionsTotal: stats.actionCount ?? 0,
|
|
182
|
+
assertionsPassed: stats.assertionsPassed ?? 0,
|
|
183
|
+
assertionsFailed: stats.assertionsFailed ?? 0,
|
|
184
|
+
retries: stats.retryCount ?? 0
|
|
185
|
+
}
|
|
186
|
+
: { reason: "start_failed" })
|
|
187
|
+
});
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
192
|
+
const isBatchRun = instructionsTarget?.type === "directory";
|
|
193
|
+
if (isBatchRun) {
|
|
194
|
+
console.log(`\nRunning ${runs.length} tests from: ${instructionsTarget.path}\n`);
|
|
195
|
+
}
|
|
196
|
+
let failures = 0;
|
|
197
|
+
for (const [index, run] of runs.entries()) {
|
|
198
|
+
const runId = createHeadlessRunId(index);
|
|
199
|
+
const testScreenshotDir = getRecordScreenshotDirForTest(screenshotDir, run.testName, isBatchRun);
|
|
200
|
+
if (isBatchRun) {
|
|
201
|
+
console.log(`\n[${index + 1}/${runs.length}] ${run.testName}`);
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
const instructions = run.instructions ?? await loadHeadlessInstructions(run.testPath);
|
|
205
|
+
if (recordScreenshots && testScreenshotDir) {
|
|
206
|
+
await mkdir(testScreenshotDir, { recursive: true });
|
|
207
|
+
}
|
|
208
|
+
const result = await runHeadlessTest({
|
|
209
|
+
runId,
|
|
210
|
+
testPath: run.testPath,
|
|
211
|
+
testName: run.testName,
|
|
212
|
+
instructions,
|
|
213
|
+
session,
|
|
214
|
+
deviceInfo,
|
|
215
|
+
headlessConfig,
|
|
216
|
+
appContext,
|
|
217
|
+
runtimeContext,
|
|
218
|
+
headlessDebug,
|
|
219
|
+
recordScreenshots,
|
|
220
|
+
screenshotDir: testScreenshotDir,
|
|
221
|
+
isBrowserTarget,
|
|
222
|
+
isLambdaTestTarget
|
|
223
|
+
});
|
|
224
|
+
if (!result.success) {
|
|
225
|
+
failures += 1;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
failures += 1;
|
|
230
|
+
if (!isBatchRun) {
|
|
231
|
+
throw error;
|
|
232
|
+
}
|
|
233
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
234
|
+
console.error(`\nTest failed: ${run.testName}: ${message}`);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
if (isBatchRun) {
|
|
238
|
+
const passed = runs.length - failures;
|
|
239
|
+
console.log(`\nFolder run complete: ${passed}/${runs.length} passed`);
|
|
240
|
+
}
|
|
241
|
+
return {
|
|
242
|
+
exitCode: failures === 0 ? 0 : 1,
|
|
243
|
+
failures
|
|
244
|
+
};
|
|
245
|
+
}
|
|
@@ -21,12 +21,12 @@ export async function handleLoadmill(args, session, context) {
|
|
|
21
21
|
addOutput({ type: 'info', text: ' /loadmill <command> Execute a Loadmill flow' });
|
|
22
22
|
addOutput({ type: 'info', text: '' });
|
|
23
23
|
addOutput({ type: 'info', text: 'Examples:' });
|
|
24
|
-
addOutput({ type: 'info', text: ' /loadmill
|
|
24
|
+
addOutput({ type: 'info', text: ' /loadmill login flow' });
|
|
25
25
|
addOutput({ type: 'info', text: ' /loadmill run checkout flow with user=test123' });
|
|
26
26
|
addOutput({ type: 'info', text: ' /loadmill run payment test with amount=100' });
|
|
27
27
|
addOutput({ type: 'info', text: '' });
|
|
28
28
|
addOutput({ type: 'info', text: 'In test scripts (.dcua files):' });
|
|
29
|
-
addOutput({ type: 'info', text: ' loadmill
|
|
29
|
+
addOutput({ type: 'info', text: ' run loadmill user authentication flow' });
|
|
30
30
|
addOutput({ type: 'info', text: '' });
|
|
31
31
|
addOutput({ type: 'info', text: 'Configuration:' });
|
|
32
32
|
addOutput({ type: 'info', text: ' Set LOADMILL_API_TOKEN in your .env file' });
|
|
@@ -68,25 +68,9 @@ export async function handleLoadmill(args, session, context) {
|
|
|
68
68
|
}
|
|
69
69
|
// Display results
|
|
70
70
|
if (result.success) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
const name = flow.description || flow.name || "Unknown";
|
|
75
|
-
const suite = flow.testSuiteDescription ? ` (Suite: ${flow.testSuiteDescription})` : '';
|
|
76
|
-
addOutput({ type: 'info', text: ` ${i + 1}. ${name}${suite}` });
|
|
77
|
-
addOutput({ type: 'info', text: ` ID: ${flow.id}` });
|
|
78
|
-
});
|
|
79
|
-
if (result.result.selectedFlow) {
|
|
80
|
-
const selectedName = result.result.selectedFlow.description || result.result.selectedFlow.name || "Unknown";
|
|
81
|
-
addOutput({ type: 'info', text: '' });
|
|
82
|
-
addOutput({ type: 'info', text: `Best match: "${selectedName}" (confidence: ${(result.result.confidence * 100).toFixed(0)}%)` });
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
|
|
87
|
-
if (result.runId) {
|
|
88
|
-
addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
|
|
89
|
-
}
|
|
71
|
+
addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
|
|
72
|
+
if (result.runId) {
|
|
73
|
+
addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
|
|
90
74
|
}
|
|
91
75
|
}
|
|
92
76
|
else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
|
|
3
|
-
import { getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
|
|
3
|
+
import { closeCurrentTab as closeCurrentBrowserTab, getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
|
|
4
4
|
import { handleModelAction } from "../device/actions.js";
|
|
5
5
|
import { sendCUARequest } from "../device/openai.js";
|
|
6
6
|
import { executeLoadmillInstruction } from "../device/loadmill.js";
|
|
@@ -11,6 +11,7 @@ import { CAPTURE_SOURCES } from "./capture-sources.js";
|
|
|
11
11
|
import { ScreenshotStore } from "./screenshot-store.js";
|
|
12
12
|
import { handlePreviewClick } from "./tool-handlers/preview-click.js";
|
|
13
13
|
import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
|
|
14
|
+
import { handleCloseCurrentTabToolCall } from "./tool-handlers/close-current-tab.js";
|
|
14
15
|
import { handleAssertionToolCall } from "./tool-handlers/assert.js";
|
|
15
16
|
import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
|
|
16
17
|
import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
|
|
@@ -73,6 +74,7 @@ export class ExecutionEngine {
|
|
|
73
74
|
this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
|
|
74
75
|
this.getBrowserInfo = options.getBrowserInfo || getBrowserInfo;
|
|
75
76
|
this.navigateToUrl = options.navigateToUrl || navigateBrowserToUrl;
|
|
77
|
+
this.closeCurrentTab = options.closeCurrentTab || closeCurrentBrowserTab;
|
|
76
78
|
this.executeLoadmillInstruction = options.executeLoadmillInstruction || executeLoadmillInstruction;
|
|
77
79
|
this.toolCapabilities = options.toolCapabilities || {};
|
|
78
80
|
this.baseSystemPrompt = options.baseSystemPrompt || session?.systemPrompt || null;
|
|
@@ -102,6 +104,8 @@ export class ExecutionEngine {
|
|
|
102
104
|
return await handleAssertionToolCall(handlerArgs);
|
|
103
105
|
case "navigate":
|
|
104
106
|
return await handleNavigateToolCall(handlerArgs);
|
|
107
|
+
case "close_current_tab":
|
|
108
|
+
return await handleCloseCurrentTabToolCall(handlerArgs);
|
|
105
109
|
case "loadmill":
|
|
106
110
|
return await handleLoadmillToolCall(handlerArgs);
|
|
107
111
|
case "test_design_complete":
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export function updateBrowserSessionInfo(session, browserInfo, fallbackUrl = null) {
|
|
2
|
+
session.browserInfo = browserInfo;
|
|
3
|
+
session.deviceInfo = browserInfo;
|
|
4
|
+
session.currentUrl = browserInfo.currentUrl || fallbackUrl || null;
|
|
5
|
+
session.browserName = browserInfo.browserName || session.currentUrl;
|
|
6
|
+
session.deviceName = session.browserName;
|
|
7
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { normalizeBrowserInfoForSession } from "../../browser/normalize.js";
|
|
2
|
+
import { CAPTURE_SOURCES } from "../capture-sources.js";
|
|
3
|
+
import { printCliOutput } from "../../utils/console-output.js";
|
|
4
|
+
import { updateBrowserSessionInfo } from "./browser-session.js";
|
|
5
|
+
import { buildFunctionCallOutputItem, buildPreviewOutputContent, buildTextToolOutputItem, } from "./utils.js";
|
|
6
|
+
export async function handleCloseCurrentTabToolCall({ engine, callId, context = null, stepContext = null }) {
|
|
7
|
+
const session = engine.session;
|
|
8
|
+
if (session?.targetKind !== "browser" && session?.deviceInfo?.platform !== "browser") {
|
|
9
|
+
return {
|
|
10
|
+
kind: "continue",
|
|
11
|
+
outputItem: buildTextToolOutputItem(callId, "close_current_tab is only supported for web runs."),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
const addOutput = context?.addOutput || printCliOutput;
|
|
15
|
+
const result = await engine.closeCurrentTab(session.browserSessionId);
|
|
16
|
+
if (!result.closed) {
|
|
17
|
+
return {
|
|
18
|
+
kind: "continue",
|
|
19
|
+
outputItem: buildTextToolOutputItem(callId, result.message || "No previous browser tab is available. The current tab was left open."),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
addOutput({
|
|
23
|
+
type: "action",
|
|
24
|
+
text: "Close current tab",
|
|
25
|
+
eventType: "tool_call",
|
|
26
|
+
runId: context?.runId,
|
|
27
|
+
stepId: stepContext?.stepId,
|
|
28
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
29
|
+
payload: {
|
|
30
|
+
platform: session.deviceInfo?.platform,
|
|
31
|
+
toolName: "close_current_tab",
|
|
32
|
+
restoredUrl: result.restoredUrl,
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
const browserInfo = normalizeBrowserInfoForSession(await engine.getBrowserInfo(session.browserSessionId));
|
|
36
|
+
updateBrowserSessionInfo(session, browserInfo, result.restoredUrl);
|
|
37
|
+
const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
|
|
38
|
+
const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
|
|
39
|
+
runId: context?.runId,
|
|
40
|
+
sessionId: context?.sessionId,
|
|
41
|
+
stepId: stepContext?.stepId,
|
|
42
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
43
|
+
captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
|
|
44
|
+
});
|
|
45
|
+
return {
|
|
46
|
+
kind: "continue",
|
|
47
|
+
outputItem: buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
|
|
48
|
+
text: result.message,
|
|
49
|
+
annotatedScreenshotBase64: screenshotBase64,
|
|
50
|
+
})),
|
|
51
|
+
currentScreenshot,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -2,6 +2,7 @@ import { normalizeBrowserInfoForSession } from "../../browser/normalize.js";
|
|
|
2
2
|
import { buildExecutionNavigationRecoveryPrompt } from "../../prompts/execution.js";
|
|
3
3
|
import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
4
4
|
import { printCliOutput } from "../../utils/console-output.js";
|
|
5
|
+
import { updateBrowserSessionInfo } from "./browser-session.js";
|
|
5
6
|
import { buildDebugContext, buildFunctionCallOutputItem, buildPreviewOutputContent, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
|
|
6
7
|
export async function handleNavigateToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
|
|
7
8
|
const session = engine.session;
|
|
@@ -42,11 +43,7 @@ export async function handleNavigateToolCall({ engine, callId, argumentsText, co
|
|
|
42
43
|
try {
|
|
43
44
|
const normalizedUrl = await engine.navigateToUrl(session.browserSessionId, navigationTarget);
|
|
44
45
|
const browserInfo = normalizeBrowserInfoForSession(await engine.getBrowserInfo(session.browserSessionId));
|
|
45
|
-
session
|
|
46
|
-
session.deviceInfo = browserInfo;
|
|
47
|
-
session.currentUrl = browserInfo.currentUrl || normalizedUrl;
|
|
48
|
-
session.browserName = browserInfo.browserName || session.currentUrl;
|
|
49
|
-
session.deviceName = session.browserName;
|
|
46
|
+
updateBrowserSessionInfo(session, browserInfo, normalizedUrl);
|
|
50
47
|
emitDesktopDebug("runner.navigate.completed", isDesignContext(context) ? "design" : "execution", buildDebugContext(context, stepContext), {
|
|
51
48
|
instruction: navigationTarget,
|
|
52
49
|
target: navigationTarget,
|
|
@@ -3,6 +3,8 @@ import sharp from "sharp";
|
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
4
|
import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
5
5
|
import { execAdb, execEmulator, spawnAdb, spawnEmulator } from "./tools.js";
|
|
6
|
+
const AVD_READY_TIMEOUT_MS = 180000;
|
|
7
|
+
const AVD_READY_TIMEOUT_SECONDS = AVD_READY_TIMEOUT_MS / 1000;
|
|
6
8
|
async function listConnectedDevices() {
|
|
7
9
|
const { stdout } = await execAdb(["devices"]);
|
|
8
10
|
return stdout
|
|
@@ -65,6 +67,12 @@ async function waitForWindowServiceReady(deviceId, timeoutMs = 60000) {
|
|
|
65
67
|
}
|
|
66
68
|
return false;
|
|
67
69
|
}
|
|
70
|
+
function remainingTimeoutMs(deadline) {
|
|
71
|
+
return Math.max(0, deadline - Date.now());
|
|
72
|
+
}
|
|
73
|
+
function emulatorLaunchTimeoutError() {
|
|
74
|
+
return new Error(`Emulator launch timed out after ${AVD_READY_TIMEOUT_SECONDS}s. Check AVD health and retry.`);
|
|
75
|
+
}
|
|
68
76
|
async function findRunningEmulatorByAvdName(avdName) {
|
|
69
77
|
const devices = await listConnectedDevices();
|
|
70
78
|
const readyEmulators = devices.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-"));
|
|
@@ -76,7 +84,8 @@ async function findRunningEmulatorByAvdName(avdName) {
|
|
|
76
84
|
}
|
|
77
85
|
return null;
|
|
78
86
|
}
|
|
79
|
-
async function launchAndWaitForAvd(avdName, timeoutMs =
|
|
87
|
+
async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, options = {}) {
|
|
88
|
+
const log = options.log || console.log;
|
|
80
89
|
let avdExists = false;
|
|
81
90
|
try {
|
|
82
91
|
const { stdout } = await execEmulator(["-list-avds"]);
|
|
@@ -92,7 +101,7 @@ async function launchAndWaitForAvd(avdName, timeoutMs = 60000) {
|
|
|
92
101
|
}
|
|
93
102
|
const baseline = await listConnectedDevices();
|
|
94
103
|
const baselineReadyEmulators = new Set(baseline.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-")).map((entry) => entry.id));
|
|
95
|
-
|
|
104
|
+
log(`Launching Android emulator "${avdName}"...`);
|
|
96
105
|
const emulatorProcess = spawnEmulator(["-avd", avdName], { detached: true, stdio: "ignore" });
|
|
97
106
|
emulatorProcess.unref();
|
|
98
107
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -108,9 +117,10 @@ async function launchAndWaitForAvd(avdName, timeoutMs = 60000) {
|
|
|
108
117
|
}
|
|
109
118
|
await wait(2000);
|
|
110
119
|
}
|
|
111
|
-
throw
|
|
120
|
+
throw emulatorLaunchTimeoutError();
|
|
112
121
|
}
|
|
113
|
-
export async function connectToDevice(deviceTargetId) {
|
|
122
|
+
export async function connectToDevice(deviceTargetId, options = {}) {
|
|
123
|
+
const log = options.log || console.log;
|
|
114
124
|
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "start", targetId: deviceTargetId || null });
|
|
115
125
|
const parsedTarget = parseTargetToken(deviceTargetId || "");
|
|
116
126
|
const devices = await listConnectedDevices();
|
|
@@ -126,27 +136,28 @@ export async function connectToDevice(deviceTargetId) {
|
|
|
126
136
|
throw new Error("No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.");
|
|
127
137
|
}
|
|
128
138
|
const selected = readyDevices[0].id;
|
|
129
|
-
|
|
139
|
+
log(`Using connected Android device: ${selected}`);
|
|
130
140
|
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: selected, reused: true });
|
|
131
141
|
return selected;
|
|
132
142
|
}
|
|
133
143
|
if (parsedTarget.kind === "avd") {
|
|
144
|
+
const deadline = Date.now() + AVD_READY_TIMEOUT_MS;
|
|
134
145
|
const alreadyRunning = await findRunningEmulatorByAvdName(parsedTarget.avdName);
|
|
135
146
|
if (alreadyRunning) {
|
|
136
|
-
const ready = await waitForWindowServiceReady(alreadyRunning,
|
|
147
|
+
const ready = await waitForWindowServiceReady(alreadyRunning, remainingTimeoutMs(deadline));
|
|
137
148
|
if (!ready) {
|
|
138
|
-
throw
|
|
149
|
+
throw emulatorLaunchTimeoutError();
|
|
139
150
|
}
|
|
140
|
-
|
|
151
|
+
log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${alreadyRunning}`);
|
|
141
152
|
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: alreadyRunning, reused: true });
|
|
142
153
|
return alreadyRunning;
|
|
143
154
|
}
|
|
144
|
-
const launchedId = await launchAndWaitForAvd(parsedTarget.avdName,
|
|
145
|
-
const ready = await waitForWindowServiceReady(launchedId,
|
|
155
|
+
const launchedId = await launchAndWaitForAvd(parsedTarget.avdName, remainingTimeoutMs(deadline), { log });
|
|
156
|
+
const ready = await waitForWindowServiceReady(launchedId, remainingTimeoutMs(deadline));
|
|
146
157
|
if (!ready) {
|
|
147
|
-
throw
|
|
158
|
+
throw emulatorLaunchTimeoutError();
|
|
148
159
|
}
|
|
149
|
-
|
|
160
|
+
log(`Connected to launched emulator "${parsedTarget.avdName}" as ${launchedId}`);
|
|
150
161
|
emitDesktopDebug("device.connect", "device", {}, {
|
|
151
162
|
platform: "android",
|
|
152
163
|
stage: "success",
|
|
@@ -163,7 +174,7 @@ export async function connectToDevice(deviceTargetId) {
|
|
|
163
174
|
if (!ready) {
|
|
164
175
|
throw new Error("Selected device is connected but not ready yet. Wait a moment and retry.");
|
|
165
176
|
}
|
|
166
|
-
|
|
177
|
+
log(`Using selected Android device: ${targetId}`);
|
|
167
178
|
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: targetId, reused: true });
|
|
168
179
|
return targetId;
|
|
169
180
|
}
|
|
@@ -23,7 +23,8 @@ function buildCloudCapabilities(options) {
|
|
|
23
23
|
}
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
|
-
export async function connectToDevice(options) {
|
|
26
|
+
export async function connectToDevice(options, logOptions = {}) {
|
|
27
|
+
const log = logOptions.log || console.log;
|
|
27
28
|
if (!options || typeof options !== "object") {
|
|
28
29
|
throw new Error("Cloud device connection requires a provider payload.");
|
|
29
30
|
}
|
|
@@ -45,7 +46,7 @@ export async function connectToDevice(options) {
|
|
|
45
46
|
sessionName
|
|
46
47
|
});
|
|
47
48
|
const client = new CloudAppiumClient(platform, adapter, credentials);
|
|
48
|
-
|
|
49
|
+
log(`Creating ${adapter.displayName} session on ${deviceName} (${osVersion})...`);
|
|
49
50
|
const session = await client.createSession(capabilities);
|
|
50
51
|
activeSession = {
|
|
51
52
|
client,
|
|
@@ -55,7 +56,7 @@ export async function connectToDevice(options) {
|
|
|
55
56
|
osVersion,
|
|
56
57
|
sessionId: session.sessionId
|
|
57
58
|
};
|
|
58
|
-
|
|
59
|
+
log(`Connected to ${adapter.displayName} device "${deviceName}" (${session.sessionId})`);
|
|
59
60
|
return session.sessionId;
|
|
60
61
|
}
|
|
61
62
|
export async function getDeviceInfo(sessionId) {
|
|
@@ -12,33 +12,35 @@ let currentBackend = null;
|
|
|
12
12
|
* @param {string} platform - Optional platform override ('android' or 'ios')
|
|
13
13
|
* @returns {Promise<string>} Device ID
|
|
14
14
|
*/
|
|
15
|
-
export async function connectToDevice(deviceNameOrOptions, platform = null) {
|
|
15
|
+
export async function connectToDevice(deviceNameOrOptions, platform = null, options = {}) {
|
|
16
16
|
if (deviceNameOrOptions && typeof deviceNameOrOptions === "object" && !Array.isArray(deviceNameOrOptions)) {
|
|
17
17
|
const cloudOptions = deviceNameOrOptions;
|
|
18
18
|
const detectedPlatform = cloudOptions.platform || detectPlatform(cloudOptions.deviceName);
|
|
19
19
|
setCurrentSource(cloudOptions.source || "local");
|
|
20
20
|
setCurrentPlatform(detectedPlatform);
|
|
21
21
|
currentBackend = getDeviceBackend(detectedPlatform);
|
|
22
|
-
console.log
|
|
23
|
-
|
|
22
|
+
const log = options.log || console.log;
|
|
23
|
+
log(`Platform: ${detectedPlatform}`);
|
|
24
|
+
return currentBackend.connectToDevice(cloudOptions, options);
|
|
24
25
|
}
|
|
25
26
|
const detectedPlatform = platform || detectPlatform(deviceNameOrOptions);
|
|
26
27
|
setCurrentSource("local");
|
|
27
28
|
setCurrentPlatform(detectedPlatform);
|
|
28
29
|
currentBackend = getDeviceBackend(detectedPlatform);
|
|
29
|
-
console.log
|
|
30
|
-
|
|
30
|
+
const log = options.log || console.log;
|
|
31
|
+
log(`Platform: ${detectedPlatform}`);
|
|
32
|
+
return currentBackend.connectToDevice(deviceNameOrOptions, options);
|
|
31
33
|
}
|
|
32
34
|
/**
|
|
33
35
|
* Get device info (screen dimensions and scale factor)
|
|
34
36
|
* @param {string} deviceId
|
|
35
37
|
* @returns {Promise<object>}
|
|
36
38
|
*/
|
|
37
|
-
export async function getDeviceInfo(deviceId) {
|
|
39
|
+
export async function getDeviceInfo(deviceId, options = {}) {
|
|
38
40
|
if (!currentBackend) {
|
|
39
41
|
throw new Error("Not connected to a device. Call connectToDevice first.");
|
|
40
42
|
}
|
|
41
|
-
return currentBackend.getDeviceInfo(deviceId);
|
|
43
|
+
return currentBackend.getDeviceInfo(deviceId, options);
|
|
42
44
|
}
|
|
43
45
|
/**
|
|
44
46
|
* Get screenshot as base64 string
|
|
@@ -89,13 +89,14 @@ async function waitForAppiumReady(timeoutMs = 30000) {
|
|
|
89
89
|
* Start Appium server if not already running
|
|
90
90
|
* @returns {Promise<void>}
|
|
91
91
|
*/
|
|
92
|
-
export async function startAppium() {
|
|
92
|
+
export async function startAppium(options = {}) {
|
|
93
|
+
const log = options.log || console.log;
|
|
93
94
|
if (await isAppiumRunning()) {
|
|
94
|
-
|
|
95
|
+
log("Appium server is already running");
|
|
95
96
|
return;
|
|
96
97
|
}
|
|
97
98
|
const appiumCommand = resolveAppiumCommand();
|
|
98
|
-
|
|
99
|
+
log(`Starting Appium server (${appiumCommand})...`);
|
|
99
100
|
try {
|
|
100
101
|
appiumProcess = await spawnAppiumProcess(appiumCommand);
|
|
101
102
|
}
|
|
@@ -110,7 +111,7 @@ export async function startAppium() {
|
|
|
110
111
|
if (!ready) {
|
|
111
112
|
throw new Error("Appium server failed to start within 30 seconds");
|
|
112
113
|
}
|
|
113
|
-
|
|
114
|
+
log("Appium server is ready");
|
|
114
115
|
}
|
|
115
116
|
/**
|
|
116
117
|
* Stop the Appium server if we started it
|