@loadmill/droid-cua 2.10.1-rc.9 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -2
- package/build/index.js +133 -16
- package/build/src/browser/playwright/connection.js +11 -4
- package/build/src/browser/playwright/installed-browser.js +36 -26
- package/build/src/cli/headless-instructions.js +38 -2
- package/build/src/cli/headless-target.js +124 -1
- package/build/src/cli/lambdatest-app-upload.js +66 -0
- package/build/src/commands/help.js +7 -0
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/session.js +1 -0
- package/build/src/core/tool-handlers/assert.js +114 -27
- package/build/src/core/tool-handlers/loadmill.js +8 -1
- package/build/src/device/android/actions.js +11 -28
- package/build/src/device/assertions.js +1 -5
- package/build/src/device/cloud/actions.js +33 -49
- package/build/src/device/cloud/connection.js +7 -5
- package/build/src/device/cloud/lambdatest/adapter.js +51 -4
- package/build/src/device/ios/actions.js +22 -33
- package/build/src/device/loadmill.js +47 -6
- package/build/src/device/mobile-keypress.js +117 -0
- package/build/src/device/openai.js +6 -2
- package/build/src/integrations/loadmill/client.js +11 -0
- package/build/src/integrations/loadmill/executor.js +121 -2
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/parameters.js +127 -0
- package/build/src/modes/execution-mode.js +7 -1
- package/build/src/prompts/execution.js +25 -8
- package/build/src/utils/cua-debug-tracer.js +24 -2
- package/package.json +1 -1
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { stat } from "fs/promises";
|
|
3
|
+
import { lambdaTestAdapter } from "../device/cloud/lambdatest/adapter.js";
|
|
4
|
+
function trimNullableString(value) {
|
|
5
|
+
if (typeof value !== "string") {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
const trimmed = value.trim();
|
|
9
|
+
return trimmed || null;
|
|
10
|
+
}
|
|
11
|
+
export function resolveLambdaTestCredentials(env = process.env) {
|
|
12
|
+
const username = trimNullableString(env.LAMBDATEST_USERNAME);
|
|
13
|
+
const accessKey = trimNullableString(env.LAMBDATEST_ACCESS_KEY);
|
|
14
|
+
if (!username || !accessKey) {
|
|
15
|
+
throw new Error("LAMBDATEST_USERNAME and LAMBDATEST_ACCESS_KEY are required for --device-source lambdatest.");
|
|
16
|
+
}
|
|
17
|
+
return {
|
|
18
|
+
username,
|
|
19
|
+
accessKey,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export function validateLambdaTestAppPath(appPath, platform) {
|
|
23
|
+
const normalizedPlatform = trimNullableString(platform)?.toLowerCase();
|
|
24
|
+
if (normalizedPlatform !== "android" && normalizedPlatform !== "ios") {
|
|
25
|
+
throw new Error("--platform must be one of: android, ios.");
|
|
26
|
+
}
|
|
27
|
+
const normalizedAppPath = trimNullableString(appPath);
|
|
28
|
+
if (!normalizedAppPath) {
|
|
29
|
+
throw new Error("--app is required with --device-source lambdatest.");
|
|
30
|
+
}
|
|
31
|
+
const resolvedPath = path.resolve(normalizedAppPath);
|
|
32
|
+
const extension = path.extname(resolvedPath).toLowerCase();
|
|
33
|
+
if (extension === ".app") {
|
|
34
|
+
throw new Error("LambdaTest real-device sessions require an .apk for Android or an .ipa for iOS. .app bundles are not supported.");
|
|
35
|
+
}
|
|
36
|
+
if (normalizedPlatform === "android" && extension !== ".apk") {
|
|
37
|
+
throw new Error("LambdaTest Android sessions require an .apk build.");
|
|
38
|
+
}
|
|
39
|
+
if (normalizedPlatform === "ios" && extension !== ".ipa") {
|
|
40
|
+
throw new Error("LambdaTest iOS sessions require an .ipa build.");
|
|
41
|
+
}
|
|
42
|
+
return resolvedPath;
|
|
43
|
+
}
|
|
44
|
+
export async function uploadLambdaTestCliApp({ appPath, platform, credentials, adapter = lambdaTestAdapter, } = {}) {
|
|
45
|
+
const resolvedPath = validateLambdaTestAppPath(appPath, platform);
|
|
46
|
+
let fileStats;
|
|
47
|
+
try {
|
|
48
|
+
fileStats = await stat(resolvedPath);
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
const message = error instanceof Error ? error.message : "App file is unavailable.";
|
|
52
|
+
throw new Error(`App file is unavailable: ${message}`);
|
|
53
|
+
}
|
|
54
|
+
if (!fileStats.isFile()) {
|
|
55
|
+
throw new Error("Selected app path is not a file.");
|
|
56
|
+
}
|
|
57
|
+
const uploadedRef = await adapter.uploadApp(credentials, resolvedPath);
|
|
58
|
+
if (!uploadedRef?.remotePath || typeof uploadedRef.remotePath !== "string") {
|
|
59
|
+
throw new Error("LambdaTest did not return a valid app reference.");
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
appRef: uploadedRef.remotePath,
|
|
63
|
+
appPath: resolvedPath,
|
|
64
|
+
uploadedRef,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -15,11 +15,16 @@ export async function handleHelp(args, session, context) {
|
|
|
15
15
|
addOutput({ type: 'info', text: 'Usage:' });
|
|
16
16
|
addOutput({ type: 'info', text: ' droid-cua --avd <device-name> [options]' });
|
|
17
17
|
addOutput({ type: 'info', text: ' droid-cua --target web --instructions <file> [options]' });
|
|
18
|
+
addOutput({ type: 'info', text: ' droid-cua --device-source lambdatest --instructions <file> [options]' });
|
|
18
19
|
addOutput({ type: 'info', text: '' });
|
|
19
20
|
addOutput({ type: 'info', text: 'Options:' });
|
|
20
21
|
addOutput({ type: 'info', text: ' --target <kind> Target kind: mobile (default) or web' });
|
|
21
22
|
addOutput({ type: 'info', text: ' --avd <name> Device name (Android device ID/serial or iOS Simulator)' });
|
|
22
23
|
addOutput({ type: 'info', text: ' --platform <platform> Force platform: android or ios' });
|
|
24
|
+
addOutput({ type: 'info', text: ' --device-source <src> Mobile source: local (default) or lambdatest' });
|
|
25
|
+
addOutput({ type: 'info', text: ' --device-name <name> LambdaTest real device name' });
|
|
26
|
+
addOutput({ type: 'info', text: ' --os-version <version> LambdaTest OS version' });
|
|
27
|
+
addOutput({ type: 'info', text: ' --app <path> LambdaTest app package path (.apk or .ipa)' });
|
|
23
28
|
addOutput({ type: 'info', text: ' --instructions <file> Run test file in headless mode' });
|
|
24
29
|
addOutput({ type: 'info', text: ' --session-mode <mode> Browser session mode for web runs: persistent or fresh' });
|
|
25
30
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
@@ -53,6 +58,7 @@ export async function handleHelp(args, session, context) {
|
|
|
53
58
|
addOutput({ type: 'info', text: 'Platform Support:' });
|
|
54
59
|
addOutput({ type: 'info', text: ' Android: Uses ADB to communicate with Android devices (physical or emulator)' });
|
|
55
60
|
addOutput({ type: 'info', text: ' iOS: Uses Appium + XCUITest for iOS Simulator automation' });
|
|
61
|
+
addOutput({ type: 'info', text: ' LambdaTest: Uses real cloud devices for headless mobile runs' });
|
|
56
62
|
addOutput({ type: 'info', text: ' Web: Uses Playwright with installed Google Chrome or Microsoft Edge for browser execution' });
|
|
57
63
|
addOutput({ type: 'info', text: '' });
|
|
58
64
|
addOutput({ type: 'info', text: 'Platform Detection:' });
|
|
@@ -74,6 +80,7 @@ export async function handleHelp(args, session, context) {
|
|
|
74
80
|
addOutput({ type: 'info', text: ' droid-cua --platform ios --avd MySim (Force iOS platform)' });
|
|
75
81
|
addOutput({ type: 'info', text: ' droid-cua --instructions tests/login.dcua --context app/context.md' });
|
|
76
82
|
addOutput({ type: 'info', text: ' droid-cua --target web --instructions tests/search.dcua' });
|
|
83
|
+
addOutput({ type: 'info', text: ' droid-cua --device-source lambdatest --platform android --device-name "Galaxy S24" --os-version 14 --app app.apk --instructions tests/login.dcua' });
|
|
77
84
|
addOutput({ type: 'info', text: ' /create login-test (design a new test)' });
|
|
78
85
|
addOutput({ type: 'info', text: ' /list (see all tests)' });
|
|
79
86
|
addOutput({ type: 'info', text: ' /view login-test (view test contents)' });
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
|
|
2
|
-
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD =
|
|
2
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 30000;
|
|
3
3
|
export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
|
|
4
4
|
export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
|
|
5
5
|
export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";
|
|
@@ -9,6 +9,7 @@ export class Session {
|
|
|
9
9
|
this.deviceName = targetInfo?.device_name || null;
|
|
10
10
|
this.browserSessionId = targetInfo?.platform === "browser" ? targetSessionId : null;
|
|
11
11
|
this.browserInfo = targetInfo?.platform === "browser" ? targetInfo : null;
|
|
12
|
+
this.browserFamily = targetInfo?.browserFamily || null;
|
|
12
13
|
this.browserName = targetInfo?.browserName || null;
|
|
13
14
|
this.currentUrl = targetInfo?.currentUrl || null;
|
|
14
15
|
this.sessionMode = targetInfo?.sessionMode || null;
|
|
@@ -2,19 +2,100 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { buildAssertionSystemPrompt, checkAssertionResult, extractFailureDetails, handleAssertionFailure, handleAssertionSuccess, } from "../../device/assertions.js";
|
|
3
3
|
import { CAPTURE_SOURCES } from "../capture-sources.js";
|
|
4
4
|
import { printCliOutput } from "../../utils/console-output.js";
|
|
5
|
-
import {
|
|
5
|
+
import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
6
|
+
import { buildNestedToolDebugContext, buildTextToolOutputItem, isDesignContext, parseJsonArguments, } from "./utils.js";
|
|
7
|
+
function parseAssertionArguments(argumentsText) {
|
|
8
|
+
const parsed = parseJsonArguments(argumentsText);
|
|
9
|
+
if (!parsed.ok || typeof parsed.value !== "object" || !parsed.value) {
|
|
10
|
+
return {
|
|
11
|
+
ok: false,
|
|
12
|
+
message: 'Invalid assert arguments: expected a JSON object with string "condition" and "context" fields.',
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
const condition = parsed.value.condition;
|
|
16
|
+
const assertionContext = parsed.value.context;
|
|
17
|
+
if (typeof condition !== "string" || !condition.trim()) {
|
|
18
|
+
return {
|
|
19
|
+
ok: false,
|
|
20
|
+
message: 'Invalid assert arguments: "condition" must be a non-empty string.',
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
if (typeof assertionContext !== "string" || !assertionContext.trim()) {
|
|
24
|
+
return {
|
|
25
|
+
ok: false,
|
|
26
|
+
message: 'Invalid assert arguments: "context" must be a non-empty string.',
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
ok: true,
|
|
31
|
+
condition: condition.trim(),
|
|
32
|
+
context: assertionContext.trim(),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function extractResponseText(response) {
|
|
36
|
+
if (typeof response?.output_text === "string" && response.output_text.trim()) {
|
|
37
|
+
return response.output_text.trim();
|
|
38
|
+
}
|
|
39
|
+
const parts = [];
|
|
40
|
+
for (const item of response?.output || []) {
|
|
41
|
+
if (typeof item?.text === "string" && item.text.trim()) {
|
|
42
|
+
parts.push(item.text.trim());
|
|
43
|
+
}
|
|
44
|
+
for (const contentItem of item?.content || []) {
|
|
45
|
+
if (typeof contentItem?.text === "string" && contentItem.text.trim()) {
|
|
46
|
+
parts.push(contentItem.text.trim());
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return parts.join("\n").trim();
|
|
51
|
+
}
|
|
52
|
+
function extractPassDetails(assertionText) {
|
|
53
|
+
const parts = assertionText.split("ASSERTION RESULT: PASS");
|
|
54
|
+
return parts[1]?.trim() || "Assertion passed.";
|
|
55
|
+
}
|
|
56
|
+
function emitAssertionToolCall({ callId, argumentsText, assertionPrompt, assertionContext, context, stepContext }) {
|
|
57
|
+
const scope = isDesignContext(context) ? "design" : "execution";
|
|
58
|
+
const ids = {
|
|
59
|
+
...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
|
|
60
|
+
...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
|
|
61
|
+
...(Number.isInteger(stepContext?.instructionIndex) ? { instructionIndex: stepContext.instructionIndex } : {}),
|
|
62
|
+
};
|
|
63
|
+
const payload = {
|
|
64
|
+
toolName: "assert",
|
|
65
|
+
callId,
|
|
66
|
+
condition: assertionPrompt,
|
|
67
|
+
context: assertionContext,
|
|
68
|
+
arguments: argumentsText,
|
|
69
|
+
};
|
|
70
|
+
emitDesktopDebug("tool_call", scope, ids, payload);
|
|
71
|
+
const addOutput = context?.addOutput || null;
|
|
72
|
+
if (addOutput) {
|
|
73
|
+
addOutput({
|
|
74
|
+
type: "muted",
|
|
75
|
+
text: `Validating assertion: ${assertionPrompt}`,
|
|
76
|
+
eventType: "tool_call",
|
|
77
|
+
runId: context?.runId,
|
|
78
|
+
stepId: stepContext?.stepId,
|
|
79
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
80
|
+
payload,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
}
|
|
6
84
|
export async function handleAssertionToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
|
|
7
85
|
const session = engine.session;
|
|
8
|
-
const parsed =
|
|
86
|
+
const parsed = parseAssertionArguments(argumentsText);
|
|
9
87
|
if (!parsed.ok) {
|
|
10
88
|
return {
|
|
11
89
|
kind: "continue",
|
|
12
90
|
outputItem: buildTextToolOutputItem(callId, parsed.message),
|
|
13
91
|
};
|
|
14
92
|
}
|
|
15
|
-
const assertionPrompt = parsed.
|
|
16
|
-
const
|
|
93
|
+
const assertionPrompt = parsed.condition;
|
|
94
|
+
const assertionContext = parsed.context;
|
|
95
|
+
emitAssertionToolCall({ callId, argumentsText, assertionPrompt, assertionContext, context, stepContext });
|
|
96
|
+
const transcriptBeforeAssertion = session.getTranscriptText();
|
|
17
97
|
session.addToTranscript(`[Assertion] ${assertionPrompt}`);
|
|
98
|
+
session.addToTranscript(`[Assertion Context] ${assertionContext}`);
|
|
18
99
|
const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
|
|
19
100
|
const assertionEventId = randomUUID();
|
|
20
101
|
const assertionScreenshot = await engine.recordScreenshot(screenshotBase64, {
|
|
@@ -35,8 +116,11 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
35
116
|
...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
|
|
36
117
|
...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {}),
|
|
37
118
|
};
|
|
38
|
-
const assertionSystemPrompt = buildAssertionSystemPrompt(engine.baseSystemPrompt || session.systemPrompt || "", assertionPrompt
|
|
39
|
-
|
|
119
|
+
const assertionSystemPrompt = buildAssertionSystemPrompt(engine.baseSystemPrompt || session.systemPrompt || "", assertionPrompt, {
|
|
120
|
+
assertionContext,
|
|
121
|
+
transcript: transcriptBeforeAssertion,
|
|
122
|
+
});
|
|
123
|
+
const assertionDebugContext = buildNestedToolDebugContext(context, stepContext, "assert", callId);
|
|
40
124
|
const assertionResponse = await engine.sendCUARequest({
|
|
41
125
|
messages: [
|
|
42
126
|
{ role: "system", content: assertionSystemPrompt },
|
|
@@ -45,30 +129,30 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
45
129
|
screenshotBase64,
|
|
46
130
|
previousResponseId: null,
|
|
47
131
|
deviceInfo: session.deviceInfo,
|
|
48
|
-
debugContext:
|
|
132
|
+
debugContext: assertionDebugContext,
|
|
133
|
+
includeComputerTool: false,
|
|
49
134
|
toolCapabilities: {
|
|
135
|
+
previewClick: false,
|
|
50
136
|
assert: false,
|
|
51
137
|
navigate: false,
|
|
52
138
|
loadmill: false,
|
|
53
139
|
},
|
|
54
140
|
});
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
const nestedTurn = await engine.runFullTurn(assertionResponse, () => false, context, stepContext, {
|
|
59
|
-
suppressAssistantOutput: true,
|
|
60
|
-
debugContextOverride: nestedDebugContext,
|
|
61
|
-
currentScreenshot: assertionScreenshot,
|
|
62
|
-
});
|
|
63
|
-
const currentScreenshot = nestedTurn.currentScreenshot ?? assertionScreenshot;
|
|
64
|
-
const assertionTranscript = session.transcript.slice(transcriptStartIndex);
|
|
141
|
+
const assertionText = extractResponseText(assertionResponse);
|
|
142
|
+
session.addToTranscript(`[Assistant] ${assertionText}`);
|
|
143
|
+
const assertionTranscript = [assertionText];
|
|
65
144
|
const result = checkAssertionResult(assertionTranscript);
|
|
145
|
+
const failureDetails = extractFailureDetails(assertionTranscript);
|
|
146
|
+
const passDetails = extractPassDetails(assertionText);
|
|
66
147
|
if (isDesignContext(context)) {
|
|
67
148
|
const details = result.passed
|
|
68
|
-
?
|
|
69
|
-
: `Assertion failed: ${
|
|
149
|
+
? passDetails
|
|
150
|
+
: `Assertion failed: ${failureDetails}`;
|
|
70
151
|
if (result.passed) {
|
|
71
|
-
handleAssertionSuccess(assertionPrompt, context, stepContext,
|
|
152
|
+
handleAssertionSuccess(assertionPrompt, context, stepContext, {
|
|
153
|
+
details: passDetails,
|
|
154
|
+
...assertionEventPayload,
|
|
155
|
+
});
|
|
72
156
|
}
|
|
73
157
|
else {
|
|
74
158
|
const addOutput = context?.addOutput || printCliOutput;
|
|
@@ -82,7 +166,7 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
82
166
|
payload: {
|
|
83
167
|
assertion: assertionPrompt,
|
|
84
168
|
passed: false,
|
|
85
|
-
details:
|
|
169
|
+
details: failureDetails,
|
|
86
170
|
...assertionEventPayload,
|
|
87
171
|
},
|
|
88
172
|
});
|
|
@@ -92,7 +176,7 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
92
176
|
outputItem: buildTextToolOutputItem(callId, result.passed
|
|
93
177
|
? `ASSERTION RESULT: PASS\n${details}`
|
|
94
178
|
: `ASSERTION RESULT: FAIL\n${details}`),
|
|
95
|
-
currentScreenshot,
|
|
179
|
+
currentScreenshot: assertionScreenshot,
|
|
96
180
|
};
|
|
97
181
|
}
|
|
98
182
|
if (result.failed) {
|
|
@@ -134,9 +218,9 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
134
218
|
if (choice === "skip" || choice === "s") {
|
|
135
219
|
return {
|
|
136
220
|
kind: "close_turn",
|
|
137
|
-
outputItem: buildTextToolOutputItem(callId, `ASSERTION RESULT: FAIL\n${
|
|
221
|
+
outputItem: buildTextToolOutputItem(callId, `ASSERTION RESULT: FAIL\n${failureDetails}`),
|
|
138
222
|
summaryUpdates: { assertionStatus: "failed_skipped" },
|
|
139
|
-
currentScreenshot,
|
|
223
|
+
currentScreenshot: assertionScreenshot,
|
|
140
224
|
};
|
|
141
225
|
}
|
|
142
226
|
return {
|
|
@@ -145,11 +229,14 @@ export async function handleAssertionToolCall({ engine, callId, argumentsText, c
|
|
|
145
229
|
summaryUpdates: { assertionStatus: "failed" },
|
|
146
230
|
};
|
|
147
231
|
}
|
|
148
|
-
handleAssertionSuccess(assertionPrompt, context, stepContext,
|
|
232
|
+
handleAssertionSuccess(assertionPrompt, context, stepContext, {
|
|
233
|
+
details: passDetails,
|
|
234
|
+
...assertionEventPayload,
|
|
235
|
+
});
|
|
149
236
|
return {
|
|
150
237
|
kind: "close_turn",
|
|
151
|
-
outputItem: buildTextToolOutputItem(callId,
|
|
238
|
+
outputItem: buildTextToolOutputItem(callId, `ASSERTION RESULT: PASS\n${passDetails}`),
|
|
152
239
|
summaryUpdates: { assertionStatus: "passed" },
|
|
153
|
-
currentScreenshot,
|
|
240
|
+
currentScreenshot: assertionScreenshot,
|
|
154
241
|
};
|
|
155
242
|
}
|
|
@@ -10,7 +10,11 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
|
|
|
10
10
|
}
|
|
11
11
|
const loadmillCommand = parsed.value;
|
|
12
12
|
session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
|
|
13
|
-
const result = await engine.executeLoadmillInstruction(loadmillCommand, engine.isHeadlessMode,
|
|
13
|
+
const result = await engine.executeLoadmillInstruction(loadmillCommand, engine.isHeadlessMode, {
|
|
14
|
+
...context,
|
|
15
|
+
session,
|
|
16
|
+
futureInstructions: context?.futureInstructions || [],
|
|
17
|
+
}, stepContext);
|
|
14
18
|
if (result.retry) {
|
|
15
19
|
return {
|
|
16
20
|
kind: "retry_instruction",
|
|
@@ -25,5 +29,8 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
|
|
|
25
29
|
return {
|
|
26
30
|
kind: "close_turn",
|
|
27
31
|
outputItem: buildTextToolOutputItem(callId, "Loadmill command completed successfully."),
|
|
32
|
+
summaryUpdates: result.runtimeParameterContext?.note
|
|
33
|
+
? { preserveSessionMessages: true }
|
|
34
|
+
: undefined,
|
|
28
35
|
};
|
|
29
36
|
}
|
|
@@ -2,36 +2,12 @@ import { logger } from "../../utils/logger.js";
|
|
|
2
2
|
import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
|
|
3
3
|
import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
|
|
4
4
|
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
5
|
+
import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
|
|
5
6
|
import { resolveScrollGesture } from "../scroll-gesture.js";
|
|
6
7
|
import { execAdb } from "./tools.js";
|
|
7
8
|
function adbShell(deviceId, command) {
|
|
8
9
|
return execAdb(["-s", deviceId, "shell", command]);
|
|
9
10
|
}
|
|
10
|
-
function normalizeMobileKeypress(keys = []) {
|
|
11
|
-
if (!Array.isArray(keys) || keys.length === 0) {
|
|
12
|
-
throw new Error("Keypress action is missing keys");
|
|
13
|
-
}
|
|
14
|
-
if (keys.length > 1) {
|
|
15
|
-
throw new Error(`Unsupported mobile key chord: ${keys.join(", ")}. Use taps and text entry instead.`);
|
|
16
|
-
}
|
|
17
|
-
const key = String(keys[0]).trim().toUpperCase();
|
|
18
|
-
const mobileKeyMap = {
|
|
19
|
-
ESC: "KEYCODE_BACK",
|
|
20
|
-
ESCAPE: "KEYCODE_BACK",
|
|
21
|
-
HOME: "KEYCODE_HOME",
|
|
22
|
-
BACK: "KEYCODE_BACK",
|
|
23
|
-
ENTER: "KEYCODE_ENTER",
|
|
24
|
-
RETURN: "KEYCODE_ENTER",
|
|
25
|
-
BACKSPACE: "KEYCODE_DEL",
|
|
26
|
-
DELETE: "KEYCODE_DEL",
|
|
27
|
-
SPACE: "KEYCODE_SPACE"
|
|
28
|
-
};
|
|
29
|
-
const mappedKey = mobileKeyMap[key];
|
|
30
|
-
if (!mappedKey) {
|
|
31
|
-
throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
|
|
32
|
-
}
|
|
33
|
-
return { originalKey: keys[0], mappedKey };
|
|
34
|
-
}
|
|
35
11
|
export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
|
|
36
12
|
const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
|
|
37
13
|
const meta = (payload = {}) => buildActionEventMeta({
|
|
@@ -109,9 +85,16 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
109
85
|
break;
|
|
110
86
|
case "keypress":
|
|
111
87
|
{
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
88
|
+
const normalized = normalizeMobileKeypress("android", keys);
|
|
89
|
+
const transport = getAndroidAdbTransport(normalized);
|
|
90
|
+
addOutput({
|
|
91
|
+
type: 'action',
|
|
92
|
+
text: normalized.logText,
|
|
93
|
+
...meta(buildMobileKeypressPayload(normalized, transport, {
|
|
94
|
+
mappedKeys: [transport.transportValue]
|
|
95
|
+
}))
|
|
96
|
+
});
|
|
97
|
+
await adbShell(deviceId, `input keyevent ${transport.transportValue}`);
|
|
115
98
|
}
|
|
116
99
|
break;
|
|
117
100
|
case "wait":
|
|
@@ -54,11 +54,7 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
|
|
|
54
54
|
addOutput({ type: 'error', text: `Assertion: ${assertionPrompt}`, ...meta });
|
|
55
55
|
addOutput({ type: 'error', text: `Details: ${details}`, ...meta });
|
|
56
56
|
if (isHeadlessMode) {
|
|
57
|
-
|
|
58
|
-
if (context?.exit) {
|
|
59
|
-
context.exit();
|
|
60
|
-
}
|
|
61
|
-
process.exit(1);
|
|
57
|
+
return;
|
|
62
58
|
}
|
|
63
59
|
// Interactive mode: caller should clear remaining instructions
|
|
64
60
|
}
|
|
@@ -2,48 +2,9 @@ import { logger } from "../../utils/logger.js";
|
|
|
2
2
|
import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
|
|
3
3
|
import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
|
|
4
4
|
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
5
|
+
import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
|
|
5
6
|
import { getActiveSession, getDevicePixelRatio } from "./connection.js";
|
|
6
7
|
import { resolveScrollGesture } from "../scroll-gesture.js";
|
|
7
|
-
function normalizeMobileKeypress(platform, keys = []) {
|
|
8
|
-
if (!Array.isArray(keys) || keys.length === 0) {
|
|
9
|
-
throw new Error("Keypress action is missing keys");
|
|
10
|
-
}
|
|
11
|
-
if (keys.length > 1) {
|
|
12
|
-
throw new Error(`Unsupported mobile key chord: ${keys.join(", ")}. Use taps and text entry instead.`);
|
|
13
|
-
}
|
|
14
|
-
const key = String(keys[0]).trim().toUpperCase();
|
|
15
|
-
if (platform === "ios") {
|
|
16
|
-
if (key === "ESC" || key === "ESCAPE" || key === "HOME") {
|
|
17
|
-
return { kind: "ios-button", originalKey: keys[0], mapped: "home" };
|
|
18
|
-
}
|
|
19
|
-
if (key === "ENTER" || key === "RETURN") {
|
|
20
|
-
return { kind: "text", originalKey: keys[0], mapped: "\n", label: "Return key" };
|
|
21
|
-
}
|
|
22
|
-
if (key === "BACKSPACE" || key === "DELETE") {
|
|
23
|
-
return { kind: "text", originalKey: keys[0], mapped: "\b", label: "Delete key" };
|
|
24
|
-
}
|
|
25
|
-
if (key === "SPACE") {
|
|
26
|
-
return { kind: "text", originalKey: keys[0], mapped: " ", label: "Space key" };
|
|
27
|
-
}
|
|
28
|
-
throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
|
|
29
|
-
}
|
|
30
|
-
const androidKeyMap = {
|
|
31
|
-
ESC: 3,
|
|
32
|
-
ESCAPE: 3,
|
|
33
|
-
HOME: 3,
|
|
34
|
-
BACK: 4,
|
|
35
|
-
ENTER: 66,
|
|
36
|
-
RETURN: 66,
|
|
37
|
-
BACKSPACE: 67,
|
|
38
|
-
DELETE: 67,
|
|
39
|
-
SPACE: 62
|
|
40
|
-
};
|
|
41
|
-
const keycode = androidKeyMap[key];
|
|
42
|
-
if (!keycode) {
|
|
43
|
-
throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
|
|
44
|
-
}
|
|
45
|
-
return { kind: "android-keycode", originalKey: keys[0], keycode };
|
|
46
|
-
}
|
|
47
8
|
export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
|
|
48
9
|
const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
|
|
49
10
|
const session = getActiveSession();
|
|
@@ -128,17 +89,40 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
128
89
|
}
|
|
129
90
|
case "keypress": {
|
|
130
91
|
const normalized = normalizeMobileKeypress(platform, action.keys);
|
|
131
|
-
if (
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
92
|
+
if (platform === "ios") {
|
|
93
|
+
const transport = getIosTransport(normalized);
|
|
94
|
+
if (transport.transportType === "noop") {
|
|
95
|
+
addOutput({
|
|
96
|
+
type: "info",
|
|
97
|
+
text: normalized.logText,
|
|
98
|
+
...meta(buildMobileKeypressPayload(normalized, transport, { ignored: true }))
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
else if (transport.transportType === "ios-button") {
|
|
102
|
+
addOutput({
|
|
103
|
+
type: "action",
|
|
104
|
+
text: normalized.logText,
|
|
105
|
+
...meta(buildMobileKeypressPayload(normalized, transport))
|
|
106
|
+
});
|
|
107
|
+
await session.client.pressIosButton(session.sessionId, transport.transportValue);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
addOutput({
|
|
111
|
+
type: "action",
|
|
112
|
+
text: normalized.logText,
|
|
113
|
+
...meta(buildMobileKeypressPayload(normalized, transport))
|
|
114
|
+
});
|
|
115
|
+
await session.client.type(session.sessionId, transport.transportValue);
|
|
116
|
+
}
|
|
138
117
|
}
|
|
139
118
|
else {
|
|
140
|
-
|
|
141
|
-
|
|
119
|
+
const transport = getCloudAndroidTransport(normalized);
|
|
120
|
+
addOutput({
|
|
121
|
+
type: "action",
|
|
122
|
+
text: normalized.logText,
|
|
123
|
+
...meta(buildMobileKeypressPayload(normalized, transport, { keycode: transport.keycode }))
|
|
124
|
+
});
|
|
125
|
+
await session.client.pressAndroidKeyCode(session.sessionId, transport.keycode);
|
|
142
126
|
}
|
|
143
127
|
break;
|
|
144
128
|
}
|
|
@@ -3,7 +3,7 @@ import { getCloudProviderAdapter } from "./registry.js";
|
|
|
3
3
|
import { CloudAppiumClient } from "./cloud-appium-client.js";
|
|
4
4
|
let activeSession = null;
|
|
5
5
|
function buildCloudCapabilities(options) {
|
|
6
|
-
const { providerId, platform, deviceName, osVersion, appRef, credentials } = options;
|
|
6
|
+
const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName } = options;
|
|
7
7
|
const adapter = getCloudProviderAdapter(providerId);
|
|
8
8
|
if (!adapter) {
|
|
9
9
|
throw new Error(`Cloud provider "${providerId}" is not registered.`);
|
|
@@ -18,8 +18,8 @@ function buildCloudCapabilities(options) {
|
|
|
18
18
|
username: credentials?.username,
|
|
19
19
|
accessKey: credentials?.accessKey,
|
|
20
20
|
projectName: "droid-cua",
|
|
21
|
-
buildName: `desktop-${new Date().toISOString().slice(0, 10)}`,
|
|
22
|
-
sessionName: `${deviceName} (${platform})`
|
|
21
|
+
buildName: buildName || `desktop-${new Date().toISOString().slice(0, 10)}`,
|
|
22
|
+
sessionName: sessionName || `${deviceName} (${platform})`
|
|
23
23
|
}
|
|
24
24
|
};
|
|
25
25
|
}
|
|
@@ -27,7 +27,7 @@ export async function connectToDevice(options) {
|
|
|
27
27
|
if (!options || typeof options !== "object") {
|
|
28
28
|
throw new Error("Cloud device connection requires a provider payload.");
|
|
29
29
|
}
|
|
30
|
-
const { providerId, platform, deviceName, osVersion, appRef, credentials } = options;
|
|
30
|
+
const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName } = options;
|
|
31
31
|
if (!providerId || !platform || !deviceName || !osVersion || !appRef) {
|
|
32
32
|
throw new Error("Cloud device connection is missing required session details.");
|
|
33
33
|
}
|
|
@@ -40,7 +40,9 @@ export async function connectToDevice(options) {
|
|
|
40
40
|
deviceName,
|
|
41
41
|
osVersion,
|
|
42
42
|
appRef,
|
|
43
|
-
credentials
|
|
43
|
+
credentials,
|
|
44
|
+
buildName,
|
|
45
|
+
sessionName
|
|
44
46
|
});
|
|
45
47
|
const client = new CloudAppiumClient(platform, adapter, credentials);
|
|
46
48
|
console.log(`Creating ${adapter.displayName} session on ${deviceName} (${osVersion})...`);
|
|
@@ -5,6 +5,7 @@ const DEVICE_REGIONS = ["us", "ap", "eu"];
|
|
|
5
5
|
const DEVICE_LIST_URL = "https://mobile-api.lambdatest.com/mobile-automation/api/v1/list";
|
|
6
6
|
const APP_UPLOAD_URL = "https://manual-api.lambdatest.com/app/upload/realDevice";
|
|
7
7
|
const APP_DATA_URL = "https://manual-api.lambdatest.com/app/data";
|
|
8
|
+
const SESSION_DATA_URL = "https://mobile-api.lambdatest.com/mobile-automation/api/v1/sessions";
|
|
8
9
|
function notImplemented(methodName) {
|
|
9
10
|
throw new Error(`LambdaTest adapter stub: ${methodName} is not implemented yet.`);
|
|
10
11
|
}
|
|
@@ -307,6 +308,54 @@ async function lookupUploadedApp(creds, ref) {
|
|
|
307
308
|
}
|
|
308
309
|
return null;
|
|
309
310
|
}
|
|
311
|
+
function unwrapSessionPayload(payload) {
|
|
312
|
+
if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {
|
|
313
|
+
return null;
|
|
314
|
+
}
|
|
315
|
+
if (typeof payload.data === "object" && payload.data !== null && !Array.isArray(payload.data)) {
|
|
316
|
+
return payload.data;
|
|
317
|
+
}
|
|
318
|
+
return payload;
|
|
319
|
+
}
|
|
320
|
+
async function fetchSessionArtifacts(creds, sessionId) {
|
|
321
|
+
const fallbackDashboardUrl = `https://automation.lambdatest.com/logs/?sessionID=${encodeURIComponent(sessionId)}`;
|
|
322
|
+
const response = await fetch(`${SESSION_DATA_URL}/${encodeURIComponent(sessionId)}`, {
|
|
323
|
+
method: "GET",
|
|
324
|
+
headers: {
|
|
325
|
+
Authorization: lambdaTestAdapter.getAuthHeader(creds)
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
if (!response.ok) {
|
|
329
|
+
return {
|
|
330
|
+
dashboardUrl: fallbackDashboardUrl
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
const session = unwrapSessionPayload(await response.json());
|
|
334
|
+
const testId = normalizeString(session?.test_id) ?? normalizeString(session?.testID) ?? normalizeString(session?.testId);
|
|
335
|
+
const buildId = normalizeString(session?.build_id) ??
|
|
336
|
+
normalizeString(session?.buildID) ??
|
|
337
|
+
normalizeString(session?.buildId) ??
|
|
338
|
+
(Number.isFinite(session?.build_id) ? String(session.build_id) : undefined) ??
|
|
339
|
+
(Number.isFinite(session?.buildId) ? String(session.buildId) : undefined);
|
|
340
|
+
if (!testId || !buildId) {
|
|
341
|
+
return {
|
|
342
|
+
dashboardUrl: fallbackDashboardUrl
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
const params = new URLSearchParams({
|
|
346
|
+
build: buildId,
|
|
347
|
+
testID: testId,
|
|
348
|
+
});
|
|
349
|
+
return {
|
|
350
|
+
dashboardUrl: `https://appautomation.lambdatest.com/test?${params.toString()}`,
|
|
351
|
+
consoleLogsUrl: normalizeString(session?.console_logs_url),
|
|
352
|
+
networkLogsUrl: normalizeString(session?.network_logs_url),
|
|
353
|
+
commandLogsUrl: normalizeString(session?.command_logs_url),
|
|
354
|
+
appiumLogsUrl: normalizeString(session?.appium_logs_url),
|
|
355
|
+
deviceLogsUrl: normalizeString(session?.device_logs_url),
|
|
356
|
+
crashLogsUrl: normalizeString(session?.crash_logs_url)
|
|
357
|
+
};
|
|
358
|
+
}
|
|
310
359
|
/** @type {import("../adapter").CloudProviderAdapter} */
|
|
311
360
|
export const lambdaTestAdapter = {
|
|
312
361
|
id: "lambdatest",
|
|
@@ -391,10 +440,8 @@ export const lambdaTestAdapter = {
|
|
|
391
440
|
const accessKey = typeof creds.accessKey === "string" ? creds.accessKey : "";
|
|
392
441
|
return `Basic ${Buffer.from(`${username}:${accessKey}`).toString("base64")}`;
|
|
393
442
|
},
|
|
394
|
-
async getSessionArtifacts(
|
|
395
|
-
return
|
|
396
|
-
dashboardUrl: `https://automation.lambdatest.com/logs/?sessionID=${encodeURIComponent(sessionId)}`
|
|
397
|
-
};
|
|
443
|
+
async getSessionArtifacts(creds, sessionId) {
|
|
444
|
+
return fetchSessionArtifacts(creds, sessionId);
|
|
398
445
|
},
|
|
399
446
|
async setSessionStatus(_creds, _sessionId, _status) {
|
|
400
447
|
return notImplemented("setSessionStatus");
|