@loadmill/droid-cua 2.22.0 → 2.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -0
- package/build/index.js +43 -1
- package/build/src/browser/playwright/actions.js +65 -1
- package/build/src/cli/skill-command.js +240 -0
- package/build/src/commands/help.js +2 -1
- package/build/src/core/cua-models.js +29 -11
- package/build/src/core/droid-cua-run-usage.js +16 -7
- package/build/src/core/execution-engine.js +334 -5
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/report/render-html-report.js +8 -3
- package/build/src/core/report/run-log-model.js +42 -0
- package/build/src/core/runtime-config.js +23 -41
- package/build/src/core/session.js +6 -0
- package/build/src/core/tool-handlers/loadmill.js +3 -0
- package/build/src/core/tool-handlers/test-design-complete.js +1 -0
- package/build/src/core/tool-handlers/type-sensitive-text.js +1 -0
- package/build/src/device/android/actions.js +6 -2
- package/build/src/device/android/connection.js +58 -23
- package/build/src/device/assertions.js +15 -8
- package/build/src/device/ios/connection.js +36 -30
- package/build/src/device/openai.js +68 -2
- package/build/src/device/scroll-gesture.js +22 -9
- package/build/src/integrations/loadmill/client.js +20 -4
- package/build/src/modes/design-mode-ink.js +10 -1
- package/build/src/modes/design-mode.js +7 -1
- package/build/src/modes/execution-mode.js +84 -8
- package/build/src/prompts/design.js +21 -1
- package/build/src/utils/console-output.js +5 -0
- package/build/src/utils/cua-debug-tracer.js +50 -7
- package/package.json +2 -1
- package/skill/droid-cua/SKILL.md +36 -0
- package/skill/droid-cua/agents/openai.yaml +6 -0
- package/skill/droid-cua/assets/droid-cua-app-icon.png +0 -0
- package/skill/droid-cua/references/authoring-tests.md +39 -0
- package/skill/droid-cua/references/onboarding-and-troubleshooting.md +74 -0
- package/skill/droid-cua/references/project-and-auth.md +38 -0
- package/skill/droid-cua/references/running-and-healing.md +70 -0
- package/build/src/core/cua-backends.js +0 -11
|
@@ -56,6 +56,12 @@ export class Session {
|
|
|
56
56
|
getTranscriptText() {
|
|
57
57
|
return this.transcript.join("\n");
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Clear the per-instruction transcript.
|
|
61
|
+
*/
|
|
62
|
+
clearTranscript() {
|
|
63
|
+
this.transcript = [];
|
|
64
|
+
}
|
|
59
65
|
/**
|
|
60
66
|
* Update the previous response ID for chaining
|
|
61
67
|
*/
|
|
@@ -29,6 +29,9 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
|
|
|
29
29
|
return {
|
|
30
30
|
kind: "close_turn",
|
|
31
31
|
outputItem: buildTextToolOutputItem(callId, "Loadmill command completed successfully."),
|
|
32
|
+
nextTurnConfig: {
|
|
33
|
+
attachFreshScreenshotOnContinuation: true,
|
|
34
|
+
},
|
|
32
35
|
summaryUpdates: result.runtimeParameterContext?.note
|
|
33
36
|
? { preserveSessionMessages: true }
|
|
34
37
|
: undefined,
|
|
@@ -51,8 +51,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
51
51
|
case "scroll":
|
|
52
52
|
const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
|
|
53
53
|
scale,
|
|
54
|
-
fallbackStartX: 500,
|
|
55
|
-
fallbackStartY: 500
|
|
54
|
+
fallbackStartX: Number.isFinite(context?.fallbackScrollStartX) ? Math.round(context.fallbackScrollStartX) : 500,
|
|
55
|
+
fallbackStartY: Number.isFinite(context?.fallbackScrollStartY) ? Math.round(context.fallbackScrollStartY) : 500,
|
|
56
|
+
minX: 0,
|
|
57
|
+
maxX: Number.isFinite(context?.deviceWidth) ? Math.round(context.deviceWidth) - 1 : undefined,
|
|
58
|
+
minY: 0,
|
|
59
|
+
maxY: Number.isFinite(context?.deviceHeight) ? Math.round(context.deviceHeight) - 1 : undefined
|
|
56
60
|
});
|
|
57
61
|
addOutput({
|
|
58
62
|
type: 'action',
|
|
@@ -21,6 +21,18 @@ async function listConnectedDevices() {
|
|
|
21
21
|
})
|
|
22
22
|
.filter(entry => entry.id.length > 0);
|
|
23
23
|
}
|
|
24
|
+
export function getAndroidConnectErrorMessage({ state, avdName } = {}) {
|
|
25
|
+
if (state === "offline" && avdName) {
|
|
26
|
+
return `Emulator "${avdName}" is running but offline. Restart adb or cold boot the emulator, then retry.`;
|
|
27
|
+
}
|
|
28
|
+
if (state === "offline") {
|
|
29
|
+
return "Device is offline. Reconnect device or restart adb.";
|
|
30
|
+
}
|
|
31
|
+
if (state === "unauthorized") {
|
|
32
|
+
return "Authorize this computer on the device and retry.";
|
|
33
|
+
}
|
|
34
|
+
return "Selected device is no longer connected. Refresh and select again.";
|
|
35
|
+
}
|
|
24
36
|
function wait(ms) {
|
|
25
37
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
26
38
|
}
|
|
@@ -35,6 +47,14 @@ function parseTargetToken(rawTarget) {
|
|
|
35
47
|
}
|
|
36
48
|
return { kind: "legacy", value: rawTarget };
|
|
37
49
|
}
|
|
50
|
+
function normalizeAvdName(value) {
|
|
51
|
+
const parts = value
|
|
52
|
+
.split(/\r?\n/)
|
|
53
|
+
.map((line) => line.trim())
|
|
54
|
+
.filter(Boolean)
|
|
55
|
+
.filter((line) => line.toUpperCase() !== "OK");
|
|
56
|
+
return parts[0] || "";
|
|
57
|
+
}
|
|
38
58
|
async function getEmulatorAvdName(serial) {
|
|
39
59
|
const candidates = [
|
|
40
60
|
["-s", serial, "emu", "avd", "name"],
|
|
@@ -45,7 +65,7 @@ async function getEmulatorAvdName(serial) {
|
|
|
45
65
|
for (const args of candidates) {
|
|
46
66
|
try {
|
|
47
67
|
const { stdout } = await execAdb(args);
|
|
48
|
-
const value = stdout
|
|
68
|
+
const value = normalizeAvdName(stdout);
|
|
49
69
|
if (value)
|
|
50
70
|
return value;
|
|
51
71
|
}
|
|
@@ -74,16 +94,23 @@ function emulatorLaunchTimeoutError() {
|
|
|
74
94
|
return new Error(`Emulator launch timed out after ${AVD_READY_TIMEOUT_SECONDS}s. Check AVD health and retry.`);
|
|
75
95
|
}
|
|
76
96
|
async function findRunningEmulatorByAvdName(avdName) {
|
|
97
|
+
const match = await findConnectedEmulatorByAvdName(avdName, { includeOffline: false });
|
|
98
|
+
return match?.state === "device" ? match.id : null;
|
|
99
|
+
}
|
|
100
|
+
async function findConnectedEmulatorByAvdName(avdName, options = {}) {
|
|
77
101
|
const devices = await listConnectedDevices();
|
|
78
|
-
const
|
|
79
|
-
for (const entry of
|
|
102
|
+
const emulatorEntries = devices.filter((entry) => entry.id.startsWith("emulator-") && (options.includeOffline || entry.state === "device"));
|
|
103
|
+
for (const entry of emulatorEntries) {
|
|
80
104
|
const runningAvdName = await getEmulatorAvdName(entry.id);
|
|
81
105
|
if (runningAvdName === avdName) {
|
|
82
|
-
return entry
|
|
106
|
+
return entry;
|
|
83
107
|
}
|
|
84
108
|
}
|
|
85
109
|
return null;
|
|
86
110
|
}
|
|
111
|
+
function offlineEmulatorError(avdName) {
|
|
112
|
+
return new Error(getAndroidConnectErrorMessage({ state: "offline", avdName }));
|
|
113
|
+
}
|
|
87
114
|
async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, options = {}) {
|
|
88
115
|
const log = options.log || console.log;
|
|
89
116
|
let avdExists = false;
|
|
@@ -99,6 +126,10 @@ async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, op
|
|
|
99
126
|
if (!avdExists) {
|
|
100
127
|
throw new Error(`AVD "${avdName}" was not found. Create it in Android Studio and retry.`);
|
|
101
128
|
}
|
|
129
|
+
const offlineMatch = await findConnectedEmulatorByAvdName(avdName, { includeOffline: true });
|
|
130
|
+
if (offlineMatch?.state === "offline") {
|
|
131
|
+
throw offlineEmulatorError(avdName);
|
|
132
|
+
}
|
|
102
133
|
const baseline = await listConnectedDevices();
|
|
103
134
|
const baselineReadyEmulators = new Set(baseline.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-")).map((entry) => entry.id));
|
|
104
135
|
log(`Launching Android emulator "${avdName}"...`);
|
|
@@ -128,12 +159,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
128
159
|
const readyDevices = devices.filter((entry) => entry.state === "device");
|
|
129
160
|
if (parsedTarget.kind === "auto") {
|
|
130
161
|
if (readyDevices.length === 0) {
|
|
162
|
+
const offlineEmulator = devices.find((entry) => entry.state === "offline" && entry.id.startsWith("emulator-"));
|
|
163
|
+
const message = offlineEmulator
|
|
164
|
+
? "Android emulator is running but offline. Restart adb or cold boot the emulator, then retry."
|
|
165
|
+
: "No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.";
|
|
131
166
|
emitDesktopDebug("device.error", "device", {}, {
|
|
132
167
|
platform: "android",
|
|
133
168
|
operation: "connect",
|
|
134
|
-
message
|
|
169
|
+
message
|
|
135
170
|
});
|
|
136
|
-
throw new Error(
|
|
171
|
+
throw new Error(message);
|
|
137
172
|
}
|
|
138
173
|
const selected = readyDevices[0].id;
|
|
139
174
|
log(`Using connected Android device: ${selected}`);
|
|
@@ -142,15 +177,18 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
142
177
|
}
|
|
143
178
|
if (parsedTarget.kind === "avd") {
|
|
144
179
|
const deadline = Date.now() + AVD_READY_TIMEOUT_MS;
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
147
|
-
|
|
180
|
+
const existingEmulator = await findConnectedEmulatorByAvdName(parsedTarget.avdName, { includeOffline: true });
|
|
181
|
+
if (existingEmulator?.state === "offline") {
|
|
182
|
+
throw offlineEmulatorError(parsedTarget.avdName);
|
|
183
|
+
}
|
|
184
|
+
if (existingEmulator?.state === "device") {
|
|
185
|
+
const ready = await waitForWindowServiceReady(existingEmulator.id, remainingTimeoutMs(deadline));
|
|
148
186
|
if (!ready) {
|
|
149
187
|
throw emulatorLaunchTimeoutError();
|
|
150
188
|
}
|
|
151
|
-
log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${
|
|
152
|
-
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId:
|
|
153
|
-
return
|
|
189
|
+
log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${existingEmulator.id}`);
|
|
190
|
+
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: existingEmulator.id, reused: true });
|
|
191
|
+
return existingEmulator.id;
|
|
154
192
|
}
|
|
155
193
|
const launchedId = await launchAndWaitForAvd(parsedTarget.avdName, remainingTimeoutMs(deadline), { log });
|
|
156
194
|
const ready = await waitForWindowServiceReady(launchedId, remainingTimeoutMs(deadline));
|
|
@@ -179,19 +217,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
179
217
|
return targetId;
|
|
180
218
|
}
|
|
181
219
|
if (parsedTarget.kind === "legacy" && targetId) {
|
|
182
|
-
const
|
|
183
|
-
if (
|
|
184
|
-
|
|
185
|
-
|
|
220
|
+
const matchedLegacyAvd = await findConnectedEmulatorByAvdName(targetId, { includeOffline: true });
|
|
221
|
+
if (matchedLegacyAvd?.state === "offline") {
|
|
222
|
+
throw offlineEmulatorError(targetId);
|
|
223
|
+
}
|
|
224
|
+
if (matchedLegacyAvd?.state === "device") {
|
|
225
|
+
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: matchedLegacyAvd.id, reused: true });
|
|
226
|
+
return matchedLegacyAvd.id;
|
|
186
227
|
}
|
|
187
228
|
}
|
|
188
|
-
|
|
189
|
-
if (state === "offline") {
|
|
190
|
-
message = "Device is offline. Reconnect device or restart adb.";
|
|
191
|
-
}
|
|
192
|
-
else if (state === "unauthorized") {
|
|
193
|
-
message = "Authorize this computer on the device and retry.";
|
|
194
|
-
}
|
|
229
|
+
const message = getAndroidConnectErrorMessage({ state });
|
|
195
230
|
emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "connect", message, targetId: targetId || deviceTargetId, state: state || "missing" });
|
|
196
231
|
throw new Error(message);
|
|
197
232
|
}
|
|
@@ -3,21 +3,28 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { printCliOutput } from "../utils/console-output.js";
|
|
5
5
|
export { buildAssertionSystemPrompt } from "../prompts/execution.js";
|
|
6
|
+
const ASSERTION_PREFIXES = ["assert", "verify", "validate", "ensure"];
|
|
7
|
+
function hasAssertionPrefix(lower) {
|
|
8
|
+
return ASSERTION_PREFIXES.some((prefix) => lower === prefix || lower.startsWith(`${prefix}:`) || lower.startsWith(`${prefix} `));
|
|
9
|
+
}
|
|
6
10
|
export function isAssertion(userInput) {
|
|
7
11
|
const trimmed = userInput.trim();
|
|
8
12
|
const lower = trimmed.toLowerCase();
|
|
9
|
-
return
|
|
13
|
+
return hasAssertionPrefix(lower);
|
|
10
14
|
}
|
|
11
15
|
export function extractAssertionPrompt(userInput) {
|
|
12
16
|
const trimmed = userInput.trim();
|
|
13
17
|
const lower = trimmed.toLowerCase();
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
for (const prefix of ASSERTION_PREFIXES) {
|
|
19
|
+
if (lower === prefix) {
|
|
20
|
+
return trimmed;
|
|
21
|
+
}
|
|
22
|
+
if (lower.startsWith(`${prefix}:`)) {
|
|
23
|
+
return trimmed.substring(prefix.length + 1).trim();
|
|
24
|
+
}
|
|
25
|
+
if (lower.startsWith(`${prefix} `)) {
|
|
26
|
+
return trimmed.substring(prefix.length).trim();
|
|
27
|
+
}
|
|
21
28
|
}
|
|
22
29
|
return trimmed;
|
|
23
30
|
}
|
|
@@ -14,11 +14,11 @@ const execAsync = promisify(exec);
|
|
|
14
14
|
// Active session state
|
|
15
15
|
let activeSession = null;
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
18
|
-
* @param {string}
|
|
19
|
-
* @returns {Promise<string|null>}
|
|
17
|
+
* Resolve simulator by UDID or name (name matches prefer already-booted simulators).
|
|
18
|
+
* @param {string} simulatorNameOrUdid
|
|
19
|
+
* @returns {Promise<{name: string, udid: string, state: string}|null>}
|
|
20
20
|
*/
|
|
21
|
-
async function
|
|
21
|
+
async function resolveSimulator(simulatorNameOrUdid) {
|
|
22
22
|
try {
|
|
23
23
|
const { stdout } = await execAsync("xcrun simctl list devices --json");
|
|
24
24
|
const data = JSON.parse(stdout);
|
|
@@ -26,25 +26,31 @@ async function getSimulatorUDID(simulatorName) {
|
|
|
26
26
|
let bootedMatch = null;
|
|
27
27
|
for (const runtime of Object.values(data.devices)) {
|
|
28
28
|
for (const device of runtime) {
|
|
29
|
-
if (device.
|
|
29
|
+
if (!device.isAvailable) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (device.udid === simulatorNameOrUdid) {
|
|
33
|
+
return { name: device.name, udid: device.udid, state: device.state };
|
|
34
|
+
}
|
|
35
|
+
if (device.name === simulatorNameOrUdid) {
|
|
36
|
+
const match = { name: device.name, udid: device.udid, state: device.state };
|
|
30
37
|
if (!firstMatch) {
|
|
31
|
-
firstMatch =
|
|
38
|
+
firstMatch = match;
|
|
32
39
|
}
|
|
33
|
-
// Prefer the already-booted one
|
|
34
40
|
if (device.state === "Booted") {
|
|
35
|
-
bootedMatch =
|
|
41
|
+
bootedMatch = match;
|
|
36
42
|
}
|
|
37
43
|
}
|
|
38
44
|
}
|
|
39
45
|
}
|
|
40
46
|
if (bootedMatch) {
|
|
41
|
-
logger.debug(`Found booted simulator: ${bootedMatch}`);
|
|
47
|
+
logger.debug(`Found booted simulator: ${bootedMatch.udid}`);
|
|
42
48
|
return bootedMatch;
|
|
43
49
|
}
|
|
44
50
|
return firstMatch;
|
|
45
51
|
}
|
|
46
52
|
catch (error) {
|
|
47
|
-
logger.error("Failed to
|
|
53
|
+
logger.error("Failed to resolve simulator", { error: error.message });
|
|
48
54
|
}
|
|
49
55
|
return null;
|
|
50
56
|
}
|
|
@@ -84,33 +90,33 @@ async function findAnySimulator() {
|
|
|
84
90
|
*/
|
|
85
91
|
async function bootSimulator(simulatorName, options = {}) {
|
|
86
92
|
const log = options.log || console.log;
|
|
87
|
-
const
|
|
88
|
-
if (!
|
|
93
|
+
const simulator = await resolveSimulator(simulatorName);
|
|
94
|
+
if (!simulator) {
|
|
89
95
|
throw new Error(`Simulator "${simulatorName}" not found. Run "xcrun simctl list devices" to see available simulators.`);
|
|
90
96
|
}
|
|
91
|
-
log(`Found simulator "${
|
|
97
|
+
log(`Found simulator "${simulator.name}" with UDID: ${simulator.udid}`);
|
|
92
98
|
// Check if already booted
|
|
93
99
|
const { stdout } = await execAsync("xcrun simctl list devices --json");
|
|
94
100
|
const data = JSON.parse(stdout);
|
|
95
101
|
for (const runtime of Object.values(data.devices)) {
|
|
96
102
|
for (const device of runtime) {
|
|
97
|
-
if (device.udid === udid) {
|
|
103
|
+
if (device.udid === simulator.udid) {
|
|
98
104
|
if (device.state === "Booted") {
|
|
99
|
-
log(`Simulator "${
|
|
100
|
-
return
|
|
105
|
+
log(`Simulator "${simulator.name}" is already booted`);
|
|
106
|
+
return simulator;
|
|
101
107
|
}
|
|
102
108
|
break;
|
|
103
109
|
}
|
|
104
110
|
}
|
|
105
111
|
}
|
|
106
|
-
log(`Booting simulator "${
|
|
107
|
-
await execAsync(`xcrun simctl boot ${udid}`);
|
|
112
|
+
log(`Booting simulator "${simulator.name}"...`);
|
|
113
|
+
await execAsync(`xcrun simctl boot ${simulator.udid}`);
|
|
108
114
|
// Open Simulator app to show the device
|
|
109
115
|
await execAsync("open -a Simulator");
|
|
110
116
|
// Wait for boot to complete
|
|
111
|
-
await waitForSimulatorBoot(udid);
|
|
112
|
-
log(`Simulator "${
|
|
113
|
-
return
|
|
117
|
+
await waitForSimulatorBoot(simulator.udid);
|
|
118
|
+
log(`Simulator "${simulator.name}" is fully booted`);
|
|
119
|
+
return simulator;
|
|
114
120
|
}
|
|
115
121
|
/**
|
|
116
122
|
* Wait for simulator to finish booting
|
|
@@ -163,30 +169,30 @@ export async function connectToDevice(simulatorName, options = {}) {
|
|
|
163
169
|
log(`No simulator specified, using: ${simulatorName}`);
|
|
164
170
|
}
|
|
165
171
|
// Boot simulator
|
|
166
|
-
const
|
|
172
|
+
const simulator = await bootSimulator(simulatorName, { log });
|
|
167
173
|
// Create Appium session
|
|
168
174
|
log("Creating Appium session...");
|
|
169
175
|
const session = await appium.createSession({
|
|
170
176
|
platformName: "iOS",
|
|
171
177
|
"appium:automationName": "XCUITest",
|
|
172
|
-
"appium:deviceName":
|
|
173
|
-
"appium:udid": udid,
|
|
178
|
+
"appium:deviceName": simulator.name,
|
|
179
|
+
"appium:udid": simulator.udid,
|
|
174
180
|
"appium:noReset": true,
|
|
175
181
|
"appium:shouldTerminateApp": false,
|
|
176
182
|
});
|
|
177
183
|
activeSession = {
|
|
178
184
|
sessionId: session.sessionId,
|
|
179
|
-
udid,
|
|
180
|
-
simulatorName,
|
|
185
|
+
udid: simulator.udid,
|
|
186
|
+
simulatorName: simulator.name,
|
|
181
187
|
};
|
|
182
|
-
log(`Connected to simulator "${
|
|
188
|
+
log(`Connected to simulator "${simulator.name}" (${simulator.udid})`);
|
|
183
189
|
emitDesktopDebug("device.connect", "device", {}, {
|
|
184
190
|
platform: "ios",
|
|
185
191
|
stage: "success",
|
|
186
|
-
simulatorName,
|
|
187
|
-
udid
|
|
192
|
+
simulatorName: simulator.name,
|
|
193
|
+
udid: simulator.udid
|
|
188
194
|
});
|
|
189
|
-
return udid;
|
|
195
|
+
return simulator.udid;
|
|
190
196
|
}
|
|
191
197
|
/**
|
|
192
198
|
* Get device info (screen dimensions and scale)
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
|
|
2
2
|
import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
|
|
3
|
+
import { LOADMILL_CUA_MODELS } from "../core/cua-models.js";
|
|
3
4
|
import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
4
5
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
|
|
5
6
|
import { logger } from "../utils/logger.js";
|
|
6
7
|
import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
|
|
7
8
|
const cuaDebugTracer = new CuaDebugTracer(logger);
|
|
9
|
+
const LOADMILL_DEVICE_METADATA_MODELS = new Set([
|
|
10
|
+
...LOADMILL_CUA_MODELS,
|
|
11
|
+
LOADMILL_MODEL_ALIASES.SMART,
|
|
12
|
+
LOADMILL_MODEL_ALIASES.BEACON,
|
|
13
|
+
]);
|
|
8
14
|
function resolveOpenAiBaseUrl(client) {
|
|
9
15
|
return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
|
|
10
16
|
}
|
|
@@ -121,6 +127,8 @@ const typeSensitiveTextTool = {
|
|
|
121
127
|
additionalProperties: false,
|
|
122
128
|
},
|
|
123
129
|
};
|
|
130
|
+
// Design completion is the handoff into script generation; it is separate from
|
|
131
|
+
// execution step-completion markers used by model adapters.
|
|
124
132
|
const testDesignCompleteTool = {
|
|
125
133
|
type: "function",
|
|
126
134
|
name: "test_design_complete",
|
|
@@ -235,7 +243,64 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
|
|
|
235
243
|
}
|
|
236
244
|
return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
|
|
237
245
|
}
|
|
238
|
-
|
|
246
|
+
function normalizeDevicePlatform(deviceInfo = {}) {
|
|
247
|
+
return typeof deviceInfo?.platform === "string"
|
|
248
|
+
? deviceInfo.platform.trim().toLowerCase()
|
|
249
|
+
: "";
|
|
250
|
+
}
|
|
251
|
+
function resolveComputerToolDeviceType(deviceInfo = {}) {
|
|
252
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
253
|
+
if (platform === "browser") {
|
|
254
|
+
return "web";
|
|
255
|
+
}
|
|
256
|
+
if (platform === "ios") {
|
|
257
|
+
return "iphone";
|
|
258
|
+
}
|
|
259
|
+
if (platform === "android") {
|
|
260
|
+
return "android";
|
|
261
|
+
}
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
function resolveComputerToolEnvironment(deviceInfo = {}) {
|
|
265
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
266
|
+
if (platform === "browser") {
|
|
267
|
+
return "browser";
|
|
268
|
+
}
|
|
269
|
+
if (platform === "ios" || platform === "android") {
|
|
270
|
+
return "mobile";
|
|
271
|
+
}
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
function resolveDisplayDimension(value) {
|
|
275
|
+
return Number.isFinite(value) && value > 0 ? Math.round(value) : null;
|
|
276
|
+
}
|
|
277
|
+
function usesLoadmillDeviceMetadata(cuaModel) {
|
|
278
|
+
return LOADMILL_DEVICE_METADATA_MODELS.has(cuaModel);
|
|
279
|
+
}
|
|
280
|
+
function buildComputerTool({ cuaModel, deviceInfo } = {}) {
|
|
281
|
+
const tool = { type: "computer" };
|
|
282
|
+
if (!usesLoadmillDeviceMetadata(cuaModel)) {
|
|
283
|
+
return tool;
|
|
284
|
+
}
|
|
285
|
+
const environment = resolveComputerToolEnvironment(deviceInfo);
|
|
286
|
+
const deviceType = resolveComputerToolDeviceType(deviceInfo);
|
|
287
|
+
const displayWidth = resolveDisplayDimension(deviceInfo?.scaled_width);
|
|
288
|
+
const displayHeight = resolveDisplayDimension(deviceInfo?.scaled_height);
|
|
289
|
+
if (environment) {
|
|
290
|
+
tool.environment = environment;
|
|
291
|
+
}
|
|
292
|
+
if (deviceType) {
|
|
293
|
+
tool.device_type = deviceType;
|
|
294
|
+
}
|
|
295
|
+
if (displayWidth) {
|
|
296
|
+
tool.display_width = displayWidth;
|
|
297
|
+
}
|
|
298
|
+
if (displayHeight) {
|
|
299
|
+
tool.display_height = displayHeight;
|
|
300
|
+
}
|
|
301
|
+
return tool;
|
|
302
|
+
}
|
|
303
|
+
export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, deviceInfo = null, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
|
|
239
304
|
const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
|
|
240
305
|
// Keep Responses API storage enabled on every turn so execution can chain
|
|
241
306
|
// reliably through previous_response_id across the whole run.
|
|
@@ -248,7 +313,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
|
|
|
248
313
|
store: true,
|
|
249
314
|
truncation: "auto",
|
|
250
315
|
tools: [
|
|
251
|
-
...(includeComputerTool ? [{
|
|
316
|
+
...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
|
|
252
317
|
...buildFunctionTools({ aimAssistMode, toolCapabilities }),
|
|
253
318
|
],
|
|
254
319
|
};
|
|
@@ -294,6 +359,7 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
|
|
|
294
359
|
aimAssistMode,
|
|
295
360
|
previousResponseId,
|
|
296
361
|
input,
|
|
362
|
+
deviceInfo,
|
|
297
363
|
includeComputerTool,
|
|
298
364
|
contextOptimizationEnabled,
|
|
299
365
|
contextOptimizationThreshold,
|
|
@@ -1,20 +1,33 @@
|
|
|
1
1
|
// The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
|
|
2
2
|
// anchor coordinates `x` / `y`. This helper converts those model coordinates
|
|
3
3
|
// into backend gesture coordinates: startX/startY and endX/endY.
|
|
4
|
-
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
|
|
4
|
+
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0, minX, maxX, minY, maxY } = {}) {
|
|
5
|
+
const clamp = (value, min, max) => {
|
|
6
|
+
if (!Number.isFinite(value)) {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) {
|
|
10
|
+
return value;
|
|
11
|
+
}
|
|
12
|
+
return Math.min(Math.max(value, min), max);
|
|
13
|
+
};
|
|
5
14
|
const divisor = scale * dpr;
|
|
6
|
-
const
|
|
7
|
-
const
|
|
15
|
+
const requestedScrollX = Math.round((action?.scroll_x ?? 0) / divisor);
|
|
16
|
+
const requestedScrollY = Math.round((action?.scroll_y ?? 0) / divisor);
|
|
8
17
|
const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
|
|
9
|
-
const
|
|
10
|
-
const
|
|
18
|
+
const unclampedStartX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
|
|
19
|
+
const unclampedStartY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
|
|
20
|
+
const startX = clamp(unclampedStartX, minX, maxX);
|
|
21
|
+
const startY = clamp(unclampedStartY, minY, maxY);
|
|
22
|
+
const endX = clamp(startX + requestedScrollX, minX, maxX);
|
|
23
|
+
const endY = clamp(startY - requestedScrollY, minY, maxY);
|
|
11
24
|
return {
|
|
12
|
-
scrollX,
|
|
13
|
-
scrollY,
|
|
25
|
+
scrollX: endX - startX,
|
|
26
|
+
scrollY: startY - endY,
|
|
14
27
|
startX,
|
|
15
28
|
startY,
|
|
16
|
-
endX
|
|
17
|
-
endY
|
|
29
|
+
endX,
|
|
30
|
+
endY,
|
|
18
31
|
hasAnchor
|
|
19
32
|
};
|
|
20
33
|
}
|
|
@@ -138,10 +138,10 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
138
138
|
body: JSON.stringify(payload),
|
|
139
139
|
});
|
|
140
140
|
}
|
|
141
|
-
async function
|
|
141
|
+
async function validateApiToken() {
|
|
142
142
|
const resolvedToken = getApiToken();
|
|
143
143
|
if (!resolvedToken) {
|
|
144
|
-
return
|
|
144
|
+
return { valid: false, reason: "missing" };
|
|
145
145
|
}
|
|
146
146
|
const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
|
|
147
147
|
method: "GET",
|
|
@@ -150,11 +150,26 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
150
150
|
Authorization: `Bearer ${resolvedToken}`,
|
|
151
151
|
},
|
|
152
152
|
});
|
|
153
|
+
if (response.status === 401 || response.status === 403) {
|
|
154
|
+
return { valid: false, reason: "invalid", status: response.status };
|
|
155
|
+
}
|
|
153
156
|
if (!response.ok) {
|
|
154
|
-
return
|
|
157
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
const user = extractLoadmillSessionUser(await response.json());
|
|
161
|
+
return user
|
|
162
|
+
? { valid: true, user }
|
|
163
|
+
: { valid: false, reason: "unavailable", status: response.status };
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
155
167
|
}
|
|
168
|
+
}
|
|
169
|
+
async function getCurrentUser() {
|
|
156
170
|
try {
|
|
157
|
-
|
|
171
|
+
const validation = await validateApiToken();
|
|
172
|
+
return validation.valid ? validation.user : null;
|
|
158
173
|
}
|
|
159
174
|
catch {
|
|
160
175
|
return null;
|
|
@@ -172,6 +187,7 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
172
187
|
getTestSuiteFlowRun,
|
|
173
188
|
explainFailure,
|
|
174
189
|
reportDroidRun,
|
|
190
|
+
validateApiToken,
|
|
175
191
|
getCurrentUser,
|
|
176
192
|
};
|
|
177
193
|
}
|
|
@@ -206,7 +206,7 @@ export class DesignModeInk {
|
|
|
206
206
|
toolCapabilities: this.engine.toolCapabilities,
|
|
207
207
|
});
|
|
208
208
|
// Run turn with action tracking and interruption checking
|
|
209
|
-
const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
|
|
209
|
+
const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
|
|
210
210
|
// Check if user interrupted - stop immediately
|
|
211
211
|
if (this.inputQueue.length > 0) {
|
|
212
212
|
return true; // Signal to stop execution
|
|
@@ -223,6 +223,15 @@ export class DesignModeInk {
|
|
|
223
223
|
}
|
|
224
224
|
return false; // Continue execution
|
|
225
225
|
}, this.context, null, { currentScreenshot });
|
|
226
|
+
if (turnSummary.failed) {
|
|
227
|
+
if (this.context.setAgentWorking) {
|
|
228
|
+
this.context.setAgentWorking(false);
|
|
229
|
+
}
|
|
230
|
+
addOutput({ type: 'error', text: turnSummary.error || 'Design mode stopped.' });
|
|
231
|
+
this.conversationActive = false;
|
|
232
|
+
this.cleanup();
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
226
235
|
this.session.updateResponseId(newResponseId);
|
|
227
236
|
this.consecutiveErrorCount = 0;
|
|
228
237
|
// Clear agent working status
|
|
@@ -170,7 +170,7 @@ export class DesignMode {
|
|
|
170
170
|
toolCapabilities: this.engine.toolCapabilities,
|
|
171
171
|
});
|
|
172
172
|
// Run turn with action tracking and interruption checking
|
|
173
|
-
const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
|
|
173
|
+
const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
|
|
174
174
|
// Check if user pressed ESC - stop immediately
|
|
175
175
|
if (this.escPressed) {
|
|
176
176
|
return true; // Signal to stop execution
|
|
@@ -188,6 +188,12 @@ export class DesignMode {
|
|
|
188
188
|
}
|
|
189
189
|
return false; // Continue execution
|
|
190
190
|
}, null, null, { currentScreenshot });
|
|
191
|
+
if (turnSummary.failed) {
|
|
192
|
+
this.cleanupEscDetection(keypressHandler);
|
|
193
|
+
console.log(`\n${turnSummary.error || "Design mode stopped."}`);
|
|
194
|
+
this.conversationActive = false;
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
191
197
|
this.session.updateResponseId(newResponseId);
|
|
192
198
|
this.consecutiveErrorCount = 0;
|
|
193
199
|
// Cleanup ESC detection
|