@loadmill/droid-cua 2.21.0 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +31 -1
- package/build/src/browser/playwright/actions.js +65 -1
- package/build/src/cli/headless-runner.js +174 -12
- package/build/src/commands/help.js +1 -1
- package/build/src/core/cua-models.js +11 -4
- package/build/src/core/droid-cua-run-usage.js +106 -0
- package/build/src/core/droid-cua-segment-events.js +32 -0
- package/build/src/core/execution-engine.js +334 -5
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/report/render-html-report.js +8 -3
- package/build/src/core/report/run-log-model.js +42 -0
- package/build/src/core/runtime-config.js +24 -39
- package/build/src/core/session.js +6 -0
- package/build/src/core/tool-handlers/loadmill.js +3 -0
- package/build/src/core/tool-handlers/test-design-complete.js +1 -0
- package/build/src/core/tool-handlers/type-sensitive-text.js +1 -0
- package/build/src/device/android/actions.js +6 -2
- package/build/src/device/android/connection.js +58 -23
- package/build/src/device/assertions.js +15 -8
- package/build/src/device/ios/connection.js +36 -30
- package/build/src/device/openai.js +68 -2
- package/build/src/device/scroll-gesture.js +22 -9
- package/build/src/integrations/loadmill/client.js +62 -0
- package/build/src/integrations/loadmill/run-reporter.js +113 -0
- package/build/src/integrations/loadmill/session-user.js +39 -0
- package/build/src/integrations/segment/droid-cua-segment.js +64 -0
- package/build/src/modes/design-mode-ink.js +10 -1
- package/build/src/modes/design-mode.js +7 -1
- package/build/src/modes/execution-mode.js +84 -8
- package/build/src/prompts/design.js +21 -1
- package/build/src/utils/console-output.js +5 -0
- package/build/src/utils/cua-debug-tracer.js +50 -7
- package/package.json +2 -1
- package/build/src/core/cua-backends.js +0 -11
|
@@ -56,6 +56,12 @@ export class Session {
|
|
|
56
56
|
getTranscriptText() {
|
|
57
57
|
return this.transcript.join("\n");
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Clear the per-instruction transcript.
|
|
61
|
+
*/
|
|
62
|
+
clearTranscript() {
|
|
63
|
+
this.transcript = [];
|
|
64
|
+
}
|
|
59
65
|
/**
|
|
60
66
|
* Update the previous response ID for chaining
|
|
61
67
|
*/
|
|
@@ -29,6 +29,9 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
|
|
|
29
29
|
return {
|
|
30
30
|
kind: "close_turn",
|
|
31
31
|
outputItem: buildTextToolOutputItem(callId, "Loadmill command completed successfully."),
|
|
32
|
+
nextTurnConfig: {
|
|
33
|
+
attachFreshScreenshotOnContinuation: true,
|
|
34
|
+
},
|
|
32
35
|
summaryUpdates: result.runtimeParameterContext?.note
|
|
33
36
|
? { preserveSessionMessages: true }
|
|
34
37
|
: undefined,
|
|
@@ -51,8 +51,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
51
51
|
case "scroll":
|
|
52
52
|
const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
|
|
53
53
|
scale,
|
|
54
|
-
fallbackStartX: 500,
|
|
55
|
-
fallbackStartY: 500
|
|
54
|
+
fallbackStartX: Number.isFinite(context?.fallbackScrollStartX) ? Math.round(context.fallbackScrollStartX) : 500,
|
|
55
|
+
fallbackStartY: Number.isFinite(context?.fallbackScrollStartY) ? Math.round(context.fallbackScrollStartY) : 500,
|
|
56
|
+
minX: 0,
|
|
57
|
+
maxX: Number.isFinite(context?.deviceWidth) ? Math.round(context.deviceWidth) - 1 : undefined,
|
|
58
|
+
minY: 0,
|
|
59
|
+
maxY: Number.isFinite(context?.deviceHeight) ? Math.round(context.deviceHeight) - 1 : undefined
|
|
56
60
|
});
|
|
57
61
|
addOutput({
|
|
58
62
|
type: 'action',
|
|
@@ -21,6 +21,18 @@ async function listConnectedDevices() {
|
|
|
21
21
|
})
|
|
22
22
|
.filter(entry => entry.id.length > 0);
|
|
23
23
|
}
|
|
24
|
+
export function getAndroidConnectErrorMessage({ state, avdName } = {}) {
|
|
25
|
+
if (state === "offline" && avdName) {
|
|
26
|
+
return `Emulator "${avdName}" is running but offline. Restart adb or cold boot the emulator, then retry.`;
|
|
27
|
+
}
|
|
28
|
+
if (state === "offline") {
|
|
29
|
+
return "Device is offline. Reconnect device or restart adb.";
|
|
30
|
+
}
|
|
31
|
+
if (state === "unauthorized") {
|
|
32
|
+
return "Authorize this computer on the device and retry.";
|
|
33
|
+
}
|
|
34
|
+
return "Selected device is no longer connected. Refresh and select again.";
|
|
35
|
+
}
|
|
24
36
|
function wait(ms) {
|
|
25
37
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
26
38
|
}
|
|
@@ -35,6 +47,14 @@ function parseTargetToken(rawTarget) {
|
|
|
35
47
|
}
|
|
36
48
|
return { kind: "legacy", value: rawTarget };
|
|
37
49
|
}
|
|
50
|
+
function normalizeAvdName(value) {
|
|
51
|
+
const parts = value
|
|
52
|
+
.split(/\r?\n/)
|
|
53
|
+
.map((line) => line.trim())
|
|
54
|
+
.filter(Boolean)
|
|
55
|
+
.filter((line) => line.toUpperCase() !== "OK");
|
|
56
|
+
return parts[0] || "";
|
|
57
|
+
}
|
|
38
58
|
async function getEmulatorAvdName(serial) {
|
|
39
59
|
const candidates = [
|
|
40
60
|
["-s", serial, "emu", "avd", "name"],
|
|
@@ -45,7 +65,7 @@ async function getEmulatorAvdName(serial) {
|
|
|
45
65
|
for (const args of candidates) {
|
|
46
66
|
try {
|
|
47
67
|
const { stdout } = await execAdb(args);
|
|
48
|
-
const value = stdout
|
|
68
|
+
const value = normalizeAvdName(stdout);
|
|
49
69
|
if (value)
|
|
50
70
|
return value;
|
|
51
71
|
}
|
|
@@ -74,16 +94,23 @@ function emulatorLaunchTimeoutError() {
|
|
|
74
94
|
return new Error(`Emulator launch timed out after ${AVD_READY_TIMEOUT_SECONDS}s. Check AVD health and retry.`);
|
|
75
95
|
}
|
|
76
96
|
async function findRunningEmulatorByAvdName(avdName) {
|
|
97
|
+
const match = await findConnectedEmulatorByAvdName(avdName, { includeOffline: false });
|
|
98
|
+
return match?.state === "device" ? match.id : null;
|
|
99
|
+
}
|
|
100
|
+
async function findConnectedEmulatorByAvdName(avdName, options = {}) {
|
|
77
101
|
const devices = await listConnectedDevices();
|
|
78
|
-
const
|
|
79
|
-
for (const entry of
|
|
102
|
+
const emulatorEntries = devices.filter((entry) => entry.id.startsWith("emulator-") && (options.includeOffline || entry.state === "device"));
|
|
103
|
+
for (const entry of emulatorEntries) {
|
|
80
104
|
const runningAvdName = await getEmulatorAvdName(entry.id);
|
|
81
105
|
if (runningAvdName === avdName) {
|
|
82
|
-
return entry
|
|
106
|
+
return entry;
|
|
83
107
|
}
|
|
84
108
|
}
|
|
85
109
|
return null;
|
|
86
110
|
}
|
|
111
|
+
function offlineEmulatorError(avdName) {
|
|
112
|
+
return new Error(getAndroidConnectErrorMessage({ state: "offline", avdName }));
|
|
113
|
+
}
|
|
87
114
|
async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, options = {}) {
|
|
88
115
|
const log = options.log || console.log;
|
|
89
116
|
let avdExists = false;
|
|
@@ -99,6 +126,10 @@ async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, op
|
|
|
99
126
|
if (!avdExists) {
|
|
100
127
|
throw new Error(`AVD "${avdName}" was not found. Create it in Android Studio and retry.`);
|
|
101
128
|
}
|
|
129
|
+
const offlineMatch = await findConnectedEmulatorByAvdName(avdName, { includeOffline: true });
|
|
130
|
+
if (offlineMatch?.state === "offline") {
|
|
131
|
+
throw offlineEmulatorError(avdName);
|
|
132
|
+
}
|
|
102
133
|
const baseline = await listConnectedDevices();
|
|
103
134
|
const baselineReadyEmulators = new Set(baseline.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-")).map((entry) => entry.id));
|
|
104
135
|
log(`Launching Android emulator "${avdName}"...`);
|
|
@@ -128,12 +159,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
128
159
|
const readyDevices = devices.filter((entry) => entry.state === "device");
|
|
129
160
|
if (parsedTarget.kind === "auto") {
|
|
130
161
|
if (readyDevices.length === 0) {
|
|
162
|
+
const offlineEmulator = devices.find((entry) => entry.state === "offline" && entry.id.startsWith("emulator-"));
|
|
163
|
+
const message = offlineEmulator
|
|
164
|
+
? "Android emulator is running but offline. Restart adb or cold boot the emulator, then retry."
|
|
165
|
+
: "No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.";
|
|
131
166
|
emitDesktopDebug("device.error", "device", {}, {
|
|
132
167
|
platform: "android",
|
|
133
168
|
operation: "connect",
|
|
134
|
-
message
|
|
169
|
+
message
|
|
135
170
|
});
|
|
136
|
-
throw new Error(
|
|
171
|
+
throw new Error(message);
|
|
137
172
|
}
|
|
138
173
|
const selected = readyDevices[0].id;
|
|
139
174
|
log(`Using connected Android device: ${selected}`);
|
|
@@ -142,15 +177,18 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
142
177
|
}
|
|
143
178
|
if (parsedTarget.kind === "avd") {
|
|
144
179
|
const deadline = Date.now() + AVD_READY_TIMEOUT_MS;
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
147
|
-
|
|
180
|
+
const existingEmulator = await findConnectedEmulatorByAvdName(parsedTarget.avdName, { includeOffline: true });
|
|
181
|
+
if (existingEmulator?.state === "offline") {
|
|
182
|
+
throw offlineEmulatorError(parsedTarget.avdName);
|
|
183
|
+
}
|
|
184
|
+
if (existingEmulator?.state === "device") {
|
|
185
|
+
const ready = await waitForWindowServiceReady(existingEmulator.id, remainingTimeoutMs(deadline));
|
|
148
186
|
if (!ready) {
|
|
149
187
|
throw emulatorLaunchTimeoutError();
|
|
150
188
|
}
|
|
151
|
-
log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${
|
|
152
|
-
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId:
|
|
153
|
-
return
|
|
189
|
+
log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${existingEmulator.id}`);
|
|
190
|
+
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: existingEmulator.id, reused: true });
|
|
191
|
+
return existingEmulator.id;
|
|
154
192
|
}
|
|
155
193
|
const launchedId = await launchAndWaitForAvd(parsedTarget.avdName, remainingTimeoutMs(deadline), { log });
|
|
156
194
|
const ready = await waitForWindowServiceReady(launchedId, remainingTimeoutMs(deadline));
|
|
@@ -179,19 +217,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
|
|
|
179
217
|
return targetId;
|
|
180
218
|
}
|
|
181
219
|
if (parsedTarget.kind === "legacy" && targetId) {
|
|
182
|
-
const
|
|
183
|
-
if (
|
|
184
|
-
|
|
185
|
-
|
|
220
|
+
const matchedLegacyAvd = await findConnectedEmulatorByAvdName(targetId, { includeOffline: true });
|
|
221
|
+
if (matchedLegacyAvd?.state === "offline") {
|
|
222
|
+
throw offlineEmulatorError(targetId);
|
|
223
|
+
}
|
|
224
|
+
if (matchedLegacyAvd?.state === "device") {
|
|
225
|
+
emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: matchedLegacyAvd.id, reused: true });
|
|
226
|
+
return matchedLegacyAvd.id;
|
|
186
227
|
}
|
|
187
228
|
}
|
|
188
|
-
|
|
189
|
-
if (state === "offline") {
|
|
190
|
-
message = "Device is offline. Reconnect device or restart adb.";
|
|
191
|
-
}
|
|
192
|
-
else if (state === "unauthorized") {
|
|
193
|
-
message = "Authorize this computer on the device and retry.";
|
|
194
|
-
}
|
|
229
|
+
const message = getAndroidConnectErrorMessage({ state });
|
|
195
230
|
emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "connect", message, targetId: targetId || deviceTargetId, state: state || "missing" });
|
|
196
231
|
throw new Error(message);
|
|
197
232
|
}
|
|
@@ -3,21 +3,28 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { printCliOutput } from "../utils/console-output.js";
|
|
5
5
|
export { buildAssertionSystemPrompt } from "../prompts/execution.js";
|
|
6
|
+
const ASSERTION_PREFIXES = ["assert", "verify", "validate", "ensure"];
|
|
7
|
+
function hasAssertionPrefix(lower) {
|
|
8
|
+
return ASSERTION_PREFIXES.some((prefix) => lower === prefix || lower.startsWith(`${prefix}:`) || lower.startsWith(`${prefix} `));
|
|
9
|
+
}
|
|
6
10
|
export function isAssertion(userInput) {
|
|
7
11
|
const trimmed = userInput.trim();
|
|
8
12
|
const lower = trimmed.toLowerCase();
|
|
9
|
-
return
|
|
13
|
+
return hasAssertionPrefix(lower);
|
|
10
14
|
}
|
|
11
15
|
export function extractAssertionPrompt(userInput) {
|
|
12
16
|
const trimmed = userInput.trim();
|
|
13
17
|
const lower = trimmed.toLowerCase();
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
for (const prefix of ASSERTION_PREFIXES) {
|
|
19
|
+
if (lower === prefix) {
|
|
20
|
+
return trimmed;
|
|
21
|
+
}
|
|
22
|
+
if (lower.startsWith(`${prefix}:`)) {
|
|
23
|
+
return trimmed.substring(prefix.length + 1).trim();
|
|
24
|
+
}
|
|
25
|
+
if (lower.startsWith(`${prefix} `)) {
|
|
26
|
+
return trimmed.substring(prefix.length).trim();
|
|
27
|
+
}
|
|
21
28
|
}
|
|
22
29
|
return trimmed;
|
|
23
30
|
}
|
|
@@ -14,11 +14,11 @@ const execAsync = promisify(exec);
|
|
|
14
14
|
// Active session state
|
|
15
15
|
let activeSession = null;
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
18
|
-
* @param {string}
|
|
19
|
-
* @returns {Promise<string|null>}
|
|
17
|
+
* Resolve simulator by UDID or name (name matches prefer already-booted simulators).
|
|
18
|
+
* @param {string} simulatorNameOrUdid
|
|
19
|
+
* @returns {Promise<{name: string, udid: string, state: string}|null>}
|
|
20
20
|
*/
|
|
21
|
-
async function
|
|
21
|
+
async function resolveSimulator(simulatorNameOrUdid) {
|
|
22
22
|
try {
|
|
23
23
|
const { stdout } = await execAsync("xcrun simctl list devices --json");
|
|
24
24
|
const data = JSON.parse(stdout);
|
|
@@ -26,25 +26,31 @@ async function getSimulatorUDID(simulatorName) {
|
|
|
26
26
|
let bootedMatch = null;
|
|
27
27
|
for (const runtime of Object.values(data.devices)) {
|
|
28
28
|
for (const device of runtime) {
|
|
29
|
-
if (device.
|
|
29
|
+
if (!device.isAvailable) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (device.udid === simulatorNameOrUdid) {
|
|
33
|
+
return { name: device.name, udid: device.udid, state: device.state };
|
|
34
|
+
}
|
|
35
|
+
if (device.name === simulatorNameOrUdid) {
|
|
36
|
+
const match = { name: device.name, udid: device.udid, state: device.state };
|
|
30
37
|
if (!firstMatch) {
|
|
31
|
-
firstMatch =
|
|
38
|
+
firstMatch = match;
|
|
32
39
|
}
|
|
33
|
-
// Prefer the already-booted one
|
|
34
40
|
if (device.state === "Booted") {
|
|
35
|
-
bootedMatch =
|
|
41
|
+
bootedMatch = match;
|
|
36
42
|
}
|
|
37
43
|
}
|
|
38
44
|
}
|
|
39
45
|
}
|
|
40
46
|
if (bootedMatch) {
|
|
41
|
-
logger.debug(`Found booted simulator: ${bootedMatch}`);
|
|
47
|
+
logger.debug(`Found booted simulator: ${bootedMatch.udid}`);
|
|
42
48
|
return bootedMatch;
|
|
43
49
|
}
|
|
44
50
|
return firstMatch;
|
|
45
51
|
}
|
|
46
52
|
catch (error) {
|
|
47
|
-
logger.error("Failed to
|
|
53
|
+
logger.error("Failed to resolve simulator", { error: error.message });
|
|
48
54
|
}
|
|
49
55
|
return null;
|
|
50
56
|
}
|
|
@@ -84,33 +90,33 @@ async function findAnySimulator() {
|
|
|
84
90
|
*/
|
|
85
91
|
async function bootSimulator(simulatorName, options = {}) {
|
|
86
92
|
const log = options.log || console.log;
|
|
87
|
-
const
|
|
88
|
-
if (!
|
|
93
|
+
const simulator = await resolveSimulator(simulatorName);
|
|
94
|
+
if (!simulator) {
|
|
89
95
|
throw new Error(`Simulator "${simulatorName}" not found. Run "xcrun simctl list devices" to see available simulators.`);
|
|
90
96
|
}
|
|
91
|
-
log(`Found simulator "${
|
|
97
|
+
log(`Found simulator "${simulator.name}" with UDID: ${simulator.udid}`);
|
|
92
98
|
// Check if already booted
|
|
93
99
|
const { stdout } = await execAsync("xcrun simctl list devices --json");
|
|
94
100
|
const data = JSON.parse(stdout);
|
|
95
101
|
for (const runtime of Object.values(data.devices)) {
|
|
96
102
|
for (const device of runtime) {
|
|
97
|
-
if (device.udid === udid) {
|
|
103
|
+
if (device.udid === simulator.udid) {
|
|
98
104
|
if (device.state === "Booted") {
|
|
99
|
-
log(`Simulator "${
|
|
100
|
-
return
|
|
105
|
+
log(`Simulator "${simulator.name}" is already booted`);
|
|
106
|
+
return simulator;
|
|
101
107
|
}
|
|
102
108
|
break;
|
|
103
109
|
}
|
|
104
110
|
}
|
|
105
111
|
}
|
|
106
|
-
log(`Booting simulator "${
|
|
107
|
-
await execAsync(`xcrun simctl boot ${udid}`);
|
|
112
|
+
log(`Booting simulator "${simulator.name}"...`);
|
|
113
|
+
await execAsync(`xcrun simctl boot ${simulator.udid}`);
|
|
108
114
|
// Open Simulator app to show the device
|
|
109
115
|
await execAsync("open -a Simulator");
|
|
110
116
|
// Wait for boot to complete
|
|
111
|
-
await waitForSimulatorBoot(udid);
|
|
112
|
-
log(`Simulator "${
|
|
113
|
-
return
|
|
117
|
+
await waitForSimulatorBoot(simulator.udid);
|
|
118
|
+
log(`Simulator "${simulator.name}" is fully booted`);
|
|
119
|
+
return simulator;
|
|
114
120
|
}
|
|
115
121
|
/**
|
|
116
122
|
* Wait for simulator to finish booting
|
|
@@ -163,30 +169,30 @@ export async function connectToDevice(simulatorName, options = {}) {
|
|
|
163
169
|
log(`No simulator specified, using: ${simulatorName}`);
|
|
164
170
|
}
|
|
165
171
|
// Boot simulator
|
|
166
|
-
const
|
|
172
|
+
const simulator = await bootSimulator(simulatorName, { log });
|
|
167
173
|
// Create Appium session
|
|
168
174
|
log("Creating Appium session...");
|
|
169
175
|
const session = await appium.createSession({
|
|
170
176
|
platformName: "iOS",
|
|
171
177
|
"appium:automationName": "XCUITest",
|
|
172
|
-
"appium:deviceName":
|
|
173
|
-
"appium:udid": udid,
|
|
178
|
+
"appium:deviceName": simulator.name,
|
|
179
|
+
"appium:udid": simulator.udid,
|
|
174
180
|
"appium:noReset": true,
|
|
175
181
|
"appium:shouldTerminateApp": false,
|
|
176
182
|
});
|
|
177
183
|
activeSession = {
|
|
178
184
|
sessionId: session.sessionId,
|
|
179
|
-
udid,
|
|
180
|
-
simulatorName,
|
|
185
|
+
udid: simulator.udid,
|
|
186
|
+
simulatorName: simulator.name,
|
|
181
187
|
};
|
|
182
|
-
log(`Connected to simulator "${
|
|
188
|
+
log(`Connected to simulator "${simulator.name}" (${simulator.udid})`);
|
|
183
189
|
emitDesktopDebug("device.connect", "device", {}, {
|
|
184
190
|
platform: "ios",
|
|
185
191
|
stage: "success",
|
|
186
|
-
simulatorName,
|
|
187
|
-
udid
|
|
192
|
+
simulatorName: simulator.name,
|
|
193
|
+
udid: simulator.udid
|
|
188
194
|
});
|
|
189
|
-
return udid;
|
|
195
|
+
return simulator.udid;
|
|
190
196
|
}
|
|
191
197
|
/**
|
|
192
198
|
* Get device info (screen dimensions and scale)
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
|
|
2
2
|
import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
|
|
3
|
+
import { LOADMILL_CUA_MODELS } from "../core/cua-models.js";
|
|
3
4
|
import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
4
5
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
|
|
5
6
|
import { logger } from "../utils/logger.js";
|
|
6
7
|
import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
|
|
7
8
|
const cuaDebugTracer = new CuaDebugTracer(logger);
|
|
9
|
+
const LOADMILL_DEVICE_METADATA_MODELS = new Set([
|
|
10
|
+
...LOADMILL_CUA_MODELS,
|
|
11
|
+
LOADMILL_MODEL_ALIASES.SMART,
|
|
12
|
+
LOADMILL_MODEL_ALIASES.BEACON,
|
|
13
|
+
]);
|
|
8
14
|
function resolveOpenAiBaseUrl(client) {
|
|
9
15
|
return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
|
|
10
16
|
}
|
|
@@ -121,6 +127,8 @@ const typeSensitiveTextTool = {
|
|
|
121
127
|
additionalProperties: false,
|
|
122
128
|
},
|
|
123
129
|
};
|
|
130
|
+
// Design completion is the handoff into script generation; it is separate from
|
|
131
|
+
// execution step-completion markers used by model adapters.
|
|
124
132
|
const testDesignCompleteTool = {
|
|
125
133
|
type: "function",
|
|
126
134
|
name: "test_design_complete",
|
|
@@ -235,7 +243,64 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
|
|
|
235
243
|
}
|
|
236
244
|
return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
|
|
237
245
|
}
|
|
238
|
-
|
|
246
|
+
function normalizeDevicePlatform(deviceInfo = {}) {
|
|
247
|
+
return typeof deviceInfo?.platform === "string"
|
|
248
|
+
? deviceInfo.platform.trim().toLowerCase()
|
|
249
|
+
: "";
|
|
250
|
+
}
|
|
251
|
+
function resolveComputerToolDeviceType(deviceInfo = {}) {
|
|
252
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
253
|
+
if (platform === "browser") {
|
|
254
|
+
return "web";
|
|
255
|
+
}
|
|
256
|
+
if (platform === "ios") {
|
|
257
|
+
return "iphone";
|
|
258
|
+
}
|
|
259
|
+
if (platform === "android") {
|
|
260
|
+
return "android";
|
|
261
|
+
}
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
function resolveComputerToolEnvironment(deviceInfo = {}) {
|
|
265
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
266
|
+
if (platform === "browser") {
|
|
267
|
+
return "browser";
|
|
268
|
+
}
|
|
269
|
+
if (platform === "ios" || platform === "android") {
|
|
270
|
+
return "mobile";
|
|
271
|
+
}
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
function resolveDisplayDimension(value) {
|
|
275
|
+
return Number.isFinite(value) && value > 0 ? Math.round(value) : null;
|
|
276
|
+
}
|
|
277
|
+
function usesLoadmillDeviceMetadata(cuaModel) {
|
|
278
|
+
return LOADMILL_DEVICE_METADATA_MODELS.has(cuaModel);
|
|
279
|
+
}
|
|
280
|
+
function buildComputerTool({ cuaModel, deviceInfo } = {}) {
|
|
281
|
+
const tool = { type: "computer" };
|
|
282
|
+
if (!usesLoadmillDeviceMetadata(cuaModel)) {
|
|
283
|
+
return tool;
|
|
284
|
+
}
|
|
285
|
+
const environment = resolveComputerToolEnvironment(deviceInfo);
|
|
286
|
+
const deviceType = resolveComputerToolDeviceType(deviceInfo);
|
|
287
|
+
const displayWidth = resolveDisplayDimension(deviceInfo?.scaled_width);
|
|
288
|
+
const displayHeight = resolveDisplayDimension(deviceInfo?.scaled_height);
|
|
289
|
+
if (environment) {
|
|
290
|
+
tool.environment = environment;
|
|
291
|
+
}
|
|
292
|
+
if (deviceType) {
|
|
293
|
+
tool.device_type = deviceType;
|
|
294
|
+
}
|
|
295
|
+
if (displayWidth) {
|
|
296
|
+
tool.display_width = displayWidth;
|
|
297
|
+
}
|
|
298
|
+
if (displayHeight) {
|
|
299
|
+
tool.display_height = displayHeight;
|
|
300
|
+
}
|
|
301
|
+
return tool;
|
|
302
|
+
}
|
|
303
|
+
export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, deviceInfo = null, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
|
|
239
304
|
const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
|
|
240
305
|
// Keep Responses API storage enabled on every turn so execution can chain
|
|
241
306
|
// reliably through previous_response_id across the whole run.
|
|
@@ -248,7 +313,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
|
|
|
248
313
|
store: true,
|
|
249
314
|
truncation: "auto",
|
|
250
315
|
tools: [
|
|
251
|
-
...(includeComputerTool ? [{
|
|
316
|
+
...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
|
|
252
317
|
...buildFunctionTools({ aimAssistMode, toolCapabilities }),
|
|
253
318
|
],
|
|
254
319
|
};
|
|
@@ -294,6 +359,7 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
|
|
|
294
359
|
aimAssistMode,
|
|
295
360
|
previousResponseId,
|
|
296
361
|
input,
|
|
362
|
+
deviceInfo,
|
|
297
363
|
includeComputerTool,
|
|
298
364
|
contextOptimizationEnabled,
|
|
299
365
|
contextOptimizationThreshold,
|
|
@@ -1,20 +1,33 @@
|
|
|
1
1
|
// The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
|
|
2
2
|
// anchor coordinates `x` / `y`. This helper converts those model coordinates
|
|
3
3
|
// into backend gesture coordinates: startX/startY and endX/endY.
|
|
4
|
-
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
|
|
4
|
+
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0, minX, maxX, minY, maxY } = {}) {
|
|
5
|
+
const clamp = (value, min, max) => {
|
|
6
|
+
if (!Number.isFinite(value)) {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) {
|
|
10
|
+
return value;
|
|
11
|
+
}
|
|
12
|
+
return Math.min(Math.max(value, min), max);
|
|
13
|
+
};
|
|
5
14
|
const divisor = scale * dpr;
|
|
6
|
-
const
|
|
7
|
-
const
|
|
15
|
+
const requestedScrollX = Math.round((action?.scroll_x ?? 0) / divisor);
|
|
16
|
+
const requestedScrollY = Math.round((action?.scroll_y ?? 0) / divisor);
|
|
8
17
|
const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
|
|
9
|
-
const
|
|
10
|
-
const
|
|
18
|
+
const unclampedStartX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
|
|
19
|
+
const unclampedStartY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
|
|
20
|
+
const startX = clamp(unclampedStartX, minX, maxX);
|
|
21
|
+
const startY = clamp(unclampedStartY, minY, maxY);
|
|
22
|
+
const endX = clamp(startX + requestedScrollX, minX, maxX);
|
|
23
|
+
const endY = clamp(startY - requestedScrollY, minY, maxY);
|
|
11
24
|
return {
|
|
12
|
-
scrollX,
|
|
13
|
-
scrollY,
|
|
25
|
+
scrollX: endX - startX,
|
|
26
|
+
scrollY: startY - endY,
|
|
14
27
|
startX,
|
|
15
28
|
startY,
|
|
16
|
-
endX
|
|
17
|
-
endY
|
|
29
|
+
endX,
|
|
30
|
+
endY,
|
|
18
31
|
hasAnchor
|
|
19
32
|
};
|
|
20
33
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Loadmill API client for interacting with test flows
|
|
3
3
|
*/
|
|
4
|
+
import { extractLoadmillSessionUser } from "./session-user.js";
|
|
4
5
|
const DEFAULT_BASE_URL = "https://app.loadmill.com/api";
|
|
5
6
|
function normalizeApiBaseUrl(rawBaseUrl) {
|
|
6
7
|
const candidate = (rawBaseUrl || DEFAULT_BASE_URL).trim();
|
|
@@ -19,6 +20,10 @@ function normalizeApiBaseUrl(rawBaseUrl) {
|
|
|
19
20
|
return DEFAULT_BASE_URL;
|
|
20
21
|
}
|
|
21
22
|
}
|
|
23
|
+
function normalizeSiteBaseUrl(rawBaseUrl) {
|
|
24
|
+
const apiBaseUrl = normalizeApiBaseUrl(rawBaseUrl);
|
|
25
|
+
return apiBaseUrl.replace(/\/api$/, "");
|
|
26
|
+
}
|
|
22
27
|
/**
|
|
23
28
|
* @param {{
|
|
24
29
|
* token?: string | null,
|
|
@@ -33,6 +38,9 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
33
38
|
function getBaseUrl() {
|
|
34
39
|
return normalizeApiBaseUrl(baseUrl);
|
|
35
40
|
}
|
|
41
|
+
function getSiteBaseUrl() {
|
|
42
|
+
return normalizeSiteBaseUrl(baseUrl);
|
|
43
|
+
}
|
|
36
44
|
async function apiRequest(endpoint, options = {}) {
|
|
37
45
|
const resolvedToken = getApiToken();
|
|
38
46
|
if (!resolvedToken) {
|
|
@@ -124,9 +132,53 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
124
132
|
}),
|
|
125
133
|
});
|
|
126
134
|
}
|
|
135
|
+
async function reportDroidRun(payload) {
|
|
136
|
+
return await apiRequest(`/droid-cua/runs`, {
|
|
137
|
+
method: "POST",
|
|
138
|
+
body: JSON.stringify(payload),
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
async function validateApiToken() {
|
|
142
|
+
const resolvedToken = getApiToken();
|
|
143
|
+
if (!resolvedToken) {
|
|
144
|
+
return { valid: false, reason: "missing" };
|
|
145
|
+
}
|
|
146
|
+
const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
|
|
147
|
+
method: "GET",
|
|
148
|
+
headers: {
|
|
149
|
+
Accept: "application/json",
|
|
150
|
+
Authorization: `Bearer ${resolvedToken}`,
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
if (response.status === 401 || response.status === 403) {
|
|
154
|
+
return { valid: false, reason: "invalid", status: response.status };
|
|
155
|
+
}
|
|
156
|
+
if (!response.ok) {
|
|
157
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
const user = extractLoadmillSessionUser(await response.json());
|
|
161
|
+
return user
|
|
162
|
+
? { valid: true, user }
|
|
163
|
+
: { valid: false, reason: "unavailable", status: response.status };
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
async function getCurrentUser() {
|
|
170
|
+
try {
|
|
171
|
+
const validation = await validateApiToken();
|
|
172
|
+
return validation.valid ? validation.user : null;
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
127
178
|
return {
|
|
128
179
|
getApiToken,
|
|
129
180
|
getBaseUrl,
|
|
181
|
+
getSiteBaseUrl,
|
|
130
182
|
searchFlows,
|
|
131
183
|
runTestSuite,
|
|
132
184
|
runTestFlow,
|
|
@@ -134,6 +186,9 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
134
186
|
getFlowRunStatus,
|
|
135
187
|
getTestSuiteFlowRun,
|
|
136
188
|
explainFailure,
|
|
189
|
+
reportDroidRun,
|
|
190
|
+
validateApiToken,
|
|
191
|
+
getCurrentUser,
|
|
137
192
|
};
|
|
138
193
|
}
|
|
139
194
|
export function getApiToken() {
|
|
@@ -185,6 +240,13 @@ export async function getFlowRunStatus(flowRunId) {
|
|
|
185
240
|
onUnauthorized: globalThis.__DROID_CUA_HANDLE_LOADMILL_UNAUTHORIZED__,
|
|
186
241
|
}).getFlowRunStatus(flowRunId);
|
|
187
242
|
}
|
|
243
|
+
export async function reportDroidRun(payload) {
|
|
244
|
+
return await createLoadmillApiClient({
|
|
245
|
+
token: process.env.LOADMILL_API_TOKEN,
|
|
246
|
+
baseUrl: process.env.LOADMILL_BASE_URL,
|
|
247
|
+
onUnauthorized: globalThis.__DROID_CUA_HANDLE_LOADMILL_UNAUTHORIZED__,
|
|
248
|
+
}).reportDroidRun(payload);
|
|
249
|
+
}
|
|
188
250
|
export async function getTestSuiteFlowRun(flowRunId) {
|
|
189
251
|
return await createLoadmillApiClient({
|
|
190
252
|
token: process.env.LOADMILL_API_TOKEN,
|