@loadmill/droid-cua 2.22.0 → 2.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +25 -0
  2. package/build/index.js +43 -1
  3. package/build/src/browser/playwright/actions.js +65 -1
  4. package/build/src/cli/skill-command.js +240 -0
  5. package/build/src/commands/help.js +2 -1
  6. package/build/src/core/cua-models.js +29 -11
  7. package/build/src/core/droid-cua-run-usage.js +16 -7
  8. package/build/src/core/execution-engine.js +334 -5
  9. package/build/src/core/prompts.js +1 -1
  10. package/build/src/core/report/render-html-report.js +8 -3
  11. package/build/src/core/report/run-log-model.js +42 -0
  12. package/build/src/core/runtime-config.js +23 -41
  13. package/build/src/core/session.js +6 -0
  14. package/build/src/core/tool-handlers/loadmill.js +3 -0
  15. package/build/src/core/tool-handlers/test-design-complete.js +1 -0
  16. package/build/src/core/tool-handlers/type-sensitive-text.js +1 -0
  17. package/build/src/device/android/actions.js +6 -2
  18. package/build/src/device/android/connection.js +58 -23
  19. package/build/src/device/assertions.js +15 -8
  20. package/build/src/device/ios/connection.js +36 -30
  21. package/build/src/device/openai.js +68 -2
  22. package/build/src/device/scroll-gesture.js +22 -9
  23. package/build/src/integrations/loadmill/client.js +20 -4
  24. package/build/src/modes/design-mode-ink.js +10 -1
  25. package/build/src/modes/design-mode.js +7 -1
  26. package/build/src/modes/execution-mode.js +84 -8
  27. package/build/src/prompts/design.js +21 -1
  28. package/build/src/utils/console-output.js +5 -0
  29. package/build/src/utils/cua-debug-tracer.js +50 -7
  30. package/package.json +2 -1
  31. package/skill/droid-cua/SKILL.md +36 -0
  32. package/skill/droid-cua/agents/openai.yaml +6 -0
  33. package/skill/droid-cua/assets/droid-cua-app-icon.png +0 -0
  34. package/skill/droid-cua/references/authoring-tests.md +39 -0
  35. package/skill/droid-cua/references/onboarding-and-troubleshooting.md +74 -0
  36. package/skill/droid-cua/references/project-and-auth.md +38 -0
  37. package/skill/droid-cua/references/running-and-healing.md +70 -0
  38. package/build/src/core/cua-backends.js +0 -11
@@ -56,6 +56,12 @@ export class Session {
56
56
  getTranscriptText() {
57
57
  return this.transcript.join("\n");
58
58
  }
59
+ /**
60
+ * Clear the per-instruction transcript.
61
+ */
62
+ clearTranscript() {
63
+ this.transcript = [];
64
+ }
59
65
  /**
60
66
  * Update the previous response ID for chaining
61
67
  */
@@ -29,6 +29,9 @@ export async function handleLoadmillToolCall({ engine, callId, argumentsText, co
29
29
  return {
30
30
  kind: "close_turn",
31
31
  outputItem: buildTextToolOutputItem(callId, "Loadmill command completed successfully."),
32
+ nextTurnConfig: {
33
+ attachFreshScreenshotOnContinuation: true,
34
+ },
32
35
  summaryUpdates: result.runtimeParameterContext?.note
33
36
  ? { preserveSessionMessages: true }
34
37
  : undefined,
@@ -88,6 +88,7 @@ export async function handleTestDesignCompleteToolCall({ engine, callId, argumen
88
88
  testDesignComplete: false,
89
89
  },
90
90
  includeComputerToolOverride: false,
91
+ continuationMode: "fresh",
91
92
  },
92
93
  };
93
94
  }
@@ -58,5 +58,6 @@ export async function handleTypeSensitiveTextToolCall({ engine, callId, argument
58
58
  imageBase64: screenshotBase64,
59
59
  })),
60
60
  currentScreenshot,
61
+ executedActions: [action],
61
62
  };
62
63
  }
@@ -51,8 +51,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
51
51
  case "scroll":
52
52
  const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
53
53
  scale,
54
- fallbackStartX: 500,
55
- fallbackStartY: 500
54
+ fallbackStartX: Number.isFinite(context?.fallbackScrollStartX) ? Math.round(context.fallbackScrollStartX) : 500,
55
+ fallbackStartY: Number.isFinite(context?.fallbackScrollStartY) ? Math.round(context.fallbackScrollStartY) : 500,
56
+ minX: 0,
57
+ maxX: Number.isFinite(context?.deviceWidth) ? Math.round(context.deviceWidth) - 1 : undefined,
58
+ minY: 0,
59
+ maxY: Number.isFinite(context?.deviceHeight) ? Math.round(context.deviceHeight) - 1 : undefined
56
60
  });
57
61
  addOutput({
58
62
  type: 'action',
@@ -21,6 +21,18 @@ async function listConnectedDevices() {
21
21
  })
22
22
  .filter(entry => entry.id.length > 0);
23
23
  }
24
+ export function getAndroidConnectErrorMessage({ state, avdName } = {}) {
25
+ if (state === "offline" && avdName) {
26
+ return `Emulator "${avdName}" is running but offline. Restart adb or cold boot the emulator, then retry.`;
27
+ }
28
+ if (state === "offline") {
29
+ return "Device is offline. Reconnect device or restart adb.";
30
+ }
31
+ if (state === "unauthorized") {
32
+ return "Authorize this computer on the device and retry.";
33
+ }
34
+ return "Selected device is no longer connected. Refresh and select again.";
35
+ }
24
36
  function wait(ms) {
25
37
  return new Promise((resolve) => setTimeout(resolve, ms));
26
38
  }
@@ -35,6 +47,14 @@ function parseTargetToken(rawTarget) {
35
47
  }
36
48
  return { kind: "legacy", value: rawTarget };
37
49
  }
50
+ function normalizeAvdName(value) {
51
+ const parts = value
52
+ .split(/\r?\n/)
53
+ .map((line) => line.trim())
54
+ .filter(Boolean)
55
+ .filter((line) => line.toUpperCase() !== "OK");
56
+ return parts[0] || "";
57
+ }
38
58
  async function getEmulatorAvdName(serial) {
39
59
  const candidates = [
40
60
  ["-s", serial, "emu", "avd", "name"],
@@ -45,7 +65,7 @@ async function getEmulatorAvdName(serial) {
45
65
  for (const args of candidates) {
46
66
  try {
47
67
  const { stdout } = await execAdb(args);
48
- const value = stdout.trim();
68
+ const value = normalizeAvdName(stdout);
49
69
  if (value)
50
70
  return value;
51
71
  }
@@ -74,16 +94,23 @@ function emulatorLaunchTimeoutError() {
74
94
  return new Error(`Emulator launch timed out after ${AVD_READY_TIMEOUT_SECONDS}s. Check AVD health and retry.`);
75
95
  }
76
96
  async function findRunningEmulatorByAvdName(avdName) {
97
+ const match = await findConnectedEmulatorByAvdName(avdName, { includeOffline: false });
98
+ return match?.state === "device" ? match.id : null;
99
+ }
100
+ async function findConnectedEmulatorByAvdName(avdName, options = {}) {
77
101
  const devices = await listConnectedDevices();
78
- const readyEmulators = devices.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-"));
79
- for (const entry of readyEmulators) {
102
+ const emulatorEntries = devices.filter((entry) => entry.id.startsWith("emulator-") && (options.includeOffline || entry.state === "device"));
103
+ for (const entry of emulatorEntries) {
80
104
  const runningAvdName = await getEmulatorAvdName(entry.id);
81
105
  if (runningAvdName === avdName) {
82
- return entry.id;
106
+ return entry;
83
107
  }
84
108
  }
85
109
  return null;
86
110
  }
111
+ function offlineEmulatorError(avdName) {
112
+ return new Error(getAndroidConnectErrorMessage({ state: "offline", avdName }));
113
+ }
87
114
  async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, options = {}) {
88
115
  const log = options.log || console.log;
89
116
  let avdExists = false;
@@ -99,6 +126,10 @@ async function launchAndWaitForAvd(avdName, timeoutMs = AVD_READY_TIMEOUT_MS, op
99
126
  if (!avdExists) {
100
127
  throw new Error(`AVD "${avdName}" was not found. Create it in Android Studio and retry.`);
101
128
  }
129
+ const offlineMatch = await findConnectedEmulatorByAvdName(avdName, { includeOffline: true });
130
+ if (offlineMatch?.state === "offline") {
131
+ throw offlineEmulatorError(avdName);
132
+ }
102
133
  const baseline = await listConnectedDevices();
103
134
  const baselineReadyEmulators = new Set(baseline.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-")).map((entry) => entry.id));
104
135
  log(`Launching Android emulator "${avdName}"...`);
@@ -128,12 +159,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
128
159
  const readyDevices = devices.filter((entry) => entry.state === "device");
129
160
  if (parsedTarget.kind === "auto") {
130
161
  if (readyDevices.length === 0) {
162
+ const offlineEmulator = devices.find((entry) => entry.state === "offline" && entry.id.startsWith("emulator-"));
163
+ const message = offlineEmulator
164
+ ? "Android emulator is running but offline. Restart adb or cold boot the emulator, then retry."
165
+ : "No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.";
131
166
  emitDesktopDebug("device.error", "device", {}, {
132
167
  platform: "android",
133
168
  operation: "connect",
134
- message: "No ready Android devices detected."
169
+ message
135
170
  });
136
- throw new Error("No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.");
171
+ throw new Error(message);
137
172
  }
138
173
  const selected = readyDevices[0].id;
139
174
  log(`Using connected Android device: ${selected}`);
@@ -142,15 +177,18 @@ export async function connectToDevice(deviceTargetId, options = {}) {
142
177
  }
143
178
  if (parsedTarget.kind === "avd") {
144
179
  const deadline = Date.now() + AVD_READY_TIMEOUT_MS;
145
- const alreadyRunning = await findRunningEmulatorByAvdName(parsedTarget.avdName);
146
- if (alreadyRunning) {
147
- const ready = await waitForWindowServiceReady(alreadyRunning, remainingTimeoutMs(deadline));
180
+ const existingEmulator = await findConnectedEmulatorByAvdName(parsedTarget.avdName, { includeOffline: true });
181
+ if (existingEmulator?.state === "offline") {
182
+ throw offlineEmulatorError(parsedTarget.avdName);
183
+ }
184
+ if (existingEmulator?.state === "device") {
185
+ const ready = await waitForWindowServiceReady(existingEmulator.id, remainingTimeoutMs(deadline));
148
186
  if (!ready) {
149
187
  throw emulatorLaunchTimeoutError();
150
188
  }
151
- log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${alreadyRunning}`);
152
- emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: alreadyRunning, reused: true });
153
- return alreadyRunning;
189
+ log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${existingEmulator.id}`);
190
+ emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: existingEmulator.id, reused: true });
191
+ return existingEmulator.id;
154
192
  }
155
193
  const launchedId = await launchAndWaitForAvd(parsedTarget.avdName, remainingTimeoutMs(deadline), { log });
156
194
  const ready = await waitForWindowServiceReady(launchedId, remainingTimeoutMs(deadline));
@@ -179,19 +217,16 @@ export async function connectToDevice(deviceTargetId, options = {}) {
179
217
  return targetId;
180
218
  }
181
219
  if (parsedTarget.kind === "legacy" && targetId) {
182
- const runningLegacyAvd = await findRunningEmulatorByAvdName(targetId);
183
- if (runningLegacyAvd) {
184
- emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: runningLegacyAvd, reused: true });
185
- return runningLegacyAvd;
220
+ const matchedLegacyAvd = await findConnectedEmulatorByAvdName(targetId, { includeOffline: true });
221
+ if (matchedLegacyAvd?.state === "offline") {
222
+ throw offlineEmulatorError(targetId);
223
+ }
224
+ if (matchedLegacyAvd?.state === "device") {
225
+ emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: matchedLegacyAvd.id, reused: true });
226
+ return matchedLegacyAvd.id;
186
227
  }
187
228
  }
188
- let message = "Selected device is no longer connected. Refresh and select again.";
189
- if (state === "offline") {
190
- message = "Device is offline. Reconnect device or restart adb.";
191
- }
192
- else if (state === "unauthorized") {
193
- message = "Authorize this computer on the device and retry.";
194
- }
229
+ const message = getAndroidConnectErrorMessage({ state });
195
230
  emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "connect", message, targetId: targetId || deviceTargetId, state: state || "missing" });
196
231
  throw new Error(message);
197
232
  }
@@ -3,21 +3,28 @@
3
3
  */
4
4
  import { printCliOutput } from "../utils/console-output.js";
5
5
  export { buildAssertionSystemPrompt } from "../prompts/execution.js";
6
+ const ASSERTION_PREFIXES = ["assert", "verify", "validate", "ensure"];
7
+ function hasAssertionPrefix(lower) {
8
+ return ASSERTION_PREFIXES.some((prefix) => lower === prefix || lower.startsWith(`${prefix}:`) || lower.startsWith(`${prefix} `));
9
+ }
6
10
  export function isAssertion(userInput) {
7
11
  const trimmed = userInput.trim();
8
12
  const lower = trimmed.toLowerCase();
9
- return lower.startsWith("assert:") || lower.startsWith("assert ");
13
+ return hasAssertionPrefix(lower);
10
14
  }
11
15
  export function extractAssertionPrompt(userInput) {
12
16
  const trimmed = userInput.trim();
13
17
  const lower = trimmed.toLowerCase();
14
- // Handle "assert:" or "Assert:"
15
- if (lower.startsWith("assert:")) {
16
- return trimmed.substring("assert:".length).trim();
17
- }
18
- // Handle "assert " or "Assert "
19
- if (lower.startsWith("assert ")) {
20
- return trimmed.substring("assert".length).trim();
18
+ for (const prefix of ASSERTION_PREFIXES) {
19
+ if (lower === prefix) {
20
+ return trimmed;
21
+ }
22
+ if (lower.startsWith(`${prefix}:`)) {
23
+ return trimmed.substring(prefix.length + 1).trim();
24
+ }
25
+ if (lower.startsWith(`${prefix} `)) {
26
+ return trimmed.substring(prefix.length).trim();
27
+ }
21
28
  }
22
29
  return trimmed;
23
30
  }
@@ -14,11 +14,11 @@ const execAsync = promisify(exec);
14
14
  // Active session state
15
15
  let activeSession = null;
16
16
  /**
17
- * Get simulator UDID by name (prefers already-booted simulator)
18
- * @param {string} simulatorName
19
- * @returns {Promise<string|null>}
17
+ * Resolve simulator by UDID or name (name matches prefer already-booted simulators).
18
+ * @param {string} simulatorNameOrUdid
19
+ * @returns {Promise<{name: string, udid: string, state: string}|null>}
20
20
  */
21
- async function getSimulatorUDID(simulatorName) {
21
+ async function resolveSimulator(simulatorNameOrUdid) {
22
22
  try {
23
23
  const { stdout } = await execAsync("xcrun simctl list devices --json");
24
24
  const data = JSON.parse(stdout);
@@ -26,25 +26,31 @@ async function getSimulatorUDID(simulatorName) {
26
26
  let bootedMatch = null;
27
27
  for (const runtime of Object.values(data.devices)) {
28
28
  for (const device of runtime) {
29
- if (device.name === simulatorName && device.isAvailable) {
29
+ if (!device.isAvailable) {
30
+ continue;
31
+ }
32
+ if (device.udid === simulatorNameOrUdid) {
33
+ return { name: device.name, udid: device.udid, state: device.state };
34
+ }
35
+ if (device.name === simulatorNameOrUdid) {
36
+ const match = { name: device.name, udid: device.udid, state: device.state };
30
37
  if (!firstMatch) {
31
- firstMatch = device.udid;
38
+ firstMatch = match;
32
39
  }
33
- // Prefer the already-booted one
34
40
  if (device.state === "Booted") {
35
- bootedMatch = device.udid;
41
+ bootedMatch = match;
36
42
  }
37
43
  }
38
44
  }
39
45
  }
40
46
  if (bootedMatch) {
41
- logger.debug(`Found booted simulator: ${bootedMatch}`);
47
+ logger.debug(`Found booted simulator: ${bootedMatch.udid}`);
42
48
  return bootedMatch;
43
49
  }
44
50
  return firstMatch;
45
51
  }
46
52
  catch (error) {
47
- logger.error("Failed to get simulator UDID", { error: error.message });
53
+ logger.error("Failed to resolve simulator", { error: error.message });
48
54
  }
49
55
  return null;
50
56
  }
@@ -84,33 +90,33 @@ async function findAnySimulator() {
84
90
  */
85
91
  async function bootSimulator(simulatorName, options = {}) {
86
92
  const log = options.log || console.log;
87
- const udid = await getSimulatorUDID(simulatorName);
88
- if (!udid) {
93
+ const simulator = await resolveSimulator(simulatorName);
94
+ if (!simulator) {
89
95
  throw new Error(`Simulator "${simulatorName}" not found. Run "xcrun simctl list devices" to see available simulators.`);
90
96
  }
91
- log(`Found simulator "${simulatorName}" with UDID: ${udid}`);
97
+ log(`Found simulator "${simulator.name}" with UDID: ${simulator.udid}`);
92
98
  // Check if already booted
93
99
  const { stdout } = await execAsync("xcrun simctl list devices --json");
94
100
  const data = JSON.parse(stdout);
95
101
  for (const runtime of Object.values(data.devices)) {
96
102
  for (const device of runtime) {
97
- if (device.udid === udid) {
103
+ if (device.udid === simulator.udid) {
98
104
  if (device.state === "Booted") {
99
- log(`Simulator "${simulatorName}" is already booted`);
100
- return udid;
105
+ log(`Simulator "${simulator.name}" is already booted`);
106
+ return simulator;
101
107
  }
102
108
  break;
103
109
  }
104
110
  }
105
111
  }
106
- log(`Booting simulator "${simulatorName}"...`);
107
- await execAsync(`xcrun simctl boot ${udid}`);
112
+ log(`Booting simulator "${simulator.name}"...`);
113
+ await execAsync(`xcrun simctl boot ${simulator.udid}`);
108
114
  // Open Simulator app to show the device
109
115
  await execAsync("open -a Simulator");
110
116
  // Wait for boot to complete
111
- await waitForSimulatorBoot(udid);
112
- log(`Simulator "${simulatorName}" is fully booted`);
113
- return udid;
117
+ await waitForSimulatorBoot(simulator.udid);
118
+ log(`Simulator "${simulator.name}" is fully booted`);
119
+ return simulator;
114
120
  }
115
121
  /**
116
122
  * Wait for simulator to finish booting
@@ -163,30 +169,30 @@ export async function connectToDevice(simulatorName, options = {}) {
163
169
  log(`No simulator specified, using: ${simulatorName}`);
164
170
  }
165
171
  // Boot simulator
166
- const udid = await bootSimulator(simulatorName, { log });
172
+ const simulator = await bootSimulator(simulatorName, { log });
167
173
  // Create Appium session
168
174
  log("Creating Appium session...");
169
175
  const session = await appium.createSession({
170
176
  platformName: "iOS",
171
177
  "appium:automationName": "XCUITest",
172
- "appium:deviceName": simulatorName,
173
- "appium:udid": udid,
178
+ "appium:deviceName": simulator.name,
179
+ "appium:udid": simulator.udid,
174
180
  "appium:noReset": true,
175
181
  "appium:shouldTerminateApp": false,
176
182
  });
177
183
  activeSession = {
178
184
  sessionId: session.sessionId,
179
- udid,
180
- simulatorName,
185
+ udid: simulator.udid,
186
+ simulatorName: simulator.name,
181
187
  };
182
- log(`Connected to simulator "${simulatorName}" (${udid})`);
188
+ log(`Connected to simulator "${simulator.name}" (${simulator.udid})`);
183
189
  emitDesktopDebug("device.connect", "device", {}, {
184
190
  platform: "ios",
185
191
  stage: "success",
186
- simulatorName,
187
- udid
192
+ simulatorName: simulator.name,
193
+ udid: simulator.udid
188
194
  });
189
- return udid;
195
+ return simulator.udid;
190
196
  }
191
197
  /**
192
198
  * Get device info (screen dimensions and scale)
@@ -1,10 +1,16 @@
1
1
  import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
2
2
  import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
3
+ import { LOADMILL_CUA_MODELS } from "../core/cua-models.js";
3
4
  import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
4
5
  import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
5
6
  import { logger } from "../utils/logger.js";
6
7
  import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
7
8
  const cuaDebugTracer = new CuaDebugTracer(logger);
9
+ const LOADMILL_DEVICE_METADATA_MODELS = new Set([
10
+ ...LOADMILL_CUA_MODELS,
11
+ LOADMILL_MODEL_ALIASES.SMART,
12
+ LOADMILL_MODEL_ALIASES.BEACON,
13
+ ]);
8
14
  function resolveOpenAiBaseUrl(client) {
9
15
  return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
10
16
  }
@@ -121,6 +127,8 @@ const typeSensitiveTextTool = {
121
127
  additionalProperties: false,
122
128
  },
123
129
  };
130
+ // Design completion is the handoff into script generation; it is separate from
131
+ // execution step-completion markers used by model adapters.
124
132
  const testDesignCompleteTool = {
125
133
  type: "function",
126
134
  name: "test_design_complete",
@@ -235,7 +243,64 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
235
243
  }
236
244
  return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
237
245
  }
238
- export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
246
+ function normalizeDevicePlatform(deviceInfo = {}) {
247
+ return typeof deviceInfo?.platform === "string"
248
+ ? deviceInfo.platform.trim().toLowerCase()
249
+ : "";
250
+ }
251
+ function resolveComputerToolDeviceType(deviceInfo = {}) {
252
+ const platform = normalizeDevicePlatform(deviceInfo);
253
+ if (platform === "browser") {
254
+ return "web";
255
+ }
256
+ if (platform === "ios") {
257
+ return "iphone";
258
+ }
259
+ if (platform === "android") {
260
+ return "android";
261
+ }
262
+ return null;
263
+ }
264
+ function resolveComputerToolEnvironment(deviceInfo = {}) {
265
+ const platform = normalizeDevicePlatform(deviceInfo);
266
+ if (platform === "browser") {
267
+ return "browser";
268
+ }
269
+ if (platform === "ios" || platform === "android") {
270
+ return "mobile";
271
+ }
272
+ return null;
273
+ }
274
+ function resolveDisplayDimension(value) {
275
+ return Number.isFinite(value) && value > 0 ? Math.round(value) : null;
276
+ }
277
+ function usesLoadmillDeviceMetadata(cuaModel) {
278
+ return LOADMILL_DEVICE_METADATA_MODELS.has(cuaModel);
279
+ }
280
+ function buildComputerTool({ cuaModel, deviceInfo } = {}) {
281
+ const tool = { type: "computer" };
282
+ if (!usesLoadmillDeviceMetadata(cuaModel)) {
283
+ return tool;
284
+ }
285
+ const environment = resolveComputerToolEnvironment(deviceInfo);
286
+ const deviceType = resolveComputerToolDeviceType(deviceInfo);
287
+ const displayWidth = resolveDisplayDimension(deviceInfo?.scaled_width);
288
+ const displayHeight = resolveDisplayDimension(deviceInfo?.scaled_height);
289
+ if (environment) {
290
+ tool.environment = environment;
291
+ }
292
+ if (deviceType) {
293
+ tool.device_type = deviceType;
294
+ }
295
+ if (displayWidth) {
296
+ tool.display_width = displayWidth;
297
+ }
298
+ if (displayHeight) {
299
+ tool.display_height = displayHeight;
300
+ }
301
+ return tool;
302
+ }
303
+ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, deviceInfo = null, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
239
304
  const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
240
305
  // Keep Responses API storage enabled on every turn so execution can chain
241
306
  // reliably through previous_response_id across the whole run.
@@ -248,7 +313,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
248
313
  store: true,
249
314
  truncation: "auto",
250
315
  tools: [
251
- ...(includeComputerTool ? [{ type: "computer" }] : []),
316
+ ...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
252
317
  ...buildFunctionTools({ aimAssistMode, toolCapabilities }),
253
318
  ],
254
319
  };
@@ -294,6 +359,7 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
294
359
  aimAssistMode,
295
360
  previousResponseId,
296
361
  input,
362
+ deviceInfo,
297
363
  includeComputerTool,
298
364
  contextOptimizationEnabled,
299
365
  contextOptimizationThreshold,
@@ -1,20 +1,33 @@
1
1
  // The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
2
2
  // anchor coordinates `x` / `y`. This helper converts those model coordinates
3
3
  // into backend gesture coordinates: startX/startY and endX/endY.
4
- export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
4
+ export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0, minX, maxX, minY, maxY } = {}) {
5
+ const clamp = (value, min, max) => {
6
+ if (!Number.isFinite(value)) {
7
+ return value;
8
+ }
9
+ if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) {
10
+ return value;
11
+ }
12
+ return Math.min(Math.max(value, min), max);
13
+ };
5
14
  const divisor = scale * dpr;
6
- const scrollX = Math.round((action?.scroll_x ?? 0) / divisor);
7
- const scrollY = Math.round((action?.scroll_y ?? 0) / divisor);
15
+ const requestedScrollX = Math.round((action?.scroll_x ?? 0) / divisor);
16
+ const requestedScrollY = Math.round((action?.scroll_y ?? 0) / divisor);
8
17
  const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
9
- const startX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
10
- const startY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
18
+ const unclampedStartX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
19
+ const unclampedStartY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
20
+ const startX = clamp(unclampedStartX, minX, maxX);
21
+ const startY = clamp(unclampedStartY, minY, maxY);
22
+ const endX = clamp(startX + requestedScrollX, minX, maxX);
23
+ const endY = clamp(startY - requestedScrollY, minY, maxY);
11
24
  return {
12
- scrollX,
13
- scrollY,
25
+ scrollX: endX - startX,
26
+ scrollY: startY - endY,
14
27
  startX,
15
28
  startY,
16
- endX: startX + scrollX,
17
- endY: startY - scrollY,
29
+ endX,
30
+ endY,
18
31
  hasAnchor
19
32
  };
20
33
  }
@@ -138,10 +138,10 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
138
138
  body: JSON.stringify(payload),
139
139
  });
140
140
  }
141
- async function getCurrentUser() {
141
+ async function validateApiToken() {
142
142
  const resolvedToken = getApiToken();
143
143
  if (!resolvedToken) {
144
- return null;
144
+ return { valid: false, reason: "missing" };
145
145
  }
146
146
  const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
147
147
  method: "GET",
@@ -150,11 +150,26 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
150
150
  Authorization: `Bearer ${resolvedToken}`,
151
151
  },
152
152
  });
153
+ if (response.status === 401 || response.status === 403) {
154
+ return { valid: false, reason: "invalid", status: response.status };
155
+ }
153
156
  if (!response.ok) {
154
- return null;
157
+ return { valid: false, reason: "unavailable", status: response.status };
158
+ }
159
+ try {
160
+ const user = extractLoadmillSessionUser(await response.json());
161
+ return user
162
+ ? { valid: true, user }
163
+ : { valid: false, reason: "unavailable", status: response.status };
164
+ }
165
+ catch {
166
+ return { valid: false, reason: "unavailable", status: response.status };
155
167
  }
168
+ }
169
+ async function getCurrentUser() {
156
170
  try {
157
- return extractLoadmillSessionUser(await response.json());
171
+ const validation = await validateApiToken();
172
+ return validation.valid ? validation.user : null;
158
173
  }
159
174
  catch {
160
175
  return null;
@@ -172,6 +187,7 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
172
187
  getTestSuiteFlowRun,
173
188
  explainFailure,
174
189
  reportDroidRun,
190
+ validateApiToken,
175
191
  getCurrentUser,
176
192
  };
177
193
  }
@@ -206,7 +206,7 @@ export class DesignModeInk {
206
206
  toolCapabilities: this.engine.toolCapabilities,
207
207
  });
208
208
  // Run turn with action tracking and interruption checking
209
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
209
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
210
210
  // Check if user interrupted - stop immediately
211
211
  if (this.inputQueue.length > 0) {
212
212
  return true; // Signal to stop execution
@@ -223,6 +223,15 @@ export class DesignModeInk {
223
223
  }
224
224
  return false; // Continue execution
225
225
  }, this.context, null, { currentScreenshot });
226
+ if (turnSummary.failed) {
227
+ if (this.context.setAgentWorking) {
228
+ this.context.setAgentWorking(false);
229
+ }
230
+ addOutput({ type: 'error', text: turnSummary.error || 'Design mode stopped.' });
231
+ this.conversationActive = false;
232
+ this.cleanup();
233
+ return;
234
+ }
226
235
  this.session.updateResponseId(newResponseId);
227
236
  this.consecutiveErrorCount = 0;
228
237
  // Clear agent working status
@@ -170,7 +170,7 @@ export class DesignMode {
170
170
  toolCapabilities: this.engine.toolCapabilities,
171
171
  });
172
172
  // Run turn with action tracking and interruption checking
173
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
173
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
174
174
  // Check if user pressed ESC - stop immediately
175
175
  if (this.escPressed) {
176
176
  return true; // Signal to stop execution
@@ -188,6 +188,12 @@ export class DesignMode {
188
188
  }
189
189
  return false; // Continue execution
190
190
  }, null, null, { currentScreenshot });
191
+ if (turnSummary.failed) {
192
+ this.cleanupEscDetection(keypressHandler);
193
+ console.log(`\n${turnSummary.error || "Design mode stopped."}`);
194
+ this.conversationActive = false;
195
+ return;
196
+ }
191
197
  this.session.updateResponseId(newResponseId);
192
198
  this.consecutiveErrorCount = 0;
193
199
  // Cleanup ESC detection