@loadmill/droid-cua 2.22.0 → 2.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,16 @@
1
1
  import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
2
2
  import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
3
+ import { LOADMILL_CUA_MODELS } from "../core/cua-models.js";
3
4
  import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
4
5
  import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
5
6
  import { logger } from "../utils/logger.js";
6
7
  import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
7
8
  const cuaDebugTracer = new CuaDebugTracer(logger);
9
+ const LOADMILL_DEVICE_METADATA_MODELS = new Set([
10
+ ...LOADMILL_CUA_MODELS,
11
+ LOADMILL_MODEL_ALIASES.SMART,
12
+ LOADMILL_MODEL_ALIASES.BEACON,
13
+ ]);
8
14
  function resolveOpenAiBaseUrl(client) {
9
15
  return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
10
16
  }
@@ -121,6 +127,8 @@ const typeSensitiveTextTool = {
121
127
  additionalProperties: false,
122
128
  },
123
129
  };
130
+ // Design completion is the handoff into script generation; it is separate from
131
+ // execution step-completion markers used by model adapters.
124
132
  const testDesignCompleteTool = {
125
133
  type: "function",
126
134
  name: "test_design_complete",
@@ -235,7 +243,64 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
235
243
  }
236
244
  return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
237
245
  }
238
- export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
246
+ function normalizeDevicePlatform(deviceInfo = {}) {
247
+ return typeof deviceInfo?.platform === "string"
248
+ ? deviceInfo.platform.trim().toLowerCase()
249
+ : "";
250
+ }
251
+ function resolveComputerToolDeviceType(deviceInfo = {}) {
252
+ const platform = normalizeDevicePlatform(deviceInfo);
253
+ if (platform === "browser") {
254
+ return "web";
255
+ }
256
+ if (platform === "ios") {
257
+ return "iphone";
258
+ }
259
+ if (platform === "android") {
260
+ return "android";
261
+ }
262
+ return null;
263
+ }
264
+ function resolveComputerToolEnvironment(deviceInfo = {}) {
265
+ const platform = normalizeDevicePlatform(deviceInfo);
266
+ if (platform === "browser") {
267
+ return "browser";
268
+ }
269
+ if (platform === "ios" || platform === "android") {
270
+ return "mobile";
271
+ }
272
+ return null;
273
+ }
274
+ function resolveDisplayDimension(value) {
275
+ return Number.isFinite(value) && value > 0 ? Math.round(value) : null;
276
+ }
277
+ function usesLoadmillDeviceMetadata(cuaModel) {
278
+ return LOADMILL_DEVICE_METADATA_MODELS.has(cuaModel);
279
+ }
280
+ function buildComputerTool({ cuaModel, deviceInfo } = {}) {
281
+ const tool = { type: "computer" };
282
+ if (!usesLoadmillDeviceMetadata(cuaModel)) {
283
+ return tool;
284
+ }
285
+ const environment = resolveComputerToolEnvironment(deviceInfo);
286
+ const deviceType = resolveComputerToolDeviceType(deviceInfo);
287
+ const displayWidth = resolveDisplayDimension(deviceInfo?.scaled_width);
288
+ const displayHeight = resolveDisplayDimension(deviceInfo?.scaled_height);
289
+ if (environment) {
290
+ tool.environment = environment;
291
+ }
292
+ if (deviceType) {
293
+ tool.device_type = deviceType;
294
+ }
295
+ if (displayWidth) {
296
+ tool.display_width = displayWidth;
297
+ }
298
+ if (displayHeight) {
299
+ tool.display_height = displayHeight;
300
+ }
301
+ return tool;
302
+ }
303
+ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, deviceInfo = null, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
239
304
  const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
240
305
  // Keep Responses API storage enabled on every turn so execution can chain
241
306
  // reliably through previous_response_id across the whole run.
@@ -248,7 +313,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
248
313
  store: true,
249
314
  truncation: "auto",
250
315
  tools: [
251
- ...(includeComputerTool ? [{ type: "computer" }] : []),
316
+ ...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
252
317
  ...buildFunctionTools({ aimAssistMode, toolCapabilities }),
253
318
  ],
254
319
  };
@@ -294,6 +359,7 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
294
359
  aimAssistMode,
295
360
  previousResponseId,
296
361
  input,
362
+ deviceInfo,
297
363
  includeComputerTool,
298
364
  contextOptimizationEnabled,
299
365
  contextOptimizationThreshold,
@@ -1,20 +1,33 @@
1
1
  // The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
2
2
  // anchor coordinates `x` / `y`. This helper converts those model coordinates
3
3
  // into backend gesture coordinates: startX/startY and endX/endY.
4
- export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
4
+ export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0, minX, maxX, minY, maxY } = {}) {
5
+ const clamp = (value, min, max) => {
6
+ if (!Number.isFinite(value)) {
7
+ return value;
8
+ }
9
+ if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) {
10
+ return value;
11
+ }
12
+ return Math.min(Math.max(value, min), max);
13
+ };
5
14
  const divisor = scale * dpr;
6
- const scrollX = Math.round((action?.scroll_x ?? 0) / divisor);
7
- const scrollY = Math.round((action?.scroll_y ?? 0) / divisor);
15
+ const requestedScrollX = Math.round((action?.scroll_x ?? 0) / divisor);
16
+ const requestedScrollY = Math.round((action?.scroll_y ?? 0) / divisor);
8
17
  const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
9
- const startX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
10
- const startY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
18
+ const unclampedStartX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
19
+ const unclampedStartY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
20
+ const startX = clamp(unclampedStartX, minX, maxX);
21
+ const startY = clamp(unclampedStartY, minY, maxY);
22
+ const endX = clamp(startX + requestedScrollX, minX, maxX);
23
+ const endY = clamp(startY - requestedScrollY, minY, maxY);
11
24
  return {
12
- scrollX,
13
- scrollY,
25
+ scrollX: endX - startX,
26
+ scrollY: startY - endY,
14
27
  startX,
15
28
  startY,
16
- endX: startX + scrollX,
17
- endY: startY - scrollY,
29
+ endX,
30
+ endY,
18
31
  hasAnchor
19
32
  };
20
33
  }
@@ -138,10 +138,10 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
138
138
  body: JSON.stringify(payload),
139
139
  });
140
140
  }
141
- async function getCurrentUser() {
141
+ async function validateApiToken() {
142
142
  const resolvedToken = getApiToken();
143
143
  if (!resolvedToken) {
144
- return null;
144
+ return { valid: false, reason: "missing" };
145
145
  }
146
146
  const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
147
147
  method: "GET",
@@ -150,11 +150,26 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
150
150
  Authorization: `Bearer ${resolvedToken}`,
151
151
  },
152
152
  });
153
+ if (response.status === 401 || response.status === 403) {
154
+ return { valid: false, reason: "invalid", status: response.status };
155
+ }
153
156
  if (!response.ok) {
154
- return null;
157
+ return { valid: false, reason: "unavailable", status: response.status };
158
+ }
159
+ try {
160
+ const user = extractLoadmillSessionUser(await response.json());
161
+ return user
162
+ ? { valid: true, user }
163
+ : { valid: false, reason: "unavailable", status: response.status };
164
+ }
165
+ catch {
166
+ return { valid: false, reason: "unavailable", status: response.status };
155
167
  }
168
+ }
169
+ async function getCurrentUser() {
156
170
  try {
157
- return extractLoadmillSessionUser(await response.json());
171
+ const validation = await validateApiToken();
172
+ return validation.valid ? validation.user : null;
158
173
  }
159
174
  catch {
160
175
  return null;
@@ -172,6 +187,7 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
172
187
  getTestSuiteFlowRun,
173
188
  explainFailure,
174
189
  reportDroidRun,
190
+ validateApiToken,
175
191
  getCurrentUser,
176
192
  };
177
193
  }
@@ -206,7 +206,7 @@ export class DesignModeInk {
206
206
  toolCapabilities: this.engine.toolCapabilities,
207
207
  });
208
208
  // Run turn with action tracking and interruption checking
209
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
209
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
210
210
  // Check if user interrupted - stop immediately
211
211
  if (this.inputQueue.length > 0) {
212
212
  return true; // Signal to stop execution
@@ -223,6 +223,15 @@ export class DesignModeInk {
223
223
  }
224
224
  return false; // Continue execution
225
225
  }, this.context, null, { currentScreenshot });
226
+ if (turnSummary.failed) {
227
+ if (this.context.setAgentWorking) {
228
+ this.context.setAgentWorking(false);
229
+ }
230
+ addOutput({ type: 'error', text: turnSummary.error || 'Design mode stopped.' });
231
+ this.conversationActive = false;
232
+ this.cleanup();
233
+ return;
234
+ }
226
235
  this.session.updateResponseId(newResponseId);
227
236
  this.consecutiveErrorCount = 0;
228
237
  // Clear agent working status
@@ -170,7 +170,7 @@ export class DesignMode {
170
170
  toolCapabilities: this.engine.toolCapabilities,
171
171
  });
172
172
  // Run turn with action tracking and interruption checking
173
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
173
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
174
174
  // Check if user pressed ESC - stop immediately
175
175
  if (this.escPressed) {
176
176
  return true; // Signal to stop execution
@@ -188,6 +188,12 @@ export class DesignMode {
188
188
  }
189
189
  return false; // Continue execution
190
190
  }, null, null, { currentScreenshot });
191
+ if (turnSummary.failed) {
192
+ this.cleanupEscDetection(keypressHandler);
193
+ console.log(`\n${turnSummary.error || "Design mode stopped."}`);
194
+ this.conversationActive = false;
195
+ return;
196
+ }
191
197
  this.session.updateResponseId(newResponseId);
192
198
  this.consecutiveErrorCount = 0;
193
199
  // Cleanup ESC detection
@@ -4,15 +4,15 @@ import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurre
4
4
  import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
5
5
  import { sendCUARequest } from "../device/openai.js";
6
6
  import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
7
- import { isAssertion } from "../device/assertions.js";
7
+ import { extractAssertionPrompt, isAssertion } from "../device/assertions.js";
8
+ import { handleAssertionToolCall } from "../core/tool-handlers/assert.js";
8
9
  import { logger } from "../utils/logger.js";
9
10
  import { emitDesktopDebug } from "../utils/desktop-debug.js";
10
11
  import { printCliOutput } from "../utils/console-output.js";
11
- // Mirrors the codes emitted by the Loadmill LLM gateway (see services/app/private/llm
12
- // in the server repo): a transient burst limit plus day/week/fixed-budget request and
13
- // token quotas. Free plans run on the fixed-budget quota.
12
+ // Mirrors the quota codes emitted by the Loadmill LLM gateway (see
13
+ // services/app/private/llm in the server repo). Free plans run on the
14
+ // fixed-budget quota.
14
15
  const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
15
- "llm_rate_limit_exceeded",
16
16
  "llm_day_quota_exceeded",
17
17
  "llm_week_quota_exceeded",
18
18
  "llm_fixed_budget_quota_exceeded",
@@ -21,6 +21,8 @@ const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
21
21
  "llm_fixed_budget_token_quota_exceeded",
22
22
  ]);
23
23
  const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
24
+ const LLM_REQUEST_RATE_LIMIT_ERROR_CODE = "llm_rate_limit_exceeded";
25
+ const DEFAULT_ASSERTION_CONTEXT = "No additional prior context is needed; validate directly from the current screen.";
24
26
  function isLlmUsageLimitError(err) {
25
27
  if (!err || typeof err !== "object") {
26
28
  return false;
@@ -30,9 +32,15 @@ function isLlmUsageLimitError(err) {
30
32
  }
31
33
  return err.status === 429 && err.type === "rate_limit_error";
32
34
  }
35
+ function isLlmRequestRateLimitError(err) {
36
+ return err && typeof err === "object" && err.code === LLM_REQUEST_RATE_LIMIT_ERROR_CODE;
37
+ }
33
38
  function getLlmUsageLimitMessage() {
34
39
  return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
35
40
  }
41
+ function getLlmRequestRateLimitMessage() {
42
+ return "Execution stopped: too many AI requests are running for this account right now. Please retry shortly or reduce parallel test runs.";
43
+ }
36
44
  /**
37
45
  * Execution Mode - Run test scripts line-by-line
38
46
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -105,10 +113,11 @@ export class ExecutionMode {
105
113
  lines.push(` Retries: ${this.stats.retryCount}`);
106
114
  return lines;
107
115
  }
108
- buildStepContext(instructionIndex) {
116
+ buildStepContext(instructionIndex, instruction = null) {
109
117
  return {
110
118
  instructionIndex,
111
119
  stepId: `step-${String(instructionIndex + 1).padStart(4, "0")}`,
120
+ ...(typeof instruction === "string" ? { instruction } : {}),
112
121
  };
113
122
  }
114
123
  emit(addOutput, type, text, context = {}, stepContext = null, extra = {}) {
@@ -123,6 +132,45 @@ export class ExecutionMode {
123
132
  payload: extra.payload
124
133
  });
125
134
  }
135
+ resetInstructionConversationState() {
136
+ this.session.updateResponseId(null);
137
+ if (typeof this.session.clearTranscript === "function") {
138
+ this.session.clearTranscript();
139
+ return;
140
+ }
141
+ if (Array.isArray(this.session.transcript)) {
142
+ this.session.transcript = [];
143
+ }
144
+ }
145
+ async executeVerificationInstruction(instruction, context, stepContext = null) {
146
+ const toolResult = await handleAssertionToolCall({
147
+ engine: this.engine,
148
+ callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
149
+ argumentsText: JSON.stringify({
150
+ condition: extractAssertionPrompt(instruction),
151
+ context: DEFAULT_ASSERTION_CONTEXT,
152
+ }),
153
+ context,
154
+ stepContext,
155
+ });
156
+ const assertionStatus = toolResult.summaryUpdates?.assertionStatus;
157
+ if (assertionStatus === "passed") {
158
+ this.stats.assertionsPassed++;
159
+ }
160
+ else if (assertionStatus === "failed" || assertionStatus === "failed_skipped") {
161
+ this.stats.assertionsFailed++;
162
+ }
163
+ if (toolResult.kind === "retry_instruction") {
164
+ this.stats.retryCount++;
165
+ this.session.clearMessages();
166
+ return await this.executeInstruction(instruction, context, 0, stepContext);
167
+ }
168
+ if (toolResult.kind === "fail") {
169
+ return { success: false, error: toolResult.error || "Assertion failed." };
170
+ }
171
+ this.session.clearMessages();
172
+ return { success: true };
173
+ }
126
174
  /**
127
175
  * Execute all instructions in the test script
128
176
  * @param {Object} context - Additional context (Ink context with addOutput)
@@ -137,7 +185,8 @@ export class ExecutionMode {
137
185
  // Start timing
138
186
  this.stats.startTime = Date.now();
139
187
  for (let i = 0; i < this.instructions.length; i++) {
140
- const stepContext = this.buildStepContext(i);
188
+ const instruction = this.instructions[i];
189
+ const stepContext = this.buildStepContext(i, instruction);
141
190
  // Check if execution should be stopped
142
191
  if (this.shouldStop) {
143
192
  this.emit(addOutput, 'info', 'Test execution stopped by user.', runContext, stepContext, {
@@ -158,7 +207,6 @@ export class ExecutionMode {
158
207
  return { success: false, error: 'Stopped by user' };
159
208
  }
160
209
  }
161
- const instruction = this.instructions[i];
162
210
  this.emit(addOutput, 'user', instruction, runContext, stepContext, {
163
211
  eventType: 'instruction_started',
164
212
  payload: {
@@ -236,10 +284,17 @@ export class ExecutionMode {
236
284
  const MAX_RETRIES = 10;
237
285
  const instructionContext = {
238
286
  ...context,
287
+ currentInstruction: instruction,
239
288
  futureInstructions: this.instructions.slice((stepContext?.instructionIndex ?? -1) + 1),
240
289
  };
290
+ if (retryCount === 0) {
291
+ this.resetInstructionConversationState();
292
+ }
241
293
  this.session.addToTranscript(`[User] ${instruction}`);
242
294
  this.session.addMessage("user", instruction);
295
+ if (isAssertion(instruction)) {
296
+ return await this.executeVerificationInstruction(instruction, instructionContext, stepContext);
297
+ }
243
298
  try {
244
299
  const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
245
300
  const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
@@ -309,6 +364,26 @@ export class ExecutionMode {
309
364
  stack: err.stack
310
365
  });
311
366
  const addOutput = context.addOutput || printCliOutput;
367
+ if (isLlmRequestRateLimitError(err)) {
368
+ const stopMessage = getLlmRequestRateLimitMessage();
369
+ logger.debug('Stopping execution because an LLM request rate limit was reached', {
370
+ instruction,
371
+ message: err.message,
372
+ status: err.status,
373
+ code: err.code,
374
+ type: err.type,
375
+ });
376
+ this.emit(addOutput, 'error', stopMessage, context, stepContext, {
377
+ eventType: 'error',
378
+ payload: {
379
+ message: stopMessage,
380
+ status: err.status,
381
+ code: err.code,
382
+ type: err.type,
383
+ }
384
+ });
385
+ return { success: false, error: stopMessage, usageLimitReached: true };
386
+ }
312
387
  if (isLlmUsageLimitError(err)) {
313
388
  const stopMessage = getLlmUsageLimitMessage();
314
389
  logger.debug('Stopping execution because an LLM usage limit was reached', {
@@ -449,6 +524,7 @@ export class ExecutionMode {
449
524
  });
450
525
  // Replace the system message with the enhanced one
451
526
  this.session.installRecoverySystemPrompt(recoverySystemPrompt);
527
+ this.session.updateResponseId(null);
452
528
  // Wait a bit before retrying
453
529
  await new Promise(resolve => setTimeout(resolve, 1000));
454
530
  // Retry the same instruction with incremented counter
@@ -139,7 +139,9 @@ Test design exploration is complete. Use the validated completion summary below
139
139
  Do not call tools.
140
140
  Do not continue exploring.
141
141
  Do not add explanation.
142
- Output only the final result in the required design-mode format.
142
+ Output only the exact final result format below: one "Suggested test name:" line, then one fenced code block.
143
+ Do not use TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, or bullets.
144
+ The fenced code block must contain only executable test instructions, one per line, ending with exit.
143
145
 
144
146
  Completion summary:
145
147
  Suggested test name: ${suggestedTestName || "(choose a short kebab-case name)"}
@@ -151,3 +153,21 @@ Verification points:
151
153
  ${formatStringList(summary.verificationPoints)}
152
154
  ${notesForScript ? `Notes for script:\n${notesForScript}\n` : ""}`;
153
155
  }
156
+ export function buildDesignScriptFormatRetryPrompt(previousOutput = "") {
157
+ const previousOutputText = typeof previousOutput === "string" ? previousOutput.trim() : "";
158
+ return `AUTHORING FORMAT CORRECTION:
159
+ Your previous answer did not use the required design-mode format.
160
+
161
+ Output only:
162
+ Suggested test name: short-kebab-case-name
163
+
164
+ \`\`\`
165
+ one executable instruction per line
166
+ exit
167
+ \`\`\`
168
+
169
+ Do not include TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, bullets, or explanation.${previousOutputText ? `
170
+
171
+ Previous output to reformat:
172
+ ${previousOutputText}` : ""}`;
173
+ }
@@ -1,4 +1,5 @@
1
1
  import { classifyCaptureSource } from "../core/capture-sources.js";
2
+ import { stepCompletionData } from "../core/report/run-log-model.js";
2
3
  function isHiddenExecutionOutput(item) {
3
4
  const captureSource = item?.payload?.captureSource;
4
5
  return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
@@ -24,6 +25,10 @@ export function formatCliOutput(item) {
24
25
  if (!text || text.trim().length === 0) {
25
26
  return null;
26
27
  }
28
+ const stepCompletion = stepCompletionData(text);
29
+ if (stepCompletion) {
30
+ return `assistant: Step complete — ${stepCompletion.instruction}`;
31
+ }
27
32
  return text
28
33
  .split("\n")
29
34
  .map((line) => `assistant: ${line}`)
@@ -92,6 +92,54 @@ function summarizeRequestInput(input) {
92
92
  };
93
93
  });
94
94
  }
95
+ function extractImageBase64Length(imageUrl) {
96
+ return typeof imageUrl === "string" ? imageUrl.replace("data:image/png;base64,", "").length : 0;
97
+ }
98
+ function summarizeRequestScreenshots(input, deviceInfo) {
99
+ const screenshots = [];
100
+ for (const item of input) {
101
+ if (item?.role && Array.isArray(item.content)) {
102
+ for (const entry of item.content) {
103
+ if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
104
+ continue;
105
+ screenshots.push({
106
+ source: `message:${item.role}`,
107
+ base64Length: extractImageBase64Length(entry.image_url),
108
+ });
109
+ }
110
+ continue;
111
+ }
112
+ if (item?.type === "computer_call_output" && typeof item.output?.image_url === "string") {
113
+ screenshots.push({
114
+ source: "computer_call_output",
115
+ callId: item.call_id,
116
+ base64Length: extractImageBase64Length(item.output.image_url),
117
+ });
118
+ continue;
119
+ }
120
+ if (item?.type === "function_call_output" && Array.isArray(item.output)) {
121
+ for (const entry of item.output) {
122
+ if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
123
+ continue;
124
+ screenshots.push({
125
+ source: "function_call_output",
126
+ callId: item.call_id,
127
+ base64Length: extractImageBase64Length(entry.image_url),
128
+ });
129
+ }
130
+ }
131
+ }
132
+ if (screenshots.length === 0) {
133
+ return null;
134
+ }
135
+ return {
136
+ count: screenshots.length,
137
+ sources: screenshots.map((entry) => entry.source),
138
+ base64Lengths: screenshots.map((entry) => entry.base64Length),
139
+ width: deviceInfo?.scaled_width ?? null,
140
+ height: deviceInfo?.scaled_height ?? null,
141
+ };
142
+ }
95
143
  function redactRequestForFileLog(requestParams) {
96
144
  return {
97
145
  ...requestParams,
@@ -223,6 +271,7 @@ export class CuaDebugTracer {
223
271
  const localRequestId = crypto.randomUUID();
224
272
  const requestLog = redactRequestForFileLog(requestParams);
225
273
  const messages = summarizeRequestInput(input);
274
+ const screenshot = summarizeRequestScreenshots(input, deviceInfo);
226
275
  const acknowledgedSafetyChecksSent = input
227
276
  .filter((item) => item?.type === "computer_call_output")
228
277
  .flatMap((item) => {
@@ -260,13 +309,7 @@ export class CuaDebugTracer {
260
309
  inputItemTypes,
261
310
  inputCallIds,
262
311
  messages,
263
- screenshot: screenshotBase64
264
- ? {
265
- width: deviceInfo?.scaled_width ?? null,
266
- height: deviceInfo?.scaled_height ?? null,
267
- base64Length: screenshotBase64.length
268
- }
269
- : null,
312
+ screenshot,
270
313
  safetyChecks: {
271
314
  previousPending: previousPendingSafetyChecks,
272
315
  previousPendingCount: previousPendingSafetyChecks.length,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loadmill/droid-cua",
3
- "version": "2.22.0",
3
+ "version": "2.23.0",
4
4
  "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
5
5
  "main": "build/index.js",
6
6
  "type": "module",
@@ -1,11 +0,0 @@
1
- export const CUA_BACKENDS = ["openai", "fara"];
2
- export const DEFAULT_CUA_BACKEND = "openai";
3
- export const DEFAULT_FARA_BASE_URL = "http://localhost:5000/v1";
4
- export const DEFAULT_FARA_MODEL = "Fara-7B";
5
- const VALID_CUA_BACKENDS = new Set(CUA_BACKENDS);
6
- export function isValidCuaBackend(value) {
7
- return VALID_CUA_BACKENDS.has(value);
8
- }
9
- export function resolveCuaBackend(value) {
10
- return isValidCuaBackend(value) ? value : DEFAULT_CUA_BACKEND;
11
- }