@loadmill/droid-cua 2.21.0 → 2.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/build/index.js +31 -1
  2. package/build/src/browser/playwright/actions.js +65 -1
  3. package/build/src/cli/headless-runner.js +174 -12
  4. package/build/src/commands/help.js +1 -1
  5. package/build/src/core/cua-models.js +11 -4
  6. package/build/src/core/droid-cua-run-usage.js +106 -0
  7. package/build/src/core/droid-cua-segment-events.js +32 -0
  8. package/build/src/core/execution-engine.js +334 -5
  9. package/build/src/core/prompts.js +1 -1
  10. package/build/src/core/report/render-html-report.js +8 -3
  11. package/build/src/core/report/run-log-model.js +42 -0
  12. package/build/src/core/runtime-config.js +24 -39
  13. package/build/src/core/session.js +6 -0
  14. package/build/src/core/tool-handlers/loadmill.js +3 -0
  15. package/build/src/core/tool-handlers/test-design-complete.js +1 -0
  16. package/build/src/core/tool-handlers/type-sensitive-text.js +1 -0
  17. package/build/src/device/android/actions.js +6 -2
  18. package/build/src/device/android/connection.js +58 -23
  19. package/build/src/device/assertions.js +15 -8
  20. package/build/src/device/ios/connection.js +36 -30
  21. package/build/src/device/openai.js +68 -2
  22. package/build/src/device/scroll-gesture.js +22 -9
  23. package/build/src/integrations/loadmill/client.js +62 -0
  24. package/build/src/integrations/loadmill/run-reporter.js +113 -0
  25. package/build/src/integrations/loadmill/session-user.js +39 -0
  26. package/build/src/integrations/segment/droid-cua-segment.js +64 -0
  27. package/build/src/modes/design-mode-ink.js +10 -1
  28. package/build/src/modes/design-mode.js +7 -1
  29. package/build/src/modes/execution-mode.js +84 -8
  30. package/build/src/prompts/design.js +21 -1
  31. package/build/src/utils/console-output.js +5 -0
  32. package/build/src/utils/cua-debug-tracer.js +50 -7
  33. package/package.json +2 -1
  34. package/build/src/core/cua-backends.js +0 -11
@@ -0,0 +1,113 @@
1
+ const VALID_STATUSES = new Set(["pass", "fail", "error", "skipped", "stopped"]);
2
+ function trimToNull(value) {
3
+ if (typeof value !== "string") {
4
+ return null;
5
+ }
6
+ const trimmed = value.trim();
7
+ return trimmed || null;
8
+ }
9
+ function nonNegativeNumberOrNull(value) {
10
+ return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null;
11
+ }
12
+ function nonNegativeIntegerOrNull(value) {
13
+ const normalized = nonNegativeNumberOrNull(value);
14
+ return normalized == null ? null : Math.floor(normalized);
15
+ }
16
+ function normalizeSource(value) {
17
+ return value === "ci" ? "ci" : "desktop";
18
+ }
19
+ function normalizePlatform(value) {
20
+ const platform = trimToNull(value);
21
+ return platform === "browser" ? "web" : platform;
22
+ }
23
+ function normalizeTestName(value) {
24
+ return value ? value.replace(/\.dcua$/i, "") : null;
25
+ }
26
+ function normalizeRunType(value, testCases) {
27
+ if (value === "project" || value === "test") {
28
+ return value;
29
+ }
30
+ return testCases > 1 ? "project" : "test";
31
+ }
32
+ export function normalizeDroidRunStatus({ status = null, outcome = null, success = null, error = null } = {}) {
33
+ if (VALID_STATUSES.has(status)) {
34
+ return status;
35
+ }
36
+ if (outcome === "pass" || success === true) {
37
+ return "pass";
38
+ }
39
+ if (error === "Stopped by user") {
40
+ return "stopped";
41
+ }
42
+ if (outcome === "skipped") {
43
+ return "skipped";
44
+ }
45
+ if (outcome === "failure" || (typeof error === "string" && error.startsWith("Assertion failed:"))) {
46
+ return "fail";
47
+ }
48
+ return "error";
49
+ }
50
+ export function buildDroidRunReportPayload(input = {}) {
51
+ const completedAt = trimToNull(input.completedAt) || new Date().toISOString();
52
+ const testCases = nonNegativeIntegerOrNull(input.testCases);
53
+ const payload = {
54
+ runType: normalizeRunType(input.runType, testCases),
55
+ testCases,
56
+ durationMs: nonNegativeIntegerOrNull(input.durationMs),
57
+ platform: normalizePlatform(input.platform),
58
+ deviceSource: trimToNull(input.deviceSource),
59
+ deviceName: trimToNull(input.deviceName),
60
+ osVersion: trimToNull(input.osVersion),
61
+ instructionsTotal: nonNegativeIntegerOrNull(input.instructionsTotal),
62
+ instructionsCompleted: nonNegativeIntegerOrNull(input.instructionsCompleted),
63
+ actionsTotal: nonNegativeIntegerOrNull(input.actionsTotal),
64
+ assertionsPassed: nonNegativeIntegerOrNull(input.assertionsPassed),
65
+ assertionsFailed: nonNegativeIntegerOrNull(input.assertionsFailed),
66
+ retries: nonNegativeIntegerOrNull(input.retries),
67
+ error: trimToNull(input.error),
68
+ };
69
+ for (const key of Object.keys(payload)) {
70
+ if (payload[key] == null) {
71
+ delete payload[key];
72
+ }
73
+ }
74
+ return {
75
+ runId: trimToNull(input.runId),
76
+ source: normalizeSource(input.source),
77
+ projectName: trimToNull(input.projectName),
78
+ testName: normalizeTestName(trimToNull(input.testName)),
79
+ status: normalizeDroidRunStatus(input),
80
+ completedAt,
81
+ payload,
82
+ };
83
+ }
84
+ /**
85
+ * @param {{
86
+ * loadmillClient?: { getApiToken?: () => string | null, reportDroidRun?: (payload: object) => Promise<unknown> } | null,
87
+ * onWarning?: ((message: string) => void) | null,
88
+ * [key: string]: unknown,
89
+ * }} input
90
+ */
91
+ export async function reportDroidRunCompletion({ loadmillClient, onWarning = null, ...input } = {}) {
92
+ if (!loadmillClient?.getApiToken?.()) {
93
+ return { reported: false, reason: "missing_token" };
94
+ }
95
+ if (typeof loadmillClient.reportDroidRun !== "function") {
96
+ return { reported: false, reason: "unsupported_client" };
97
+ }
98
+ const payload = buildDroidRunReportPayload(input);
99
+ if (!payload.runId) {
100
+ return { reported: false, reason: "missing_run_id" };
101
+ }
102
+ try {
103
+ await loadmillClient.reportDroidRun(payload);
104
+ return { reported: true };
105
+ }
106
+ catch (error) {
107
+ const message = error instanceof Error ? error.message : String(error);
108
+ if (typeof onWarning === "function") {
109
+ onWarning(`Failed to report Droid run to Loadmill: ${message}`);
110
+ }
111
+ return { reported: false, reason: "request_failed", error: message };
112
+ }
113
+ }
@@ -0,0 +1,39 @@
1
+ function trimToNull(value) {
2
+ if (typeof value !== "string") {
3
+ return null;
4
+ }
5
+ const trimmed = value.trim();
6
+ return trimmed || null;
7
+ }
8
+ function extractUserDisplayNameFromRecord(record) {
9
+ for (const key of ["displayName", "fullName", "name", "email"]) {
10
+ const value = trimToNull(record[key]);
11
+ if (value) {
12
+ return value;
13
+ }
14
+ }
15
+ const nested = record.user;
16
+ if (nested && typeof nested === "object") {
17
+ return extractUserDisplayNameFromRecord(nested);
18
+ }
19
+ return null;
20
+ }
21
+ export function extractLoadmillSessionUser(payload) {
22
+ if (!payload || typeof payload !== "object") {
23
+ return null;
24
+ }
25
+ const record = payload;
26
+ const id = trimToNull(record.id) || trimToNull(record.userId);
27
+ const email = trimToNull(record.email);
28
+ const displayName = extractUserDisplayNameFromRecord(record);
29
+ const teamId = trimToNull(record.teamId);
30
+ if (!id) {
31
+ return null;
32
+ }
33
+ return {
34
+ id,
35
+ email,
36
+ displayName,
37
+ teamId,
38
+ };
39
+ }
@@ -0,0 +1,64 @@
1
+ import { DroidCuaSegmentEvent, buildDroidCuaTestRunCompletedProperties, } from "../../core/droid-cua-segment-events.js";
2
+ const DEFAULT_DROID_CUA_SEGMENT_WRITE_KEY = "ZuHGZkEXjhEkvzVPMnVDJfWiNsD1nGhG";
3
+ const SEGMENT_TRACK_URL = "https://api.segment.io/v1/track";
4
+ function trimToNull(value) {
5
+ if (typeof value !== "string") {
6
+ return null;
7
+ }
8
+ const trimmed = value.trim();
9
+ return trimmed || null;
10
+ }
11
+ function getSegmentWriteKey() {
12
+ if (process.env.DROID_CUA_SEGMENT_DISABLED === "1") {
13
+ return null;
14
+ }
15
+ if (process.env.DROID_CUA_SEGMENT_DISABLED === "true") {
16
+ return null;
17
+ }
18
+ return trimToNull(process.env.SEGMENT_WRITE_KEY) || DEFAULT_DROID_CUA_SEGMENT_WRITE_KEY;
19
+ }
20
+ function buildBasicAuthHeader(writeKey) {
21
+ return `Basic ${Buffer.from(`${writeKey}:`).toString("base64")}`;
22
+ }
23
+ function warnSegmentError(scope, error) {
24
+ console.warn(`[segment] ${scope}`, error);
25
+ }
26
+ export async function reportDroidCuaCiTestRunCompleted(input = {}) {
27
+ const writeKey = getSegmentWriteKey();
28
+ if (!writeKey) {
29
+ return { reported: false, reason: "disabled" };
30
+ }
31
+ const userId = trimToNull(input.userId);
32
+ const anonymousId = trimToNull(input.anonymousId) || (input.runId ? `droid-cua-ci:${input.runId}` : null);
33
+ if (!userId && !anonymousId) {
34
+ return { reported: false, reason: "missing_identity" };
35
+ }
36
+ const body = {
37
+ ...(userId ? { userId } : { anonymousId }),
38
+ event: DroidCuaSegmentEvent.TEST_RUN_COMPLETED,
39
+ properties: buildDroidCuaTestRunCompletedProperties(input),
40
+ };
41
+ try {
42
+ const response = await fetch(SEGMENT_TRACK_URL, {
43
+ method: "POST",
44
+ headers: {
45
+ Authorization: buildBasicAuthHeader(writeKey),
46
+ "Content-Type": "application/json",
47
+ },
48
+ body: JSON.stringify(body),
49
+ });
50
+ if (!response.ok) {
51
+ const detail = await response.text();
52
+ return { reported: false, reason: "request_failed", error: `Segment API error (${response.status}): ${detail}` };
53
+ }
54
+ return { reported: true };
55
+ }
56
+ catch (error) {
57
+ warnSegmentError("CI test run completed reporting failed", error);
58
+ return {
59
+ reported: false,
60
+ reason: "request_failed",
61
+ error: error instanceof Error ? error.message : String(error),
62
+ };
63
+ }
64
+ }
@@ -206,7 +206,7 @@ export class DesignModeInk {
206
206
  toolCapabilities: this.engine.toolCapabilities,
207
207
  });
208
208
  // Run turn with action tracking and interruption checking
209
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
209
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
210
210
  // Check if user interrupted - stop immediately
211
211
  if (this.inputQueue.length > 0) {
212
212
  return true; // Signal to stop execution
@@ -223,6 +223,15 @@ export class DesignModeInk {
223
223
  }
224
224
  return false; // Continue execution
225
225
  }, this.context, null, { currentScreenshot });
226
+ if (turnSummary.failed) {
227
+ if (this.context.setAgentWorking) {
228
+ this.context.setAgentWorking(false);
229
+ }
230
+ addOutput({ type: 'error', text: turnSummary.error || 'Design mode stopped.' });
231
+ this.conversationActive = false;
232
+ this.cleanup();
233
+ return;
234
+ }
226
235
  this.session.updateResponseId(newResponseId);
227
236
  this.consecutiveErrorCount = 0;
228
237
  // Clear agent working status
@@ -170,7 +170,7 @@ export class DesignMode {
170
170
  toolCapabilities: this.engine.toolCapabilities,
171
171
  });
172
172
  // Run turn with action tracking and interruption checking
173
- const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
173
+ const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
174
174
  // Check if user pressed ESC - stop immediately
175
175
  if (this.escPressed) {
176
176
  return true; // Signal to stop execution
@@ -188,6 +188,12 @@ export class DesignMode {
188
188
  }
189
189
  return false; // Continue execution
190
190
  }, null, null, { currentScreenshot });
191
+ if (turnSummary.failed) {
192
+ this.cleanupEscDetection(keypressHandler);
193
+ console.log(`\n${turnSummary.error || "Design mode stopped."}`);
194
+ this.conversationActive = false;
195
+ return;
196
+ }
191
197
  this.session.updateResponseId(newResponseId);
192
198
  this.consecutiveErrorCount = 0;
193
199
  // Cleanup ESC detection
@@ -4,15 +4,15 @@ import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurre
4
4
  import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
5
5
  import { sendCUARequest } from "../device/openai.js";
6
6
  import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
7
- import { isAssertion } from "../device/assertions.js";
7
+ import { extractAssertionPrompt, isAssertion } from "../device/assertions.js";
8
+ import { handleAssertionToolCall } from "../core/tool-handlers/assert.js";
8
9
  import { logger } from "../utils/logger.js";
9
10
  import { emitDesktopDebug } from "../utils/desktop-debug.js";
10
11
  import { printCliOutput } from "../utils/console-output.js";
11
- // Mirrors the codes emitted by the Loadmill LLM gateway (see services/app/private/llm
12
- // in the server repo): a transient burst limit plus day/week/fixed-budget request and
13
- // token quotas. Free plans run on the fixed-budget quota.
12
+ // Mirrors the quota codes emitted by the Loadmill LLM gateway (see
13
+ // services/app/private/llm in the server repo). Free plans run on the
14
+ // fixed-budget quota.
14
15
  const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
15
- "llm_rate_limit_exceeded",
16
16
  "llm_day_quota_exceeded",
17
17
  "llm_week_quota_exceeded",
18
18
  "llm_fixed_budget_quota_exceeded",
@@ -21,6 +21,8 @@ const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
21
21
  "llm_fixed_budget_token_quota_exceeded",
22
22
  ]);
23
23
  const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
24
+ const LLM_REQUEST_RATE_LIMIT_ERROR_CODE = "llm_rate_limit_exceeded";
25
+ const DEFAULT_ASSERTION_CONTEXT = "No additional prior context is needed; validate directly from the current screen.";
24
26
  function isLlmUsageLimitError(err) {
25
27
  if (!err || typeof err !== "object") {
26
28
  return false;
@@ -30,9 +32,15 @@ function isLlmUsageLimitError(err) {
30
32
  }
31
33
  return err.status === 429 && err.type === "rate_limit_error";
32
34
  }
35
+ function isLlmRequestRateLimitError(err) {
36
+ return err && typeof err === "object" && err.code === LLM_REQUEST_RATE_LIMIT_ERROR_CODE;
37
+ }
33
38
  function getLlmUsageLimitMessage() {
34
39
  return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
35
40
  }
41
+ function getLlmRequestRateLimitMessage() {
42
+ return "Execution stopped: too many AI requests are running for this account right now. Please retry shortly or reduce parallel test runs.";
43
+ }
36
44
  /**
37
45
  * Execution Mode - Run test scripts line-by-line
38
46
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -105,10 +113,11 @@ export class ExecutionMode {
105
113
  lines.push(` Retries: ${this.stats.retryCount}`);
106
114
  return lines;
107
115
  }
108
- buildStepContext(instructionIndex) {
116
+ buildStepContext(instructionIndex, instruction = null) {
109
117
  return {
110
118
  instructionIndex,
111
119
  stepId: `step-${String(instructionIndex + 1).padStart(4, "0")}`,
120
+ ...(typeof instruction === "string" ? { instruction } : {}),
112
121
  };
113
122
  }
114
123
  emit(addOutput, type, text, context = {}, stepContext = null, extra = {}) {
@@ -123,6 +132,45 @@ export class ExecutionMode {
123
132
  payload: extra.payload
124
133
  });
125
134
  }
135
+ resetInstructionConversationState() {
136
+ this.session.updateResponseId(null);
137
+ if (typeof this.session.clearTranscript === "function") {
138
+ this.session.clearTranscript();
139
+ return;
140
+ }
141
+ if (Array.isArray(this.session.transcript)) {
142
+ this.session.transcript = [];
143
+ }
144
+ }
145
+ async executeVerificationInstruction(instruction, context, stepContext = null) {
146
+ const toolResult = await handleAssertionToolCall({
147
+ engine: this.engine,
148
+ callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
149
+ argumentsText: JSON.stringify({
150
+ condition: extractAssertionPrompt(instruction),
151
+ context: DEFAULT_ASSERTION_CONTEXT,
152
+ }),
153
+ context,
154
+ stepContext,
155
+ });
156
+ const assertionStatus = toolResult.summaryUpdates?.assertionStatus;
157
+ if (assertionStatus === "passed") {
158
+ this.stats.assertionsPassed++;
159
+ }
160
+ else if (assertionStatus === "failed" || assertionStatus === "failed_skipped") {
161
+ this.stats.assertionsFailed++;
162
+ }
163
+ if (toolResult.kind === "retry_instruction") {
164
+ this.stats.retryCount++;
165
+ this.session.clearMessages();
166
+ return await this.executeInstruction(instruction, context, 0, stepContext);
167
+ }
168
+ if (toolResult.kind === "fail") {
169
+ return { success: false, error: toolResult.error || "Assertion failed." };
170
+ }
171
+ this.session.clearMessages();
172
+ return { success: true };
173
+ }
126
174
  /**
127
175
  * Execute all instructions in the test script
128
176
  * @param {Object} context - Additional context (Ink context with addOutput)
@@ -137,7 +185,8 @@ export class ExecutionMode {
137
185
  // Start timing
138
186
  this.stats.startTime = Date.now();
139
187
  for (let i = 0; i < this.instructions.length; i++) {
140
- const stepContext = this.buildStepContext(i);
188
+ const instruction = this.instructions[i];
189
+ const stepContext = this.buildStepContext(i, instruction);
141
190
  // Check if execution should be stopped
142
191
  if (this.shouldStop) {
143
192
  this.emit(addOutput, 'info', 'Test execution stopped by user.', runContext, stepContext, {
@@ -158,7 +207,6 @@ export class ExecutionMode {
158
207
  return { success: false, error: 'Stopped by user' };
159
208
  }
160
209
  }
161
- const instruction = this.instructions[i];
162
210
  this.emit(addOutput, 'user', instruction, runContext, stepContext, {
163
211
  eventType: 'instruction_started',
164
212
  payload: {
@@ -236,10 +284,17 @@ export class ExecutionMode {
236
284
  const MAX_RETRIES = 10;
237
285
  const instructionContext = {
238
286
  ...context,
287
+ currentInstruction: instruction,
239
288
  futureInstructions: this.instructions.slice((stepContext?.instructionIndex ?? -1) + 1),
240
289
  };
290
+ if (retryCount === 0) {
291
+ this.resetInstructionConversationState();
292
+ }
241
293
  this.session.addToTranscript(`[User] ${instruction}`);
242
294
  this.session.addMessage("user", instruction);
295
+ if (isAssertion(instruction)) {
296
+ return await this.executeVerificationInstruction(instruction, instructionContext, stepContext);
297
+ }
243
298
  try {
244
299
  const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
245
300
  const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
@@ -309,6 +364,26 @@ export class ExecutionMode {
309
364
  stack: err.stack
310
365
  });
311
366
  const addOutput = context.addOutput || printCliOutput;
367
+ if (isLlmRequestRateLimitError(err)) {
368
+ const stopMessage = getLlmRequestRateLimitMessage();
369
+ logger.debug('Stopping execution because an LLM request rate limit was reached', {
370
+ instruction,
371
+ message: err.message,
372
+ status: err.status,
373
+ code: err.code,
374
+ type: err.type,
375
+ });
376
+ this.emit(addOutput, 'error', stopMessage, context, stepContext, {
377
+ eventType: 'error',
378
+ payload: {
379
+ message: stopMessage,
380
+ status: err.status,
381
+ code: err.code,
382
+ type: err.type,
383
+ }
384
+ });
385
+ return { success: false, error: stopMessage, usageLimitReached: true };
386
+ }
312
387
  if (isLlmUsageLimitError(err)) {
313
388
  const stopMessage = getLlmUsageLimitMessage();
314
389
  logger.debug('Stopping execution because an LLM usage limit was reached', {
@@ -449,6 +524,7 @@ export class ExecutionMode {
449
524
  });
450
525
  // Replace the system message with the enhanced one
451
526
  this.session.installRecoverySystemPrompt(recoverySystemPrompt);
527
+ this.session.updateResponseId(null);
452
528
  // Wait a bit before retrying
453
529
  await new Promise(resolve => setTimeout(resolve, 1000));
454
530
  // Retry the same instruction with incremented counter
@@ -139,7 +139,9 @@ Test design exploration is complete. Use the validated completion summary below
139
139
  Do not call tools.
140
140
  Do not continue exploring.
141
141
  Do not add explanation.
142
- Output only the final result in the required design-mode format.
142
+ Output only the exact final result format below: one "Suggested test name:" line, then one fenced code block.
143
+ Do not use TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, or bullets.
144
+ The fenced code block must contain only executable test instructions, one per line, ending with exit.
143
145
 
144
146
  Completion summary:
145
147
  Suggested test name: ${suggestedTestName || "(choose a short kebab-case name)"}
@@ -151,3 +153,21 @@ Verification points:
151
153
  ${formatStringList(summary.verificationPoints)}
152
154
  ${notesForScript ? `Notes for script:\n${notesForScript}\n` : ""}`;
153
155
  }
156
+ export function buildDesignScriptFormatRetryPrompt(previousOutput = "") {
157
+ const previousOutputText = typeof previousOutput === "string" ? previousOutput.trim() : "";
158
+ return `AUTHORING FORMAT CORRECTION:
159
+ Your previous answer did not use the required design-mode format.
160
+
161
+ Output only:
162
+ Suggested test name: short-kebab-case-name
163
+
164
+ \`\`\`
165
+ one executable instruction per line
166
+ exit
167
+ \`\`\`
168
+
169
+ Do not include TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, bullets, or explanation.${previousOutputText ? `
170
+
171
+ Previous output to reformat:
172
+ ${previousOutputText}` : ""}`;
173
+ }
@@ -1,4 +1,5 @@
1
1
  import { classifyCaptureSource } from "../core/capture-sources.js";
2
+ import { stepCompletionData } from "../core/report/run-log-model.js";
2
3
  function isHiddenExecutionOutput(item) {
3
4
  const captureSource = item?.payload?.captureSource;
4
5
  return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
@@ -24,6 +25,10 @@ export function formatCliOutput(item) {
24
25
  if (!text || text.trim().length === 0) {
25
26
  return null;
26
27
  }
28
+ const stepCompletion = stepCompletionData(text);
29
+ if (stepCompletion) {
30
+ return `assistant: Step complete — ${stepCompletion.instruction}`;
31
+ }
27
32
  return text
28
33
  .split("\n")
29
34
  .map((line) => `assistant: ${line}`)
@@ -92,6 +92,54 @@ function summarizeRequestInput(input) {
92
92
  };
93
93
  });
94
94
  }
95
+ function extractImageBase64Length(imageUrl) {
96
+ return typeof imageUrl === "string" ? imageUrl.replace("data:image/png;base64,", "").length : 0;
97
+ }
98
+ function summarizeRequestScreenshots(input, deviceInfo) {
99
+ const screenshots = [];
100
+ for (const item of input) {
101
+ if (item?.role && Array.isArray(item.content)) {
102
+ for (const entry of item.content) {
103
+ if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
104
+ continue;
105
+ screenshots.push({
106
+ source: `message:${item.role}`,
107
+ base64Length: extractImageBase64Length(entry.image_url),
108
+ });
109
+ }
110
+ continue;
111
+ }
112
+ if (item?.type === "computer_call_output" && typeof item.output?.image_url === "string") {
113
+ screenshots.push({
114
+ source: "computer_call_output",
115
+ callId: item.call_id,
116
+ base64Length: extractImageBase64Length(item.output.image_url),
117
+ });
118
+ continue;
119
+ }
120
+ if (item?.type === "function_call_output" && Array.isArray(item.output)) {
121
+ for (const entry of item.output) {
122
+ if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
123
+ continue;
124
+ screenshots.push({
125
+ source: "function_call_output",
126
+ callId: item.call_id,
127
+ base64Length: extractImageBase64Length(entry.image_url),
128
+ });
129
+ }
130
+ }
131
+ }
132
+ if (screenshots.length === 0) {
133
+ return null;
134
+ }
135
+ return {
136
+ count: screenshots.length,
137
+ sources: screenshots.map((entry) => entry.source),
138
+ base64Lengths: screenshots.map((entry) => entry.base64Length),
139
+ width: deviceInfo?.scaled_width ?? null,
140
+ height: deviceInfo?.scaled_height ?? null,
141
+ };
142
+ }
95
143
  function redactRequestForFileLog(requestParams) {
96
144
  return {
97
145
  ...requestParams,
@@ -223,6 +271,7 @@ export class CuaDebugTracer {
223
271
  const localRequestId = crypto.randomUUID();
224
272
  const requestLog = redactRequestForFileLog(requestParams);
225
273
  const messages = summarizeRequestInput(input);
274
+ const screenshot = summarizeRequestScreenshots(input, deviceInfo);
226
275
  const acknowledgedSafetyChecksSent = input
227
276
  .filter((item) => item?.type === "computer_call_output")
228
277
  .flatMap((item) => {
@@ -260,13 +309,7 @@ export class CuaDebugTracer {
260
309
  inputItemTypes,
261
310
  inputCallIds,
262
311
  messages,
263
- screenshot: screenshotBase64
264
- ? {
265
- width: deviceInfo?.scaled_width ?? null,
266
- height: deviceInfo?.scaled_height ?? null,
267
- base64Length: screenshotBase64.length
268
- }
269
- : null,
312
+ screenshot,
270
313
  safetyChecks: {
271
314
  previousPending: previousPendingSafetyChecks,
272
315
  previousPendingCount: previousPendingSafetyChecks.length,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loadmill/droid-cua",
3
- "version": "2.21.0",
3
+ "version": "2.23.0",
4
4
  "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
5
5
  "main": "build/index.js",
6
6
  "type": "module",
@@ -20,6 +20,7 @@
20
20
  "bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
21
21
  "bench:report": "node scripts/benchmark-report.js",
22
22
  "bench:report:web": "node scripts/benchmark-report-web.js",
23
+ "obs:droid": "node scripts/obs-droid-demo.mjs",
23
24
  "execution:profile": "node scripts/execution-profile.js",
24
25
  "test": "node --test unit-tests/*.test.js",
25
26
  "test:aws-device-farm:android": "node integration-tests/aws-device-farm-android.test.js",
@@ -1,11 +0,0 @@
1
- export const CUA_BACKENDS = ["openai", "fara"];
2
- export const DEFAULT_CUA_BACKEND = "openai";
3
- export const DEFAULT_FARA_BASE_URL = "http://localhost:5000/v1";
4
- export const DEFAULT_FARA_MODEL = "Fara-7B";
5
- const VALID_CUA_BACKENDS = new Set(CUA_BACKENDS);
6
- export function isValidCuaBackend(value) {
7
- return VALID_CUA_BACKENDS.has(value);
8
- }
9
- export function resolveCuaBackend(value) {
10
- return isValidCuaBackend(value) ? value : DEFAULT_CUA_BACKEND;
11
- }