@loadmill/droid-cua 2.8.0 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +7 -6
  2. package/build/index.js +89 -71
  3. package/build/src/brand/loadmill-mark.js +20 -0
  4. package/build/src/browser/playwright/connection.js +160 -12
  5. package/build/src/browser/playwright/welcome-page.js +161 -0
  6. package/build/src/cli/headless-execution-config.js +18 -58
  7. package/build/src/cli/headless-target.js +0 -9
  8. package/build/src/cli/ink-shell.js +5 -2
  9. package/build/src/commands/help.js +5 -8
  10. package/build/src/commands/loadmill.js +10 -1
  11. package/build/src/commands/run.js +17 -28
  12. package/build/src/core/aim-assist.js +5 -0
  13. package/build/src/core/app-context.js +57 -33
  14. package/build/src/core/context-optimization.js +1 -1
  15. package/build/src/core/cua-models.js +9 -0
  16. package/build/src/core/execution-engine.js +121 -199
  17. package/build/src/core/runtime-config.js +95 -0
  18. package/build/src/device/loadmill.js +15 -9
  19. package/build/src/device/openai.js +76 -142
  20. package/build/src/integrations/loadmill/client.js +149 -131
  21. package/build/src/integrations/loadmill/executor.js +17 -11
  22. package/build/src/integrations/loadmill/index.js +1 -1
  23. package/build/src/integrations/loadmill/interpreter.js +4 -14
  24. package/build/src/modes/design-mode-ink.js +5 -6
  25. package/build/src/modes/design-mode.js +5 -6
  26. package/build/src/modes/execution-mode.js +26 -0
  27. package/build/src/prompts/base.js +44 -30
  28. package/build/src/prompts/design.js +3 -4
  29. package/build/src/prompts/execution.js +13 -110
  30. package/build/src/utils/console-output.js +8 -0
  31. package/build/src/utils/cua-debug-tracer.js +0 -1
  32. package/build/src/utils/structured-debug-log-manager.js +4 -1
  33. package/package.json +1 -1
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Loadmill integration public exports
3
3
  */
4
- export { getApiToken, searchFlows, runTestFlow, getFlowRunStatus } from "./client.js";
4
+ export { createLoadmillApiClient, getApiToken, searchFlows, runTestFlow, getFlowRunStatus } from "./client.js";
5
5
  export { interpretLoadmillCommand, selectBestFlow } from "./interpreter.js";
6
6
  export { executeLoadmillCommand } from "./executor.js";
@@ -1,24 +1,14 @@
1
1
  /**
2
2
  * AI-powered text interpretation for Loadmill commands
3
3
  */
4
- import OpenAI from "openai";
5
4
  import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionMessages, } from "../../prompts/loadmill.js";
6
- let openai = null;
7
- function getOpenAI() {
8
- if (!openai) {
9
- openai = new OpenAI({
10
- apiKey: process.env.OPENAI_API_KEY,
11
- });
12
- }
13
- return openai;
14
- }
15
5
  /**
16
6
  * Interpret a natural language Loadmill command into structured data
17
7
  * @param {string} userInput - Natural language command
18
8
  * @returns {Promise<{searchQuery: string, parameters: Object, action: 'run'|'search'}>}
19
9
  */
20
- export async function interpretLoadmillCommand(userInput) {
21
- const response = await getOpenAI().chat.completions.create({
10
+ export async function interpretLoadmillCommand(userInput, { client }) {
11
+ const response = await client.chat.completions.create({
22
12
  model: "gpt-4o-mini",
23
13
  messages: buildLoadmillCommandInterpretationMessages(userInput),
24
14
  response_format: { type: "json_object" }
@@ -37,7 +27,7 @@ export async function interpretLoadmillCommand(userInput) {
37
27
  * @param {string} originalQuery - Original user query
38
28
  * @returns {Promise<{selectedFlow: Object|null, confidence: number}>}
39
29
  */
40
- export async function selectBestFlow(flows, originalQuery) {
30
+ export async function selectBestFlow(flows, originalQuery, { client }) {
41
31
  // Ensure flows is an array
42
32
  if (!flows || !Array.isArray(flows) || flows.length === 0) {
43
33
  return { selectedFlow: null, confidence: 0 };
@@ -51,7 +41,7 @@ export async function selectBestFlow(flows, originalQuery) {
51
41
  const suite = f.testSuiteDescription || "";
52
42
  return `${i + 1}. ID: ${f.id}, Name: "${name}"${suite ? `, Suite: "${suite}"` : ""}`;
53
43
  }).join("\n");
54
- const response = await getOpenAI().chat.completions.create({
44
+ const response = await client.chat.completions.create({
55
45
  model: "gpt-4o-mini",
56
46
  messages: buildLoadmillFlowSelectionMessages(originalQuery, flowList),
57
47
  response_format: { type: "json_object" }
@@ -1,5 +1,4 @@
1
1
  import { getScreenshotAsBase64 } from "../device/connection.js";
2
- import { sendCUARequest, reviseTestScript } from "../device/openai.js";
3
2
  import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
4
3
  import { saveTest } from "../test-store/test-manager.js";
5
4
  import { logger } from "../utils/logger.js";
@@ -23,6 +22,8 @@ export class DesignModeInk {
23
22
  this.baseDesignPrompt = null;
24
23
  this.consecutiveErrorCount = 0;
25
24
  this.maxConsecutiveErrors = 3;
25
+ this.sendCUARequest = executionEngine.sendCUARequest;
26
+ this.reviseTestScript = executionEngine.reviseTestScript;
26
27
  }
27
28
  /**
28
29
  * Start design mode conversation
@@ -31,9 +32,7 @@ export class DesignModeInk {
31
32
  async start() {
32
33
  const addOutput = this.context.addOutput;
33
34
  // Set design mode system prompt
34
- const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, {
35
- strictMode: Boolean(this.engine?.strictMode)
36
- });
35
+ const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
37
36
  this.baseDesignPrompt = designPrompt;
38
37
  this.session.setSystemPrompt(designPrompt);
39
38
  // Update UI
@@ -183,7 +182,7 @@ export class DesignModeInk {
183
182
  }
184
183
  // Get screenshot and send to model
185
184
  const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
186
- const response = await sendCUARequest({
185
+ const response = await this.sendCUARequest({
187
186
  messages: this.session.messages,
188
187
  screenshotBase64,
189
188
  previousResponseId: this.session.previousResponseId,
@@ -271,7 +270,7 @@ export class DesignModeInk {
271
270
  const revision = confirm;
272
271
  // Use simple chat completion for revision instead of CUA
273
272
  addOutput({ type: 'info', text: 'Revising test script...' });
274
- currentScript = await reviseTestScript(currentScript, revision);
273
+ currentScript = await this.reviseTestScript(currentScript, revision);
275
274
  // Display the revised script
276
275
  addOutput({ type: 'info', text: '' });
277
276
  addOutput({ type: 'system', text: '=== Generated Test Script ===' });
@@ -1,6 +1,5 @@
1
1
  import readline from "readline";
2
2
  import { getScreenshotAsBase64 } from "../device/connection.js";
3
- import { sendCUARequest, reviseTestScript } from "../device/openai.js";
4
3
  import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
5
4
  import { saveTest } from "../test-store/test-manager.js";
6
5
  import { logger } from "../utils/logger.js";
@@ -20,6 +19,8 @@ export class DesignMode {
20
19
  this.baseDesignPrompt = null;
21
20
  this.consecutiveErrorCount = 0;
22
21
  this.maxConsecutiveErrors = 3;
22
+ this.sendCUARequest = executionEngine.sendCUARequest;
23
+ this.reviseTestScript = executionEngine.reviseTestScript;
23
24
  }
24
25
  /**
25
26
  * Start design mode conversation
@@ -28,9 +29,7 @@ export class DesignMode {
28
29
  */
29
30
  async start(context) {
30
31
  // Set design mode system prompt
31
- const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, {
32
- strictMode: Boolean(this.engine?.strictMode)
33
- });
32
+ const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
34
33
  this.baseDesignPrompt = designPrompt;
35
34
  this.session.setSystemPrompt(designPrompt);
36
35
  console.log(`\n=== Design Mode: Creating test "${this.testName}" ===`);
@@ -147,7 +146,7 @@ export class DesignMode {
147
146
  const keypressHandler = this.setupEscDetection();
148
147
  // Get screenshot and send to model
149
148
  const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
150
- const response = await sendCUARequest({
149
+ const response = await this.sendCUARequest({
151
150
  messages: this.session.messages,
152
151
  screenshotBase64,
153
152
  previousResponseId: this.session.previousResponseId,
@@ -262,7 +261,7 @@ export class DesignMode {
262
261
  const revision = confirm;
263
262
  // Use simple chat completion for revision instead of CUA
264
263
  console.log("Revising test script...");
265
- currentScript = await reviseTestScript(currentScript, revision);
264
+ currentScript = await this.reviseTestScript(currentScript, revision);
266
265
  // Display the revised script
267
266
  console.log("\n=== Generated Test Script ===");
268
267
  console.log(currentScript);
@@ -228,6 +228,14 @@ export class ExecutionMode {
228
228
  }
229
229
  const navigationTarget = extractNavigateTarget(instruction);
230
230
  this.session.addToTranscript(`[Navigate] ${navigationTarget}`);
231
+ emitDesktopDebug("runner.navigate.started", "execution", {
232
+ runId: context?.runId,
233
+ stepId: stepContext?.stepId,
234
+ instructionIndex: stepContext?.instructionIndex
235
+ }, {
236
+ instruction,
237
+ target: navigationTarget
238
+ });
231
239
  this.emit(addOutput, "system", `Navigating to ${navigationTarget}...`, context, stepContext, {
232
240
  eventType: "system_message",
233
241
  payload: {
@@ -250,6 +258,15 @@ export class ExecutionMode {
250
258
  navigatedUrl: this.session.currentUrl,
251
259
  });
252
260
  this.session.installRecoverySystemPrompt(navigationRecoveryPrompt);
261
+ emitDesktopDebug("runner.navigate.completed", "execution", {
262
+ runId: context?.runId,
263
+ stepId: stepContext?.stepId,
264
+ instructionIndex: stepContext?.instructionIndex
265
+ }, {
266
+ instruction,
267
+ target: navigationTarget,
268
+ url: normalizedUrl
269
+ });
253
270
  this.emit(addOutput, "success", `Navigation complete: ${normalizedUrl}`, context, stepContext, {
254
271
  eventType: "system_message",
255
272
  payload: {
@@ -266,6 +283,15 @@ export class ExecutionMode {
266
283
  retryCount,
267
284
  message,
268
285
  });
286
+ emitDesktopDebug("runner.navigate.failed", "execution", {
287
+ runId: context?.runId,
288
+ stepId: stepContext?.stepId,
289
+ instructionIndex: stepContext?.instructionIndex
290
+ }, {
291
+ instruction,
292
+ target: navigationTarget,
293
+ message
294
+ });
269
295
  this.emit(addOutput, "error", `Navigation failed: ${message}`, context, stepContext, {
270
296
  eventType: "error",
271
297
  payload: {
@@ -27,16 +27,6 @@ function appendCustomSections(prompt, sections = []) {
27
27
  ${customSection}
28
28
  `;
29
29
  }
30
- function buildStrictModeRuntimeSection(runtimeOptions = {}) {
31
- if (runtimeOptions?.strictMode !== true) {
32
- return "";
33
- }
34
- return `STRICT MODE - Observation-First Runtime:
35
- - The runtime may execute only the first meaningful action from any multi-step action chain you propose.
36
- - After that first action, it will re-observe the ${runtimeOptions?.platform === "browser" ? "browser" : "device"} and continue from a fresh screenshot.
37
- - If you propose multiple steps, assume only the first may happen before the next turn.
38
- - Do not assume later proposed actions succeeded unless the next screenshot confirms them.`;
39
- }
40
30
  function isBrowserTarget(deviceInfo = {}) {
41
31
  return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
42
32
  }
@@ -55,7 +45,42 @@ function describeControlledTarget(deviceInfo = {}) {
55
45
  }
56
46
  return "a mobile device";
57
47
  }
58
- function buildBrowserBaseSystemPrompt(deviceInfo) {
48
+ function buildAlwaysAimAssistGuidance(targetKind) {
49
+ return [
50
+ "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
51
+ "- Review the returned annotated screenshot before clicking.",
52
+ `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable" : "tappable"} control.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
53
+ `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
54
+ "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
55
+ "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
56
+ "- Click precisely based on your visual estimate.",
57
+ ];
58
+ }
59
+ function buildAimAssistGuidance(targetKind, aimAssistMode = "always") {
60
+ if (aimAssistMode === "off") {
61
+ return [
62
+ "- Click directly based on your best visual estimate.",
63
+ "- If a click has no effect, treat it as a likely misclick and try a meaningfully different coordinate.",
64
+ "- Click precisely based on your visual estimate.",
65
+ ].join("\n ");
66
+ }
67
+ if (aimAssistMode === "smart") {
68
+ const targetWord = targetKind === "browser" ? "clickable" : "tappable";
69
+ return [
70
+ ...buildAlwaysAimAssistGuidance(targetKind),
71
+ "- In Smart mode, you may skip preview only when the target is broad, isolated, stable, and unambiguous, and you are confident the click point is clearly inside the intended target.",
72
+ "- Good skip examples are large empty text fields, large search fields, large CTA buttons with generous padding, launcher icons, and wide rows with generous spacing and no nearby competing actions.",
73
+ `- Do not skip preview when the target is small, dense, visually ambiguous, close to neighboring controls, or you are not confident the marker will land clearly inside the intended ${targetWord} target.`,
74
+ "- If a dropdown, expanded option list, popover, or select menu is visible, always use preview before selecting an option.",
75
+ "- Do not skip preview in search results, bottom sheets, bottom cards, modals, overlays, keyboard-driven flows, or after recent screen changes.",
76
+ "- Do not skip preview for known tricky UI such as dense lists, date pickers, time pickers, calendar cells, wheels, segmented selectors, and small inline controls inside larger rows or cards.",
77
+ "- If the target is one option among many visually similar options, use preview before clicking.",
78
+ ].join("\n ");
79
+ }
80
+ return buildAlwaysAimAssistGuidance(targetKind).join("\n ");
81
+ }
82
+ function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
83
+ const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
59
84
  return `
60
85
  You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
61
86
  Follow the user's instructions to interact with the browser.
@@ -67,13 +92,7 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
67
92
 
68
93
  When aiming for visual targets:
69
94
  - Reason carefully about the approximate pixel position.
70
- - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
71
- - Review the returned annotated screenshot before clicking.
72
- - Click if the marker is clearly inside the intended clickable target.
73
- - Adjust and preview again only if the marker is outside the target, on the edge, or closer to a neighboring target than the intended one.
74
- - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
75
- - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
76
- - Click precisely based on your visual estimate.
95
+ ${buildAimAssistGuidance("browser", aimAssistMode)}
77
96
 
78
97
  Available actions: click, scroll, type, keypress, wait, screenshot.
79
98
 
@@ -129,7 +148,8 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
129
148
  - CTRL/CMD/ALT/OPTION/SHIFT may be used when they are the most reliable path
130
149
  `;
131
150
  }
132
- function buildMobileBaseSystemPrompt(deviceInfo) {
151
+ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
152
+ const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
133
153
  return `
134
154
  You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
135
155
  Follow the user's instructions to interact with the device.
@@ -142,13 +162,7 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
142
162
 
143
163
  When aiming for visual targets:
144
164
  - Reason carefully about the approximate pixel position.
145
- - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
146
- - Review the returned annotated screenshot before clicking.
147
- - Click if the marker is clearly inside the intended tappable control. Text inside a button, field, row, or bottom bar is okay.
148
- - Adjust and preview again only if the marker is outside the target, on the target border/edge, or closer to a neighboring target than the intended one.
149
- - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
150
- - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
151
- - Click precisely based on your visual estimate.
165
+ ${buildAimAssistGuidance("mobile", aimAssistMode)}
152
166
 
153
167
  Available actions: click, scroll, type, keypress, wait, screenshot.
154
168
 
@@ -209,9 +223,9 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
209
223
  - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
210
224
  `;
211
225
  }
212
- function buildBaseSystemPrompt(deviceInfo) {
226
+ function buildBaseSystemPrompt(deviceInfo, options = {}) {
213
227
  return isBrowserTarget(deviceInfo)
214
- ? buildBrowserBaseSystemPrompt(deviceInfo)
215
- : buildMobileBaseSystemPrompt(deviceInfo);
228
+ ? buildBrowserBaseSystemPrompt(deviceInfo, options)
229
+ : buildMobileBaseSystemPrompt(deviceInfo, options);
216
230
  }
217
- export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, buildStrictModeRuntimeSection, };
231
+ export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };
@@ -1,8 +1,7 @@
1
- import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
2
- export function buildDesignModePrompt(deviceInfo, customInstructions = {}, runtimeOptions = {}) {
1
+ import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
2
+ export function buildDesignModePrompt(deviceInfo, customInstructions = {}) {
3
3
  const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
4
4
  const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
5
- const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
6
5
  const browserTarget = isBrowserTarget(deviceInfo);
7
6
  const prompt = `${basePrompt}
8
7
 
@@ -87,7 +86,7 @@ WRONG Example (DON'T DO THIS):
87
86
  \`\`\`
88
87
 
89
88
  Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
90
- ` + (strictModeSection ? `\n\n${strictModeSection}` : "");
89
+ `;
91
90
  return appendCustomSections(prompt, [
92
91
  { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
93
92
  { title: "Design Mode Instructions", text: designCustomText }
@@ -1,20 +1,21 @@
1
- import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
2
- export function buildAppContextSection(briefing) {
3
- const text = typeof briefing === "string" ? briefing.trim() : "";
4
- if (!text) {
1
+ import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
2
+ import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
3
+ export function buildAppContextSection(text) {
4
+ const trimmed = typeof text === "string" ? text.trim() : "";
5
+ if (!trimmed) {
5
6
  return "";
6
7
  }
7
- return `APP CONTEXT BRIEFING:
8
- The following is a condensed description of the app you are testing, relevant to the current task.
8
+ return `APP CONTEXT:
9
+ The following describes the app you are testing.
9
10
  Use this to understand screen layouts, terminology, navigation, and expected behavior.
10
11
 
11
- ${text}`;
12
+ ${trimmed}`;
12
13
  }
13
- export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextBriefing = "", runtimeOptions = {}) {
14
+ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
14
15
  const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
15
- const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
16
- const appContextSection = buildAppContextSection(appContextBriefing);
17
- const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
16
+ const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
17
+ const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
18
+ const appContextSection = buildAppContextSection(appContextText);
18
19
  const browserTarget = isBrowserTarget(deviceInfo);
19
20
  const prompt = `${basePrompt}
20
21
 
@@ -45,7 +46,7 @@ Your process:
45
46
  6. Stop immediately - no commentary, no questions
46
47
 
47
48
  Each instruction is independent. Do not reference previous instructions or ask about next steps.
48
- ${appContextSection ? `\n\n${appContextSection}` : ""}` + (strictModeSection ? `\n\n${strictModeSection}` : "");
49
+ ${appContextSection ? `\n\n${appContextSection}` : ""}`;
49
50
  return appendCustomSections(prompt, [
50
51
  { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
51
52
  { title: "Execution Mode Instructions", text: executionCustomText }
@@ -104,101 +105,3 @@ Remember:
104
105
  - You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
105
106
  - Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
106
107
  }
107
- export function buildAppContextCompactionInput({ contextDocument, taskDescription, tokenBudget }) {
108
- return [
109
- {
110
- role: "system",
111
- content: [{
112
- type: "input_text",
113
- text: `You are compressing an app context document for an app testing agent.
114
-
115
- You will receive:
116
- 1. A context document
117
- 2. A test task
118
-
119
- Your job is to SELECT only the facts from the context document that are useful for the given task.
120
- The output will be injected into a system prompt with a strict token budget.
121
-
122
- CRITICAL:
123
- - Use only facts explicitly supported by the context document
124
- - Never invent, infer, normalize, substitute, or improve credentials, labels, screen names, button names, or numeric values
125
- - Preserve exact values verbatim when present in the source
126
- - Prefer facts that help the agent act correctly when they are not obvious from the task alone
127
- - Do not restate, paraphrase, summarize, or reorganize the test task
128
- - The output must not read like instructions or a test plan
129
- - Do not describe what the agent should do
130
- - Output only reference knowledge about the app
131
- - If a line could be copied from the task with minor wording changes, omit it
132
- - Prefer copying source facts verbatim or near-verbatim over rewriting them
133
- - Do not collapse multiple specific source facts into one generic summary if that removes useful distinctions
134
-
135
- Selection priority:
136
- 1. Facts the agent would NOT know from the test script alone
137
- 2. Facts that are hard to infer from screenshots
138
- 3. Non-obvious navigation or interaction details
139
- 4. Exact visible labels needed to act correctly
140
- 5. Credentials and other exact values
141
-
142
- High-value facts:
143
- - exact UI labels
144
- - how state, mode, or account selection is performed
145
- - where logout is located
146
- - hidden or non-obvious navigation
147
- - which menu items are decorative or non-functional
148
- - screen titles and section labels used to confirm location
149
- - exact credentials and role labels
150
-
151
- Low-value facts:
152
- - restating the test steps
153
- - repeating literal values already present in the task
154
- - generic summaries like "approve the transaction"
155
-
156
- When the task involves authentication, switching state or mode, opening menus, or moving between major areas of the app, strongly prefer including:
157
- - how account, state, or mode selection is performed
158
- - exact visible labels for the relevant controls
159
- - where exit or sign-out actions are located
160
- - the screen or section labels that confirm the agent is in the right place
161
-
162
- Rules:
163
- - Output plain text only
164
- - No markdown, no bullet symbols, no numbering, no headers
165
- - Use terse, factual language: one fact per line, no filler words
166
- - Blank lines only to separate logical groups
167
- - Prefer exact visible UI labels over summaries
168
- - Do not describe step-by-step procedures
169
- - Do not restate the test workflow
170
- - State only facts about screens, elements, hidden interactions, entities, credentials, and navigation
171
- - If a useful fact is not explicitly stated in the context document, omit it
172
- - Include only information relevant to this task
173
- - Do not waste space repeating the task itself
174
- - If the task already states a value or action, include it only when the context adds non-obvious execution details
175
- - Return a short result or an empty string if little is relevant
176
- - Target: under ${tokenBudget} tokens
177
-
178
- Bad output patterns to avoid:
179
- - generic summaries that remove actionable details
180
- - lines that restate the task in generic prose
181
- - lines that describe obvious workflow steps instead of app knowledge
182
- - lines that replace exact source labels or mechanisms with broad summaries
183
-
184
- Good output characteristics:
185
- - preserves the exact label or mechanism from the source when it matters
186
- - keeps distinctions like dropdown vs tabs, drawer vs visible button, exact section titles, exact button text
187
- - includes hidden or non-obvious navigation details when relevant
188
-
189
- Return only the briefing text.`
190
- }]
191
- },
192
- {
193
- role: "user",
194
- content: [{
195
- type: "input_text",
196
- text: `APP CONTEXT DOCUMENT:
197
- ${contextDocument}
198
-
199
- TASK:
200
- ${taskDescription}`
201
- }]
202
- }
203
- ];
204
- }
@@ -1,3 +1,8 @@
1
+ import { classifyCaptureSource } from "../core/capture-sources.js";
2
+ function isHiddenExecutionOutput(item) {
3
+ const captureSource = item?.payload?.captureSource;
4
+ return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
5
+ }
1
6
  export function formatCliOutput(item) {
2
7
  if (item == null) {
3
8
  return null;
@@ -8,6 +13,9 @@ export function formatCliOutput(item) {
8
13
  if (typeof item !== "object") {
9
14
  return String(item);
10
15
  }
16
+ if (isHiddenExecutionOutput(item)) {
17
+ return null;
18
+ }
11
19
  const text = typeof item.text === "string" ? item.text : null;
12
20
  const type = typeof item.type === "string" ? item.type : "";
13
21
  const eventType = typeof item.eventType === "string" ? item.eventType : "";
@@ -94,7 +94,6 @@ function redactRequestForFileLog(requestParams) {
94
94
  ...item.output,
95
95
  image_url: `data:image/png;base64,[${base64Data.length} chars]`
96
96
  },
97
- current_url: item.current_url,
98
97
  acknowledged_safety_checks: item.acknowledged_safety_checks
99
98
  };
100
99
  }
@@ -20,7 +20,10 @@ function compactTimestamp(iso) {
20
20
  }
21
21
  function sanitizeValue(value, pathParts = [], eventName = "") {
22
22
  if (typeof value === "string") {
23
- if (eventName === "cua.request.full" || eventName === "cua.response.full" || eventName === "app_context.briefing.full") {
23
+ if (eventName === "cua.request.full" ||
24
+ eventName === "cua.response.full" ||
25
+ eventName === "app_context.briefing.full" ||
26
+ eventName === "app_context.loaded") {
24
27
  return value;
25
28
  }
26
29
  if (isPromptContentPath(pathParts)) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loadmill/droid-cua",
3
- "version": "2.8.0",
3
+ "version": "2.10.0",
4
4
  "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
5
5
  "main": "build/index.js",
6
6
  "type": "module",