@loadmill/droid-cua 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/build/index.js +86 -63
- package/build/src/browser/playwright/connection.js +13 -2
- package/build/src/cli/headless-execution-config.js +4 -36
- package/build/src/cli/ink-shell.js +5 -2
- package/build/src/commands/help.js +2 -3
- package/build/src/commands/loadmill.js +10 -1
- package/build/src/commands/run.js +17 -28
- package/build/src/core/app-context.js +57 -33
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/cua-models.js +9 -0
- package/build/src/core/execution-engine.js +2 -8
- package/build/src/core/runtime-config.js +96 -0
- package/build/src/device/loadmill.js +15 -9
- package/build/src/device/openai.js +67 -139
- package/build/src/integrations/loadmill/client.js +149 -131
- package/build/src/integrations/loadmill/executor.js +17 -11
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/interpreter.js +4 -14
- package/build/src/modes/design-mode-ink.js +4 -3
- package/build/src/modes/design-mode.js +4 -3
- package/build/src/modes/execution-mode.js +26 -0
- package/build/src/prompts/execution.js +8 -106
- package/build/src/utils/console-output.js +8 -0
- package/build/src/utils/cua-debug-tracer.js +0 -1
- package/build/src/utils/structured-debug-log-manager.js +4 -1
- package/package.json +1 -1
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
|
|
2
|
-
export function buildAppContextSection(
|
|
3
|
-
const
|
|
4
|
-
if (!
|
|
2
|
+
export function buildAppContextSection(text) {
|
|
3
|
+
const trimmed = typeof text === "string" ? text.trim() : "";
|
|
4
|
+
if (!trimmed) {
|
|
5
5
|
return "";
|
|
6
6
|
}
|
|
7
|
-
return `APP CONTEXT
|
|
8
|
-
The following
|
|
7
|
+
return `APP CONTEXT:
|
|
8
|
+
The following describes the app you are testing.
|
|
9
9
|
Use this to understand screen layouts, terminology, navigation, and expected behavior.
|
|
10
10
|
|
|
11
|
-
${
|
|
11
|
+
${trimmed}`;
|
|
12
12
|
}
|
|
13
|
-
export function buildExecutionModePrompt(deviceInfo, customInstructions = {},
|
|
13
|
+
export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", runtimeOptions = {}) {
|
|
14
14
|
const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
|
|
15
15
|
const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
|
|
16
|
-
const appContextSection = buildAppContextSection(
|
|
16
|
+
const appContextSection = buildAppContextSection(appContextText);
|
|
17
17
|
const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
|
|
18
18
|
const browserTarget = isBrowserTarget(deviceInfo);
|
|
19
19
|
const prompt = `${basePrompt}
|
|
@@ -104,101 +104,3 @@ Remember:
|
|
|
104
104
|
- You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
|
|
105
105
|
- Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
|
|
106
106
|
}
|
|
107
|
-
export function buildAppContextCompactionInput({ contextDocument, taskDescription, tokenBudget }) {
|
|
108
|
-
return [
|
|
109
|
-
{
|
|
110
|
-
role: "system",
|
|
111
|
-
content: [{
|
|
112
|
-
type: "input_text",
|
|
113
|
-
text: `You are compressing an app context document for an app testing agent.
|
|
114
|
-
|
|
115
|
-
You will receive:
|
|
116
|
-
1. A context document
|
|
117
|
-
2. A test task
|
|
118
|
-
|
|
119
|
-
Your job is to SELECT only the facts from the context document that are useful for the given task.
|
|
120
|
-
The output will be injected into a system prompt with a strict token budget.
|
|
121
|
-
|
|
122
|
-
CRITICAL:
|
|
123
|
-
- Use only facts explicitly supported by the context document
|
|
124
|
-
- Never invent, infer, normalize, substitute, or improve credentials, labels, screen names, button names, or numeric values
|
|
125
|
-
- Preserve exact values verbatim when present in the source
|
|
126
|
-
- Prefer facts that help the agent act correctly when they are not obvious from the task alone
|
|
127
|
-
- Do not restate, paraphrase, summarize, or reorganize the test task
|
|
128
|
-
- The output must not read like instructions or a test plan
|
|
129
|
-
- Do not describe what the agent should do
|
|
130
|
-
- Output only reference knowledge about the app
|
|
131
|
-
- If a line could be copied from the task with minor wording changes, omit it
|
|
132
|
-
- Prefer copying source facts verbatim or near-verbatim over rewriting them
|
|
133
|
-
- Do not collapse multiple specific source facts into one generic summary if that removes useful distinctions
|
|
134
|
-
|
|
135
|
-
Selection priority:
|
|
136
|
-
1. Facts the agent would NOT know from the test script alone
|
|
137
|
-
2. Facts that are hard to infer from screenshots
|
|
138
|
-
3. Non-obvious navigation or interaction details
|
|
139
|
-
4. Exact visible labels needed to act correctly
|
|
140
|
-
5. Credentials and other exact values
|
|
141
|
-
|
|
142
|
-
High-value facts:
|
|
143
|
-
- exact UI labels
|
|
144
|
-
- how state, mode, or account selection is performed
|
|
145
|
-
- where logout is located
|
|
146
|
-
- hidden or non-obvious navigation
|
|
147
|
-
- which menu items are decorative or non-functional
|
|
148
|
-
- screen titles and section labels used to confirm location
|
|
149
|
-
- exact credentials and role labels
|
|
150
|
-
|
|
151
|
-
Low-value facts:
|
|
152
|
-
- restating the test steps
|
|
153
|
-
- repeating literal values already present in the task
|
|
154
|
-
- generic summaries like "approve the transaction"
|
|
155
|
-
|
|
156
|
-
When the task involves authentication, switching state or mode, opening menus, or moving between major areas of the app, strongly prefer including:
|
|
157
|
-
- how account, state, or mode selection is performed
|
|
158
|
-
- exact visible labels for the relevant controls
|
|
159
|
-
- where exit or sign-out actions are located
|
|
160
|
-
- the screen or section labels that confirm the agent is in the right place
|
|
161
|
-
|
|
162
|
-
Rules:
|
|
163
|
-
- Output plain text only
|
|
164
|
-
- No markdown, no bullet symbols, no numbering, no headers
|
|
165
|
-
- Use terse, factual language: one fact per line, no filler words
|
|
166
|
-
- Blank lines only to separate logical groups
|
|
167
|
-
- Prefer exact visible UI labels over summaries
|
|
168
|
-
- Do not describe step-by-step procedures
|
|
169
|
-
- Do not restate the test workflow
|
|
170
|
-
- State only facts about screens, elements, hidden interactions, entities, credentials, and navigation
|
|
171
|
-
- If a useful fact is not explicitly stated in the context document, omit it
|
|
172
|
-
- Include only information relevant to this task
|
|
173
|
-
- Do not waste space repeating the task itself
|
|
174
|
-
- If the task already states a value or action, include it only when the context adds non-obvious execution details
|
|
175
|
-
- Return a short result or an empty string if little is relevant
|
|
176
|
-
- Target: under ${tokenBudget} tokens
|
|
177
|
-
|
|
178
|
-
Bad output patterns to avoid:
|
|
179
|
-
- generic summaries that remove actionable details
|
|
180
|
-
- lines that restate the task in generic prose
|
|
181
|
-
- lines that describe obvious workflow steps instead of app knowledge
|
|
182
|
-
- lines that replace exact source labels or mechanisms with broad summaries
|
|
183
|
-
|
|
184
|
-
Good output characteristics:
|
|
185
|
-
- preserves the exact label or mechanism from the source when it matters
|
|
186
|
-
- keeps distinctions like dropdown vs tabs, drawer vs visible button, exact section titles, exact button text
|
|
187
|
-
- includes hidden or non-obvious navigation details when relevant
|
|
188
|
-
|
|
189
|
-
Return only the briefing text.`
|
|
190
|
-
}]
|
|
191
|
-
},
|
|
192
|
-
{
|
|
193
|
-
role: "user",
|
|
194
|
-
content: [{
|
|
195
|
-
type: "input_text",
|
|
196
|
-
text: `APP CONTEXT DOCUMENT:
|
|
197
|
-
${contextDocument}
|
|
198
|
-
|
|
199
|
-
TASK:
|
|
200
|
-
${taskDescription}`
|
|
201
|
-
}]
|
|
202
|
-
}
|
|
203
|
-
];
|
|
204
|
-
}
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import { classifyCaptureSource } from "../core/capture-sources.js";
|
|
2
|
+
function isHiddenExecutionOutput(item) {
|
|
3
|
+
const captureSource = item?.payload?.captureSource;
|
|
4
|
+
return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
|
|
5
|
+
}
|
|
1
6
|
export function formatCliOutput(item) {
|
|
2
7
|
if (item == null) {
|
|
3
8
|
return null;
|
|
@@ -8,6 +13,9 @@ export function formatCliOutput(item) {
|
|
|
8
13
|
if (typeof item !== "object") {
|
|
9
14
|
return String(item);
|
|
10
15
|
}
|
|
16
|
+
if (isHiddenExecutionOutput(item)) {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
11
19
|
const text = typeof item.text === "string" ? item.text : null;
|
|
12
20
|
const type = typeof item.type === "string" ? item.type : "";
|
|
13
21
|
const eventType = typeof item.eventType === "string" ? item.eventType : "";
|
|
@@ -20,7 +20,10 @@ function compactTimestamp(iso) {
|
|
|
20
20
|
}
|
|
21
21
|
function sanitizeValue(value, pathParts = [], eventName = "") {
|
|
22
22
|
if (typeof value === "string") {
|
|
23
|
-
if (eventName === "cua.request.full" ||
|
|
23
|
+
if (eventName === "cua.request.full" ||
|
|
24
|
+
eventName === "cua.response.full" ||
|
|
25
|
+
eventName === "app_context.briefing.full" ||
|
|
26
|
+
eventName === "app_context.loaded") {
|
|
24
27
|
return value;
|
|
25
28
|
}
|
|
26
29
|
if (isPromptContentPath(pathParts)) {
|