@loadmill/droid-cua 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/build/index.js +89 -71
- package/build/src/brand/loadmill-mark.js +20 -0
- package/build/src/browser/playwright/connection.js +160 -12
- package/build/src/browser/playwright/welcome-page.js +161 -0
- package/build/src/cli/headless-execution-config.js +18 -58
- package/build/src/cli/headless-target.js +0 -9
- package/build/src/cli/ink-shell.js +5 -2
- package/build/src/commands/help.js +5 -8
- package/build/src/commands/loadmill.js +10 -1
- package/build/src/commands/run.js +17 -28
- package/build/src/core/aim-assist.js +5 -0
- package/build/src/core/app-context.js +57 -33
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/cua-models.js +9 -0
- package/build/src/core/execution-engine.js +121 -199
- package/build/src/core/runtime-config.js +95 -0
- package/build/src/device/loadmill.js +15 -9
- package/build/src/device/openai.js +76 -142
- package/build/src/integrations/loadmill/client.js +149 -131
- package/build/src/integrations/loadmill/executor.js +17 -11
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/interpreter.js +4 -14
- package/build/src/modes/design-mode-ink.js +5 -6
- package/build/src/modes/design-mode.js +5 -6
- package/build/src/modes/execution-mode.js +26 -0
- package/build/src/prompts/base.js +44 -30
- package/build/src/prompts/design.js +3 -4
- package/build/src/prompts/execution.js +13 -110
- package/build/src/utils/console-output.js +8 -0
- package/build/src/utils/cua-debug-tracer.js +0 -1
- package/build/src/utils/structured-debug-log-manager.js +4 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Loadmill integration public exports
|
|
3
3
|
*/
|
|
4
|
-
export { getApiToken, searchFlows, runTestFlow, getFlowRunStatus } from "./client.js";
|
|
4
|
+
export { createLoadmillApiClient, getApiToken, searchFlows, runTestFlow, getFlowRunStatus } from "./client.js";
|
|
5
5
|
export { interpretLoadmillCommand, selectBestFlow } from "./interpreter.js";
|
|
6
6
|
export { executeLoadmillCommand } from "./executor.js";
|
|
@@ -1,24 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* AI-powered text interpretation for Loadmill commands
|
|
3
3
|
*/
|
|
4
|
-
import OpenAI from "openai";
|
|
5
4
|
import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionMessages, } from "../../prompts/loadmill.js";
|
|
6
|
-
let openai = null;
|
|
7
|
-
function getOpenAI() {
|
|
8
|
-
if (!openai) {
|
|
9
|
-
openai = new OpenAI({
|
|
10
|
-
apiKey: process.env.OPENAI_API_KEY,
|
|
11
|
-
});
|
|
12
|
-
}
|
|
13
|
-
return openai;
|
|
14
|
-
}
|
|
15
5
|
/**
|
|
16
6
|
* Interpret a natural language Loadmill command into structured data
|
|
17
7
|
* @param {string} userInput - Natural language command
|
|
18
8
|
* @returns {Promise<{searchQuery: string, parameters: Object, action: 'run'|'search'}>}
|
|
19
9
|
*/
|
|
20
|
-
export async function interpretLoadmillCommand(userInput) {
|
|
21
|
-
const response = await
|
|
10
|
+
export async function interpretLoadmillCommand(userInput, { client }) {
|
|
11
|
+
const response = await client.chat.completions.create({
|
|
22
12
|
model: "gpt-4o-mini",
|
|
23
13
|
messages: buildLoadmillCommandInterpretationMessages(userInput),
|
|
24
14
|
response_format: { type: "json_object" }
|
|
@@ -37,7 +27,7 @@ export async function interpretLoadmillCommand(userInput) {
|
|
|
37
27
|
* @param {string} originalQuery - Original user query
|
|
38
28
|
* @returns {Promise<{selectedFlow: Object|null, confidence: number}>}
|
|
39
29
|
*/
|
|
40
|
-
export async function selectBestFlow(flows, originalQuery) {
|
|
30
|
+
export async function selectBestFlow(flows, originalQuery, { client }) {
|
|
41
31
|
// Ensure flows is an array
|
|
42
32
|
if (!flows || !Array.isArray(flows) || flows.length === 0) {
|
|
43
33
|
return { selectedFlow: null, confidence: 0 };
|
|
@@ -51,7 +41,7 @@ export async function selectBestFlow(flows, originalQuery) {
|
|
|
51
41
|
const suite = f.testSuiteDescription || "";
|
|
52
42
|
return `${i + 1}. ID: ${f.id}, Name: "${name}"${suite ? `, Suite: "${suite}"` : ""}`;
|
|
53
43
|
}).join("\n");
|
|
54
|
-
const response = await
|
|
44
|
+
const response = await client.chat.completions.create({
|
|
55
45
|
model: "gpt-4o-mini",
|
|
56
46
|
messages: buildLoadmillFlowSelectionMessages(originalQuery, flowList),
|
|
57
47
|
response_format: { type: "json_object" }
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { getScreenshotAsBase64 } from "../device/connection.js";
|
|
2
|
-
import { sendCUARequest, reviseTestScript } from "../device/openai.js";
|
|
3
2
|
import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
|
|
4
3
|
import { saveTest } from "../test-store/test-manager.js";
|
|
5
4
|
import { logger } from "../utils/logger.js";
|
|
@@ -23,6 +22,8 @@ export class DesignModeInk {
|
|
|
23
22
|
this.baseDesignPrompt = null;
|
|
24
23
|
this.consecutiveErrorCount = 0;
|
|
25
24
|
this.maxConsecutiveErrors = 3;
|
|
25
|
+
this.sendCUARequest = executionEngine.sendCUARequest;
|
|
26
|
+
this.reviseTestScript = executionEngine.reviseTestScript;
|
|
26
27
|
}
|
|
27
28
|
/**
|
|
28
29
|
* Start design mode conversation
|
|
@@ -31,9 +32,7 @@ export class DesignModeInk {
|
|
|
31
32
|
async start() {
|
|
32
33
|
const addOutput = this.context.addOutput;
|
|
33
34
|
// Set design mode system prompt
|
|
34
|
-
const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}
|
|
35
|
-
strictMode: Boolean(this.engine?.strictMode)
|
|
36
|
-
});
|
|
35
|
+
const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
|
|
37
36
|
this.baseDesignPrompt = designPrompt;
|
|
38
37
|
this.session.setSystemPrompt(designPrompt);
|
|
39
38
|
// Update UI
|
|
@@ -183,7 +182,7 @@ export class DesignModeInk {
|
|
|
183
182
|
}
|
|
184
183
|
// Get screenshot and send to model
|
|
185
184
|
const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
|
|
186
|
-
const response = await sendCUARequest({
|
|
185
|
+
const response = await this.sendCUARequest({
|
|
187
186
|
messages: this.session.messages,
|
|
188
187
|
screenshotBase64,
|
|
189
188
|
previousResponseId: this.session.previousResponseId,
|
|
@@ -271,7 +270,7 @@ export class DesignModeInk {
|
|
|
271
270
|
const revision = confirm;
|
|
272
271
|
// Use simple chat completion for revision instead of CUA
|
|
273
272
|
addOutput({ type: 'info', text: 'Revising test script...' });
|
|
274
|
-
currentScript = await reviseTestScript(currentScript, revision);
|
|
273
|
+
currentScript = await this.reviseTestScript(currentScript, revision);
|
|
275
274
|
// Display the revised script
|
|
276
275
|
addOutput({ type: 'info', text: '' });
|
|
277
276
|
addOutput({ type: 'system', text: '=== Generated Test Script ===' });
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import readline from "readline";
|
|
2
2
|
import { getScreenshotAsBase64 } from "../device/connection.js";
|
|
3
|
-
import { sendCUARequest, reviseTestScript } from "../device/openai.js";
|
|
4
3
|
import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
|
|
5
4
|
import { saveTest } from "../test-store/test-manager.js";
|
|
6
5
|
import { logger } from "../utils/logger.js";
|
|
@@ -20,6 +19,8 @@ export class DesignMode {
|
|
|
20
19
|
this.baseDesignPrompt = null;
|
|
21
20
|
this.consecutiveErrorCount = 0;
|
|
22
21
|
this.maxConsecutiveErrors = 3;
|
|
22
|
+
this.sendCUARequest = executionEngine.sendCUARequest;
|
|
23
|
+
this.reviseTestScript = executionEngine.reviseTestScript;
|
|
23
24
|
}
|
|
24
25
|
/**
|
|
25
26
|
* Start design mode conversation
|
|
@@ -28,9 +29,7 @@ export class DesignMode {
|
|
|
28
29
|
*/
|
|
29
30
|
async start(context) {
|
|
30
31
|
// Set design mode system prompt
|
|
31
|
-
const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}
|
|
32
|
-
strictMode: Boolean(this.engine?.strictMode)
|
|
33
|
-
});
|
|
32
|
+
const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
|
|
34
33
|
this.baseDesignPrompt = designPrompt;
|
|
35
34
|
this.session.setSystemPrompt(designPrompt);
|
|
36
35
|
console.log(`\n=== Design Mode: Creating test "${this.testName}" ===`);
|
|
@@ -147,7 +146,7 @@ export class DesignMode {
|
|
|
147
146
|
const keypressHandler = this.setupEscDetection();
|
|
148
147
|
// Get screenshot and send to model
|
|
149
148
|
const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
|
|
150
|
-
const response = await sendCUARequest({
|
|
149
|
+
const response = await this.sendCUARequest({
|
|
151
150
|
messages: this.session.messages,
|
|
152
151
|
screenshotBase64,
|
|
153
152
|
previousResponseId: this.session.previousResponseId,
|
|
@@ -262,7 +261,7 @@ export class DesignMode {
|
|
|
262
261
|
const revision = confirm;
|
|
263
262
|
// Use simple chat completion for revision instead of CUA
|
|
264
263
|
console.log("Revising test script...");
|
|
265
|
-
currentScript = await reviseTestScript(currentScript, revision);
|
|
264
|
+
currentScript = await this.reviseTestScript(currentScript, revision);
|
|
266
265
|
// Display the revised script
|
|
267
266
|
console.log("\n=== Generated Test Script ===");
|
|
268
267
|
console.log(currentScript);
|
|
@@ -228,6 +228,14 @@ export class ExecutionMode {
|
|
|
228
228
|
}
|
|
229
229
|
const navigationTarget = extractNavigateTarget(instruction);
|
|
230
230
|
this.session.addToTranscript(`[Navigate] ${navigationTarget}`);
|
|
231
|
+
emitDesktopDebug("runner.navigate.started", "execution", {
|
|
232
|
+
runId: context?.runId,
|
|
233
|
+
stepId: stepContext?.stepId,
|
|
234
|
+
instructionIndex: stepContext?.instructionIndex
|
|
235
|
+
}, {
|
|
236
|
+
instruction,
|
|
237
|
+
target: navigationTarget
|
|
238
|
+
});
|
|
231
239
|
this.emit(addOutput, "system", `Navigating to ${navigationTarget}...`, context, stepContext, {
|
|
232
240
|
eventType: "system_message",
|
|
233
241
|
payload: {
|
|
@@ -250,6 +258,15 @@ export class ExecutionMode {
|
|
|
250
258
|
navigatedUrl: this.session.currentUrl,
|
|
251
259
|
});
|
|
252
260
|
this.session.installRecoverySystemPrompt(navigationRecoveryPrompt);
|
|
261
|
+
emitDesktopDebug("runner.navigate.completed", "execution", {
|
|
262
|
+
runId: context?.runId,
|
|
263
|
+
stepId: stepContext?.stepId,
|
|
264
|
+
instructionIndex: stepContext?.instructionIndex
|
|
265
|
+
}, {
|
|
266
|
+
instruction,
|
|
267
|
+
target: navigationTarget,
|
|
268
|
+
url: normalizedUrl
|
|
269
|
+
});
|
|
253
270
|
this.emit(addOutput, "success", `Navigation complete: ${normalizedUrl}`, context, stepContext, {
|
|
254
271
|
eventType: "system_message",
|
|
255
272
|
payload: {
|
|
@@ -266,6 +283,15 @@ export class ExecutionMode {
|
|
|
266
283
|
retryCount,
|
|
267
284
|
message,
|
|
268
285
|
});
|
|
286
|
+
emitDesktopDebug("runner.navigate.failed", "execution", {
|
|
287
|
+
runId: context?.runId,
|
|
288
|
+
stepId: stepContext?.stepId,
|
|
289
|
+
instructionIndex: stepContext?.instructionIndex
|
|
290
|
+
}, {
|
|
291
|
+
instruction,
|
|
292
|
+
target: navigationTarget,
|
|
293
|
+
message
|
|
294
|
+
});
|
|
269
295
|
this.emit(addOutput, "error", `Navigation failed: ${message}`, context, stepContext, {
|
|
270
296
|
eventType: "error",
|
|
271
297
|
payload: {
|
|
@@ -27,16 +27,6 @@ function appendCustomSections(prompt, sections = []) {
|
|
|
27
27
|
${customSection}
|
|
28
28
|
`;
|
|
29
29
|
}
|
|
30
|
-
function buildStrictModeRuntimeSection(runtimeOptions = {}) {
|
|
31
|
-
if (runtimeOptions?.strictMode !== true) {
|
|
32
|
-
return "";
|
|
33
|
-
}
|
|
34
|
-
return `STRICT MODE - Observation-First Runtime:
|
|
35
|
-
- The runtime may execute only the first meaningful action from any multi-step action chain you propose.
|
|
36
|
-
- After that first action, it will re-observe the ${runtimeOptions?.platform === "browser" ? "browser" : "device"} and continue from a fresh screenshot.
|
|
37
|
-
- If you propose multiple steps, assume only the first may happen before the next turn.
|
|
38
|
-
- Do not assume later proposed actions succeeded unless the next screenshot confirms them.`;
|
|
39
|
-
}
|
|
40
30
|
function isBrowserTarget(deviceInfo = {}) {
|
|
41
31
|
return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
|
|
42
32
|
}
|
|
@@ -55,7 +45,42 @@ function describeControlledTarget(deviceInfo = {}) {
|
|
|
55
45
|
}
|
|
56
46
|
return "a mobile device";
|
|
57
47
|
}
|
|
58
|
-
function
|
|
48
|
+
function buildAlwaysAimAssistGuidance(targetKind) {
|
|
49
|
+
return [
|
|
50
|
+
"- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
|
|
51
|
+
"- Review the returned annotated screenshot before clicking.",
|
|
52
|
+
`- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable" : "tappable"} control.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
|
|
53
|
+
`- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
|
|
54
|
+
"- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
|
|
55
|
+
"- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
|
|
56
|
+
"- Click precisely based on your visual estimate.",
|
|
57
|
+
];
|
|
58
|
+
}
|
|
59
|
+
function buildAimAssistGuidance(targetKind, aimAssistMode = "always") {
|
|
60
|
+
if (aimAssistMode === "off") {
|
|
61
|
+
return [
|
|
62
|
+
"- Click directly based on your best visual estimate.",
|
|
63
|
+
"- If a click has no effect, treat it as a likely misclick and try a meaningfully different coordinate.",
|
|
64
|
+
"- Click precisely based on your visual estimate.",
|
|
65
|
+
].join("\n ");
|
|
66
|
+
}
|
|
67
|
+
if (aimAssistMode === "smart") {
|
|
68
|
+
const targetWord = targetKind === "browser" ? "clickable" : "tappable";
|
|
69
|
+
return [
|
|
70
|
+
...buildAlwaysAimAssistGuidance(targetKind),
|
|
71
|
+
"- In Smart mode, you may skip preview only when the target is broad, isolated, stable, and unambiguous, and you are confident the click point is clearly inside the intended target.",
|
|
72
|
+
"- Good skip examples are large empty text fields, large search fields, large CTA buttons with generous padding, launcher icons, and wide rows with generous spacing and no nearby competing actions.",
|
|
73
|
+
`- Do not skip preview when the target is small, dense, visually ambiguous, close to neighboring controls, or you are not confident the marker will land clearly inside the intended ${targetWord} target.`,
|
|
74
|
+
"- If a dropdown, expanded option list, popover, or select menu is visible, always use preview before selecting an option.",
|
|
75
|
+
"- Do not skip preview in search results, bottom sheets, bottom cards, modals, overlays, keyboard-driven flows, or after recent screen changes.",
|
|
76
|
+
"- Do not skip preview for known tricky UI such as dense lists, date pickers, time pickers, calendar cells, wheels, segmented selectors, and small inline controls inside larger rows or cards.",
|
|
77
|
+
"- If the target is one option among many visually similar options, use preview before clicking.",
|
|
78
|
+
].join("\n ");
|
|
79
|
+
}
|
|
80
|
+
return buildAlwaysAimAssistGuidance(targetKind).join("\n ");
|
|
81
|
+
}
|
|
82
|
+
function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
|
|
83
|
+
const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
|
|
59
84
|
return `
|
|
60
85
|
You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
|
|
61
86
|
Follow the user's instructions to interact with the browser.
|
|
@@ -67,13 +92,7 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
|
|
|
67
92
|
|
|
68
93
|
When aiming for visual targets:
|
|
69
94
|
- Reason carefully about the approximate pixel position.
|
|
70
|
-
|
|
71
|
-
- Review the returned annotated screenshot before clicking.
|
|
72
|
-
- Click if the marker is clearly inside the intended clickable target.
|
|
73
|
-
- Adjust and preview again only if the marker is outside the target, on the edge, or closer to a neighboring target than the intended one.
|
|
74
|
-
- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
|
|
75
|
-
- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
|
|
76
|
-
- Click precisely based on your visual estimate.
|
|
95
|
+
${buildAimAssistGuidance("browser", aimAssistMode)}
|
|
77
96
|
|
|
78
97
|
Available actions: click, scroll, type, keypress, wait, screenshot.
|
|
79
98
|
|
|
@@ -129,7 +148,8 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
|
|
|
129
148
|
- CTRL/CMD/ALT/OPTION/SHIFT may be used when they are the most reliable path
|
|
130
149
|
`;
|
|
131
150
|
}
|
|
132
|
-
function buildMobileBaseSystemPrompt(deviceInfo) {
|
|
151
|
+
function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
|
|
152
|
+
const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
|
|
133
153
|
return `
|
|
134
154
|
You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
|
|
135
155
|
Follow the user's instructions to interact with the device.
|
|
@@ -142,13 +162,7 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
|
|
|
142
162
|
|
|
143
163
|
When aiming for visual targets:
|
|
144
164
|
- Reason carefully about the approximate pixel position.
|
|
145
|
-
|
|
146
|
-
- Review the returned annotated screenshot before clicking.
|
|
147
|
-
- Click if the marker is clearly inside the intended tappable control. Text inside a button, field, row, or bottom bar is okay.
|
|
148
|
-
- Adjust and preview again only if the marker is outside the target, on the target border/edge, or closer to a neighboring target than the intended one.
|
|
149
|
-
- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
|
|
150
|
-
- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
|
|
151
|
-
- Click precisely based on your visual estimate.
|
|
165
|
+
${buildAimAssistGuidance("mobile", aimAssistMode)}
|
|
152
166
|
|
|
153
167
|
Available actions: click, scroll, type, keypress, wait, screenshot.
|
|
154
168
|
|
|
@@ -209,9 +223,9 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
|
|
|
209
223
|
- Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
|
|
210
224
|
`;
|
|
211
225
|
}
|
|
212
|
-
function buildBaseSystemPrompt(deviceInfo) {
|
|
226
|
+
function buildBaseSystemPrompt(deviceInfo, options = {}) {
|
|
213
227
|
return isBrowserTarget(deviceInfo)
|
|
214
|
-
? buildBrowserBaseSystemPrompt(deviceInfo)
|
|
215
|
-
: buildMobileBaseSystemPrompt(deviceInfo);
|
|
228
|
+
? buildBrowserBaseSystemPrompt(deviceInfo, options)
|
|
229
|
+
: buildMobileBaseSystemPrompt(deviceInfo, options);
|
|
216
230
|
}
|
|
217
|
-
export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget,
|
|
231
|
+
export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { appendCustomSections, buildBaseSystemPrompt,
|
|
2
|
-
export function buildDesignModePrompt(deviceInfo, customInstructions = {}
|
|
1
|
+
import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
|
|
2
|
+
export function buildDesignModePrompt(deviceInfo, customInstructions = {}) {
|
|
3
3
|
const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
|
|
4
4
|
const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
|
|
5
|
-
const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
|
|
6
5
|
const browserTarget = isBrowserTarget(deviceInfo);
|
|
7
6
|
const prompt = `${basePrompt}
|
|
8
7
|
|
|
@@ -87,7 +86,7 @@ WRONG Example (DON'T DO THIS):
|
|
|
87
86
|
\`\`\`
|
|
88
87
|
|
|
89
88
|
Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
|
|
90
|
-
|
|
89
|
+
`;
|
|
91
90
|
return appendCustomSections(prompt, [
|
|
92
91
|
{ title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
|
|
93
92
|
{ title: "Design Mode Instructions", text: designCustomText }
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
2
|
+
import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
|
|
3
|
+
export function buildAppContextSection(text) {
|
|
4
|
+
const trimmed = typeof text === "string" ? text.trim() : "";
|
|
5
|
+
if (!trimmed) {
|
|
5
6
|
return "";
|
|
6
7
|
}
|
|
7
|
-
return `APP CONTEXT
|
|
8
|
-
The following
|
|
8
|
+
return `APP CONTEXT:
|
|
9
|
+
The following describes the app you are testing.
|
|
9
10
|
Use this to understand screen layouts, terminology, navigation, and expected behavior.
|
|
10
11
|
|
|
11
|
-
${
|
|
12
|
+
${trimmed}`;
|
|
12
13
|
}
|
|
13
|
-
export function buildExecutionModePrompt(deviceInfo, customInstructions = {},
|
|
14
|
+
export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
|
|
14
15
|
const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
16
|
+
const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
|
|
17
|
+
const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
|
|
18
|
+
const appContextSection = buildAppContextSection(appContextText);
|
|
18
19
|
const browserTarget = isBrowserTarget(deviceInfo);
|
|
19
20
|
const prompt = `${basePrompt}
|
|
20
21
|
|
|
@@ -45,7 +46,7 @@ Your process:
|
|
|
45
46
|
6. Stop immediately - no commentary, no questions
|
|
46
47
|
|
|
47
48
|
Each instruction is independent. Do not reference previous instructions or ask about next steps.
|
|
48
|
-
${appContextSection ? `\n\n${appContextSection}` : ""}
|
|
49
|
+
${appContextSection ? `\n\n${appContextSection}` : ""}`;
|
|
49
50
|
return appendCustomSections(prompt, [
|
|
50
51
|
{ title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
|
|
51
52
|
{ title: "Execution Mode Instructions", text: executionCustomText }
|
|
@@ -104,101 +105,3 @@ Remember:
|
|
|
104
105
|
- You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
|
|
105
106
|
- Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
|
|
106
107
|
}
|
|
107
|
-
export function buildAppContextCompactionInput({ contextDocument, taskDescription, tokenBudget }) {
|
|
108
|
-
return [
|
|
109
|
-
{
|
|
110
|
-
role: "system",
|
|
111
|
-
content: [{
|
|
112
|
-
type: "input_text",
|
|
113
|
-
text: `You are compressing an app context document for an app testing agent.
|
|
114
|
-
|
|
115
|
-
You will receive:
|
|
116
|
-
1. A context document
|
|
117
|
-
2. A test task
|
|
118
|
-
|
|
119
|
-
Your job is to SELECT only the facts from the context document that are useful for the given task.
|
|
120
|
-
The output will be injected into a system prompt with a strict token budget.
|
|
121
|
-
|
|
122
|
-
CRITICAL:
|
|
123
|
-
- Use only facts explicitly supported by the context document
|
|
124
|
-
- Never invent, infer, normalize, substitute, or improve credentials, labels, screen names, button names, or numeric values
|
|
125
|
-
- Preserve exact values verbatim when present in the source
|
|
126
|
-
- Prefer facts that help the agent act correctly when they are not obvious from the task alone
|
|
127
|
-
- Do not restate, paraphrase, summarize, or reorganize the test task
|
|
128
|
-
- The output must not read like instructions or a test plan
|
|
129
|
-
- Do not describe what the agent should do
|
|
130
|
-
- Output only reference knowledge about the app
|
|
131
|
-
- If a line could be copied from the task with minor wording changes, omit it
|
|
132
|
-
- Prefer copying source facts verbatim or near-verbatim over rewriting them
|
|
133
|
-
- Do not collapse multiple specific source facts into one generic summary if that removes useful distinctions
|
|
134
|
-
|
|
135
|
-
Selection priority:
|
|
136
|
-
1. Facts the agent would NOT know from the test script alone
|
|
137
|
-
2. Facts that are hard to infer from screenshots
|
|
138
|
-
3. Non-obvious navigation or interaction details
|
|
139
|
-
4. Exact visible labels needed to act correctly
|
|
140
|
-
5. Credentials and other exact values
|
|
141
|
-
|
|
142
|
-
High-value facts:
|
|
143
|
-
- exact UI labels
|
|
144
|
-
- how state, mode, or account selection is performed
|
|
145
|
-
- where logout is located
|
|
146
|
-
- hidden or non-obvious navigation
|
|
147
|
-
- which menu items are decorative or non-functional
|
|
148
|
-
- screen titles and section labels used to confirm location
|
|
149
|
-
- exact credentials and role labels
|
|
150
|
-
|
|
151
|
-
Low-value facts:
|
|
152
|
-
- restating the test steps
|
|
153
|
-
- repeating literal values already present in the task
|
|
154
|
-
- generic summaries like "approve the transaction"
|
|
155
|
-
|
|
156
|
-
When the task involves authentication, switching state or mode, opening menus, or moving between major areas of the app, strongly prefer including:
|
|
157
|
-
- how account, state, or mode selection is performed
|
|
158
|
-
- exact visible labels for the relevant controls
|
|
159
|
-
- where exit or sign-out actions are located
|
|
160
|
-
- the screen or section labels that confirm the agent is in the right place
|
|
161
|
-
|
|
162
|
-
Rules:
|
|
163
|
-
- Output plain text only
|
|
164
|
-
- No markdown, no bullet symbols, no numbering, no headers
|
|
165
|
-
- Use terse, factual language: one fact per line, no filler words
|
|
166
|
-
- Blank lines only to separate logical groups
|
|
167
|
-
- Prefer exact visible UI labels over summaries
|
|
168
|
-
- Do not describe step-by-step procedures
|
|
169
|
-
- Do not restate the test workflow
|
|
170
|
-
- State only facts about screens, elements, hidden interactions, entities, credentials, and navigation
|
|
171
|
-
- If a useful fact is not explicitly stated in the context document, omit it
|
|
172
|
-
- Include only information relevant to this task
|
|
173
|
-
- Do not waste space repeating the task itself
|
|
174
|
-
- If the task already states a value or action, include it only when the context adds non-obvious execution details
|
|
175
|
-
- Return a short result or an empty string if little is relevant
|
|
176
|
-
- Target: under ${tokenBudget} tokens
|
|
177
|
-
|
|
178
|
-
Bad output patterns to avoid:
|
|
179
|
-
- generic summaries that remove actionable details
|
|
180
|
-
- lines that restate the task in generic prose
|
|
181
|
-
- lines that describe obvious workflow steps instead of app knowledge
|
|
182
|
-
- lines that replace exact source labels or mechanisms with broad summaries
|
|
183
|
-
|
|
184
|
-
Good output characteristics:
|
|
185
|
-
- preserves the exact label or mechanism from the source when it matters
|
|
186
|
-
- keeps distinctions like dropdown vs tabs, drawer vs visible button, exact section titles, exact button text
|
|
187
|
-
- includes hidden or non-obvious navigation details when relevant
|
|
188
|
-
|
|
189
|
-
Return only the briefing text.`
|
|
190
|
-
}]
|
|
191
|
-
},
|
|
192
|
-
{
|
|
193
|
-
role: "user",
|
|
194
|
-
content: [{
|
|
195
|
-
type: "input_text",
|
|
196
|
-
text: `APP CONTEXT DOCUMENT:
|
|
197
|
-
${contextDocument}
|
|
198
|
-
|
|
199
|
-
TASK:
|
|
200
|
-
${taskDescription}`
|
|
201
|
-
}]
|
|
202
|
-
}
|
|
203
|
-
];
|
|
204
|
-
}
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import { classifyCaptureSource } from "../core/capture-sources.js";
|
|
2
|
+
function isHiddenExecutionOutput(item) {
|
|
3
|
+
const captureSource = item?.payload?.captureSource;
|
|
4
|
+
return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
|
|
5
|
+
}
|
|
1
6
|
export function formatCliOutput(item) {
|
|
2
7
|
if (item == null) {
|
|
3
8
|
return null;
|
|
@@ -8,6 +13,9 @@ export function formatCliOutput(item) {
|
|
|
8
13
|
if (typeof item !== "object") {
|
|
9
14
|
return String(item);
|
|
10
15
|
}
|
|
16
|
+
if (isHiddenExecutionOutput(item)) {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
11
19
|
const text = typeof item.text === "string" ? item.text : null;
|
|
12
20
|
const type = typeof item.type === "string" ? item.type : "";
|
|
13
21
|
const eventType = typeof item.eventType === "string" ? item.eventType : "";
|
|
@@ -20,7 +20,10 @@ function compactTimestamp(iso) {
|
|
|
20
20
|
}
|
|
21
21
|
function sanitizeValue(value, pathParts = [], eventName = "") {
|
|
22
22
|
if (typeof value === "string") {
|
|
23
|
-
if (eventName === "cua.request.full" ||
|
|
23
|
+
if (eventName === "cua.request.full" ||
|
|
24
|
+
eventName === "cua.response.full" ||
|
|
25
|
+
eventName === "app_context.briefing.full" ||
|
|
26
|
+
eventName === "app_context.loaded") {
|
|
24
27
|
return value;
|
|
25
28
|
}
|
|
26
29
|
if (isPromptContentPath(pathParts)) {
|