@loadmill/droid-cua 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/build/index.js +89 -71
- package/build/src/brand/loadmill-mark.js +20 -0
- package/build/src/browser/playwright/connection.js +160 -12
- package/build/src/browser/playwright/welcome-page.js +161 -0
- package/build/src/cli/headless-execution-config.js +18 -58
- package/build/src/cli/headless-target.js +0 -9
- package/build/src/cli/ink-shell.js +5 -2
- package/build/src/commands/help.js +5 -8
- package/build/src/commands/loadmill.js +10 -1
- package/build/src/commands/run.js +17 -28
- package/build/src/core/aim-assist.js +5 -0
- package/build/src/core/app-context.js +57 -33
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/cua-models.js +9 -0
- package/build/src/core/execution-engine.js +121 -199
- package/build/src/core/runtime-config.js +95 -0
- package/build/src/device/loadmill.js +15 -9
- package/build/src/device/openai.js +76 -142
- package/build/src/integrations/loadmill/client.js +149 -131
- package/build/src/integrations/loadmill/executor.js +17 -11
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/interpreter.js +4 -14
- package/build/src/modes/design-mode-ink.js +5 -6
- package/build/src/modes/design-mode.js +5 -6
- package/build/src/modes/execution-mode.js +26 -0
- package/build/src/prompts/base.js +44 -30
- package/build/src/prompts/design.js +3 -4
- package/build/src/prompts/execution.js +13 -110
- package/build/src/utils/console-output.js +8 -0
- package/build/src/utils/cua-debug-tracer.js +0 -1
- package/build/src/utils/structured-debug-log-manager.js +4 -1
- package/package.json +1 -1
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
|
+
import { pathToFileURL } from "node:url";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import { LOADMILL_MARK_PATHS } from "../../brand/loadmill-mark.js";
|
|
6
|
+
const WELCOME_PAGE_DIR = path.join(os.homedir(), ".droid-cua", "browser-assets");
|
|
7
|
+
const WELCOME_PAGE_PATH = path.join(WELCOME_PAGE_DIR, "loadmill-droid-cua-welcome.html");
|
|
8
|
+
const WELCOME_PAGE_URL = pathToFileURL(WELCOME_PAGE_PATH).toString();
|
|
9
|
+
let ensureWelcomePagePromise = null;
|
|
10
|
+
let welcomePageTestHooks = null;
|
|
11
|
+
function renderLogoSvg() {
|
|
12
|
+
const paths = LOADMILL_MARK_PATHS.map(({ d, fill }) => `<path d="${d}" fill="${fill}" />`).join("");
|
|
13
|
+
return `<svg viewBox="0 0 1035 1035" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">${paths}</svg>`;
|
|
14
|
+
}
|
|
15
|
+
function buildWelcomePageHtml() {
|
|
16
|
+
return `<!doctype html>
|
|
17
|
+
<html lang="en">
|
|
18
|
+
<head>
|
|
19
|
+
<meta charset="utf-8" />
|
|
20
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
21
|
+
<title>Loadmill Droid CUA</title>
|
|
22
|
+
<style>
|
|
23
|
+
* { box-sizing: border-box; }
|
|
24
|
+
html, body { height: 100%; margin: 0; }
|
|
25
|
+
|
|
26
|
+
body {
|
|
27
|
+
font-family: "SF Pro Text", "Inter Tight", "Segoe UI", sans-serif;
|
|
28
|
+
background: linear-gradient(135deg, #f3f7fd 0%, #d4e2f3 55%, #b8cce8 100%);
|
|
29
|
+
color: #475569;
|
|
30
|
+
position: relative;
|
|
31
|
+
overflow-x: hidden;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
body::before,
|
|
35
|
+
body::after {
|
|
36
|
+
content: "";
|
|
37
|
+
position: absolute;
|
|
38
|
+
width: 168px;
|
|
39
|
+
height: 132px;
|
|
40
|
+
background-image: radial-gradient(rgba(91, 142, 226, 0.36) 1.4px, transparent 1.6px);
|
|
41
|
+
background-size: 18px 18px;
|
|
42
|
+
pointer-events: none;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
body::before { bottom: 7vh; left: 5vw; }
|
|
46
|
+
body::after { top: 7vh; right: 5vw; }
|
|
47
|
+
|
|
48
|
+
main {
|
|
49
|
+
position: relative;
|
|
50
|
+
min-height: 100vh;
|
|
51
|
+
display: flex;
|
|
52
|
+
align-items: flex-start;
|
|
53
|
+
justify-content: center;
|
|
54
|
+
padding: clamp(10vh, 13vh, 18vh) 32px 32px;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
.content { width: min(100%, 720px); text-align: center; }
|
|
58
|
+
|
|
59
|
+
.brand { margin: 0 auto 32px; }
|
|
60
|
+
.logo { width: 72px; height: 72px; margin: 0 auto 14px; }
|
|
61
|
+
.wordmark {
|
|
62
|
+
font-size: 17px;
|
|
63
|
+
letter-spacing: 0.32em;
|
|
64
|
+
font-weight: 600;
|
|
65
|
+
color: #0f172a;
|
|
66
|
+
text-indent: 0.32em;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
h1 {
|
|
70
|
+
margin: 0;
|
|
71
|
+
font-size: clamp(40px, 5.2vw, 56px);
|
|
72
|
+
line-height: 1.05;
|
|
73
|
+
letter-spacing: -0.035em;
|
|
74
|
+
color: #0f172a;
|
|
75
|
+
font-weight: 700;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
p { margin: 0; }
|
|
79
|
+
|
|
80
|
+
.subhead { margin-top: 22px; font-size: 18px; line-height: 1.65; }
|
|
81
|
+
|
|
82
|
+
.divider {
|
|
83
|
+
width: 48px;
|
|
84
|
+
height: 2px;
|
|
85
|
+
margin: 28px auto;
|
|
86
|
+
background: #c7d3e5;
|
|
87
|
+
border-radius: 999px;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
.pointer { font-size: 16px; line-height: 1.65; color: #64748b; }
|
|
91
|
+
|
|
92
|
+
.tip {
|
|
93
|
+
margin-top: 36px;
|
|
94
|
+
font-size: 13px;
|
|
95
|
+
line-height: 1.5;
|
|
96
|
+
color: #7c8aa0;
|
|
97
|
+
display: inline-flex;
|
|
98
|
+
align-items: center;
|
|
99
|
+
gap: 8px;
|
|
100
|
+
}
|
|
101
|
+
.tip svg { width: 14px; height: 14px; flex-shrink: 0; opacity: 0.8; }
|
|
102
|
+
|
|
103
|
+
@media (max-width: 720px) {
|
|
104
|
+
main { padding: 10vh 24px 24px; }
|
|
105
|
+
body::before, body::after { width: 120px; height: 96px; }
|
|
106
|
+
body::before { bottom: 4vh; left: 4vw; }
|
|
107
|
+
body::after { top: 4vh; right: 4vw; }
|
|
108
|
+
.logo { width: 60px; height: 60px; }
|
|
109
|
+
.wordmark { font-size: 15px; }
|
|
110
|
+
.subhead { font-size: 16px; }
|
|
111
|
+
.pointer { font-size: 15px; }
|
|
112
|
+
}
|
|
113
|
+
</style>
|
|
114
|
+
</head>
|
|
115
|
+
<body>
|
|
116
|
+
<main>
|
|
117
|
+
<div class="content">
|
|
118
|
+
<div class="brand">
|
|
119
|
+
<div class="logo">${renderLogoSvg()}</div>
|
|
120
|
+
<div class="wordmark">LOADMILL</div>
|
|
121
|
+
</div>
|
|
122
|
+
<h1>Test Browser Ready</h1>
|
|
123
|
+
<p class="subhead">This is the browser the Loadmill Droid CUA agent will drive when you start a test.</p>
|
|
124
|
+
<div class="divider" role="presentation"></div>
|
|
125
|
+
<p class="pointer">Switch back to Loadmill Droid CUA to design or run a test — you should keep this window in the background.</p>
|
|
126
|
+
<p class="tip">
|
|
127
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
|
128
|
+
<circle cx="12" cy="12" r="10"></circle>
|
|
129
|
+
<line x1="12" y1="16" x2="12" y2="12"></line>
|
|
130
|
+
<line x1="12" y1="8" x2="12.01" y2="8"></line>
|
|
131
|
+
</svg>
|
|
132
|
+
Want to explore manually first? Use the address bar above.
|
|
133
|
+
</p>
|
|
134
|
+
</div>
|
|
135
|
+
</main>
|
|
136
|
+
</body>
|
|
137
|
+
</html>`;
|
|
138
|
+
}
|
|
139
|
+
export async function getWelcomePageUrl() {
|
|
140
|
+
if (typeof welcomePageTestHooks?.getWelcomePageUrl === "function") {
|
|
141
|
+
return await welcomePageTestHooks.getWelcomePageUrl();
|
|
142
|
+
}
|
|
143
|
+
if (!ensureWelcomePagePromise) {
|
|
144
|
+
ensureWelcomePagePromise = (async () => {
|
|
145
|
+
await mkdir(WELCOME_PAGE_DIR, { recursive: true });
|
|
146
|
+
await writeFile(WELCOME_PAGE_PATH, buildWelcomePageHtml(), "utf8");
|
|
147
|
+
return WELCOME_PAGE_URL;
|
|
148
|
+
})();
|
|
149
|
+
}
|
|
150
|
+
return await ensureWelcomePagePromise;
|
|
151
|
+
}
|
|
152
|
+
export function isWelcomePageUrl(url) {
|
|
153
|
+
if (typeof welcomePageTestHooks?.isWelcomePageUrl === "function") {
|
|
154
|
+
return welcomePageTestHooks.isWelcomePageUrl(url);
|
|
155
|
+
}
|
|
156
|
+
return typeof url === "string" && url === WELCOME_PAGE_URL;
|
|
157
|
+
}
|
|
158
|
+
export function __setWelcomePageTestHooks(hooks = null) {
|
|
159
|
+
welcomePageTestHooks = hooks;
|
|
160
|
+
ensureWelcomePagePromise = null;
|
|
161
|
+
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
|
-
import { DEFAULT_APP_CONTEXT_BUDGET, MAX_APP_CONTEXT_BUDGET, MIN_APP_CONTEXT_BUDGET, } from "../core/app-context.js";
|
|
4
3
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
import { AIM_ASSIST_MODES, DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
5
|
+
import { CUA_MODELS, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
|
|
7
6
|
function createEmptyPromptCustomizations() {
|
|
8
7
|
return {
|
|
9
8
|
basePromptInstructions: "",
|
|
@@ -20,41 +19,19 @@ function assertNonEmptyString(value, label) {
|
|
|
20
19
|
}
|
|
21
20
|
}
|
|
22
21
|
function validateCuaModel(value, label) {
|
|
23
|
-
if (typeof value !== "string" || !
|
|
24
|
-
throw new Error(`${label} must be one of: ${
|
|
22
|
+
if (typeof value !== "string" || !isValidCuaModel(value)) {
|
|
23
|
+
throw new Error(`${label} must be one of: ${CUA_MODELS.join(", ")}.`);
|
|
25
24
|
}
|
|
26
25
|
return value;
|
|
27
26
|
}
|
|
28
|
-
function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return "gpt-5.4";
|
|
27
|
+
function validateAimAssistMode(value, label) {
|
|
28
|
+
if (typeof value !== "string" || !AIM_ASSIST_MODES.includes(value)) {
|
|
29
|
+
throw new Error(`${label} must be one of: ${AIM_ASSIST_MODES.join(", ")}.`);
|
|
32
30
|
}
|
|
33
|
-
return
|
|
34
|
-
}
|
|
35
|
-
function parseBooleanFlagState(args, key) {
|
|
36
|
-
return {
|
|
37
|
-
enabled: args[key] === true,
|
|
38
|
-
disabled: args[`no-${key}`] === true || args[key] === false,
|
|
39
|
-
};
|
|
31
|
+
return value;
|
|
40
32
|
}
|
|
41
|
-
function
|
|
42
|
-
|
|
43
|
-
if (typeof rawValue === "number") {
|
|
44
|
-
return rawValue;
|
|
45
|
-
}
|
|
46
|
-
if (typeof rawValue === "string" && /^-?\d+$/.test(rawValue.trim())) {
|
|
47
|
-
return Number.parseInt(rawValue, 10);
|
|
48
|
-
}
|
|
49
|
-
return Number.NaN;
|
|
50
|
-
})();
|
|
51
|
-
if (!Number.isInteger(numericValue)) {
|
|
52
|
-
throw new Error(`${label} must be an integer between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
|
|
53
|
-
}
|
|
54
|
-
if (numericValue < MIN_APP_CONTEXT_BUDGET || numericValue > MAX_APP_CONTEXT_BUDGET) {
|
|
55
|
-
throw new Error(`${label} must be between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
|
|
56
|
-
}
|
|
57
|
-
return numericValue;
|
|
33
|
+
function readEnvDefaultCuaModel() {
|
|
34
|
+
return resolveCuaModel(process.env.OPENAI_CUA_MODEL);
|
|
58
35
|
}
|
|
59
36
|
function parseContextOptimizationThreshold(rawValue, label) {
|
|
60
37
|
const numericValue = (() => {
|
|
@@ -131,17 +108,19 @@ async function loadConfigFromFile(configPath) {
|
|
|
131
108
|
const normalized = {
|
|
132
109
|
configPath: absoluteConfigPath,
|
|
133
110
|
cuaModel: undefined,
|
|
111
|
+
aimAssistMode: undefined,
|
|
134
112
|
promptCustomizations: createEmptyPromptCustomizations(),
|
|
135
113
|
appContextEnabled: undefined,
|
|
136
|
-
appContextBudget: undefined,
|
|
137
114
|
appContextPath: undefined,
|
|
138
115
|
contextOptimizationEnabled: undefined,
|
|
139
116
|
contextOptimizationThreshold: undefined,
|
|
140
|
-
strictMode: undefined,
|
|
141
117
|
};
|
|
142
118
|
if ("cuaModel" in rawConfig) {
|
|
143
119
|
normalized.cuaModel = validateCuaModel(rawConfig.cuaModel, "config.cuaModel");
|
|
144
120
|
}
|
|
121
|
+
if ("aimAssistMode" in rawConfig) {
|
|
122
|
+
normalized.aimAssistMode = validateAimAssistMode(rawConfig.aimAssistMode, "config.aimAssistMode");
|
|
123
|
+
}
|
|
145
124
|
if ("promptCustomizations" in rawConfig) {
|
|
146
125
|
normalized.promptCustomizations = normalizePromptCustomizations(rawConfig.promptCustomizations, "config.promptCustomizations");
|
|
147
126
|
}
|
|
@@ -151,9 +130,6 @@ async function loadConfigFromFile(configPath) {
|
|
|
151
130
|
}
|
|
152
131
|
normalized.appContextEnabled = rawConfig.appContextEnabled;
|
|
153
132
|
}
|
|
154
|
-
if ("appContextBudget" in rawConfig) {
|
|
155
|
-
normalized.appContextBudget = parseBudgetValue(rawConfig.appContextBudget, "config.appContextBudget");
|
|
156
|
-
}
|
|
157
133
|
if ("appContextPath" in rawConfig && rawConfig.appContextPath != null) {
|
|
158
134
|
assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
|
|
159
135
|
normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
|
|
@@ -167,12 +143,7 @@ async function loadConfigFromFile(configPath) {
|
|
|
167
143
|
if ("contextOptimizationThreshold" in rawConfig) {
|
|
168
144
|
normalized.contextOptimizationThreshold = parseContextOptimizationThreshold(rawConfig.contextOptimizationThreshold, "config.contextOptimizationThreshold");
|
|
169
145
|
}
|
|
170
|
-
|
|
171
|
-
if (typeof rawConfig.strictMode !== "boolean") {
|
|
172
|
-
throw new Error("config.strictMode must be a boolean.");
|
|
173
|
-
}
|
|
174
|
-
normalized.strictMode = rawConfig.strictMode;
|
|
175
|
-
}
|
|
146
|
+
// Legacy strictMode is silently accepted and ignored.
|
|
176
147
|
return normalized;
|
|
177
148
|
}
|
|
178
149
|
export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
@@ -183,7 +154,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
183
154
|
const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
|
|
184
155
|
const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
|
|
185
156
|
const hasContextOptimizationThresholdFlag = args["context-optimization-threshold"] != null;
|
|
186
|
-
const { enabled: strictEnabledByFlag, disabled: strictDisabledByFlag } = parseBooleanFlagState(args, "strict");
|
|
187
157
|
const basePromptText = typeof args["base-prompt"] === "string" ? args["base-prompt"] : null;
|
|
188
158
|
const executionPromptText = typeof args["execution-prompt"] === "string" ? args["execution-prompt"] : null;
|
|
189
159
|
const basePromptFilePath = typeof args["base-prompt-file"] === "string" ? path.resolve(cwd, args["base-prompt-file"]) : null;
|
|
@@ -191,9 +161,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
191
161
|
if (explicitContextPath && noContext) {
|
|
192
162
|
throw new Error("--context and --no-context cannot be used together.");
|
|
193
163
|
}
|
|
194
|
-
if (strictEnabledByFlag && strictDisabledByFlag) {
|
|
195
|
-
throw new Error("--strict and --no-strict cannot be used together.");
|
|
196
|
-
}
|
|
197
164
|
if (contextOptimizationEnabledByFlag) {
|
|
198
165
|
throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
|
|
199
166
|
}
|
|
@@ -208,19 +175,18 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
208
175
|
const resolved = {
|
|
209
176
|
configPath: fileConfig?.configPath || null,
|
|
210
177
|
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
178
|
+
aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
|
|
211
179
|
promptCustomizations,
|
|
212
180
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
213
|
-
appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
|
|
214
181
|
appContextPath: fileConfig?.appContextPath || null,
|
|
215
182
|
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
216
183
|
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
217
|
-
strictMode: fileConfig?.strictMode ?? false,
|
|
218
184
|
};
|
|
219
185
|
if (typeof args["cua-model"] === "string") {
|
|
220
186
|
resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
|
|
221
187
|
}
|
|
222
|
-
if (args["
|
|
223
|
-
resolved.
|
|
188
|
+
if (typeof args["aim-assist"] === "string") {
|
|
189
|
+
resolved.aimAssistMode = validateAimAssistMode(args["aim-assist"], "--aim-assist");
|
|
224
190
|
}
|
|
225
191
|
if (hasContextOptimizationThresholdFlag) {
|
|
226
192
|
resolved.contextOptimizationEnabled = true;
|
|
@@ -249,11 +215,5 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
249
215
|
if (noContextOptimization) {
|
|
250
216
|
resolved.contextOptimizationEnabled = false;
|
|
251
217
|
}
|
|
252
|
-
if (strictEnabledByFlag) {
|
|
253
|
-
resolved.strictMode = true;
|
|
254
|
-
}
|
|
255
|
-
else if (strictDisabledByFlag) {
|
|
256
|
-
resolved.strictMode = false;
|
|
257
|
-
}
|
|
258
218
|
return resolved;
|
|
259
219
|
}
|
|
@@ -30,33 +30,24 @@ function hasNonEmptyString(value) {
|
|
|
30
30
|
export function resolveHeadlessTarget(args = {}) {
|
|
31
31
|
const target = normalizeTarget(args.target);
|
|
32
32
|
const sessionMode = normalizeSessionMode(args["session-mode"]);
|
|
33
|
-
const url = hasNonEmptyString(args.url) ? args.url.trim() : null;
|
|
34
33
|
const avdName = hasNonEmptyString(args.avd) ? args.avd.trim() : null;
|
|
35
34
|
const platform = hasNonEmptyString(args.platform) ? args.platform.trim() : null;
|
|
36
35
|
if (target === "web") {
|
|
37
|
-
if (!url) {
|
|
38
|
-
throw new Error("--target web requires --url.");
|
|
39
|
-
}
|
|
40
36
|
if (avdName || platform) {
|
|
41
37
|
throw new Error("--avd and --platform cannot be used with --target web.");
|
|
42
38
|
}
|
|
43
39
|
return {
|
|
44
40
|
target,
|
|
45
|
-
url,
|
|
46
41
|
sessionMode,
|
|
47
42
|
avdName: null,
|
|
48
43
|
platform: null,
|
|
49
44
|
};
|
|
50
45
|
}
|
|
51
|
-
if (hasNonEmptyString(args.url)) {
|
|
52
|
-
throw new Error("--url can only be used with --target web.");
|
|
53
|
-
}
|
|
54
46
|
if (args["session-mode"] != null) {
|
|
55
47
|
throw new Error("--session-mode can only be used with --target web.");
|
|
56
48
|
}
|
|
57
49
|
return {
|
|
58
50
|
target,
|
|
59
|
-
url: null,
|
|
60
51
|
sessionMode: null,
|
|
61
52
|
avdName,
|
|
62
53
|
platform,
|
|
@@ -7,7 +7,7 @@ import { routeCommand } from '../commands/index.js';
|
|
|
7
7
|
* Start the Ink-based conversational shell
|
|
8
8
|
* @param {Object} session - Session object with device info
|
|
9
9
|
* @param {Object} executionEngine - Execution engine instance
|
|
10
|
-
* @param {{ appContextPath?: string | null,
|
|
10
|
+
* @param {{ appContextPath?: string | null, runtimeContext?: Object | null }} [options]
|
|
11
11
|
* @returns {Promise<void>}
|
|
12
12
|
*/
|
|
13
13
|
export async function startInkShell(session, executionEngine, options = {}) {
|
|
@@ -34,7 +34,10 @@ export async function startInkShell(session, executionEngine, options = {}) {
|
|
|
34
34
|
...context,
|
|
35
35
|
engine: executionEngine,
|
|
36
36
|
appContextPath: options.appContextPath ?? null,
|
|
37
|
-
|
|
37
|
+
runtimeContext: options.runtimeContext ?? null,
|
|
38
|
+
openAiClient: options.runtimeContext?.openAiClient ?? null,
|
|
39
|
+
loadmillClient: options.runtimeContext?.loadmillClient ?? null,
|
|
40
|
+
runtimeCredentials: options.runtimeContext?.credentials ?? null
|
|
38
41
|
});
|
|
39
42
|
if (!shouldContinue) {
|
|
40
43
|
shouldExit = true;
|
|
@@ -14,19 +14,18 @@ export async function handleHelp(args, session, context) {
|
|
|
14
14
|
addOutput({ type: 'info', text: '' });
|
|
15
15
|
addOutput({ type: 'info', text: 'Usage:' });
|
|
16
16
|
addOutput({ type: 'info', text: ' droid-cua --avd <device-name> [options]' });
|
|
17
|
-
addOutput({ type: 'info', text: ' droid-cua --target web --
|
|
17
|
+
addOutput({ type: 'info', text: ' droid-cua --target web --instructions <file> [options]' });
|
|
18
18
|
addOutput({ type: 'info', text: '' });
|
|
19
19
|
addOutput({ type: 'info', text: 'Options:' });
|
|
20
20
|
addOutput({ type: 'info', text: ' --target <kind> Target kind: mobile (default) or web' });
|
|
21
21
|
addOutput({ type: 'info', text: ' --avd <name> Device name (Android device ID/serial or iOS Simulator)' });
|
|
22
22
|
addOutput({ type: 'info', text: ' --platform <platform> Force platform: android or ios' });
|
|
23
23
|
addOutput({ type: 'info', text: ' --instructions <file> Run test file in headless mode' });
|
|
24
|
-
addOutput({ type: 'info', text: ' --url <value> Required starting URL for --target web headless runs' });
|
|
25
24
|
addOutput({ type: 'info', text: ' --session-mode <mode> Browser session mode for web runs: persistent or fresh' });
|
|
26
25
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
27
|
-
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4
|
|
28
|
-
addOutput({ type: 'info', text: ' --
|
|
29
|
-
addOutput({ type: 'info', text: ' --
|
|
26
|
+
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4 or gpt-5.4-mini' });
|
|
27
|
+
addOutput({ type: 'info', text: ' --aim-assist <mode> Headless Aim Assist mode: off, smart, or always' });
|
|
28
|
+
addOutput({ type: 'info', text: ' --context <file> Optional app context file injected into execution runs' });
|
|
30
29
|
addOutput({ type: 'info', text: ' --no-context Disable app context for headless execution' });
|
|
31
30
|
addOutput({ type: 'info', text: ' --context-optimization-threshold <tokens> Headless Context Optimization compaction threshold' });
|
|
32
31
|
addOutput({ type: 'info', text: ' --no-context-optimization Disable Context Optimization for headless execution' });
|
|
@@ -34,8 +33,6 @@ export async function handleHelp(args, session, context) {
|
|
|
34
33
|
addOutput({ type: 'info', text: ' --execution-prompt <text> Headless execution prompt customization text' });
|
|
35
34
|
addOutput({ type: 'info', text: ' --base-prompt-file Headless base prompt customization file' });
|
|
36
35
|
addOutput({ type: 'info', text: ' --execution-prompt-file Headless execution prompt customization file' });
|
|
37
|
-
addOutput({ type: 'info', text: ' --strict Strict Mode: re-observe after the first action in a chain' });
|
|
38
|
-
addOutput({ type: 'info', text: ' --no-strict Disable Strict Mode for headless config-driven runs' });
|
|
39
36
|
addOutput({ type: 'info', text: ' --record Record screenshots during execution' });
|
|
40
37
|
addOutput({ type: 'info', text: ' --debug Enable structured JSONL debug artifacts' });
|
|
41
38
|
addOutput({ type: 'info', text: '' });
|
|
@@ -76,7 +73,7 @@ export async function handleHelp(args, session, context) {
|
|
|
76
73
|
addOutput({ type: 'info', text: ' droid-cua --avd "iPhone 16" (iOS Simulator, auto-detected)' });
|
|
77
74
|
addOutput({ type: 'info', text: ' droid-cua --platform ios --avd MySim (Force iOS platform)' });
|
|
78
75
|
addOutput({ type: 'info', text: ' droid-cua --instructions tests/login.dcua --context app/context.md' });
|
|
79
|
-
addOutput({ type: 'info', text: ' droid-cua --target web --
|
|
76
|
+
addOutput({ type: 'info', text: ' droid-cua --target web --instructions tests/search.dcua' });
|
|
80
77
|
addOutput({ type: 'info', text: ' /create login-test (design a new test)' });
|
|
81
78
|
addOutput({ type: 'info', text: ' /list (see all tests)' });
|
|
82
79
|
addOutput({ type: 'info', text: ' /view login-test (view test contents)' });
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Loadmill command handler
|
|
3
3
|
*/
|
|
4
|
-
import { executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
|
|
4
|
+
import { createLoadmillApiClient, executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
|
|
5
|
+
import { createOpenAiClient, resolveRuntimeCredentialsFromEnv } from "../core/runtime-config.js";
|
|
5
6
|
/**
|
|
6
7
|
* Handle /loadmill command
|
|
7
8
|
* @param {string} args - Command arguments (natural language command)
|
|
@@ -47,8 +48,16 @@ export async function handleLoadmill(args, session, context) {
|
|
|
47
48
|
if (context.setAgentWorking) {
|
|
48
49
|
context.setAgentWorking(true, 'Running Loadmill flow...');
|
|
49
50
|
}
|
|
51
|
+
const runtimeCredentials = context.runtimeCredentials || resolveRuntimeCredentialsFromEnv();
|
|
52
|
+
const loadmillClient = context.loadmillClient || createLoadmillApiClient({
|
|
53
|
+
token: runtimeCredentials.loadmillApiToken,
|
|
54
|
+
baseUrl: runtimeCredentials.loadmillBaseUrl,
|
|
55
|
+
});
|
|
56
|
+
const openAiClient = context.openAiClient || createOpenAiClient(runtimeCredentials.openAiApiKey);
|
|
50
57
|
// Execute the command
|
|
51
58
|
const result = await executeLoadmillCommand(command, {
|
|
59
|
+
loadmillClient,
|
|
60
|
+
openAiClient,
|
|
52
61
|
onProgress: ({ message }) => {
|
|
53
62
|
addOutput({ type: 'info', text: `[Loadmill] ${message}` });
|
|
54
63
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { loadTest, listTests, testExists } from "../test-store/test-manager.js";
|
|
5
5
|
import { ExecutionMode } from "../modes/execution-mode.js";
|
|
6
6
|
import { buildExecutionModePrompt } from "../core/prompts.js";
|
|
7
|
-
import {
|
|
7
|
+
import { resolveExecutionAppContext } from "../core/app-context.js";
|
|
8
8
|
import { logger } from "../utils/logger.js";
|
|
9
9
|
/**
|
|
10
10
|
* Handle /run command
|
|
@@ -51,7 +51,6 @@ export async function handleRun(args, session, context) {
|
|
|
51
51
|
// Load test instructions
|
|
52
52
|
addOutput({ type: 'system', text: `Loading test: ${testName}` });
|
|
53
53
|
const instructions = await loadTest(testName);
|
|
54
|
-
const taskText = instructions.join("\n");
|
|
55
54
|
addOutput({ type: 'info', text: `Loaded ${instructions.length} instructions` });
|
|
56
55
|
addOutput({ type: 'info', text: '' });
|
|
57
56
|
// Disable free-form input during execution (only allow commands like /exit)
|
|
@@ -72,34 +71,24 @@ export async function handleRun(args, session, context) {
|
|
|
72
71
|
// Each test instruction should execute in isolation
|
|
73
72
|
session.updateResponseId(undefined);
|
|
74
73
|
session.clearMessages();
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
logger.error('CLI app context compaction failed', {
|
|
91
|
-
contextPath: context.appContextPath,
|
|
92
|
-
testName,
|
|
93
|
-
message,
|
|
94
|
-
});
|
|
95
|
-
addOutput({
|
|
96
|
-
type: 'warning',
|
|
97
|
-
text: `Warning: could not load app context from ${context.appContextPath}. Running without briefing.`,
|
|
98
|
-
});
|
|
99
|
-
}
|
|
74
|
+
const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
|
|
75
|
+
const appContextText = appContext.text;
|
|
76
|
+
if (appContext.status === 'loaded' && appContext.contextPath) {
|
|
77
|
+
addOutput({ type: 'info', text: `Loaded app context from: ${appContext.contextPath}` });
|
|
78
|
+
}
|
|
79
|
+
else if (appContext.status === 'failed') {
|
|
80
|
+
logger.error('CLI app context load failed', {
|
|
81
|
+
contextPath: appContext.contextPath,
|
|
82
|
+
testName,
|
|
83
|
+
message: appContext.message,
|
|
84
|
+
});
|
|
85
|
+
addOutput({
|
|
86
|
+
type: 'warning',
|
|
87
|
+
text: `Warning: could not load app context from ${appContext.contextPath}. Running without it.`,
|
|
88
|
+
});
|
|
100
89
|
}
|
|
101
90
|
// Set execution mode system prompt (replaces any design mode prompt)
|
|
102
|
-
const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {},
|
|
91
|
+
const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText);
|
|
103
92
|
session.setSystemPrompt(executionPrompt);
|
|
104
93
|
// Create execution mode
|
|
105
94
|
const executionMode = new ExecutionMode(session, context.engine, instructions);
|
|
@@ -1,24 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { access, readFile } from "fs/promises";
|
|
3
3
|
import { constants as fsConstants } from "fs";
|
|
4
|
-
import { compactAppContext } from "../device/openai.js";
|
|
5
4
|
export const APP_CONTEXT_FILENAME = "context.md";
|
|
6
|
-
export const DEFAULT_APP_CONTEXT_BUDGET = 300;
|
|
7
|
-
export const MIN_APP_CONTEXT_BUDGET = 100;
|
|
8
|
-
export const MAX_APP_CONTEXT_BUDGET = 2000;
|
|
9
|
-
export function normalizeAppContextBudget(value) {
|
|
10
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
11
|
-
return DEFAULT_APP_CONTEXT_BUDGET;
|
|
12
|
-
}
|
|
13
|
-
const normalized = Math.round(value);
|
|
14
|
-
if (normalized < MIN_APP_CONTEXT_BUDGET) {
|
|
15
|
-
return MIN_APP_CONTEXT_BUDGET;
|
|
16
|
-
}
|
|
17
|
-
if (normalized > MAX_APP_CONTEXT_BUDGET) {
|
|
18
|
-
return MAX_APP_CONTEXT_BUDGET;
|
|
19
|
-
}
|
|
20
|
-
return normalized;
|
|
21
|
-
}
|
|
22
5
|
export function getDefaultProjectContextPath(projectPath) {
|
|
23
6
|
return path.join(projectPath, APP_CONTEXT_FILENAME);
|
|
24
7
|
}
|
|
@@ -35,23 +18,64 @@ export async function appContextFileExists(filePath) {
|
|
|
35
18
|
return false;
|
|
36
19
|
}
|
|
37
20
|
}
|
|
38
|
-
|
|
21
|
+
/**
|
|
22
|
+
* @param {{ contextPath?: string | null }} [options]
|
|
23
|
+
*/
|
|
24
|
+
export async function loadAppContext({ contextPath }) {
|
|
25
|
+
if (!contextPath) {
|
|
26
|
+
return { text: "", contextPath: null };
|
|
27
|
+
}
|
|
28
|
+
const text = await readAppContextFile(contextPath);
|
|
29
|
+
return { text, contextPath };
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* @param {{
|
|
33
|
+
* enabled?: boolean,
|
|
34
|
+
* contextPath?: string | null,
|
|
35
|
+
* missingStatus?: string,
|
|
36
|
+
* missingFileStatus?: string | null,
|
|
37
|
+
* }} [options]
|
|
38
|
+
*/
|
|
39
|
+
export async function resolveExecutionAppContext({ enabled = true, contextPath = null, missingStatus = "missing", missingFileStatus = null, } = {}) {
|
|
40
|
+
if (!enabled) {
|
|
41
|
+
return {
|
|
42
|
+
status: "disabled",
|
|
43
|
+
text: "",
|
|
44
|
+
contextPath: null,
|
|
45
|
+
message: null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
39
48
|
if (!contextPath) {
|
|
40
|
-
return {
|
|
49
|
+
return {
|
|
50
|
+
status: missingStatus,
|
|
51
|
+
text: "",
|
|
52
|
+
contextPath: null,
|
|
53
|
+
message: null,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
const result = await loadAppContext({ contextPath });
|
|
58
|
+
return {
|
|
59
|
+
status: result.text.trim().length > 0 ? "loaded" : "empty",
|
|
60
|
+
text: result.text,
|
|
61
|
+
contextPath: result.contextPath,
|
|
62
|
+
message: null,
|
|
63
|
+
};
|
|
41
64
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
65
|
+
catch (error) {
|
|
66
|
+
if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
|
|
67
|
+
return {
|
|
68
|
+
status: missingFileStatus,
|
|
69
|
+
text: "",
|
|
70
|
+
contextPath,
|
|
71
|
+
message: null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
status: "failed",
|
|
76
|
+
text: "",
|
|
77
|
+
contextPath,
|
|
78
|
+
message: error instanceof Error ? error.message : "Unknown app context error",
|
|
79
|
+
};
|
|
45
80
|
}
|
|
46
|
-
const rawContext = await readAppContextFile(contextPath);
|
|
47
|
-
const result = await compactAppContext({
|
|
48
|
-
contextDocument: rawContext,
|
|
49
|
-
taskDescription: taskText,
|
|
50
|
-
tokenBudget: normalizedBudget,
|
|
51
|
-
});
|
|
52
|
-
return {
|
|
53
|
-
briefing: result.briefing,
|
|
54
|
-
outputTokens: result.outputTokens,
|
|
55
|
-
contextPath,
|
|
56
|
-
};
|
|
57
81
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
|
|
2
|
-
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD =
|
|
2
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 20000;
|
|
3
3
|
export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
|
|
4
4
|
export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
|
|
5
5
|
export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export const CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
|
|
2
|
+
export const DEFAULT_CUA_MODEL = "gpt-5.4";
|
|
3
|
+
const VALID_CUA_MODELS = new Set(CUA_MODELS);
|
|
4
|
+
export function isValidCuaModel(value) {
|
|
5
|
+
return VALID_CUA_MODELS.has(value);
|
|
6
|
+
}
|
|
7
|
+
export function resolveCuaModel(value) {
|
|
8
|
+
return isValidCuaModel(value) ? value : DEFAULT_CUA_MODEL;
|
|
9
|
+
}
|