@loadmill/droid-cua 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/build/index.js +86 -63
- package/build/src/browser/playwright/connection.js +13 -2
- package/build/src/cli/headless-execution-config.js +4 -36
- package/build/src/cli/ink-shell.js +5 -2
- package/build/src/commands/help.js +2 -3
- package/build/src/commands/loadmill.js +10 -1
- package/build/src/commands/run.js +17 -28
- package/build/src/core/app-context.js +57 -33
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/cua-models.js +9 -0
- package/build/src/core/execution-engine.js +2 -8
- package/build/src/core/runtime-config.js +96 -0
- package/build/src/device/loadmill.js +15 -9
- package/build/src/device/openai.js +67 -139
- package/build/src/integrations/loadmill/client.js +149 -131
- package/build/src/integrations/loadmill/executor.js +17 -11
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/interpreter.js +4 -14
- package/build/src/modes/design-mode-ink.js +4 -3
- package/build/src/modes/design-mode.js +4 -3
- package/build/src/modes/execution-mode.js +26 -0
- package/build/src/prompts/execution.js +8 -106
- package/build/src/utils/console-output.js +8 -0
- package/build/src/utils/cua-debug-tracer.js +0 -1
- package/build/src/utils/structured-debug-log-manager.js +4 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -40,6 +40,7 @@ Under the hood, `droid-cua` uses an AI agent to explore your app, execute action
|
|
|
40
40
|
|
|
41
41
|
- Mac: [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA.dmg)
|
|
42
42
|
- Mac (Intel): [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA-intel.dmg)
|
|
43
|
+
- Windows: [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA-Setup.exe)
|
|
43
44
|
|
|
44
45
|
**2. Launch the app**
|
|
45
46
|
|
|
@@ -67,6 +68,10 @@ You can also keep project run history in a results folder and review past runs f
|
|
|
67
68
|
- **Android** - Physical devices and emulators
|
|
68
69
|
- **iOS** - Simulators on macOS
|
|
69
70
|
|
|
71
|
+
Desktop app support:
|
|
72
|
+
- macOS: Android and iOS simulator workflows
|
|
73
|
+
- Windows: Android workflows only
|
|
74
|
+
|
|
70
75
|
## Requirements
|
|
71
76
|
|
|
72
77
|
**All platforms:**
|
|
@@ -154,10 +159,9 @@ Example headless config:
|
|
|
154
159
|
"executionModeInstructions": ""
|
|
155
160
|
},
|
|
156
161
|
"appContextEnabled": true,
|
|
157
|
-
"appContextBudget": 300,
|
|
158
162
|
"appContextPath": "../tests/context.md",
|
|
159
163
|
"contextOptimizationEnabled": true,
|
|
160
|
-
"contextOptimizationThreshold":
|
|
164
|
+
"contextOptimizationThreshold": 20000
|
|
161
165
|
}
|
|
162
166
|
```
|
|
163
167
|
|
|
@@ -177,7 +181,6 @@ Supported CLI options include:
|
|
|
177
181
|
- `--config`
|
|
178
182
|
- `--cua-model`
|
|
179
183
|
- `--context`
|
|
180
|
-
- `--app-context-budget`
|
|
181
184
|
- `--no-context`
|
|
182
185
|
- `--context-optimization-threshold`
|
|
183
186
|
- `--no-context-optimization`
|
|
@@ -195,7 +198,7 @@ Config and precedence rules:
|
|
|
195
198
|
- CLI flags override config file values.
|
|
196
199
|
- `--context` overrides the config app-context path.
|
|
197
200
|
- `--no-context` disables app context entirely.
|
|
198
|
-
- Context Optimization is enabled by default with a `
|
|
201
|
+
- Context Optimization is enabled by default with a `20000` input-token threshold.
|
|
199
202
|
- `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
|
|
200
203
|
- `--no-context-optimization` disables Context Optimization.
|
|
201
204
|
- `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
|
package/build/index.js
CHANGED
|
@@ -11,16 +11,18 @@ import { ExecutionEngine } from "./src/core/execution-engine.js";
|
|
|
11
11
|
import { buildBaseSystemPrompt, buildExecutionModePrompt } from "./src/core/prompts.js";
|
|
12
12
|
import { startInkShell } from "./src/cli/ink-shell.js";
|
|
13
13
|
import { ExecutionMode } from "./src/modes/execution-mode.js";
|
|
14
|
+
import { createOpenAiExecutor } from "./src/device/openai.js";
|
|
14
15
|
import { logger } from "./src/utils/logger.js";
|
|
15
16
|
import { selectDevice } from "./src/cli/device-selector.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
17
|
+
import { resolveExecutionAppContext } from "./src/core/app-context.js";
|
|
18
|
+
import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
|
|
18
19
|
import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
|
|
19
20
|
import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
|
|
20
21
|
import { printCliOutput } from "./src/utils/console-output.js";
|
|
21
22
|
import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
|
|
22
23
|
import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
|
|
23
24
|
import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
|
|
25
|
+
import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
|
|
24
26
|
dotenv.config();
|
|
25
27
|
const args = minimist(process.argv.slice(2));
|
|
26
28
|
const targetConfig = resolveHeadlessTarget(args);
|
|
@@ -38,6 +40,28 @@ if (strictFlag && noStrictFlag) {
|
|
|
38
40
|
const screenshotDir = path.join("droid-cua-recording-" + Date.now());
|
|
39
41
|
if (recordScreenshots)
|
|
40
42
|
await mkdir(screenshotDir, { recursive: true });
|
|
43
|
+
function createCliRuntimeContext(configOverrides = {}) {
|
|
44
|
+
// CLI env/config is bootstrap input only. After this boundary the shared
|
|
45
|
+
// runtime uses explicit run-scoped config and credentials objects.
|
|
46
|
+
const context = createRunContext({
|
|
47
|
+
config: resolveRuntimeConfigFromEnv(process.env, configOverrides),
|
|
48
|
+
credentials: resolveRuntimeCredentialsFromEnv(process.env),
|
|
49
|
+
});
|
|
50
|
+
const openAi = createOpenAiExecutor({
|
|
51
|
+
client: context.clients.openai,
|
|
52
|
+
config: context.config,
|
|
53
|
+
});
|
|
54
|
+
const loadmillClient = createLoadmillApiClient({
|
|
55
|
+
token: context.credentials.loadmillApiToken,
|
|
56
|
+
baseUrl: context.credentials.loadmillBaseUrl,
|
|
57
|
+
});
|
|
58
|
+
return {
|
|
59
|
+
...context,
|
|
60
|
+
openAi,
|
|
61
|
+
openAiClient: context.clients.openai,
|
|
62
|
+
loadmillClient,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
41
65
|
async function main() {
|
|
42
66
|
const isHeadlessInstructionsRun = Boolean(instructionsFile);
|
|
43
67
|
const isBrowserTarget = targetConfig.target === "web";
|
|
@@ -102,71 +126,55 @@ async function main() {
|
|
|
102
126
|
}
|
|
103
127
|
console.log(`\nRunning test from: ${instructionsFile}\n`);
|
|
104
128
|
instructions = await loadHeadlessInstructions(instructionsFile);
|
|
105
|
-
const taskText = instructions.join("\n");
|
|
106
129
|
const headlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
130
|
+
const appContext = await resolveExecutionAppContext({
|
|
131
|
+
enabled: headlessConfig.appContextEnabled,
|
|
132
|
+
contextPath: headlessConfig.appContextPath,
|
|
133
|
+
});
|
|
134
|
+
const appContextText = appContext.text;
|
|
135
|
+
const runtimeContext = createCliRuntimeContext({
|
|
136
|
+
cuaModel: headlessConfig.cuaModel,
|
|
137
|
+
strictMode: headlessConfig.strictMode,
|
|
138
|
+
promptCustomizations: headlessConfig.promptCustomizations,
|
|
139
|
+
contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
|
|
140
|
+
contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
|
|
141
|
+
appContext: {
|
|
142
|
+
enabled: headlessConfig.appContextEnabled,
|
|
143
|
+
path: appContext.contextPath,
|
|
144
|
+
status: appContext.status,
|
|
145
|
+
text: appContextText,
|
|
146
|
+
message: appContext.message,
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
|
|
150
|
+
if (appContext.status === "loaded" || appContext.status === "empty") {
|
|
151
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
114
152
|
source: "cli_context_flag",
|
|
115
|
-
contextPath:
|
|
116
|
-
|
|
117
|
-
|
|
153
|
+
contextPath: appContext.contextPath,
|
|
154
|
+
status: appContext.status,
|
|
155
|
+
text: appContextText
|
|
118
156
|
});
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
try {
|
|
122
|
-
const result = await buildAppContextBriefing({
|
|
123
|
-
contextPath: headlessConfig.appContextPath,
|
|
124
|
-
taskText,
|
|
125
|
-
budget: headlessConfig.appContextBudget,
|
|
126
|
-
});
|
|
127
|
-
appContextBriefing = result.briefing;
|
|
128
|
-
emitDesktopDebug("app_context.status", "execution", { runId }, {
|
|
129
|
-
source: "cli_context_flag",
|
|
130
|
-
contextPath: result.contextPath,
|
|
131
|
-
budget: headlessConfig.appContextBudget,
|
|
132
|
-
outputTokens: result.outputTokens,
|
|
133
|
-
status: appContextBriefing.trim().length > 0 ? "loaded" : "empty"
|
|
134
|
-
});
|
|
135
|
-
if (appContextBriefing) {
|
|
136
|
-
emitDesktopDebug("app_context.briefing.full", "execution", { runId }, {
|
|
137
|
-
source: "cli_context_flag",
|
|
138
|
-
contextPath: result.contextPath,
|
|
139
|
-
budget: headlessConfig.appContextBudget,
|
|
140
|
-
outputTokens: result.outputTokens,
|
|
141
|
-
briefing: appContextBriefing
|
|
142
|
-
});
|
|
143
|
-
console.log(`Using app context briefing from: ${result.contextPath}`);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
catch (error) {
|
|
147
|
-
const message = error instanceof Error ? error.message : "Unknown app context error";
|
|
148
|
-
emitDesktopDebug("app_context.status", "execution", { runId }, {
|
|
149
|
-
source: "cli_context_flag",
|
|
150
|
-
contextPath: headlessConfig.appContextPath,
|
|
151
|
-
budget: headlessConfig.appContextBudget,
|
|
152
|
-
status: "failed",
|
|
153
|
-
message
|
|
154
|
-
});
|
|
155
|
-
console.warn(`Warning: could not load app context from ${headlessConfig.appContextPath}. Running without briefing.`);
|
|
157
|
+
if (appContextText) {
|
|
158
|
+
console.log(`Using app context from: ${appContext.contextPath}`);
|
|
156
159
|
}
|
|
157
160
|
}
|
|
161
|
+
else if (appContext.status === "failed") {
|
|
162
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
163
|
+
source: "cli_context_flag",
|
|
164
|
+
contextPath: appContext.contextPath,
|
|
165
|
+
status: "failed",
|
|
166
|
+
message: appContext.message
|
|
167
|
+
});
|
|
168
|
+
console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
|
|
169
|
+
}
|
|
158
170
|
else {
|
|
159
|
-
emitDesktopDebug("app_context.
|
|
171
|
+
emitDesktopDebug("app_context.loaded", "execution", { runId }, {
|
|
160
172
|
source: "cli_context_flag",
|
|
161
|
-
contextPath:
|
|
162
|
-
|
|
163
|
-
status: "missing"
|
|
173
|
+
contextPath: appContext.contextPath,
|
|
174
|
+
status: appContext.status
|
|
164
175
|
});
|
|
165
|
-
if (isBrowserTarget && deviceInfo.currentUrl) {
|
|
166
|
-
appContextBriefing = `Starting URL: ${deviceInfo.currentUrl}`;
|
|
167
|
-
}
|
|
168
176
|
}
|
|
169
|
-
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations,
|
|
177
|
+
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { strictMode: headlessConfig.strictMode });
|
|
170
178
|
session.setSystemPrompt(executionPrompt);
|
|
171
179
|
const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
|
|
172
180
|
runId,
|
|
@@ -177,12 +185,18 @@ async function main() {
|
|
|
177
185
|
? {
|
|
178
186
|
recordScreenshots: true,
|
|
179
187
|
screenshotRecorder,
|
|
180
|
-
strictMode:
|
|
188
|
+
strictMode: runtimeContext.config.strictMode,
|
|
189
|
+
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
190
|
+
sendCUARequest: runtimeContext.openAi.sendCUARequest,
|
|
191
|
+
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
181
192
|
}
|
|
182
193
|
: {
|
|
183
194
|
recordScreenshots,
|
|
184
195
|
screenshotDir,
|
|
185
|
-
strictMode:
|
|
196
|
+
strictMode: runtimeContext.config.strictMode,
|
|
197
|
+
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
198
|
+
sendCUARequest: runtimeContext.openAi.sendCUARequest,
|
|
199
|
+
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
186
200
|
};
|
|
187
201
|
if (isBrowserTarget) {
|
|
188
202
|
engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
|
|
@@ -193,7 +207,10 @@ async function main() {
|
|
|
193
207
|
executionMode = new ExecutionMode(session, engine, instructions, true);
|
|
194
208
|
const result = await executionMode.execute({
|
|
195
209
|
runId,
|
|
196
|
-
addOutput: printCliOutput
|
|
210
|
+
addOutput: printCliOutput,
|
|
211
|
+
openAiClient: runtimeContext.openAiClient,
|
|
212
|
+
loadmillClient: runtimeContext.loadmillClient,
|
|
213
|
+
runtimeCredentials: runtimeContext.credentials,
|
|
197
214
|
});
|
|
198
215
|
const stats = executionMode.stats || {};
|
|
199
216
|
const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
|
|
@@ -238,15 +255,21 @@ async function main() {
|
|
|
238
255
|
throw error;
|
|
239
256
|
}
|
|
240
257
|
}
|
|
258
|
+
const runtimeContext = createCliRuntimeContext({
|
|
259
|
+
strictMode: strictFlag,
|
|
260
|
+
});
|
|
241
261
|
const engine = new ExecutionEngine(session, {
|
|
242
262
|
recordScreenshots,
|
|
243
263
|
screenshotDir,
|
|
244
|
-
strictMode:
|
|
264
|
+
strictMode: runtimeContext.config.strictMode,
|
|
265
|
+
stepDelayMs: runtimeContext.config.stepDelayMs,
|
|
266
|
+
sendCUARequest: runtimeContext.openAi.sendCUARequest,
|
|
267
|
+
reviseTestScript: runtimeContext.openAi.reviseTestScript,
|
|
245
268
|
});
|
|
246
269
|
// Otherwise, start interactive Ink shell
|
|
247
270
|
await startInkShell(session, engine, {
|
|
248
271
|
appContextPath: appContextPath ? path.resolve(appContextPath) : null,
|
|
249
|
-
|
|
272
|
+
runtimeContext,
|
|
250
273
|
});
|
|
251
274
|
}
|
|
252
275
|
main().catch((error) => {
|
|
@@ -8,6 +8,17 @@ import { clearActiveBrowserSession, getActiveBrowserSession, setActiveBrowserSes
|
|
|
8
8
|
import { normalizeAutomationUrl, normalizeOptionalAutomationUrl } from "./url.js";
|
|
9
9
|
const DEFAULT_VIEWPORT = { width: 1440, height: 900 };
|
|
10
10
|
let browserConnectionTestHooks = null;
|
|
11
|
+
async function gotoWithTimeoutTolerance(page, url) {
|
|
12
|
+
try {
|
|
13
|
+
await page.goto(url, { waitUntil: "load" });
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (!(error instanceof Error) || error.name !== "TimeoutError") {
|
|
17
|
+
throw error;
|
|
18
|
+
}
|
|
19
|
+
console.warn(`Page load did not finish within the timeout for ${url}. Continuing with the current page state.`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
11
22
|
function normalizeKnownPageUrl(url) {
|
|
12
23
|
if (typeof url !== "string" || url.trim().length === 0 || url.trim() === "about:blank") {
|
|
13
24
|
return null;
|
|
@@ -133,7 +144,7 @@ export async function connectToBrowser(targetOrOptions) {
|
|
|
133
144
|
});
|
|
134
145
|
});
|
|
135
146
|
if (normalizedUrl) {
|
|
136
|
-
await primaryPage
|
|
147
|
+
await gotoWithTimeoutTolerance(primaryPage, normalizedUrl);
|
|
137
148
|
}
|
|
138
149
|
setActiveBrowserSession({
|
|
139
150
|
sessionId,
|
|
@@ -169,7 +180,7 @@ export async function getScreenshotAsBase64(browserSessionId) {
|
|
|
169
180
|
export async function navigateToUrl(browserSessionId, url) {
|
|
170
181
|
const session = getActiveBrowserSession(browserSessionId);
|
|
171
182
|
const normalizedUrl = normalizeAutomationUrl(url);
|
|
172
|
-
await session.page
|
|
183
|
+
await gotoWithTimeoutTolerance(session.page, normalizedUrl);
|
|
173
184
|
session.url = normalizedUrl;
|
|
174
185
|
emitDesktopDebug("browser.navigate", "device", { browserSessionId }, {
|
|
175
186
|
platform: "browser",
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
|
-
import { DEFAULT_APP_CONTEXT_BUDGET, MAX_APP_CONTEXT_BUDGET, MIN_APP_CONTEXT_BUDGET, } from "../core/app-context.js";
|
|
4
3
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
|
|
5
|
-
|
|
6
|
-
const VALID_CUA_MODEL_LIST = ["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"];
|
|
4
|
+
import { CUA_MODELS, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
|
|
7
5
|
function createEmptyPromptCustomizations() {
|
|
8
6
|
return {
|
|
9
7
|
basePromptInstructions: "",
|
|
@@ -20,17 +18,13 @@ function assertNonEmptyString(value, label) {
|
|
|
20
18
|
}
|
|
21
19
|
}
|
|
22
20
|
function validateCuaModel(value, label) {
|
|
23
|
-
if (typeof value !== "string" || !
|
|
24
|
-
throw new Error(`${label} must be one of: ${
|
|
21
|
+
if (typeof value !== "string" || !isValidCuaModel(value)) {
|
|
22
|
+
throw new Error(`${label} must be one of: ${CUA_MODELS.join(", ")}.`);
|
|
25
23
|
}
|
|
26
24
|
return value;
|
|
27
25
|
}
|
|
28
26
|
function readEnvDefaultCuaModel() {
|
|
29
|
-
|
|
30
|
-
if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
|
|
31
|
-
return "gpt-5.4";
|
|
32
|
-
}
|
|
33
|
-
return VALID_CUA_MODELS.has(rawValue) ? rawValue : "gpt-5.4";
|
|
27
|
+
return resolveCuaModel(process.env.OPENAI_CUA_MODEL);
|
|
34
28
|
}
|
|
35
29
|
function parseBooleanFlagState(args, key) {
|
|
36
30
|
return {
|
|
@@ -38,24 +32,6 @@ function parseBooleanFlagState(args, key) {
|
|
|
38
32
|
disabled: args[`no-${key}`] === true || args[key] === false,
|
|
39
33
|
};
|
|
40
34
|
}
|
|
41
|
-
function parseBudgetValue(rawValue, label) {
|
|
42
|
-
const numericValue = (() => {
|
|
43
|
-
if (typeof rawValue === "number") {
|
|
44
|
-
return rawValue;
|
|
45
|
-
}
|
|
46
|
-
if (typeof rawValue === "string" && /^-?\d+$/.test(rawValue.trim())) {
|
|
47
|
-
return Number.parseInt(rawValue, 10);
|
|
48
|
-
}
|
|
49
|
-
return Number.NaN;
|
|
50
|
-
})();
|
|
51
|
-
if (!Number.isInteger(numericValue)) {
|
|
52
|
-
throw new Error(`${label} must be an integer between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
|
|
53
|
-
}
|
|
54
|
-
if (numericValue < MIN_APP_CONTEXT_BUDGET || numericValue > MAX_APP_CONTEXT_BUDGET) {
|
|
55
|
-
throw new Error(`${label} must be between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
|
|
56
|
-
}
|
|
57
|
-
return numericValue;
|
|
58
|
-
}
|
|
59
35
|
function parseContextOptimizationThreshold(rawValue, label) {
|
|
60
36
|
const numericValue = (() => {
|
|
61
37
|
if (typeof rawValue === "number") {
|
|
@@ -133,7 +109,6 @@ async function loadConfigFromFile(configPath) {
|
|
|
133
109
|
cuaModel: undefined,
|
|
134
110
|
promptCustomizations: createEmptyPromptCustomizations(),
|
|
135
111
|
appContextEnabled: undefined,
|
|
136
|
-
appContextBudget: undefined,
|
|
137
112
|
appContextPath: undefined,
|
|
138
113
|
contextOptimizationEnabled: undefined,
|
|
139
114
|
contextOptimizationThreshold: undefined,
|
|
@@ -151,9 +126,6 @@ async function loadConfigFromFile(configPath) {
|
|
|
151
126
|
}
|
|
152
127
|
normalized.appContextEnabled = rawConfig.appContextEnabled;
|
|
153
128
|
}
|
|
154
|
-
if ("appContextBudget" in rawConfig) {
|
|
155
|
-
normalized.appContextBudget = parseBudgetValue(rawConfig.appContextBudget, "config.appContextBudget");
|
|
156
|
-
}
|
|
157
129
|
if ("appContextPath" in rawConfig && rawConfig.appContextPath != null) {
|
|
158
130
|
assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
|
|
159
131
|
normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
|
|
@@ -210,7 +182,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
210
182
|
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
211
183
|
promptCustomizations,
|
|
212
184
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
213
|
-
appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
|
|
214
185
|
appContextPath: fileConfig?.appContextPath || null,
|
|
215
186
|
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
216
187
|
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
@@ -219,9 +190,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
219
190
|
if (typeof args["cua-model"] === "string") {
|
|
220
191
|
resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
|
|
221
192
|
}
|
|
222
|
-
if (args["app-context-budget"] != null) {
|
|
223
|
-
resolved.appContextBudget = parseBudgetValue(args["app-context-budget"], "--app-context-budget");
|
|
224
|
-
}
|
|
225
193
|
if (hasContextOptimizationThresholdFlag) {
|
|
226
194
|
resolved.contextOptimizationEnabled = true;
|
|
227
195
|
resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
|
|
@@ -7,7 +7,7 @@ import { routeCommand } from '../commands/index.js';
|
|
|
7
7
|
* Start the Ink-based conversational shell
|
|
8
8
|
* @param {Object} session - Session object with device info
|
|
9
9
|
* @param {Object} executionEngine - Execution engine instance
|
|
10
|
-
* @param {{ appContextPath?: string | null,
|
|
10
|
+
* @param {{ appContextPath?: string | null, runtimeContext?: Object | null }} [options]
|
|
11
11
|
* @returns {Promise<void>}
|
|
12
12
|
*/
|
|
13
13
|
export async function startInkShell(session, executionEngine, options = {}) {
|
|
@@ -34,7 +34,10 @@ export async function startInkShell(session, executionEngine, options = {}) {
|
|
|
34
34
|
...context,
|
|
35
35
|
engine: executionEngine,
|
|
36
36
|
appContextPath: options.appContextPath ?? null,
|
|
37
|
-
|
|
37
|
+
runtimeContext: options.runtimeContext ?? null,
|
|
38
|
+
openAiClient: options.runtimeContext?.openAiClient ?? null,
|
|
39
|
+
loadmillClient: options.runtimeContext?.loadmillClient ?? null,
|
|
40
|
+
runtimeCredentials: options.runtimeContext?.credentials ?? null
|
|
38
41
|
});
|
|
39
42
|
if (!shouldContinue) {
|
|
40
43
|
shouldExit = true;
|
|
@@ -24,9 +24,8 @@ export async function handleHelp(args, session, context) {
|
|
|
24
24
|
addOutput({ type: 'info', text: ' --url <value> Required starting URL for --target web headless runs' });
|
|
25
25
|
addOutput({ type: 'info', text: ' --session-mode <mode> Browser session mode for web runs: persistent or fresh' });
|
|
26
26
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
27
|
-
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4
|
|
28
|
-
addOutput({ type: 'info', text: ' --context <file> Optional app context file
|
|
29
|
-
addOutput({ type: 'info', text: ' --app-context-budget Headless app context token budget override' });
|
|
27
|
+
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4 or gpt-5.4-mini' });
|
|
28
|
+
addOutput({ type: 'info', text: ' --context <file> Optional app context file injected into execution runs' });
|
|
30
29
|
addOutput({ type: 'info', text: ' --no-context Disable app context for headless execution' });
|
|
31
30
|
addOutput({ type: 'info', text: ' --context-optimization-threshold <tokens> Headless Context Optimization compaction threshold' });
|
|
32
31
|
addOutput({ type: 'info', text: ' --no-context-optimization Disable Context Optimization for headless execution' });
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Loadmill command handler
|
|
3
3
|
*/
|
|
4
|
-
import { executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
|
|
4
|
+
import { createLoadmillApiClient, executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
|
|
5
|
+
import { createOpenAiClient, resolveRuntimeCredentialsFromEnv } from "../core/runtime-config.js";
|
|
5
6
|
/**
|
|
6
7
|
* Handle /loadmill command
|
|
7
8
|
* @param {string} args - Command arguments (natural language command)
|
|
@@ -47,8 +48,16 @@ export async function handleLoadmill(args, session, context) {
|
|
|
47
48
|
if (context.setAgentWorking) {
|
|
48
49
|
context.setAgentWorking(true, 'Running Loadmill flow...');
|
|
49
50
|
}
|
|
51
|
+
const runtimeCredentials = context.runtimeCredentials || resolveRuntimeCredentialsFromEnv();
|
|
52
|
+
const loadmillClient = context.loadmillClient || createLoadmillApiClient({
|
|
53
|
+
token: runtimeCredentials.loadmillApiToken,
|
|
54
|
+
baseUrl: runtimeCredentials.loadmillBaseUrl,
|
|
55
|
+
});
|
|
56
|
+
const openAiClient = context.openAiClient || createOpenAiClient(runtimeCredentials.openAiApiKey);
|
|
50
57
|
// Execute the command
|
|
51
58
|
const result = await executeLoadmillCommand(command, {
|
|
59
|
+
loadmillClient,
|
|
60
|
+
openAiClient,
|
|
52
61
|
onProgress: ({ message }) => {
|
|
53
62
|
addOutput({ type: 'info', text: `[Loadmill] ${message}` });
|
|
54
63
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { loadTest, listTests, testExists } from "../test-store/test-manager.js";
|
|
5
5
|
import { ExecutionMode } from "../modes/execution-mode.js";
|
|
6
6
|
import { buildExecutionModePrompt } from "../core/prompts.js";
|
|
7
|
-
import {
|
|
7
|
+
import { resolveExecutionAppContext } from "../core/app-context.js";
|
|
8
8
|
import { logger } from "../utils/logger.js";
|
|
9
9
|
/**
|
|
10
10
|
* Handle /run command
|
|
@@ -51,7 +51,6 @@ export async function handleRun(args, session, context) {
|
|
|
51
51
|
// Load test instructions
|
|
52
52
|
addOutput({ type: 'system', text: `Loading test: ${testName}` });
|
|
53
53
|
const instructions = await loadTest(testName);
|
|
54
|
-
const taskText = instructions.join("\n");
|
|
55
54
|
addOutput({ type: 'info', text: `Loaded ${instructions.length} instructions` });
|
|
56
55
|
addOutput({ type: 'info', text: '' });
|
|
57
56
|
// Disable free-form input during execution (only allow commands like /exit)
|
|
@@ -72,34 +71,24 @@ export async function handleRun(args, session, context) {
|
|
|
72
71
|
// Each test instruction should execute in isolation
|
|
73
72
|
session.updateResponseId(undefined);
|
|
74
73
|
session.clearMessages();
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
logger.error('CLI app context compaction failed', {
|
|
91
|
-
contextPath: context.appContextPath,
|
|
92
|
-
testName,
|
|
93
|
-
message,
|
|
94
|
-
});
|
|
95
|
-
addOutput({
|
|
96
|
-
type: 'warning',
|
|
97
|
-
text: `Warning: could not load app context from ${context.appContextPath}. Running without briefing.`,
|
|
98
|
-
});
|
|
99
|
-
}
|
|
74
|
+
const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
|
|
75
|
+
const appContextText = appContext.text;
|
|
76
|
+
if (appContext.status === 'loaded' && appContext.contextPath) {
|
|
77
|
+
addOutput({ type: 'info', text: `Loaded app context from: ${appContext.contextPath}` });
|
|
78
|
+
}
|
|
79
|
+
else if (appContext.status === 'failed') {
|
|
80
|
+
logger.error('CLI app context load failed', {
|
|
81
|
+
contextPath: appContext.contextPath,
|
|
82
|
+
testName,
|
|
83
|
+
message: appContext.message,
|
|
84
|
+
});
|
|
85
|
+
addOutput({
|
|
86
|
+
type: 'warning',
|
|
87
|
+
text: `Warning: could not load app context from ${appContext.contextPath}. Running without it.`,
|
|
88
|
+
});
|
|
100
89
|
}
|
|
101
90
|
// Set execution mode system prompt (replaces any design mode prompt)
|
|
102
|
-
const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {},
|
|
91
|
+
const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, { strictMode: Boolean(context.engine?.strictMode) });
|
|
103
92
|
session.setSystemPrompt(executionPrompt);
|
|
104
93
|
// Create execution mode
|
|
105
94
|
const executionMode = new ExecutionMode(session, context.engine, instructions);
|
|
@@ -1,24 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { access, readFile } from "fs/promises";
|
|
3
3
|
import { constants as fsConstants } from "fs";
|
|
4
|
-
import { compactAppContext } from "../device/openai.js";
|
|
5
4
|
export const APP_CONTEXT_FILENAME = "context.md";
|
|
6
|
-
export const DEFAULT_APP_CONTEXT_BUDGET = 300;
|
|
7
|
-
export const MIN_APP_CONTEXT_BUDGET = 100;
|
|
8
|
-
export const MAX_APP_CONTEXT_BUDGET = 2000;
|
|
9
|
-
export function normalizeAppContextBudget(value) {
|
|
10
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
11
|
-
return DEFAULT_APP_CONTEXT_BUDGET;
|
|
12
|
-
}
|
|
13
|
-
const normalized = Math.round(value);
|
|
14
|
-
if (normalized < MIN_APP_CONTEXT_BUDGET) {
|
|
15
|
-
return MIN_APP_CONTEXT_BUDGET;
|
|
16
|
-
}
|
|
17
|
-
if (normalized > MAX_APP_CONTEXT_BUDGET) {
|
|
18
|
-
return MAX_APP_CONTEXT_BUDGET;
|
|
19
|
-
}
|
|
20
|
-
return normalized;
|
|
21
|
-
}
|
|
22
5
|
export function getDefaultProjectContextPath(projectPath) {
|
|
23
6
|
return path.join(projectPath, APP_CONTEXT_FILENAME);
|
|
24
7
|
}
|
|
@@ -35,23 +18,64 @@ export async function appContextFileExists(filePath) {
|
|
|
35
18
|
return false;
|
|
36
19
|
}
|
|
37
20
|
}
|
|
38
|
-
|
|
21
|
+
/**
|
|
22
|
+
* @param {{ contextPath?: string | null }} [options]
|
|
23
|
+
*/
|
|
24
|
+
export async function loadAppContext({ contextPath }) {
|
|
25
|
+
if (!contextPath) {
|
|
26
|
+
return { text: "", contextPath: null };
|
|
27
|
+
}
|
|
28
|
+
const text = await readAppContextFile(contextPath);
|
|
29
|
+
return { text, contextPath };
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* @param {{
|
|
33
|
+
* enabled?: boolean,
|
|
34
|
+
* contextPath?: string | null,
|
|
35
|
+
* missingStatus?: string,
|
|
36
|
+
* missingFileStatus?: string | null,
|
|
37
|
+
* }} [options]
|
|
38
|
+
*/
|
|
39
|
+
export async function resolveExecutionAppContext({ enabled = true, contextPath = null, missingStatus = "missing", missingFileStatus = null, } = {}) {
|
|
40
|
+
if (!enabled) {
|
|
41
|
+
return {
|
|
42
|
+
status: "disabled",
|
|
43
|
+
text: "",
|
|
44
|
+
contextPath: null,
|
|
45
|
+
message: null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
39
48
|
if (!contextPath) {
|
|
40
|
-
return {
|
|
49
|
+
return {
|
|
50
|
+
status: missingStatus,
|
|
51
|
+
text: "",
|
|
52
|
+
contextPath: null,
|
|
53
|
+
message: null,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
const result = await loadAppContext({ contextPath });
|
|
58
|
+
return {
|
|
59
|
+
status: result.text.trim().length > 0 ? "loaded" : "empty",
|
|
60
|
+
text: result.text,
|
|
61
|
+
contextPath: result.contextPath,
|
|
62
|
+
message: null,
|
|
63
|
+
};
|
|
41
64
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
65
|
+
catch (error) {
|
|
66
|
+
if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
|
|
67
|
+
return {
|
|
68
|
+
status: missingFileStatus,
|
|
69
|
+
text: "",
|
|
70
|
+
contextPath,
|
|
71
|
+
message: null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
status: "failed",
|
|
76
|
+
text: "",
|
|
77
|
+
contextPath,
|
|
78
|
+
message: error instanceof Error ? error.message : "Unknown app context error",
|
|
79
|
+
};
|
|
45
80
|
}
|
|
46
|
-
const rawContext = await readAppContextFile(contextPath);
|
|
47
|
-
const result = await compactAppContext({
|
|
48
|
-
contextDocument: rawContext,
|
|
49
|
-
taskDescription: taskText,
|
|
50
|
-
tokenBudget: normalizedBudget,
|
|
51
|
-
});
|
|
52
|
-
return {
|
|
53
|
-
briefing: result.briefing,
|
|
54
|
-
outputTokens: result.outputTokens,
|
|
55
|
-
contextPath,
|
|
56
|
-
};
|
|
57
81
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
|
|
2
|
-
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD =
|
|
2
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 20000;
|
|
3
3
|
export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
|
|
4
4
|
export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
|
|
5
5
|
export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export const CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
|
|
2
|
+
export const DEFAULT_CUA_MODEL = "gpt-5.4";
|
|
3
|
+
const VALID_CUA_MODELS = new Set(CUA_MODELS);
|
|
4
|
+
export function isValidCuaModel(value) {
|
|
5
|
+
return VALID_CUA_MODELS.has(value);
|
|
6
|
+
}
|
|
7
|
+
export function resolveCuaModel(value) {
|
|
8
|
+
return isValidCuaModel(value) ? value : DEFAULT_CUA_MODEL;
|
|
9
|
+
}
|