@loadmill/droid-cua 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +11 -2
- package/build/index.js +7 -6
- package/build/src/cli/headless-execution-config.js +59 -3
- package/build/src/cli/headless-instructions.js +12 -0
- package/build/src/commands/help.js +3 -1
- package/build/src/core/context-optimization.js +36 -0
- package/build/src/core/execution-engine.js +383 -130
- package/build/src/device/android/actions.js +5 -1
- package/build/src/device/assertions.js +6 -4
- package/build/src/device/cloud/actions.js +5 -1
- package/build/src/device/ios/actions.js +5 -1
- package/build/src/device/ios/appium-server.js +4 -3
- package/build/src/device/ios/node-cli-env.js +28 -0
- package/build/src/device/openai.js +127 -48
- package/build/src/modes/execution-mode.js +36 -3
- package/build/src/prompts/base.js +6 -0
- package/build/src/utils/cua-debug-tracer.js +48 -5
- package/build/src/utils/run-screenshot-recorder.js +161 -0
- package/package.json +6 -1
package/LICENSE
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
©
|
|
1
|
+
© 2026 Loadmill. All rights reserved.
|
package/README.md
CHANGED
|
@@ -155,7 +155,9 @@ Example headless config:
|
|
|
155
155
|
},
|
|
156
156
|
"appContextEnabled": true,
|
|
157
157
|
"appContextBudget": 300,
|
|
158
|
-
"appContextPath": "../tests/context.md"
|
|
158
|
+
"appContextPath": "../tests/context.md",
|
|
159
|
+
"contextOptimizationEnabled": true,
|
|
160
|
+
"contextOptimizationThreshold": 30000
|
|
159
161
|
}
|
|
160
162
|
```
|
|
161
163
|
|
|
@@ -177,10 +179,14 @@ Supported CLI options include:
|
|
|
177
179
|
- `--context`
|
|
178
180
|
- `--app-context-budget`
|
|
179
181
|
- `--no-context`
|
|
182
|
+
- `--context-optimization-threshold`
|
|
183
|
+
- `--no-context-optimization`
|
|
180
184
|
- `--base-prompt`
|
|
181
185
|
- `--execution-prompt`
|
|
182
186
|
- `--base-prompt-file`
|
|
183
187
|
- `--execution-prompt-file`
|
|
188
|
+
- `--strict`
|
|
189
|
+
- `--no-strict`
|
|
184
190
|
- `--record`
|
|
185
191
|
- `--debug`
|
|
186
192
|
|
|
@@ -189,6 +195,9 @@ Config and precedence rules:
|
|
|
189
195
|
- CLI flags override config file values.
|
|
190
196
|
- `--context` overrides the config app-context path.
|
|
191
197
|
- `--no-context` disables app context entirely.
|
|
198
|
+
- Context Optimization is enabled by default with a `30000` input-token threshold.
|
|
199
|
+
- `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
|
|
200
|
+
- `--no-context-optimization` disables Context Optimization.
|
|
192
201
|
- `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
|
|
193
202
|
- `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
|
|
194
203
|
- If both inline and file-based prompt overrides are provided, the inline prompt flags win.
|
|
@@ -216,4 +225,4 @@ Current headless behavior is documented in [docs/headless-cli-spec.md](docs/head
|
|
|
216
225
|
|
|
217
226
|
<h2 id="license">📄 License</h2>
|
|
218
227
|
|
|
219
|
-
©
|
|
228
|
+
© 2026 Loadmill. All rights reserved.
|
package/build/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import minimist from "minimist";
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import path from "path";
|
|
4
|
-
import { mkdir
|
|
4
|
+
import { mkdir } from "fs/promises";
|
|
5
5
|
import { connectToDevice, getDeviceInfo } from "./src/device/connection.js";
|
|
6
6
|
import { Session } from "./src/core/session.js";
|
|
7
7
|
import { ExecutionEngine } from "./src/core/execution-engine.js";
|
|
@@ -11,10 +11,12 @@ import { ExecutionMode } from "./src/modes/execution-mode.js";
|
|
|
11
11
|
import { logger } from "./src/utils/logger.js";
|
|
12
12
|
import { selectDevice } from "./src/cli/device-selector.js";
|
|
13
13
|
import { buildAppContextBriefing, DEFAULT_APP_CONTEXT_BUDGET } from "./src/core/app-context.js";
|
|
14
|
+
import { CONTEXT_OPTIMIZATION_ENABLED_ENV, CONTEXT_OPTIMIZATION_THRESHOLD_ENV, } from "./src/core/context-optimization.js";
|
|
14
15
|
import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
|
|
15
16
|
import { printCliOutput } from "./src/utils/console-output.js";
|
|
16
17
|
import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
|
|
17
18
|
import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
|
|
19
|
+
import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
|
|
18
20
|
dotenv.config();
|
|
19
21
|
const args = minimist(process.argv.slice(2));
|
|
20
22
|
let avdName = args["avd"];
|
|
@@ -74,14 +76,13 @@ async function main() {
|
|
|
74
76
|
console.log(`Debug logging enabled: ${currentLogFilePath}`);
|
|
75
77
|
}
|
|
76
78
|
console.log(`\nRunning test from: ${instructionsFile}\n`);
|
|
77
|
-
|
|
78
|
-
instructions = content
|
|
79
|
-
.split("\n")
|
|
80
|
-
.map(line => line.trim())
|
|
81
|
-
.filter(line => line.length > 0);
|
|
79
|
+
instructions = await loadHeadlessInstructions(instructionsFile);
|
|
82
80
|
const taskText = instructions.join("\n");
|
|
83
81
|
const headlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
84
82
|
process.env.OPENAI_CUA_MODEL = headlessConfig.cuaModel;
|
|
83
|
+
process.env[CONTEXT_OPTIMIZATION_ENABLED_ENV] = String(headlessConfig.contextOptimizationEnabled);
|
|
84
|
+
process.env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV] = String(headlessConfig.contextOptimizationThreshold);
|
|
85
|
+
console.log(`Using CUA model: ${headlessConfig.cuaModel}`);
|
|
85
86
|
let appContextBriefing = "";
|
|
86
87
|
if (!headlessConfig.appContextEnabled) {
|
|
87
88
|
emitDesktopDebug("app_context.status", "execution", { runId }, {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
3
|
import { DEFAULT_APP_CONTEXT_BUDGET, MAX_APP_CONTEXT_BUDGET, MIN_APP_CONTEXT_BUDGET, } from "../core/app-context.js";
|
|
4
|
-
|
|
4
|
+
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
|
|
5
|
+
const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
|
|
6
|
+
const VALID_CUA_MODEL_LIST = ["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"];
|
|
5
7
|
function createEmptyPromptCustomizations() {
|
|
6
8
|
return {
|
|
7
9
|
basePromptInstructions: "",
|
|
@@ -19,10 +21,17 @@ function assertNonEmptyString(value, label) {
|
|
|
19
21
|
}
|
|
20
22
|
function validateCuaModel(value, label) {
|
|
21
23
|
if (typeof value !== "string" || !VALID_CUA_MODELS.has(value)) {
|
|
22
|
-
throw new Error(`${label} must be one of:
|
|
24
|
+
throw new Error(`${label} must be one of: ${VALID_CUA_MODEL_LIST.join(", ")}.`);
|
|
23
25
|
}
|
|
24
26
|
return value;
|
|
25
27
|
}
|
|
28
|
+
function readEnvDefaultCuaModel() {
|
|
29
|
+
const rawValue = process.env.OPENAI_CUA_MODEL;
|
|
30
|
+
if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
|
|
31
|
+
return "gpt-5.4";
|
|
32
|
+
}
|
|
33
|
+
return VALID_CUA_MODELS.has(rawValue) ? rawValue : "gpt-5.4";
|
|
34
|
+
}
|
|
26
35
|
function parseBooleanFlagState(args, key) {
|
|
27
36
|
return {
|
|
28
37
|
enabled: args[key] === true,
|
|
@@ -47,6 +56,24 @@ function parseBudgetValue(rawValue, label) {
|
|
|
47
56
|
}
|
|
48
57
|
return numericValue;
|
|
49
58
|
}
|
|
59
|
+
function parseContextOptimizationThreshold(rawValue, label) {
|
|
60
|
+
const numericValue = (() => {
|
|
61
|
+
if (typeof rawValue === "number") {
|
|
62
|
+
return rawValue;
|
|
63
|
+
}
|
|
64
|
+
if (typeof rawValue === "string" && /^-?\d+$/.test(rawValue.trim())) {
|
|
65
|
+
return Number.parseInt(rawValue, 10);
|
|
66
|
+
}
|
|
67
|
+
return Number.NaN;
|
|
68
|
+
})();
|
|
69
|
+
if (!Number.isInteger(numericValue)) {
|
|
70
|
+
throw new Error(`${label} must be an integer between ${MIN_CONTEXT_OPTIMIZATION_THRESHOLD} and ${MAX_CONTEXT_OPTIMIZATION_THRESHOLD}.`);
|
|
71
|
+
}
|
|
72
|
+
if (numericValue < MIN_CONTEXT_OPTIMIZATION_THRESHOLD || numericValue > MAX_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
73
|
+
throw new Error(`${label} must be between ${MIN_CONTEXT_OPTIMIZATION_THRESHOLD} and ${MAX_CONTEXT_OPTIMIZATION_THRESHOLD}.`);
|
|
74
|
+
}
|
|
75
|
+
return numericValue;
|
|
76
|
+
}
|
|
50
77
|
function normalizePromptCustomizations(rawValue, label) {
|
|
51
78
|
if (rawValue == null) {
|
|
52
79
|
return createEmptyPromptCustomizations();
|
|
@@ -108,6 +135,8 @@ async function loadConfigFromFile(configPath) {
|
|
|
108
135
|
appContextEnabled: undefined,
|
|
109
136
|
appContextBudget: undefined,
|
|
110
137
|
appContextPath: undefined,
|
|
138
|
+
contextOptimizationEnabled: undefined,
|
|
139
|
+
contextOptimizationThreshold: undefined,
|
|
111
140
|
strictMode: undefined,
|
|
112
141
|
};
|
|
113
142
|
if ("cuaModel" in rawConfig) {
|
|
@@ -129,6 +158,15 @@ async function loadConfigFromFile(configPath) {
|
|
|
129
158
|
assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
|
|
130
159
|
normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
|
|
131
160
|
}
|
|
161
|
+
if ("contextOptimizationEnabled" in rawConfig) {
|
|
162
|
+
if (typeof rawConfig.contextOptimizationEnabled !== "boolean") {
|
|
163
|
+
throw new Error("config.contextOptimizationEnabled must be a boolean.");
|
|
164
|
+
}
|
|
165
|
+
normalized.contextOptimizationEnabled = rawConfig.contextOptimizationEnabled;
|
|
166
|
+
}
|
|
167
|
+
if ("contextOptimizationThreshold" in rawConfig) {
|
|
168
|
+
normalized.contextOptimizationThreshold = parseContextOptimizationThreshold(rawConfig.contextOptimizationThreshold, "config.contextOptimizationThreshold");
|
|
169
|
+
}
|
|
132
170
|
if ("strictMode" in rawConfig) {
|
|
133
171
|
if (typeof rawConfig.strictMode !== "boolean") {
|
|
134
172
|
throw new Error("config.strictMode must be a boolean.");
|
|
@@ -142,6 +180,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
142
180
|
const configPath = typeof args.config === "string" ? args.config : null;
|
|
143
181
|
const explicitContextPath = typeof args.context === "string" ? path.resolve(cwd, args.context) : null;
|
|
144
182
|
const noContext = args["no-context"] === true || args.context === false;
|
|
183
|
+
const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
|
|
184
|
+
const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
|
|
185
|
+
const hasContextOptimizationThresholdFlag = args["context-optimization-threshold"] != null;
|
|
145
186
|
const { enabled: strictEnabledByFlag, disabled: strictDisabledByFlag } = parseBooleanFlagState(args, "strict");
|
|
146
187
|
const basePromptText = typeof args["base-prompt"] === "string" ? args["base-prompt"] : null;
|
|
147
188
|
const executionPromptText = typeof args["execution-prompt"] === "string" ? args["execution-prompt"] : null;
|
|
@@ -153,6 +194,12 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
153
194
|
if (strictEnabledByFlag && strictDisabledByFlag) {
|
|
154
195
|
throw new Error("--strict and --no-strict cannot be used together.");
|
|
155
196
|
}
|
|
197
|
+
if (contextOptimizationEnabledByFlag) {
|
|
198
|
+
throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
|
|
199
|
+
}
|
|
200
|
+
if (noContextOptimization && hasContextOptimizationThresholdFlag) {
|
|
201
|
+
throw new Error("--context-optimization-threshold and --no-context-optimization cannot be used together.");
|
|
202
|
+
}
|
|
156
203
|
const fileConfig = configPath ? await loadConfigFromFile(configPath) : null;
|
|
157
204
|
const promptCustomizations = {
|
|
158
205
|
...createEmptyPromptCustomizations(),
|
|
@@ -160,11 +207,13 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
160
207
|
};
|
|
161
208
|
const resolved = {
|
|
162
209
|
configPath: fileConfig?.configPath || null,
|
|
163
|
-
cuaModel: fileConfig?.cuaModel ||
|
|
210
|
+
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
164
211
|
promptCustomizations,
|
|
165
212
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
166
213
|
appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
|
|
167
214
|
appContextPath: fileConfig?.appContextPath || null,
|
|
215
|
+
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
216
|
+
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
168
217
|
strictMode: fileConfig?.strictMode ?? false,
|
|
169
218
|
};
|
|
170
219
|
if (typeof args["cua-model"] === "string") {
|
|
@@ -173,6 +222,10 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
173
222
|
if (args["app-context-budget"] != null) {
|
|
174
223
|
resolved.appContextBudget = parseBudgetValue(args["app-context-budget"], "--app-context-budget");
|
|
175
224
|
}
|
|
225
|
+
if (hasContextOptimizationThresholdFlag) {
|
|
226
|
+
resolved.contextOptimizationEnabled = true;
|
|
227
|
+
resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
|
|
228
|
+
}
|
|
176
229
|
if (basePromptFilePath) {
|
|
177
230
|
resolved.promptCustomizations.basePromptInstructions = await readTextFile(basePromptFilePath, "--base-prompt-file");
|
|
178
231
|
}
|
|
@@ -193,6 +246,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
193
246
|
resolved.appContextEnabled = false;
|
|
194
247
|
resolved.appContextPath = null;
|
|
195
248
|
}
|
|
249
|
+
if (noContextOptimization) {
|
|
250
|
+
resolved.contextOptimizationEnabled = false;
|
|
251
|
+
}
|
|
196
252
|
if (strictEnabledByFlag) {
|
|
197
253
|
resolved.strictMode = true;
|
|
198
254
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { readFile } from "fs/promises";
|
|
2
|
+
import { parseTestScript } from "../test-store/test-script.js";
|
|
3
|
+
/**
|
|
4
|
+
* Load runnable instructions from a headless .dcua script file.
|
|
5
|
+
*
|
|
6
|
+
* @param {string} instructionsPath
|
|
7
|
+
* @returns {Promise<string[]>}
|
|
8
|
+
*/
|
|
9
|
+
export async function loadHeadlessInstructions(instructionsPath) {
|
|
10
|
+
const content = await readFile(instructionsPath, "utf-8");
|
|
11
|
+
return parseTestScript(content);
|
|
12
|
+
}
|
|
@@ -20,10 +20,12 @@ export async function handleHelp(args, session, context) {
|
|
|
20
20
|
addOutput({ type: 'info', text: ' --platform <platform> Force platform: android or ios' });
|
|
21
21
|
addOutput({ type: 'info', text: ' --instructions <file> Run test file in headless mode' });
|
|
22
22
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
23
|
-
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4 or computer-use-preview' });
|
|
23
|
+
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4, gpt-5.4-mini, or computer-use-preview' });
|
|
24
24
|
addOutput({ type: 'info', text: ' --context <file> Optional app context file used to brief execution runs' });
|
|
25
25
|
addOutput({ type: 'info', text: ' --app-context-budget Headless app context token budget override' });
|
|
26
26
|
addOutput({ type: 'info', text: ' --no-context Disable app context for headless execution' });
|
|
27
|
+
addOutput({ type: 'info', text: ' --context-optimization-threshold <tokens> Headless Context Optimization compaction threshold' });
|
|
28
|
+
addOutput({ type: 'info', text: ' --no-context-optimization Disable Context Optimization for headless execution' });
|
|
27
29
|
addOutput({ type: 'info', text: ' --base-prompt <text> Headless base prompt customization text' });
|
|
28
30
|
addOutput({ type: 'info', text: ' --execution-prompt <text> Headless execution prompt customization text' });
|
|
29
31
|
addOutput({ type: 'info', text: ' --base-prompt-file Headless base prompt customization file' });
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
|
|
2
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 30000;
|
|
3
|
+
export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
|
|
4
|
+
export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
|
|
5
|
+
export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";
|
|
6
|
+
export const CONTEXT_OPTIMIZATION_THRESHOLD_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_THRESHOLD";
|
|
7
|
+
export function normalizeContextOptimizationEnabled(value) {
|
|
8
|
+
if (typeof value !== "boolean") {
|
|
9
|
+
return DEFAULT_CONTEXT_OPTIMIZATION_ENABLED;
|
|
10
|
+
}
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
export function normalizeContextOptimizationThreshold(value) {
|
|
14
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
15
|
+
return DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
16
|
+
}
|
|
17
|
+
const normalized = Math.round(value);
|
|
18
|
+
if (normalized < MIN_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
19
|
+
return MIN_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
20
|
+
}
|
|
21
|
+
if (normalized > MAX_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
22
|
+
return MAX_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
23
|
+
}
|
|
24
|
+
return normalized;
|
|
25
|
+
}
|
|
26
|
+
export function readContextOptimizationEnv(env = process.env) {
|
|
27
|
+
const rawEnabled = env[CONTEXT_OPTIMIZATION_ENABLED_ENV];
|
|
28
|
+
const enabled = rawEnabled == null
|
|
29
|
+
? DEFAULT_CONTEXT_OPTIMIZATION_ENABLED
|
|
30
|
+
: !["0", "false", "off", "no"].includes(String(rawEnabled).trim().toLowerCase());
|
|
31
|
+
const rawThreshold = env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV];
|
|
32
|
+
const threshold = rawThreshold == null || String(rawThreshold).trim() === ""
|
|
33
|
+
? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD
|
|
34
|
+
: normalizeContextOptimizationThreshold(Number(rawThreshold));
|
|
35
|
+
return { enabled, threshold };
|
|
36
|
+
}
|