@loadmill/droid-cua 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +11 -2
- package/build/index.js +7 -6
- package/build/src/cli/headless-execution-config.js +59 -3
- package/build/src/cli/headless-instructions.js +12 -0
- package/build/src/commands/help.js +3 -1
- package/build/src/core/context-optimization.js +36 -0
- package/build/src/core/execution-engine.js +4 -1
- package/build/src/device/ios/appium-server.js +4 -3
- package/build/src/device/ios/node-cli-env.js +28 -0
- package/build/src/device/openai.js +47 -2
- package/build/src/prompts/base.js +3 -1
- package/package.json +6 -1
package/LICENSE
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
©
|
|
1
|
+
© 2026 Loadmill. All rights reserved.
|
package/README.md
CHANGED
|
@@ -155,7 +155,9 @@ Example headless config:
|
|
|
155
155
|
},
|
|
156
156
|
"appContextEnabled": true,
|
|
157
157
|
"appContextBudget": 300,
|
|
158
|
-
"appContextPath": "../tests/context.md"
|
|
158
|
+
"appContextPath": "../tests/context.md",
|
|
159
|
+
"contextOptimizationEnabled": true,
|
|
160
|
+
"contextOptimizationThreshold": 30000
|
|
159
161
|
}
|
|
160
162
|
```
|
|
161
163
|
|
|
@@ -177,10 +179,14 @@ Supported CLI options include:
|
|
|
177
179
|
- `--context`
|
|
178
180
|
- `--app-context-budget`
|
|
179
181
|
- `--no-context`
|
|
182
|
+
- `--context-optimization-threshold`
|
|
183
|
+
- `--no-context-optimization`
|
|
180
184
|
- `--base-prompt`
|
|
181
185
|
- `--execution-prompt`
|
|
182
186
|
- `--base-prompt-file`
|
|
183
187
|
- `--execution-prompt-file`
|
|
188
|
+
- `--strict`
|
|
189
|
+
- `--no-strict`
|
|
184
190
|
- `--record`
|
|
185
191
|
- `--debug`
|
|
186
192
|
|
|
@@ -189,6 +195,9 @@ Config and precedence rules:
|
|
|
189
195
|
- CLI flags override config file values.
|
|
190
196
|
- `--context` overrides the config app-context path.
|
|
191
197
|
- `--no-context` disables app context entirely.
|
|
198
|
+
- Context Optimization is enabled by default with a `30000` input-token threshold.
|
|
199
|
+
- `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
|
|
200
|
+
- `--no-context-optimization` disables Context Optimization.
|
|
192
201
|
- `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
|
|
193
202
|
- `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
|
|
194
203
|
- If both inline and file-based prompt overrides are provided, the inline prompt flags win.
|
|
@@ -216,4 +225,4 @@ Current headless behavior is documented in [docs/headless-cli-spec.md](docs/head
|
|
|
216
225
|
|
|
217
226
|
<h2 id="license">📄 License</h2>
|
|
218
227
|
|
|
219
|
-
©
|
|
228
|
+
© 2026 Loadmill. All rights reserved.
|
package/build/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import minimist from "minimist";
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import path from "path";
|
|
4
|
-
import { mkdir
|
|
4
|
+
import { mkdir } from "fs/promises";
|
|
5
5
|
import { connectToDevice, getDeviceInfo } from "./src/device/connection.js";
|
|
6
6
|
import { Session } from "./src/core/session.js";
|
|
7
7
|
import { ExecutionEngine } from "./src/core/execution-engine.js";
|
|
@@ -11,10 +11,12 @@ import { ExecutionMode } from "./src/modes/execution-mode.js";
|
|
|
11
11
|
import { logger } from "./src/utils/logger.js";
|
|
12
12
|
import { selectDevice } from "./src/cli/device-selector.js";
|
|
13
13
|
import { buildAppContextBriefing, DEFAULT_APP_CONTEXT_BUDGET } from "./src/core/app-context.js";
|
|
14
|
+
import { CONTEXT_OPTIMIZATION_ENABLED_ENV, CONTEXT_OPTIMIZATION_THRESHOLD_ENV, } from "./src/core/context-optimization.js";
|
|
14
15
|
import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
|
|
15
16
|
import { printCliOutput } from "./src/utils/console-output.js";
|
|
16
17
|
import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
|
|
17
18
|
import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
|
|
19
|
+
import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
|
|
18
20
|
dotenv.config();
|
|
19
21
|
const args = minimist(process.argv.slice(2));
|
|
20
22
|
let avdName = args["avd"];
|
|
@@ -74,14 +76,13 @@ async function main() {
|
|
|
74
76
|
console.log(`Debug logging enabled: ${currentLogFilePath}`);
|
|
75
77
|
}
|
|
76
78
|
console.log(`\nRunning test from: ${instructionsFile}\n`);
|
|
77
|
-
|
|
78
|
-
instructions = content
|
|
79
|
-
.split("\n")
|
|
80
|
-
.map(line => line.trim())
|
|
81
|
-
.filter(line => line.length > 0);
|
|
79
|
+
instructions = await loadHeadlessInstructions(instructionsFile);
|
|
82
80
|
const taskText = instructions.join("\n");
|
|
83
81
|
const headlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
84
82
|
process.env.OPENAI_CUA_MODEL = headlessConfig.cuaModel;
|
|
83
|
+
process.env[CONTEXT_OPTIMIZATION_ENABLED_ENV] = String(headlessConfig.contextOptimizationEnabled);
|
|
84
|
+
process.env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV] = String(headlessConfig.contextOptimizationThreshold);
|
|
85
|
+
console.log(`Using CUA model: ${headlessConfig.cuaModel}`);
|
|
85
86
|
let appContextBriefing = "";
|
|
86
87
|
if (!headlessConfig.appContextEnabled) {
|
|
87
88
|
emitDesktopDebug("app_context.status", "execution", { runId }, {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
3
|
import { DEFAULT_APP_CONTEXT_BUDGET, MAX_APP_CONTEXT_BUDGET, MIN_APP_CONTEXT_BUDGET, } from "../core/app-context.js";
|
|
4
|
-
|
|
4
|
+
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
|
|
5
|
+
const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
|
|
6
|
+
const VALID_CUA_MODEL_LIST = ["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"];
|
|
5
7
|
function createEmptyPromptCustomizations() {
|
|
6
8
|
return {
|
|
7
9
|
basePromptInstructions: "",
|
|
@@ -19,10 +21,17 @@ function assertNonEmptyString(value, label) {
|
|
|
19
21
|
}
|
|
20
22
|
function validateCuaModel(value, label) {
|
|
21
23
|
if (typeof value !== "string" || !VALID_CUA_MODELS.has(value)) {
|
|
22
|
-
throw new Error(`${label} must be one of:
|
|
24
|
+
throw new Error(`${label} must be one of: ${VALID_CUA_MODEL_LIST.join(", ")}.`);
|
|
23
25
|
}
|
|
24
26
|
return value;
|
|
25
27
|
}
|
|
28
|
+
function readEnvDefaultCuaModel() {
|
|
29
|
+
const rawValue = process.env.OPENAI_CUA_MODEL;
|
|
30
|
+
if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
|
|
31
|
+
return "gpt-5.4";
|
|
32
|
+
}
|
|
33
|
+
return VALID_CUA_MODELS.has(rawValue) ? rawValue : "gpt-5.4";
|
|
34
|
+
}
|
|
26
35
|
function parseBooleanFlagState(args, key) {
|
|
27
36
|
return {
|
|
28
37
|
enabled: args[key] === true,
|
|
@@ -47,6 +56,24 @@ function parseBudgetValue(rawValue, label) {
|
|
|
47
56
|
}
|
|
48
57
|
return numericValue;
|
|
49
58
|
}
|
|
59
|
+
function parseContextOptimizationThreshold(rawValue, label) {
|
|
60
|
+
const numericValue = (() => {
|
|
61
|
+
if (typeof rawValue === "number") {
|
|
62
|
+
return rawValue;
|
|
63
|
+
}
|
|
64
|
+
if (typeof rawValue === "string" && /^-?\d+$/.test(rawValue.trim())) {
|
|
65
|
+
return Number.parseInt(rawValue, 10);
|
|
66
|
+
}
|
|
67
|
+
return Number.NaN;
|
|
68
|
+
})();
|
|
69
|
+
if (!Number.isInteger(numericValue)) {
|
|
70
|
+
throw new Error(`${label} must be an integer between ${MIN_CONTEXT_OPTIMIZATION_THRESHOLD} and ${MAX_CONTEXT_OPTIMIZATION_THRESHOLD}.`);
|
|
71
|
+
}
|
|
72
|
+
if (numericValue < MIN_CONTEXT_OPTIMIZATION_THRESHOLD || numericValue > MAX_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
73
|
+
throw new Error(`${label} must be between ${MIN_CONTEXT_OPTIMIZATION_THRESHOLD} and ${MAX_CONTEXT_OPTIMIZATION_THRESHOLD}.`);
|
|
74
|
+
}
|
|
75
|
+
return numericValue;
|
|
76
|
+
}
|
|
50
77
|
function normalizePromptCustomizations(rawValue, label) {
|
|
51
78
|
if (rawValue == null) {
|
|
52
79
|
return createEmptyPromptCustomizations();
|
|
@@ -108,6 +135,8 @@ async function loadConfigFromFile(configPath) {
|
|
|
108
135
|
appContextEnabled: undefined,
|
|
109
136
|
appContextBudget: undefined,
|
|
110
137
|
appContextPath: undefined,
|
|
138
|
+
contextOptimizationEnabled: undefined,
|
|
139
|
+
contextOptimizationThreshold: undefined,
|
|
111
140
|
strictMode: undefined,
|
|
112
141
|
};
|
|
113
142
|
if ("cuaModel" in rawConfig) {
|
|
@@ -129,6 +158,15 @@ async function loadConfigFromFile(configPath) {
|
|
|
129
158
|
assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
|
|
130
159
|
normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
|
|
131
160
|
}
|
|
161
|
+
if ("contextOptimizationEnabled" in rawConfig) {
|
|
162
|
+
if (typeof rawConfig.contextOptimizationEnabled !== "boolean") {
|
|
163
|
+
throw new Error("config.contextOptimizationEnabled must be a boolean.");
|
|
164
|
+
}
|
|
165
|
+
normalized.contextOptimizationEnabled = rawConfig.contextOptimizationEnabled;
|
|
166
|
+
}
|
|
167
|
+
if ("contextOptimizationThreshold" in rawConfig) {
|
|
168
|
+
normalized.contextOptimizationThreshold = parseContextOptimizationThreshold(rawConfig.contextOptimizationThreshold, "config.contextOptimizationThreshold");
|
|
169
|
+
}
|
|
132
170
|
if ("strictMode" in rawConfig) {
|
|
133
171
|
if (typeof rawConfig.strictMode !== "boolean") {
|
|
134
172
|
throw new Error("config.strictMode must be a boolean.");
|
|
@@ -142,6 +180,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
142
180
|
const configPath = typeof args.config === "string" ? args.config : null;
|
|
143
181
|
const explicitContextPath = typeof args.context === "string" ? path.resolve(cwd, args.context) : null;
|
|
144
182
|
const noContext = args["no-context"] === true || args.context === false;
|
|
183
|
+
const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
|
|
184
|
+
const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
|
|
185
|
+
const hasContextOptimizationThresholdFlag = args["context-optimization-threshold"] != null;
|
|
145
186
|
const { enabled: strictEnabledByFlag, disabled: strictDisabledByFlag } = parseBooleanFlagState(args, "strict");
|
|
146
187
|
const basePromptText = typeof args["base-prompt"] === "string" ? args["base-prompt"] : null;
|
|
147
188
|
const executionPromptText = typeof args["execution-prompt"] === "string" ? args["execution-prompt"] : null;
|
|
@@ -153,6 +194,12 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
153
194
|
if (strictEnabledByFlag && strictDisabledByFlag) {
|
|
154
195
|
throw new Error("--strict and --no-strict cannot be used together.");
|
|
155
196
|
}
|
|
197
|
+
if (contextOptimizationEnabledByFlag) {
|
|
198
|
+
throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
|
|
199
|
+
}
|
|
200
|
+
if (noContextOptimization && hasContextOptimizationThresholdFlag) {
|
|
201
|
+
throw new Error("--context-optimization-threshold and --no-context-optimization cannot be used together.");
|
|
202
|
+
}
|
|
156
203
|
const fileConfig = configPath ? await loadConfigFromFile(configPath) : null;
|
|
157
204
|
const promptCustomizations = {
|
|
158
205
|
...createEmptyPromptCustomizations(),
|
|
@@ -160,11 +207,13 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
160
207
|
};
|
|
161
208
|
const resolved = {
|
|
162
209
|
configPath: fileConfig?.configPath || null,
|
|
163
|
-
cuaModel: fileConfig?.cuaModel ||
|
|
210
|
+
cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
|
|
164
211
|
promptCustomizations,
|
|
165
212
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
166
213
|
appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
|
|
167
214
|
appContextPath: fileConfig?.appContextPath || null,
|
|
215
|
+
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
216
|
+
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
168
217
|
strictMode: fileConfig?.strictMode ?? false,
|
|
169
218
|
};
|
|
170
219
|
if (typeof args["cua-model"] === "string") {
|
|
@@ -173,6 +222,10 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
173
222
|
if (args["app-context-budget"] != null) {
|
|
174
223
|
resolved.appContextBudget = parseBudgetValue(args["app-context-budget"], "--app-context-budget");
|
|
175
224
|
}
|
|
225
|
+
if (hasContextOptimizationThresholdFlag) {
|
|
226
|
+
resolved.contextOptimizationEnabled = true;
|
|
227
|
+
resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");
|
|
228
|
+
}
|
|
176
229
|
if (basePromptFilePath) {
|
|
177
230
|
resolved.promptCustomizations.basePromptInstructions = await readTextFile(basePromptFilePath, "--base-prompt-file");
|
|
178
231
|
}
|
|
@@ -193,6 +246,9 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
193
246
|
resolved.appContextEnabled = false;
|
|
194
247
|
resolved.appContextPath = null;
|
|
195
248
|
}
|
|
249
|
+
if (noContextOptimization) {
|
|
250
|
+
resolved.contextOptimizationEnabled = false;
|
|
251
|
+
}
|
|
196
252
|
if (strictEnabledByFlag) {
|
|
197
253
|
resolved.strictMode = true;
|
|
198
254
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { readFile } from "fs/promises";
|
|
2
|
+
import { parseTestScript } from "../test-store/test-script.js";
|
|
3
|
+
/**
|
|
4
|
+
* Load runnable instructions from a headless .dcua script file.
|
|
5
|
+
*
|
|
6
|
+
* @param {string} instructionsPath
|
|
7
|
+
* @returns {Promise<string[]>}
|
|
8
|
+
*/
|
|
9
|
+
export async function loadHeadlessInstructions(instructionsPath) {
|
|
10
|
+
const content = await readFile(instructionsPath, "utf-8");
|
|
11
|
+
return parseTestScript(content);
|
|
12
|
+
}
|
|
@@ -20,10 +20,12 @@ export async function handleHelp(args, session, context) {
|
|
|
20
20
|
addOutput({ type: 'info', text: ' --platform <platform> Force platform: android or ios' });
|
|
21
21
|
addOutput({ type: 'info', text: ' --instructions <file> Run test file in headless mode' });
|
|
22
22
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
23
|
-
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4 or computer-use-preview' });
|
|
23
|
+
addOutput({ type: 'info', text: ' --cua-model <model> Headless CUA model override: gpt-5.4, gpt-5.4-mini, or computer-use-preview' });
|
|
24
24
|
addOutput({ type: 'info', text: ' --context <file> Optional app context file used to brief execution runs' });
|
|
25
25
|
addOutput({ type: 'info', text: ' --app-context-budget Headless app context token budget override' });
|
|
26
26
|
addOutput({ type: 'info', text: ' --no-context Disable app context for headless execution' });
|
|
27
|
+
addOutput({ type: 'info', text: ' --context-optimization-threshold <tokens> Headless Context Optimization compaction threshold' });
|
|
28
|
+
addOutput({ type: 'info', text: ' --no-context-optimization Disable Context Optimization for headless execution' });
|
|
27
29
|
addOutput({ type: 'info', text: ' --base-prompt <text> Headless base prompt customization text' });
|
|
28
30
|
addOutput({ type: 'info', text: ' --execution-prompt <text> Headless execution prompt customization text' });
|
|
29
31
|
addOutput({ type: 'info', text: ' --base-prompt-file Headless base prompt customization file' });
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
|
|
2
|
+
export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 30000;
|
|
3
|
+
export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
|
|
4
|
+
export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
|
|
5
|
+
export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";
|
|
6
|
+
export const CONTEXT_OPTIMIZATION_THRESHOLD_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_THRESHOLD";
|
|
7
|
+
export function normalizeContextOptimizationEnabled(value) {
|
|
8
|
+
if (typeof value !== "boolean") {
|
|
9
|
+
return DEFAULT_CONTEXT_OPTIMIZATION_ENABLED;
|
|
10
|
+
}
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
export function normalizeContextOptimizationThreshold(value) {
|
|
14
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
15
|
+
return DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
16
|
+
}
|
|
17
|
+
const normalized = Math.round(value);
|
|
18
|
+
if (normalized < MIN_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
19
|
+
return MIN_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
20
|
+
}
|
|
21
|
+
if (normalized > MAX_CONTEXT_OPTIMIZATION_THRESHOLD) {
|
|
22
|
+
return MAX_CONTEXT_OPTIMIZATION_THRESHOLD;
|
|
23
|
+
}
|
|
24
|
+
return normalized;
|
|
25
|
+
}
|
|
26
|
+
export function readContextOptimizationEnv(env = process.env) {
|
|
27
|
+
const rawEnabled = env[CONTEXT_OPTIMIZATION_ENABLED_ENV];
|
|
28
|
+
const enabled = rawEnabled == null
|
|
29
|
+
? DEFAULT_CONTEXT_OPTIMIZATION_ENABLED
|
|
30
|
+
: !["0", "false", "off", "no"].includes(String(rawEnabled).trim().toLowerCase());
|
|
31
|
+
const rawThreshold = env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV];
|
|
32
|
+
const threshold = rawThreshold == null || String(rawThreshold).trim() === ""
|
|
33
|
+
? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD
|
|
34
|
+
: normalizeContextOptimizationThreshold(Number(rawThreshold));
|
|
35
|
+
return { enabled, threshold };
|
|
36
|
+
}
|
|
@@ -8,6 +8,7 @@ import { emitDesktopDebug } from "../utils/desktop-debug.js";
|
|
|
8
8
|
import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
|
|
9
9
|
import { printCliOutput } from "../utils/console-output.js";
|
|
10
10
|
import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
|
|
11
|
+
const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
|
|
11
12
|
function extractActionableOutputItems(items) {
|
|
12
13
|
const entries = [];
|
|
13
14
|
for (const item of items) {
|
|
@@ -548,7 +549,9 @@ export class ExecutionEngine {
|
|
|
548
549
|
note: runtimeNote
|
|
549
550
|
});
|
|
550
551
|
}
|
|
551
|
-
const selectedCuaModel = process.env.OPENAI_CUA_MODEL
|
|
552
|
+
const selectedCuaModel = VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL)
|
|
553
|
+
? process.env.OPENAI_CUA_MODEL
|
|
554
|
+
: "gpt-5.4";
|
|
552
555
|
const input = [{
|
|
553
556
|
type: "computer_call_output",
|
|
554
557
|
call_id,
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import { spawn } from "child_process";
|
|
8
8
|
import { existsSync, readdirSync } from "fs";
|
|
9
9
|
import path from "path";
|
|
10
|
+
import { COMMON_NODE_BIN_DIRS, makeNodeCliCommandEnv } from "./node-cli-env.js";
|
|
10
11
|
let appiumProcess = null;
|
|
11
12
|
let cleanupRegistered = false;
|
|
12
13
|
function getAppiumBaseUrl() {
|
|
@@ -35,8 +36,7 @@ export function resolveAppiumCommand() {
|
|
|
35
36
|
return explicit;
|
|
36
37
|
}
|
|
37
38
|
const commonCandidates = [
|
|
38
|
-
"
|
|
39
|
-
"/usr/local/bin/appium",
|
|
39
|
+
...COMMON_NODE_BIN_DIRS.map((dir) => path.join(dir, "appium")),
|
|
40
40
|
...collectNvmAppiumCandidates()
|
|
41
41
|
];
|
|
42
42
|
for (const candidate of commonCandidates) {
|
|
@@ -50,7 +50,8 @@ async function spawnAppiumProcess(command) {
|
|
|
50
50
|
return await new Promise((resolve, reject) => {
|
|
51
51
|
const child = spawn(command, [], {
|
|
52
52
|
detached: true,
|
|
53
|
-
stdio: "ignore"
|
|
53
|
+
stdio: "ignore",
|
|
54
|
+
env: makeNodeCliCommandEnv(command)
|
|
54
55
|
});
|
|
55
56
|
child.once("error", reject);
|
|
56
57
|
child.once("spawn", () => resolve(child));
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
export const COMMON_NODE_BIN_DIRS = ["/opt/homebrew/bin", "/usr/local/bin", "/opt/local/bin"];
|
|
3
|
+
function prependPathEntries(entries, originalPath) {
|
|
4
|
+
const seen = new Set();
|
|
5
|
+
const next = [];
|
|
6
|
+
const allEntries = [
|
|
7
|
+
...entries,
|
|
8
|
+
...(originalPath
|
|
9
|
+
?.split(path.delimiter)
|
|
10
|
+
.map((entry) => entry.trim())
|
|
11
|
+
.filter(Boolean) ?? [])
|
|
12
|
+
];
|
|
13
|
+
for (const entry of allEntries) {
|
|
14
|
+
if (seen.has(entry))
|
|
15
|
+
continue;
|
|
16
|
+
seen.add(entry);
|
|
17
|
+
next.push(entry);
|
|
18
|
+
}
|
|
19
|
+
return next.join(path.delimiter);
|
|
20
|
+
}
|
|
21
|
+
export function makeNodeCliCommandEnv(command) {
|
|
22
|
+
const binDirs = path.isAbsolute(command) ? [path.dirname(command), ...COMMON_NODE_BIN_DIRS] : COMMON_NODE_BIN_DIRS;
|
|
23
|
+
return {
|
|
24
|
+
...process.env,
|
|
25
|
+
// Finder-launched apps may miss Homebrew paths; npm/Appium shims use "#!/usr/bin/env node".
|
|
26
|
+
PATH: prependPathEntries(binDirs, process.env.PATH)
|
|
27
|
+
};
|
|
28
|
+
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import OpenAI from "openai";
|
|
2
|
+
import { readContextOptimizationEnv } from "../core/context-optimization.js";
|
|
2
3
|
import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
|
|
3
4
|
import { buildAppContextCompactionInput } from "../prompts/execution.js";
|
|
4
5
|
import { logger } from "../utils/logger.js";
|
|
5
6
|
import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
|
|
6
7
|
let openai = null;
|
|
7
8
|
const cuaDebugTracer = new CuaDebugTracer(logger);
|
|
9
|
+
const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
|
|
8
10
|
function getSelectedCuaModel() {
|
|
9
|
-
return process.env.OPENAI_CUA_MODEL
|
|
11
|
+
return VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL) ? process.env.OPENAI_CUA_MODEL : "gpt-5.4";
|
|
10
12
|
}
|
|
11
13
|
function normalizeMessageContent(content) {
|
|
12
14
|
if (Array.isArray(content)) {
|
|
@@ -88,7 +90,18 @@ export function buildCuaInput({ messages, screenshotBase64, previousResponseId,
|
|
|
88
90
|
}
|
|
89
91
|
return input;
|
|
90
92
|
}
|
|
93
|
+
function buildContextManagement(cuaModel) {
|
|
94
|
+
if (cuaModel === "computer-use-preview") {
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
const { enabled, threshold } = readContextOptimizationEnv();
|
|
98
|
+
if (!enabled) {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
return [{ type: "compaction", compact_threshold: threshold }];
|
|
102
|
+
}
|
|
91
103
|
export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
|
|
104
|
+
const contextManagement = buildContextManagement(cuaModel);
|
|
92
105
|
if (cuaModel === "computer-use-preview") {
|
|
93
106
|
return {
|
|
94
107
|
model: cuaModel,
|
|
@@ -111,6 +124,7 @@ export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo
|
|
|
111
124
|
return {
|
|
112
125
|
model: cuaModel,
|
|
113
126
|
reasoning: { effort: "medium" },
|
|
127
|
+
...(contextManagement ? { context_management: contextManagement } : {}),
|
|
114
128
|
previous_response_id: previousResponseId || undefined,
|
|
115
129
|
input,
|
|
116
130
|
store: true,
|
|
@@ -137,12 +151,41 @@ function shouldMapPreviewAccessError(err) {
|
|
|
137
151
|
return (message.includes("computer-use-preview") &&
|
|
138
152
|
(message.includes("access") || message.includes("permission") || message.includes("not found") || message.includes("unsupported")));
|
|
139
153
|
}
|
|
154
|
+
export function shouldMapMiniCompatibilityError(err) {
|
|
155
|
+
const status = Number(err?.status);
|
|
156
|
+
const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
|
|
157
|
+
const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
|
|
158
|
+
const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
|
|
159
|
+
const mentionsMiniModel = message.includes("gpt-5.4-mini");
|
|
160
|
+
const mentionsComputerTool = message.includes("computer") || message.includes("tool");
|
|
161
|
+
const mentionsUnsupported = (message.includes("unsupported") ||
|
|
162
|
+
message.includes("not supported") ||
|
|
163
|
+
message.includes("does not support"));
|
|
164
|
+
if (code.includes("model_not_found")) {
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
if (status === 404) {
|
|
168
|
+
return mentionsMiniModel && (message.includes("not found") || mentionsUnsupported);
|
|
169
|
+
}
|
|
170
|
+
if (code.includes("unsupported")) {
|
|
171
|
+
return mentionsMiniModel || mentionsComputerTool;
|
|
172
|
+
}
|
|
173
|
+
if (!type.includes("invalid")) {
|
|
174
|
+
return false;
|
|
175
|
+
}
|
|
176
|
+
return mentionsUnsupported && (mentionsMiniModel || mentionsComputerTool);
|
|
177
|
+
}
|
|
140
178
|
function mapCuaError(err, cuaModel) {
|
|
141
179
|
if (cuaModel === "computer-use-preview" && shouldMapPreviewAccessError(err)) {
|
|
142
180
|
const mapped = new Error("OpenAI API key does not have access to computer-use-preview. Switch to gpt-5.4 in Settings > CUA Model.");
|
|
143
181
|
mapped.cause = err;
|
|
144
182
|
return mapped;
|
|
145
183
|
}
|
|
184
|
+
if (cuaModel === "gpt-5.4-mini" && shouldMapMiniCompatibilityError(err)) {
|
|
185
|
+
const mapped = new Error("OpenAI rejected gpt-5.4-mini for computer-use execution. Switch to gpt-5.4 or computer-use-preview in Settings > CUA Model.");
|
|
186
|
+
mapped.cause = err;
|
|
187
|
+
return mapped;
|
|
188
|
+
}
|
|
146
189
|
return err;
|
|
147
190
|
}
|
|
148
191
|
export function isNonRetryableCuaError(err) {
|
|
@@ -157,7 +200,9 @@ export function isNonRetryableCuaError(err) {
|
|
|
157
200
|
code.includes("permission") ||
|
|
158
201
|
type.includes("permission") ||
|
|
159
202
|
message.includes("does not have access to computer-use-preview") ||
|
|
160
|
-
message.includes("switch to gpt-5.4 in settings > cua model")
|
|
203
|
+
message.includes("switch to gpt-5.4 in settings > cua model") ||
|
|
204
|
+
message.includes("openai rejected gpt-5.4-mini for computer-use execution") ||
|
|
205
|
+
message.includes("switch to gpt-5.4 or computer-use-preview in settings > cua model"));
|
|
161
206
|
}
|
|
162
207
|
function getOpenAI() {
|
|
163
208
|
if (!openai) {
|
|
@@ -64,7 +64,9 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
|
|
|
64
64
|
- Reason carefully about the approximate pixel position.
|
|
65
65
|
- Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
|
|
66
66
|
- Review the returned annotated screenshot before clicking.
|
|
67
|
-
-
|
|
67
|
+
- Click if the marker is clearly inside the intended tappable control. Text inside a button, field, row, or bottom bar is okay.
|
|
68
|
+
- Adjust and preview again only if the marker is outside the target, on the target border/edge, or closer to a neighboring target than the intended one.
|
|
69
|
+
- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
|
|
68
70
|
- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
|
|
69
71
|
- Click precisely based on your visual estimate.
|
|
70
72
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loadmill/droid-cua",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
],
|
|
16
16
|
"scripts": {
|
|
17
17
|
"start": "tsx index.js",
|
|
18
|
+
"bench": "npm run bench:preflight && npm run bench:run && npm run bench:report",
|
|
19
|
+
"bench:preflight": "node scripts/benchmark-preflight.js",
|
|
20
|
+
"bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
|
|
21
|
+
"bench:report": "node scripts/benchmark-report.js",
|
|
22
|
+
"execution:profile": "node scripts/execution-profile.js",
|
|
18
23
|
"test": "node --test unit-tests/*.test.js",
|
|
19
24
|
"build": "tsc",
|
|
20
25
|
"clean": "rm -rf build",
|