@loadmill/droid-cua 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +7 -4
  2. package/build/index.js +131 -74
  3. package/build/src/browser/actions.js +9 -0
  4. package/build/src/browser/connection.js +37 -0
  5. package/build/src/browser/factory.js +22 -0
  6. package/build/src/browser/interface.js +14 -0
  7. package/build/src/browser/navigation.js +11 -0
  8. package/build/src/browser/normalize.js +36 -0
  9. package/build/src/browser/playwright/actions.js +199 -0
  10. package/build/src/browser/playwright/connection.js +207 -0
  11. package/build/src/browser/playwright/installed-browser.js +131 -0
  12. package/build/src/browser/playwright/session-state.js +19 -0
  13. package/build/src/browser/playwright/url.js +79 -0
  14. package/build/src/cli/headless-execution-config.js +4 -36
  15. package/build/src/cli/headless-target.js +64 -0
  16. package/build/src/cli/ink-shell.js +5 -2
  17. package/build/src/commands/help.js +9 -4
  18. package/build/src/commands/loadmill.js +10 -1
  19. package/build/src/commands/run.js +17 -28
  20. package/build/src/core/app-context.js +57 -33
  21. package/build/src/core/capture-sources.js +76 -0
  22. package/build/src/core/context-optimization.js +1 -1
  23. package/build/src/core/cua-models.js +9 -0
  24. package/build/src/core/execution-engine.js +43 -19
  25. package/build/src/core/prompts.js +1 -1
  26. package/build/src/core/runtime-config.js +96 -0
  27. package/build/src/core/session.js +18 -4
  28. package/build/src/device/action-event-meta.js +18 -0
  29. package/build/src/device/android/actions.js +6 -13
  30. package/build/src/device/cloud/actions.js +9 -14
  31. package/build/src/device/cloud/connection.js +8 -0
  32. package/build/src/device/ios/actions.js +6 -13
  33. package/build/src/device/ios/connection.js +8 -0
  34. package/build/src/device/loadmill.js +15 -9
  35. package/build/src/device/openai.js +67 -139
  36. package/build/src/integrations/loadmill/client.js +149 -131
  37. package/build/src/integrations/loadmill/executor.js +17 -11
  38. package/build/src/integrations/loadmill/index.js +1 -1
  39. package/build/src/integrations/loadmill/interpreter.js +4 -14
  40. package/build/src/modes/design-mode-ink.js +4 -3
  41. package/build/src/modes/design-mode.js +4 -3
  42. package/build/src/modes/execution-mode.js +139 -21
  43. package/build/src/prompts/base.js +92 -20
  44. package/build/src/prompts/design.js +6 -11
  45. package/build/src/prompts/execution.js +33 -109
  46. package/build/src/utils/console-output.js +8 -0
  47. package/build/src/utils/cua-debug-tracer.js +0 -1
  48. package/build/src/utils/run-screenshot-recorder.js +1 -1
  49. package/build/src/utils/structured-debug-log-manager.js +4 -1
  50. package/package.json +2 -1
@@ -0,0 +1,96 @@
1
+ import OpenAI from "openai";
2
+ import { readContextOptimizationEnv, normalizeContextOptimizationEnabled, normalizeContextOptimizationThreshold, } from "./context-optimization.js";
3
+ import { resolveCuaModel } from "./cua-models.js";
4
+ import { normalizeStepDelayMs } from "../utils/step-delay.js";
5
+ export function createEmptyPromptCustomizations() {
6
+ return {
7
+ basePromptInstructions: "",
8
+ designModeInstructions: "",
9
+ executionModeInstructions: "",
10
+ };
11
+ }
12
+ export function resolvePromptCustomizations(rawValue = null) {
13
+ const resolved = createEmptyPromptCustomizations();
14
+ if (!rawValue || typeof rawValue !== "object" || Array.isArray(rawValue)) {
15
+ return resolved;
16
+ }
17
+ for (const key of Object.keys(resolved)) {
18
+ if (typeof rawValue[key] === "string") {
19
+ resolved[key] = rawValue[key];
20
+ }
21
+ }
22
+ return resolved;
23
+ }
24
+ function trimNullableString(value) {
25
+ if (typeof value !== "string") {
26
+ return null;
27
+ }
28
+ const trimmed = value.trim();
29
+ return trimmed || null;
30
+ }
31
+ export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToken = null, loadmillBaseUrl = null, } = {}) {
32
+ return {
33
+ openAiApiKey: trimNullableString(openAiApiKey),
34
+ loadmillApiToken: trimNullableString(loadmillApiToken),
35
+ loadmillBaseUrl: trimNullableString(loadmillBaseUrl),
36
+ };
37
+ }
38
+ export function resolveRuntimeConfig({ cuaModel = null, strictMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
39
+ return {
40
+ cuaModel: resolveCuaModel(cuaModel),
41
+ strictMode: strictMode === true,
42
+ stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
43
+ contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
44
+ contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),
45
+ promptCustomizations: resolvePromptCustomizations(promptCustomizations),
46
+ appContext: appContext && typeof appContext === "object"
47
+ ? {
48
+ enabled: appContext.enabled !== false,
49
+ path: trimNullableString(appContext.path ?? appContext.contextPath),
50
+ status: typeof appContext.status === "string" ? appContext.status : null,
51
+ text: typeof appContext.text === "string" ? appContext.text : "",
52
+ message: trimNullableString(appContext.message),
53
+ }
54
+ : {
55
+ enabled: true,
56
+ path: null,
57
+ status: null,
58
+ text: "",
59
+ message: null,
60
+ },
61
+ };
62
+ }
63
+ export function resolveRuntimeConfigFromEnv(env = process.env, overrides = {}) {
64
+ const contextOptimization = readContextOptimizationEnv(env);
65
+ return resolveRuntimeConfig({
66
+ cuaModel: env.OPENAI_CUA_MODEL,
67
+ stepDelayMs: env.DROID_CUA_STEP_DELAY_MS,
68
+ contextOptimizationEnabled: contextOptimization.enabled,
69
+ contextOptimizationThreshold: contextOptimization.threshold,
70
+ ...overrides,
71
+ });
72
+ }
73
+ export function resolveRuntimeCredentialsFromEnv(env = process.env, overrides = {}) {
74
+ return resolveRuntimeCredentials({
75
+ openAiApiKey: env.OPENAI_API_KEY,
76
+ loadmillApiToken: env.LOADMILL_API_TOKEN,
77
+ loadmillBaseUrl: env.LOADMILL_BASE_URL,
78
+ ...overrides,
79
+ });
80
+ }
81
+ export function createOpenAiClient(apiKey) {
82
+ return new OpenAI({
83
+ apiKey: trimNullableString(apiKey) || undefined,
84
+ });
85
+ }
86
+ export function createRunContext({ config, credentials }) {
87
+ const resolvedConfig = resolveRuntimeConfig(config);
88
+ const resolvedCredentials = resolveRuntimeCredentials(credentials);
89
+ return {
90
+ config: resolvedConfig,
91
+ credentials: resolvedCredentials,
92
+ clients: {
93
+ openai: createOpenAiClient(resolvedCredentials.openAiApiKey),
94
+ },
95
+ };
96
+ }
@@ -1,11 +1,18 @@
1
1
  /**
2
2
  * Session manages the state for a single CLI session
3
- * Includes device info, message history, transcript, and response chaining
3
+ * Includes target info, message history, transcript, and response chaining
4
4
  */
5
5
  export class Session {
6
- constructor(deviceId, deviceInfo) {
7
- this.deviceId = deviceId;
8
- this.deviceInfo = deviceInfo;
6
+ constructor(targetSessionId, targetInfo) {
7
+ this.deviceId = targetSessionId;
8
+ this.deviceInfo = targetInfo;
9
+ this.deviceName = targetInfo?.device_name || null;
10
+ this.browserSessionId = targetInfo?.platform === "browser" ? targetSessionId : null;
11
+ this.browserInfo = targetInfo?.platform === "browser" ? targetInfo : null;
12
+ this.browserName = targetInfo?.browserName || null;
13
+ this.currentUrl = targetInfo?.currentUrl || null;
14
+ this.sessionMode = targetInfo?.sessionMode || null;
15
+ this.targetKind = targetInfo?.platform === "browser" ? "browser" : "mobile";
9
16
  this.messages = [];
10
17
  this.previousResponseId = null;
11
18
  this.transcript = [];
@@ -54,4 +61,11 @@ export class Session {
54
61
  updateResponseId(responseId) {
55
62
  this.previousResponseId = responseId;
56
63
  }
64
+ /**
65
+ * Replace the active system prompt for the next turn and start a fresh response chain.
66
+ */
67
+ installRecoverySystemPrompt(prompt) {
68
+ this.messages = [{ role: "system", content: prompt }];
69
+ this.updateResponseId(null);
70
+ }
57
71
  }
@@ -0,0 +1,18 @@
1
+ export function buildActionEventMeta({ context, action, platform, payload = {}, extraPayload = {} }) {
2
+ return {
3
+ eventType: "tool_call",
4
+ actionType: action?.type,
5
+ runId: context?.runId,
6
+ stepId: context?.stepId,
7
+ instructionIndex: context?.instructionIndex,
8
+ payload: {
9
+ eventId: context?.eventId,
10
+ screenshotPath: context?.screenshotPath,
11
+ screenshotDataUrl: context?.screenshotDataUrl,
12
+ captureSource: context?.captureSource,
13
+ platform,
14
+ ...extraPayload,
15
+ ...payload,
16
+ },
17
+ };
18
+ }
@@ -1,6 +1,7 @@
1
1
  import { logger } from "../../utils/logger.js";
2
2
  import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
3
3
  import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
4
+ import { buildActionEventMeta } from "../action-event-meta.js";
4
5
  import { resolveScrollGesture } from "../scroll-gesture.js";
5
6
  import { execAdb } from "./tools.js";
6
7
  function adbShell(deviceId, command) {
@@ -33,19 +34,11 @@ function normalizeMobileKeypress(keys = []) {
33
34
  }
34
35
  export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
35
36
  const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
36
- const meta = (payload = {}) => ({
37
- eventType: 'tool_call',
38
- actionType: action?.type,
39
- runId: context?.runId,
40
- stepId: context?.stepId,
41
- instructionIndex: context?.instructionIndex,
42
- payload: {
43
- eventId: context?.eventId,
44
- screenshotPath: context?.screenshotPath,
45
- screenshotDataUrl: context?.screenshotDataUrl,
46
- platform: 'android',
47
- ...payload
48
- }
37
+ const meta = (payload = {}) => buildActionEventMeta({
38
+ context,
39
+ action,
40
+ platform: 'android',
41
+ payload
49
42
  });
50
43
  try {
51
44
  const { x, y, x1, y1, x2, y2, text, keys, path } = action;
@@ -1,6 +1,7 @@
1
1
  import { logger } from "../../utils/logger.js";
2
2
  import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
3
3
  import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
4
+ import { buildActionEventMeta } from "../action-event-meta.js";
4
5
  import { getActiveSession, getDevicePixelRatio } from "./connection.js";
5
6
  import { resolveScrollGesture } from "../scroll-gesture.js";
6
7
  function normalizeMobileKeypress(platform, keys = []) {
@@ -47,20 +48,14 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
47
48
  const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
48
49
  const session = getActiveSession();
49
50
  const platform = session?.platform;
50
- const meta = (payload = {}) => ({
51
- eventType: "tool_call",
52
- actionType: action?.type,
53
- runId: context?.runId,
54
- stepId: context?.stepId,
55
- instructionIndex: context?.instructionIndex,
56
- payload: {
57
- eventId: context?.eventId,
58
- screenshotPath: context?.screenshotPath,
59
- screenshotDataUrl: context?.screenshotDataUrl,
60
- platform,
61
- source: session?.providerId,
62
- ...payload
63
- }
51
+ const meta = (payload = {}) => buildActionEventMeta({
52
+ context,
53
+ action,
54
+ platform,
55
+ extraPayload: {
56
+ source: session?.providerId
57
+ },
58
+ payload
64
59
  });
65
60
  if (!session || !platform) {
66
61
  throw new Error("No active cloud session");
@@ -103,6 +103,14 @@ export function getActiveSession() {
103
103
  export function getDevicePixelRatio() {
104
104
  return activeSession?.devicePixelRatio || 1;
105
105
  }
106
+ // Test-only — do not call from production code. Used by unit tests to install
107
+ // a fake session without going through the real cloud-provider connect flow.
108
+ export function setActiveSessionForTests(session) {
109
+ activeSession = session;
110
+ }
111
+ export function clearActiveSessionForTests() {
112
+ activeSession = null;
113
+ }
106
114
  export async function disconnect() {
107
115
  if (!activeSession) {
108
116
  return;
@@ -4,6 +4,7 @@
4
4
  * Executes CUA model actions on iOS Simulator via Appium.
5
5
  */
6
6
  import * as appium from "./appium-client.js";
7
+ import { buildActionEventMeta } from "../action-event-meta.js";
7
8
  import { getActiveSession, getDevicePixelRatio } from "./connection.js";
8
9
  import { logger } from "../../utils/logger.js";
9
10
  import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
@@ -44,19 +45,11 @@ function normalizeMobileKeypress(keys = []) {
44
45
  export async function handleModelAction(simulatorId, action, scale = 1.0, context = null) {
45
46
  const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
46
47
  const session = getActiveSession();
47
- const meta = (payload = {}) => ({
48
- eventType: "tool_call",
49
- actionType: action?.type,
50
- runId: context?.runId,
51
- stepId: context?.stepId,
52
- instructionIndex: context?.instructionIndex,
53
- payload: {
54
- eventId: context?.eventId,
55
- screenshotPath: context?.screenshotPath,
56
- screenshotDataUrl: context?.screenshotDataUrl,
57
- platform: "ios",
58
- ...payload
59
- }
48
+ const meta = (payload = {}) => buildActionEventMeta({
49
+ context,
50
+ action,
51
+ platform: "ios",
52
+ payload
60
53
  });
61
54
  if (!session) {
62
55
  throw new Error("No active iOS session");
@@ -299,6 +299,14 @@ export function getActiveSession() {
299
299
  export function getDevicePixelRatio() {
300
300
  return activeSession?.devicePixelRatio || 3;
301
301
  }
302
+ // Test-only — do not call from production code. Used by unit tests to install
303
+ // a fake session without going through the real Appium/simulator boot flow.
304
+ export function setActiveSessionForTests(session) {
305
+ activeSession = session;
306
+ }
307
+ export function clearActiveSessionForTests() {
308
+ activeSession = null;
309
+ }
302
310
  /**
303
311
  * Disconnect and cleanup
304
312
  */
@@ -3,8 +3,8 @@
3
3
  */
4
4
  import { executeLoadmillCommand } from "../integrations/loadmill/index.js";
5
5
  import { printCliOutput } from "../utils/console-output.js";
6
- function getLoadmillSiteBaseUrl() {
7
- const rawBaseUrl = process.env.LOADMILL_BASE_URL || "https://app.loadmill.com/api";
6
+ function getLoadmillSiteBaseUrl(loadmillClient = null) {
7
+ const rawBaseUrl = loadmillClient?.getBaseUrl?.() || "https://app.loadmill.com/api";
8
8
  return rawBaseUrl.replace(/\/api\/?$/, "");
9
9
  }
10
10
  /**
@@ -45,6 +45,8 @@ export function extractLoadmillCommand(userInput) {
45
45
  */
46
46
  export async function executeLoadmillInstruction(command, isHeadlessMode, context, stepContext = null) {
47
47
  const addOutput = context?.addOutput || printCliOutput;
48
+ const loadmillClient = context?.loadmillClient || null;
49
+ const openAiClient = context?.openAiClient || null;
48
50
  const meta = {
49
51
  runId: context?.runId,
50
52
  stepId: stepContext?.stepId,
@@ -54,14 +56,16 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
54
56
  type: 'info',
55
57
  text: `[Loadmill] Executing: ${command}`,
56
58
  eventType: 'system_message',
57
- payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() },
59
+ payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) },
58
60
  ...meta
59
61
  });
60
62
  const result = await executeLoadmillCommand(command, {
63
+ loadmillClient,
64
+ openAiClient,
61
65
  onProgress: ({ message, runId }) => {
62
66
  const payload = runId
63
- ? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
64
- : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
67
+ ? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
68
+ : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
65
69
  addOutput({ type: 'info', text: `[Loadmill] ${message}`, eventType: 'system_message', payload, ...meta });
66
70
  }
67
71
  });
@@ -84,14 +88,15 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
84
88
  */
85
89
  export async function handleLoadmillFailure(command, error, isHeadlessMode, context, stepContext = null, suiteRunId = null) {
86
90
  const addOutput = context?.addOutput || printCliOutput;
91
+ const loadmillClient = context?.loadmillClient || null;
87
92
  const meta = {
88
93
  runId: context?.runId,
89
94
  stepId: stepContext?.stepId,
90
95
  instructionIndex: stepContext?.instructionIndex
91
96
  };
92
97
  const payload = suiteRunId
93
- ? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
94
- : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
98
+ ? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
99
+ : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
95
100
  addOutput({ type: 'error', text: '[Loadmill] FAILED', eventType: 'error', payload, ...meta });
96
101
  addOutput({ type: 'error', text: `Command: ${command}`, eventType: 'error', payload, ...meta });
97
102
  addOutput({ type: 'error', text: `Error: ${error}`, eventType: 'error', payload, ...meta });
@@ -148,14 +153,15 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
148
153
  */
149
154
  export function handleLoadmillSuccess(command, result, context, stepContext = null) {
150
155
  const addOutput = context?.addOutput || printCliOutput;
156
+ const loadmillClient = context?.loadmillClient || null;
151
157
  const meta = {
152
158
  runId: context?.runId,
153
159
  stepId: stepContext?.stepId,
154
160
  instructionIndex: stepContext?.instructionIndex
155
161
  };
156
162
  const payload = result.runId
157
- ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
158
- : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
163
+ ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
164
+ : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
159
165
  if (result.action === "search") {
160
166
  addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)`, eventType: 'system_message', payload, ...meta });
161
167
  result.result.flows.forEach((flow, i) => {
@@ -1,15 +1,9 @@
1
- import OpenAI from "openai";
2
- import { readContextOptimizationEnv } from "../core/context-optimization.js";
3
1
  import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
4
- import { buildAppContextCompactionInput } from "../prompts/execution.js";
2
+ import { createOpenAiClient, resolveRuntimeConfig } from "../core/runtime-config.js";
3
+ import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
5
4
  import { logger } from "../utils/logger.js";
6
5
  import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
7
- let openai = null;
8
6
  const cuaDebugTracer = new CuaDebugTracer(logger);
9
- const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
10
- function getSelectedCuaModel() {
11
- return VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL) ? process.env.OPENAI_CUA_MODEL : "gpt-5.4";
12
- }
13
7
  function normalizeMessageContent(content) {
14
8
  if (Array.isArray(content)) {
15
9
  return content.map((item) => {
@@ -90,37 +84,14 @@ export function buildCuaInput({ messages, screenshotBase64, previousResponseId,
90
84
  }
91
85
  return input;
92
86
  }
93
- function buildContextManagement(cuaModel) {
94
- if (cuaModel === "computer-use-preview") {
95
- return undefined;
96
- }
97
- const { enabled, threshold } = readContextOptimizationEnv();
98
- if (!enabled) {
87
+ function buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold) {
88
+ if (!contextOptimizationEnabled) {
99
89
  return undefined;
100
90
  }
101
- return [{ type: "compaction", compact_threshold: threshold }];
91
+ return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
102
92
  }
103
- export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
104
- const contextManagement = buildContextManagement(cuaModel);
105
- if (cuaModel === "computer-use-preview") {
106
- return {
107
- model: cuaModel,
108
- previous_response_id: previousResponseId || undefined,
109
- input,
110
- store: true,
111
- truncation: "auto",
112
- tools: [
113
- {
114
- type: "computer_use_preview",
115
- display_width: deviceInfo.scaled_width,
116
- display_height: deviceInfo.scaled_height,
117
- environment: "browser",
118
- },
119
- previewClickTool,
120
- ],
121
- reasoning: { summary: "concise" },
122
- };
123
- }
93
+ export function buildCuaRequestParams({ cuaModel, previousResponseId, input, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, }) {
94
+ const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
124
95
  return {
125
96
  model: cuaModel,
126
97
  reasoning: { effort: "medium" },
@@ -130,97 +101,23 @@ export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo
130
101
  store: true,
131
102
  truncation: "auto",
132
103
  tools: [
133
- {
134
- type: "computer",
135
- },
104
+ { type: "computer" },
136
105
  previewClickTool,
137
106
  ],
138
107
  };
139
108
  }
140
- function shouldMapPreviewAccessError(err) {
141
- const status = err?.status;
142
- const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
143
- const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
144
- const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
145
- if (status === 403 || status === 404)
146
- return true;
147
- if (code.includes("model_not_found") || code.includes("permission"))
148
- return true;
149
- if (type.includes("permission"))
150
- return true;
151
- return (message.includes("computer-use-preview") &&
152
- (message.includes("access") || message.includes("permission") || message.includes("not found") || message.includes("unsupported")));
153
- }
154
- export function shouldMapMiniCompatibilityError(err) {
155
- const status = Number(err?.status);
156
- const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
157
- const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
158
- const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
159
- const mentionsMiniModel = message.includes("gpt-5.4-mini");
160
- const mentionsComputerTool = message.includes("computer") || message.includes("tool");
161
- const mentionsUnsupported = (message.includes("unsupported") ||
162
- message.includes("not supported") ||
163
- message.includes("does not support"));
164
- if (code.includes("model_not_found")) {
165
- return true;
166
- }
167
- if (status === 404) {
168
- return mentionsMiniModel && (message.includes("not found") || mentionsUnsupported);
169
- }
170
- if (code.includes("unsupported")) {
171
- return mentionsMiniModel || mentionsComputerTool;
172
- }
173
- if (!type.includes("invalid")) {
174
- return false;
175
- }
176
- return mentionsUnsupported && (mentionsMiniModel || mentionsComputerTool);
177
- }
178
- function mapCuaError(err, cuaModel) {
179
- if (cuaModel === "computer-use-preview" && shouldMapPreviewAccessError(err)) {
180
- const mapped = new Error("OpenAI API key does not have access to computer-use-preview. Switch to gpt-5.4 in Settings > CUA Model.");
181
- mapped.cause = err;
182
- return mapped;
183
- }
184
- if (cuaModel === "gpt-5.4-mini" && shouldMapMiniCompatibilityError(err)) {
185
- const mapped = new Error("OpenAI rejected gpt-5.4-mini for computer-use execution. Switch to gpt-5.4 or computer-use-preview in Settings > CUA Model.");
186
- mapped.cause = err;
187
- return mapped;
188
- }
189
- return err;
190
- }
191
- export function isNonRetryableCuaError(err) {
192
- const status = Number(err?.status);
193
- const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
194
- const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
195
- const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
196
- if ([401, 403, 404].includes(status)) {
197
- return true;
198
- }
199
- return (code.includes("model_not_found") ||
200
- code.includes("permission") ||
201
- type.includes("permission") ||
202
- message.includes("does not have access to computer-use-preview") ||
203
- message.includes("switch to gpt-5.4 in settings > cua model") ||
204
- message.includes("openai rejected gpt-5.4-mini for computer-use execution") ||
205
- message.includes("switch to gpt-5.4 or computer-use-preview in settings > cua model"));
206
- }
207
- function getOpenAI() {
208
- if (!openai) {
209
- openai = new OpenAI({
210
- apiKey: process.env.OPENAI_API_KEY,
211
- });
212
- }
213
- return openai;
214
- }
215
109
  /**
216
110
  * Revise a test script based on user feedback using simple chat completion
217
- * @param {string} originalScript - The original test script
218
- * @param {string} revisionRequest - User's requested changes
111
+ * @param {{
112
+ * client: import("openai").default,
113
+ * originalScript: string,
114
+ * revisionRequest: string,
115
+ * }} options
219
116
  * @returns {Promise<string>} - The revised test script
220
117
  */
221
- export async function reviseTestScript(originalScript, revisionRequest) {
222
- const response = await getOpenAI().chat.completions.create({
223
- model: "gpt-4o",
118
+ export async function reviseTestScript({ client, originalScript, revisionRequest }) {
119
+ const response = await client.chat.completions.create({
120
+ model: "gpt-5.4",
224
121
  messages: [{
225
122
  role: "system",
226
123
  content: buildTestRevisionSystemPrompt(originalScript, revisionRequest)
@@ -228,29 +125,28 @@ export async function reviseTestScript(originalScript, revisionRequest) {
228
125
  });
229
126
  return response.choices[0].message.content.trim();
230
127
  }
231
- export async function compactAppContext({ contextDocument, taskDescription, tokenBudget }) {
232
- const response = await getOpenAI().responses.create({
233
- model: "gpt-5.4",
234
- temperature: 0,
235
- input: buildAppContextCompactionInput({
236
- contextDocument,
237
- taskDescription,
238
- tokenBudget,
239
- })
240
- });
241
- return {
242
- briefing: typeof response.output_text === "string" ? response.output_text.trim() : "",
243
- outputTokens: typeof response.usage?.output_tokens === "number" ? response.usage.output_tokens : null,
244
- };
245
- }
246
- export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
247
- const cuaModel = getSelectedCuaModel();
128
+ /**
129
+ * @param {{
130
+ * client: import("openai").default,
131
+ * cuaModel: string,
132
+ * contextOptimizationEnabled?: boolean,
133
+ * contextOptimizationThreshold?: number,
134
+ * messages: Array<unknown>,
135
+ * screenshotBase64?: string,
136
+ * previousResponseId?: string | null,
137
+ * callId?: string | null,
138
+ * deviceInfo?: unknown,
139
+ * debugContext?: unknown,
140
+ * }} options
141
+ */
142
+ export async function sendCUARequest({ client, cuaModel, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
248
143
  const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
249
144
  const requestParams = buildCuaRequestParams({
250
145
  cuaModel,
251
146
  previousResponseId,
252
- deviceInfo,
253
147
  input,
148
+ contextOptimizationEnabled,
149
+ contextOptimizationThreshold,
254
150
  });
255
151
  const trace = cuaDebugTracer.startTurn({
256
152
  requestParams,
@@ -262,12 +158,44 @@ export async function sendCUARequest({ messages, screenshotBase64, previousRespo
262
158
  });
263
159
  logger.debug("CUA Request:", trace.requestLog);
264
160
  try {
265
- const response = await getOpenAI().responses.create(requestParams);
161
+ const response = await client.responses.create(requestParams);
266
162
  cuaDebugTracer.onResponse(trace, response);
267
163
  return response;
268
164
  }
269
165
  catch (err) {
270
166
  cuaDebugTracer.onError(trace, err);
271
- throw mapCuaError(err, cuaModel);
167
+ throw err;
272
168
  }
273
169
  }
170
+ /**
171
+ * @param {{
172
+ * client?: import("openai").default | null,
173
+ * config?: Record<string, unknown>,
174
+ * }} options
175
+ */
176
+ export function createOpenAiExecutor({ client, config }) {
177
+ const runtimeConfig = resolveRuntimeConfig(config);
178
+ const openAiClient = client || createOpenAiClient(null);
179
+ return {
180
+ client: openAiClient,
181
+ config: runtimeConfig,
182
+ async sendCUARequest(args) {
183
+ // Bind model and request-shaping config to the active run so execution
184
+ // stays deterministic even if settings change after the run starts.
185
+ return await sendCUARequest({
186
+ client: openAiClient,
187
+ cuaModel: runtimeConfig.cuaModel,
188
+ contextOptimizationEnabled: runtimeConfig.contextOptimizationEnabled,
189
+ contextOptimizationThreshold: runtimeConfig.contextOptimizationThreshold,
190
+ ...args,
191
+ });
192
+ },
193
+ async reviseTestScript(originalScript, revisionRequest) {
194
+ return await reviseTestScript({
195
+ client: openAiClient,
196
+ originalScript,
197
+ revisionRequest,
198
+ });
199
+ },
200
+ };
201
+ }