@loadmill/droid-cua 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/build/index.js +131 -74
- package/build/src/browser/actions.js +9 -0
- package/build/src/browser/connection.js +37 -0
- package/build/src/browser/factory.js +22 -0
- package/build/src/browser/interface.js +14 -0
- package/build/src/browser/navigation.js +11 -0
- package/build/src/browser/normalize.js +36 -0
- package/build/src/browser/playwright/actions.js +199 -0
- package/build/src/browser/playwright/connection.js +207 -0
- package/build/src/browser/playwright/installed-browser.js +131 -0
- package/build/src/browser/playwright/session-state.js +19 -0
- package/build/src/browser/playwright/url.js +79 -0
- package/build/src/cli/headless-execution-config.js +4 -36
- package/build/src/cli/headless-target.js +64 -0
- package/build/src/cli/ink-shell.js +5 -2
- package/build/src/commands/help.js +9 -4
- package/build/src/commands/loadmill.js +10 -1
- package/build/src/commands/run.js +17 -28
- package/build/src/core/app-context.js +57 -33
- package/build/src/core/capture-sources.js +76 -0
- package/build/src/core/context-optimization.js +1 -1
- package/build/src/core/cua-models.js +9 -0
- package/build/src/core/execution-engine.js +43 -19
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/runtime-config.js +96 -0
- package/build/src/core/session.js +18 -4
- package/build/src/device/action-event-meta.js +18 -0
- package/build/src/device/android/actions.js +6 -13
- package/build/src/device/cloud/actions.js +9 -14
- package/build/src/device/cloud/connection.js +8 -0
- package/build/src/device/ios/actions.js +6 -13
- package/build/src/device/ios/connection.js +8 -0
- package/build/src/device/loadmill.js +15 -9
- package/build/src/device/openai.js +67 -139
- package/build/src/integrations/loadmill/client.js +149 -131
- package/build/src/integrations/loadmill/executor.js +17 -11
- package/build/src/integrations/loadmill/index.js +1 -1
- package/build/src/integrations/loadmill/interpreter.js +4 -14
- package/build/src/modes/design-mode-ink.js +4 -3
- package/build/src/modes/design-mode.js +4 -3
- package/build/src/modes/execution-mode.js +139 -21
- package/build/src/prompts/base.js +92 -20
- package/build/src/prompts/design.js +6 -11
- package/build/src/prompts/execution.js +33 -109
- package/build/src/utils/console-output.js +8 -0
- package/build/src/utils/cua-debug-tracer.js +0 -1
- package/build/src/utils/run-screenshot-recorder.js +1 -1
- package/build/src/utils/structured-debug-log-manager.js +4 -1
- package/package.json +2 -1
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import OpenAI from "openai";
|
|
2
|
+
import { readContextOptimizationEnv, normalizeContextOptimizationEnabled, normalizeContextOptimizationThreshold, } from "./context-optimization.js";
|
|
3
|
+
import { resolveCuaModel } from "./cua-models.js";
|
|
4
|
+
import { normalizeStepDelayMs } from "../utils/step-delay.js";
|
|
5
|
+
export function createEmptyPromptCustomizations() {
|
|
6
|
+
return {
|
|
7
|
+
basePromptInstructions: "",
|
|
8
|
+
designModeInstructions: "",
|
|
9
|
+
executionModeInstructions: "",
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function resolvePromptCustomizations(rawValue = null) {
|
|
13
|
+
const resolved = createEmptyPromptCustomizations();
|
|
14
|
+
if (!rawValue || typeof rawValue !== "object" || Array.isArray(rawValue)) {
|
|
15
|
+
return resolved;
|
|
16
|
+
}
|
|
17
|
+
for (const key of Object.keys(resolved)) {
|
|
18
|
+
if (typeof rawValue[key] === "string") {
|
|
19
|
+
resolved[key] = rawValue[key];
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return resolved;
|
|
23
|
+
}
|
|
24
|
+
function trimNullableString(value) {
|
|
25
|
+
if (typeof value !== "string") {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const trimmed = value.trim();
|
|
29
|
+
return trimmed || null;
|
|
30
|
+
}
|
|
31
|
+
export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToken = null, loadmillBaseUrl = null, } = {}) {
|
|
32
|
+
return {
|
|
33
|
+
openAiApiKey: trimNullableString(openAiApiKey),
|
|
34
|
+
loadmillApiToken: trimNullableString(loadmillApiToken),
|
|
35
|
+
loadmillBaseUrl: trimNullableString(loadmillBaseUrl),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
export function resolveRuntimeConfig({ cuaModel = null, strictMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
|
|
39
|
+
return {
|
|
40
|
+
cuaModel: resolveCuaModel(cuaModel),
|
|
41
|
+
strictMode: strictMode === true,
|
|
42
|
+
stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
|
|
43
|
+
contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
|
|
44
|
+
contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),
|
|
45
|
+
promptCustomizations: resolvePromptCustomizations(promptCustomizations),
|
|
46
|
+
appContext: appContext && typeof appContext === "object"
|
|
47
|
+
? {
|
|
48
|
+
enabled: appContext.enabled !== false,
|
|
49
|
+
path: trimNullableString(appContext.path ?? appContext.contextPath),
|
|
50
|
+
status: typeof appContext.status === "string" ? appContext.status : null,
|
|
51
|
+
text: typeof appContext.text === "string" ? appContext.text : "",
|
|
52
|
+
message: trimNullableString(appContext.message),
|
|
53
|
+
}
|
|
54
|
+
: {
|
|
55
|
+
enabled: true,
|
|
56
|
+
path: null,
|
|
57
|
+
status: null,
|
|
58
|
+
text: "",
|
|
59
|
+
message: null,
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export function resolveRuntimeConfigFromEnv(env = process.env, overrides = {}) {
|
|
64
|
+
const contextOptimization = readContextOptimizationEnv(env);
|
|
65
|
+
return resolveRuntimeConfig({
|
|
66
|
+
cuaModel: env.OPENAI_CUA_MODEL,
|
|
67
|
+
stepDelayMs: env.DROID_CUA_STEP_DELAY_MS,
|
|
68
|
+
contextOptimizationEnabled: contextOptimization.enabled,
|
|
69
|
+
contextOptimizationThreshold: contextOptimization.threshold,
|
|
70
|
+
...overrides,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
export function resolveRuntimeCredentialsFromEnv(env = process.env, overrides = {}) {
|
|
74
|
+
return resolveRuntimeCredentials({
|
|
75
|
+
openAiApiKey: env.OPENAI_API_KEY,
|
|
76
|
+
loadmillApiToken: env.LOADMILL_API_TOKEN,
|
|
77
|
+
loadmillBaseUrl: env.LOADMILL_BASE_URL,
|
|
78
|
+
...overrides,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
export function createOpenAiClient(apiKey) {
|
|
82
|
+
return new OpenAI({
|
|
83
|
+
apiKey: trimNullableString(apiKey) || undefined,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
export function createRunContext({ config, credentials }) {
|
|
87
|
+
const resolvedConfig = resolveRuntimeConfig(config);
|
|
88
|
+
const resolvedCredentials = resolveRuntimeCredentials(credentials);
|
|
89
|
+
return {
|
|
90
|
+
config: resolvedConfig,
|
|
91
|
+
credentials: resolvedCredentials,
|
|
92
|
+
clients: {
|
|
93
|
+
openai: createOpenAiClient(resolvedCredentials.openAiApiKey),
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Session manages the state for a single CLI session
|
|
3
|
-
* Includes
|
|
3
|
+
* Includes target info, message history, transcript, and response chaining
|
|
4
4
|
*/
|
|
5
5
|
export class Session {
|
|
6
|
-
constructor(
|
|
7
|
-
this.deviceId =
|
|
8
|
-
this.deviceInfo =
|
|
6
|
+
constructor(targetSessionId, targetInfo) {
|
|
7
|
+
this.deviceId = targetSessionId;
|
|
8
|
+
this.deviceInfo = targetInfo;
|
|
9
|
+
this.deviceName = targetInfo?.device_name || null;
|
|
10
|
+
this.browserSessionId = targetInfo?.platform === "browser" ? targetSessionId : null;
|
|
11
|
+
this.browserInfo = targetInfo?.platform === "browser" ? targetInfo : null;
|
|
12
|
+
this.browserName = targetInfo?.browserName || null;
|
|
13
|
+
this.currentUrl = targetInfo?.currentUrl || null;
|
|
14
|
+
this.sessionMode = targetInfo?.sessionMode || null;
|
|
15
|
+
this.targetKind = targetInfo?.platform === "browser" ? "browser" : "mobile";
|
|
9
16
|
this.messages = [];
|
|
10
17
|
this.previousResponseId = null;
|
|
11
18
|
this.transcript = [];
|
|
@@ -54,4 +61,11 @@ export class Session {
|
|
|
54
61
|
updateResponseId(responseId) {
|
|
55
62
|
this.previousResponseId = responseId;
|
|
56
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Replace the active system prompt for the next turn and start a fresh response chain.
|
|
66
|
+
*/
|
|
67
|
+
installRecoverySystemPrompt(prompt) {
|
|
68
|
+
this.messages = [{ role: "system", content: prompt }];
|
|
69
|
+
this.updateResponseId(null);
|
|
70
|
+
}
|
|
57
71
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export function buildActionEventMeta({ context, action, platform, payload = {}, extraPayload = {} }) {
|
|
2
|
+
return {
|
|
3
|
+
eventType: "tool_call",
|
|
4
|
+
actionType: action?.type,
|
|
5
|
+
runId: context?.runId,
|
|
6
|
+
stepId: context?.stepId,
|
|
7
|
+
instructionIndex: context?.instructionIndex,
|
|
8
|
+
payload: {
|
|
9
|
+
eventId: context?.eventId,
|
|
10
|
+
screenshotPath: context?.screenshotPath,
|
|
11
|
+
screenshotDataUrl: context?.screenshotDataUrl,
|
|
12
|
+
captureSource: context?.captureSource,
|
|
13
|
+
platform,
|
|
14
|
+
...extraPayload,
|
|
15
|
+
...payload,
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { logger } from "../../utils/logger.js";
|
|
2
2
|
import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
|
|
3
3
|
import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
|
|
4
|
+
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
4
5
|
import { resolveScrollGesture } from "../scroll-gesture.js";
|
|
5
6
|
import { execAdb } from "./tools.js";
|
|
6
7
|
function adbShell(deviceId, command) {
|
|
@@ -33,19 +34,11 @@ function normalizeMobileKeypress(keys = []) {
|
|
|
33
34
|
}
|
|
34
35
|
export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
|
|
35
36
|
const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
|
|
36
|
-
const meta = (payload = {}) => ({
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
instructionIndex: context?.instructionIndex,
|
|
42
|
-
payload: {
|
|
43
|
-
eventId: context?.eventId,
|
|
44
|
-
screenshotPath: context?.screenshotPath,
|
|
45
|
-
screenshotDataUrl: context?.screenshotDataUrl,
|
|
46
|
-
platform: 'android',
|
|
47
|
-
...payload
|
|
48
|
-
}
|
|
37
|
+
const meta = (payload = {}) => buildActionEventMeta({
|
|
38
|
+
context,
|
|
39
|
+
action,
|
|
40
|
+
platform: 'android',
|
|
41
|
+
payload
|
|
49
42
|
});
|
|
50
43
|
try {
|
|
51
44
|
const { x, y, x1, y1, x2, y2, text, keys, path } = action;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { logger } from "../../utils/logger.js";
|
|
2
2
|
import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
|
|
3
3
|
import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
|
|
4
|
+
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
4
5
|
import { getActiveSession, getDevicePixelRatio } from "./connection.js";
|
|
5
6
|
import { resolveScrollGesture } from "../scroll-gesture.js";
|
|
6
7
|
function normalizeMobileKeypress(platform, keys = []) {
|
|
@@ -47,20 +48,14 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
47
48
|
const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
|
|
48
49
|
const session = getActiveSession();
|
|
49
50
|
const platform = session?.platform;
|
|
50
|
-
const meta = (payload = {}) => ({
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
screenshotPath: context?.screenshotPath,
|
|
59
|
-
screenshotDataUrl: context?.screenshotDataUrl,
|
|
60
|
-
platform,
|
|
61
|
-
source: session?.providerId,
|
|
62
|
-
...payload
|
|
63
|
-
}
|
|
51
|
+
const meta = (payload = {}) => buildActionEventMeta({
|
|
52
|
+
context,
|
|
53
|
+
action,
|
|
54
|
+
platform,
|
|
55
|
+
extraPayload: {
|
|
56
|
+
source: session?.providerId
|
|
57
|
+
},
|
|
58
|
+
payload
|
|
64
59
|
});
|
|
65
60
|
if (!session || !platform) {
|
|
66
61
|
throw new Error("No active cloud session");
|
|
@@ -103,6 +103,14 @@ export function getActiveSession() {
|
|
|
103
103
|
export function getDevicePixelRatio() {
|
|
104
104
|
return activeSession?.devicePixelRatio || 1;
|
|
105
105
|
}
|
|
106
|
+
// Test-only — do not call from production code. Used by unit tests to install
|
|
107
|
+
// a fake session without going through the real cloud-provider connect flow.
|
|
108
|
+
export function setActiveSessionForTests(session) {
|
|
109
|
+
activeSession = session;
|
|
110
|
+
}
|
|
111
|
+
export function clearActiveSessionForTests() {
|
|
112
|
+
activeSession = null;
|
|
113
|
+
}
|
|
106
114
|
export async function disconnect() {
|
|
107
115
|
if (!activeSession) {
|
|
108
116
|
return;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Executes CUA model actions on iOS Simulator via Appium.
|
|
5
5
|
*/
|
|
6
6
|
import * as appium from "./appium-client.js";
|
|
7
|
+
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
7
8
|
import { getActiveSession, getDevicePixelRatio } from "./connection.js";
|
|
8
9
|
import { logger } from "../../utils/logger.js";
|
|
9
10
|
import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
|
|
@@ -44,19 +45,11 @@ function normalizeMobileKeypress(keys = []) {
|
|
|
44
45
|
export async function handleModelAction(simulatorId, action, scale = 1.0, context = null) {
|
|
45
46
|
const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
|
|
46
47
|
const session = getActiveSession();
|
|
47
|
-
const meta = (payload = {}) => ({
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
instructionIndex: context?.instructionIndex,
|
|
53
|
-
payload: {
|
|
54
|
-
eventId: context?.eventId,
|
|
55
|
-
screenshotPath: context?.screenshotPath,
|
|
56
|
-
screenshotDataUrl: context?.screenshotDataUrl,
|
|
57
|
-
platform: "ios",
|
|
58
|
-
...payload
|
|
59
|
-
}
|
|
48
|
+
const meta = (payload = {}) => buildActionEventMeta({
|
|
49
|
+
context,
|
|
50
|
+
action,
|
|
51
|
+
platform: "ios",
|
|
52
|
+
payload
|
|
60
53
|
});
|
|
61
54
|
if (!session) {
|
|
62
55
|
throw new Error("No active iOS session");
|
|
@@ -299,6 +299,14 @@ export function getActiveSession() {
|
|
|
299
299
|
export function getDevicePixelRatio() {
|
|
300
300
|
return activeSession?.devicePixelRatio || 3;
|
|
301
301
|
}
|
|
302
|
+
// Test-only — do not call from production code. Used by unit tests to install
|
|
303
|
+
// a fake session without going through the real Appium/simulator boot flow.
|
|
304
|
+
export function setActiveSessionForTests(session) {
|
|
305
|
+
activeSession = session;
|
|
306
|
+
}
|
|
307
|
+
export function clearActiveSessionForTests() {
|
|
308
|
+
activeSession = null;
|
|
309
|
+
}
|
|
302
310
|
/**
|
|
303
311
|
* Disconnect and cleanup
|
|
304
312
|
*/
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { executeLoadmillCommand } from "../integrations/loadmill/index.js";
|
|
5
5
|
import { printCliOutput } from "../utils/console-output.js";
|
|
6
|
-
function getLoadmillSiteBaseUrl() {
|
|
7
|
-
const rawBaseUrl =
|
|
6
|
+
function getLoadmillSiteBaseUrl(loadmillClient = null) {
|
|
7
|
+
const rawBaseUrl = loadmillClient?.getBaseUrl?.() || "https://app.loadmill.com/api";
|
|
8
8
|
return rawBaseUrl.replace(/\/api\/?$/, "");
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
@@ -45,6 +45,8 @@ export function extractLoadmillCommand(userInput) {
|
|
|
45
45
|
*/
|
|
46
46
|
export async function executeLoadmillInstruction(command, isHeadlessMode, context, stepContext = null) {
|
|
47
47
|
const addOutput = context?.addOutput || printCliOutput;
|
|
48
|
+
const loadmillClient = context?.loadmillClient || null;
|
|
49
|
+
const openAiClient = context?.openAiClient || null;
|
|
48
50
|
const meta = {
|
|
49
51
|
runId: context?.runId,
|
|
50
52
|
stepId: stepContext?.stepId,
|
|
@@ -54,14 +56,16 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
|
|
|
54
56
|
type: 'info',
|
|
55
57
|
text: `[Loadmill] Executing: ${command}`,
|
|
56
58
|
eventType: 'system_message',
|
|
57
|
-
payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() },
|
|
59
|
+
payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) },
|
|
58
60
|
...meta
|
|
59
61
|
});
|
|
60
62
|
const result = await executeLoadmillCommand(command, {
|
|
63
|
+
loadmillClient,
|
|
64
|
+
openAiClient,
|
|
61
65
|
onProgress: ({ message, runId }) => {
|
|
62
66
|
const payload = runId
|
|
63
|
-
? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
|
|
64
|
-
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
|
|
67
|
+
? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
|
|
68
|
+
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
|
|
65
69
|
addOutput({ type: 'info', text: `[Loadmill] ${message}`, eventType: 'system_message', payload, ...meta });
|
|
66
70
|
}
|
|
67
71
|
});
|
|
@@ -84,14 +88,15 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
|
|
|
84
88
|
*/
|
|
85
89
|
export async function handleLoadmillFailure(command, error, isHeadlessMode, context, stepContext = null, suiteRunId = null) {
|
|
86
90
|
const addOutput = context?.addOutput || printCliOutput;
|
|
91
|
+
const loadmillClient = context?.loadmillClient || null;
|
|
87
92
|
const meta = {
|
|
88
93
|
runId: context?.runId,
|
|
89
94
|
stepId: stepContext?.stepId,
|
|
90
95
|
instructionIndex: stepContext?.instructionIndex
|
|
91
96
|
};
|
|
92
97
|
const payload = suiteRunId
|
|
93
|
-
? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
|
|
94
|
-
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
|
|
98
|
+
? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
|
|
99
|
+
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
|
|
95
100
|
addOutput({ type: 'error', text: '[Loadmill] FAILED', eventType: 'error', payload, ...meta });
|
|
96
101
|
addOutput({ type: 'error', text: `Command: ${command}`, eventType: 'error', payload, ...meta });
|
|
97
102
|
addOutput({ type: 'error', text: `Error: ${error}`, eventType: 'error', payload, ...meta });
|
|
@@ -148,14 +153,15 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
|
|
|
148
153
|
*/
|
|
149
154
|
export function handleLoadmillSuccess(command, result, context, stepContext = null) {
|
|
150
155
|
const addOutput = context?.addOutput || printCliOutput;
|
|
156
|
+
const loadmillClient = context?.loadmillClient || null;
|
|
151
157
|
const meta = {
|
|
152
158
|
runId: context?.runId,
|
|
153
159
|
stepId: stepContext?.stepId,
|
|
154
160
|
instructionIndex: stepContext?.instructionIndex
|
|
155
161
|
};
|
|
156
162
|
const payload = result.runId
|
|
157
|
-
? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
|
|
158
|
-
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
|
|
163
|
+
? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
|
|
164
|
+
: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
|
|
159
165
|
if (result.action === "search") {
|
|
160
166
|
addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)`, eventType: 'system_message', payload, ...meta });
|
|
161
167
|
result.result.flows.forEach((flow, i) => {
|
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
import OpenAI from "openai";
|
|
2
|
-
import { readContextOptimizationEnv } from "../core/context-optimization.js";
|
|
3
1
|
import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
|
|
4
|
-
import {
|
|
2
|
+
import { createOpenAiClient, resolveRuntimeConfig } from "../core/runtime-config.js";
|
|
3
|
+
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
|
|
5
4
|
import { logger } from "../utils/logger.js";
|
|
6
5
|
import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
|
|
7
|
-
let openai = null;
|
|
8
6
|
const cuaDebugTracer = new CuaDebugTracer(logger);
|
|
9
|
-
const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
|
|
10
|
-
function getSelectedCuaModel() {
|
|
11
|
-
return VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL) ? process.env.OPENAI_CUA_MODEL : "gpt-5.4";
|
|
12
|
-
}
|
|
13
7
|
function normalizeMessageContent(content) {
|
|
14
8
|
if (Array.isArray(content)) {
|
|
15
9
|
return content.map((item) => {
|
|
@@ -90,37 +84,14 @@ export function buildCuaInput({ messages, screenshotBase64, previousResponseId,
|
|
|
90
84
|
}
|
|
91
85
|
return input;
|
|
92
86
|
}
|
|
93
|
-
function buildContextManagement(
|
|
94
|
-
if (
|
|
95
|
-
return undefined;
|
|
96
|
-
}
|
|
97
|
-
const { enabled, threshold } = readContextOptimizationEnv();
|
|
98
|
-
if (!enabled) {
|
|
87
|
+
function buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold) {
|
|
88
|
+
if (!contextOptimizationEnabled) {
|
|
99
89
|
return undefined;
|
|
100
90
|
}
|
|
101
|
-
return [{ type: "compaction", compact_threshold:
|
|
91
|
+
return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
|
|
102
92
|
}
|
|
103
|
-
export function buildCuaRequestParams({ cuaModel, previousResponseId,
|
|
104
|
-
const contextManagement = buildContextManagement(
|
|
105
|
-
if (cuaModel === "computer-use-preview") {
|
|
106
|
-
return {
|
|
107
|
-
model: cuaModel,
|
|
108
|
-
previous_response_id: previousResponseId || undefined,
|
|
109
|
-
input,
|
|
110
|
-
store: true,
|
|
111
|
-
truncation: "auto",
|
|
112
|
-
tools: [
|
|
113
|
-
{
|
|
114
|
-
type: "computer_use_preview",
|
|
115
|
-
display_width: deviceInfo.scaled_width,
|
|
116
|
-
display_height: deviceInfo.scaled_height,
|
|
117
|
-
environment: "browser",
|
|
118
|
-
},
|
|
119
|
-
previewClickTool,
|
|
120
|
-
],
|
|
121
|
-
reasoning: { summary: "concise" },
|
|
122
|
-
};
|
|
123
|
-
}
|
|
93
|
+
export function buildCuaRequestParams({ cuaModel, previousResponseId, input, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, }) {
|
|
94
|
+
const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
|
|
124
95
|
return {
|
|
125
96
|
model: cuaModel,
|
|
126
97
|
reasoning: { effort: "medium" },
|
|
@@ -130,97 +101,23 @@ export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo
|
|
|
130
101
|
store: true,
|
|
131
102
|
truncation: "auto",
|
|
132
103
|
tools: [
|
|
133
|
-
{
|
|
134
|
-
type: "computer",
|
|
135
|
-
},
|
|
104
|
+
{ type: "computer" },
|
|
136
105
|
previewClickTool,
|
|
137
106
|
],
|
|
138
107
|
};
|
|
139
108
|
}
|
|
140
|
-
function shouldMapPreviewAccessError(err) {
|
|
141
|
-
const status = err?.status;
|
|
142
|
-
const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
|
|
143
|
-
const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
|
|
144
|
-
const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
|
|
145
|
-
if (status === 403 || status === 404)
|
|
146
|
-
return true;
|
|
147
|
-
if (code.includes("model_not_found") || code.includes("permission"))
|
|
148
|
-
return true;
|
|
149
|
-
if (type.includes("permission"))
|
|
150
|
-
return true;
|
|
151
|
-
return (message.includes("computer-use-preview") &&
|
|
152
|
-
(message.includes("access") || message.includes("permission") || message.includes("not found") || message.includes("unsupported")));
|
|
153
|
-
}
|
|
154
|
-
export function shouldMapMiniCompatibilityError(err) {
|
|
155
|
-
const status = Number(err?.status);
|
|
156
|
-
const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
|
|
157
|
-
const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
|
|
158
|
-
const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
|
|
159
|
-
const mentionsMiniModel = message.includes("gpt-5.4-mini");
|
|
160
|
-
const mentionsComputerTool = message.includes("computer") || message.includes("tool");
|
|
161
|
-
const mentionsUnsupported = (message.includes("unsupported") ||
|
|
162
|
-
message.includes("not supported") ||
|
|
163
|
-
message.includes("does not support"));
|
|
164
|
-
if (code.includes("model_not_found")) {
|
|
165
|
-
return true;
|
|
166
|
-
}
|
|
167
|
-
if (status === 404) {
|
|
168
|
-
return mentionsMiniModel && (message.includes("not found") || mentionsUnsupported);
|
|
169
|
-
}
|
|
170
|
-
if (code.includes("unsupported")) {
|
|
171
|
-
return mentionsMiniModel || mentionsComputerTool;
|
|
172
|
-
}
|
|
173
|
-
if (!type.includes("invalid")) {
|
|
174
|
-
return false;
|
|
175
|
-
}
|
|
176
|
-
return mentionsUnsupported && (mentionsMiniModel || mentionsComputerTool);
|
|
177
|
-
}
|
|
178
|
-
function mapCuaError(err, cuaModel) {
|
|
179
|
-
if (cuaModel === "computer-use-preview" && shouldMapPreviewAccessError(err)) {
|
|
180
|
-
const mapped = new Error("OpenAI API key does not have access to computer-use-preview. Switch to gpt-5.4 in Settings > CUA Model.");
|
|
181
|
-
mapped.cause = err;
|
|
182
|
-
return mapped;
|
|
183
|
-
}
|
|
184
|
-
if (cuaModel === "gpt-5.4-mini" && shouldMapMiniCompatibilityError(err)) {
|
|
185
|
-
const mapped = new Error("OpenAI rejected gpt-5.4-mini for computer-use execution. Switch to gpt-5.4 or computer-use-preview in Settings > CUA Model.");
|
|
186
|
-
mapped.cause = err;
|
|
187
|
-
return mapped;
|
|
188
|
-
}
|
|
189
|
-
return err;
|
|
190
|
-
}
|
|
191
|
-
export function isNonRetryableCuaError(err) {
|
|
192
|
-
const status = Number(err?.status);
|
|
193
|
-
const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
|
|
194
|
-
const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
|
|
195
|
-
const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
|
|
196
|
-
if ([401, 403, 404].includes(status)) {
|
|
197
|
-
return true;
|
|
198
|
-
}
|
|
199
|
-
return (code.includes("model_not_found") ||
|
|
200
|
-
code.includes("permission") ||
|
|
201
|
-
type.includes("permission") ||
|
|
202
|
-
message.includes("does not have access to computer-use-preview") ||
|
|
203
|
-
message.includes("switch to gpt-5.4 in settings > cua model") ||
|
|
204
|
-
message.includes("openai rejected gpt-5.4-mini for computer-use execution") ||
|
|
205
|
-
message.includes("switch to gpt-5.4 or computer-use-preview in settings > cua model"));
|
|
206
|
-
}
|
|
207
|
-
function getOpenAI() {
|
|
208
|
-
if (!openai) {
|
|
209
|
-
openai = new OpenAI({
|
|
210
|
-
apiKey: process.env.OPENAI_API_KEY,
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
return openai;
|
|
214
|
-
}
|
|
215
109
|
/**
|
|
216
110
|
* Revise a test script based on user feedback using simple chat completion
|
|
217
|
-
* @param {
|
|
218
|
-
*
|
|
111
|
+
* @param {{
|
|
112
|
+
* client: import("openai").default,
|
|
113
|
+
* originalScript: string,
|
|
114
|
+
* revisionRequest: string,
|
|
115
|
+
* }} options
|
|
219
116
|
* @returns {Promise<string>} - The revised test script
|
|
220
117
|
*/
|
|
221
|
-
export async function reviseTestScript(originalScript, revisionRequest) {
|
|
222
|
-
const response = await
|
|
223
|
-
model: "gpt-
|
|
118
|
+
export async function reviseTestScript({ client, originalScript, revisionRequest }) {
|
|
119
|
+
const response = await client.chat.completions.create({
|
|
120
|
+
model: "gpt-5.4",
|
|
224
121
|
messages: [{
|
|
225
122
|
role: "system",
|
|
226
123
|
content: buildTestRevisionSystemPrompt(originalScript, revisionRequest)
|
|
@@ -228,29 +125,28 @@ export async function reviseTestScript(originalScript, revisionRequest) {
|
|
|
228
125
|
});
|
|
229
126
|
return response.choices[0].message.content.trim();
|
|
230
127
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
246
|
-
export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
|
|
247
|
-
const cuaModel = getSelectedCuaModel();
|
|
128
|
+
/**
|
|
129
|
+
* @param {{
|
|
130
|
+
* client: import("openai").default,
|
|
131
|
+
* cuaModel: string,
|
|
132
|
+
* contextOptimizationEnabled?: boolean,
|
|
133
|
+
* contextOptimizationThreshold?: number,
|
|
134
|
+
* messages: Array<unknown>,
|
|
135
|
+
* screenshotBase64?: string,
|
|
136
|
+
* previousResponseId?: string | null,
|
|
137
|
+
* callId?: string | null,
|
|
138
|
+
* deviceInfo?: unknown,
|
|
139
|
+
* debugContext?: unknown,
|
|
140
|
+
* }} options
|
|
141
|
+
*/
|
|
142
|
+
export async function sendCUARequest({ client, cuaModel, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
|
|
248
143
|
const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
|
|
249
144
|
const requestParams = buildCuaRequestParams({
|
|
250
145
|
cuaModel,
|
|
251
146
|
previousResponseId,
|
|
252
|
-
deviceInfo,
|
|
253
147
|
input,
|
|
148
|
+
contextOptimizationEnabled,
|
|
149
|
+
contextOptimizationThreshold,
|
|
254
150
|
});
|
|
255
151
|
const trace = cuaDebugTracer.startTurn({
|
|
256
152
|
requestParams,
|
|
@@ -262,12 +158,44 @@ export async function sendCUARequest({ messages, screenshotBase64, previousRespo
|
|
|
262
158
|
});
|
|
263
159
|
logger.debug("CUA Request:", trace.requestLog);
|
|
264
160
|
try {
|
|
265
|
-
const response = await
|
|
161
|
+
const response = await client.responses.create(requestParams);
|
|
266
162
|
cuaDebugTracer.onResponse(trace, response);
|
|
267
163
|
return response;
|
|
268
164
|
}
|
|
269
165
|
catch (err) {
|
|
270
166
|
cuaDebugTracer.onError(trace, err);
|
|
271
|
-
throw
|
|
167
|
+
throw err;
|
|
272
168
|
}
|
|
273
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* @param {{
|
|
172
|
+
* client?: import("openai").default | null,
|
|
173
|
+
* config?: Record<string, unknown>,
|
|
174
|
+
* }} options
|
|
175
|
+
*/
|
|
176
|
+
export function createOpenAiExecutor({ client, config }) {
|
|
177
|
+
const runtimeConfig = resolveRuntimeConfig(config);
|
|
178
|
+
const openAiClient = client || createOpenAiClient(null);
|
|
179
|
+
return {
|
|
180
|
+
client: openAiClient,
|
|
181
|
+
config: runtimeConfig,
|
|
182
|
+
async sendCUARequest(args) {
|
|
183
|
+
// Bind model and request-shaping config to the active run so execution
|
|
184
|
+
// stays deterministic even if settings change after the run starts.
|
|
185
|
+
return await sendCUARequest({
|
|
186
|
+
client: openAiClient,
|
|
187
|
+
cuaModel: runtimeConfig.cuaModel,
|
|
188
|
+
contextOptimizationEnabled: runtimeConfig.contextOptimizationEnabled,
|
|
189
|
+
contextOptimizationThreshold: runtimeConfig.contextOptimizationThreshold,
|
|
190
|
+
...args,
|
|
191
|
+
});
|
|
192
|
+
},
|
|
193
|
+
async reviseTestScript(originalScript, revisionRequest) {
|
|
194
|
+
return await reviseTestScript({
|
|
195
|
+
client: openAiClient,
|
|
196
|
+
originalScript,
|
|
197
|
+
revisionRequest,
|
|
198
|
+
});
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
}
|