@loadmill/droid-cua 2.22.0 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +31 -1
- package/build/src/browser/playwright/actions.js +65 -1
- package/build/src/commands/help.js +1 -1
- package/build/src/core/cua-models.js +11 -4
- package/build/src/core/execution-engine.js +334 -5
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/report/render-html-report.js +8 -3
- package/build/src/core/report/run-log-model.js +42 -0
- package/build/src/core/runtime-config.js +24 -39
- package/build/src/core/session.js +6 -0
- package/build/src/core/tool-handlers/loadmill.js +3 -0
- package/build/src/core/tool-handlers/test-design-complete.js +1 -0
- package/build/src/core/tool-handlers/type-sensitive-text.js +1 -0
- package/build/src/device/android/actions.js +6 -2
- package/build/src/device/android/connection.js +58 -23
- package/build/src/device/assertions.js +15 -8
- package/build/src/device/ios/connection.js +36 -30
- package/build/src/device/openai.js +68 -2
- package/build/src/device/scroll-gesture.js +22 -9
- package/build/src/integrations/loadmill/client.js +20 -4
- package/build/src/modes/design-mode-ink.js +10 -1
- package/build/src/modes/design-mode.js +7 -1
- package/build/src/modes/execution-mode.js +84 -8
- package/build/src/prompts/design.js +21 -1
- package/build/src/utils/console-output.js +5 -0
- package/build/src/utils/cua-debug-tracer.js +50 -7
- package/package.json +1 -1
- package/build/src/core/cua-backends.js +0 -11
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
|
|
2
2
|
import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
|
|
3
|
+
import { LOADMILL_CUA_MODELS } from "../core/cua-models.js";
|
|
3
4
|
import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
|
|
4
5
|
import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
|
|
5
6
|
import { logger } from "../utils/logger.js";
|
|
6
7
|
import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
|
|
7
8
|
const cuaDebugTracer = new CuaDebugTracer(logger);
|
|
9
|
+
const LOADMILL_DEVICE_METADATA_MODELS = new Set([
|
|
10
|
+
...LOADMILL_CUA_MODELS,
|
|
11
|
+
LOADMILL_MODEL_ALIASES.SMART,
|
|
12
|
+
LOADMILL_MODEL_ALIASES.BEACON,
|
|
13
|
+
]);
|
|
8
14
|
function resolveOpenAiBaseUrl(client) {
|
|
9
15
|
return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
|
|
10
16
|
}
|
|
@@ -121,6 +127,8 @@ const typeSensitiveTextTool = {
|
|
|
121
127
|
additionalProperties: false,
|
|
122
128
|
},
|
|
123
129
|
};
|
|
130
|
+
// Design completion is the handoff into script generation; it is separate from
|
|
131
|
+
// execution step-completion markers used by model adapters.
|
|
124
132
|
const testDesignCompleteTool = {
|
|
125
133
|
type: "function",
|
|
126
134
|
name: "test_design_complete",
|
|
@@ -235,7 +243,64 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
|
|
|
235
243
|
}
|
|
236
244
|
return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
|
|
237
245
|
}
|
|
238
|
-
|
|
246
|
+
function normalizeDevicePlatform(deviceInfo = {}) {
|
|
247
|
+
return typeof deviceInfo?.platform === "string"
|
|
248
|
+
? deviceInfo.platform.trim().toLowerCase()
|
|
249
|
+
: "";
|
|
250
|
+
}
|
|
251
|
+
function resolveComputerToolDeviceType(deviceInfo = {}) {
|
|
252
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
253
|
+
if (platform === "browser") {
|
|
254
|
+
return "web";
|
|
255
|
+
}
|
|
256
|
+
if (platform === "ios") {
|
|
257
|
+
return "iphone";
|
|
258
|
+
}
|
|
259
|
+
if (platform === "android") {
|
|
260
|
+
return "android";
|
|
261
|
+
}
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
function resolveComputerToolEnvironment(deviceInfo = {}) {
|
|
265
|
+
const platform = normalizeDevicePlatform(deviceInfo);
|
|
266
|
+
if (platform === "browser") {
|
|
267
|
+
return "browser";
|
|
268
|
+
}
|
|
269
|
+
if (platform === "ios" || platform === "android") {
|
|
270
|
+
return "mobile";
|
|
271
|
+
}
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
function resolveDisplayDimension(value) {
|
|
275
|
+
return Number.isFinite(value) && value > 0 ? Math.round(value) : null;
|
|
276
|
+
}
|
|
277
|
+
function usesLoadmillDeviceMetadata(cuaModel) {
|
|
278
|
+
return LOADMILL_DEVICE_METADATA_MODELS.has(cuaModel);
|
|
279
|
+
}
|
|
280
|
+
function buildComputerTool({ cuaModel, deviceInfo } = {}) {
|
|
281
|
+
const tool = { type: "computer" };
|
|
282
|
+
if (!usesLoadmillDeviceMetadata(cuaModel)) {
|
|
283
|
+
return tool;
|
|
284
|
+
}
|
|
285
|
+
const environment = resolveComputerToolEnvironment(deviceInfo);
|
|
286
|
+
const deviceType = resolveComputerToolDeviceType(deviceInfo);
|
|
287
|
+
const displayWidth = resolveDisplayDimension(deviceInfo?.scaled_width);
|
|
288
|
+
const displayHeight = resolveDisplayDimension(deviceInfo?.scaled_height);
|
|
289
|
+
if (environment) {
|
|
290
|
+
tool.environment = environment;
|
|
291
|
+
}
|
|
292
|
+
if (deviceType) {
|
|
293
|
+
tool.device_type = deviceType;
|
|
294
|
+
}
|
|
295
|
+
if (displayWidth) {
|
|
296
|
+
tool.display_width = displayWidth;
|
|
297
|
+
}
|
|
298
|
+
if (displayHeight) {
|
|
299
|
+
tool.display_height = displayHeight;
|
|
300
|
+
}
|
|
301
|
+
return tool;
|
|
302
|
+
}
|
|
303
|
+
export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, deviceInfo = null, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
|
|
239
304
|
const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
|
|
240
305
|
// Keep Responses API storage enabled on every turn so execution can chain
|
|
241
306
|
// reliably through previous_response_id across the whole run.
|
|
@@ -248,7 +313,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
|
|
|
248
313
|
store: true,
|
|
249
314
|
truncation: "auto",
|
|
250
315
|
tools: [
|
|
251
|
-
...(includeComputerTool ? [{
|
|
316
|
+
...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
|
|
252
317
|
...buildFunctionTools({ aimAssistMode, toolCapabilities }),
|
|
253
318
|
],
|
|
254
319
|
};
|
|
@@ -294,6 +359,7 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
|
|
|
294
359
|
aimAssistMode,
|
|
295
360
|
previousResponseId,
|
|
296
361
|
input,
|
|
362
|
+
deviceInfo,
|
|
297
363
|
includeComputerTool,
|
|
298
364
|
contextOptimizationEnabled,
|
|
299
365
|
contextOptimizationThreshold,
|
|
@@ -1,20 +1,33 @@
|
|
|
1
1
|
// The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
|
|
2
2
|
// anchor coordinates `x` / `y`. This helper converts those model coordinates
|
|
3
3
|
// into backend gesture coordinates: startX/startY and endX/endY.
|
|
4
|
-
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
|
|
4
|
+
export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0, minX, maxX, minY, maxY } = {}) {
|
|
5
|
+
const clamp = (value, min, max) => {
|
|
6
|
+
if (!Number.isFinite(value)) {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) {
|
|
10
|
+
return value;
|
|
11
|
+
}
|
|
12
|
+
return Math.min(Math.max(value, min), max);
|
|
13
|
+
};
|
|
5
14
|
const divisor = scale * dpr;
|
|
6
|
-
const
|
|
7
|
-
const
|
|
15
|
+
const requestedScrollX = Math.round((action?.scroll_x ?? 0) / divisor);
|
|
16
|
+
const requestedScrollY = Math.round((action?.scroll_y ?? 0) / divisor);
|
|
8
17
|
const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
|
|
9
|
-
const
|
|
10
|
-
const
|
|
18
|
+
const unclampedStartX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
|
|
19
|
+
const unclampedStartY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
|
|
20
|
+
const startX = clamp(unclampedStartX, minX, maxX);
|
|
21
|
+
const startY = clamp(unclampedStartY, minY, maxY);
|
|
22
|
+
const endX = clamp(startX + requestedScrollX, minX, maxX);
|
|
23
|
+
const endY = clamp(startY - requestedScrollY, minY, maxY);
|
|
11
24
|
return {
|
|
12
|
-
scrollX,
|
|
13
|
-
scrollY,
|
|
25
|
+
scrollX: endX - startX,
|
|
26
|
+
scrollY: startY - endY,
|
|
14
27
|
startX,
|
|
15
28
|
startY,
|
|
16
|
-
endX
|
|
17
|
-
endY
|
|
29
|
+
endX,
|
|
30
|
+
endY,
|
|
18
31
|
hasAnchor
|
|
19
32
|
};
|
|
20
33
|
}
|
|
@@ -138,10 +138,10 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
138
138
|
body: JSON.stringify(payload),
|
|
139
139
|
});
|
|
140
140
|
}
|
|
141
|
-
async function
|
|
141
|
+
async function validateApiToken() {
|
|
142
142
|
const resolvedToken = getApiToken();
|
|
143
143
|
if (!resolvedToken) {
|
|
144
|
-
return
|
|
144
|
+
return { valid: false, reason: "missing" };
|
|
145
145
|
}
|
|
146
146
|
const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
|
|
147
147
|
method: "GET",
|
|
@@ -150,11 +150,26 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
150
150
|
Authorization: `Bearer ${resolvedToken}`,
|
|
151
151
|
},
|
|
152
152
|
});
|
|
153
|
+
if (response.status === 401 || response.status === 403) {
|
|
154
|
+
return { valid: false, reason: "invalid", status: response.status };
|
|
155
|
+
}
|
|
153
156
|
if (!response.ok) {
|
|
154
|
-
return
|
|
157
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
const user = extractLoadmillSessionUser(await response.json());
|
|
161
|
+
return user
|
|
162
|
+
? { valid: true, user }
|
|
163
|
+
: { valid: false, reason: "unavailable", status: response.status };
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return { valid: false, reason: "unavailable", status: response.status };
|
|
155
167
|
}
|
|
168
|
+
}
|
|
169
|
+
async function getCurrentUser() {
|
|
156
170
|
try {
|
|
157
|
-
|
|
171
|
+
const validation = await validateApiToken();
|
|
172
|
+
return validation.valid ? validation.user : null;
|
|
158
173
|
}
|
|
159
174
|
catch {
|
|
160
175
|
return null;
|
|
@@ -172,6 +187,7 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
|
|
|
172
187
|
getTestSuiteFlowRun,
|
|
173
188
|
explainFailure,
|
|
174
189
|
reportDroidRun,
|
|
190
|
+
validateApiToken,
|
|
175
191
|
getCurrentUser,
|
|
176
192
|
};
|
|
177
193
|
}
|
|
@@ -206,7 +206,7 @@ export class DesignModeInk {
|
|
|
206
206
|
toolCapabilities: this.engine.toolCapabilities,
|
|
207
207
|
});
|
|
208
208
|
// Run turn with action tracking and interruption checking
|
|
209
|
-
const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
|
|
209
|
+
const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
|
|
210
210
|
// Check if user interrupted - stop immediately
|
|
211
211
|
if (this.inputQueue.length > 0) {
|
|
212
212
|
return true; // Signal to stop execution
|
|
@@ -223,6 +223,15 @@ export class DesignModeInk {
|
|
|
223
223
|
}
|
|
224
224
|
return false; // Continue execution
|
|
225
225
|
}, this.context, null, { currentScreenshot });
|
|
226
|
+
if (turnSummary.failed) {
|
|
227
|
+
if (this.context.setAgentWorking) {
|
|
228
|
+
this.context.setAgentWorking(false);
|
|
229
|
+
}
|
|
230
|
+
addOutput({ type: 'error', text: turnSummary.error || 'Design mode stopped.' });
|
|
231
|
+
this.conversationActive = false;
|
|
232
|
+
this.cleanup();
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
226
235
|
this.session.updateResponseId(newResponseId);
|
|
227
236
|
this.consecutiveErrorCount = 0;
|
|
228
237
|
// Clear agent working status
|
|
@@ -170,7 +170,7 @@ export class DesignMode {
|
|
|
170
170
|
toolCapabilities: this.engine.toolCapabilities,
|
|
171
171
|
});
|
|
172
172
|
// Run turn with action tracking and interruption checking
|
|
173
|
-
const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
|
|
173
|
+
const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, (action) => {
|
|
174
174
|
// Check if user pressed ESC - stop immediately
|
|
175
175
|
if (this.escPressed) {
|
|
176
176
|
return true; // Signal to stop execution
|
|
@@ -188,6 +188,12 @@ export class DesignMode {
|
|
|
188
188
|
}
|
|
189
189
|
return false; // Continue execution
|
|
190
190
|
}, null, null, { currentScreenshot });
|
|
191
|
+
if (turnSummary.failed) {
|
|
192
|
+
this.cleanupEscDetection(keypressHandler);
|
|
193
|
+
console.log(`\n${turnSummary.error || "Design mode stopped."}`);
|
|
194
|
+
this.conversationActive = false;
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
191
197
|
this.session.updateResponseId(newResponseId);
|
|
192
198
|
this.consecutiveErrorCount = 0;
|
|
193
199
|
// Cleanup ESC detection
|
|
@@ -4,15 +4,15 @@ import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurre
|
|
|
4
4
|
import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
|
|
5
5
|
import { sendCUARequest } from "../device/openai.js";
|
|
6
6
|
import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
|
|
7
|
-
import { isAssertion } from "../device/assertions.js";
|
|
7
|
+
import { extractAssertionPrompt, isAssertion } from "../device/assertions.js";
|
|
8
|
+
import { handleAssertionToolCall } from "../core/tool-handlers/assert.js";
|
|
8
9
|
import { logger } from "../utils/logger.js";
|
|
9
10
|
import { emitDesktopDebug } from "../utils/desktop-debug.js";
|
|
10
11
|
import { printCliOutput } from "../utils/console-output.js";
|
|
11
|
-
// Mirrors the codes emitted by the Loadmill LLM gateway (see
|
|
12
|
-
// in the server repo)
|
|
13
|
-
//
|
|
12
|
+
// Mirrors the quota codes emitted by the Loadmill LLM gateway (see
|
|
13
|
+
// services/app/private/llm in the server repo). Free plans run on the
|
|
14
|
+
// fixed-budget quota.
|
|
14
15
|
const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
|
|
15
|
-
"llm_rate_limit_exceeded",
|
|
16
16
|
"llm_day_quota_exceeded",
|
|
17
17
|
"llm_week_quota_exceeded",
|
|
18
18
|
"llm_fixed_budget_quota_exceeded",
|
|
@@ -21,6 +21,8 @@ const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
|
|
|
21
21
|
"llm_fixed_budget_token_quota_exceeded",
|
|
22
22
|
]);
|
|
23
23
|
const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
|
|
24
|
+
const LLM_REQUEST_RATE_LIMIT_ERROR_CODE = "llm_rate_limit_exceeded";
|
|
25
|
+
const DEFAULT_ASSERTION_CONTEXT = "No additional prior context is needed; validate directly from the current screen.";
|
|
24
26
|
function isLlmUsageLimitError(err) {
|
|
25
27
|
if (!err || typeof err !== "object") {
|
|
26
28
|
return false;
|
|
@@ -30,9 +32,15 @@ function isLlmUsageLimitError(err) {
|
|
|
30
32
|
}
|
|
31
33
|
return err.status === 429 && err.type === "rate_limit_error";
|
|
32
34
|
}
|
|
35
|
+
function isLlmRequestRateLimitError(err) {
|
|
36
|
+
return err && typeof err === "object" && err.code === LLM_REQUEST_RATE_LIMIT_ERROR_CODE;
|
|
37
|
+
}
|
|
33
38
|
function getLlmUsageLimitMessage() {
|
|
34
39
|
return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
|
|
35
40
|
}
|
|
41
|
+
function getLlmRequestRateLimitMessage() {
|
|
42
|
+
return "Execution stopped: too many AI requests are running for this account right now. Please retry shortly or reduce parallel test runs.";
|
|
43
|
+
}
|
|
36
44
|
/**
|
|
37
45
|
* Execution Mode - Run test scripts line-by-line
|
|
38
46
|
* Each instruction is executed in isolation (messages cleared after each turn)
|
|
@@ -105,10 +113,11 @@ export class ExecutionMode {
|
|
|
105
113
|
lines.push(` Retries: ${this.stats.retryCount}`);
|
|
106
114
|
return lines;
|
|
107
115
|
}
|
|
108
|
-
buildStepContext(instructionIndex) {
|
|
116
|
+
buildStepContext(instructionIndex, instruction = null) {
|
|
109
117
|
return {
|
|
110
118
|
instructionIndex,
|
|
111
119
|
stepId: `step-${String(instructionIndex + 1).padStart(4, "0")}`,
|
|
120
|
+
...(typeof instruction === "string" ? { instruction } : {}),
|
|
112
121
|
};
|
|
113
122
|
}
|
|
114
123
|
emit(addOutput, type, text, context = {}, stepContext = null, extra = {}) {
|
|
@@ -123,6 +132,45 @@ export class ExecutionMode {
|
|
|
123
132
|
payload: extra.payload
|
|
124
133
|
});
|
|
125
134
|
}
|
|
135
|
+
resetInstructionConversationState() {
|
|
136
|
+
this.session.updateResponseId(null);
|
|
137
|
+
if (typeof this.session.clearTranscript === "function") {
|
|
138
|
+
this.session.clearTranscript();
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
if (Array.isArray(this.session.transcript)) {
|
|
142
|
+
this.session.transcript = [];
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async executeVerificationInstruction(instruction, context, stepContext = null) {
|
|
146
|
+
const toolResult = await handleAssertionToolCall({
|
|
147
|
+
engine: this.engine,
|
|
148
|
+
callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
|
|
149
|
+
argumentsText: JSON.stringify({
|
|
150
|
+
condition: extractAssertionPrompt(instruction),
|
|
151
|
+
context: DEFAULT_ASSERTION_CONTEXT,
|
|
152
|
+
}),
|
|
153
|
+
context,
|
|
154
|
+
stepContext,
|
|
155
|
+
});
|
|
156
|
+
const assertionStatus = toolResult.summaryUpdates?.assertionStatus;
|
|
157
|
+
if (assertionStatus === "passed") {
|
|
158
|
+
this.stats.assertionsPassed++;
|
|
159
|
+
}
|
|
160
|
+
else if (assertionStatus === "failed" || assertionStatus === "failed_skipped") {
|
|
161
|
+
this.stats.assertionsFailed++;
|
|
162
|
+
}
|
|
163
|
+
if (toolResult.kind === "retry_instruction") {
|
|
164
|
+
this.stats.retryCount++;
|
|
165
|
+
this.session.clearMessages();
|
|
166
|
+
return await this.executeInstruction(instruction, context, 0, stepContext);
|
|
167
|
+
}
|
|
168
|
+
if (toolResult.kind === "fail") {
|
|
169
|
+
return { success: false, error: toolResult.error || "Assertion failed." };
|
|
170
|
+
}
|
|
171
|
+
this.session.clearMessages();
|
|
172
|
+
return { success: true };
|
|
173
|
+
}
|
|
126
174
|
/**
|
|
127
175
|
* Execute all instructions in the test script
|
|
128
176
|
* @param {Object} context - Additional context (Ink context with addOutput)
|
|
@@ -137,7 +185,8 @@ export class ExecutionMode {
|
|
|
137
185
|
// Start timing
|
|
138
186
|
this.stats.startTime = Date.now();
|
|
139
187
|
for (let i = 0; i < this.instructions.length; i++) {
|
|
140
|
-
const
|
|
188
|
+
const instruction = this.instructions[i];
|
|
189
|
+
const stepContext = this.buildStepContext(i, instruction);
|
|
141
190
|
// Check if execution should be stopped
|
|
142
191
|
if (this.shouldStop) {
|
|
143
192
|
this.emit(addOutput, 'info', 'Test execution stopped by user.', runContext, stepContext, {
|
|
@@ -158,7 +207,6 @@ export class ExecutionMode {
|
|
|
158
207
|
return { success: false, error: 'Stopped by user' };
|
|
159
208
|
}
|
|
160
209
|
}
|
|
161
|
-
const instruction = this.instructions[i];
|
|
162
210
|
this.emit(addOutput, 'user', instruction, runContext, stepContext, {
|
|
163
211
|
eventType: 'instruction_started',
|
|
164
212
|
payload: {
|
|
@@ -236,10 +284,17 @@ export class ExecutionMode {
|
|
|
236
284
|
const MAX_RETRIES = 10;
|
|
237
285
|
const instructionContext = {
|
|
238
286
|
...context,
|
|
287
|
+
currentInstruction: instruction,
|
|
239
288
|
futureInstructions: this.instructions.slice((stepContext?.instructionIndex ?? -1) + 1),
|
|
240
289
|
};
|
|
290
|
+
if (retryCount === 0) {
|
|
291
|
+
this.resetInstructionConversationState();
|
|
292
|
+
}
|
|
241
293
|
this.session.addToTranscript(`[User] ${instruction}`);
|
|
242
294
|
this.session.addMessage("user", instruction);
|
|
295
|
+
if (isAssertion(instruction)) {
|
|
296
|
+
return await this.executeVerificationInstruction(instruction, instructionContext, stepContext);
|
|
297
|
+
}
|
|
243
298
|
try {
|
|
244
299
|
const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
|
|
245
300
|
const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
|
|
@@ -309,6 +364,26 @@ export class ExecutionMode {
|
|
|
309
364
|
stack: err.stack
|
|
310
365
|
});
|
|
311
366
|
const addOutput = context.addOutput || printCliOutput;
|
|
367
|
+
if (isLlmRequestRateLimitError(err)) {
|
|
368
|
+
const stopMessage = getLlmRequestRateLimitMessage();
|
|
369
|
+
logger.debug('Stopping execution because an LLM request rate limit was reached', {
|
|
370
|
+
instruction,
|
|
371
|
+
message: err.message,
|
|
372
|
+
status: err.status,
|
|
373
|
+
code: err.code,
|
|
374
|
+
type: err.type,
|
|
375
|
+
});
|
|
376
|
+
this.emit(addOutput, 'error', stopMessage, context, stepContext, {
|
|
377
|
+
eventType: 'error',
|
|
378
|
+
payload: {
|
|
379
|
+
message: stopMessage,
|
|
380
|
+
status: err.status,
|
|
381
|
+
code: err.code,
|
|
382
|
+
type: err.type,
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
return { success: false, error: stopMessage, usageLimitReached: true };
|
|
386
|
+
}
|
|
312
387
|
if (isLlmUsageLimitError(err)) {
|
|
313
388
|
const stopMessage = getLlmUsageLimitMessage();
|
|
314
389
|
logger.debug('Stopping execution because an LLM usage limit was reached', {
|
|
@@ -449,6 +524,7 @@ export class ExecutionMode {
|
|
|
449
524
|
});
|
|
450
525
|
// Replace the system message with the enhanced one
|
|
451
526
|
this.session.installRecoverySystemPrompt(recoverySystemPrompt);
|
|
527
|
+
this.session.updateResponseId(null);
|
|
452
528
|
// Wait a bit before retrying
|
|
453
529
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
454
530
|
// Retry the same instruction with incremented counter
|
|
@@ -139,7 +139,9 @@ Test design exploration is complete. Use the validated completion summary below
|
|
|
139
139
|
Do not call tools.
|
|
140
140
|
Do not continue exploring.
|
|
141
141
|
Do not add explanation.
|
|
142
|
-
Output only the final result
|
|
142
|
+
Output only the exact final result format below: one "Suggested test name:" line, then one fenced code block.
|
|
143
|
+
Do not use TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, or bullets.
|
|
144
|
+
The fenced code block must contain only executable test instructions, one per line, ending with exit.
|
|
143
145
|
|
|
144
146
|
Completion summary:
|
|
145
147
|
Suggested test name: ${suggestedTestName || "(choose a short kebab-case name)"}
|
|
@@ -151,3 +153,21 @@ Verification points:
|
|
|
151
153
|
${formatStringList(summary.verificationPoints)}
|
|
152
154
|
${notesForScript ? `Notes for script:\n${notesForScript}\n` : ""}`;
|
|
153
155
|
}
|
|
156
|
+
export function buildDesignScriptFormatRetryPrompt(previousOutput = "") {
|
|
157
|
+
const previousOutputText = typeof previousOutput === "string" ? previousOutput.trim() : "";
|
|
158
|
+
return `AUTHORING FORMAT CORRECTION:
|
|
159
|
+
Your previous answer did not use the required design-mode format.
|
|
160
|
+
|
|
161
|
+
Output only:
|
|
162
|
+
Suggested test name: short-kebab-case-name
|
|
163
|
+
|
|
164
|
+
\`\`\`
|
|
165
|
+
one executable instruction per line
|
|
166
|
+
exit
|
|
167
|
+
\`\`\`
|
|
168
|
+
|
|
169
|
+
Do not include TEST NAME, GOAL, STEPS, VERIFICATIONS, FINAL STATE, markdown headings, numbered lists, bullets, or explanation.${previousOutputText ? `
|
|
170
|
+
|
|
171
|
+
Previous output to reformat:
|
|
172
|
+
${previousOutputText}` : ""}`;
|
|
173
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { classifyCaptureSource } from "../core/capture-sources.js";
|
|
2
|
+
import { stepCompletionData } from "../core/report/run-log-model.js";
|
|
2
3
|
function isHiddenExecutionOutput(item) {
|
|
3
4
|
const captureSource = item?.payload?.captureSource;
|
|
4
5
|
return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
|
|
@@ -24,6 +25,10 @@ export function formatCliOutput(item) {
|
|
|
24
25
|
if (!text || text.trim().length === 0) {
|
|
25
26
|
return null;
|
|
26
27
|
}
|
|
28
|
+
const stepCompletion = stepCompletionData(text);
|
|
29
|
+
if (stepCompletion) {
|
|
30
|
+
return `assistant: Step complete — ${stepCompletion.instruction}`;
|
|
31
|
+
}
|
|
27
32
|
return text
|
|
28
33
|
.split("\n")
|
|
29
34
|
.map((line) => `assistant: ${line}`)
|
|
@@ -92,6 +92,54 @@ function summarizeRequestInput(input) {
|
|
|
92
92
|
};
|
|
93
93
|
});
|
|
94
94
|
}
|
|
95
|
+
function extractImageBase64Length(imageUrl) {
|
|
96
|
+
return typeof imageUrl === "string" ? imageUrl.replace("data:image/png;base64,", "").length : 0;
|
|
97
|
+
}
|
|
98
|
+
function summarizeRequestScreenshots(input, deviceInfo) {
|
|
99
|
+
const screenshots = [];
|
|
100
|
+
for (const item of input) {
|
|
101
|
+
if (item?.role && Array.isArray(item.content)) {
|
|
102
|
+
for (const entry of item.content) {
|
|
103
|
+
if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
|
|
104
|
+
continue;
|
|
105
|
+
screenshots.push({
|
|
106
|
+
source: `message:${item.role}`,
|
|
107
|
+
base64Length: extractImageBase64Length(entry.image_url),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (item?.type === "computer_call_output" && typeof item.output?.image_url === "string") {
|
|
113
|
+
screenshots.push({
|
|
114
|
+
source: "computer_call_output",
|
|
115
|
+
callId: item.call_id,
|
|
116
|
+
base64Length: extractImageBase64Length(item.output.image_url),
|
|
117
|
+
});
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (item?.type === "function_call_output" && Array.isArray(item.output)) {
|
|
121
|
+
for (const entry of item.output) {
|
|
122
|
+
if (entry?.type !== "input_image" || typeof entry.image_url !== "string")
|
|
123
|
+
continue;
|
|
124
|
+
screenshots.push({
|
|
125
|
+
source: "function_call_output",
|
|
126
|
+
callId: item.call_id,
|
|
127
|
+
base64Length: extractImageBase64Length(entry.image_url),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
if (screenshots.length === 0) {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
count: screenshots.length,
|
|
137
|
+
sources: screenshots.map((entry) => entry.source),
|
|
138
|
+
base64Lengths: screenshots.map((entry) => entry.base64Length),
|
|
139
|
+
width: deviceInfo?.scaled_width ?? null,
|
|
140
|
+
height: deviceInfo?.scaled_height ?? null,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
95
143
|
function redactRequestForFileLog(requestParams) {
|
|
96
144
|
return {
|
|
97
145
|
...requestParams,
|
|
@@ -223,6 +271,7 @@ export class CuaDebugTracer {
|
|
|
223
271
|
const localRequestId = crypto.randomUUID();
|
|
224
272
|
const requestLog = redactRequestForFileLog(requestParams);
|
|
225
273
|
const messages = summarizeRequestInput(input);
|
|
274
|
+
const screenshot = summarizeRequestScreenshots(input, deviceInfo);
|
|
226
275
|
const acknowledgedSafetyChecksSent = input
|
|
227
276
|
.filter((item) => item?.type === "computer_call_output")
|
|
228
277
|
.flatMap((item) => {
|
|
@@ -260,13 +309,7 @@ export class CuaDebugTracer {
|
|
|
260
309
|
inputItemTypes,
|
|
261
310
|
inputCallIds,
|
|
262
311
|
messages,
|
|
263
|
-
screenshot
|
|
264
|
-
? {
|
|
265
|
-
width: deviceInfo?.scaled_width ?? null,
|
|
266
|
-
height: deviceInfo?.scaled_height ?? null,
|
|
267
|
-
base64Length: screenshotBase64.length
|
|
268
|
-
}
|
|
269
|
-
: null,
|
|
312
|
+
screenshot,
|
|
270
313
|
safetyChecks: {
|
|
271
314
|
previousPending: previousPendingSafetyChecks,
|
|
272
315
|
previousPendingCount: previousPendingSafetyChecks.length,
|
package/package.json
CHANGED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
export const CUA_BACKENDS = ["openai", "fara"];
|
|
2
|
-
export const DEFAULT_CUA_BACKEND = "openai";
|
|
3
|
-
export const DEFAULT_FARA_BASE_URL = "http://localhost:5000/v1";
|
|
4
|
-
export const DEFAULT_FARA_MODEL = "Fara-7B";
|
|
5
|
-
const VALID_CUA_BACKENDS = new Set(CUA_BACKENDS);
|
|
6
|
-
export function isValidCuaBackend(value) {
|
|
7
|
-
return VALID_CUA_BACKENDS.has(value);
|
|
8
|
-
}
|
|
9
|
-
export function resolveCuaBackend(value) {
|
|
10
|
-
return isValidCuaBackend(value) ? value : DEFAULT_CUA_BACKEND;
|
|
11
|
-
}
|