@loadmill/droid-cua 2.16.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/build/index.js +88 -1
- package/build/src/browser/playwright/actions.js +17 -8
- package/build/src/cli/headless-execution-config.js +21 -0
- package/build/src/cli/headless-report.js +113 -0
- package/build/src/cli/headless-runner.js +57 -11
- package/build/src/cli/scenario-planner.js +1 -0
- package/build/src/commands/help.js +4 -0
- package/build/src/commands/run.js +2 -1
- package/build/src/core/execution-engine.js +57 -2
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/report/generate-html-report.js +88 -0
- package/build/src/core/report/parse-run-report.js +222 -0
- package/build/src/core/report/render-html-report.js +317 -0
- package/build/src/core/report/report-file-name.js +31 -0
- package/build/src/core/report/run-log-model.js +144 -0
- package/build/src/core/report/run-report-model.js +143 -0
- package/build/src/core/scenario-planner.js +85 -0
- package/build/src/core/secrets.js +116 -0
- package/build/src/core/test-data-import.js +62 -0
- package/build/src/core/test-data.js +103 -0
- package/build/src/core/tool-handlers/close-current-tab.js +3 -3
- package/build/src/core/tool-handlers/navigate.js +3 -3
- package/build/src/core/tool-handlers/preview-click.js +6 -6
- package/build/src/core/tool-handlers/type-sensitive-text.js +62 -0
- package/build/src/core/tool-handlers/utils.js +3 -3
- package/build/src/device/android/actions.js +13 -8
- package/build/src/device/cloud/actions.js +17 -8
- package/build/src/device/ios/actions.js +17 -8
- package/build/src/device/openai.js +17 -0
- package/build/src/device/sensitive-action.js +61 -0
- package/build/src/modes/design-mode-ink.js +3 -1
- package/build/src/modes/design-mode.js +3 -1
- package/build/src/modes/execution-mode.js +1 -0
- package/build/src/prompts/design.js +4 -3
- package/build/src/prompts/execution.js +27 -1
- package/build/src/prompts/scenario-planner.js +48 -0
- package/build/src/prompts/test-data-import.js +49 -0
- package/package.json +1 -1
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { resolveEffectiveHelperModel } from "./runtime-config.js";
|
|
2
|
+
import { buildScenarioPlannerMessages } from "../prompts/scenario-planner.js";
|
|
3
|
+
function parseScenarioPlannerContent(content) {
|
|
4
|
+
try {
|
|
5
|
+
return JSON.parse(content);
|
|
6
|
+
}
|
|
7
|
+
catch (error) {
|
|
8
|
+
const message = error instanceof Error ? error.message : "Invalid JSON.";
|
|
9
|
+
throw new Error(`Could not parse scenario planner response: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function normalizeCaseName(value, fallback) {
|
|
13
|
+
const trimmed = typeof value === "string" ? value.trim() : "";
|
|
14
|
+
return trimmed || fallback;
|
|
15
|
+
}
|
|
16
|
+
function dedupeCaseNames(cases) {
|
|
17
|
+
const usedNames = new Set();
|
|
18
|
+
const baseCounts = new Map();
|
|
19
|
+
return cases.map((entry) => {
|
|
20
|
+
const baseName = entry.caseName;
|
|
21
|
+
let nextIndex = baseCounts.get(baseName) || 1;
|
|
22
|
+
let caseName = baseName;
|
|
23
|
+
while (usedNames.has(caseName)) {
|
|
24
|
+
nextIndex += 1;
|
|
25
|
+
caseName = `${baseName} (${nextIndex})`;
|
|
26
|
+
}
|
|
27
|
+
baseCounts.set(baseName, nextIndex);
|
|
28
|
+
usedNames.add(caseName);
|
|
29
|
+
return {
|
|
30
|
+
...entry,
|
|
31
|
+
caseName,
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
export function validateScenarioCases(rawCases, candidateTests) {
|
|
36
|
+
if (!Array.isArray(rawCases) || rawCases.length === 0) {
|
|
37
|
+
throw new Error("Scenario planner did not return any cases.");
|
|
38
|
+
}
|
|
39
|
+
const candidateNames = new Set(candidateTests.map((test) => test.testName));
|
|
40
|
+
const normalizedCases = rawCases.map((rawCase, index) => {
|
|
41
|
+
if (!rawCase || typeof rawCase !== "object" || Array.isArray(rawCase)) {
|
|
42
|
+
throw new Error(`Scenario case ${index + 1} must be an object.`);
|
|
43
|
+
}
|
|
44
|
+
const testName = typeof rawCase.testName === "string" ? rawCase.testName.trim() : "";
|
|
45
|
+
if (!candidateNames.has(testName)) {
|
|
46
|
+
throw new Error(`Scenario case ${index + 1} references unknown test "${testName || "(missing)"}".`);
|
|
47
|
+
}
|
|
48
|
+
const caseName = normalizeCaseName(rawCase.caseName, `${testName} / Case ${index + 1}`);
|
|
49
|
+
const testDataContext = typeof rawCase.testDataContext === "string" ? rawCase.testDataContext.trim() : "";
|
|
50
|
+
return {
|
|
51
|
+
testName,
|
|
52
|
+
caseName,
|
|
53
|
+
testDataContext,
|
|
54
|
+
};
|
|
55
|
+
});
|
|
56
|
+
return dedupeCaseNames(normalizedCases);
|
|
57
|
+
}
|
|
58
|
+
export async function planScenarioRuns({ client, runtimeConfig, scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
|
|
59
|
+
if (!client?.chat?.completions?.create) {
|
|
60
|
+
throw new Error("Scenario planning requires an LLM client.");
|
|
61
|
+
}
|
|
62
|
+
const prompt = typeof scenarioPrompt === "string" ? scenarioPrompt.trim() : "";
|
|
63
|
+
if (!prompt) {
|
|
64
|
+
throw new Error("--scenario must be a non-empty string.");
|
|
65
|
+
}
|
|
66
|
+
if (!Array.isArray(tests) || tests.length === 0) {
|
|
67
|
+
throw new Error("Scenario planning requires at least one candidate test.");
|
|
68
|
+
}
|
|
69
|
+
const response = await client.chat.completions.create({
|
|
70
|
+
model: resolveEffectiveHelperModel({
|
|
71
|
+
llmProviderMode: runtimeConfig?.llmProviderMode,
|
|
72
|
+
cuaModel: runtimeConfig?.cuaModel,
|
|
73
|
+
}),
|
|
74
|
+
messages: buildScenarioPlannerMessages({
|
|
75
|
+
scenarioPrompt: prompt,
|
|
76
|
+
tests,
|
|
77
|
+
appContextText,
|
|
78
|
+
testDataText,
|
|
79
|
+
}),
|
|
80
|
+
response_format: { type: "json_object" },
|
|
81
|
+
});
|
|
82
|
+
const content = response.choices?.[0]?.message?.content || "{}";
|
|
83
|
+
const parsed = parseScenarioPlannerContent(content);
|
|
84
|
+
return validateScenarioCases(parsed.cases, tests);
|
|
85
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { parse as parseDotenv } from "dotenv";
|
|
4
|
+
export const SECRETS_FILE_NAME = ".secrets";
|
|
5
|
+
export function normalizeSecretKeys(keys) {
|
|
6
|
+
if (!Array.isArray(keys))
|
|
7
|
+
return [];
|
|
8
|
+
return keys
|
|
9
|
+
.map((key) => (typeof key === "string" ? key.trim() : ""))
|
|
10
|
+
.filter(Boolean)
|
|
11
|
+
.sort((a, b) => a.localeCompare(b));
|
|
12
|
+
}
|
|
13
|
+
function sortedKeys(secretsByKey = {}) {
|
|
14
|
+
return normalizeSecretKeys(Object.keys(secretsByKey));
|
|
15
|
+
}
|
|
16
|
+
function mergeSecretMaps(...maps) {
|
|
17
|
+
const merged = {};
|
|
18
|
+
for (const map of maps) {
|
|
19
|
+
if (!map || typeof map !== "object")
|
|
20
|
+
continue;
|
|
21
|
+
for (const [key, value] of Object.entries(map)) {
|
|
22
|
+
if (!key)
|
|
23
|
+
continue;
|
|
24
|
+
merged[key] = String(value ?? "");
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return merged;
|
|
28
|
+
}
|
|
29
|
+
export function getDefaultSecretsPath(basePath = process.cwd()) {
|
|
30
|
+
return path.join(basePath, SECRETS_FILE_NAME);
|
|
31
|
+
}
|
|
32
|
+
export function parseSecretsOverrideString(rawValue) {
|
|
33
|
+
if (rawValue == null || rawValue === "") {
|
|
34
|
+
return {};
|
|
35
|
+
}
|
|
36
|
+
if (typeof rawValue !== "string") {
|
|
37
|
+
throw new Error("--secrets must be a comma-separated KEY=value string.");
|
|
38
|
+
}
|
|
39
|
+
const parsed = {};
|
|
40
|
+
for (const rawEntry of rawValue.split(",")) {
|
|
41
|
+
const entry = rawEntry.trim();
|
|
42
|
+
if (!entry)
|
|
43
|
+
continue;
|
|
44
|
+
const separatorIndex = entry.indexOf("=");
|
|
45
|
+
if (separatorIndex < 0) {
|
|
46
|
+
throw new Error("--secrets entries must use KEY=value format.");
|
|
47
|
+
}
|
|
48
|
+
const key = entry.slice(0, separatorIndex).trim();
|
|
49
|
+
if (!key) {
|
|
50
|
+
throw new Error("--secrets entries must include a non-empty key.");
|
|
51
|
+
}
|
|
52
|
+
parsed[key] = entry.slice(separatorIndex + 1);
|
|
53
|
+
}
|
|
54
|
+
return parsed;
|
|
55
|
+
}
|
|
56
|
+
export async function loadSecretsFile(filePath) {
|
|
57
|
+
if (!filePath) {
|
|
58
|
+
return {
|
|
59
|
+
status: "missing",
|
|
60
|
+
secretsByKey: {},
|
|
61
|
+
path: null,
|
|
62
|
+
message: null,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
const content = await readFile(filePath, "utf-8");
|
|
67
|
+
return {
|
|
68
|
+
status: "loaded",
|
|
69
|
+
secretsByKey: parseDotenv(content),
|
|
70
|
+
path: filePath,
|
|
71
|
+
message: null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error?.code === "ENOENT") {
|
|
76
|
+
return {
|
|
77
|
+
status: "missing",
|
|
78
|
+
secretsByKey: {},
|
|
79
|
+
path: filePath,
|
|
80
|
+
message: null,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const message = error instanceof Error ? error.message : "Unknown file read error.";
|
|
84
|
+
return {
|
|
85
|
+
status: "failed",
|
|
86
|
+
secretsByKey: {},
|
|
87
|
+
path: filePath,
|
|
88
|
+
message,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* @param {Object} [options]
|
|
94
|
+
* @param {string|null} [options.secretsPath] Path to a `.secrets` file. Pass `null`
|
|
95
|
+
* to skip file loading entirely (resolves to no file-based secrets).
|
|
96
|
+
* @param {string|null} [options.cliSecrets] Comma-separated KEY=value overrides.
|
|
97
|
+
*/
|
|
98
|
+
export async function resolveExecutionSecrets({ secretsPath = getDefaultSecretsPath(), cliSecrets = null, } = {}) {
|
|
99
|
+
const fileResult = await loadSecretsFile(secretsPath);
|
|
100
|
+
if (fileResult.status === "failed") {
|
|
101
|
+
throw new Error(`Could not read secrets file ${fileResult.path}: ${fileResult.message}`);
|
|
102
|
+
}
|
|
103
|
+
const cliSecretsByKey = parseSecretsOverrideString(cliSecrets);
|
|
104
|
+
const secretsByKey = mergeSecretMaps(fileResult.secretsByKey, cliSecretsByKey);
|
|
105
|
+
const availableSecretKeys = sortedKeys(secretsByKey);
|
|
106
|
+
return {
|
|
107
|
+
secretsByKey,
|
|
108
|
+
availableSecretKeys,
|
|
109
|
+
sourceMetadata: {
|
|
110
|
+
filePath: fileResult.path,
|
|
111
|
+
fileStatus: fileResult.status,
|
|
112
|
+
fileKeyCount: sortedKeys(fileResult.secretsByKey).length,
|
|
113
|
+
cliKeyCount: sortedKeys(cliSecretsByKey).length,
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { resolveEffectiveHelperModel } from "./runtime-config.js";
|
|
3
|
+
import { buildTestDataImportMessages } from "../prompts/test-data-import.js";
|
|
4
|
+
export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt"];
|
|
5
|
+
export function isSupportedTestDataImportPath(filePath) {
|
|
6
|
+
return TEST_DATA_IMPORT_EXTENSIONS.includes(path.extname(String(filePath ?? "")).toLowerCase());
|
|
7
|
+
}
|
|
8
|
+
export function assertSupportedTestDataImportPath(filePath) {
|
|
9
|
+
if (!isSupportedTestDataImportPath(filePath)) {
|
|
10
|
+
throw new Error("Test data import supports CSV and TXT files only.");
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function parseTestDataImportContent(content) {
|
|
14
|
+
try {
|
|
15
|
+
return JSON.parse(content);
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
const message = error instanceof Error ? error.message : "Invalid JSON.";
|
|
19
|
+
throw new Error(`Could not parse test data import response: ${message}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function normalizeGeneratedContent(value) {
|
|
23
|
+
const content = typeof value === "string" ? value.trim() : "";
|
|
24
|
+
if (!content) {
|
|
25
|
+
throw new Error("Test data import returned empty content.");
|
|
26
|
+
}
|
|
27
|
+
if (/^```/.test(content)) {
|
|
28
|
+
throw new Error("Test data import returned code-fenced content instead of markdown.");
|
|
29
|
+
}
|
|
30
|
+
return content.endsWith("\n") ? content : `${content}\n`;
|
|
31
|
+
}
|
|
32
|
+
export async function importTestDataMarkdown({ client, runtimeConfig, currentContent = "", importedFileName, importedFileText, instructions = "", }) {
|
|
33
|
+
if (!client?.chat?.completions?.create) {
|
|
34
|
+
throw new Error("Test data import requires an LLM client.");
|
|
35
|
+
}
|
|
36
|
+
const fileName = typeof importedFileName === "string" ? importedFileName.trim() : "";
|
|
37
|
+
if (!fileName) {
|
|
38
|
+
throw new Error("Imported file name is required.");
|
|
39
|
+
}
|
|
40
|
+
const fileText = typeof importedFileText === "string" ? importedFileText : "";
|
|
41
|
+
if (!fileText.trim()) {
|
|
42
|
+
throw new Error("Selected import file is empty.");
|
|
43
|
+
}
|
|
44
|
+
const response = await client.chat.completions.create({
|
|
45
|
+
model: resolveEffectiveHelperModel({
|
|
46
|
+
llmProviderMode: runtimeConfig?.llmProviderMode,
|
|
47
|
+
cuaModel: runtimeConfig?.cuaModel,
|
|
48
|
+
}),
|
|
49
|
+
messages: buildTestDataImportMessages({
|
|
50
|
+
currentContent,
|
|
51
|
+
importedFileName: fileName,
|
|
52
|
+
importedFileText: fileText,
|
|
53
|
+
instructions,
|
|
54
|
+
}),
|
|
55
|
+
response_format: { type: "json_object" },
|
|
56
|
+
});
|
|
57
|
+
const content = response.choices?.[0]?.message?.content || "{}";
|
|
58
|
+
const parsed = parseTestDataImportContent(content);
|
|
59
|
+
return {
|
|
60
|
+
content: normalizeGeneratedContent(parsed.content),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { access, readFile } from "fs/promises";
|
|
3
|
+
import { constants as fsConstants } from "fs";
|
|
4
|
+
export const TEST_DATA_FILENAME = "test-data.md";
|
|
5
|
+
export const DEFAULT_TEST_DATA_TEMPLATE = `# Test Data
|
|
6
|
+
<!-- Concrete data the agent can use during Scenario Runs. Keep entries named and specific. -->
|
|
7
|
+
<!-- Examples below are commented out so they are guidance, not usable test data. -->
|
|
8
|
+
|
|
9
|
+
## Accounts
|
|
10
|
+
<!--
|
|
11
|
+
### Standard buyer
|
|
12
|
+
Email:
|
|
13
|
+
Password:
|
|
14
|
+
Notes: When should the agent use this account?
|
|
15
|
+
-->
|
|
16
|
+
|
|
17
|
+
## Products
|
|
18
|
+
<!--
|
|
19
|
+
### In-stock product
|
|
20
|
+
Name:
|
|
21
|
+
SKU:
|
|
22
|
+
Notes:
|
|
23
|
+
-->
|
|
24
|
+
|
|
25
|
+
## Payment Methods
|
|
26
|
+
<!--
|
|
27
|
+
### Valid card
|
|
28
|
+
Number:
|
|
29
|
+
Expiration:
|
|
30
|
+
CVV:
|
|
31
|
+
Notes:
|
|
32
|
+
-->
|
|
33
|
+
|
|
34
|
+
## Roles and Permissions
|
|
35
|
+
|
|
36
|
+
## Regions and Locales
|
|
37
|
+
|
|
38
|
+
## Other Data
|
|
39
|
+
`;
|
|
40
|
+
export function getDefaultProjectTestDataPath(projectPath) {
|
|
41
|
+
return path.join(projectPath, TEST_DATA_FILENAME);
|
|
42
|
+
}
|
|
43
|
+
export function normalizeTestDataTemplateText(text) {
|
|
44
|
+
return String(text ?? "").replace(/\r\n?/g, "\n").replace(/[ \t]+$/gm, "").trimEnd();
|
|
45
|
+
}
|
|
46
|
+
export function isDefaultTestDataTemplate(text) {
|
|
47
|
+
if (!String(text ?? "").trim()) {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return normalizeTestDataTemplateText(text) === normalizeTestDataTemplateText(DEFAULT_TEST_DATA_TEMPLATE);
|
|
51
|
+
}
|
|
52
|
+
export async function readTestDataFile(filePath) {
|
|
53
|
+
await access(filePath, fsConstants.R_OK);
|
|
54
|
+
return await readFile(filePath, "utf-8");
|
|
55
|
+
}
|
|
56
|
+
export async function loadTestData({ testDataPath }) {
|
|
57
|
+
if (!testDataPath) {
|
|
58
|
+
return { text: "", testDataPath: null };
|
|
59
|
+
}
|
|
60
|
+
const text = await readTestDataFile(testDataPath);
|
|
61
|
+
return { text, testDataPath };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* @param {{
|
|
65
|
+
* testDataPath?: string | null,
|
|
66
|
+
* missingFileStatus?: string | null,
|
|
67
|
+
* }} [options]
|
|
68
|
+
*/
|
|
69
|
+
export async function resolveExecutionTestData({ testDataPath = null, missingFileStatus = null, } = {}) {
|
|
70
|
+
if (!testDataPath) {
|
|
71
|
+
return {
|
|
72
|
+
status: "omitted",
|
|
73
|
+
text: "",
|
|
74
|
+
testDataPath: null,
|
|
75
|
+
message: null,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
const result = await loadTestData({ testDataPath });
|
|
80
|
+
return {
|
|
81
|
+
status: result.text.trim().length > 0 ? "loaded" : "empty",
|
|
82
|
+
text: result.text,
|
|
83
|
+
testDataPath: result.testDataPath,
|
|
84
|
+
message: null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
|
|
89
|
+
return {
|
|
90
|
+
status: missingFileStatus,
|
|
91
|
+
text: "",
|
|
92
|
+
testDataPath,
|
|
93
|
+
message: null,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
status: "failed",
|
|
98
|
+
text: "",
|
|
99
|
+
testDataPath,
|
|
100
|
+
message: error instanceof Error ? error.message : "Unknown test data error",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -2,7 +2,7 @@ import { normalizeBrowserInfoForSession } from "../../browser/normalize.js";
|
|
|
2
2
|
import { CAPTURE_SOURCES } from "../capture-sources.js";
|
|
3
3
|
import { printCliOutput } from "../../utils/console-output.js";
|
|
4
4
|
import { updateBrowserSessionInfo } from "./browser-session.js";
|
|
5
|
-
import { buildFunctionCallOutputItem,
|
|
5
|
+
import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, } from "./utils.js";
|
|
6
6
|
export async function handleCloseCurrentTabToolCall({ engine, callId, context = null, stepContext = null }) {
|
|
7
7
|
const session = engine.session;
|
|
8
8
|
if (session?.targetKind !== "browser" && session?.deviceInfo?.platform !== "browser") {
|
|
@@ -44,9 +44,9 @@ export async function handleCloseCurrentTabToolCall({ engine, callId, context =
|
|
|
44
44
|
});
|
|
45
45
|
return {
|
|
46
46
|
kind: "continue",
|
|
47
|
-
outputItem: buildFunctionCallOutputItem(callId,
|
|
47
|
+
outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
48
48
|
text: result.message,
|
|
49
|
-
|
|
49
|
+
imageBase64: screenshotBase64,
|
|
50
50
|
})),
|
|
51
51
|
currentScreenshot,
|
|
52
52
|
};
|
|
@@ -3,7 +3,7 @@ import { buildExecutionNavigationRecoveryPrompt } from "../../prompts/execution.
|
|
|
3
3
|
import { emitDesktopDebug } from "../../utils/desktop-debug.js";
|
|
4
4
|
import { printCliOutput } from "../../utils/console-output.js";
|
|
5
5
|
import { updateBrowserSessionInfo } from "./browser-session.js";
|
|
6
|
-
import { buildDebugContext, buildFunctionCallOutputItem,
|
|
6
|
+
import { buildDebugContext, buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
|
|
7
7
|
export async function handleNavigateToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
|
|
8
8
|
const session = engine.session;
|
|
9
9
|
const parsed = parseStringToolArgument(argumentsText, "target", "navigate");
|
|
@@ -69,9 +69,9 @@ export async function handleNavigateToolCall({ engine, callId, argumentsText, co
|
|
|
69
69
|
const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
|
|
70
70
|
return {
|
|
71
71
|
kind: "continue",
|
|
72
|
-
outputItem: buildFunctionCallOutputItem(callId,
|
|
72
|
+
outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
73
73
|
text: `Navigation complete: ${normalizedUrl}`,
|
|
74
|
-
|
|
74
|
+
imageBase64: screenshotBase64,
|
|
75
75
|
})),
|
|
76
76
|
};
|
|
77
77
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { createAnnotatedScreenshotFromBase64 } from "../../utils/run-screenshot-recorder.js";
|
|
3
3
|
import { CAPTURE_SOURCES } from "../capture-sources.js";
|
|
4
|
-
import { buildFunctionCallOutputItem,
|
|
4
|
+
import { buildFunctionCallOutputItem, buildToolOutputContent, } from "./utils.js";
|
|
5
5
|
function buildPreviewClickValidationError(argumentsText) {
|
|
6
6
|
if (!argumentsText || typeof argumentsText !== "string") {
|
|
7
7
|
return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
|
|
@@ -42,7 +42,7 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
|
|
|
42
42
|
const session = engine.session;
|
|
43
43
|
const parsedArguments = parsePreviewClickArguments(argumentsText);
|
|
44
44
|
if (!parsedArguments.ok) {
|
|
45
|
-
return continueWith(buildFunctionCallOutputItem(callId,
|
|
45
|
+
return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: parsedArguments.message })));
|
|
46
46
|
}
|
|
47
47
|
try {
|
|
48
48
|
const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
|
|
@@ -52,7 +52,7 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
|
|
|
52
52
|
style: "preview",
|
|
53
53
|
});
|
|
54
54
|
if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
|
|
55
|
-
return continueWith(buildFunctionCallOutputItem(callId,
|
|
55
|
+
return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
56
56
|
text: "preview_click failed: could not annotate the current screenshot.",
|
|
57
57
|
})));
|
|
58
58
|
}
|
|
@@ -99,13 +99,13 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
|
|
|
99
99
|
},
|
|
100
100
|
});
|
|
101
101
|
}
|
|
102
|
-
return continueWith(buildFunctionCallOutputItem(callId,
|
|
102
|
+
return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
103
103
|
text: previewText,
|
|
104
|
-
|
|
104
|
+
imageBase64: previewResult.screenshotBase64,
|
|
105
105
|
})));
|
|
106
106
|
}
|
|
107
107
|
catch (error) {
|
|
108
|
-
return continueWith(buildFunctionCallOutputItem(callId,
|
|
108
|
+
return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
109
109
|
text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
|
|
110
110
|
})));
|
|
111
111
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { CAPTURE_SOURCES } from "../capture-sources.js";
|
|
3
|
+
import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, parseStringToolArgument, } from "./utils.js";
|
|
4
|
+
export async function handleTypeSensitiveTextToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
|
|
5
|
+
const parsed = parseStringToolArgument(argumentsText, "key", "type_sensitive_text");
|
|
6
|
+
if (!parsed.ok) {
|
|
7
|
+
return {
|
|
8
|
+
kind: "continue",
|
|
9
|
+
outputItem: buildTextToolOutputItem(callId, parsed.message),
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
const key = parsed.value;
|
|
13
|
+
const secretsByKey = engine.secretsByKey && typeof engine.secretsByKey === "object" ? engine.secretsByKey : {};
|
|
14
|
+
if (!Object.prototype.hasOwnProperty.call(secretsByKey, key)) {
|
|
15
|
+
return {
|
|
16
|
+
kind: "continue",
|
|
17
|
+
outputItem: buildTextToolOutputItem(callId, `Secret key not found: ${key}`),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
const action = {
|
|
21
|
+
type: "type",
|
|
22
|
+
text: String(secretsByKey[key] ?? ""),
|
|
23
|
+
sensitive: true,
|
|
24
|
+
sensitiveKey: key,
|
|
25
|
+
};
|
|
26
|
+
try {
|
|
27
|
+
const result = await engine.handleModelAction(engine.session.deviceId, action, engine.session.deviceInfo.scale, {
|
|
28
|
+
...context,
|
|
29
|
+
stepId: stepContext?.stepId,
|
|
30
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
31
|
+
eventId: randomUUID(),
|
|
32
|
+
});
|
|
33
|
+
if (result?.success === false) {
|
|
34
|
+
return {
|
|
35
|
+
kind: "fail",
|
|
36
|
+
error: `Sensitive typing failed for ${key}.`,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return {
|
|
42
|
+
kind: "fail",
|
|
43
|
+
error: `Sensitive typing failed for ${key}.`,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const screenshotBase64 = await engine.getScreenshotAsBase64(engine.session.deviceId, engine.session.deviceInfo);
|
|
47
|
+
const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
|
|
48
|
+
runId: context?.runId,
|
|
49
|
+
sessionId: context?.sessionId,
|
|
50
|
+
stepId: stepContext?.stepId,
|
|
51
|
+
instructionIndex: stepContext?.instructionIndex,
|
|
52
|
+
captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
|
|
53
|
+
});
|
|
54
|
+
return {
|
|
55
|
+
kind: "continue",
|
|
56
|
+
outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
|
|
57
|
+
text: `Typed secret: ${key}`,
|
|
58
|
+
imageBase64: screenshotBase64,
|
|
59
|
+
})),
|
|
60
|
+
currentScreenshot,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -45,12 +45,12 @@ export function buildFunctionCallOutputItem(callId, output) {
|
|
|
45
45
|
export function buildTextToolOutputItem(callId, text) {
|
|
46
46
|
return buildFunctionCallOutputItem(callId, [{ type: "input_text", text }]);
|
|
47
47
|
}
|
|
48
|
-
export function
|
|
48
|
+
export function buildToolOutputContent({ text, imageBase64 = null }) {
|
|
49
49
|
const content = [{ type: "input_text", text }];
|
|
50
|
-
if (
|
|
50
|
+
if (imageBase64) {
|
|
51
51
|
content.push({
|
|
52
52
|
type: "input_image",
|
|
53
|
-
image_url: `data:image/png;base64,${
|
|
53
|
+
image_url: `data:image/png;base64,${imageBase64}`,
|
|
54
54
|
detail: "high",
|
|
55
55
|
});
|
|
56
56
|
}
|
|
@@ -3,6 +3,7 @@ import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js
|
|
|
3
3
|
import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
|
|
4
4
|
import { buildActionEventMeta } from "../action-event-meta.js";
|
|
5
5
|
import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
|
|
6
|
+
import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
|
|
6
7
|
import { resolveScrollGesture } from "../scroll-gesture.js";
|
|
7
8
|
import { execAdb } from "./tools.js";
|
|
8
9
|
function adbShell(deviceId, command) {
|
|
@@ -34,7 +35,8 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
34
35
|
y1,
|
|
35
36
|
x2,
|
|
36
37
|
y2,
|
|
37
|
-
text:
|
|
38
|
+
text: buildDebugTextField(action, truncateForDebug),
|
|
39
|
+
...buildSensitiveDebugFields(action),
|
|
38
40
|
keyCount: Array.isArray(keys) ? keys.length : 0,
|
|
39
41
|
pathPoints: Array.isArray(path) ? path.length : 0
|
|
40
42
|
});
|
|
@@ -79,7 +81,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
79
81
|
}
|
|
80
82
|
break;
|
|
81
83
|
case "type":
|
|
82
|
-
addOutput({ type: 'action', text:
|
|
84
|
+
addOutput({ type: 'action', text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
|
|
83
85
|
const escapedText = text.replace(/(["\\$`])/g, "\\$1").replace(/ /g, "%s");
|
|
84
86
|
await adbShell(deviceId, `input text "${escapedText}"`);
|
|
85
87
|
break;
|
|
@@ -104,13 +106,15 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
104
106
|
default:
|
|
105
107
|
addOutput({ type: 'info', text: `Unknown action: ${JSON.stringify(action)}` });
|
|
106
108
|
}
|
|
109
|
+
return { success: true };
|
|
107
110
|
}
|
|
108
111
|
catch (error) {
|
|
112
|
+
const outputError = buildActionErrorOutput(action, error);
|
|
109
113
|
// Log full error details to file
|
|
110
114
|
logger.error('Action execution error', {
|
|
111
|
-
action,
|
|
112
|
-
message:
|
|
113
|
-
stack: error.stack
|
|
115
|
+
action: actionForErrorLog(action),
|
|
116
|
+
message: outputError.message,
|
|
117
|
+
stack: isSensitiveAction(action) ? undefined : error.stack
|
|
114
118
|
});
|
|
115
119
|
emitDesktopDebug("device.error", "device", {
|
|
116
120
|
runId: context?.runId,
|
|
@@ -121,12 +125,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
121
125
|
platform: "android",
|
|
122
126
|
operation: "action.execute",
|
|
123
127
|
actionType: action?.type,
|
|
124
|
-
message:
|
|
128
|
+
message: outputError.message
|
|
125
129
|
});
|
|
126
130
|
// Show user-friendly error message
|
|
127
131
|
addOutput({
|
|
128
132
|
type: 'error',
|
|
129
|
-
text:
|
|
133
|
+
text: outputError.text,
|
|
130
134
|
eventType: 'error',
|
|
131
135
|
actionType: action?.type,
|
|
132
136
|
runId: context?.runId,
|
|
@@ -134,10 +138,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
|
|
|
134
138
|
instructionIndex: context?.instructionIndex,
|
|
135
139
|
payload: {
|
|
136
140
|
eventId: context?.eventId,
|
|
137
|
-
message:
|
|
141
|
+
message: outputError.message,
|
|
138
142
|
platform: 'android'
|
|
139
143
|
}
|
|
140
144
|
});
|
|
141
145
|
addOutput({ type: 'info', text: 'Full error details have been logged to the debug log.' });
|
|
146
|
+
return { success: false, error: outputError.message };
|
|
142
147
|
}
|
|
143
148
|
}
|