@loadmill/droid-cua 2.16.0 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +18 -0
  2. package/build/index.js +88 -1
  3. package/build/src/browser/playwright/actions.js +17 -8
  4. package/build/src/cli/headless-execution-config.js +21 -0
  5. package/build/src/cli/headless-report.js +113 -0
  6. package/build/src/cli/headless-runner.js +57 -11
  7. package/build/src/cli/scenario-planner.js +1 -0
  8. package/build/src/commands/help.js +4 -0
  9. package/build/src/commands/run.js +2 -1
  10. package/build/src/core/execution-engine.js +57 -2
  11. package/build/src/core/prompts.js +1 -1
  12. package/build/src/core/report/generate-html-report.js +88 -0
  13. package/build/src/core/report/parse-run-report.js +222 -0
  14. package/build/src/core/report/render-html-report.js +317 -0
  15. package/build/src/core/report/report-file-name.js +31 -0
  16. package/build/src/core/report/run-log-model.js +144 -0
  17. package/build/src/core/report/run-report-model.js +143 -0
  18. package/build/src/core/scenario-planner.js +85 -0
  19. package/build/src/core/secrets.js +116 -0
  20. package/build/src/core/test-data-import.js +62 -0
  21. package/build/src/core/test-data.js +103 -0
  22. package/build/src/core/tool-handlers/close-current-tab.js +3 -3
  23. package/build/src/core/tool-handlers/navigate.js +3 -3
  24. package/build/src/core/tool-handlers/preview-click.js +6 -6
  25. package/build/src/core/tool-handlers/type-sensitive-text.js +62 -0
  26. package/build/src/core/tool-handlers/utils.js +3 -3
  27. package/build/src/device/android/actions.js +13 -8
  28. package/build/src/device/cloud/actions.js +17 -8
  29. package/build/src/device/ios/actions.js +17 -8
  30. package/build/src/device/openai.js +17 -0
  31. package/build/src/device/sensitive-action.js +61 -0
  32. package/build/src/modes/design-mode-ink.js +3 -1
  33. package/build/src/modes/design-mode.js +3 -1
  34. package/build/src/modes/execution-mode.js +1 -0
  35. package/build/src/prompts/design.js +4 -3
  36. package/build/src/prompts/execution.js +27 -1
  37. package/build/src/prompts/scenario-planner.js +48 -0
  38. package/build/src/prompts/test-data-import.js +49 -0
  39. package/package.json +1 -1
@@ -0,0 +1,85 @@
1
+ import { resolveEffectiveHelperModel } from "./runtime-config.js";
2
+ import { buildScenarioPlannerMessages } from "../prompts/scenario-planner.js";
3
+ function parseScenarioPlannerContent(content) {
4
+ try {
5
+ return JSON.parse(content);
6
+ }
7
+ catch (error) {
8
+ const message = error instanceof Error ? error.message : "Invalid JSON.";
9
+ throw new Error(`Could not parse scenario planner response: ${message}`);
10
+ }
11
+ }
12
+ function normalizeCaseName(value, fallback) {
13
+ const trimmed = typeof value === "string" ? value.trim() : "";
14
+ return trimmed || fallback;
15
+ }
16
+ function dedupeCaseNames(cases) {
17
+ const usedNames = new Set();
18
+ const baseCounts = new Map();
19
+ return cases.map((entry) => {
20
+ const baseName = entry.caseName;
21
+ let nextIndex = baseCounts.get(baseName) || 1;
22
+ let caseName = baseName;
23
+ while (usedNames.has(caseName)) {
24
+ nextIndex += 1;
25
+ caseName = `${baseName} (${nextIndex})`;
26
+ }
27
+ baseCounts.set(baseName, nextIndex);
28
+ usedNames.add(caseName);
29
+ return {
30
+ ...entry,
31
+ caseName,
32
+ };
33
+ });
34
+ }
35
+ export function validateScenarioCases(rawCases, candidateTests) {
36
+ if (!Array.isArray(rawCases) || rawCases.length === 0) {
37
+ throw new Error("Scenario planner did not return any cases.");
38
+ }
39
+ const candidateNames = new Set(candidateTests.map((test) => test.testName));
40
+ const normalizedCases = rawCases.map((rawCase, index) => {
41
+ if (!rawCase || typeof rawCase !== "object" || Array.isArray(rawCase)) {
42
+ throw new Error(`Scenario case ${index + 1} must be an object.`);
43
+ }
44
+ const testName = typeof rawCase.testName === "string" ? rawCase.testName.trim() : "";
45
+ if (!candidateNames.has(testName)) {
46
+ throw new Error(`Scenario case ${index + 1} references unknown test "${testName || "(missing)"}".`);
47
+ }
48
+ const caseName = normalizeCaseName(rawCase.caseName, `${testName} / Case ${index + 1}`);
49
+ const testDataContext = typeof rawCase.testDataContext === "string" ? rawCase.testDataContext.trim() : "";
50
+ return {
51
+ testName,
52
+ caseName,
53
+ testDataContext,
54
+ };
55
+ });
56
+ return dedupeCaseNames(normalizedCases);
57
+ }
58
+ export async function planScenarioRuns({ client, runtimeConfig, scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
59
+ if (!client?.chat?.completions?.create) {
60
+ throw new Error("Scenario planning requires an LLM client.");
61
+ }
62
+ const prompt = typeof scenarioPrompt === "string" ? scenarioPrompt.trim() : "";
63
+ if (!prompt) {
64
+ throw new Error("--scenario must be a non-empty string.");
65
+ }
66
+ if (!Array.isArray(tests) || tests.length === 0) {
67
+ throw new Error("Scenario planning requires at least one candidate test.");
68
+ }
69
+ const response = await client.chat.completions.create({
70
+ model: resolveEffectiveHelperModel({
71
+ llmProviderMode: runtimeConfig?.llmProviderMode,
72
+ cuaModel: runtimeConfig?.cuaModel,
73
+ }),
74
+ messages: buildScenarioPlannerMessages({
75
+ scenarioPrompt: prompt,
76
+ tests,
77
+ appContextText,
78
+ testDataText,
79
+ }),
80
+ response_format: { type: "json_object" },
81
+ });
82
+ const content = response.choices?.[0]?.message?.content || "{}";
83
+ const parsed = parseScenarioPlannerContent(content);
84
+ return validateScenarioCases(parsed.cases, tests);
85
+ }
@@ -0,0 +1,116 @@
1
+ import path from "node:path";
2
+ import { readFile } from "node:fs/promises";
3
+ import { parse as parseDotenv } from "dotenv";
4
+ export const SECRETS_FILE_NAME = ".secrets";
5
+ export function normalizeSecretKeys(keys) {
6
+ if (!Array.isArray(keys))
7
+ return [];
8
+ return keys
9
+ .map((key) => (typeof key === "string" ? key.trim() : ""))
10
+ .filter(Boolean)
11
+ .sort((a, b) => a.localeCompare(b));
12
+ }
13
+ function sortedKeys(secretsByKey = {}) {
14
+ return normalizeSecretKeys(Object.keys(secretsByKey));
15
+ }
16
+ function mergeSecretMaps(...maps) {
17
+ const merged = {};
18
+ for (const map of maps) {
19
+ if (!map || typeof map !== "object")
20
+ continue;
21
+ for (const [key, value] of Object.entries(map)) {
22
+ if (!key)
23
+ continue;
24
+ merged[key] = String(value ?? "");
25
+ }
26
+ }
27
+ return merged;
28
+ }
29
+ export function getDefaultSecretsPath(basePath = process.cwd()) {
30
+ return path.join(basePath, SECRETS_FILE_NAME);
31
+ }
32
+ export function parseSecretsOverrideString(rawValue) {
33
+ if (rawValue == null || rawValue === "") {
34
+ return {};
35
+ }
36
+ if (typeof rawValue !== "string") {
37
+ throw new Error("--secrets must be a comma-separated KEY=value string.");
38
+ }
39
+ const parsed = {};
40
+ for (const rawEntry of rawValue.split(",")) {
41
+ const entry = rawEntry.trim();
42
+ if (!entry)
43
+ continue;
44
+ const separatorIndex = entry.indexOf("=");
45
+ if (separatorIndex < 0) {
46
+ throw new Error("--secrets entries must use KEY=value format.");
47
+ }
48
+ const key = entry.slice(0, separatorIndex).trim();
49
+ if (!key) {
50
+ throw new Error("--secrets entries must include a non-empty key.");
51
+ }
52
+ parsed[key] = entry.slice(separatorIndex + 1);
53
+ }
54
+ return parsed;
55
+ }
56
+ export async function loadSecretsFile(filePath) {
57
+ if (!filePath) {
58
+ return {
59
+ status: "missing",
60
+ secretsByKey: {},
61
+ path: null,
62
+ message: null,
63
+ };
64
+ }
65
+ try {
66
+ const content = await readFile(filePath, "utf-8");
67
+ return {
68
+ status: "loaded",
69
+ secretsByKey: parseDotenv(content),
70
+ path: filePath,
71
+ message: null,
72
+ };
73
+ }
74
+ catch (error) {
75
+ if (error?.code === "ENOENT") {
76
+ return {
77
+ status: "missing",
78
+ secretsByKey: {},
79
+ path: filePath,
80
+ message: null,
81
+ };
82
+ }
83
+ const message = error instanceof Error ? error.message : "Unknown file read error.";
84
+ return {
85
+ status: "failed",
86
+ secretsByKey: {},
87
+ path: filePath,
88
+ message,
89
+ };
90
+ }
91
+ }
92
+ /**
93
+ * @param {Object} [options]
94
+ * @param {string|null} [options.secretsPath] Path to a `.secrets` file. Pass `null`
95
+ * to skip file loading entirely (resolves to no file-based secrets).
96
+ * @param {string|null} [options.cliSecrets] Comma-separated KEY=value overrides.
97
+ */
98
+ export async function resolveExecutionSecrets({ secretsPath = getDefaultSecretsPath(), cliSecrets = null, } = {}) {
99
+ const fileResult = await loadSecretsFile(secretsPath);
100
+ if (fileResult.status === "failed") {
101
+ throw new Error(`Could not read secrets file ${fileResult.path}: ${fileResult.message}`);
102
+ }
103
+ const cliSecretsByKey = parseSecretsOverrideString(cliSecrets);
104
+ const secretsByKey = mergeSecretMaps(fileResult.secretsByKey, cliSecretsByKey);
105
+ const availableSecretKeys = sortedKeys(secretsByKey);
106
+ return {
107
+ secretsByKey,
108
+ availableSecretKeys,
109
+ sourceMetadata: {
110
+ filePath: fileResult.path,
111
+ fileStatus: fileResult.status,
112
+ fileKeyCount: sortedKeys(fileResult.secretsByKey).length,
113
+ cliKeyCount: sortedKeys(cliSecretsByKey).length,
114
+ },
115
+ };
116
+ }
@@ -0,0 +1,62 @@
1
+ import path from "path";
2
+ import { resolveEffectiveHelperModel } from "./runtime-config.js";
3
+ import { buildTestDataImportMessages } from "../prompts/test-data-import.js";
4
+ export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt"];
5
+ export function isSupportedTestDataImportPath(filePath) {
6
+ return TEST_DATA_IMPORT_EXTENSIONS.includes(path.extname(String(filePath ?? "")).toLowerCase());
7
+ }
8
+ export function assertSupportedTestDataImportPath(filePath) {
9
+ if (!isSupportedTestDataImportPath(filePath)) {
10
+ throw new Error("Test data import supports CSV and TXT files only.");
11
+ }
12
+ }
13
+ function parseTestDataImportContent(content) {
14
+ try {
15
+ return JSON.parse(content);
16
+ }
17
+ catch (error) {
18
+ const message = error instanceof Error ? error.message : "Invalid JSON.";
19
+ throw new Error(`Could not parse test data import response: ${message}`);
20
+ }
21
+ }
22
+ function normalizeGeneratedContent(value) {
23
+ const content = typeof value === "string" ? value.trim() : "";
24
+ if (!content) {
25
+ throw new Error("Test data import returned empty content.");
26
+ }
27
+ if (/^```/.test(content)) {
28
+ throw new Error("Test data import returned code-fenced content instead of markdown.");
29
+ }
30
+ return content.endsWith("\n") ? content : `${content}\n`;
31
+ }
32
+ export async function importTestDataMarkdown({ client, runtimeConfig, currentContent = "", importedFileName, importedFileText, instructions = "", }) {
33
+ if (!client?.chat?.completions?.create) {
34
+ throw new Error("Test data import requires an LLM client.");
35
+ }
36
+ const fileName = typeof importedFileName === "string" ? importedFileName.trim() : "";
37
+ if (!fileName) {
38
+ throw new Error("Imported file name is required.");
39
+ }
40
+ const fileText = typeof importedFileText === "string" ? importedFileText : "";
41
+ if (!fileText.trim()) {
42
+ throw new Error("Selected import file is empty.");
43
+ }
44
+ const response = await client.chat.completions.create({
45
+ model: resolveEffectiveHelperModel({
46
+ llmProviderMode: runtimeConfig?.llmProviderMode,
47
+ cuaModel: runtimeConfig?.cuaModel,
48
+ }),
49
+ messages: buildTestDataImportMessages({
50
+ currentContent,
51
+ importedFileName: fileName,
52
+ importedFileText: fileText,
53
+ instructions,
54
+ }),
55
+ response_format: { type: "json_object" },
56
+ });
57
+ const content = response.choices?.[0]?.message?.content || "{}";
58
+ const parsed = parseTestDataImportContent(content);
59
+ return {
60
+ content: normalizeGeneratedContent(parsed.content),
61
+ };
62
+ }
@@ -0,0 +1,103 @@
1
+ import path from "path";
2
+ import { access, readFile } from "fs/promises";
3
+ import { constants as fsConstants } from "fs";
4
+ export const TEST_DATA_FILENAME = "test-data.md";
5
+ export const DEFAULT_TEST_DATA_TEMPLATE = `# Test Data
6
+ <!-- Concrete data the agent can use during Scenario Runs. Keep entries named and specific. -->
7
+ <!-- Examples below are commented out so they are guidance, not usable test data. -->
8
+
9
+ ## Accounts
10
+ <!--
11
+ ### Standard buyer
12
+ Email:
13
+ Password:
14
+ Notes: When should the agent use this account?
15
+ -->
16
+
17
+ ## Products
18
+ <!--
19
+ ### In-stock product
20
+ Name:
21
+ SKU:
22
+ Notes:
23
+ -->
24
+
25
+ ## Payment Methods
26
+ <!--
27
+ ### Valid card
28
+ Number:
29
+ Expiration:
30
+ CVV:
31
+ Notes:
32
+ -->
33
+
34
+ ## Roles and Permissions
35
+
36
+ ## Regions and Locales
37
+
38
+ ## Other Data
39
+ `;
40
+ export function getDefaultProjectTestDataPath(projectPath) {
41
+ return path.join(projectPath, TEST_DATA_FILENAME);
42
+ }
43
+ export function normalizeTestDataTemplateText(text) {
44
+ return String(text ?? "").replace(/\r\n?/g, "\n").replace(/[ \t]+$/gm, "").trimEnd();
45
+ }
46
+ export function isDefaultTestDataTemplate(text) {
47
+ if (!String(text ?? "").trim()) {
48
+ return true;
49
+ }
50
+ return normalizeTestDataTemplateText(text) === normalizeTestDataTemplateText(DEFAULT_TEST_DATA_TEMPLATE);
51
+ }
52
+ export async function readTestDataFile(filePath) {
53
+ await access(filePath, fsConstants.R_OK);
54
+ return await readFile(filePath, "utf-8");
55
+ }
56
+ export async function loadTestData({ testDataPath }) {
57
+ if (!testDataPath) {
58
+ return { text: "", testDataPath: null };
59
+ }
60
+ const text = await readTestDataFile(testDataPath);
61
+ return { text, testDataPath };
62
+ }
63
+ /**
64
+ * @param {{
65
+ * testDataPath?: string | null,
66
+ * missingFileStatus?: string | null,
67
+ * }} [options]
68
+ */
69
+ export async function resolveExecutionTestData({ testDataPath = null, missingFileStatus = null, } = {}) {
70
+ if (!testDataPath) {
71
+ return {
72
+ status: "omitted",
73
+ text: "",
74
+ testDataPath: null,
75
+ message: null,
76
+ };
77
+ }
78
+ try {
79
+ const result = await loadTestData({ testDataPath });
80
+ return {
81
+ status: result.text.trim().length > 0 ? "loaded" : "empty",
82
+ text: result.text,
83
+ testDataPath: result.testDataPath,
84
+ message: null,
85
+ };
86
+ }
87
+ catch (error) {
88
+ if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
89
+ return {
90
+ status: missingFileStatus,
91
+ text: "",
92
+ testDataPath,
93
+ message: null,
94
+ };
95
+ }
96
+ return {
97
+ status: "failed",
98
+ text: "",
99
+ testDataPath,
100
+ message: error instanceof Error ? error.message : "Unknown test data error",
101
+ };
102
+ }
103
+ }
@@ -2,7 +2,7 @@ import { normalizeBrowserInfoForSession } from "../../browser/normalize.js";
2
2
  import { CAPTURE_SOURCES } from "../capture-sources.js";
3
3
  import { printCliOutput } from "../../utils/console-output.js";
4
4
  import { updateBrowserSessionInfo } from "./browser-session.js";
5
- import { buildFunctionCallOutputItem, buildPreviewOutputContent, buildTextToolOutputItem, } from "./utils.js";
5
+ import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, } from "./utils.js";
6
6
  export async function handleCloseCurrentTabToolCall({ engine, callId, context = null, stepContext = null }) {
7
7
  const session = engine.session;
8
8
  if (session?.targetKind !== "browser" && session?.deviceInfo?.platform !== "browser") {
@@ -44,9 +44,9 @@ export async function handleCloseCurrentTabToolCall({ engine, callId, context =
44
44
  });
45
45
  return {
46
46
  kind: "continue",
47
- outputItem: buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
47
+ outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
48
48
  text: result.message,
49
- annotatedScreenshotBase64: screenshotBase64,
49
+ imageBase64: screenshotBase64,
50
50
  })),
51
51
  currentScreenshot,
52
52
  };
@@ -3,7 +3,7 @@ import { buildExecutionNavigationRecoveryPrompt } from "../../prompts/execution.
3
3
  import { emitDesktopDebug } from "../../utils/desktop-debug.js";
4
4
  import { printCliOutput } from "../../utils/console-output.js";
5
5
  import { updateBrowserSessionInfo } from "./browser-session.js";
6
- import { buildDebugContext, buildFunctionCallOutputItem, buildPreviewOutputContent, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
6
+ import { buildDebugContext, buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
7
7
  export async function handleNavigateToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
8
8
  const session = engine.session;
9
9
  const parsed = parseStringToolArgument(argumentsText, "target", "navigate");
@@ -69,9 +69,9 @@ export async function handleNavigateToolCall({ engine, callId, argumentsText, co
69
69
  const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
70
70
  return {
71
71
  kind: "continue",
72
- outputItem: buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
72
+ outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
73
73
  text: `Navigation complete: ${normalizedUrl}`,
74
- annotatedScreenshotBase64: screenshotBase64,
74
+ imageBase64: screenshotBase64,
75
75
  })),
76
76
  };
77
77
  }
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { createAnnotatedScreenshotFromBase64 } from "../../utils/run-screenshot-recorder.js";
3
3
  import { CAPTURE_SOURCES } from "../capture-sources.js";
4
- import { buildFunctionCallOutputItem, buildPreviewOutputContent, } from "./utils.js";
4
+ import { buildFunctionCallOutputItem, buildToolOutputContent, } from "./utils.js";
5
5
  function buildPreviewClickValidationError(argumentsText) {
6
6
  if (!argumentsText || typeof argumentsText !== "string") {
7
7
  return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
@@ -42,7 +42,7 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
42
42
  const session = engine.session;
43
43
  const parsedArguments = parsePreviewClickArguments(argumentsText);
44
44
  if (!parsedArguments.ok) {
45
- return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({ text: parsedArguments.message })));
45
+ return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: parsedArguments.message })));
46
46
  }
47
47
  try {
48
48
  const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
@@ -52,7 +52,7 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
52
52
  style: "preview",
53
53
  });
54
54
  if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
55
- return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
55
+ return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
56
56
  text: "preview_click failed: could not annotate the current screenshot.",
57
57
  })));
58
58
  }
@@ -99,13 +99,13 @@ export async function handlePreviewClick({ engine, callId, argumentsText, contex
99
99
  },
100
100
  });
101
101
  }
102
- return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
102
+ return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
103
103
  text: previewText,
104
- annotatedScreenshotBase64: previewResult.screenshotBase64,
104
+ imageBase64: previewResult.screenshotBase64,
105
105
  })));
106
106
  }
107
107
  catch (error) {
108
- return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
108
+ return continueWith(buildFunctionCallOutputItem(callId, buildToolOutputContent({
109
109
  text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
110
110
  })));
111
111
  }
@@ -0,0 +1,62 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { CAPTURE_SOURCES } from "../capture-sources.js";
3
+ import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, parseStringToolArgument, } from "./utils.js";
4
+ export async function handleTypeSensitiveTextToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
5
+ const parsed = parseStringToolArgument(argumentsText, "key", "type_sensitive_text");
6
+ if (!parsed.ok) {
7
+ return {
8
+ kind: "continue",
9
+ outputItem: buildTextToolOutputItem(callId, parsed.message),
10
+ };
11
+ }
12
+ const key = parsed.value;
13
+ const secretsByKey = engine.secretsByKey && typeof engine.secretsByKey === "object" ? engine.secretsByKey : {};
14
+ if (!Object.prototype.hasOwnProperty.call(secretsByKey, key)) {
15
+ return {
16
+ kind: "continue",
17
+ outputItem: buildTextToolOutputItem(callId, `Secret key not found: ${key}`),
18
+ };
19
+ }
20
+ const action = {
21
+ type: "type",
22
+ text: String(secretsByKey[key] ?? ""),
23
+ sensitive: true,
24
+ sensitiveKey: key,
25
+ };
26
+ try {
27
+ const result = await engine.handleModelAction(engine.session.deviceId, action, engine.session.deviceInfo.scale, {
28
+ ...context,
29
+ stepId: stepContext?.stepId,
30
+ instructionIndex: stepContext?.instructionIndex,
31
+ eventId: randomUUID(),
32
+ });
33
+ if (result?.success === false) {
34
+ return {
35
+ kind: "fail",
36
+ error: `Sensitive typing failed for ${key}.`,
37
+ };
38
+ }
39
+ }
40
+ catch {
41
+ return {
42
+ kind: "fail",
43
+ error: `Sensitive typing failed for ${key}.`,
44
+ };
45
+ }
46
+ const screenshotBase64 = await engine.getScreenshotAsBase64(engine.session.deviceId, engine.session.deviceInfo);
47
+ const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
48
+ runId: context?.runId,
49
+ sessionId: context?.sessionId,
50
+ stepId: stepContext?.stepId,
51
+ instructionIndex: stepContext?.instructionIndex,
52
+ captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
53
+ });
54
+ return {
55
+ kind: "continue",
56
+ outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({
57
+ text: `Typed secret: ${key}`,
58
+ imageBase64: screenshotBase64,
59
+ })),
60
+ currentScreenshot,
61
+ };
62
+ }
@@ -45,12 +45,12 @@ export function buildFunctionCallOutputItem(callId, output) {
45
45
  export function buildTextToolOutputItem(callId, text) {
46
46
  return buildFunctionCallOutputItem(callId, [{ type: "input_text", text }]);
47
47
  }
48
- export function buildPreviewOutputContent({ text, annotatedScreenshotBase64 = null }) {
48
+ export function buildToolOutputContent({ text, imageBase64 = null }) {
49
49
  const content = [{ type: "input_text", text }];
50
- if (annotatedScreenshotBase64) {
50
+ if (imageBase64) {
51
51
  content.push({
52
52
  type: "input_image",
53
- image_url: `data:image/png;base64,${annotatedScreenshotBase64}`,
53
+ image_url: `data:image/png;base64,${imageBase64}`,
54
54
  detail: "high",
55
55
  });
56
56
  }
@@ -3,6 +3,7 @@ import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js
3
3
  import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
4
4
  import { buildActionEventMeta } from "../action-event-meta.js";
5
5
  import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
6
+ import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
6
7
  import { resolveScrollGesture } from "../scroll-gesture.js";
7
8
  import { execAdb } from "./tools.js";
8
9
  function adbShell(deviceId, command) {
@@ -34,7 +35,8 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
34
35
  y1,
35
36
  x2,
36
37
  y2,
37
- text: typeof text === "string" ? truncateForDebug(text, 300) : undefined,
38
+ text: buildDebugTextField(action, truncateForDebug),
39
+ ...buildSensitiveDebugFields(action),
38
40
  keyCount: Array.isArray(keys) ? keys.length : 0,
39
41
  pathPoints: Array.isArray(path) ? path.length : 0
40
42
  });
@@ -79,7 +81,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
79
81
  }
80
82
  break;
81
83
  case "type":
82
- addOutput({ type: 'action', text: `Typing text: ${text}`, ...meta({ text }) });
84
+ addOutput({ type: 'action', text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
83
85
  const escapedText = text.replace(/(["\\$`])/g, "\\$1").replace(/ /g, "%s");
84
86
  await adbShell(deviceId, `input text "${escapedText}"`);
85
87
  break;
@@ -104,13 +106,15 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
104
106
  default:
105
107
  addOutput({ type: 'info', text: `Unknown action: ${JSON.stringify(action)}` });
106
108
  }
109
+ return { success: true };
107
110
  }
108
111
  catch (error) {
112
+ const outputError = buildActionErrorOutput(action, error);
109
113
  // Log full error details to file
110
114
  logger.error('Action execution error', {
111
- action,
112
- message: error.message,
113
- stack: error.stack
115
+ action: actionForErrorLog(action),
116
+ message: outputError.message,
117
+ stack: isSensitiveAction(action) ? undefined : error.stack
114
118
  });
115
119
  emitDesktopDebug("device.error", "device", {
116
120
  runId: context?.runId,
@@ -121,12 +125,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
121
125
  platform: "android",
122
126
  operation: "action.execute",
123
127
  actionType: action?.type,
124
- message: error.message
128
+ message: outputError.message
125
129
  });
126
130
  // Show user-friendly error message
127
131
  addOutput({
128
132
  type: 'error',
129
- text: `Error executing action: ${error.message}`,
133
+ text: outputError.text,
130
134
  eventType: 'error',
131
135
  actionType: action?.type,
132
136
  runId: context?.runId,
@@ -134,10 +138,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
134
138
  instructionIndex: context?.instructionIndex,
135
139
  payload: {
136
140
  eventId: context?.eventId,
137
- message: error.message,
141
+ message: outputError.message,
138
142
  platform: 'android'
139
143
  }
140
144
  });
141
145
  addOutput({ type: 'info', text: 'Full error details have been logged to the debug log.' });
146
+ return { success: false, error: outputError.message };
142
147
  }
143
148
  }