@loadmill/droid-cua 2.16.0 → 2.18.0

package/build/src/device/cloud/actions.js

@@ -2,6 +2,7 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
+import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
@@ -33,7 +34,8 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             deviceId,
             actionType: action?.type,
             scale,
-            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            text: buildDebugTextField(action, truncateForDebug),
+            ...buildSensitiveDebugFields(action),
             keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
             pathPoints: Array.isArray(action?.path) ? action.path.length : 0
         });
@@ -48,7 +50,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 break;
             }
             case "type": {
-                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
+                addOutput({
+                    type: "action",
+                    text: buildTypeActionText(action, action.text),
+                    ...meta(buildTypeActionPayload(action, action.text))
+                });
                 await session.client.type(session.sessionId, action.text);
                 break;
             }
@@ -133,12 +139,14 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             default:
                 addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
         }
+        return { success: true };
     }
     catch (error) {
+        const outputError = buildActionErrorOutput(action, error);
         logger.error("Cloud action execution error", {
-            action,
-            message: error.message,
-            stack: error.stack,
+            action: actionForErrorLog(action),
+            message: outputError.message,
+            stack: isSensitiveAction(action) ? undefined : error.stack,
             providerId: session.providerId,
             platform
         });
@@ -152,11 +160,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             source: session.providerId,
             operation: "action.execute",
             actionType: action?.type,
-            message: error.message
+            message: outputError.message
         });
         addOutput({
             type: "error",
-            text: `Error executing action: ${error.message}`,
+            text: outputError.text,
             eventType: "error",
             actionType: action?.type,
             runId: context?.runId,
@@ -164,11 +172,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             instructionIndex: context?.instructionIndex,
             payload: {
                 eventId: context?.eventId,
-                message: error.message,
+                message: outputError.message,
                 platform,
                 source: session.providerId
             }
         });
         addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+        return { success: false, error: outputError.message };
     }
 }

package/build/src/device/ios/actions.js

@@ -6,6 +6,7 @@
 import * as appium from "./appium-client.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
 import { buildMobileKeypressPayload, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
+import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
@@ -41,7 +42,8 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             simulatorId,
             actionType: action?.type,
             scale,
-            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            text: buildDebugTextField(action, truncateForDebug),
+            ...buildSensitiveDebugFields(action),
             keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
             pathPoints: Array.isArray(action?.path) ? action.path.length : 0
         });
@@ -59,7 +61,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 break;
             }
             case "type": {
-                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
+                addOutput({
+                    type: "action",
+                    text: buildTypeActionText(action, action.text),
+                    ...meta(buildTypeActionPayload(action, action.text))
+                });
                 await appium.type(session.sessionId, action.text);
                 break;
             }
@@ -138,12 +144,14 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             default:
                 addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
         }
+        return { success: true };
     }
     catch (error) {
+        const outputError = buildActionErrorOutput(action, error);
         logger.error("iOS action execution error", {
-            action,
-            message: error.message,
-            stack: error.stack,
+            action: actionForErrorLog(action),
+            message: outputError.message,
+            stack: isSensitiveAction(action) ? undefined : error.stack,
         });
         emitDesktopDebug("device.error", "device", {
             runId: context?.runId,
@@ -154,11 +162,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             platform: "ios",
             operation: "action.execute",
             actionType: action?.type,
-            message: error.message
+            message: outputError.message
         });
         addOutput({
             type: "error",
-            text: `Error executing action: ${error.message}`,
+            text: outputError.text,
             eventType: "error",
             actionType: action?.type,
             runId: context?.runId,
@@ -166,10 +174,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             instructionIndex: context?.instructionIndex,
             payload: {
                 eventId: context?.eventId,
-                message: error.message,
+                message: outputError.message,
                 platform: "ios"
             }
         });
         addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+        return { success: false, error: outputError.message };
     }
 }

package/build/src/device/openai.js

@@ -107,6 +107,20 @@ const loadmillTool = {
         additionalProperties: false,
     },
 };
+const typeSensitiveTextTool = {
+    type: "function",
+    name: "type_sensitive_text",
+    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            key: { type: "string" },
+        },
+        required: ["key"],
+        additionalProperties: false,
+    },
+};
 const testDesignCompleteTool = {
     type: "function",
     name: "test_design_complete",
@@ -156,6 +170,9 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
     }
+    if (toolCapabilities.typeSensitiveText) {
+        tools.push(typeSensitiveTextTool);
+    }
     if (toolCapabilities.testDesignComplete) {
         tools.push(testDesignCompleteTool);
     }

package/build/src/device/sensitive-action.js

@@ -0,0 +1,61 @@
+export function isSensitiveAction(action) {
+    return action?.sensitive === true;
+}
+export function getSensitiveKey(action) {
+    return typeof action?.sensitiveKey === "string" && action.sensitiveKey.trim()
+        ? action.sensitiveKey.trim()
+        : "secret";
+}
+export function buildTypeActionText(action, text) {
+    if (isSensitiveAction(action)) {
+        return `Typing secret: ${getSensitiveKey(action)}`;
+    }
+    return `Typing text: ${text}`;
+}
+export function buildTypeActionPayload(action, text) {
+    if (isSensitiveAction(action)) {
+        return {
+            sensitive: true,
+            secretKey: getSensitiveKey(action),
+        };
+    }
+    return { text };
+}
+export function buildDebugTextField(action, truncate) {
+    if (isSensitiveAction(action)) {
+        return undefined;
+    }
+    return typeof action?.text === "string" ? truncate(action.text, 300) : undefined;
+}
+export function buildSensitiveDebugFields(action) {
+    if (!isSensitiveAction(action)) {
+        return {};
+    }
+    return {
+        sensitive: true,
+        secretKey: getSensitiveKey(action),
+    };
+}
+export function actionForErrorLog(action) {
+    if (!isSensitiveAction(action)) {
+        return action;
+    }
+    return {
+        type: action?.type,
+        sensitive: true,
+        secretKey: getSensitiveKey(action),
+    };
+}
+export function buildActionErrorOutput(action, error) {
+    if (isSensitiveAction(action)) {
+        const key = getSensitiveKey(action);
+        return {
+            text: `Error typing secret: ${key}`,
+            message: `Sensitive typing failed for ${key}.`,
+        };
+    }
+    return {
+        text: `Error executing action: ${error.message}`,
+        message: error.message,
+    };
+}

package/build/src/modes/design-mode-ink.js

@@ -33,7 +33,8 @@ export class DesignModeInk {
     async start() {
         const addOutput = this.context.addOutput;
         // Set design mode system prompt
-        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
+        const availableSecretKeys = this.engine.getAvailableSecretKeys();
+        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, "", { availableSecretKeys });
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
         this.engine.toolCapabilities = {
@@ -41,6 +42,7 @@ export class DesignModeInk {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             loadmill: false,
+            typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,
         };
         this.engine.baseSystemPrompt = designPrompt;

package/build/src/modes/design-mode.js

@@ -30,7 +30,8 @@ export class DesignMode {
      */
     async start(context) {
         // Set design mode system prompt
-        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
+        const availableSecretKeys = this.engine.getAvailableSecretKeys();
+        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, "", { availableSecretKeys });
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
         this.engine.toolCapabilities = {
@@ -38,6 +39,7 @@ export class DesignMode {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             loadmill: false,
+            typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,
         };
         this.engine.baseSystemPrompt = designPrompt;

package/build/src/modes/execution-mode.js

@@ -70,6 +70,7 @@ export class ExecutionMode {
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
             loadmill: true,
+            typeSensitiveText: this.engine.hasSecrets(),
         };
         this.engine.baseSystemPrompt = this.initialSystemText;
         this.engine.isHeadlessMode = this.isHeadlessMode;

package/build/src/prompts/design.js

@@ -1,9 +1,10 @@
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
-import { buildAppContextSection } from "./execution.js";
-export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "") {
+import { buildAppContextSection, buildSecretKeysContextSection } from "./execution.js";
+export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
     const appContextSection = buildAppContextSection(appContextText);
+    const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -93,7 +94,7 @@ WRONG Example (DON'T DO THIS):
 3. etc...
 \`\`\`
 
-Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}
+Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
 `;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },

package/build/src/prompts/execution.js

@@ -1,4 +1,5 @@
 import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
+import { normalizeSecretKeys } from "../core/secrets.js";
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
 export function buildAppContextSection(text) {
     const trimmed = typeof text === "string" ? text.trim() : "";
@@ -11,11 +12,36 @@ Use this to understand screen layouts, terminology, navigation, and expected beh
 
 ${trimmed}`;
 }
+export function buildTestDataContextSection(text) {
+    const trimmed = typeof text === "string" ? text.trim() : "";
+    if (!trimmed) {
+        return "";
+    }
+    return `TEST DATA FOR THIS RUN:
+The following values are test data for this execution. Use them as literal run data when performing the test.
+They do not replace or override the test instructions.
+
+${trimmed}`;
+}
+export function buildSecretKeysContextSection(keys = []) {
+    const normalized = normalizeSecretKeys(keys);
+    if (normalized.length === 0) {
+        return "";
+    }
+    return `SECRET KEYS FOR THIS RUN:
+The following secret keys are available. You can see only the keys, not the values.
+Use the \`type_sensitive_text\` tool when you need to enter passwords, OTPs, tokens, API keys, or other hidden credentials.
+Never ask the user to reveal these values and never type placeholder text when a matching secret key is available.
+
+${normalized.map((key) => `- ${key}`).join("\n")}`;
+}
 export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
     const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
     const appContextSection = buildAppContextSection(appContextText);
+    const testDataContextSection = buildTestDataContextSection(options.testDataContext);
+    const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -52,7 +78,7 @@ Your process:
 7. Stop immediately - no commentary, no questions
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
-${appContextSection ? `\n\n${appContextSection}` : ""}`;
+${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
         { title: "Execution Mode Instructions", text: executionCustomText }

package/build/src/prompts/scenario-planner.js

@@ -0,0 +1,48 @@
+function formatCandidateTests(tests) {
+    return tests.map((test, index) => {
+        const content = Array.isArray(test.instructions) ? test.instructions.join("\n") : "";
+        return [
+            `TEST ${index + 1}`,
+            `Name: ${test.testName}`,
+            "Instructions:",
+            content || "(empty)",
+        ].join("\n");
+    }).join("\n\n---\n\n");
+}
+export function buildScenarioPlannerMessages({ scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
+    return [
+        {
+            role: "system",
+            content: [
+                "You prepare concrete test executions for a computer-using agent.",
+                "The agent runs manual-style app tests: it receives a test brief, app context, and optional run data, then operates the app UI.",
+                "Your job is to turn the user's scenario request into the exact ordered cases the agent should run.",
+                "Return JSON only with this shape:",
+                "{\"cases\":[{\"testName\":\"exact-test-name.dcua\",\"caseName\":\"test-name.dcua / data label\",\"testDataContext\":\"raw data text for this case\"}]}",
+                "Rules:",
+                "- testName must exactly match one of the provided candidate test names.",
+                "- Create an ordered flat list of cases to run.",
+                "- caseName is an ephemeral display name for logs and reports. Keep it short and human-readable.",
+                "- testDataContext must contain only the concrete data relevant to that case.",
+                "- Do not invent data that was not provided in test data, context, or the scenario prompt.",
+                "- If no extra data is needed for a case, use an empty string for testDataContext.",
+            ].join("\n")
+        },
+        {
+            role: "user",
+            content: [
+                "Scenario prompt:",
+                scenarioPrompt,
+                "",
+                "App context:",
+                appContextText && appContextText.trim() ? appContextText.trim() : "(none)",
+                "",
+                "Project test data:",
+                testDataText && testDataText.trim() ? testDataText.trim() : "(none)",
+                "",
+                "Candidate tests:",
+                formatCandidateTests(tests),
+            ].join("\n")
+        }
+    ];
+}

package/build/src/prompts/test-data-import.js

@@ -0,0 +1,49 @@
+import { DEFAULT_TEST_DATA_TEMPLATE, isDefaultTestDataTemplate } from "../core/test-data.js";
+export function buildTestDataImportMessages({ currentContent = "", importedFileName, importedFileText, instructions = "", }) {
+    const currentIsTemplateOnly = isDefaultTestDataTemplate(currentContent);
+    const userInstructions = typeof instructions === "string" && instructions.trim()
+        ? instructions.trim()
+        : "(none)";
+    return [
+        {
+            role: "system",
+            content: [
+                "You convert imported data into durable project test data for computer-using-agent scenario runs.",
+                "The durable format is test-data.md: readable markdown that describes concrete entities and run data the agent can use.",
+                "Return JSON only with this shape:",
+                "{\"content\":\"# Test Data\\n...complete markdown...\"}",
+                "Rules:",
+                "- Return the complete replacement content for test-data.md, not a patch.",
+                "- Do not invent values that are not present in the current file, imported file, or user instructions.",
+                "- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
+                "- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
+                "- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
+                "- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
+                "- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
+                "- Output markdown only inside the JSON content string; do not use code fences.",
+            ].join("\n")
+        },
+        {
+            role: "user",
+            content: [
+                "Current test-data.md status:",
+                currentIsTemplateOnly ? "template-only" : "existing user content",
+                "",
+                "Default template used for template-only detection:",
+                DEFAULT_TEST_DATA_TEMPLATE.trimEnd(),
+                "",
+                "Current test-data.md content:",
+                String(currentContent ?? "").trim() ? String(currentContent) : "(empty)",
+                "",
+                "Imported file:",
+                importedFileName,
+                "",
+                "Imported file text:",
+                importedFileText,
+                "",
+                "User import instructions:",
+                userInstructions,
+            ].join("\n")
+        }
+    ];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.16.0",
+  "version": "2.18.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",