@loadmill/droid-cua 2.17.0 → 2.18.0

package/build/src/device/cloud/actions.js

@@ -2,6 +2,7 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
+import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
@@ -33,7 +34,8 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             deviceId,
             actionType: action?.type,
             scale,
-            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            text: buildDebugTextField(action, truncateForDebug),
+            ...buildSensitiveDebugFields(action),
             keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
             pathPoints: Array.isArray(action?.path) ? action.path.length : 0
         });
@@ -48,7 +50,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 break;
             }
             case "type": {
-                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
+                addOutput({
+                    type: "action",
+                    text: buildTypeActionText(action, action.text),
+                    ...meta(buildTypeActionPayload(action, action.text))
+                });
                 await session.client.type(session.sessionId, action.text);
                 break;
             }
@@ -133,12 +139,14 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             default:
                 addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
         }
+        return { success: true };
     }
     catch (error) {
+        const outputError = buildActionErrorOutput(action, error);
         logger.error("Cloud action execution error", {
-            action,
-            message: error.message,
-            stack: error.stack,
+            action: actionForErrorLog(action),
+            message: outputError.message,
+            stack: isSensitiveAction(action) ? undefined : error.stack,
             providerId: session.providerId,
             platform
         });
@@ -152,11 +160,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             source: session.providerId,
             operation: "action.execute",
             actionType: action?.type,
-            message: error.message
+            message: outputError.message
         });
         addOutput({
             type: "error",
-            text: `Error executing action: ${error.message}`,
+            text: outputError.text,
             eventType: "error",
             actionType: action?.type,
             runId: context?.runId,
@@ -164,11 +172,12 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             instructionIndex: context?.instructionIndex,
             payload: {
                 eventId: context?.eventId,
-                message: error.message,
+                message: outputError.message,
                 platform,
                 source: session.providerId
             }
         });
         addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+        return { success: false, error: outputError.message };
     }
 }

package/build/src/device/ios/actions.js

@@ -6,6 +6,7 @@
 import * as appium from "./appium-client.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
 import { buildMobileKeypressPayload, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
+import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
@@ -41,7 +42,8 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             simulatorId,
             actionType: action?.type,
             scale,
-            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            text: buildDebugTextField(action, truncateForDebug),
+            ...buildSensitiveDebugFields(action),
             keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
             pathPoints: Array.isArray(action?.path) ? action.path.length : 0
         });
@@ -59,7 +61,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 break;
             }
             case "type": {
-                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
+                addOutput({
+                    type: "action",
+                    text: buildTypeActionText(action, action.text),
+                    ...meta(buildTypeActionPayload(action, action.text))
+                });
                 await appium.type(session.sessionId, action.text);
                 break;
             }
@@ -138,12 +144,14 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             default:
                 addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
         }
+        return { success: true };
     }
     catch (error) {
+        const outputError = buildActionErrorOutput(action, error);
         logger.error("iOS action execution error", {
-            action,
-            message: error.message,
-            stack: error.stack,
+            action: actionForErrorLog(action),
+            message: outputError.message,
+            stack: isSensitiveAction(action) ? undefined : error.stack,
         });
         emitDesktopDebug("device.error", "device", {
             runId: context?.runId,
@@ -154,11 +162,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             platform: "ios",
             operation: "action.execute",
             actionType: action?.type,
-            message: error.message
+            message: outputError.message
         });
         addOutput({
             type: "error",
-            text: `Error executing action: ${error.message}`,
+            text: outputError.text,
             eventType: "error",
             actionType: action?.type,
             runId: context?.runId,
@@ -166,10 +174,11 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             instructionIndex: context?.instructionIndex,
             payload: {
                 eventId: context?.eventId,
-                message: error.message,
+                message: outputError.message,
                 platform: "ios"
             }
         });
         addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+        return { success: false, error: outputError.message };
     }
 }

package/build/src/device/openai.js

@@ -107,6 +107,20 @@ const loadmillTool = {
         additionalProperties: false,
     },
 };
+const typeSensitiveTextTool = {
+    type: "function",
+    name: "type_sensitive_text",
+    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            key: { type: "string" },
+        },
+        required: ["key"],
+        additionalProperties: false,
+    },
+};
 const testDesignCompleteTool = {
     type: "function",
     name: "test_design_complete",
@@ -156,6 +170,9 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
     }
+    if (toolCapabilities.typeSensitiveText) {
+        tools.push(typeSensitiveTextTool);
+    }
     if (toolCapabilities.testDesignComplete) {
         tools.push(testDesignCompleteTool);
     }

package/build/src/device/sensitive-action.js

@@ -0,0 +1,61 @@
+export function isSensitiveAction(action) {
+    return action?.sensitive === true;
+}
+export function getSensitiveKey(action) {
+    return typeof action?.sensitiveKey === "string" && action.sensitiveKey.trim()
+        ? action.sensitiveKey.trim()
+        : "secret";
+}
+export function buildTypeActionText(action, text) {
+    if (isSensitiveAction(action)) {
+        return `Typing secret: ${getSensitiveKey(action)}`;
+    }
+    return `Typing text: ${text}`;
+}
+export function buildTypeActionPayload(action, text) {
+    if (isSensitiveAction(action)) {
+        return {
+            sensitive: true,
+            secretKey: getSensitiveKey(action),
+        };
+    }
+    return { text };
+}
+export function buildDebugTextField(action, truncate) {
+    if (isSensitiveAction(action)) {
+        return undefined;
+    }
+    return typeof action?.text === "string" ? truncate(action.text, 300) : undefined;
+}
+export function buildSensitiveDebugFields(action) {
+    if (!isSensitiveAction(action)) {
+        return {};
+    }
+    return {
+        sensitive: true,
+        secretKey: getSensitiveKey(action),
+    };
+}
+export function actionForErrorLog(action) {
+    if (!isSensitiveAction(action)) {
+        return action;
+    }
+    return {
+        type: action?.type,
+        sensitive: true,
+        secretKey: getSensitiveKey(action),
+    };
+}
+export function buildActionErrorOutput(action, error) {
+    if (isSensitiveAction(action)) {
+        const key = getSensitiveKey(action);
+        return {
+            text: `Error typing secret: ${key}`,
+            message: `Sensitive typing failed for ${key}.`,
+        };
+    }
+    return {
+        text: `Error executing action: ${error.message}`,
+        message: error.message,
+    };
+}

package/build/src/modes/design-mode-ink.js

@@ -33,7 +33,8 @@ export class DesignModeInk {
     async start() {
         const addOutput = this.context.addOutput;
         // Set design mode system prompt
-        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
+        const availableSecretKeys = this.engine.getAvailableSecretKeys();
+        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, "", { availableSecretKeys });
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
         this.engine.toolCapabilities = {
@@ -41,6 +42,7 @@ export class DesignModeInk {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             loadmill: false,
+            typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,
         };
         this.engine.baseSystemPrompt = designPrompt;

package/build/src/modes/design-mode.js

@@ -30,7 +30,8 @@ export class DesignMode {
      */
     async start(context) {
         // Set design mode system prompt
-        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
+        const availableSecretKeys = this.engine.getAvailableSecretKeys();
+        const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {}, "", { availableSecretKeys });
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
         this.engine.toolCapabilities = {
@@ -38,6 +39,7 @@ export class DesignMode {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             loadmill: false,
+            typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,
         };
         this.engine.baseSystemPrompt = designPrompt;

package/build/src/modes/execution-mode.js

@@ -70,6 +70,7 @@ export class ExecutionMode {
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
             loadmill: true,
+            typeSensitiveText: this.engine.hasSecrets(),
         };
         this.engine.baseSystemPrompt = this.initialSystemText;
         this.engine.isHeadlessMode = this.isHeadlessMode;

package/build/src/prompts/design.js

@@ -1,9 +1,10 @@
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
-import { buildAppContextSection } from "./execution.js";
-export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "") {
+import { buildAppContextSection, buildSecretKeysContextSection } from "./execution.js";
+export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
     const appContextSection = buildAppContextSection(appContextText);
+    const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -93,7 +94,7 @@ WRONG Example (DON'T DO THIS):
 3. etc...
 \`\`\`
 
-Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}
+Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
 `;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },

package/build/src/prompts/execution.js

@@ -1,4 +1,5 @@
 import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
+import { normalizeSecretKeys } from "../core/secrets.js";
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
 export function buildAppContextSection(text) {
     const trimmed = typeof text === "string" ? text.trim() : "";
@@ -22,12 +23,25 @@ They do not replace or override the test instructions.
 
 ${trimmed}`;
 }
+export function buildSecretKeysContextSection(keys = []) {
+    const normalized = normalizeSecretKeys(keys);
+    if (normalized.length === 0) {
+        return "";
+    }
+    return `SECRET KEYS FOR THIS RUN:
+The following secret keys are available. You can see only the keys, not the values.
+Use the \`type_sensitive_text\` tool when you need to enter passwords, OTPs, tokens, API keys, or other hidden credentials.
+Never ask the user to reveal these values and never type placeholder text when a matching secret key is available.
+
+${normalized.map((key) => `- ${key}`).join("\n")}`;
+}
 export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
     const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
     const appContextSection = buildAppContextSection(appContextText);
     const testDataContextSection = buildTestDataContextSection(options.testDataContext);
+    const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -64,7 +78,7 @@ Your process:
 7. Stop immediately - no commentary, no questions
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
-${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}`;
+${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
         { title: "Execution Mode Instructions", text: executionCustomText }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.17.0",
+  "version": "2.18.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",