@loadmill/droid-cua 2.2.2 → 2.4.0

package/build/src/prompts/execution.js

@@ -0,0 +1,182 @@
+import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection } from "./base.js";
+export function buildAppContextSection(briefing) {
+    const text = typeof briefing === "string" ? briefing.trim() : "";
+    if (!text) {
+        return "";
+    }
+    return `APP CONTEXT BRIEFING:
+The following is a condensed description of the app you are testing, relevant to the current task.
+Use this to understand screen layouts, terminology, navigation, and expected behavior.
+
+${text}`;
+}
+export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextBriefing = "", runtimeOptions = {}) {
+    const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
+    const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
+    const appContextSection = buildAppContextSection(appContextBriefing);
+    const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
+    const prompt = `${basePrompt}
+
+EXECUTION MODE - Critical Behavior:
+You are executing test script commands one at a time. This is NOT a conversation.
+
+CRITICAL RULES:
+- DO NOT generate conversational text or narration
+- DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
+- DO NOT describe what you see on screen
+- DO NOT say "Let me know if you need help" or similar phrases
+- Just execute the action silently and stop immediately
+- Only generate text if the action FAILED or cannot be completed
+- Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses
+- Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
+- If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
+- For form submissions, cleared fields plus a reset action button are strong success signals; stop even if the created item is not yet obvious in the visible list
+- If target is not visible, perform bounded off-screen discovery first:
+  1. Scroll the screen in the likely direction to reveal hidden controls
+  2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry
+
+Your process:
+1. Read the instruction
+2. Execute the required actions
+3. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
+4. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
+5. Stop as soon as success is visible
+6. Stop immediately - no commentary, no questions
+
+Each instruction is independent. Do not reference previous instructions or ask about next steps.
+${appContextSection ? `\n\n${appContextSection}` : ""}` + (strictModeSection ? `\n\n${strictModeSection}` : "");
+    return appendCustomSections(prompt, [
+        { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
+        { title: "Execution Mode Instructions", text: executionCustomText }
+    ]);
+}
+export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
+    if (!transcript) {
+        return basePrompt;
+    }
+    return `${basePrompt}
+
+[SESSION RECOVERY - Connection was lost. Previous actions completed before the error:]
+${transcript}
+
+[IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]`;
+}
+export function buildAssertionSystemPrompt(baseSystemPrompt, assertionPrompt) {
+    return `${baseSystemPrompt}
+
+ASSERTION MODE:
+You are now validating an assertion. The user has provided an assertion statement that you must verify.
+
+Your task:
+1. Take screenshots and perform LIMITED actions if needed to validate the assertion.
+2. Determine if the assertion is TRUE or FALSE based on the current state.
+3. You MUST respond with a clear verdict in this exact format:
+   - If the assertion is true, include the text: "ASSERTION RESULT: PASS"
+   - If the assertion is false or cannot be confidently validated, include: "ASSERTION RESULT: FAIL"
+4. After the verdict, provide a brief explanation (1-2 sentences) of why it passed or failed.
+
+The assertion to validate is: "${assertionPrompt}"
+
+Remember:
+- If you cannot confidently validate the assertion, treat it as FAIL.
+- You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
+- Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
+}
+export function buildAppContextCompactionInput({ contextDocument, taskDescription, tokenBudget }) {
+    return [
+        {
+            role: "system",
+            content: [{
+                    type: "input_text",
+                    text: `You are compressing an app context document for a mobile testing agent.
+
+You will receive:
+1. A context document
+2. A test task
+
+Your job is to SELECT only the facts from the context document that are useful for the given task.
+The output will be injected into a system prompt with a strict token budget.
+
+CRITICAL:
+- Use only facts explicitly supported by the context document
+- Never invent, infer, normalize, substitute, or improve credentials, labels, screen names, button names, or numeric values
+- Preserve exact values verbatim when present in the source
+- Prefer facts that help the agent act correctly when they are not obvious from the task alone
+- Do not restate, paraphrase, summarize, or reorganize the test task
+- The output must not read like instructions or a test plan
+- Do not describe what the agent should do
+- Output only reference knowledge about the app
+- If a line could be copied from the task with minor wording changes, omit it
+- Prefer copying source facts verbatim or near-verbatim over rewriting them
+- Do not collapse multiple specific source facts into one generic summary if that removes useful distinctions
+
+Selection priority:
+1. Facts the agent would NOT know from the test script alone
+2. Facts that are hard to infer from screenshots
+3. Non-obvious navigation or interaction details
+4. Exact visible labels needed to act correctly
+5. Credentials and other exact values
+
+High-value facts:
+- exact UI labels
+- how state, mode, or account selection is performed
+- where logout is located
+- hidden or non-obvious navigation
+- which menu items are decorative or non-functional
+- screen titles and section labels used to confirm location
+- exact credentials and role labels
+
+Low-value facts:
+- restating the test steps
+- repeating literal values already present in the task
+- generic summaries like "approve the transaction"
+
+When the task involves authentication, switching state or mode, opening menus, or moving between major areas of the app, strongly prefer including:
+- how account, state, or mode selection is performed
+- exact visible labels for the relevant controls
+- where exit or sign-out actions are located
+- the screen or section labels that confirm the agent is in the right place
+
+Rules:
+- Output plain text only
+- No markdown, no bullet symbols, no numbering, no headers
+- Use terse, factual language: one fact per line, no filler words
+- Blank lines only to separate logical groups
+- Prefer exact visible UI labels over summaries
+- Do not describe step-by-step procedures
+- Do not restate the test workflow
+- State only facts about screens, elements, hidden interactions, entities, credentials, and navigation
+- If a useful fact is not explicitly stated in the context document, omit it
+- Include only information relevant to this task
+- Do not waste space repeating the task itself
+- If the task already states a value or action, include it only when the context adds non-obvious execution details
+- Return a short result or an empty string if little is relevant
+- Target: under ${tokenBudget} tokens
+
+Bad output patterns to avoid:
+- generic summaries that remove actionable details
+- lines that restate the task in generic prose
+- lines that describe obvious workflow steps instead of app knowledge
+- lines that replace exact source labels or mechanisms with broad summaries
+
+Good output characteristics:
+- preserves the exact label or mechanism from the source when it matters
+- keeps distinctions like dropdown vs tabs, drawer vs visible button, exact section titles, exact button text
+- includes hidden or non-obvious navigation details when relevant
+
+Return only the briefing text.`
+                }]
+        },
+        {
+            role: "user",
+            content: [{
+                    type: "input_text",
+                    text: `APP CONTEXT DOCUMENT:
+${contextDocument}
+
+TASK:
+${taskDescription}`
+                }]
+        }
+    ];
+}

package/build/src/prompts/loadmill.js

@@ -0,0 +1,60 @@
+export function buildLoadmillCommandInterpretationMessages(userInput) {
+    return [
+        {
+            role: "system",
+            content: `You are a parser that extracts structured data from natural language Loadmill commands.
+
+Extract the following from the user's input:
+1. searchQuery: The flow name or description to search for (required). FIX any obvious typos or misspellings.
+2. parameters: Any key=value pairs mentioned (as an object)
+3. action: Either "run" (if user wants to execute) or "search" (if user just wants to find flows)
+
+Output JSON only, no markdown or explanation.
+
+Examples:
+Input: "run the checkout flow with user=test123"
+Output: {"searchQuery": "checkout flow", "parameters": {"user": "test123"}, "action": "run"}
+
+Input: "search for login test"
+Output: {"searchQuery": "login test", "parameters": {}, "action": "search"}
+
+Input: "run user authentication with email=test@example.com password=secret123"
+Output: {"searchQuery": "user authentication", "parameters": {"email": "test@example.com", "password": "secret123"}, "action": "run"}
+
+Input: "execute payment flow"
+Output: {"searchQuery": "payment flow", "parameters": {}, "action": "run"}
+
+Input: "create a transction with amount=200"
+Output: {"searchQuery": "transaction", "parameters": {"amount": "200"}, "action": "run"}`
+        },
+        {
+            role: "user",
+            content: userInput
+        }
+    ];
+}
+export function buildLoadmillFlowSelectionMessages(originalQuery, flowList) {
+    return [
+        {
+            role: "system",
+            content: `You are selecting the best matching test flow based on a user query.
+
+Given the user's query and a list of available flows, select the best match.
+
+Output JSON with:
+- index: 1-based index of the best matching flow
+- confidence: number between 0 and 1 indicating how confident you are
+
+If no flow seems to match well, set confidence to a low value (< 0.5).
+
+Output JSON only, no markdown.`
+        },
+        {
+            role: "user",
+            content: `Query: "${originalQuery}"
+
+Available flows:
+${flowList}`
+        }
+    ];
+}

package/build/src/utils/console-output.js

@@ -0,0 +1,35 @@
+export function formatCliOutput(item) {
+    if (item == null) {
+        return null;
+    }
+    if (typeof item === "string" || typeof item === "number" || typeof item === "boolean") {
+        return String(item);
+    }
+    if (typeof item !== "object") {
+        return String(item);
+    }
+    const text = typeof item.text === "string" ? item.text : null;
+    const type = typeof item.type === "string" ? item.type : "";
+    const eventType = typeof item.eventType === "string" ? item.eventType : "";
+    const isAssistantMessage = type === "assistant" || eventType === "assistant_message";
+    if (isAssistantMessage) {
+        if (!text || text.trim().length === 0) {
+            return null;
+        }
+        return text
+            .split("\n")
+            .map((line) => `assistant: ${line}`)
+            .join("\n");
+    }
+    if (text !== null) {
+        return text;
+    }
+    return item;
+}
+export function printCliOutput(item, consoleLike = console) {
+    const formatted = formatCliOutput(item);
+    if (formatted == null) {
+        return;
+    }
+    consoleLike.log(formatted);
+}

package/build/src/utils/run-screenshot-recorder.js

@@ -0,0 +1,98 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+function pad2(value) {
+    return String(value).padStart(2, "0");
+}
+function pad3(value) {
+    return String(value).padStart(3, "0");
+}
+export function formatArtifactTimestamp(date = new Date()) {
+    return `${date.getFullYear()}${pad2(date.getMonth() + 1)}${pad2(date.getDate())}-${pad2(date.getHours())}${pad2(date.getMinutes())}${pad2(date.getSeconds())}-${pad3(date.getMilliseconds())}`;
+}
+function sanitizeArtifactSegment(value, fallback = "item") {
+    if (typeof value !== "string") {
+        return fallback;
+    }
+    const normalized = value
+        .trim()
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^[-_.]+|[-_.]+$/g, "");
+    return normalized || fallback;
+}
+function buildInstructionSegment(instructionIndex) {
+    if (!Number.isInteger(instructionIndex) || instructionIndex < 0) {
+        return null;
+    }
+    return `instruction-${String(instructionIndex + 1).padStart(3, "0")}`;
+}
+function buildCallSegment(callId) {
+    if (typeof callId !== "string" || !callId.trim()) {
+        return null;
+    }
+    return `call-${sanitizeArtifactSegment(callId, "call").slice(0, 24)}`;
+}
+function buildScreenshotFileName(sequence, metadata = {}) {
+    const segments = [
+        String(sequence).padStart(4, "0"),
+        formatArtifactTimestamp(metadata.timestamp instanceof Date ? metadata.timestamp : new Date()),
+        sanitizeArtifactSegment(metadata.captureSource || "screenshot", "screenshot")
+    ];
+    const stepIdSegment = sanitizeArtifactSegment(metadata.stepId || "", "");
+    if (stepIdSegment) {
+        segments.push(stepIdSegment);
+    }
+    const instructionSegment = buildInstructionSegment(metadata.instructionIndex);
+    if (instructionSegment) {
+        segments.push(instructionSegment);
+    }
+    const callSegment = buildCallSegment(metadata.callId);
+    if (callSegment) {
+        segments.push(callSegment);
+    }
+    return `${segments.join("_")}.png`;
+}
+export function createDebugScreenshotRecorder({ directoryPath }) {
+    let nextSequence = 1;
+    let ensuredDirectoryPromise = null;
+    async function ensureDirectory() {
+        if (!ensuredDirectoryPromise) {
+            ensuredDirectoryPromise = mkdir(directoryPath, { recursive: true });
+        }
+        await ensuredDirectoryPromise;
+        return directoryPath;
+    }
+    return {
+        directoryPath,
+        async ensureDirectory() {
+            return await ensureDirectory();
+        },
+        async saveScreenshot(screenshotBase64, metadata = {}) {
+            if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+                return null;
+            }
+            await ensureDirectory();
+            const fileName = buildScreenshotFileName(nextSequence++, metadata);
+            const filePath = path.join(directoryPath, fileName);
+            await writeFile(filePath, Buffer.from(screenshotBase64, "base64"));
+            return filePath;
+        }
+    };
+}
+export function createCompositeScreenshotRecorder({ recorders }) {
+    const activeRecorders = Array.isArray(recorders) ? recorders.filter(Boolean) : [];
+    return {
+        directoryPath: activeRecorders.map((recorder) => recorder.directoryPath).filter(Boolean),
+        async ensureDirectory() {
+            await Promise.all(activeRecorders.map((recorder) => recorder.ensureDirectory?.()));
+            return this.directoryPath;
+        },
+        async saveScreenshot(screenshotBase64, metadata = {}) {
+            if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+                return null;
+            }
+            const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveScreenshot(screenshotBase64, metadata)));
+            return results.filter(Boolean);
+        }
+    };
+}