@loadmill/droid-cua 2.16.0 → 2.18.0

package/build/src/core/execution-engine.js

@@ -5,9 +5,11 @@ import { handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
 import { executeLoadmillInstruction } from "../device/loadmill.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
+import { createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
 import { CAPTURE_SOURCES } from "./capture-sources.js";
+import { normalizeSecretKeys } from "./secrets.js";
 import { ScreenshotStore } from "./screenshot-store.js";
 import { handlePreviewClick } from "./tool-handlers/preview-click.js";
 import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
@@ -15,6 +17,7 @@ import { handleCloseCurrentTabToolCall } from "./tool-handlers/close-current-tab
 import { handleAssertionToolCall } from "./tool-handlers/assert.js";
 import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
 import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
+import { handleTypeSensitiveTextToolCall } from "./tool-handlers/type-sensitive-text.js";
 import { buildDebugContext, buildTextToolOutputItem, } from "./tool-handlers/utils.js";
 function extractActionableOutputItems(items) {
     const entries = [];
@@ -77,11 +80,33 @@ export class ExecutionEngine {
         this.closeCurrentTab = options.closeCurrentTab || closeCurrentBrowserTab;
         this.executeLoadmillInstruction = options.executeLoadmillInstruction || executeLoadmillInstruction;
         this.toolCapabilities = options.toolCapabilities || {};
+        this.secretsByKey = options.secretsByKey && typeof options.secretsByKey === "object" ? options.secretsByKey : {};
         this.baseSystemPrompt = options.baseSystemPrompt || session?.systemPrompt || null;
         this.isHeadlessMode = options.isHeadlessMode === true;
     }
+    hasSecrets() {
+        return this.getAvailableSecretKeys().length > 0;
+    }
+    getAvailableSecretKeys() {
+        return normalizeSecretKeys(Object.keys(this.secretsByKey));
+    }
     async recordScreenshot(screenshotBase64, metadata = {}) {
-        return await this.screenshotStore.saveScreenshot(screenshotBase64, metadata);
+        const saved = await this.screenshotStore.saveScreenshot(screenshotBase64, metadata);
+        if (saved) {
+            return saved;
+        }
+        // No recorder/persistence (e.g. a plain CLI `--report` run): still hold onto the raw
+        // base64 so downstream click-annotation can render the marker in-memory for the report.
+        if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+            return null;
+        }
+        return {
+            base64: screenshotBase64,
+            artifactPaths: [],
+            annotationSourcePaths: [],
+            screenshotPath: null,
+            screenshotDataUrl: null,
+        };
     }
     async annotateScreenshot(currentScreenshot, action, metadata = {}) {
         if (!(action?.type === "click" || action?.type === "double_click")) {
@@ -89,6 +114,30 @@ export class ExecutionEngine {
         }
         return await this.screenshotStore.annotateScreenshot(currentScreenshot, { x: action.x, y: action.y, style: "click" }, { ...metadata, actionType: action.type });
     }
+    // Resolves the click-annotation data URL for an action event when a self-contained
+    // report is being produced (`includeScreenshotDataUrl`). Prefers the recorder's own
+    // annotated data URL (Desktop); otherwise renders the green "click" marker in-memory
+    // from the raw screenshot so reports work even without a disk recorder.
+    async resolveAnnotatedDataUrl(annotatedScreenshot, currentScreenshot, action, context) {
+        if (!context?.includeScreenshotDataUrl) {
+            return null;
+        }
+        if (annotatedScreenshot?.screenshotDataUrl) {
+            return annotatedScreenshot.screenshotDataUrl;
+        }
+        if (!(action?.type === "click" || action?.type === "double_click")) {
+            return null;
+        }
+        if (typeof currentScreenshot?.base64 !== "string" || !currentScreenshot.base64) {
+            return null;
+        }
+        const annotated = await createAnnotatedScreenshotFromBase64(currentScreenshot.base64, {
+            x: action.x,
+            y: action.y,
+            style: "click",
+        });
+        return annotated?.screenshotBase64 ? `data:image/png;base64,${annotated.screenshotBase64}` : null;
+    }
     /**
      * Dispatches a function-call tool to its handler. Each handler returns a
      * discriminated outcome that runFullTurn interprets below. Continue and
@@ -108,6 +157,8 @@ export class ExecutionEngine {
                 return await handleCloseCurrentTabToolCall(handlerArgs);
             case "loadmill":
                 return await handleLoadmillToolCall(handlerArgs);
+            case "type_sensitive_text":
+                return await handleTypeSensitiveTextToolCall(handlerArgs);
             case "test_design_complete":
                 return await handleTestDesignCompleteToolCall(handlerArgs);
             default:
@@ -146,6 +197,10 @@ export class ExecutionEngine {
                 eventId: actionEventId,
                 captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
             });
+            // Inline the green click marker as a data URL so self-contained reports show it.
+            // The Desktop recorder returns one from disk; for runs without a recorder (e.g. a
+            // plain CLI `--report`) render the same marker in-memory from the raw screenshot.
+            const annotatedDataUrl = await this.resolveAnnotatedDataUrl(annotatedScreenshot, currentScreenshot, action, context);
             await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
                 ...context,
                 shouldStop,
@@ -154,7 +209,7 @@ export class ExecutionEngine {
                 eventId: actionEventId,
                 captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
                 ...(annotatedScreenshot?.screenshotPath ? { screenshotPath: annotatedScreenshot.screenshotPath } : {}),
-                ...(context?.includeScreenshotDataUrl && annotatedScreenshot?.screenshotDataUrl ? { screenshotDataUrl: annotatedScreenshot.screenshotDataUrl } : {})
+                ...(context?.includeScreenshotDataUrl && annotatedDataUrl ? { screenshotDataUrl: annotatedDataUrl } : {})
             });
             if (trackAction) {
                 trackAction(action);

package/build/src/core/prompts.js

@@ -1,3 +1,3 @@
 export { buildBaseSystemPrompt } from "../prompts/base.js";
 export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
-export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";
+export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, buildSecretKeysContextSection, buildTestDataContextSection, } from "../prompts/execution.js";

package/build/src/core/report/generate-html-report.js

@@ -0,0 +1,88 @@
+// File/glue layer for the HTML report: reads a stored JUnit run-result XML, inlines
+// its sibling screenshot PNGs as base64 data URLs, and renders a self-contained HTML
+// document. Used by both the CLI (`--report`) and the Desktop App ("Share" button).
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { parseRunReport } from "./parse-run-report.js";
+import { renderHtmlReport } from "./render-html-report.js";
+const MIME_BY_EXT = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+};
+function mimeForPath(filePath) {
+    return MIME_BY_EXT[path.extname(filePath).toLowerCase()] ?? "image/png";
+}
+// Resolves a screenshotPath (relative to the results directory) to an absolute path,
+// guarding against escaping the results directory via traversal. Returns null if the
+// resolved path would fall outside `resultsDir`.
+function resolveScreenshotPath(resultsDir, screenshotPath) {
+    if (typeof screenshotPath !== "string" || screenshotPath.length === 0)
+        return null;
+    const resolved = path.resolve(resultsDir, screenshotPath);
+    const base = path.resolve(resultsDir);
+    if (resolved !== base && !resolved.startsWith(base + path.sep))
+        return null;
+    return resolved;
+}
+// Reads an image file and returns a data URL, or null if it cannot be read.
+async function toDataUrl(filePath) {
+    try {
+        const buffer = await readFile(filePath);
+        return `data:${mimeForPath(filePath)};base64,${buffer.toString("base64")}`;
+    }
+    catch {
+        return null;
+    }
+}
+// Walks every event (top-level and child tests) and replaces payload.screenshotPath
+// with an inlined payload.screenshotDataUrl read from disk. Missing files are skipped.
+async function inlineScreenshots(model, resultsDir) {
+    const allEventLists = [model.events ?? []];
+    for (const child of model.childTests ?? []) {
+        allEventLists.push(child.events ?? []);
+    }
+    for (const events of allEventLists) {
+        for (const event of events) {
+            const screenshotPath = event.payload?.screenshotPath;
+            if (typeof screenshotPath !== "string" || screenshotPath.length === 0)
+                continue;
+            if (typeof event.payload?.screenshotDataUrl === "string")
+                continue; // already inlined
+            const absolute = resolveScreenshotPath(resultsDir, screenshotPath);
+            if (!absolute)
+                continue;
+            const dataUrl = await toDataUrl(absolute);
+            if (dataUrl) {
+                event.payload = { ...event.payload, screenshotDataUrl: dataUrl };
+            }
+        }
+    }
+    return model;
+}
+// Builds the HTML report from an in-memory model. Screenshots are inlined from
+// `resultsDir` (the directory the relative screenshotPath values are based on).
+export async function generateHtmlReportFromModel(model, { resultsDir, title } = {}) {
+    if (resultsDir) {
+        await inlineScreenshots(model, resultsDir);
+    }
+    return renderHtmlReport(model, { title });
+}
+// Reads a stored run-result XML file and produces both the self-contained HTML report
+// and the parsed model (so callers can derive a filename from testName/timestamp without
+// re-reading the file). Screenshots are read from the sibling directory the relative
+// screenshotPath values reference (which is the XML file's own directory).
+export async function generateReportFromXmlFile(xmlPath, { title } = {}) {
+    const xml = await readFile(xmlPath, "utf-8");
+    const model = parseRunReport(xml);
+    const resultsDir = path.dirname(path.resolve(xmlPath));
+    const html = await generateHtmlReportFromModel(model, { resultsDir, title });
+    return { html, model };
+}
+// Convenience wrapper returning just the HTML for callers that don't need the model.
+export async function generateHtmlReportFromXmlFile(xmlPath, options = {}) {
+    const { html } = await generateReportFromXmlFile(xmlPath, options);
+    return html;
+}

package/build/src/core/report/parse-run-report.js

@@ -0,0 +1,222 @@
+// Parses a JUnit-style run-result XML (the format stored under a project's
+// results/ directory) into a normalized RunReportModel. This is the XML-acquisition
+// adapter: it does the regex/attr/NDJSON parsing, then delegates ALL model assembly
+// (event normalization, status, success rate, failure shape, final shape) to the shared
+// run-report-model.js so the desktop live view, the Share report, and the CLI report all
+// derive identically.
+import { normalizeEvent, deriveRunStatus, computeSuccessRate, makeFailureOrError, assembleRunReportModel, } from "./run-report-model.js";
+function parseAttributes(raw) {
+    const attrs = {};
+    const attrPattern = /([^\s=]+)\s*=\s*("([^"]*)"|'([^']*)')/g;
+    let match = null;
+    while ((match = attrPattern.exec(raw)) !== null) {
+        attrs[match[1]] = match[3] ?? match[4] ?? "";
+    }
+    return attrs;
+}
+function findOpenTag(xml, tagName) {
+    const pattern = new RegExp(`<${tagName}\\b([^>]*)>`, "i");
+    const match = xml.match(pattern);
+    if (!match)
+        return null;
+    return parseAttributes(match[1] ?? "");
+}
+function findFirstTagWithBody(xml, tagName) {
+    const pattern = new RegExp(`<${tagName}\\b([^>]*)>([\\s\\S]*?)<\\/${tagName}>`, "i");
+    const match = xml.match(pattern);
+    if (!match)
+        return null;
+    return { attrs: parseAttributes(match[1] ?? ""), body: match[2] ?? "" };
+}
+function findFirstTestcase(xml) {
+    const withBody = findFirstTagWithBody(xml, "testcase");
+    if (withBody)
+        return withBody;
+    const selfClosing = xml.match(/<testcase\b([^>]*)\/>/i);
+    if (!selfClosing)
+        return null;
+    return { attrs: parseAttributes(selfClosing[1] ?? ""), body: "" };
+}
+function findTestcases(xml) {
+    const testcases = [];
+    const withBodyPattern = /<testcase\b([^>]*)>([\s\S]*?)<\/testcase>/gi;
+    let match = null;
+    while ((match = withBodyPattern.exec(xml)) !== null) {
+        testcases.push({ attrs: parseAttributes(match[1] ?? ""), body: match[2] ?? "" });
+    }
+    if (testcases.length > 0)
+        return testcases;
+    const selfClosingPattern = /<testcase\b([^>]*)\/>/gi;
+    while ((match = selfClosingPattern.exec(xml)) !== null) {
+        testcases.push({ attrs: parseAttributes(match[1] ?? ""), body: "" });
+    }
+    return testcases;
+}
+function parseCount(raw) {
+    if (!raw)
+        return null;
+    const parsed = Number.parseInt(raw, 10);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function parseDuration(raw) {
+    if (!raw)
+        return null;
+    const parsed = Number.parseFloat(raw);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function countTestcases(xml) {
+    return xml.match(/<testcase\b/gi)?.length ?? 0;
+}
+function readFailureOrError(testcaseBody) {
+    const failure = findFirstTagWithBody(testcaseBody, "failure");
+    if (failure) {
+        return makeFailureOrError({
+            kind: "failure",
+            type: failure.attrs.type,
+            message: failure.attrs.message,
+            body: failure.body.trim() || undefined,
+        });
+    }
+    const error = findFirstTagWithBody(testcaseBody, "error");
+    if (error) {
+        return makeFailureOrError({
+            kind: "error",
+            type: error.attrs.type,
+            message: error.attrs.message,
+            body: error.body.trim() || undefined,
+        });
+    }
+    return undefined;
+}
+function hasStoppedMarker(value) {
+    return /stopped by user/i.test(value);
+}
+// Maps one parsed NDJSON line (producer field names `event`/`action`) to the canonical
+// event shape and runs it through the shared normalizer.
+function normalizeNdjsonEvent(value, fallbackTs, warnings) {
+    return normalizeEvent({
+        ts: value.ts,
+        kind: value.kind,
+        text: value.text,
+        eventType: value.event,
+        actionType: value.action,
+        payload: value.payload,
+    }, { fallbackTs, warnings });
+}
+function parseSystemOutEvents(systemOutBody, fallbackTs, warnings) {
+    const normalizedSystemOut = systemOutBody
+        ?.trim()
+        .replace(/^<!\[CDATA\[/, "")
+        .replace(/\]\]>$/, "");
+    if (!normalizedSystemOut || !normalizedSystemOut.trim()) {
+        warnings.push("Event log not available (older report format).");
+        return [];
+    }
+    const lines = normalizedSystemOut
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0);
+    if (lines.length === 0) {
+        warnings.push("Event log not available (older report format).");
+        return [];
+    }
+    const events = [];
+    let invalidJsonCount = 0;
+    for (const line of lines) {
+        try {
+            const parsed = JSON.parse(line);
+            if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+                invalidJsonCount += 1;
+                continue;
+            }
+            const event = normalizeNdjsonEvent(parsed, fallbackTs, warnings);
+            if (event)
+                events.push(event);
+        }
+        catch {
+            invalidJsonCount += 1;
+        }
+    }
+    if (invalidJsonCount > 0) {
+        warnings.push(`Skipped ${invalidJsonCount} malformed event line${invalidJsonCount === 1 ? "" : "s"}.`);
+    }
+    return events;
+}
+export function parseRunReport(xml, { fallbackTimestamp = new Date(0).toISOString() } = {}) {
+    const warnings = [];
+    const suitesAttrs = findOpenTag(xml, "testsuites");
+    const suite = findFirstTagWithBody(xml, "testsuite");
+    if (!suite) {
+        throw new Error("Missing <testsuite> element.");
+    }
+    const testcases = findTestcases(suite.body);
+    const testcase = testcases[0] ?? findFirstTestcase(suite.body);
+    if (!testcase) {
+        warnings.push("Missing <testcase> element; using suite-level data only.");
+    }
+    const tests = (parseCount(suite.attrs.tests ?? suitesAttrs?.tests) ?? countTestcases(suite.body)) || 1;
+    const failures = parseCount(suite.attrs.failures ?? suitesAttrs?.failures) ?? 0;
+    const errors = parseCount(suite.attrs.errors ?? suitesAttrs?.errors) ?? 0;
+    const skipped = parseCount(suite.attrs.skipped ?? suitesAttrs?.skipped) ?? 0;
+    const failureOrError = testcases.map((entry) => readFailureOrError(entry.body)).find(Boolean) ??
+        (testcase ? readFailureOrError(testcase.body) : undefined);
+    const hasSkippedNode = testcase ? /<skipped\b/i.test(testcase.body) : false;
+    const stopped = testcases.some((entry) => hasStoppedMarker(entry.body)) ||
+        (testcase ? hasStoppedMarker(testcase.body) : false);
+    const status = deriveRunStatus({
+        tests,
+        failures,
+        errors,
+        skipped,
+        stopped,
+        hasSkippedNode,
+        failureKind: failureOrError?.kind,
+    });
+    const successRate = computeSuccessRate({ tests, failures, errors, skipped, status });
+    const timestamp = suite.attrs.timestamp ?? suitesAttrs?.timestamp ?? null;
+    const detailDuration = testcases.length > 1
+        ? parseDuration(suite.attrs.time ?? suitesAttrs?.time)
+        : parseDuration(testcase?.attrs.time) ?? parseDuration(suite.attrs.time ?? suitesAttrs?.time);
+    const systemOut = findFirstTagWithBody(testcase?.body ?? suite.body, "system-out");
+    const events = parseSystemOutEvents(systemOut?.body ?? null, timestamp ?? fallbackTimestamp, warnings);
+    const childTests = testcases.length > 1
+        ? testcases.map((child, index) => {
+            const childWarnings = [];
+            const childFailureOrError = readFailureOrError(child.body);
+            const childHasSkippedNode = /<skipped\b/i.test(child.body);
+            const childStopped = hasStoppedMarker(child.body);
+            // childTests is XML-only (the CLI never produces it); successRate stays a binary
+            // pass/non-pass here by design — only the status ladder is shared.
+            const childStatus = deriveRunStatus({
+                tests: 1,
+                stopped: childStopped,
+                hasSkippedNode: childHasSkippedNode,
+                failureKind: childFailureOrError?.kind,
+            });
+            const childSystemOut = findFirstTagWithBody(child.body, "system-out");
+            const childEvents = parseSystemOutEvents(childSystemOut?.body ?? null, timestamp ?? fallbackTimestamp, childWarnings);
+            return {
+                testName: child.attrs.name ?? `Test ${index + 1}`,
+                durationSeconds: parseDuration(child.attrs.time),
+                status: childStatus,
+                successRate: childStatus === "pass" ? 100 : 0,
+                failureOrError: childFailureOrError,
+                events: childEvents,
+                warnings: childWarnings,
+            };
+        })
+        : undefined;
+    return assembleRunReportModel({
+        suiteName: suite.attrs.name ?? null,
+        testName: childTests ? suite.attrs.name ?? suitesAttrs?.name ?? null : testcase?.attrs.name ?? null,
+        timestamp,
+        durationSeconds: detailDuration,
+        status,
+        testCases: tests,
+        successRate,
+        failureOrError,
+        events,
+        childTests,
+        warnings,
+    });
+}