@loadmill/droid-cua 2.2.1 → 2.3.0

package/build/src/utils/console-output.js

@@ -0,0 +1,35 @@
+export function formatCliOutput(item) {
+    if (item == null) {
+        return null;
+    }
+    if (typeof item === "string" || typeof item === "number" || typeof item === "boolean") {
+        return String(item);
+    }
+    if (typeof item !== "object") {
+        return String(item);
+    }
+    const text = typeof item.text === "string" ? item.text : null;
+    const type = typeof item.type === "string" ? item.type : "";
+    const eventType = typeof item.eventType === "string" ? item.eventType : "";
+    const isAssistantMessage = type === "assistant" || eventType === "assistant_message";
+    if (isAssistantMessage) {
+        if (!text || text.trim().length === 0) {
+            return null;
+        }
+        return text
+            .split("\n")
+            .map((line) => `assistant: ${line}`)
+            .join("\n");
+    }
+    if (text !== null) {
+        return text;
+    }
+    return item;
+}
+export function printCliOutput(item, consoleLike = console) {
+    const formatted = formatCliOutput(item);
+    if (formatted == null) {
+        return;
+    }
+    consoleLike.log(formatted);
+}

package/build/src/utils/run-screenshot-recorder.js

@@ -0,0 +1,98 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+function pad2(value) {
+    return String(value).padStart(2, "0");
+}
+function pad3(value) {
+    return String(value).padStart(3, "0");
+}
+export function formatArtifactTimestamp(date = new Date()) {
+    return `${date.getFullYear()}${pad2(date.getMonth() + 1)}${pad2(date.getDate())}-${pad2(date.getHours())}${pad2(date.getMinutes())}${pad2(date.getSeconds())}-${pad3(date.getMilliseconds())}`;
+}
+function sanitizeArtifactSegment(value, fallback = "item") {
+    if (typeof value !== "string") {
+        return fallback;
+    }
+    const normalized = value
+        .trim()
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^[-_.]+|[-_.]+$/g, "");
+    return normalized || fallback;
+}
+function buildInstructionSegment(instructionIndex) {
+    if (!Number.isInteger(instructionIndex) || instructionIndex < 0) {
+        return null;
+    }
+    return `instruction-${String(instructionIndex + 1).padStart(3, "0")}`;
+}
+function buildCallSegment(callId) {
+    if (typeof callId !== "string" || !callId.trim()) {
+        return null;
+    }
+    return `call-${sanitizeArtifactSegment(callId, "call").slice(0, 24)}`;
+}
+function buildScreenshotFileName(sequence, metadata = {}) {
+    const segments = [
+        String(sequence).padStart(4, "0"),
+        formatArtifactTimestamp(metadata.timestamp instanceof Date ? metadata.timestamp : new Date()),
+        sanitizeArtifactSegment(metadata.captureSource || "screenshot", "screenshot")
+    ];
+    const stepIdSegment = sanitizeArtifactSegment(metadata.stepId || "", "");
+    if (stepIdSegment) {
+        segments.push(stepIdSegment);
+    }
+    const instructionSegment = buildInstructionSegment(metadata.instructionIndex);
+    if (instructionSegment) {
+        segments.push(instructionSegment);
+    }
+    const callSegment = buildCallSegment(metadata.callId);
+    if (callSegment) {
+        segments.push(callSegment);
+    }
+    return `${segments.join("_")}.png`;
+}
+export function createDebugScreenshotRecorder({ directoryPath }) {
+    let nextSequence = 1;
+    let ensuredDirectoryPromise = null;
+    async function ensureDirectory() {
+        if (!ensuredDirectoryPromise) {
+            ensuredDirectoryPromise = mkdir(directoryPath, { recursive: true });
+        }
+        await ensuredDirectoryPromise;
+        return directoryPath;
+    }
+    return {
+        directoryPath,
+        async ensureDirectory() {
+            return await ensureDirectory();
+        },
+        async saveScreenshot(screenshotBase64, metadata = {}) {
+            if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+                return null;
+            }
+            await ensureDirectory();
+            const fileName = buildScreenshotFileName(nextSequence++, metadata);
+            const filePath = path.join(directoryPath, fileName);
+            await writeFile(filePath, Buffer.from(screenshotBase64, "base64"));
+            return filePath;
+        }
+    };
+}
+export function createCompositeScreenshotRecorder({ recorders }) {
+    const activeRecorders = Array.isArray(recorders) ? recorders.filter(Boolean) : [];
+    return {
+        directoryPath: activeRecorders.map((recorder) => recorder.directoryPath).filter(Boolean),
+        async ensureDirectory() {
+            await Promise.all(activeRecorders.map((recorder) => recorder.ensureDirectory?.()));
+            return this.directoryPath;
+        },
+        async saveScreenshot(screenshotBase64, metadata = {}) {
+            if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+                return null;
+            }
+            const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveScreenshot(screenshotBase64, metadata)));
+            return results.filter(Boolean);
+        }
+    };
+}

package/build/src/utils/structured-debug-log-manager.js

@@ -0,0 +1,325 @@
+import path from "node:path";
+import { constants } from "node:fs";
+import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
+export const WORKSPACE_DEBUG_BRIDGE_KEY = "__DROID_DESKTOP_DEBUG_LOG_EVENT";
+export const DEVICE_DEBUG_LOG_NAME = "device-events.jsonl";
+const MAX_STRING_LENGTH = 1600;
+function isPromptContentPath(pathParts = []) {
+    if (pathParts.length < 2)
+        return false;
+    const last = pathParts[pathParts.length - 1];
+    if (last !== "content")
+        return false;
+    return pathParts.includes("messages");
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+function compactTimestamp(iso) {
+    return iso.replace(/[:.]/g, "-");
+}
+function sanitizeValue(value, pathParts = [], eventName = "") {
+    if (typeof value === "string") {
+        if (eventName === "cua.request.full" || eventName === "cua.response.full" || eventName === "app_context.briefing.full") {
+            return value;
+        }
+        if (isPromptContentPath(pathParts)) {
+            return value;
+        }
+        if (value.length <= MAX_STRING_LENGTH)
+            return value;
+        return `${value.slice(0, MAX_STRING_LENGTH)}...<truncated:${value.length - MAX_STRING_LENGTH}>`;
+    }
+    if (Array.isArray(value)) {
+        return value.map((item, index) => sanitizeValue(item, [...pathParts, index], eventName));
+    }
+    if (value && typeof value === "object") {
+        const output = {};
+        for (const [key, item] of Object.entries(value)) {
+            output[key] = sanitizeValue(item, [...pathParts, key], eventName);
+        }
+        return output;
+    }
+    return value;
+}
+function serializeLine(event) {
+    const payload = {
+        ts: nowIso(),
+        event: event.event,
+        scope: event.scope,
+        ids: sanitizeValue(event.ids ?? {}, [], event.event),
+        data: sanitizeValue(event.data ?? {}, [], event.event)
+    };
+    return JSON.stringify(payload) + "\n";
+}
+function buildArtifactsDirPath(filePath) {
+    const ext = path.extname(filePath);
+    const stem = path.basename(filePath, ext);
+    return path.join(path.dirname(filePath), stem);
+}
+async function fileExists(filePath) {
+    try {
+        await access(filePath, constants.F_OK);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * @param {{ enabled?: boolean, logsDirPath?: string | null }} [options]
+ */
+export function createStructuredDebugLogManager({ enabled = false, logsDirPath = null } = {}) {
+    const executionSessions = new Map();
+    const designSessions = new Map();
+    let state = {
+        enabled,
+        logsDirPath,
+        deviceLogPath: null,
+        currentLogFilePath: null
+    };
+    function setLogsDirPath(nextLogsDirPath) {
+        if (state.logsDirPath === nextLogsDirPath) {
+            return;
+        }
+        state.logsDirPath = nextLogsDirPath;
+        state.deviceLogPath = null;
+        state.currentLogFilePath = null;
+    }
+    function getActiveSessionFilePaths() {
+        const all = [...executionSessions.values(), ...designSessions.values()];
+        return all.map((record) => record.filePath);
+    }
+    function resolveSessionFile(scope, id) {
+        if (scope === "execution") {
+            return executionSessions.get(id)?.filePath ?? null;
+        }
+        return designSessions.get(id)?.filePath ?? null;
+    }
+    function pickCurrentFile() {
+        const activeExecution = [...executionSessions.values()].at(-1)?.filePath ?? null;
+        if (activeExecution)
+            return activeExecution;
+        const activeDesign = [...designSessions.values()].at(-1)?.filePath ?? null;
+        if (activeDesign)
+            return activeDesign;
+        return state.deviceLogPath;
+    }
+    async function ensureLogsDir() {
+        if (!state.logsDirPath) {
+            throw new Error("Structured debug logs directory is not configured.");
+        }
+        await mkdir(state.logsDirPath, { recursive: true });
+        return state.logsDirPath;
+    }
+    async function ensureDeviceLogFile() {
+        if (state.deviceLogPath)
+            return state.deviceLogPath;
+        const dir = await ensureLogsDir();
+        state.deviceLogPath = path.join(dir, DEVICE_DEBUG_LOG_NAME);
+        return state.deviceLogPath;
+    }
+    async function appendEvent(filePath, event) {
+        await appendFile(filePath, serializeLine(event), "utf-8");
+    }
+    async function appendDeviceEvent(event) {
+        const logFile = await ensureDeviceLogFile();
+        await appendEvent(logFile, event);
+        for (const sessionFilePath of getActiveSessionFilePaths()) {
+            if (sessionFilePath === logFile)
+                continue;
+            await appendEvent(sessionFilePath, event);
+        }
+        state.currentLogFilePath = getActiveSessionFilePaths().at(-1) ?? logFile;
+    }
+    return {
+        /**
+         * @param {{ enabled?: boolean, logsDirPath?: string | null }} [options]
+         */
+        async configure({ enabled: nextEnabled = state.enabled, logsDirPath: nextLogsDirPath = state.logsDirPath } = {}) {
+            state.enabled = Boolean(nextEnabled);
+            setLogsDirPath(nextLogsDirPath ?? null);
+            if (state.enabled) {
+                await ensureLogsDir();
+            }
+            return state.enabled;
+        },
+        isEnabled() {
+            return state.enabled;
+        },
+        installWorkspaceDebugBridge(target = globalThis) {
+            if (!target || typeof target !== "object") {
+                return;
+            }
+            target[WORKSPACE_DEBUG_BRIDGE_KEY] = (event) => {
+                void this.logWorkspaceEvent(event);
+            };
+        },
+        async startExecutionSession(runId, data = {}) {
+            if (!state.enabled)
+                return;
+            const dir = await ensureLogsDir();
+            const startedAt = nowIso();
+            const filename = `execution-${runId}-${compactTimestamp(startedAt)}.jsonl`;
+            const filePath = path.join(dir, filename);
+            const artifactsDirPath = buildArtifactsDirPath(filePath);
+            await writeFile(filePath, "", "utf-8");
+            await mkdir(artifactsDirPath, { recursive: true });
+            executionSessions.set(runId, {
+                id: runId,
+                scope: "execution",
+                filePath,
+                artifactsDirPath,
+                startedAt
+            });
+            state.currentLogFilePath = filePath;
+            await appendEvent(filePath, {
+                event: "session.started",
+                scope: "execution",
+                ids: { runId },
+                data: {
+                    startedAt,
+                    artifactsDirPath,
+                    ...data
+                }
+            });
+        },
+        async endExecutionSession(runId, data = {}) {
+            if (!state.enabled)
+                return;
+            const session = executionSessions.get(runId);
+            if (!session)
+                return;
+            await appendEvent(session.filePath, {
+                event: "session.ended",
+                scope: "execution",
+                ids: { runId },
+                data: {
+                    startedAt: session.startedAt,
+                    endedAt: nowIso(),
+                    ...data
+                }
+            });
+            executionSessions.delete(runId);
+            state.currentLogFilePath = pickCurrentFile();
+        },
+        async startDesignSession(sessionId, data = {}) {
+            if (!state.enabled)
+                return;
+            const dir = await ensureLogsDir();
+            const startedAt = nowIso();
+            const filename = `design-${sessionId}-${compactTimestamp(startedAt)}.jsonl`;
+            const filePath = path.join(dir, filename);
+            const artifactsDirPath = buildArtifactsDirPath(filePath);
+            await writeFile(filePath, "", "utf-8");
+            await mkdir(artifactsDirPath, { recursive: true });
+            designSessions.set(sessionId, {
+                id: sessionId,
+                scope: "design",
+                filePath,
+                artifactsDirPath,
+                startedAt
+            });
+            state.currentLogFilePath = filePath;
+            await appendEvent(filePath, {
+                event: "session.started",
+                scope: "design",
+                ids: { sessionId },
+                data: {
+                    startedAt,
+                    artifactsDirPath,
+                    ...data
+                }
+            });
+        },
+        async endDesignSession(sessionId, data = {}) {
+            if (!state.enabled)
+                return;
+            const session = designSessions.get(sessionId);
+            if (!session)
+                return;
+            await appendEvent(session.filePath, {
+                event: "session.ended",
+                scope: "design",
+                ids: { sessionId },
+                data: {
+                    startedAt: session.startedAt,
+                    endedAt: nowIso(),
+                    ...data
+                }
+            });
+            designSessions.delete(sessionId);
+            state.currentLogFilePath = pickCurrentFile();
+        },
+        async logDeviceEvent(event) {
+            if (!state.enabled)
+                return;
+            await appendDeviceEvent({
+                event: event.event,
+                scope: "device",
+                ids: event.ids,
+                data: event.data
+            });
+        },
+        async logSessionEvent(scope, id, event) {
+            if (!state.enabled)
+                return;
+            const sessionFile = resolveSessionFile(scope, id);
+            if (!sessionFile)
+                return;
+            await appendEvent(sessionFile, {
+                event: event.event,
+                scope,
+                ids: event.ids,
+                data: event.data
+            });
+        },
+        async logWorkspaceEvent(rawEvent) {
+            if (!state.enabled)
+                return;
+            const event = typeof rawEvent?.event === "string" ? rawEvent.event : "";
+            const scope = rawEvent?.scope === "execution" || rawEvent?.scope === "design" || rawEvent?.scope === "device"
+                ? rawEvent.scope
+                : null;
+            const ids = rawEvent?.ids && typeof rawEvent.ids === "object" ? rawEvent.ids : {};
+            const data = rawEvent?.data && typeof rawEvent.data === "object" ? rawEvent.data : {};
+            if (!event || !scope)
+                return;
+            if (scope === "device") {
+                await this.logDeviceEvent({ event, ids, data });
+                return;
+            }
+            const idKey = scope === "execution" ? "runId" : "sessionId";
+            const idValue = typeof ids[idKey] === "string" ? ids[idKey] : "";
+            if (idValue) {
+                await this.logSessionEvent(scope, idValue, { event, ids, data });
+                return;
+            }
+            const sessionFilePath = scope === "execution"
+                ? [...executionSessions.values()].at(-1)?.filePath ?? null
+                : [...designSessions.values()].at(-1)?.filePath ?? null;
+            if (!sessionFilePath)
+                return;
+            await appendEvent(sessionFilePath, { event, scope, ids, data });
+        },
+        async getCurrentLogFilePath() {
+            const current = state.currentLogFilePath ?? pickCurrentFile();
+            if (current && (await fileExists(current))) {
+                return current;
+            }
+            if (state.deviceLogPath && (await fileExists(state.deviceLogPath))) {
+                return state.deviceLogPath;
+            }
+            return null;
+        },
+        getExecutionSessionArtifactsDir(runId) {
+            return executionSessions.get(runId)?.artifactsDirPath ?? null;
+        },
+        getDesignSessionArtifactsDir(sessionId) {
+            return designSessions.get(sessionId)?.artifactsDirPath ?? null;
+        },
+        async getLogsDirPath() {
+            return ensureLogsDir();
+        }
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -15,6 +15,7 @@
   ],
   "scripts": {
     "start": "tsx index.js",
+    "test": "node --test unit-tests/*.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
     "prerelease": "npm run build",