@loadmill/droid-cua 2.5.0 → 2.7.0

package/build/src/utils/run-screenshot-recorder.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { mkdir, writeFile } from "node:fs/promises";
+import sharp from "sharp";
 function pad2(value) {
     return String(value).padStart(2, "0");
 }
@@ -32,6 +33,12 @@ function buildCallSegment(callId) {
     }
     return `call-${sanitizeArtifactSegment(callId, "call").slice(0, 24)}`;
 }
+function buildEventSegment(eventId) {
+    if (typeof eventId !== "string" || !eventId.trim()) {
+        return null;
+    }
+    return `event-${sanitizeArtifactSegment(eventId, "event").slice(0, 36)}`;
+}
 function buildScreenshotFileName(sequence, metadata = {}) {
     const segments = [
         String(sequence).padStart(4, "0"),
@@ -50,8 +57,119 @@ function buildScreenshotFileName(sequence, metadata = {}) {
     if (callSegment) {
         segments.push(callSegment);
     }
+    const eventSegment = buildEventSegment(metadata.eventId);
+    if (eventSegment) {
+        segments.push(eventSegment);
+    }
     return `${segments.join("_")}.png`;
 }
+function clampCoordinate(value, max) {
+    if (!Number.isFinite(value) || max <= 0) {
+        return 0;
+    }
+    return Math.max(0, Math.min(max - 1, Math.round(value)));
+}
+function buildMarkerPlacement(marker = {}, width, height) {
+    const requestedX = Number(marker.x);
+    const requestedY = Number(marker.y);
+    const x = clampCoordinate(requestedX, width);
+    const y = clampCoordinate(requestedY, height);
+    return {
+        requestedX,
+        requestedY,
+        x,
+        y,
+        wasClamped: !Number.isFinite(requestedX) || !Number.isFinite(requestedY) || x !== Math.round(requestedX) || y !== Math.round(requestedY),
+    };
+}
+const MARKER_APPEARANCES = {
+    click: {
+        haloFill: "#39ff14",
+        haloOpacity: 0.28,
+        ringStroke: "#39ff14",
+        centerDotFill: "#07131f",
+    },
+    preview: {
+        haloFill: "#00d8ff",
+        haloOpacity: 0.28,
+        ringStroke: "#00d8ff",
+        centerDotFill: "#07131f",
+    },
+};
+function getMarkerAppearance(marker = {}) {
+    if (marker?.style === "preview") {
+        return MARKER_APPEARANCES.preview;
+    }
+    return MARKER_APPEARANCES.click;
+}
+function buildClickMarkerSvg({ width, height, x, y, radius, marker }) {
+    const haloRadius = radius + 8;
+    const whiteStrokeWidth = Math.max(5, Math.round(radius * 0.34));
+    const ringStrokeWidth = Math.max(3, Math.round(radius * 0.18));
+    const centerDotRadius = Math.max(2, Math.round(radius * 0.22));
+    const centerDotStrokeWidth = Math.max(2, Math.round(radius * 0.1));
+    const appearance = getMarkerAppearance(marker);
+    return `
+    <svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">
+      <circle cx="${x}" cy="${y}" r="${haloRadius}" fill="${appearance.haloFill}" opacity="${appearance.haloOpacity}" />
+      <circle cx="${x}" cy="${y}" r="${radius}" fill="none" stroke="#ffffff" stroke-width="${whiteStrokeWidth}" />
+      <circle cx="${x}" cy="${y}" r="${radius}" fill="none" stroke="${appearance.ringStroke}" stroke-width="${ringStrokeWidth}" />
+      <circle cx="${x}" cy="${y}" r="${centerDotRadius}" fill="${appearance.centerDotFill}" stroke="#ffffff" stroke-width="${centerDotStrokeWidth}" />
+    </svg>
+  `.trim();
+}
+async function renderAnnotatedScreenshot(image, marker = {}) {
+    const metadata = await image.metadata();
+    const width = Number(metadata.width) || 0;
+    const height = Number(metadata.height) || 0;
+    if (width <= 0 || height <= 0) {
+        return null;
+    }
+    const placement = buildMarkerPlacement(marker, width, height);
+    const radius = Math.max(14, Math.round(Math.min(width, height) * 0.055));
+    const overlay = Buffer.from(buildClickMarkerSvg({ width, height, x: placement.x, y: placement.y, radius, marker }));
+    const outputBuffer = await image
+        .composite([{ input: overlay }])
+        .png()
+        .toBuffer();
+    return {
+        outputBuffer,
+        width,
+        height,
+        marker: placement,
+    };
+}
+export async function createAnnotatedScreenshotBuffer(filePath, marker = {}) {
+    if (typeof filePath !== "string" || !filePath.trim()) {
+        return null;
+    }
+    const result = await renderAnnotatedScreenshot(sharp(filePath), marker);
+    if (!result) {
+        return null;
+    }
+    return result.outputBuffer;
+}
+export async function createAnnotatedScreenshotFromBase64(screenshotBase64, marker = {}) {
+    if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+        return null;
+    }
+    const result = await renderAnnotatedScreenshot(sharp(Buffer.from(screenshotBase64, "base64")), marker);
+    if (!result) {
+        return null;
+    }
+    return {
+        ...result,
+        screenshotBase64: result.outputBuffer.toString("base64"),
+    };
+}
+export async function annotateScreenshotArtifact(filePath, marker = {}) {
+    const outputBuffer = await createAnnotatedScreenshotBuffer(filePath, marker);
+    if (!outputBuffer) {
+        return null;
+    }
+    await writeFile(filePath, outputBuffer);
+    return filePath;
+}
 export function createDebugScreenshotRecorder({ directoryPath }) {
     let nextSequence = 1;
     let ensuredDirectoryPromise = null;
@@ -76,6 +194,23 @@ export function createDebugScreenshotRecorder({ directoryPath }) {
             const filePath = path.join(directoryPath, fileName);
             await writeFile(filePath, Buffer.from(screenshotBase64, "base64"));
             return filePath;
+        },
+        async saveAnnotatedScreenshot(sourceFilePath, marker = {}, metadata = {}) {
+            if (typeof sourceFilePath !== "string" || !sourceFilePath.trim()) {
+                return null;
+            }
+            await ensureDirectory();
+            const outputBuffer = await createAnnotatedScreenshotBuffer(sourceFilePath, marker);
+            if (!outputBuffer) {
+                return null;
+            }
+            const fileName = buildScreenshotFileName(nextSequence++, metadata);
+            const filePath = path.join(directoryPath, fileName);
+            await writeFile(filePath, outputBuffer);
+            return {
+                artifactPaths: [filePath],
+                annotationSourcePaths: [sourceFilePath],
+            };
         }
     };
 }
@@ -93,6 +228,32 @@ export function createCompositeScreenshotRecorder({ recorders }) {
             }
             const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveScreenshot(screenshotBase64, metadata)));
             return results.filter(Boolean);
+        },
+        async saveAnnotatedScreenshot(sourceFilePath, marker = {}, metadata = {}) {
+            if (typeof sourceFilePath !== "string" || !sourceFilePath.trim()) {
+                return null;
+            }
+            const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveAnnotatedScreenshot?.(sourceFilePath, marker, metadata)));
+            const filteredResults = results.filter(Boolean);
+            if (filteredResults.length === 0) {
+                return null;
+            }
+            return {
+                artifactPaths: filteredResults.flatMap((result) => {
+                    if (result && typeof result === "object" && !Array.isArray(result)) {
+                        return Array.isArray(result.artifactPaths) ? result.artifactPaths.filter(Boolean) : [];
+                    }
+                    return typeof result === "string" ? [result] : [];
+                }),
+                annotationSourcePaths: filteredResults.flatMap((result) => {
+                    if (result && typeof result === "object" && !Array.isArray(result) && Array.isArray(result.annotationSourcePaths)) {
+                        return result.annotationSourcePaths.filter(Boolean);
+                    }
+                    return [];
+                }),
+                screenshotPath: filteredResults.find((result) => result?.screenshotPath)?.screenshotPath ?? null,
+                screenshotDataUrl: filteredResults.find((result) => result?.screenshotDataUrl)?.screenshotDataUrl ?? null,
+            };
         }
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.5.0",
+  "version": "2.7.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -15,6 +15,11 @@
   ],
   "scripts": {
     "start": "tsx index.js",
+    "bench": "npm run bench:preflight && npm run bench:run && npm run bench:report",
+    "bench:preflight": "node scripts/benchmark-preflight.js",
+    "bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
+    "bench:report": "node scripts/benchmark-report.js",
+    "execution:profile": "node scripts/execution-profile.js",
     "test": "node --test unit-tests/*.test.js",
     "build": "tsc",
     "clean": "rm -rf build",