gsd-pi 2.33.1 → 2.34.0-dev.6769913

package/dist/resources/extensions/browser-tools/tools/visual-diff.js

@@ -0,0 +1,174 @@
+import { Type } from "@sinclair/typebox";
+/**
+ * Visual regression diffing — compare current page screenshot against a stored baseline.
+ */
+const BASELINE_DIR = ".gsd/browser-baselines";
+export function registerVisualDiffTools(pi, deps) {
+    pi.registerTool({
+        name: "browser_visual_diff",
+        label: "Browser Visual Diff",
+        description: "Compare current page screenshot against a stored baseline pixel-by-pixel. " +
+            "Returns similarity score (0–1), diff pixel count, and optionally generates a diff image highlighting changes. " +
+            "On first run with no baseline, saves the current screenshot as the baseline. " +
+            "Baselines are stored in .gsd/browser-baselines/ (gitignored, environment-specific).",
+        parameters: Type.Object({
+            name: Type.Optional(Type.String({
+                description: "Baseline name (default: auto-generated from URL + viewport). " +
+                    "Use consistent names to compare the same view across runs.",
+            })),
+            selector: Type.Optional(Type.String({
+                description: "CSS selector to scope comparison to a specific element instead of full viewport.",
+            })),
+            threshold: Type.Optional(Type.Number({
+                description: "Pixel matching threshold 0–1 (default: 0.1). " +
+                    "Higher values are more tolerant of anti-aliasing and rendering differences.",
+            })),
+            updateBaseline: Type.Optional(Type.Boolean({
+                description: "If true, overwrite the existing baseline with the current screenshot (default: false).",
+            })),
+        }),
+        async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+            try {
+                const { page: p } = await deps.ensureBrowser();
+                const { mkdir, readFile, writeFile } = await import("node:fs/promises");
+                const pathMod = await import("node:path");
+                const baselineDir = pathMod.resolve(process.cwd(), BASELINE_DIR);
+                await mkdir(baselineDir, { recursive: true });
+                // Ensure .gitignore
+                const gitignorePath = pathMod.join(baselineDir, ".gitignore");
+                await writeFile(gitignorePath, "*\n!.gitignore\n").catch(() => { });
+                // Generate baseline name
+                const url = p.url();
+                const viewport = p.viewportSize();
+                const vpSuffix = viewport ? `${viewport.width}x${viewport.height}` : "unknown";
+                const autoName = deps.sanitizeArtifactName(`${new URL(url).pathname.replace(/\//g, "-")}-${vpSuffix}`, `baseline-${vpSuffix}`);
+                const name = deps.sanitizeArtifactName(params.name ?? autoName, autoName);
+                const baselinePath = pathMod.join(baselineDir, `${name}.png`);
+                const diffPath = pathMod.join(baselineDir, `${name}-diff.png`);
+                // Capture current screenshot as PNG (needed for pixel comparison)
+                let currentBuffer;
+                if (params.selector) {
+                    const locator = p.locator(params.selector).first();
+                    currentBuffer = await locator.screenshot({ type: "png" });
+                }
+                else {
+                    currentBuffer = await p.screenshot({ type: "png", fullPage: false });
+                }
+                // Check if baseline exists
+                let baselineBuffer = null;
+                try {
+                    baselineBuffer = await readFile(baselinePath);
+                }
+                catch {
+                    // No baseline yet
+                }
+                if (!baselineBuffer || params.updateBaseline) {
+                    // Save as new baseline
+                    await writeFile(baselinePath, currentBuffer);
+                    return {
+                        content: [{
+                                type: "text",
+                                text: baselineBuffer
+                                    ? `Baseline updated: ${baselinePath}\nSize: ${(currentBuffer.length / 1024).toFixed(1)} KB`
+                                    : `Baseline created (first run): ${baselinePath}\nSize: ${(currentBuffer.length / 1024).toFixed(1)} KB\nRe-run to compare against this baseline.`,
+                            }],
+                        details: {
+                            baselinePath,
+                            baselineCreated: !baselineBuffer,
+                            baselineUpdated: !!baselineBuffer,
+                            sizeBytes: currentBuffer.length,
+                        },
+                    };
+                }
+                // Perform pixel comparison using sharp for PNG decoding
+                const sharp = (await import("sharp")).default;
+                const baselineMeta = await sharp(baselineBuffer).metadata();
+                const currentMeta = await sharp(currentBuffer).metadata();
+                const bWidth = baselineMeta.width ?? 0;
+                const bHeight = baselineMeta.height ?? 0;
+                const cWidth = currentMeta.width ?? 0;
+                const cHeight = currentMeta.height ?? 0;
+                // If dimensions differ, report mismatch
+                if (bWidth !== cWidth || bHeight !== cHeight) {
+                    return {
+                        content: [{
+                                type: "text",
+                                text: `Dimension mismatch: baseline is ${bWidth}x${bHeight}, current is ${cWidth}x${cHeight}. Cannot compare.\nUse updateBaseline: true to reset.`,
+                            }],
+                        details: {
+                            match: false,
+                            dimensionMismatch: true,
+                            baselineDimensions: { width: bWidth, height: bHeight },
+                            currentDimensions: { width: cWidth, height: cHeight },
+                        },
+                    };
+                }
+                // Extract raw RGBA pixel data
+                const baselineRaw = await sharp(baselineBuffer).ensureAlpha().raw().toBuffer();
+                const currentRaw = await sharp(currentBuffer).ensureAlpha().raw().toBuffer();
+                const width = bWidth;
+                const height = bHeight;
+                const totalPixels = width * height;
+                const threshold = params.threshold ?? 0.1;
+                // Simple pixel-by-pixel comparison (avoiding pixelmatch dependency)
+                const diffData = Buffer.alloc(width * height * 4);
+                let diffPixels = 0;
+                const thresholdSq = threshold * threshold * 255 * 255 * 3;
+                for (let i = 0; i < totalPixels; i++) {
+                    const offset = i * 4;
+                    const dr = baselineRaw[offset] - currentRaw[offset];
+                    const dg = baselineRaw[offset + 1] - currentRaw[offset + 1];
+                    const db = baselineRaw[offset + 2] - currentRaw[offset + 2];
+                    const distSq = dr * dr + dg * dg + db * db;
+                    if (distSq > thresholdSq) {
+                        diffPixels++;
+                        // Mark diff pixels as red
+                        diffData[offset] = 255; // R
+                        diffData[offset + 1] = 0; // G
+                        diffData[offset + 2] = 0; // B
+                        diffData[offset + 3] = 255; // A
+                    }
+                    else {
+                        // Dim unchanged pixels
+                        diffData[offset] = currentRaw[offset] >> 1;
+                        diffData[offset + 1] = currentRaw[offset + 1] >> 1;
+                        diffData[offset + 2] = currentRaw[offset + 2] >> 1;
+                        diffData[offset + 3] = 255;
+                    }
+                }
+                const similarity = 1 - (diffPixels / totalPixels);
+                const match = diffPixels === 0;
+                // Save diff image
+                await sharp(diffData, { raw: { width, height, channels: 4 } })
+                    .png()
+                    .toFile(diffPath);
+                return {
+                    content: [{
+                            type: "text",
+                            text: match
+                                ? `Visual diff: MATCH (100% similar)\nBaseline: ${baselinePath}`
+                                : `Visual diff: ${(similarity * 100).toFixed(2)}% similar\nDiff pixels: ${diffPixels} of ${totalPixels} (${((diffPixels / totalPixels) * 100).toFixed(2)}%)\nDiff image: ${diffPath}\nBaseline: ${baselinePath}`,
+                        }],
+                    details: {
+                        match,
+                        similarity,
+                        diffPixels,
+                        totalPixels,
+                        diffPercentage: (diffPixels / totalPixels) * 100,
+                        dimensions: { width, height },
+                        baselinePath,
+                        diffImagePath: match ? undefined : diffPath,
+                        threshold,
+                    },
+                };
+            }
+            catch (err) {
+                return {
+                    content: [{ type: "text", text: `Visual diff failed: ${err.message}` }],
+                    details: { error: err.message },
+                    isError: true,
+                };
+            }
+        },
+    });
+}

package/dist/resources/extensions/browser-tools/tools/wait.js

@@ -0,0 +1,201 @@
+import { Type } from "@sinclair/typebox";
+import { StringEnum } from "@gsd/pi-ai";
+import { validateWaitParams, createRegionStableScript, parseThreshold, includesNeedle, } from "../core.js";
+import { getConsoleLogs, } from "../state.js";
+export function registerWaitTools(pi, deps) {
+    pi.registerTool({
+        name: "browser_wait_for",
+        label: "Browser Wait For",
+        description: "Wait for a condition before continuing. Use after actions that trigger async updates — data fetches, route changes, animations, loading spinners. Choose the appropriate condition: 'selector_visible' waits for an element to appear, 'selector_hidden' waits for it to disappear, 'url_contains' waits for the URL to match, 'network_idle' waits for all network requests to finish, 'delay' waits a fixed number of milliseconds, 'text_visible' waits for text to appear in the page body, 'text_hidden' waits for text to disappear from the page body, 'request_completed' waits for a network response whose URL contains the given substring, 'console_message' waits for a console log message containing the given substring, 'element_count' waits for the number of elements matching the CSS selector in 'value' to satisfy the 'threshold' expression (e.g. '>=3', '==0', '<5'), 'region_stable' waits for the DOM region matching the CSS selector in 'value' to stop changing.",
+        parameters: Type.Object({
+            condition: StringEnum([
+                "selector_visible",
+                "selector_hidden",
+                "url_contains",
+                "network_idle",
+                "delay",
+                "text_visible",
+                "text_hidden",
+                "request_completed",
+                "console_message",
+                "element_count",
+                "region_stable",
+            ]),
+            value: Type.Optional(Type.String({
+                description: "For selector_visible/selector_hidden/element_count/region_stable: CSS selector. For url_contains/request_completed: URL substring. For text_visible/text_hidden/console_message: text substring. For delay: milliseconds as a string (e.g. '1000'). Not used for network_idle.",
+            })),
+            threshold: Type.Optional(Type.String({
+                description: "Threshold expression for element_count (e.g. '>=3', '==0', '<5', or bare '3' which defaults to >=). Only used with element_count condition.",
+            })),
+            timeout: Type.Optional(Type.Number({
+                description: "Maximum milliseconds to wait before failing (default: 10000)",
+            })),
+        }),
+        async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+            try {
+                const { page: p } = await deps.ensureBrowser();
+                const target = deps.getActiveTarget();
+                const timeout = params.timeout ?? 10000;
+                const validation = validateWaitParams({ condition: params.condition, value: params.value, threshold: params.threshold });
+                if (validation) {
+                    return {
+                        content: [{ type: "text", text: validation.error }],
+                        details: { error: validation.error, condition: params.condition },
+                        isError: true,
+                    };
+                }
+                switch (params.condition) {
+                    case "selector_visible": {
+                        if (!params.value) {
+                            return {
+                                content: [{ type: "text", text: "selector_visible requires a value (CSS selector)" }],
+                                details: {},
+                                isError: true,
+                            };
+                        }
+                        await target.waitForSelector(params.value, { state: "visible", timeout });
+                        return {
+                            content: [{ type: "text", text: `Element "${params.value}" is now visible` }],
+                            details: { condition: params.condition, value: params.value },
+                        };
+                    }
+                    case "selector_hidden": {
+                        if (!params.value) {
+                            return {
+                                content: [{ type: "text", text: "selector_hidden requires a value (CSS selector)" }],
+                                details: {},
+                                isError: true,
+                            };
+                        }
+                        await target.waitForSelector(params.value, { state: "hidden", timeout });
+                        return {
+                            content: [{ type: "text", text: `Element "${params.value}" is now hidden` }],
+                            details: { condition: params.condition, value: params.value },
+                        };
+                    }
+                    case "url_contains": {
+                        if (!params.value) {
+                            return {
+                                content: [{ type: "text", text: "url_contains requires a value (URL substring)" }],
+                                details: {},
+                                isError: true,
+                            };
+                        }
+                        await p.waitForURL((url) => url.toString().includes(params.value), { timeout });
+                        return {
+                            content: [{ type: "text", text: `URL now contains "${params.value}". Current URL: ${p.url()}` }],
+                            details: { condition: params.condition, value: params.value, url: p.url() },
+                        };
+                    }
+                    case "network_idle": {
+                        await p.waitForLoadState("networkidle", { timeout });
+                        return {
+                            content: [{ type: "text", text: "Network is idle" }],
+                            details: { condition: params.condition },
+                        };
+                    }
+                    case "delay": {
+                        const ms = parseInt(params.value ?? "1000", 10);
+                        if (isNaN(ms)) {
+                            return {
+                                content: [{ type: "text", text: "delay requires a numeric value (milliseconds)" }],
+                                details: {},
+                                isError: true,
+                            };
+                        }
+                        await new Promise((resolve) => setTimeout(resolve, ms));
+                        return {
+                            content: [{ type: "text", text: `Waited ${ms}ms` }],
+                            details: { condition: params.condition, ms },
+                        };
+                    }
+                    case "text_visible": {
+                        await target.waitForFunction((needle) => {
+                            const body = document.body?.innerText ?? "";
+                            return body.toLowerCase().includes(needle.toLowerCase());
+                        }, params.value, { timeout });
+                        return {
+                            content: [{ type: "text", text: `Text "${params.value}" is now visible on the page` }],
+                            details: { condition: params.condition, value: params.value },
+                        };
+                    }
+                    case "text_hidden": {
+                        await target.waitForFunction((needle) => {
+                            const body = document.body?.innerText ?? "";
+                            return !body.toLowerCase().includes(needle.toLowerCase());
+                        }, params.value, { timeout });
+                        return {
+                            content: [{ type: "text", text: `Text "${params.value}" is no longer visible on the page` }],
+                            details: { condition: params.condition, value: params.value },
+                        };
+                    }
+                    case "request_completed": {
+                        const response = await deps.getActivePage().waitForResponse((resp) => resp.url().includes(params.value), { timeout });
+                        return {
+                            content: [{ type: "text", text: `Request completed: ${response.url()} (status ${response.status()})` }],
+                            details: { condition: params.condition, value: params.value, url: response.url(), status: response.status() },
+                        };
+                    }
+                    case "console_message": {
+                        const needle = params.value;
+                        const startTime = Date.now();
+                        while (Date.now() - startTime < timeout) {
+                            const match = getConsoleLogs().find((entry) => includesNeedle(entry.text, needle));
+                            if (match) {
+                                return {
+                                    content: [{ type: "text", text: `Console message matching "${needle}" found: "${match.text}"` }],
+                                    details: { condition: params.condition, value: needle, matchedText: match.text, matchedType: match.type },
+                                };
+                            }
+                            await new Promise((resolve) => setTimeout(resolve, 100));
+                        }
+                        throw new Error(`Timed out waiting for console message matching "${needle}" (${timeout}ms)`);
+                    }
+                    case "element_count": {
+                        const threshold = parseThreshold(params.threshold ?? ">=1");
+                        if (!threshold) {
+                            return {
+                                content: [{ type: "text", text: `element_count threshold is malformed: "${params.threshold}"` }],
+                                details: { error: "malformed threshold", condition: params.condition },
+                                isError: true,
+                            };
+                        }
+                        const selector = params.value;
+                        const op = threshold.op;
+                        const n = threshold.n;
+                        await target.waitForFunction(({ selector, op, n }) => {
+                            const count = document.querySelectorAll(selector).length;
+                            switch (op) {
+                                case ">=": return count >= n;
+                                case "<=": return count <= n;
+                                case "==": return count === n;
+                                case ">": return count > n;
+                                case "<": return count < n;
+                                default: return false;
+                            }
+                        }, { selector, op, n }, { timeout });
+                        return {
+                            content: [{ type: "text", text: `Element count for "${selector}" satisfies ${op}${n}` }],
+                            details: { condition: params.condition, value: selector, threshold: `${op}${n}` },
+                        };
+                    }
+                    case "region_stable": {
+                        const script = createRegionStableScript(params.value);
+                        await target.waitForFunction(script, undefined, { timeout, polling: 200 });
+                        return {
+                            content: [{ type: "text", text: `Region "${params.value}" is now stable` }],
+                            details: { condition: params.condition, value: params.value },
+                        };
+                    }
+                }
+            }
+            catch (err) {
+                return {
+                    content: [{ type: "text", text: `Wait failed: ${err.message}` }],
+                    details: { error: err.message, condition: params.condition, value: params.value },
+                    isError: true,
+                };
+            }
+        },
+    });
+}

package/dist/resources/extensions/browser-tools/tools/zoom.js

@@ -0,0 +1,90 @@
+import { Type } from "@sinclair/typebox";
+/**
+ * Region zoom / high-res capture — capture and upscale specific page regions.
+ */
+export function registerZoomTools(pi, deps) {
+    pi.registerTool({
+        name: "browser_zoom_region",
+        label: "Browser Zoom Region",
+        description: "Capture and optionally upscale a specific rectangular region of the page for detailed inspection. " +
+            "Useful for dense UIs where full-page screenshots have text too small to read. " +
+            "Returns the region as an inline image, same as browser_screenshot.",
+        parameters: Type.Object({
+            x: Type.Number({ description: "Left coordinate of the region in CSS pixels." }),
+            y: Type.Number({ description: "Top coordinate of the region in CSS pixels." }),
+            width: Type.Number({ description: "Width of the region in CSS pixels." }),
+            height: Type.Number({ description: "Height of the region in CSS pixels." }),
+            scale: Type.Optional(Type.Number({
+                description: "Upscale factor (default: 2). Use 1 for native resolution, 2-4 for zoomed detail.",
+            })),
+        }),
+        async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+            try {
+                const { page: p } = await deps.ensureBrowser();
+                const { x, y, width, height } = params;
+                const scale = params.scale ?? 2;
+                // Validate dimensions
+                if (width <= 0 || height <= 0) {
+                    return {
+                        content: [{ type: "text", text: "Width and height must be positive." }],
+                        details: { error: "invalid_dimensions" },
+                        isError: true,
+                    };
+                }
+                // Capture the region using Playwright's clip option
+                const regionBuffer = await p.screenshot({
+                    type: "png",
+                    clip: { x, y, width, height },
+                });
+                let outputBuffer = regionBuffer;
+                let outputMime = "image/png";
+                // Upscale if scale > 1
+                if (scale > 1) {
+                    const sharp = (await import("sharp")).default;
+                    const targetWidth = Math.round(width * scale);
+                    const targetHeight = Math.round(height * scale);
+                    outputBuffer = await sharp(regionBuffer)
+                        .resize(targetWidth, targetHeight, {
+                        kernel: "lanczos3",
+                        fit: "fill",
+                    })
+                        .png()
+                        .toBuffer();
+                }
+                const base64Data = outputBuffer.toString("base64");
+                const title = await p.title();
+                const url = p.url();
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Region capture: ${width}x${height} at (${x},${y})${scale > 1 ? ` upscaled ${scale}x to ${Math.round(width * scale)}x${Math.round(height * scale)}` : ""}\nPage: ${title}\nURL: ${url}`,
+                        },
+                        {
+                            type: "image",
+                            data: base64Data,
+                            mimeType: outputMime,
+                        },
+                    ],
+                    details: {
+                        region: { x, y, width, height },
+                        scale,
+                        outputDimensions: {
+                            width: Math.round(width * scale),
+                            height: Math.round(height * scale),
+                        },
+                        title,
+                        url,
+                    },
+                };
+            }
+            catch (err) {
+                return {
+                    content: [{ type: "text", text: `Region zoom failed: ${err.message}` }],
+                    details: { error: err.message },
+                    isError: true,
+                };
+            }
+        },
+    });
+}