electron-verify-mcp 0.0.1

package/dist/core/verify.js

@@ -0,0 +1,182 @@
+import { PNG } from "pngjs";
+import { waitUntilReady } from "./readiness.js";
+import { annotateHoles, captureRawScreenshot, downscaleIfNeeded, } from "./screenshot.js";
+import { captureOsWindow, captureWindow } from "./windowCapture.js";
+import { collectSignals } from "./signals/collect.js";
+import { computeVerdict } from "./verdict.js";
+import { detectHoles, selectTargetWindow } from "./windows.js";
+/**
+ * Orchestrate one verification pass:
+ * select window → (optional freshLoad) → readiness → drain error buffer →
+ * capture content (for signals) + optional window capture → collect → verdict.
+ */
+export async function verify(session, opts = {}) {
+    const selected = await selectTargetWindow(session.app, {
+        windowUrlMatch: opts.windowUrlMatch ?? session.windowUrlMatch,
+    });
+    session.setPage(selected.page);
+    const page = selected.page;
+    const pageUrl = page.url();
+    // freshLoad: explicit wins; else default TRUE when no interaction since last verify.
+    const freshLoad = opts.freshLoad !== undefined
+        ? opts.freshLoad
+        : !session.interactionSinceVerify;
+    await session.consoleCollector.installErrorBuffer(page);
+    if (freshLoad) {
+        try {
+            await page.reload({
+                waitUntil: "load",
+                timeout: opts.readinessTimeoutMs ?? 15_000,
+            });
+        }
+        catch {
+            // Some shells reject reload or hang past load; readiness still runs.
+        }
+    }
+    const readiness = await waitUntilReady(page, {
+        timeoutMs: opts.readinessTimeoutMs,
+        changeThreshold: opts.changeThreshold,
+    });
+    await session.consoleCollector.drainBufferedErrors(page);
+    // BrowserView bounds for the SELECTED window (not always getAllWindows()[0]).
+    const dipHoles = await detectHoles(session.app, pageUrl);
+    const dpr = await readDevicePixelRatio(page);
+    const holes = scaleHolesToPhysical(dipHoles, dpr);
+    // Signal analysis ALWAYS uses content capture.
+    const rawPng = await captureRawScreenshot(page);
+    const rawDecoded = PNG.sync.read(rawPng);
+    const annotated = holes.length > 0 ? annotateHoles(rawDecoded, holes) : rawDecoded;
+    let contentPng = PNG.sync.write(annotated);
+    const maxW = opts.maxScreenshotWidth ?? 1280;
+    contentPng = downscaleIfNeeded(contentPng, maxW);
+    const signals = await collectSignals(session, {
+        readiness,
+        png: rawPng,
+        holes,
+        criticalSelectors: opts.criticalSelectors ?? session.criticalSelectors,
+        srcGlobs: opts.srcGlobs ?? session.srcGlobs,
+        page,
+        blankCoverage: opts.blankCoverage,
+    });
+    // Advance delta windows so the next verify only sees subsequent errors/failures.
+    session.consoleCollector.markVerifyStart();
+    session.resourcesCollector.markVerifyStart();
+    session.clearInteractionFlag();
+    const strictConsole = opts.strictConsole ?? session.strictConsole;
+    const result = computeVerdict(signals, {
+        expect: opts.expect,
+        strictConsole,
+    });
+    // Human-facing capture mode (default: window).
+    const want = opts.captureMode ?? "window";
+    const packed = await packHumanScreenshots({
+        session,
+        pageUrl,
+        contentPng,
+        holes,
+        want,
+        maxW,
+    });
+    return {
+        ...result,
+        screenshot: packed.primary,
+        windowScreenshot: packed.windowPng,
+        contentScreenshot: packed.contentPng,
+        captureMode: packed.mode,
+        captureNote: packed.note,
+        windowStrategy: selected.strategy,
+        windowWarning: selected.warning,
+        holes: (want === "window" && packed.mode === "window") ||
+            (want === "os-window" &&
+                (packed.mode === "os-window" || packed.mode === "os-window-fallback:window"))
+            ? []
+            : holes,
+    };
+}
+async function packHumanScreenshots(args) {
+    const { session, pageUrl, contentPng, want, maxW } = args;
+    if (want === "content") {
+        return { primary: contentPng, mode: "content", contentPng };
+    }
+    if (want === "os-window") {
+        const os = await captureOsWindow(session.app, pageUrl);
+        let osPng = os.png;
+        if (osPng)
+            osPng = downscaleIfNeeded(osPng, maxW);
+        if (osPng) {
+            return { primary: osPng, mode: "os-window" };
+        }
+        // Fall back to composited window, then content.
+        const win = await captureWindow(session.app, pageUrl);
+        let windowPng = win.png;
+        if (windowPng)
+            windowPng = downscaleIfNeeded(windowPng, maxW);
+        if (windowPng) {
+            return {
+                primary: windowPng,
+                mode: "os-window-fallback:window",
+                note: os.note ?? "os-window capture unavailable; used window",
+            };
+        }
+        return {
+            primary: contentPng,
+            mode: "content-fallback",
+            note: [os.note, win.note].filter(Boolean).join("; ") ||
+                "os-window and window capture unavailable",
+        };
+    }
+    const win = await captureWindow(session.app, pageUrl);
+    let windowPng = win.png;
+    if (windowPng) {
+        windowPng = downscaleIfNeeded(windowPng, maxW);
+    }
+    if (want === "both") {
+        if (windowPng) {
+            return {
+                primary: windowPng,
+                mode: "window",
+                windowPng,
+                contentPng,
+            };
+        }
+        return {
+            primary: contentPng,
+            mode: "content-fallback",
+            note: win.note ?? "window capture unavailable",
+            contentPng,
+        };
+    }
+    // want === "window"
+    if (windowPng) {
+        return {
+            primary: windowPng,
+            mode: "window",
+            // No hole annotation — composited capture shows real pixels.
+        };
+    }
+    return {
+        primary: contentPng,
+        mode: "content-fallback",
+        note: win.note ?? "window capture unavailable",
+    };
+}
+async function readDevicePixelRatio(page) {
+    try {
+        const dpr = await page.evaluate(() => window.devicePixelRatio || 1);
+        return typeof dpr === "number" && dpr > 0 ? dpr : 1;
+    }
+    catch {
+        return 1;
+    }
+}
+/** Multiply DIP hole rects by devicePixelRatio for physical-pixel screenshot space. */
+export function scaleHolesToPhysical(holes, dpr) {
+    if (dpr === 1)
+        return holes;
+    return holes.map((h) => ({
+        x: h.x * dpr,
+        y: h.y * dpr,
+        width: h.width * dpr,
+        height: h.height * dpr,
+    }));
+}

package/dist/core/windowCapture.d.ts

@@ -0,0 +1,27 @@
+import type { ElectronApplication } from "playwright-core";
+export interface WindowCaptureResult {
+    png: Buffer | null;
+    note?: string;
+}
+/**
+ * Capture the composited Electron window via desktopCapturer window sources
+ * (menu bar + content; typically NO OS caption / min-max-close buttons).
+ *
+ * Returns null on permission failure, missing source, zero-size, or
+ * all-black/empty thumbnail so callers can fall back to content capture.
+ *
+ * Uses Playwright's `app.evaluate` first-arg Electron namespace (no require —
+ * the main-process evaluate context is ESM-like and `require` is undefined).
+ */
+export declare function captureWindow(app: ElectronApplication, selectedPageUrl: string): Promise<WindowCaptureResult>;
+/**
+ * Capture the FULL OS window including the caption bar (min/max/close buttons)
+ * and borders, by cropping a screen-source thumbnail to the window's outer
+ * bounds (getBounds includes the frame).
+ *
+ * Cross-platform, in-process, no native addon. Caveats:
+ * - Another window on top can intrude (we raise/focus best-effort first).
+ * - macOS needs Screen Recording permission (all-black → null → fallback).
+ * - Multi-monitor: uses getDisplayMatching + display-relative crop.
+ */
+export declare function captureOsWindow(app: ElectronApplication, selectedPageUrl: string): Promise<WindowCaptureResult>;

package/dist/core/windowCapture.js

@@ -0,0 +1,236 @@
+import { PNG } from "pngjs";
+/**
+ * Capture the composited Electron window via desktopCapturer window sources
+ * (menu bar + content; typically NO OS caption / min-max-close buttons).
+ *
+ * Returns null on permission failure, missing source, zero-size, or
+ * all-black/empty thumbnail so callers can fall back to content capture.
+ *
+ * Uses Playwright's `app.evaluate` first-arg Electron namespace (no require —
+ * the main-process evaluate context is ESM-like and `require` is undefined).
+ */
+export async function captureWindow(app, selectedPageUrl) {
+    try {
+        const b64 = await app.evaluate(async (electron, pageUrl) => {
+            const { desktopCapturer, screen, BrowserWindow, } = electron;
+            const wins = BrowserWindow.getAllWindows().filter((w) => !w.isDestroyed());
+            const win = wins.find((w) => {
+                try {
+                    return w.webContents.getURL() === pageUrl;
+                }
+                catch {
+                    return false;
+                }
+            }) ?? wins[0];
+            if (!win)
+                return null;
+            if (win.isMinimized())
+                win.restore();
+            const b = win.getBounds();
+            if (b.width <= 0 || b.height <= 0)
+                return null;
+            const sf = screen.getDisplayMatching(b).scaleFactor || 1;
+            const sourceId = win.getMediaSourceId();
+            const sources = await desktopCapturer.getSources({
+                types: ["window"],
+                thumbnailSize: {
+                    width: Math.ceil(b.width * sf),
+                    height: Math.ceil(b.height * sf),
+                },
+            });
+            // Match by ID ONLY — never title (titles collide / change).
+            const src = sources.find((s) => s.id === sourceId);
+            if (!src)
+                return null;
+            if (src.thumbnail.isEmpty())
+                return null;
+            const size = src.thumbnail.getSize();
+            if (size.width < 2 || size.height < 2)
+                return null;
+            const pngBuf = src.thumbnail.toPNG();
+            // Buffer may be a Node Buffer or Uint8Array depending on Electron version.
+            const u8 = pngBuf instanceof Uint8Array ? pngBuf : new Uint8Array(pngBuf);
+            let binary = "";
+            const chunk = 0x8000;
+            for (let i = 0; i < u8.length; i += chunk) {
+                binary += String.fromCharCode(...u8.subarray(i, i + chunk));
+            }
+            return btoa(binary);
+        }, selectedPageUrl);
+        if (!b64 || typeof b64 !== "string") {
+            return {
+                png: null,
+                note: "window capture unavailable: no matching source (permission/minimized/offscreen)",
+            };
+        }
+        const png = Buffer.from(b64, "base64");
+        if (png.length < 50) {
+            return { png: null, note: "window capture unavailable: empty thumbnail" };
+        }
+        if (isAllBlackOrEmpty(png)) {
+            return {
+                png: null,
+                note: "window capture unavailable: all-black thumbnail (permission/offscreen)",
+            };
+        }
+        return { png };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return {
+            png: null,
+            note: `window capture unavailable: ${msg.slice(0, 200)}`,
+        };
+    }
+}
+/**
+ * Capture the FULL OS window including the caption bar (min/max/close buttons)
+ * and borders, by cropping a screen-source thumbnail to the window's outer
+ * bounds (getBounds includes the frame).
+ *
+ * Cross-platform, in-process, no native addon. Caveats:
+ * - Another window on top can intrude (we raise/focus best-effort first).
+ * - macOS needs Screen Recording permission (all-black → null → fallback).
+ * - Multi-monitor: uses getDisplayMatching + display-relative crop.
+ */
+export async function captureOsWindow(app, selectedPageUrl) {
+    try {
+        const b64 = await app.evaluate(async (electron, pageUrl) => {
+            const { desktopCapturer, screen, BrowserWindow, } = electron;
+            const wins = BrowserWindow.getAllWindows().filter((w) => !w.isDestroyed());
+            const win = wins.find((w) => {
+                try {
+                    return w.webContents.getURL() === pageUrl;
+                }
+                catch {
+                    return false;
+                }
+            }) ?? wins[0];
+            if (!win)
+                return null;
+            // Raise to avoid occlusion (best-effort).
+            try {
+                if (win.isMinimized())
+                    win.restore();
+                win.show();
+                win.focus();
+                if (typeof win.moveTop === "function")
+                    win.moveTop();
+            }
+            catch {
+                /* ignore raise failures */
+            }
+            // Brief tick so the compositor paints the raised window.
+            await new Promise((r) => setTimeout(r, 80));
+            const b = win.getBounds();
+            if (b.width <= 0 || b.height <= 0)
+                return null;
+            const display = screen.getDisplayMatching(b);
+            const sf = display.scaleFactor || 1;
+            const thumbW = Math.round(display.size.width * sf);
+            const thumbH = Math.round(display.size.height * sf);
+            if (thumbW < 2 || thumbH < 2)
+                return null;
+            const sources = await desktopCapturer.getSources({
+                types: ["screen"],
+                thumbnailSize: { width: thumbW, height: thumbH },
+            });
+            if (!sources.length)
+                return null;
+            const displayIdStr = String(display.id);
+            const src = sources.find((s) => s.display_id === displayIdStr) ?? sources[0];
+            if (!src || src.thumbnail.isEmpty())
+                return null;
+            // Window rect relative to the display, in physical pixels.
+            const cropX = Math.round((b.x - display.bounds.x) * sf);
+            const cropY = Math.round((b.y - display.bounds.y) * sf);
+            const cropW = Math.round(b.width * sf);
+            const cropH = Math.round(b.height * sf);
+            const thumbSize = src.thumbnail.getSize();
+            // Reject crop that is outside the thumbnail / zero-size.
+            if (cropW < 2 ||
+                cropH < 2 ||
+                cropX + cropW <= 0 ||
+                cropY + cropH <= 0 ||
+                cropX >= thumbSize.width ||
+                cropY >= thumbSize.height) {
+                return null;
+            }
+            // Clamp crop to thumbnail bounds (partial off-screen windows).
+            const x = Math.max(0, cropX);
+            const y = Math.max(0, cropY);
+            const width = Math.min(cropW - (x - cropX), thumbSize.width - x);
+            const height = Math.min(cropH - (y - cropY), thumbSize.height - y);
+            if (width < 2 || height < 2)
+                return null;
+            const cropped = src.thumbnail.crop({ x, y, width, height });
+            if (cropped.isEmpty())
+                return null;
+            const cSize = cropped.getSize();
+            if (cSize.width < 2 || cSize.height < 2)
+                return null;
+            const pngBuf = cropped.toPNG();
+            const u8 = pngBuf instanceof Uint8Array ? pngBuf : new Uint8Array(pngBuf);
+            let binary = "";
+            const chunk = 0x8000;
+            for (let i = 0; i < u8.length; i += chunk) {
+                binary += String.fromCharCode(...u8.subarray(i, i + chunk));
+            }
+            return btoa(binary);
+        }, selectedPageUrl);
+        if (!b64 || typeof b64 !== "string") {
+            return {
+                png: null,
+                note: "os-window capture unavailable: no screen source / crop failed (permission/minimized/offscreen)",
+            };
+        }
+        const png = Buffer.from(b64, "base64");
+        if (png.length < 50) {
+            return { png: null, note: "os-window capture unavailable: empty crop" };
+        }
+        if (isAllBlackOrEmpty(png)) {
+            return {
+                png: null,
+                note: "os-window capture unavailable: all-black image (Screen Recording permission / offscreen)",
+            };
+        }
+        return { png };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return {
+            png: null,
+            note: `os-window capture unavailable: ${msg.slice(0, 200)}`,
+        };
+    }
+}
+/** Detect zero-size / fully black-or-transparent captures (macOS permission). */
+function isAllBlackOrEmpty(pngBuf) {
+    try {
+        const img = PNG.sync.read(pngBuf);
+        if (img.width < 2 || img.height < 2)
+            return true;
+        let nonBlack = 0;
+        let samples = 0;
+        const stepX = Math.max(1, Math.floor(img.width / 16));
+        const stepY = Math.max(1, Math.floor(img.height / 16));
+        for (let y = 0; y < img.height; y += stepY) {
+            for (let x = 0; x < img.width; x += stepX) {
+                const i = (img.width * y + x) << 2;
+                const r = img.data[i];
+                const g = img.data[i + 1];
+                const b = img.data[i + 2];
+                const a = img.data[i + 3];
+                samples++;
+                if (a > 16 && (r > 8 || g > 8 || b > 8))
+                    nonBlack++;
+            }
+        }
+        if (samples === 0)
+            return true;
+        return nonBlack / samples < 0.01;
+    }
+    catch {
+        return true;
+    }
+}

package/dist/core/windows.d.ts

@@ -0,0 +1,29 @@
+import type { ElectronApplication, Page } from "playwright-core";
+import type { HoleRect } from "./types.js";
+export interface SelectWindowOptions {
+    /**
+     * Substring or full URL that the target Page must match.
+     * When set, takes priority over the non-http heuristic.
+     */
+    windowUrlMatch?: string;
+}
+export interface SelectWindowResult {
+    page: Page;
+    /** How the window was chosen (for diagnostics). */
+    strategy: "windowUrlMatch" | "nonHttpUrl" | "mainProcessTop" | "firstWindow";
+    warning?: string;
+}
+/**
+ * Pick the app-owned top-level window rather than an embedded BrowserView.
+ *
+ * Pursuit (and similar shells) expose the BrowserView (https://…) first;
+ * `firstWindow()` returns that. The real app window is often a `data:` URL
+ * (or file://). Prefer non-http(s) pages, cross-check via main-process
+ * BrowserWindow, allow config override.
+ */
+export declare function selectTargetWindow(app: ElectronApplication, opts?: SelectWindowOptions): Promise<SelectWindowResult>;
+/**
+ * Read BrowserView bounds from the main process for the window whose
+ * webContents URL matches `selectedPageUrl` (falls back to first window).
+ */
+export declare function detectHoles(app: ElectronApplication, selectedPageUrl?: string): Promise<HoleRect[]>;

package/dist/core/windows.js

@@ -0,0 +1,94 @@
+/**
+ * Pick the app-owned top-level window rather than an embedded BrowserView.
+ *
+ * Pursuit (and similar shells) expose the BrowserView (https://…) first;
+ * `firstWindow()` returns that. The real app window is often a `data:` URL
+ * (or file://). Prefer non-http(s) pages, cross-check via main-process
+ * BrowserWindow, allow config override.
+ */
+export async function selectTargetWindow(app, opts = {}) {
+    const pages = app.windows();
+    if (pages.length === 0) {
+        const page = await app.firstWindow({ timeout: 30_000 });
+        return {
+            page,
+            strategy: "firstWindow",
+            warning: "No windows listed yet; fell back to firstWindow().",
+        };
+    }
+    // 1. Explicit config override.
+    if (opts.windowUrlMatch) {
+        const needle = opts.windowUrlMatch;
+        const match = pages.find((p) => p.url().includes(needle));
+        if (match) {
+            return { page: match, strategy: "windowUrlMatch" };
+        }
+    }
+    // 2. Prefer app-owned top-level: URL does NOT match ^https?:
+    //    Verified on Pursuit: data: URL shell vs https BrowserView.
+    const nonHttp = pages.find((p) => !/^https?:/i.test(p.url()));
+    if (nonHttp) {
+        return { page: nonHttp, strategy: "nonHttpUrl" };
+    }
+    // 3. Cross-check via main process: match BrowserWindow by selected heuristics.
+    try {
+        const topUrl = await app.evaluate(({ BrowserWindow }) => {
+            const wins = BrowserWindow.getAllWindows();
+            // Prefer the focused / first non-destroyed top-level window.
+            const w = wins.find((x) => !x.isDestroyed()) ?? wins[0];
+            return w ? w.webContents.getURL() : null;
+        });
+        if (typeof topUrl === "string" && topUrl.length > 0) {
+            const match = pages.find((p) => p.url() === topUrl);
+            if (match) {
+                return { page: match, strategy: "mainProcessTop" };
+            }
+        }
+    }
+    catch {
+        // Main-process evaluate can fail if the app is mid-quit; fall through.
+    }
+    // 4. Last resort.
+    const page = pages[0] ?? (await app.firstWindow({ timeout: 5_000 }));
+    return {
+        page,
+        strategy: "firstWindow",
+        warning: "Could not identify app-owned window via non-http URL or main process; using firstWindow().",
+    };
+}
+/**
+ * Read BrowserView bounds from the main process for the window whose
+ * webContents URL matches `selectedPageUrl` (falls back to first window).
+ */
+export async function detectHoles(app, selectedPageUrl) {
+    try {
+        const holes = await app.evaluate(({ BrowserWindow }, pageUrl) => {
+            const wins = BrowserWindow.getAllWindows().filter((w) => !w.isDestroyed());
+            const w = (pageUrl
+                ? wins.find((x) => {
+                    try {
+                        return x.webContents.getURL() === pageUrl;
+                    }
+                    catch {
+                        return false;
+                    }
+                })
+                : undefined) ?? wins[0];
+            if (!w)
+                return [];
+            const views = typeof w.getBrowserViews === "function"
+                ? w.getBrowserViews()
+                : [];
+            return views.map((v) => v.getBounds());
+        }, selectedPageUrl ?? null);
+        return (holes ?? []).filter((h) => typeof h?.x === "number" &&
+            typeof h?.y === "number" &&
+            typeof h?.width === "number" &&
+            typeof h?.height === "number" &&
+            h.width > 0 &&
+            h.height > 0);
+    }
+    catch {
+        return [];
+    }
+}

package/dist/mcp.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};