@webcrawl/mcp 1.4.3 → 1.4.4

package/dist/core/engine/chrome-cdp.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Minimal Chrome DevTools Protocol client for local JS rendering.
+ *
+ * Drive a real Chrome over CDP instead of dump-dom, wait until the DOM is
+ * actually ready, and always bound the session with a hard timeout. No
+ * puppeteer/playwright dependency.
+ */
+import { spawn } from "node:child_process";
+import { type ChromeHeadlessMode } from "./chrome-launch.js";
+export interface CdpSnapshot {
+    html: string;
+    title: string;
+    textLen: number;
+    readyState: string;
+}
+export interface CdpRenderOptions {
+    timeoutMs: number;
+    headless: ChromeHeadlessMode;
+    /** Injected for tests. */
+    spawnChrome?: typeof spawn;
+}
+export declare function renderUrlViaCdp(chromePath: string, url: string, opts: CdpRenderOptions): Promise<CdpSnapshot>;

package/dist/core/engine/chrome-cdp.js

@@ -0,0 +1,318 @@
+/**
+ * Minimal Chrome DevTools Protocol client for local JS rendering.
+ *
+ * Drive a real Chrome over CDP instead of dump-dom, wait until the DOM is
+ * actually ready, and always bound the session with a hard timeout. No
+ * puppeteer/playwright dependency.
+ */
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { chromeCdpArgs, killProcessGroup } from "./chrome-launch.js";
+const SNAPSHOT_JS = `(() => {
+  const body = document.body;
+  const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
+  return {
+    html: document.documentElement ? document.documentElement.outerHTML : "",
+    title: document.title || "",
+    textLen: text.length,
+    readyState: document.readyState || "",
+  };
+})()`;
+/** mutationIdle (200ms quiet) plus readyState complete, capped. */
+const WAIT_FOR_CONTENT_JS = `(async () => {
+  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+  const snap = () => {
+    const body = document.body;
+    const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
+    return {
+      html: document.documentElement ? document.documentElement.outerHTML : "",
+      title: document.title || "",
+      textLen: text.length,
+      readyState: document.readyState || "",
+      elemCount: document.getElementsByTagName("*").length,
+    };
+  };
+  let lastMut = Date.now();
+  let observer = null;
+  try {
+    observer = new MutationObserver(() => { lastMut = Date.now(); });
+    if (document.documentElement) {
+      observer.observe(document.documentElement, {
+        subtree: true,
+        childList: true,
+        characterData: true,
+      });
+    }
+  } catch (_) { /* ignore */ }
+  const start = Date.now();
+  let last = snap();
+  while (Date.now() - start < 2500) {
+    await sleep(150);
+    const now = snap();
+    const quiet = Date.now() - lastMut >= 200;
+    const grown = now.elemCount > last.elemCount + 2 || now.textLen > last.textLen + 20;
+    last = now;
+    if (
+      now.readyState === "complete" &&
+      now.textLen >= 80 &&
+      quiet &&
+      !grown
+    ) break;
+  }
+  try { observer && observer.disconnect(); } catch (_) { /* ignore */ }
+  return last;
+})()`;
+export async function renderUrlViaCdp(chromePath, url, opts) {
+    const timeoutMs = Math.max(1_500, opts.timeoutMs);
+    const profile = mkdtempSync(join(tmpdir(), "wc-cdp-"));
+    const port = 40000 + Math.floor(Math.random() * 20000);
+    const spawnFn = opts.spawnChrome ?? spawn;
+    const args = chromeCdpArgs({
+        port,
+        profile,
+        headless: opts.headless,
+        binaryPath: chromePath,
+    });
+    let child;
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), timeoutMs);
+    try {
+        child = spawnFn(chromePath, args, {
+            stdio: ["ignore", "ignore", "pipe"],
+            detached: true,
+        });
+        if (!child.pid) {
+            throw new Error("Chrome failed to start (no pid)");
+        }
+        let cdpReady = false;
+        child.once("exit", () => {
+            if (!cdpReady && !ac.signal.aborted)
+                ac.abort();
+        });
+        const wsUrl = await waitForPageWebSocket(port, timeoutMs, ac.signal);
+        cdpReady = true;
+        const snapshot = await withCdpSession(wsUrl, url, timeoutMs, ac.signal);
+        if (!snapshot.html || snapshot.html.length < 40) {
+            throw new Error("CDP returned empty HTML");
+        }
+        return snapshot;
+    }
+    catch (err) {
+        if (ac.signal.aborted) {
+            throw new Error(`CDP timed out after ${timeoutMs}ms`);
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timer);
+        if (child?.pid)
+            killProcessGroup(child.pid, child);
+        try {
+            rmSync(profile, { recursive: true, force: true });
+        }
+        catch {
+            /* leftover profile is in /tmp */
+        }
+    }
+}
+async function waitForPageWebSocket(port, timeoutMs, signal) {
+    const deadline = Date.now() + Math.min(timeoutMs, 8_000);
+    let lastErr = "CDP HTTP endpoint never became ready";
+    while (Date.now() < deadline) {
+        if (signal.aborted)
+            throw new Error("Chrome CDP timed out while starting");
+        try {
+            const res = await fetch(`http://127.0.0.1:${port}/json/list`, {
+                signal: AbortSignal.timeout(400),
+            });
+            if (res.ok) {
+                const json = (await res.json());
+                const page = (Array.isArray(json) ? json : []).find((t) => t.webSocketDebuggerUrl && (t.type === "page" || !t.type));
+                if (page?.webSocketDebuggerUrl)
+                    return page.webSocketDebuggerUrl;
+                lastErr = "CDP /json/list had no page websocket";
+            }
+            else {
+                lastErr = `CDP /json/list HTTP ${res.status}`;
+            }
+        }
+        catch (err) {
+            lastErr = err instanceof Error ? err.message : String(err);
+        }
+        await delay(80, signal);
+    }
+    throw new Error(lastErr);
+}
+async function withCdpSession(wsUrl, url, timeoutMs, signal) {
+    const client = await CdpClient.connect(wsUrl, signal);
+    const onAbort = () => client.close();
+    signal.addEventListener("abort", onAbort);
+    try {
+        await client.send("Page.enable");
+        await client.send("Runtime.enable");
+        const load = client.waitForEvent("Page.loadEventFired", Math.min(timeoutMs, 12_000));
+        const nav = await client.send("Page.navigate", { url });
+        if (nav.errorText) {
+            throw new Error(`Page.navigate failed: ${String(nav.errorText)}`);
+        }
+        try {
+            await load;
+        }
+        catch {
+            /* take a snapshot anyway if navigation load times out */
+        }
+        let snapshot = await evaluateSnapshot(client, WAIT_FOR_CONTENT_JS);
+        if (!snapshot || snapshot.textLen < 40 || snapshot.html.length < 80) {
+            snapshot = await evaluateSnapshot(client, SNAPSHOT_JS);
+        }
+        if (!snapshot)
+            throw new Error("Runtime.evaluate returned no snapshot");
+        return snapshot;
+    }
+    finally {
+        signal.removeEventListener("abort", onAbort);
+        client.close();
+    }
+}
+async function evaluateSnapshot(client, expression) {
+    const result = await client.send("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
+    const value = result.result?.value;
+    if (!value || typeof value.html !== "string")
+        return null;
+    return {
+        html: value.html,
+        title: String(value.title ?? ""),
+        textLen: Number(value.textLen ?? 0),
+        readyState: String(value.readyState ?? ""),
+    };
+}
+class CdpClient {
+    ws;
+    nextId = 0;
+    pending = new Map();
+    eventWaiters = [];
+    constructor(ws) {
+        this.ws = ws;
+        ws.addEventListener("message", (ev) => {
+            let msg;
+            try {
+                msg = JSON.parse(String(ev.data));
+            }
+            catch {
+                return;
+            }
+            if (msg.id != null) {
+                const p = this.pending.get(msg.id);
+                if (p) {
+                    this.pending.delete(msg.id);
+                    if (msg.error) {
+                        p.reject(new Error(msg.error.message || `CDP error ${msg.error.code ?? ""}`));
+                    }
+                    else {
+                        p.resolve(msg.result ?? {});
+                    }
+                }
+                return;
+            }
+            if (msg.method) {
+                for (const w of [...this.eventWaiters]) {
+                    if (w.method === msg.method) {
+                        this.eventWaiters = this.eventWaiters.filter((x) => x !== w);
+                        w.resolve();
+                    }
+                }
+            }
+        });
+    }
+    static connect(url, signal) {
+        return new Promise((resolve, reject) => {
+            const ws = new WebSocket(url);
+            const onAbort = () => {
+                try {
+                    ws.close();
+                }
+                catch {
+                    /* ignore */
+                }
+                reject(new Error("Chrome CDP websocket timed out"));
+            };
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
+            signal.addEventListener("abort", onAbort, { once: true });
+            ws.addEventListener("open", () => {
+                signal.removeEventListener("abort", onAbort);
+                resolve(new CdpClient(ws));
+            });
+            ws.addEventListener("error", () => {
+                signal.removeEventListener("abort", onAbort);
+                reject(new Error("Chrome CDP websocket failed"));
+            });
+        });
+    }
+    send(method, params = {}) {
+        const id = ++this.nextId;
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            try {
+                this.ws.send(JSON.stringify({ id, method, params }));
+            }
+            catch (err) {
+                this.pending.delete(id);
+                reject(err instanceof Error ? err : new Error(String(err)));
+            }
+        });
+    }
+    waitForEvent(method, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.eventWaiters = this.eventWaiters.filter((w) => w !== waiter);
+                reject(new Error(`Timed out waiting for ${method}`));
+            }, timeoutMs);
+            const waiter = {
+                method,
+                resolve: () => {
+                    clearTimeout(timer);
+                    resolve();
+                },
+                reject: (err) => {
+                    clearTimeout(timer);
+                    reject(err);
+                },
+            };
+            this.eventWaiters.push(waiter);
+        });
+    }
+    close() {
+        for (const p of this.pending.values()) {
+            p.reject(new Error("CDP session closed"));
+        }
+        this.pending.clear();
+        for (const w of this.eventWaiters) {
+            w.reject(new Error("CDP session closed"));
+        }
+        this.eventWaiters = [];
+        try {
+            this.ws.close();
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+function delay(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal.aborted) {
+            reject(new Error("aborted"));
+            return;
+        }
+        const t = setTimeout(resolve, ms);
+        signal.addEventListener("abort", () => {
+            clearTimeout(t);
+            reject(new Error("aborted"));
+        }, { once: true });
+    });
+}

package/dist/core/engine/chrome-launch.d.ts

@@ -0,0 +1,38 @@
+import type { ChildProcess } from "node:child_process";
+export declare const DUMP_MAX_BYTES: number;
+export declare const DEFAULT_TIMEOUT_MS = 18000;
+export declare const VIRTUAL_TIME_MS = 10000;
+export type ChromeHeadlessMode = "new" | "old" | "shell";
+export declare function killProcessGroup(pid: number | undefined, child?: Pick<ChildProcess, "kill">): void;
+export declare function headlessFlag(mode: ChromeHeadlessMode, binaryPath: string): string[];
+export declare function chromeCdpArgs(opts: {
+    port: number;
+    profile: string;
+    headless: ChromeHeadlessMode;
+    binaryPath?: string;
+}): string[];
+export declare function chromeDumpDomArgs(opts: {
+    profile: string;
+    headless: ChromeHeadlessMode;
+    url: string;
+    timeoutMs: number;
+    binaryPath?: string;
+}): string[];
+/**
+ * First line of Chrome's DevToolsActivePort file is the chosen port when
+ * `--remote-debugging-port=0`.
+ */
+export declare function parseDevToolsActivePort(text: string): number | null;
+export declare function defaultChromeInstallPaths(): string[];
+/** Playwright's chrome-headless-shell, if a previous `playwright install` left it on disk. */
+export declare function findPlaywrightHeadlessShell(): string[];
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks). Env overrides are returned
+ * even if the file is missing so tests can point at a stub binary.
+ */
+export declare function installedChromePath(): string | null;
+export declare function isLocalChromeAvailable(): boolean;
+/** Distinct binaries to try, env override first, then system Chrome, then headless-shell. */
+export declare function discoverChromeBinaries(): string[];

package/dist/core/engine/chrome-launch.js

@@ -0,0 +1,214 @@
+/**
+ * Chrome process helpers shared by the CDP driver and the dump-dom fallback.
+ */
+import { existsSync, readdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+export const DUMP_MAX_BYTES = 2 * 1024 * 1024;
+export const DEFAULT_TIMEOUT_MS = 18_000;
+export const VIRTUAL_TIME_MS = 10_000;
+export function killProcessGroup(pid, child) {
+    if (pid && pid > 0) {
+        try {
+            process.kill(-pid, "SIGKILL");
+            return;
+        }
+        catch {
+            /* process group kill is Unix-only; fall through */
+        }
+        try {
+            process.kill(pid, "SIGKILL");
+            return;
+        }
+        catch {
+            /* already gone */
+        }
+    }
+    try {
+        child?.kill("SIGKILL");
+    }
+    catch {
+        /* already gone */
+    }
+}
+const COMMON_CHROME_ARGS = [
+    "--no-sandbox",
+    "--disable-gpu",
+    "--disable-dev-shm-usage",
+    "--no-first-run",
+    "--no-default-browser-check",
+    "--disable-crash-reporter",
+    "--disable-breakpad",
+    "--disable-background-networking",
+    "--disable-sync",
+    "--disable-extensions",
+    "--disable-popup-blocking",
+    "--mute-audio",
+    "--hide-scrollbars",
+    "--disable-blink-features=AutomationControlled",
+];
+export function headlessFlag(mode, binaryPath) {
+    const isShell = /headless[-_]?shell/i.test(binaryPath);
+    if (isShell || mode === "shell") {
+        // chrome-headless-shell is already a headless binary; --headless=new hangs
+        // on some Chrome 156 installs (the v1.4.3 smoke-test failure).
+        return ["--headless"];
+    }
+    if (mode === "old")
+        return ["--headless=old"];
+    return ["--headless=new"];
+}
+export function chromeCdpArgs(opts) {
+    return [
+        ...headlessFlag(opts.headless, opts.binaryPath ?? ""),
+        ...COMMON_CHROME_ARGS,
+        `--remote-debugging-port=${opts.port}`,
+        "--remote-debugging-address=127.0.0.1",
+        `--user-data-dir=${opts.profile}`,
+        `--window-size=1280,800`,
+        "about:blank",
+    ];
+}
+export function chromeDumpDomArgs(opts) {
+    return [
+        ...headlessFlag(opts.headless, opts.binaryPath ?? ""),
+        ...COMMON_CHROME_ARGS,
+        `--user-data-dir=${opts.profile}`,
+        `--virtual-time-budget=${VIRTUAL_TIME_MS}`,
+        `--timeout=${opts.timeoutMs}`,
+        "--dump-dom",
+        opts.url,
+    ];
+}
+/**
+ * First line of Chrome's DevToolsActivePort file is the chosen port when
+ * `--remote-debugging-port=0`.
+ */
+export function parseDevToolsActivePort(text) {
+    const line = String(text ?? "")
+        .split(/\r?\n/)
+        .map((s) => s.trim())
+        .find(Boolean);
+    if (!line)
+        return null;
+    const port = Number(line);
+    return Number.isInteger(port) && port > 0 && port < 65536 ? port : null;
+}
+function envChromePath() {
+    const fromEnv = process.env.WEBCRAWL_CHROME_PATH?.trim() ||
+        process.env.INTERACT_CHROME_PATH?.trim() ||
+        process.env.CHROME_PATH?.trim();
+    return fromEnv || null;
+}
+export function defaultChromeInstallPaths() {
+    if (process.platform === "darwin") {
+        return [
+            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+            `${process.env.HOME || homedir()}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`,
+            "/Applications/Chromium.app/Contents/MacOS/Chromium",
+        ];
+    }
+    if (process.platform === "win32") {
+        return [
+            `${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`,
+            `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`,
+        ];
+    }
+    return [
+        "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
+        "/usr/bin/chromium-browser",
+        "/usr/bin/chromium",
+        "/snap/bin/chromium",
+    ];
+}
+/** Playwright's chrome-headless-shell, if a previous `playwright install` left it on disk. */
+export function findPlaywrightHeadlessShell() {
+    const roots = [
+        process.env.PLAYWRIGHT_BROWSERS_PATH,
+        join(homedir(), ".cache", "ms-playwright"),
+        join(homedir(), "Library", "Caches", "ms-playwright"),
+        process.env.LOCALAPPDATA
+            ? join(process.env.LOCALAPPDATA, "ms-playwright")
+            : "",
+    ].filter(Boolean);
+    const out = [];
+    for (const root of roots) {
+        if (!existsSync(root))
+            continue;
+        let entries = [];
+        try {
+            entries = readdirSync(root);
+        }
+        catch {
+            continue;
+        }
+        for (const name of entries) {
+            if (!name.startsWith("chromium_headless_shell") && !name.startsWith("chromium-headless-shell")) {
+                continue;
+            }
+            const dir = join(root, name);
+            const candidates = [
+                join(dir, "chrome-headless-shell-linux64", "chrome-headless-shell"),
+                join(dir, "chrome-headless-shell-linux-arm64", "chrome-headless-shell"),
+                join(dir, "chrome-headless-shell-mac-arm64", "chrome-headless-shell"),
+                join(dir, "chrome-headless-shell-mac-x64", "chrome-headless-shell"),
+                join(dir, "chrome-headless-shell-win64", "chrome-headless-shell.exe"),
+            ];
+            for (const c of candidates) {
+                if (existsSync(c))
+                    out.push(c);
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks). Env overrides are returned
+ * even if the file is missing so tests can point at a stub binary.
+ */
+export function installedChromePath() {
+    const fromEnv = envChromePath();
+    if (fromEnv)
+        return fromEnv;
+    for (const p of defaultChromeInstallPaths()) {
+        try {
+            if (p && existsSync(p))
+                return p;
+        }
+        catch {
+            /* not installed here */
+        }
+    }
+    const shells = findPlaywrightHeadlessShell();
+    return shells[0] ?? null;
+}
+export function isLocalChromeAvailable() {
+    return installedChromePath() !== null;
+}
+/** Distinct binaries to try, env override first, then system Chrome, then headless-shell. */
+export function discoverChromeBinaries() {
+    const seen = new Set();
+    const out = [];
+    const add = (p) => {
+        if (!p || seen.has(p))
+            return;
+        seen.add(p);
+        out.push(p);
+    };
+    add(envChromePath());
+    for (const p of defaultChromeInstallPaths()) {
+        try {
+            if (p && existsSync(p))
+                add(p);
+        }
+        catch {
+            /* skip */
+        }
+    }
+    for (const p of findPlaywrightHeadlessShell())
+        add(p);
+    return out;
+}

package/dist/core/engine/chrome.d.ts

@@ -1,10 +1,16 @@
-/**
- * Path to a real installed Chrome/Chromium, or null.
- * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
- * defaults (same locations `webcrawl doctor` checks).
- */
-export declare function installedChromePath(): string | null;
-export declare function isLocalChromeAvailable(): boolean;
+export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, type ChromeHeadlessMode, } from "./chrome-launch.js";
+export interface ChromeRenderAttempt {
+    ok: boolean;
+    method: string;
+    ms: number;
+    error?: string;
+}
+export interface ChromeRenderResult {
+    html: string | null;
+    method?: string;
+    error?: string;
+    attempts: ChromeRenderAttempt[];
+}
 /**
  * Render `url` in local Chrome and return the hydrated HTML, or null when
  * Chrome is missing, times out, or produces nothing.
@@ -12,3 +18,11 @@ export declare function isLocalChromeAvailable(): boolean;
 export declare function renderViaLocalChrome(url: string, opts?: {
     timeoutMs?: number;
 }): Promise<string | null>;
+/**
+ * Same as `renderViaLocalChrome` but with the attempt log and an actionable
+ * error when every driver fails. Callers that need to report honest JS-render
+ * failures should use this.
+ */
+export declare function renderViaLocalChromeDetailed(url: string, opts?: {
+    timeoutMs?: number;
+}): Promise<ChromeRenderResult>;