@webcrawl/mcp 1.4.4 → 1.4.5

package/dist/core/engine/chrome-cdp.d.ts

@@ -7,6 +7,11 @@
  */
 import { spawn } from "node:child_process";
 import { type ChromeHeadlessMode } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export declare class ChromeConnectError extends Error {
+    readonly hung: boolean;
+    constructor(message: string, hung?: boolean);
+}
 export interface CdpSnapshot {
     html: string;
     title: string;
@@ -15,6 +20,8 @@ export interface CdpSnapshot {
 }
 export interface CdpRenderOptions {
     timeoutMs: number;
+    /** CDP launch/connect budget. Defaults to 4s (`WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`). */
+    connectTimeoutMs?: number;
     headless: ChromeHeadlessMode;
     /** Injected for tests. */
     spawnChrome?: typeof spawn;

package/dist/core/engine/chrome-cdp.js

@@ -9,7 +9,16 @@ import { spawn } from "node:child_process";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { chromeCdpArgs, killProcessGroup } from "./chrome-launch.js";
+import { chromeCdpArgs, killProcessGroup, resolveChromeConnectTimeoutMs, } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export class ChromeConnectError extends Error {
+    hung;
+    constructor(message, hung = false) {
+        super(message);
+        this.name = "ChromeConnectError";
+        this.hung = hung;
+    }
+}
 const SNAPSHOT_JS = `(() => {
   const body = document.body;
   const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
@@ -64,8 +73,12 @@ const WAIT_FOR_CONTENT_JS = `(async () => {
   try { observer && observer.disconnect(); } catch (_) { /* ignore */ }
   return last;
 })()`;
+function childStillRunning(child) {
+    return Boolean(child && child.pid && child.exitCode == null && !child.killed);
+}
 export async function renderUrlViaCdp(chromePath, url, opts) {
     const timeoutMs = Math.max(1_500, opts.timeoutMs);
+    const connectTimeoutMs = resolveChromeConnectTimeoutMs(opts.connectTimeoutMs);
     const profile = mkdtempSync(join(tmpdir(), "wc-cdp-"));
     const port = 40000 + Math.floor(Math.random() * 20000);
     const spawnFn = opts.spawnChrome ?? spawn;
@@ -76,7 +89,9 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         binaryPath: chromePath,
     });
     let child;
+    let cdpReady = false;
     const ac = new AbortController();
+    // Page budget covers navigation after connect; connect has its own deadline.
     const timer = setTimeout(() => ac.abort(), timeoutMs);
     try {
         child = spawnFn(chromePath, args, {
@@ -86,12 +101,11 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         if (!child.pid) {
             throw new Error("Chrome failed to start (no pid)");
         }
-        let cdpReady = false;
         child.once("exit", () => {
             if (!cdpReady && !ac.signal.aborted)
                 ac.abort();
         });
-        const wsUrl = await waitForPageWebSocket(port, timeoutMs, ac.signal);
+        const wsUrl = await waitForPageWebSocket(port, connectTimeoutMs, ac.signal);
         cdpReady = true;
         const snapshot = await withCdpSession(wsUrl, url, timeoutMs, ac.signal);
         if (!snapshot.html || snapshot.html.length < 40) {
@@ -100,6 +114,13 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         return snapshot;
     }
     catch (err) {
+        if (!cdpReady) {
+            const hung = childStillRunning(child);
+            const detail = err instanceof Error ? err.message : String(err);
+            throw new ChromeConnectError(hung
+                ? `Chrome did not respond over CDP within ${connectTimeoutMs}ms`
+                : `Chrome CDP failed to start (${detail})`, hung);
+        }
         if (ac.signal.aborted) {
             throw new Error(`CDP timed out after ${timeoutMs}ms`);
         }
@@ -117,8 +138,8 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         }
     }
 }
-async function waitForPageWebSocket(port, timeoutMs, signal) {
-    const deadline = Date.now() + Math.min(timeoutMs, 8_000);
+async function waitForPageWebSocket(port, connectTimeoutMs, signal) {
+    const deadline = Date.now() + connectTimeoutMs;
     let lastErr = "CDP HTTP endpoint never became ready";
     while (Date.now() < deadline) {
         if (signal.aborted)

package/dist/core/engine/chrome-launch.d.ts

@@ -1,7 +1,18 @@
 import type { ChildProcess } from "node:child_process";
 export declare const DUMP_MAX_BYTES: number;
+/** Page navigation + DOM-wait budget once Chrome is answering over CDP. */
 export declare const DEFAULT_TIMEOUT_MS = 18000;
+/**
+ * How long to wait for Chrome to accept CDP after spawn. Kept short and
+ * separate from `DEFAULT_TIMEOUT_MS` so a hung binary fails fast while a
+ * slow but working JS page (often ~7s) still has time to render.
+ */
+export declare const DEFAULT_CONNECT_TIMEOUT_MS = 4000;
 export declare const VIRTUAL_TIME_MS = 10000;
+/** `opts.timeoutMs`, else `WEBCRAWL_CHROME_TIMEOUT_MS`, else 18s. */
+export declare function resolveChromeTimeoutMs(override?: number): number;
+/** `opts.connectTimeoutMs`, else `WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`, else 4s. */
+export declare function resolveChromeConnectTimeoutMs(override?: number): number;
 export type ChromeHeadlessMode = "new" | "old" | "shell";
 export declare function killProcessGroup(pid: number | undefined, child?: Pick<ChildProcess, "kill">): void;
 export declare function headlessFlag(mode: ChromeHeadlessMode, binaryPath: string): string[];

package/dist/core/engine/chrome-launch.js

@@ -5,8 +5,38 @@ import { existsSync, readdirSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 export const DUMP_MAX_BYTES = 2 * 1024 * 1024;
+/** Page navigation + DOM-wait budget once Chrome is answering over CDP. */
 export const DEFAULT_TIMEOUT_MS = 18_000;
+/**
+ * How long to wait for Chrome to accept CDP after spawn. Kept short and
+ * separate from `DEFAULT_TIMEOUT_MS` so a hung binary fails fast while a
+ * slow but working JS page (often ~7s) still has time to render.
+ */
+export const DEFAULT_CONNECT_TIMEOUT_MS = 4_000;
 export const VIRTUAL_TIME_MS = 10_000;
+function envPositiveMs(name, fallback, min) {
+    const raw = process.env[name];
+    if (raw == null || raw.trim() === "")
+        return fallback;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n <= 0)
+        return fallback;
+    return Math.max(min, n);
+}
+/** `opts.timeoutMs`, else `WEBCRAWL_CHROME_TIMEOUT_MS`, else 18s. */
+export function resolveChromeTimeoutMs(override) {
+    if (typeof override === "number" && Number.isFinite(override) && override > 0) {
+        return Math.max(1_500, override);
+    }
+    return envPositiveMs("WEBCRAWL_CHROME_TIMEOUT_MS", DEFAULT_TIMEOUT_MS, 1_500);
+}
+/** `opts.connectTimeoutMs`, else `WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`, else 4s. */
+export function resolveChromeConnectTimeoutMs(override) {
+    if (typeof override === "number" && Number.isFinite(override) && override > 0) {
+        return Math.max(500, override);
+    }
+    return envPositiveMs("WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS", DEFAULT_CONNECT_TIMEOUT_MS, 500);
+}
 export function killProcessGroup(pid, child) {
     if (pid && pid > 0) {
         try {

package/dist/core/engine/chrome.d.ts

@@ -1,4 +1,5 @@
-export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, type ChromeHeadlessMode, } from "./chrome-launch.js";
+export { DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, type ChromeHeadlessMode, } from "./chrome-launch.js";
+export { ChromeConnectError } from "./chrome-cdp.js";
 export interface ChromeRenderAttempt {
     ok: boolean;
     method: string;
@@ -17,6 +18,8 @@ export interface ChromeRenderResult {
  */
 export declare function renderViaLocalChrome(url: string, opts?: {
     timeoutMs?: number;
+    connectTimeoutMs?: number;
+    binaries?: string[];
 }): Promise<string | null>;
 /**
  * Same as `renderViaLocalChrome` but with the attempt log and an actionable
@@ -25,4 +28,6 @@ export declare function renderViaLocalChrome(url: string, opts?: {
  */
 export declare function renderViaLocalChromeDetailed(url: string, opts?: {
     timeoutMs?: number;
+    connectTimeoutMs?: number;
+    binaries?: string[];
 }): Promise<ChromeRenderResult>;

package/dist/core/engine/chrome.js

@@ -15,9 +15,10 @@ import { spawn } from "node:child_process";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { renderUrlViaCdp } from "./chrome-cdp.js";
-import { DEFAULT_TIMEOUT_MS, DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, } from "./chrome-launch.js";
-export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, } from "./chrome-launch.js";
+import { ChromeConnectError, renderUrlViaCdp } from "./chrome-cdp.js";
+import { DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, } from "./chrome-launch.js";
+export { DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, } from "./chrome-launch.js";
+export { ChromeConnectError } from "./chrome-cdp.js";
 function attemptPlan(binary) {
     const isShell = /headless[-_]?shell/i.test(binary);
     if (isShell) {
@@ -51,7 +52,7 @@ export async function renderViaLocalChrome(url, opts = {}) {
  * failures should use this.
  */
 export async function renderViaLocalChromeDetailed(url, opts = {}) {
-    const binaries = discoverChromeBinaries();
+    const binaries = opts.binaries?.length ? opts.binaries : discoverChromeBinaries();
     if (binaries.length === 0) {
         return {
             html: null,
@@ -60,15 +61,20 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
             attempts: [],
         };
     }
-    const budgetMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const budgetMs = resolveChromeTimeoutMs(opts.timeoutMs);
+    const connectTimeoutMs = resolveChromeConnectTimeoutMs(opts.connectTimeoutMs);
     const started = Date.now();
     const attempts = [];
     let lastError = "local Chrome produced no content";
     for (const binary of binaries) {
+        let hungConnect = false;
         for (const kind of attemptPlan(binary)) {
             const remaining = budgetMs - (Date.now() - started);
             if (remaining < 1_200)
                 break;
+            // A process that launched but never answered CDP will hang dump-dom too.
+            if (kind.driver === "dump-dom" && hungConnect)
+                continue;
             const method = methodName(binary, kind);
             const attemptBudget = kind.driver === "cdp"
                 ? Math.min(remaining, kind.headless === "new" ? 12_000 : 8_000)
@@ -78,6 +84,7 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
                 const html = kind.driver === "cdp"
                     ? (await renderUrlViaCdp(binary, url, {
                         timeoutMs: attemptBudget,
+                        connectTimeoutMs: Math.min(connectTimeoutMs, remaining),
                         headless: kind.headless,
                     })).html
                     : await dumpDom(binary, url, kind.headless, attemptBudget);
@@ -93,6 +100,8 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
                 const ms = Date.now() - t0;
                 lastError = `${method}: ${err instanceof Error ? err.message : String(err)}`;
                 attempts.push({ ok: false, method, ms, error: lastError });
+                if (err instanceof ChromeConnectError && err.hung)
+                    hungConnect = true;
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webcrawl/mcp",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "mcpName": "io.github.brainfish-ai/webcrawl",
   "description": "MCP server for Webcrawl Anything: search, scrape, map, crawl, parse and monitor the web from your agent. Runs locally with no API key, or on webcrawl.app cloud.",
   "type": "module",
@@ -56,8 +56,8 @@
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@modelcontextprotocol/sdk": "^1.32.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.4",
-    "@webcrawl/curl-impersonate": "^1.4.4",
+    "@webcrawl/client": "^1.4.5",
+    "@webcrawl/curl-impersonate": "^1.4.5",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",