@webcrawl/mcp 1.4.2 → 1.4.4

package/dist/core/engine/chrome.js

@@ -0,0 +1,208 @@
+/**
+ * Local Chrome renderer (JS pages the native fetch ladder cannot hydrate).
+ *
+ * Primary driver is Chrome DevTools Protocol:
+ * launch headless Chrome, navigate, wait until the DOM is content-ready
+ * (load + 200ms mutation-idle), then read `documentElement.outerHTML`.
+ *
+ * dump-dom is a last-resort fallback: Chrome 156 hangs on dump-dom with a
+ * fresh profile, which used to look like a successful scrape of the
+ * unrendered nav shell. Every attempt has a hard timeout and a process-group
+ * kill. Playwright's chrome-headless-shell is discovered automatically when
+ * present.
+ */
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { renderUrlViaCdp } from "./chrome-cdp.js";
+import { DEFAULT_TIMEOUT_MS, DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, } from "./chrome-launch.js";
+export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, } from "./chrome-launch.js";
+function attemptPlan(binary) {
+    const isShell = /headless[-_]?shell/i.test(binary);
+    if (isShell) {
+        return [
+            { driver: "cdp", headless: "shell" },
+            { driver: "dump-dom", headless: "shell" },
+        ];
+    }
+    return [
+        { driver: "cdp", headless: "new" },
+        { driver: "cdp", headless: "old" },
+        { driver: "dump-dom", headless: "old" },
+        { driver: "dump-dom", headless: "new" },
+    ];
+}
+function methodName(binary, kind) {
+    const base = binary.split(/[/\\]/).pop() || binary;
+    return `${kind.driver}/${kind.headless} (${base})`;
+}
+/**
+ * Render `url` in local Chrome and return the hydrated HTML, or null when
+ * Chrome is missing, times out, or produces nothing.
+ */
+export async function renderViaLocalChrome(url, opts = {}) {
+    const result = await renderViaLocalChromeDetailed(url, opts);
+    return result.html;
+}
+/**
+ * Same as `renderViaLocalChrome` but with the attempt log and an actionable
+ * error when every driver fails. Callers that need to report honest JS-render
+ * failures should use this.
+ */
+export async function renderViaLocalChromeDetailed(url, opts = {}) {
+    const binaries = discoverChromeBinaries();
+    if (binaries.length === 0) {
+        return {
+            html: null,
+            error: "No local Chrome/Chromium binary found. Install Chrome, set WEBCRAWL_CHROME_PATH, " +
+                "or run `npx playwright install chromium-headless-shell`. See `webcrawl doctor`.",
+            attempts: [],
+        };
+    }
+    const budgetMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const started = Date.now();
+    const attempts = [];
+    let lastError = "local Chrome produced no content";
+    for (const binary of binaries) {
+        for (const kind of attemptPlan(binary)) {
+            const remaining = budgetMs - (Date.now() - started);
+            if (remaining < 1_200)
+                break;
+            const method = methodName(binary, kind);
+            const attemptBudget = kind.driver === "cdp"
+                ? Math.min(remaining, kind.headless === "new" ? 12_000 : 8_000)
+                : Math.min(remaining, 6_000);
+            const t0 = Date.now();
+            try {
+                const html = kind.driver === "cdp"
+                    ? (await renderUrlViaCdp(binary, url, {
+                        timeoutMs: attemptBudget,
+                        headless: kind.headless,
+                    })).html
+                    : await dumpDom(binary, url, kind.headless, attemptBudget);
+                const ms = Date.now() - t0;
+                if (html && html.length > 100) {
+                    attempts.push({ ok: true, method, ms });
+                    return { html, method, attempts };
+                }
+                lastError = `${method} returned empty HTML`;
+                attempts.push({ ok: false, method, ms, error: lastError });
+            }
+            catch (err) {
+                const ms = Date.now() - t0;
+                lastError = `${method}: ${err instanceof Error ? err.message : String(err)}`;
+                attempts.push({ ok: false, method, ms, error: lastError });
+            }
+        }
+    }
+    return {
+        html: null,
+        error: `JavaScript rendering failed (${lastError}). ` +
+            "Chrome hung, timed out, or produced no content. Install Chrome, point WEBCRAWL_CHROME_PATH " +
+            "at chrome-headless-shell (`npx playwright install chromium-headless-shell`), " +
+            "run `webcrawl login` and retry with `--mode cloud`, or set WEBCRAWL_RENDER_URL.",
+        attempts,
+    };
+}
+async function dumpDom(chrome, url, headless, timeoutMs) {
+    let profile = "";
+    try {
+        profile = mkdtempSync(join(tmpdir(), "wc-chrome-"));
+    }
+    catch {
+        throw new Error("could not create Chrome profile directory");
+    }
+    const args = chromeDumpDomArgs({
+        profile,
+        headless,
+        url,
+        timeoutMs,
+        binaryPath: chrome,
+    });
+    try {
+        return await new Promise((resolve, reject) => {
+            let settled = false;
+            const done = (err, value) => {
+                if (settled)
+                    return;
+                settled = true;
+                if (err)
+                    reject(err);
+                else
+                    resolve(value ?? "");
+            };
+            let child;
+            try {
+                child = spawn(chrome, args, {
+                    stdio: ["ignore", "pipe", "pipe"],
+                    detached: true,
+                });
+            }
+            catch (err) {
+                done(err instanceof Error ? err : new Error(String(err)));
+                return;
+            }
+            const chunks = [];
+            let bytes = 0;
+            let idle = null;
+            const finish = (err) => {
+                clearTimeout(timer);
+                if (idle)
+                    clearTimeout(idle);
+                killProcessGroup(child.pid, child);
+                if (err) {
+                    done(err);
+                    return;
+                }
+                const text = Buffer.concat(chunks).toString("utf8");
+                done(null, text.trim());
+            };
+            const timer = setTimeout(() => {
+                if (bytes > 100)
+                    finish(null);
+                else
+                    finish(new Error(`dump-dom timed out after ${timeoutMs}ms`));
+            }, timeoutMs);
+            child.stdout?.on("data", (c) => {
+                if (bytes >= DUMP_MAX_BYTES)
+                    return;
+                const take = c.subarray(0, DUMP_MAX_BYTES - bytes);
+                chunks.push(take);
+                bytes += take.length;
+                // Chrome 156 often prints the DOM then never exits (`--dump-dom` hang).
+                if (idle)
+                    clearTimeout(idle);
+                idle = setTimeout(() => {
+                    if (bytes > 40)
+                        finish(null);
+                }, 400);
+            });
+            child.on("error", (err) => {
+                clearTimeout(timer);
+                if (idle)
+                    clearTimeout(idle);
+                done(err);
+            });
+            child.on("close", () => {
+                clearTimeout(timer);
+                if (idle)
+                    clearTimeout(idle);
+                if (bytes === 0) {
+                    done(new Error("dump-dom produced no stdout"));
+                    return;
+                }
+                const text = Buffer.concat(chunks).toString("utf8");
+                done(null, text.trim().length > 0 ? text : "");
+            });
+        });
+    }
+    finally {
+        try {
+            rmSync(profile, { recursive: true, force: true });
+        }
+        catch {
+            /* leftover profile is in /tmp */
+        }
+    }
+}

package/dist/core/engine/crawl-policy.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+export interface RobotsRules {
+    allows(url: string): boolean;
+    /** Crawl-delay in ms for our agent (or `*`), 0 when unset. */
+    crawlDelayMs: number;
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export declare function parseRobots(text: string | null | undefined): RobotsRules;
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export declare class AdaptiveConcurrency {
+    private readonly min;
+    private readonly max;
+    private target;
+    constructor(min?: number, max?: number, initial?: number);
+    get current(): number;
+    /** Feed the outcome of a finished wave. */
+    record(outcome: {
+        ok: number;
+        failed: number;
+        throttled: boolean;
+    }): number;
+}
+export declare const sleep: (ms: number) => Promise<void>;
+export declare function backoffDelayMs(attempt: number, baseMs?: number, capMs?: number, rand?: () => number): number;
+export declare function isRetryableStatus(status: number): boolean;
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export declare function withRetries<T extends {
+    status: number;
+}>(run: () => Promise<T>, opts?: {
+    retries?: number;
+    baseMs?: number;
+    sleepFn?: (ms: number) => Promise<void>;
+}): Promise<T>;

package/dist/core/engine/crawl-policy.js

@@ -0,0 +1,134 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+const ROBOT_TOKEN = "webcrawl";
+function patternToRegExp(path) {
+    const anchored = path.endsWith("$");
+    const body = (anchored ? path.slice(0, -1) : path)
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    return new RegExp(`^${body}${anchored ? "$" : ""}`);
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export function parseRobots(text) {
+    const groups = [];
+    let current = null;
+    let lastWasAgent = false;
+    for (const rawLine of (text ?? "").split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(":");
+        if (idx < 0)
+            continue;
+        const key = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (key === "user-agent") {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [], delay: 0 };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!current)
+            continue;
+        if (key === "disallow" || key === "allow") {
+            if (!value && key === "disallow")
+                continue; // empty Disallow = allow all
+            current.rules.push({
+                allow: key === "allow",
+                pattern: patternToRegExp(value),
+                length: value.length,
+            });
+        }
+        else if (key === "crawl-delay") {
+            const n = Number(value);
+            if (Number.isFinite(n) && n > 0)
+                current.delay = Math.min(n, 10) * 1000;
+        }
+    }
+    const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && ROBOT_TOKEN.includes(a)));
+    const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
+    const rules = chosen.flatMap((g) => g.rules);
+    const crawlDelayMs = Math.max(0, ...chosen.map((g) => g.delay));
+    return {
+        crawlDelayMs,
+        allows(url) {
+            let path;
+            try {
+                const u = new URL(url);
+                path = u.pathname + u.search;
+            }
+            catch {
+                return true;
+            }
+            let best = null;
+            for (const r of rules) {
+                if (!r.pattern.test(path))
+                    continue;
+                if (!best || r.length > best.length || (r.length === best.length && r.allow))
+                    best = r;
+            }
+            return best ? best.allow : true;
+        },
+    };
+}
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export class AdaptiveConcurrency {
+    min;
+    max;
+    target;
+    constructor(min = 1, max = 16, initial = 4) {
+        this.min = min;
+        this.max = max;
+        this.target = Math.min(Math.max(initial, min), max);
+    }
+    get current() {
+        return this.target;
+    }
+    /** Feed the outcome of a finished wave. */
+    record(outcome) {
+        const total = outcome.ok + outcome.failed;
+        if (outcome.throttled || (total > 0 && outcome.failed / total > 0.3)) {
+            this.target = Math.max(this.min, Math.floor(this.target / 2));
+        }
+        else if (outcome.ok > 0) {
+            this.target = Math.min(this.max, this.target + 1);
+        }
+        return this.target;
+    }
+}
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export function backoffDelayMs(attempt, baseMs = 250, capMs = 4000, rand = Math.random) {
+    const exp = Math.min(capMs, baseMs * 2 ** attempt);
+    return Math.round(exp / 2 + rand() * (exp / 2));
+}
+export function isRetryableStatus(status) {
+    return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export async function withRetries(run, opts = {}) {
+    const retries = opts.retries ?? 2;
+    const wait = opts.sleepFn ?? sleep;
+    let lastErr;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const res = await run();
+            if (!isRetryableStatus(res.status) || attempt === retries)
+                return res;
+        }
+        catch (err) {
+            if (err?.name === "UnsafeUrlError")
+                throw err;
+            lastErr = err;
+            if (attempt === retries)
+                throw err;
+        }
+        await wait(backoffDelayMs(attempt, opts.baseMs));
+    }
+    throw lastErr;
+}

package/dist/core/engine/crawler.d.ts

@@ -0,0 +1,152 @@
+export interface MapWebsiteOptions {
+    /** Only crawl the exact root/target domain (default: true when includeSubdomains is false and selectedDomains is empty) */
+    onlyRootDomain?: boolean;
+    /** Also crawl subdomains of the allowed domain(s), e.g. docs.example.com (default: false) */
+    includeSubdomains?: boolean;
+    /** Explicit allowlist of domains/subdomains (supports exact hosts like "docs.example.com" or wildcards like "*.example.com") */
+    selectedDomains?: string[];
+    /** Maximum number of discovered URLs to return (default: 5000) */
+    limit?: number;
+    /** Maximum number of HTML pages to fetch during multi-hop BFS crawl when sitemap is missing (default: 250) */
+    maxCrawlPages?: number;
+    /** Alias for maxCrawlPages */
+    maxCrawl?: number;
+    /** Maximum BFS hop depth from the entry URL (default: 8) */
+    maxDepth?: number;
+    /** Only keep URLs whose path matches one of these (prefix, `*` wildcards, trailing `$`). */
+    includePaths?: string[];
+    /** Drop URLs whose path matches any of these. Wins over includePaths. */
+    excludePaths?: string[];
+    /** Stay under the start URL's path when no includePaths are given (default: true). */
+    restrictToSubpath?: boolean;
+    /** Sitemaps to read first (e.g. a site whose sitemap is not at /sitemap.xml). */
+    sitemapUrls?: string[];
+    /** Overall map timeout in milliseconds (default: 25000) */
+    timeoutMs?: number;
+    /** Bypass 24-hour map cache (default: false) */
+    bypassCache?: boolean;
+    /** When a bot check blocks plain requests, find pages through a real browser (default: true) */
+    stealth?: boolean;
+    /** Time budget for the stealth browser session (default: 75000) */
+    stealthBudgetMs?: number;
+    /** Allow the paid remote stealth provider (default true). Off for unbilled runs. */
+    remoteStealth?: boolean;
+}
+export declare function mapWebsiteUrls(targetUrl: string, options?: MapWebsiteOptions): Promise<{
+    success: boolean;
+    links: string[];
+    latencyMs: number;
+    source: string;
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+    pagesCrawled?: number;
+    /** The site answered plain requests with a bot check. */
+    blocked?: boolean;
+    /** Pages were found through the stealth browser. */
+    stealth?: boolean;
+    /** What a remote stealth provider charged for discovery, in USD. */
+    stealthCostUsd?: number;
+    scope?: {
+        onlyRootDomain: boolean;
+        includeSubdomains: boolean;
+        selectedDomains: string[];
+        maxCrawlPages: number;
+        maxDepth: number;
+        includePaths: string[];
+        excludePaths: string[];
+    };
+}>;
+/**
+ * Ultra-fast HTTP status batch check for `websiteDeletedPages.ts`.
+ * Checks if URLs return 404/410 without downloading/rendering full markdown!
+ */
+export declare function checkUrlsStatusBatch(urls: string[], timeoutMs?: number): Promise<Array<{
+    url: string;
+    statusCode: number;
+    ok: boolean;
+}>>;
+/**
+ * Extracts structured JSON from a URL.
+ * Honours `prompt` / `schema` when provided; otherwise falls back to
+ * `{ app_name, product_features }` from title and headings.
+ */
+export declare function extractWebsiteData(url: string, options?: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    openaiApiKey?: string;
+    bypassCache?: boolean;
+    /** Already-extracted content (e.g. a parsed PDF); skips the HTML fetch and cache. */
+    markdown?: string;
+    /** Source HTML when the caller already fetched the page (quote cards, JSON-LD). */
+    html?: string;
+}): Promise<Record<string, unknown> & {
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+}>;
+/**
+ * Full-website multi-page crawler (`POST /v1/crawl`) backed by the shared 3-tier
+ * L1 Memory -> L2 Disk -> L3 Postgres URL store (`crawled_pages`).
+ * Pages already stored in Postgres for this website are served directly without re-crawling!
+ */
+export declare function crawlWebsitePages(targetUrl: string, options?: {
+    maxPages?: number;
+    includeSubdomains?: boolean;
+    onlyRootDomain?: boolean;
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<{
+    success: boolean;
+    rootUrl: string;
+    pagesCount: number;
+    cachedPagesCount: number;
+    postgresHits: number;
+    latencyMs: number;
+    pages: CrawledPage[];
+}>;
+export interface CrawledPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+    structuredData?: Record<string, unknown> | null;
+    statusCode?: number;
+    error?: string;
+    /** The site served a bot check instead of this page. */
+    blocked?: boolean;
+    /** Read through the stealth browser. */
+    stealth?: boolean;
+    /** Remote provider that hosted the stealth browser (billed with a surcharge). */
+    stealthProvider?: string;
+}
+/** True for markdown converted from a bot-check page (old cache entries). */
+export declare function isChallengeMarkdown(title: string, markdown: string): boolean;
+/**
+ * Re-reads blocked pages through one stealth browser session. Pages that come
+ * back clean are cached and returned as content; pages still blocked keep
+ * `blocked: true` and an honest error.
+ */
+export declare function crawlPagesStealth(entryUrl: string, urls: string[], options?: {
+    extractStructured?: boolean;
+    budgetMs?: number;
+    remote?: boolean;
+}): Promise<{
+    pages: CrawledPage[];
+    solved: boolean;
+    durationMs: number;
+    error?: string;
+    provider?: string;
+    providerCostUsd?: number;
+}>;
+/**
+ * Fetches one page for a crawl: cache first (unless bypassed), then a
+ * TLS-impersonated fetch. Never throws; failures come back with `error`.
+ * Used by `crawlWebsitePages` and by the durable crawl workflow (one call per page).
+ */
+export declare function crawlPage(pageUrl: string, options?: {
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<CrawledPage>;