@webcrawl/mcp 1.4.1 → 1.4.3

package/dist/core/engine/chrome.js

@@ -0,0 +1,140 @@
+/**
+ * Local Chrome dump-dom renderer (JS pages the native fetch ladder cannot hydrate).
+ *
+ * Uses a user-installed Chrome/Chromium binary — the same one `webcrawl doctor`
+ * reports — with `--dump-dom` + virtual time so `document.write` / client-rendered
+ * content appears in the HTML. Playwright is not required; this is the zero-key
+ * local JS tier.
+ */
+import { spawn } from "node:child_process";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+const DUMP_MAX_BYTES = 2 * 1024 * 1024;
+const DEFAULT_TIMEOUT_MS = 20_000;
+const VIRTUAL_TIME_MS = 10_000;
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks).
+ */
+export function installedChromePath() {
+    const fromEnv = process.env.WEBCRAWL_CHROME_PATH?.trim() ||
+        process.env.INTERACT_CHROME_PATH?.trim() ||
+        process.env.CHROME_PATH?.trim();
+    if (fromEnv)
+        return fromEnv;
+    const candidates = [];
+    if (process.platform === "darwin") {
+        candidates.push("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", `${process.env.HOME || ""}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`);
+    }
+    else if (process.platform === "win32") {
+        candidates.push(`${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`, `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`);
+    }
+    else {
+        candidates.push("/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/snap/bin/chromium");
+    }
+    for (const p of candidates) {
+        try {
+            if (p && existsSync(p))
+                return p;
+        }
+        catch {
+            /* not installed here */
+        }
+    }
+    return null;
+}
+export function isLocalChromeAvailable() {
+    return installedChromePath() !== null;
+}
+/**
+ * Render `url` in local Chrome and return the hydrated HTML, or null when
+ * Chrome is missing, times out, or produces nothing.
+ */
+export async function renderViaLocalChrome(url, opts = {}) {
+    const chrome = installedChromePath();
+    if (!chrome)
+        return null;
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    let profile = "";
+    try {
+        profile = mkdtempSync(join(tmpdir(), "wc-chrome-"));
+    }
+    catch {
+        return null;
+    }
+    const args = [
+        "--headless=new",
+        "--no-sandbox",
+        "--disable-gpu",
+        "--disable-dev-shm-usage",
+        "--no-first-run",
+        "--no-default-browser-check",
+        "--disable-crash-reporter",
+        "--disable-breakpad",
+        `--user-data-dir=${profile}`,
+        `--virtual-time-budget=${VIRTUAL_TIME_MS}`,
+        `--timeout=${timeoutMs}`,
+        "--dump-dom",
+        url,
+    ];
+    try {
+        return await new Promise((resolve) => {
+            let settled = false;
+            const done = (value) => {
+                if (settled)
+                    return;
+                settled = true;
+                resolve(value);
+            };
+            let child;
+            try {
+                child = spawn(chrome, args, { stdio: ["ignore", "pipe", "pipe"] });
+            }
+            catch {
+                done(null);
+                return;
+            }
+            const chunks = [];
+            let bytes = 0;
+            const timer = setTimeout(() => {
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch {
+                    /* already gone */
+                }
+                done(null);
+            }, timeoutMs + 2_000);
+            child.stdout?.on("data", (c) => {
+                if (bytes >= DUMP_MAX_BYTES)
+                    return;
+                const take = c.subarray(0, DUMP_MAX_BYTES - bytes);
+                chunks.push(take);
+                bytes += take.length;
+            });
+            child.on("error", () => {
+                clearTimeout(timer);
+                done(null);
+            });
+            child.on("close", () => {
+                clearTimeout(timer);
+                if (bytes === 0) {
+                    done(null);
+                    return;
+                }
+                const text = Buffer.concat(chunks).toString("utf8");
+                done(text.trim().length > 0 ? text : null);
+            });
+        });
+    }
+    finally {
+        try {
+            rmSync(profile, { recursive: true, force: true });
+        }
+        catch {
+            /* leftover profile is in /tmp */
+        }
+    }
+}

package/dist/core/engine/crawl-policy.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+export interface RobotsRules {
+    allows(url: string): boolean;
+    /** Crawl-delay in ms for our agent (or `*`), 0 when unset. */
+    crawlDelayMs: number;
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export declare function parseRobots(text: string | null | undefined): RobotsRules;
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export declare class AdaptiveConcurrency {
+    private readonly min;
+    private readonly max;
+    private target;
+    constructor(min?: number, max?: number, initial?: number);
+    get current(): number;
+    /** Feed the outcome of a finished wave. */
+    record(outcome: {
+        ok: number;
+        failed: number;
+        throttled: boolean;
+    }): number;
+}
+export declare const sleep: (ms: number) => Promise<void>;
+export declare function backoffDelayMs(attempt: number, baseMs?: number, capMs?: number, rand?: () => number): number;
+export declare function isRetryableStatus(status: number): boolean;
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export declare function withRetries<T extends {
+    status: number;
+}>(run: () => Promise<T>, opts?: {
+    retries?: number;
+    baseMs?: number;
+    sleepFn?: (ms: number) => Promise<void>;
+}): Promise<T>;

package/dist/core/engine/crawl-policy.js

@@ -0,0 +1,134 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+const ROBOT_TOKEN = "webcrawl";
+function patternToRegExp(path) {
+    const anchored = path.endsWith("$");
+    const body = (anchored ? path.slice(0, -1) : path)
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    return new RegExp(`^${body}${anchored ? "$" : ""}`);
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export function parseRobots(text) {
+    const groups = [];
+    let current = null;
+    let lastWasAgent = false;
+    for (const rawLine of (text ?? "").split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(":");
+        if (idx < 0)
+            continue;
+        const key = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (key === "user-agent") {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [], delay: 0 };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!current)
+            continue;
+        if (key === "disallow" || key === "allow") {
+            if (!value && key === "disallow")
+                continue; // empty Disallow = allow all
+            current.rules.push({
+                allow: key === "allow",
+                pattern: patternToRegExp(value),
+                length: value.length,
+            });
+        }
+        else if (key === "crawl-delay") {
+            const n = Number(value);
+            if (Number.isFinite(n) && n > 0)
+                current.delay = Math.min(n, 10) * 1000;
+        }
+    }
+    const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && ROBOT_TOKEN.includes(a)));
+    const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
+    const rules = chosen.flatMap((g) => g.rules);
+    const crawlDelayMs = Math.max(0, ...chosen.map((g) => g.delay));
+    return {
+        crawlDelayMs,
+        allows(url) {
+            let path;
+            try {
+                const u = new URL(url);
+                path = u.pathname + u.search;
+            }
+            catch {
+                return true;
+            }
+            let best = null;
+            for (const r of rules) {
+                if (!r.pattern.test(path))
+                    continue;
+                if (!best || r.length > best.length || (r.length === best.length && r.allow))
+                    best = r;
+            }
+            return best ? best.allow : true;
+        },
+    };
+}
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export class AdaptiveConcurrency {
+    min;
+    max;
+    target;
+    constructor(min = 1, max = 16, initial = 4) {
+        this.min = min;
+        this.max = max;
+        this.target = Math.min(Math.max(initial, min), max);
+    }
+    get current() {
+        return this.target;
+    }
+    /** Feed the outcome of a finished wave. */
+    record(outcome) {
+        const total = outcome.ok + outcome.failed;
+        if (outcome.throttled || (total > 0 && outcome.failed / total > 0.3)) {
+            this.target = Math.max(this.min, Math.floor(this.target / 2));
+        }
+        else if (outcome.ok > 0) {
+            this.target = Math.min(this.max, this.target + 1);
+        }
+        return this.target;
+    }
+}
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export function backoffDelayMs(attempt, baseMs = 250, capMs = 4000, rand = Math.random) {
+    const exp = Math.min(capMs, baseMs * 2 ** attempt);
+    return Math.round(exp / 2 + rand() * (exp / 2));
+}
+export function isRetryableStatus(status) {
+    return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export async function withRetries(run, opts = {}) {
+    const retries = opts.retries ?? 2;
+    const wait = opts.sleepFn ?? sleep;
+    let lastErr;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const res = await run();
+            if (!isRetryableStatus(res.status) || attempt === retries)
+                return res;
+        }
+        catch (err) {
+            if (err?.name === "UnsafeUrlError")
+                throw err;
+            lastErr = err;
+            if (attempt === retries)
+                throw err;
+        }
+        await wait(backoffDelayMs(attempt, opts.baseMs));
+    }
+    throw lastErr;
+}

package/dist/core/engine/crawler.d.ts

@@ -0,0 +1,152 @@
+export interface MapWebsiteOptions {
+    /** Only crawl the exact root/target domain (default: true when includeSubdomains is false and selectedDomains is empty) */
+    onlyRootDomain?: boolean;
+    /** Also crawl subdomains of the allowed domain(s), e.g. docs.example.com (default: false) */
+    includeSubdomains?: boolean;
+    /** Explicit allowlist of domains/subdomains (supports exact hosts like "docs.example.com" or wildcards like "*.example.com") */
+    selectedDomains?: string[];
+    /** Maximum number of discovered URLs to return (default: 5000) */
+    limit?: number;
+    /** Maximum number of HTML pages to fetch during multi-hop BFS crawl when sitemap is missing (default: 250) */
+    maxCrawlPages?: number;
+    /** Alias for maxCrawlPages */
+    maxCrawl?: number;
+    /** Maximum BFS hop depth from the entry URL (default: 8) */
+    maxDepth?: number;
+    /** Only keep URLs whose path matches one of these (prefix, `*` wildcards, trailing `$`). */
+    includePaths?: string[];
+    /** Drop URLs whose path matches any of these. Wins over includePaths. */
+    excludePaths?: string[];
+    /** Stay under the start URL's path when no includePaths are given (default: true). */
+    restrictToSubpath?: boolean;
+    /** Sitemaps to read first (e.g. a site whose sitemap is not at /sitemap.xml). */
+    sitemapUrls?: string[];
+    /** Overall map timeout in milliseconds (default: 25000) */
+    timeoutMs?: number;
+    /** Bypass 24-hour map cache (default: false) */
+    bypassCache?: boolean;
+    /** When a bot check blocks plain requests, find pages through a real browser (default: true) */
+    stealth?: boolean;
+    /** Time budget for the stealth browser session (default: 75000) */
+    stealthBudgetMs?: number;
+    /** Allow the paid remote stealth provider (default true). Off for unbilled runs. */
+    remoteStealth?: boolean;
+}
+export declare function mapWebsiteUrls(targetUrl: string, options?: MapWebsiteOptions): Promise<{
+    success: boolean;
+    links: string[];
+    latencyMs: number;
+    source: string;
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+    pagesCrawled?: number;
+    /** The site answered plain requests with a bot check. */
+    blocked?: boolean;
+    /** Pages were found through the stealth browser. */
+    stealth?: boolean;
+    /** What a remote stealth provider charged for discovery, in USD. */
+    stealthCostUsd?: number;
+    scope?: {
+        onlyRootDomain: boolean;
+        includeSubdomains: boolean;
+        selectedDomains: string[];
+        maxCrawlPages: number;
+        maxDepth: number;
+        includePaths: string[];
+        excludePaths: string[];
+    };
+}>;
+/**
+ * Ultra-fast HTTP status batch check for `websiteDeletedPages.ts`.
+ * Checks if URLs return 404/410 without downloading/rendering full markdown!
+ */
+export declare function checkUrlsStatusBatch(urls: string[], timeoutMs?: number): Promise<Array<{
+    url: string;
+    statusCode: number;
+    ok: boolean;
+}>>;
+/**
+ * Extracts structured JSON from a URL.
+ * Honours `prompt` / `schema` when provided; otherwise falls back to
+ * `{ app_name, product_features }` from title and headings.
+ */
+export declare function extractWebsiteData(url: string, options?: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    openaiApiKey?: string;
+    bypassCache?: boolean;
+    /** Already-extracted content (e.g. a parsed PDF); skips the HTML fetch and cache. */
+    markdown?: string;
+    /** Source HTML when the caller already fetched the page (quote cards, JSON-LD). */
+    html?: string;
+}): Promise<Record<string, unknown> & {
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+}>;
+/**
+ * Full-website multi-page crawler (`POST /v1/crawl`) backed by the shared 3-tier
+ * L1 Memory -> L2 Disk -> L3 Postgres URL store (`crawled_pages`).
+ * Pages already stored in Postgres for this website are served directly without re-crawling!
+ */
+export declare function crawlWebsitePages(targetUrl: string, options?: {
+    maxPages?: number;
+    includeSubdomains?: boolean;
+    onlyRootDomain?: boolean;
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<{
+    success: boolean;
+    rootUrl: string;
+    pagesCount: number;
+    cachedPagesCount: number;
+    postgresHits: number;
+    latencyMs: number;
+    pages: CrawledPage[];
+}>;
+export interface CrawledPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+    structuredData?: Record<string, unknown> | null;
+    statusCode?: number;
+    error?: string;
+    /** The site served a bot check instead of this page. */
+    blocked?: boolean;
+    /** Read through the stealth browser. */
+    stealth?: boolean;
+    /** Remote provider that hosted the stealth browser (billed with a surcharge). */
+    stealthProvider?: string;
+}
+/** True for markdown converted from a bot-check page (old cache entries). */
+export declare function isChallengeMarkdown(title: string, markdown: string): boolean;
+/**
+ * Re-reads blocked pages through one stealth browser session. Pages that come
+ * back clean are cached and returned as content; pages still blocked keep
+ * `blocked: true` and an honest error.
+ */
+export declare function crawlPagesStealth(entryUrl: string, urls: string[], options?: {
+    extractStructured?: boolean;
+    budgetMs?: number;
+    remote?: boolean;
+}): Promise<{
+    pages: CrawledPage[];
+    solved: boolean;
+    durationMs: number;
+    error?: string;
+    provider?: string;
+    providerCostUsd?: number;
+}>;
+/**
+ * Fetches one page for a crawl: cache first (unless bypassed), then a
+ * TLS-impersonated fetch. Never throws; failures come back with `error`.
+ * Used by `crawlWebsitePages` and by the durable crawl workflow (one call per page).
+ */
+export declare function crawlPage(pageUrl: string, options?: {
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<CrawledPage>;