@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/engine/safe-url.js

@@ -0,0 +1,109 @@
+import { isIP } from "node:net";
+import { lookup } from "node:dns/promises";
+export class UnsafeUrlError extends Error {
+    constructor(message = "That address isn't publicly reachable") {
+        super(message);
+        this.name = "UnsafeUrlError";
+    }
+}
+export function isPrivateAddress(ip) {
+    const v = isIP(ip);
+    if (v === 4) {
+        const [a, b] = ip.split(".").map(Number);
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && b === 168) ||
+            (a === 198 && (b === 18 || b === 19)) ||
+            a >= 224);
+    }
+    if (v === 6) {
+        const s = ip.toLowerCase();
+        if (s.startsWith("::ffff:"))
+            return isPrivateAddress(s.slice(7));
+        return (s === "::" ||
+            s === "::1" ||
+            s.startsWith("fc") ||
+            s.startsWith("fd") ||
+            s.startsWith("fe8") ||
+            s.startsWith("fe9") ||
+            s.startsWith("fea") ||
+            s.startsWith("feb"));
+    }
+    return true;
+}
+/**
+ * True for 198.18.0.0/15, the benchmarking range that fake-IP proxies
+ * (Clash, Surge, sing-box, some VPN clients) hand out for every hostname.
+ */
+export function isFakeIpAddress(ip) {
+    const s = ip.toLowerCase().replace(/^::ffff:/, "");
+    if (isIP(s) !== 4)
+        return false;
+    const [a, b] = s.split(".").map(Number);
+    return a === 198 && (b === 18 || b === 19);
+}
+export const FAKE_IP_HINT = "That address isn't publicly reachable: it resolved to 198.18.x.x, which fake-IP proxies " +
+    "(Clash, Surge, sing-box, some VPNs) return for every site. Switch the proxy to redir-host/real-IP " +
+    "DNS mode, or set WEBCRAWL_ALLOW_PRIVATE_URLS=true for local runs you trust.";
+const defaultResolve = async (h) => (await lookup(h, { all: true })).map((r) => r.address);
+/** Local development escape hatch (never set in production). */
+function allowPrivate() {
+    return process.env.WEBCRAWL_ALLOW_PRIVATE_URLS === "true";
+}
+/**
+ * Throws `UnsafeUrlError` unless `raw` is an http(s) URL whose host resolves
+ * only to public addresses. Call it before every request and every redirect hop.
+ */
+export async function assertSafeUrl(raw, resolve = defaultResolve) {
+    let u;
+    try {
+        u = new URL(raw);
+    }
+    catch {
+        throw new UnsafeUrlError("Invalid URL");
+    }
+    if (u.protocol !== "http:" && u.protocol !== "https:") {
+        throw new UnsafeUrlError("Only http and https addresses are supported");
+    }
+    if (allowPrivate())
+        return u;
+    const host = u.hostname.replace(/^\[|\]$/g, "").toLowerCase();
+    if (host === "localhost" ||
+        host.endsWith(".localhost") ||
+        host.endsWith(".local") ||
+        host.endsWith(".internal")) {
+        throw new UnsafeUrlError();
+    }
+    const addrs = isIP(host) ? [host] : await resolve(host).catch(() => []);
+    if (addrs.length === 0)
+        throw new UnsafeUrlError("We couldn't find that website");
+    if (addrs.some(isPrivateAddress)) {
+        // Every address in the fake-IP range means a local proxy rewrote DNS,
+        // not that the site is internal: say so and name the workaround.
+        throw new UnsafeUrlError(addrs.every(isFakeIpAddress) ? FAKE_IP_HINT : undefined);
+    }
+    return u;
+}
+export const MAX_REDIRECTS = 8;
+/**
+ * Runs `send` for `url` and follows redirects manually, validating every hop
+ * with `assertSafeUrl`. `send` must NOT follow redirects itself.
+ */
+export async function followSafely(url, send, resolve) {
+    let current = url;
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+        await assertSafeUrl(current, resolve);
+        const res = await send(current);
+        const loc = res.headers["location"];
+        if (res.status >= 300 && res.status < 400 && loc) {
+            current = new URL(loc, current).toString();
+            continue;
+        }
+        return { res, finalUrl: current };
+    }
+    throw new UnsafeUrlError("Too many redirects");
+}

package/dist/core/engine/session.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** A solved stealth session kept for reuse. */
+export interface WarmSession {
+    domain: string;
+    sandboxId: string;
+    userAgent: string;
+    /** `Cookie:` header value for the solved session (UA-paired: replay both or neither). */
+    cookieHeader: string;
+    /** Epoch ms after which the entry is stale. */
+    expiresAt: number;
+}
+export interface ExternalSessionStore {
+    getWarmSession(domain: string): Promise<WarmSession | null>;
+    saveWarmSession(session: WarmSession): Promise<void>;
+    deleteWarmSession(domain: string): Promise<void>;
+    markDomainChallenged(domain: string): Promise<void>;
+    isDomainChallenged(domain: string): Promise<boolean>;
+    /** Optional persistence for the residential-proxy memo (Tier P1). */
+    markDomainProxyNeeded?(domain: string): Promise<void>;
+    isDomainProxyNeeded?(domain: string): Promise<boolean>;
+    /** Optional persistence for the render-provider memo (Tier R). */
+    markDomainRenderNeeded?(domain: string): Promise<void>;
+    isDomainRenderNeeded?(domain: string): Promise<boolean>;
+}
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export declare const WARM_TTL_MS: number;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export declare const WARM_SANDBOX_TIMEOUT_MS: number;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export declare const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export declare const CHALLENGE_MEMO_TTL_MS: number;
+/** How long a domain stays on the residential-proxy memo (Tier P1). */
+export declare const PROXY_MEMO_TTL_MS: number;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export declare const RENDER_MEMO_TTL_MS: number;
+export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
+export interface CookieLike {
+    name: string;
+    value: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export declare function cookieHeaderFrom(cookies: CookieLike[]): string;
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export declare function getWarmSession(domain: string): Promise<WarmSession | null>;
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export declare function saveWarmSession(session: WarmSession): Promise<string[]>;
+export declare function deleteWarmSession(domain: string): Promise<void>;
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export declare function markDomainChallenged(domain: string): Promise<void>;
+export declare function isDomainChallenged(domain: string): Promise<boolean>;
+/**
+ * Marks a domain as needing the residential-proxy tier (Tier P1): every
+ * datacenter egress attempt is blocked by IP reputation (DataDome et al), so
+ * future ladder runs skip straight to the proxy hop.
+ */
+export declare function markDomainProxyNeeded(domain: string): Promise<void>;
+export declare function isDomainProxyNeeded(domain: string): Promise<boolean>;
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export declare function markDomainRenderNeeded(domain: string): Promise<void>;
+export declare function isDomainRenderNeeded(domain: string): Promise<boolean>;
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -0,0 +1,250 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export const WARM_TTL_MS = 8 * 60_000;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export const WARM_SANDBOX_TIMEOUT_MS = 10 * 60_000;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
+/** How long a domain stays on the residential-proxy memo (Tier P1). */
+export const PROXY_MEMO_TTL_MS = 60 * 60_000;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export const RENDER_MEMO_TTL_MS = 12 * 60 * 60_000;
+let external = null;
+export function registerExternalSessionStore(store) {
+    external = store;
+}
+const warm = new Map();
+const challenged = new Map();
+const proxyNeeded = new Map();
+const renderNeeded = new Map();
+function reap(now = Date.now()) {
+    for (const [d, s] of warm)
+        if (s.expiresAt <= now)
+            warm.delete(d);
+    for (const [d, t] of challenged)
+        if (t <= now)
+            challenged.delete(d);
+    for (const [d, t] of proxyNeeded)
+        if (t <= now)
+            proxyNeeded.delete(d);
+    for (const [d, t] of renderNeeded)
+        if (t <= now)
+            renderNeeded.delete(d);
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export function cookieHeaderFrom(cookies) {
+    const now = Date.now() / 1000;
+    return cookies
+        .filter((c) => c.name && c.value && (!c.expires || c.expires > now))
+        .map((c) => `${c.name}=${c.value}`)
+        .join("; ");
+}
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export async function getWarmSession(domain) {
+    if (!domain)
+        return null;
+    reap();
+    const local = warm.get(domain);
+    if (local && local.expiresAt > Date.now())
+        return local;
+    if (local)
+        warm.delete(domain);
+    if (external) {
+        try {
+            const stored = await external.getWarmSession(domain);
+            if (stored && stored.expiresAt > Date.now()) {
+                warm.set(domain, stored);
+                return stored;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return null;
+}
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export async function saveWarmSession(session) {
+    if (!session.domain)
+        return [];
+    reap();
+    warm.delete(session.domain);
+    warm.set(session.domain, { ...session });
+    const evicted = [];
+    while (warm.size > WARM_CAP) {
+        const [d, s] = warm.entries().next().value;
+        warm.delete(d);
+        if (s.sandboxId && s.sandboxId !== session.sandboxId)
+            evicted.push(s.sandboxId);
+    }
+    if (external) {
+        try {
+            await external.saveWarmSession(session);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return evicted;
+}
+export async function deleteWarmSession(domain) {
+    if (!domain)
+        return;
+    warm.delete(domain);
+    if (external) {
+        try {
+            await external.deleteWarmSession(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export async function markDomainChallenged(domain) {
+    if (!domain)
+        return;
+    challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+    if (external) {
+        try {
+            await external.markDomainChallenged(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainChallenged(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = challenged.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        challenged.delete(domain);
+    if (external) {
+        try {
+            if (await external.isDomainChallenged(domain)) {
+                challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
+/**
+ * Marks a domain as needing the residential-proxy tier (Tier P1): every
+ * datacenter egress attempt is blocked by IP reputation (DataDome et al), so
+ * future ladder runs skip straight to the proxy hop.
+ */
+export async function markDomainProxyNeeded(domain) {
+    if (!domain)
+        return;
+    proxyNeeded.set(domain, Date.now() + PROXY_MEMO_TTL_MS);
+    if (external?.markDomainProxyNeeded) {
+        try {
+            await external.markDomainProxyNeeded(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainProxyNeeded(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = proxyNeeded.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        proxyNeeded.delete(domain);
+    if (external?.isDomainProxyNeeded) {
+        try {
+            if (await external.isDomainProxyNeeded(domain)) {
+                proxyNeeded.set(domain, Date.now() + PROXY_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export async function markDomainRenderNeeded(domain) {
+    if (!domain)
+        return;
+    renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+    if (external?.markDomainRenderNeeded) {
+        try {
+            await external.markDomainRenderNeeded(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainRenderNeeded(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = renderNeeded.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        renderNeeded.delete(domain);
+    if (external?.isDomainRenderNeeded) {
+        try {
+            if (await external.isDomainRenderNeeded(domain)) {
+                renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export function resetSessionStateForTests() {
+    warm.clear();
+    challenged.clear();
+    proxyNeeded.clear();
+    renderNeeded.clear();
+}

package/dist/core/engine/stealth.d.ts

@@ -0,0 +1,83 @@
+import { type RemoteProvider } from "../interact/remote.js";
+export interface StealthDoc {
+    url: string;
+    ok: boolean;
+    status?: number | null;
+    contentType?: string;
+    body?: string;
+    finalUrl?: string;
+    title?: string;
+    blocked?: boolean;
+    error?: string;
+}
+export interface StealthResult {
+    /** True when the entry page passed the bot check. */
+    solved: boolean;
+    entry: StealthDoc | null;
+    docs: StealthDoc[];
+    durationMs: number;
+    error?: string;
+    /** Set when a remote provider (e.g. Browser Use) hosted the browser. */
+    provider?: RemoteProvider;
+    /** Provider charge for the session in USD, when it reported one. */
+    providerCostUsd?: number;
+}
+export interface StealthItem {
+    url: string;
+    /** Read as a raw document (robots.txt, sitemap XML) instead of rendering it. */
+    raw?: boolean;
+}
+/**
+ * True when a real browser can run here: our own (sandbox in prod, local
+ * Playwright in dev) or a remote provider. A remote browser still needs the
+ * sandbox or local Playwright to run the step script against it.
+ */
+export declare function stealthAvailable(): Promise<boolean>;
+/** True when this fetched document is a bot check, not the real content. */
+export declare function isBlockedResponse(status: number | null | undefined, body: string): boolean;
+/**
+ * Opens `entryUrl` in a real browser, waits for the bot check to clear, then
+ * reads `items` in order through that session. One browser per call.
+ */
+export declare function stealthFetch(entryUrl: string, items: StealthItem[], opts?: {
+    budgetMs?: number;
+    solveMs?: number;
+    followSitemaps?: number;
+    /** Allow the paid remote provider. Off for unbilled (anonymous) runs. Default true. */
+    remote?: boolean;
+    /** Keep the solved session warm (sandbox left running + cookies saved) for repeat queries. */
+    keepWarm?: boolean;
+    /** Tier P2: egress the stealth browser through this residential proxy URL. */
+    proxy?: string;
+}): Promise<StealthResult>;
+/** Extracts `<loc>` entries from a sitemap or sitemap index. */
+export declare function parseSitemapLocs(xml: string): {
+    pages: string[];
+    sitemaps: string[];
+};
+/** Sitemap URLs declared in robots.txt. */
+export declare function sitemapsFromRobots(robots: string): string[];
+/**
+ * Finds a blocked site's pages through one stealth session: robots.txt and the
+ * usual sitemap paths, following robots-declared and child sitemaps in the same
+ * session. Also returns the rendered entry page so its links count when there
+ * is no sitemap.
+ */
+export declare function stealthDiscover(entryUrl: string, opts?: {
+    budgetMs?: number;
+    sitemapUrls?: string[];
+    remote?: boolean;
+}): Promise<{
+    solved: boolean;
+    sitemapPages: string[];
+    sitemapsRead: string[];
+    entry: StealthDoc | null;
+    durationMs: number;
+    error?: string;
+    providerCostUsd?: number;
+}>;
+/** Page URLs from every readable sitemap among `docs` (robots.txt and blocked docs are skipped). */
+export declare function collectSitemapPages(docs: StealthDoc[]): {
+    pages: string[];
+    sitemapsRead: string[];
+};