@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/ranker/types.d.ts

@@ -0,0 +1,90 @@
+export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "bing-rss" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
+export type DecisionModelId = "baseten/mammoth-1" | "baseten/elephant-1" | "convaiinnovations/laya" | "auto" | "none";
+export interface RawCandidateDocument {
+    title: string;
+    url: string;
+    description: string;
+    markdown?: string;
+    links?: string[];
+    publishedDate?: string;
+    engine?: SearchEngineSource;
+    engineRank?: number;
+    sources?: SearchEngineSource[];
+    engineRanks?: Partial<Record<SearchEngineSource, number>>;
+}
+export interface UserLocationContext {
+    country?: string;
+    countryName?: string;
+    region?: string;
+    city?: string;
+    timezone?: string;
+    latitude?: number;
+    longitude?: number;
+}
+export interface SearchFilterContext {
+    category?: string;
+    subcategory?: string | string[];
+    preferLocalDomains?: boolean;
+    boostTerms?: string[];
+    mustIncludeTerms?: string[];
+    excludeTerms?: string[];
+    customInstruction?: string;
+}
+export interface DocumentScoreBreakdown {
+    rrfScore: number;
+    bm25fScore: number;
+    passageMaxSimScore: number;
+    proximityScore: number;
+    freshnessAuthorityScore: number;
+    geoAffinityScore?: number;
+    lexicalHybridScore: number;
+    decisionScore?: number;
+    decisionRawScore?: number;
+    decisionConfidence?: number;
+    decisionModelUsed?: string;
+    multiEngineConsensusCount: number;
+}
+export interface RankedDocument {
+    title: string;
+    url: string;
+    canonicalUrl: string;
+    description: string;
+    markdown?: string;
+    links?: string[];
+    publishedDate?: string;
+    score: number;
+    rank: number;
+    sources: SearchEngineSource[];
+    highlights: string[];
+    scoreBreakdown: DocumentScoreBreakdown;
+}
+export interface RerankOptions {
+    topK?: number;
+    rrfK?: number;
+    useDecisionModel?: boolean;
+    decisionModel?: DecisionModelId;
+    decisionTimeoutMs?: number;
+    decisionTopN?: number;
+    extractHighlights?: boolean;
+    maxHighlights?: number;
+    /** Maximum results per publisher/domain in topK when diverse candidates exist (default: 1 for topK <= 10, 2 for topK > 10) */
+    maxPerDomain?: number;
+    /** Maximal Marginal Relevance lambda in [0, 1] balancing relevance vs diversity (default: 0.82) */
+    mmrLambda?: number;
+    /** Optional user geolocation context for boosting local/regional sources and ccTLDs */
+    userLocation?: UserLocationContext;
+    /** Optional dynamic category, subcategory, and agent filter context */
+    filterContext?: SearchFilterContext;
+}
+export interface RerankResult {
+    results: RankedDocument[];
+    rankingMetrics: {
+        totalCandidates: number;
+        uniqueCandidates: number;
+        lexicalLatencyMs: number;
+        decisionLatencyMs: number;
+        totalRankingLatencyMs: number;
+        decisionModelUsed: string | null;
+        enginesContributing: SearchEngineSource[];
+    };
+}

package/dist/core/ranker/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/ranker.d.ts

@@ -0,0 +1 @@
+export * from "./ranker/index.js";

package/dist/core/ranker.js

@@ -0,0 +1 @@
+export * from "./ranker/index.js";

package/dist/core/sandbox.d.ts

@@ -0,0 +1,29 @@
+export interface BrowserEscalationResult {
+    html: string;
+    markdown: string;
+    title: string;
+    links: string[];
+    statusCode: number;
+    tier: "sandbox" | "cloudflare" | "browser";
+}
+/** True when an escalation result is still the bot check, not the page. */
+export declare function isChallengeResult(html: string, markdown: string, status?: number): boolean;
+/** True when a browser tier (Cloudflare Browser Rendering or Vercel Sandbox) is configured. */
+export declare function browserEscalationAvailable(): boolean;
+/**
+ * Tier 2/3 Anti-Bot Escalation:
+ * 1. Uses Cloudflare Browser Rendering `/markdown` if CF_ACCOUNT_ID + CF_API_TOKEN are set.
+ * 2. Uses `@vercel/sandbox` isolated Linux microVM (with curl-impersonate / Playwright / Node TLS fetch + proxy)
+ *    to bypass JA3/JA4 TLS blocks, solve JS challenges, and cache `cf_clearance` cookies per domain.
+ */
+export declare function escalateToSandboxOrBrowser(url: string, options?: {
+    retainImages?: "all" | "none";
+    timeoutMs?: number;
+    /**
+     * Last resort: open the page in a real browser and wait out the bot check.
+     * Slow (cold start plus up to 20s solving), so only callers with a large
+     * time budget should opt in.
+     */
+    browser?: boolean;
+    browserBudgetMs?: number;
+}): Promise<BrowserEscalationResult | null>;

package/dist/core/sandbox.js

@@ -0,0 +1,222 @@
+import { fastHtmlToMarkdown, setDomainCookie } from "./engine.js";
+import { isAntiBotChallenge } from "./engine/fetcher.js";
+import { getWarmSession, isDomainChallenged, markDomainChallenged } from "./engine/session.js";
+/** Bot-check copy that survives HTML → markdown conversion (Cloudflare `/markdown`). */
+const CHALLENGE_MARKDOWN = [
+    "just a moment",
+    "vercel security checkpoint",
+    "verify you are human",
+    "checking your browser",
+    "enable javascript and cookies to continue",
+];
+/** True when an escalation result is still the bot check, not the page. */
+export function isChallengeResult(html, markdown, status = 200) {
+    if (html && isAntiBotChallenge(html, status))
+        return true;
+    const head = markdown.slice(0, 1500).toLowerCase();
+    // Short pages only: a long article that mentions "just a moment" is real content.
+    return markdown.length < 1500 && CHALLENGE_MARKDOWN.some((s) => head.includes(s));
+}
+/**
+ * True when Vercel Sandbox can authenticate. On Vercel (VERCEL=1) the OIDC token is
+ * not in env at request time: it arrives in the `x-vercel-oidc-token` header, which
+ * @vercel/sandbox reads via @vercel/oidc. Locally it comes from `vercel env pull`.
+ */
+function vercelSandboxAvailable() {
+    return Boolean(process.env.VERCEL === "1" ||
+        process.env.VERCEL_OIDC_TOKEN ||
+        process.env.VERCEL_SANDBOX_ENABLED === "true");
+}
+/** True when a browser tier (Cloudflare Browser Rendering or Vercel Sandbox) is configured. */
+export function browserEscalationAvailable() {
+    return Boolean((process.env.CF_ACCOUNT_ID && process.env.CF_API_TOKEN) || vercelSandboxAvailable());
+}
+/**
+ * Tier 2/3 Anti-Bot Escalation:
+ * 1. Uses Cloudflare Browser Rendering `/markdown` if CF_ACCOUNT_ID + CF_API_TOKEN are set.
+ * 2. Uses `@vercel/sandbox` isolated Linux microVM (with curl-impersonate / Playwright / Node TLS fetch + proxy)
+ *    to bypass JA3/JA4 TLS blocks, solve JS challenges, and cache `cf_clearance` cookies per domain.
+ */
+export async function escalateToSandboxOrBrowser(url, options = {}) {
+    // Option A: Cloudflare Browser Rendering Quick Action (/markdown) if configured
+    const cfAccountId = process.env.CF_ACCOUNT_ID;
+    const cfApiToken = process.env.CF_API_TOKEN;
+    if (cfAccountId && cfApiToken) {
+        try {
+            const cfRes = await fetch(`https://api.cloudflare.com/client/v4/accounts/${cfAccountId}/browser-rendering/markdown`, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${cfApiToken}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    url,
+                    gotoOptions: { waitUntil: "domcontentloaded", timeout: options.timeoutMs ?? 15000 },
+                    rejectRequestPattern: [
+                        "^.*\\.(png|jpg|jpeg|gif|webp|svg|ico|woff2?|ttf|mp4|mp3|css)(\\?.*)?$",
+                        "^.*(googletagmanager|google-analytics|intercom|segment|hotjar|sentry).*$",
+                    ],
+                }),
+                signal: AbortSignal.timeout(options.timeoutMs ?? 15000),
+            });
+            if (cfRes.ok) {
+                const payload = (await cfRes.json());
+                if (payload.success && payload.result && !isChallengeResult("", payload.result)) {
+                    const titleMatch = payload.result.match(/^#\s+([^\n]+)/m);
+                    return {
+                        html: "",
+                        markdown: payload.result,
+                        title: titleMatch ? titleMatch[1].trim() : url,
+                        links: [],
+                        statusCode: 200,
+                        tier: "cloudflare",
+                    };
+                }
+            }
+        }
+        catch {
+            // Fall through to Vercel Sandbox
+        }
+    }
+    // Option B: Vercel Sandbox microVM execution (`@vercel/sandbox`)
+    if (vercelSandboxAvailable()) {
+        // Challenge memo: curl can never solve a JS challenge, so for domains known
+        // to serve one (or with a warm solved session) skip this tier entirely and
+        // save its ~4-6 s sandbox boot.
+        let host = "";
+        try {
+            host = new URL(url).hostname;
+        }
+        catch {
+            /* fall through to curl */
+        }
+        const skipCurlTier = host
+            ? (await isDomainChallenged(host).catch(() => false)) || Boolean(await getWarmSession(host).catch(() => null))
+            : false;
+        if (!skipCurlTier) {
+            try {
+                const { Sandbox } = await import("@vercel/sandbox");
+                const sandbox = await Sandbox.create({
+                    timeout: options.timeoutMs ?? 20_000,
+                });
+                try {
+                    // Execute curl with browser cipher list + cookie capture inside the clean Linux microVM egress IP
+                    const proxyArg = process.env.PROXY_URL ? `-x "${process.env.PROXY_URL}"` : "";
+                    const cmd = await sandbox.runCommand({
+                        cmd: "curl",
+                        args: [
+                            "-sSL",
+                            "--compressed",
+                            "--max-time",
+                            "12",
+                            "-D",
+                            "/tmp/headers.txt",
+                            ...(proxyArg ? ["-x", process.env.PROXY_URL] : []),
+                            "-H",
+                            "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+                            "-H",
+                            "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                            "-H",
+                            "Accept-Language: en-US,en;q=0.9",
+                            url,
+                        ],
+                    });
+                    const html = await cmd.stdout();
+                    try {
+                        const headersCmd = await sandbox.runCommand({
+                            cmd: "cat",
+                            args: ["/tmp/headers.txt"],
+                        });
+                        const headersText = await headersCmd.stdout();
+                        const setCookies = Array.from(headersText.matchAll(/^set-cookie:\s*([^;\r\n]+)/gim)).map((m) => m[1]);
+                        if (setCookies.length > 0) {
+                            setDomainCookie(new URL(url).hostname, setCookies.join("; "));
+                        }
+                    }
+                    catch {
+                        // ignore cookie extraction failure
+                    }
+                    // A challenge page is not content: never return it as a successful escalation.
+                    // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
+                    // parse first and fall through to the browser tier instead.
+                    if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
+                        const parsed = fastHtmlToMarkdown(html, url, {
+                            retainImages: options.retainImages,
+                        });
+                        if (parsed.markdown.trim().length > 0) {
+                            return {
+                                html,
+                                markdown: parsed.markdown,
+                                title: parsed.title,
+                                links: parsed.links,
+                                statusCode: 200,
+                                tier: "sandbox",
+                            };
+                        }
+                    }
+                    // The clean-VM curl could not produce content: this domain serves a JS
+                    // challenge (or similar), so memo it and skip this tier next time.
+                    if (host)
+                        await markDomainChallenged(host).catch(() => { });
+                }
+                finally {
+                    await sandbox.stop().catch(() => { });
+                }
+            }
+            catch {
+                // Sandbox unavailable or not configured in this environment
+            }
+        }
+    }
+    // Option C: real browser that waits out the bot check (opt-in, slow).
+    if (options.browser) {
+        const viaBrowser = await escalateViaStealthBrowser(url, options);
+        if (viaBrowser)
+            return viaBrowser;
+    }
+    return null;
+}
+async function escalateViaStealthBrowser(url, options) {
+    try {
+        const { stealthAvailable, stealthFetch } = await import("./engine/stealth.js");
+        if (!(await stealthAvailable()))
+            return null;
+        // Tier P2: when the domain is memoized as needing a residential egress,
+        // the stealth browser itself egresses through the proxy (resource-blocked,
+        // byte-capped inside stealthFetch) instead of a doomed datacenter IP.
+        let proxy;
+        try {
+            const { residentialProxyUrl } = await import("./engine/fetcher.js");
+            const { isDomainProxyNeeded } = await import("./engine/session.js");
+            let host = "";
+            try {
+                host = new URL(url).hostname;
+            }
+            catch { /* bad url handled below */ }
+            if (host && (await isDomainProxyNeeded(host)))
+                proxy = residentialProxyUrl() ?? undefined;
+        }
+        catch { /* proxy memo unavailable — datacenter stealth as before */ }
+        const res = await stealthFetch(url, [], { budgetMs: options.browserBudgetMs ?? 30_000, ...(proxy ? { proxy } : {}) });
+        const entry = res.entry;
+        if (!entry?.body || entry.blocked || isAntiBotChallenge(entry.body, entry.status ?? 200)) {
+            return null;
+        }
+        const parsed = fastHtmlToMarkdown(entry.body, entry.finalUrl || entry.url || url, {
+            retainImages: options.retainImages,
+        });
+        if (!parsed.markdown.trim())
+            return null;
+        return {
+            html: entry.body,
+            markdown: parsed.markdown,
+            title: entry.title || parsed.title,
+            links: parsed.links,
+            statusCode: entry.status ?? 200,
+            tier: "browser",
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/search/catalog.d.ts

@@ -0,0 +1,176 @@
+import { type AggregatedProduct, type BusinessListing, type CveRecord, type FlightOffer, type HotelListing, type ProductOffer, type PropertyListing } from "../extract/index.js";
+import type { DecisionModelId, DocumentScoreBreakdown, SearchEngineSource, UserLocationContext } from "../ranker/index.js";
+export type { AggregatedProduct, BusinessListing, CveRecord, FlightOffer, HotelListing, ProductOffer, PropertyListing, } from "../extract/index.js";
+export interface SearchAgentFilters {
+    /** Restrict to these engines (overrides the category's default engine plan) */
+    engines?: SearchEngineSource[];
+    excludeEngines?: SearchEngineSource[];
+    /** Prefer domains on the user's country-code TLD and demote other country TLDs */
+    preferLocalDomains?: boolean;
+    /** How to localize the upstream query: add city, country, region, or nothing. Default "auto" = per category policy */
+    localizeBy?: "auto" | "city" | "country" | "region" | "none";
+    mustIncludeTerms?: string[];
+    excludeTerms?: string[];
+    boostTerms?: string[];
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    /** Target specific retailer domains or URLs (e.g. ["jbhifi.com.au", "amazon.com.au"]) */
+    sites?: string[];
+    /** Enable structured product/offer extraction & grouping (default: true when category === "shopping") */
+    productMode?: boolean;
+    /** Allow headless browser / sandbox escalation on bot-gated candidate product pages (default: false) */
+    allowBrowserEscalation?: boolean;
+    /** Free-text ranking instruction passed to the decision model */
+    customInstruction?: string;
+    /** When true, mustIncludeTerms / excludeTerms hard-drop candidates instead of only lowering scores */
+    strict?: boolean;
+}
+export declare const SEARCH_PROVIDER_CHOICES: readonly ["auto", "ddg", "jina", "serper", "brave"];
+export type SearchProviderChoice = (typeof SEARCH_PROVIDER_CHOICES)[number];
+/** Providers that bill an extra credit when they serve the search. */
+export declare const PAID_SEARCH_PROVIDERS: ReadonlySet<string>;
+export declare const SEARCH_PROVIDER_ENV_KEYS: Record<string, string>;
+export interface SearchRequest {
+    query: string;
+    limit?: number;
+    tbs?: string;
+    lang?: string;
+    country?: string;
+    region?: string;
+    city?: string;
+    timezone?: string;
+    location?: string | UserLocationContext;
+    /** Any category string; see GET /v1/search/categories for the built-in catalog */
+    category?: string;
+    /** Any subcategory string(s) within the category */
+    subcategory?: string | string[];
+    /** Legacy multi-category list; first entry is used when `category` is unset */
+    categories?: string[];
+    filters?: SearchAgentFilters;
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    bypassCache?: boolean;
+    /**
+     * Pick one search provider and skip the default multi-engine pipeline.
+     * `auto` (default) uses the free engines and falls back to Jina when results are thin.
+     */
+    provider?: SearchProviderChoice;
+    rankOptions?: {
+        useDecisionModel?: boolean;
+        decisionModel?: DecisionModelId;
+        decisionTimeoutMs?: number;
+        quorumTimeoutMs?: number;
+        maxPerDomain?: number;
+    };
+    scrapeOptions?: {
+        formats?: Array<"markdown" | "links" | "html">;
+        retainImages?: "all" | "none";
+        scrapeTimeoutMs?: number;
+    };
+}
+export interface SearchResultItem {
+    title: string;
+    url: string;
+    canonicalUrl?: string;
+    description: string;
+    markdown?: string;
+    links?: string[];
+    publishedDate?: string;
+    score?: number;
+    rank?: number;
+    sources?: SearchEngineSource[];
+    highlights?: string[];
+    scoreBreakdown?: DocumentScoreBreakdown;
+    offer?: ProductOffer;
+    listings?: PropertyListing[];
+    flights?: FlightOffer[];
+    hotels?: HotelListing[];
+    businesses?: BusinessListing[];
+    /** CVE records this page is the record for, or mentions (category === "security") */
+    cves?: CveRecord[];
+}
+export interface SearchResponse {
+    success: boolean;
+    data: SearchResultItem[];
+    /** Structured product clusters with normalized multi-vendor offers and lowest-price `bestOffer` (populated by default when category === "shopping" or filters.productMode === true) */
+    products?: AggregatedProduct[];
+    /** Flat list of all extracted product offers across engines and candidate pages */
+    offers?: ProductOffer[];
+    /** Structured real-estate property listings with address, price, beds/baths/parking, propertyType, portal, and agency (populated when category === "real-estate") */
+    listings?: PropertyListing[];
+    /** Structured flight offers with airline, origin, destination, duration, stops, cabinClass, and fare price (populated when category === "flights") */
+    flights?: FlightOffer[];
+    /** Structured hotel and accommodation listings with nightly rate, star/guest ratings, address, and amenities (populated when category === "hotels") */
+    hotels?: HotelListing[];
+    /** Structured business and company listings with category, address, phone, rating, hours, and website (populated when category === "business") */
+    businesses?: BusinessListing[];
+    /** Structured CVE records from NVD and OSV: CVSS score and severity, CWE, affected products and versions, fixed versions, CISA KEV flag, dates and references (populated when category === "security") */
+    cves?: CveRecord[];
+    provider: "native-federated" | "brave" | "serper" | "jina" | "ddg-edge";
+    /** Set when a keyless search returns no hits (engines blocked or empty). */
+    error?: string;
+    /** Set when the primary engines came back thin and a paid fallback engine filled in. */
+    fallback?: "jina";
+    cached?: boolean;
+    latencyMs: number;
+    userLocation?: UserLocationContext;
+    rankingMetrics?: {
+        totalCandidates: number;
+        uniqueCandidates: number;
+        lexicalLatencyMs: number;
+        decisionLatencyMs: number;
+        totalRankingLatencyMs: number;
+        decisionModelUsed: string | null;
+        enginesContributing: SearchEngineSource[];
+    };
+}
+export interface CategoryDefinition {
+    description: string;
+    engines: SearchEngineSource[];
+    /** Where to localize the upstream query by default */
+    localizeBy: "city" | "country" | "none";
+    preferLocalDomains: boolean;
+    subcategories: Record<string, {
+        description: string;
+    }>;
+}
+/** Declarative category catalog. Categories/subcategories passed by agents are free-form; this is only the default engine plan. */
+export declare const SEARCH_CATEGORY_CATALOG: Record<string, CategoryDefinition>;
+export declare function resolveCategory(req: SearchRequest): string;
+export declare function planEngines(category: string, filters?: SearchAgentFilters): SearchEngineSource[];
+/** DuckDuckGo region param (`kl`) derived from ISO-3166 country + language. */
+export declare function ddgRegion(country?: string, lang?: string): string | undefined;
+export declare function getSearchCategoriesCatalog(): {
+    categories: {
+        subcategories: {
+            description: string;
+            id: string;
+        }[];
+        description: string;
+        engines: SearchEngineSource[];
+        /** Where to localize the upstream query by default */
+        localizeBy: "city" | "country" | "none";
+        preferLocalDomains: boolean;
+        id: string;
+    }[];
+    notes: string;
+    filters: {
+        engines: string;
+        excludeEngines: string;
+        preferLocalDomains: string;
+        localizeBy: string;
+        mustIncludeTerms: string;
+        excludeTerms: string;
+        boostTerms: string;
+        includeDomains: string;
+        excludeDomains: string;
+        sites: string;
+        productMode: string;
+        allowBrowserEscalation: string;
+        customInstruction: string;
+        strict: string;
+    };
+};
+export declare function normalizeHostList(domainsOrUrls?: string[]): string[];
+export declare function hostMatchesAny(urlOrHost: string, allowedHosts: string[]): boolean;
+export declare function resolveUserLocation(req: SearchRequest): UserLocationContext;