@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/targets/registry.js

@@ -0,0 +1,411 @@
+/**
+ * Unified Target & Extractor Registry for Webcrawl Anything.
+ *
+ * Wraps every built-in vertical extractor (`src/extract/*`), binary/media parser (`src/parse/*`),
+ * multi-tier web crawler (`src/engine/*`), and developer target (`src/targets/developer.ts`)
+ * behind a single discoverable interface with REA-style Evidence & Provenance.
+ */
+import crypto from "node:crypto";
+import { extractBusinessListingsFromHtml, extractCveIdsFromText, extractFlightOffersFromHtml, extractHotelListingsFromHtml, extractOffersFromHtml, extractPropertyListingsFromHtml, groupOffersIntoProducts, } from "../extract/index.js";
+import { fastHtmlToMarkdown } from "../engine/markdown.js";
+import { BINARY_KINDS, detectFileKind } from "../parse/detect.js";
+import { youtubeVideoId } from "../parse/youtube.js";
+import { arxivTargetPlugin, githubTargetPlugin, llmsTxtTargetPlugin, } from "./developer.js";
+function sha256(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+export const webSpaTargetPlugin = {
+    id: "web-spa",
+    name: "Websites, SPAs & Anti-Bot Pages",
+    category: "web",
+    returns: "Clean Markdown, title, outbound links, JSON-LD / OpenGraph metadata, and tier provenance",
+    requirements: "Built-in impit TLS impersonation; optional local Chrome/Playwright or webcrawl.app cloud stealth for Cloudflare/DataDome",
+    examples: [
+        "npx webcrawl scrape https://news.ycombinator.com",
+        "npx webcrawl crawl https://docs.stripe.com -n 10",
+    ],
+    matches({ url }) {
+        if (url && /^https?:\/\//i.test(url)) {
+            return { matched: true, confidence: 0.5, reason: "Generic HTTP/HTTPS web URL" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const { title, markdown, links } = fastHtmlToMarkdown(html, url);
+        const matchedSignals = ["html:fast-markdown"];
+        const limitations = [];
+        if (!markdown.trim()) {
+            limitations.push("Static HTML contained no readable body text; enable browser/stealth escalation for client-rendered SPA");
+        }
+        return {
+            targetId: "web-spa",
+            targetName: "Websites, SPAs & Anti-Bot Pages",
+            category: "web",
+            title,
+            markdown,
+            structured: { title, linkCount: links.length, links: links.slice(0, 50) },
+            evidence: {
+                targetId: "web-spa",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals,
+                limitations,
+            },
+        };
+    },
+};
+export const pdfOfficeTargetPlugin = {
+    id: "pdf-office",
+    name: "PDFs & Office Documents (DOCX, XLSX, PPTX)",
+    category: "documents",
+    returns: "Page-numbered Markdown, tables, native text extraction, automatic OCR on scanned pages, and skipped-page diagnostics",
+    requirements: "Built-in @firecrawl/pdf-inspector, mammoth, and officeparser; optional OCR provider for scanned pages",
+    examples: [
+        "npx webcrawl scrape https://arxiv.org/pdf/1706.03762.pdf",
+    ],
+    matches({ url, contentType }) {
+        if (url || contentType) {
+            const { kind } = detectFileKind({ contentType, fileName: url });
+            if (kind === "pdf" || kind === "docx" || kind === "xlsx" || kind === "pptx") {
+                return { matched: true, confidence: 0.99, reason: `Detected ${kind.toUpperCase()} document` };
+            }
+        }
+        return { matched: false, confidence: 0 };
+    },
+};
+export const youtubeMediaTargetPlugin = {
+    id: "youtube-media",
+    name: "YouTube Videos, Audio & Media Files",
+    category: "media",
+    returns: "Timestamped transcript Markdown, video title/channel/duration metadata, and spoken audio transcription",
+    requirements: "Built-in zero-key YouTube multi-client caption extractor; optional audio transcription provider for raw MP3/MP4",
+    examples: [
+        "npx webcrawl scrape https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+    ],
+    matches({ url, contentType }) {
+        if (url && youtubeVideoId(url)) {
+            return { matched: true, confidence: 0.99, reason: "YouTube video URL" };
+        }
+        if (url || contentType) {
+            const { kind } = detectFileKind({ contentType, fileName: url });
+            if (kind === "audio" || kind === "video" || (BINARY_KINDS.has(kind) && kind === "image")) {
+                return { matched: true, confidence: 0.95, reason: `Detected ${kind} media resource` };
+            }
+        }
+        return { matched: false, confidence: 0 };
+    },
+};
+export const shoppingTargetPlugin = {
+    id: "shopping",
+    name: "E-Commerce Products & Multi-Retailer Offers",
+    category: "commerce",
+    returns: "Normalized prices, currency, stock availability, seller ratings, free-shipping flags, and grouped product comparisons",
+    requirements: "Built-in JSON-LD Product/Offer, OpenGraph, and SERP shopping card extractors",
+    examples: [
+        "npx webcrawl search \"Sony WH-1000XM5 headphones\" -c shopping",
+    ],
+    matches({ category, query, url }) {
+        if (category === "shopping") {
+            return { matched: true, confidence: 0.99, reason: "Shopping search category" };
+        }
+        if (url && /\b(amazon\.|ebay\.|walmart\.|bestbuy\.|jbhifi\.|shopify)/i.test(url)) {
+            return { matched: true, confidence: 0.85, reason: "Retailer product domain" };
+        }
+        if (query && /\b(buy|price|cheapest|deal|in stock|review)\b/i.test(query)) {
+            return { matched: true, confidence: 0.65, reason: "Commercial shopping query intent" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const offers = extractOffersFromHtml(html, url, input.countryCode);
+        const products = groupOffersIntoProducts(offers, input.query ?? "");
+        const matchedSignals = offers.map((o) => `offer:${o.source}`);
+        const limitations = offers.length === 0 ? ["No JSON-LD Product/Offer or OpenGraph price tags found on page"] : [];
+        return {
+            targetId: "shopping",
+            targetName: "E-Commerce Products & Multi-Retailer Offers",
+            category: "commerce",
+            structured: { offers, products },
+            evidence: {
+                targetId: "shopping",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals: Array.from(new Set(matchedSignals)),
+                limitations,
+            },
+        };
+    },
+};
+export const flightsTargetPlugin = {
+    id: "flights",
+    name: "Airline Routes, Fares & Schedules",
+    category: "travel",
+    returns: "Origin/destination IATA codes, airline, price, cabin class, stops, duration, and trip type",
+    requirements: "Built-in IATA route parser, JSON-LD FlightReservation, and fare table extractor",
+    examples: [
+        "npx webcrawl search \"flights SYD to SFO non-stop\" -c flights",
+    ],
+    matches({ category, query }) {
+        if (category === "flights") {
+            return { matched: true, confidence: 0.99, reason: "Flights search category" };
+        }
+        if (query && /\b(flights?|airfare|non-stop|round-trip|[A-Z]{3}\s+to\s+[A-Z]{3})\b/i.test(query)) {
+            return { matched: true, confidence: 0.8, reason: "Flight route query intent" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const flights = extractFlightOffersFromHtml(html, url, input.countryCode);
+        return {
+            targetId: "flights",
+            targetName: "Airline Routes, Fares & Schedules",
+            category: "travel",
+            structured: { flights },
+            evidence: {
+                targetId: "flights",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals: flights.length > 0 ? ["dom:flight-offers"] : [],
+                limitations: flights.length === 0 ? ["No structured flight fares found in static HTML"] : [],
+            },
+        };
+    },
+};
+export const hotelsTargetPlugin = {
+    id: "hotels",
+    name: "Hotels, Stays & Nightly Rates",
+    category: "travel",
+    returns: "Hotel name, nightly rate, star rating, guest review score, neighbourhood, and amenities",
+    requirements: "Built-in LodgingBusiness / Hotel JSON-LD and booking card extractor",
+    examples: [
+        "npx webcrawl search \"boutique hotels in Tokyo Shinjuku\" -c hotels",
+    ],
+    matches({ category, query, url }) {
+        if (category === "hotels") {
+            return { matched: true, confidence: 0.99, reason: "Hotels search category" };
+        }
+        if (url && /\b(booking\.com|expedia\.|hotels\.com|airbnb\.)/i.test(url)) {
+            return { matched: true, confidence: 0.85, reason: "Hospitality domain" };
+        }
+        if (query && /\b(hotels?|resort|nightly rate|check-in|accommodation)\b/i.test(query)) {
+            return { matched: true, confidence: 0.75, reason: "Hotel search query intent" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const hotels = extractHotelListingsFromHtml(html, url, input.countryCode);
+        return {
+            targetId: "hotels",
+            targetName: "Hotels, Stays & Nightly Rates",
+            category: "travel",
+            structured: { hotels },
+            evidence: {
+                targetId: "hotels",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals: hotels.length > 0 ? ["dom:hotel-listings"] : [],
+                limitations: hotels.length === 0 ? ["No structured hotel listings found in static HTML"] : [],
+            },
+        };
+    },
+};
+export const realEstateTargetPlugin = {
+    id: "real-estate",
+    name: "Real Estate & Property Listings",
+    category: "real-estate",
+    returns: "Property address, price/rent period, bedrooms, bathrooms, parking, land size, and listing type (sale/rent/sold)",
+    requirements: "Built-in RealEstateListing / Residence JSON-LD and property portal DOM extractor",
+    examples: [
+        "npx webcrawl search \"3 bedroom house in Austin TX\" -c real-estate --sub buy",
+    ],
+    matches({ category, query, url }) {
+        if (category === "real-estate") {
+            return { matched: true, confidence: 0.99, reason: "Real estate search category" };
+        }
+        if (url && /\b(zillow\.com|realtor\.com|redfin\.com|domain\.com\.au|realestate\.com\.au|rightmove\.co\.uk)/i.test(url)) {
+            return { matched: true, confidence: 0.9, reason: "Real estate portal domain" };
+        }
+        if (query && /\b(for sale|for rent|bedrooms?| townhouses?|apartments?)\b/i.test(query)) {
+            return { matched: true, confidence: 0.7, reason: "Real estate query intent" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const listings = extractPropertyListingsFromHtml(html, url, input.countryCode);
+        return {
+            targetId: "real-estate",
+            targetName: "Real Estate & Property Listings",
+            category: "real-estate",
+            structured: { listings },
+            evidence: {
+                targetId: "real-estate",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals: listings.length > 0 ? ["dom:property-listings"] : [],
+                limitations: listings.length === 0 ? ["No structured property cards found in static HTML"] : [],
+            },
+        };
+    },
+};
+export const localBusinessTargetPlugin = {
+    id: "local-business",
+    name: "Local Businesses, Clinics & Services",
+    category: "local",
+    returns: "Business name, phone, street address, opening hours, rating, review count, and price range",
+    requirements: "Built-in LocalBusiness schema.org & contact block extractor",
+    examples: [
+        "npx webcrawl search \"24 hour emergency vet in Melbourne\" -c business",
+    ],
+    matches({ category }) {
+        if (category === "business") {
+            return { matched: true, confidence: 0.99, reason: "Local business search category" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const html = input.html ?? "";
+        const url = input.url ?? "https://example.com";
+        const businesses = extractBusinessListingsFromHtml(html, url);
+        return {
+            targetId: "local-business",
+            targetName: "Local Businesses, Clinics & Services",
+            category: "local",
+            structured: { businesses },
+            evidence: {
+                targetId: "local-business",
+                tier: "static",
+                contentHash: sha256(html),
+                sourceUrl: url,
+                matchedSignals: businesses.length > 0 ? ["dom:local-business"] : [],
+                limitations: businesses.length === 0 ? ["No LocalBusiness JSON-LD or contact card found"] : [],
+            },
+        };
+    },
+};
+export const cveSecurityTargetPlugin = {
+    id: "cve-security",
+    name: "CVEs, GHSA Advisories & Exploit Catalogs",
+    category: "security",
+    returns: "CVE/GHSA IDs, CVSS v3/v4 score & vector, severity, CWEs, affected package ranges, fixed versions, and CISA KEV status",
+    requirements: "Built-in NVD 2.0 + OSV.dev advisory correlator and page vulnerability parser (zero API key required)",
+    examples: [
+        "npx webcrawl search \"CVE-2024-3094 xz utils\" -c security",
+        "npx webcrawl search \"nextjs middleware auth bypass cve\" -c security",
+    ],
+    matches({ category, query, url }) {
+        if (category === "security") {
+            return { matched: true, confidence: 0.99, reason: "Security search category" };
+        }
+        const text = `${url ?? ""} ${query ?? ""}`;
+        if (/\b(CVE-\d{4}-\d{4,7}|GHSA-[23456789cfghjmpqrvwx]{4}-[23456789cfghjmpqrvwx]{4}-[23456789cfghjmpqrvwx]{4})\b/i.test(text)) {
+            return { matched: true, confidence: 0.98, reason: "CVE or GHSA identifier detected" };
+        }
+        return { matched: false, confidence: 0 };
+    },
+    extract(input) {
+        const text = input.markdown ?? input.html ?? input.query ?? "";
+        const cveIds = extractCveIdsFromText(text);
+        return {
+            targetId: "cve-security",
+            targetName: "CVEs, GHSA Advisories & Exploit Catalogs",
+            category: "security",
+            structured: { cveIds },
+            evidence: {
+                targetId: "cve-security",
+                tier: "api",
+                contentHash: sha256(text),
+                sourceUrl: input.url,
+                matchedSignals: cveIds.map((id) => `cve:${id}`),
+                limitations: cveIds.length === 0 ? ["No explicit CVE-YYYY-NNNN identifier found in text"] : [],
+            },
+        };
+    },
+};
+/** Default ordered registry of all built-in targets (specific targets first, generic web-spa last). */
+export const BUILTIN_TARGETS = [
+    llmsTxtTargetPlugin,
+    githubTargetPlugin,
+    arxivTargetPlugin,
+    youtubeMediaTargetPlugin,
+    pdfOfficeTargetPlugin,
+    cveSecurityTargetPlugin,
+    shoppingTargetPlugin,
+    flightsTargetPlugin,
+    hotelsTargetPlugin,
+    realEstateTargetPlugin,
+    localBusinessTargetPlugin,
+    webSpaTargetPlugin,
+];
+const customTargets = new Map();
+/** Register a custom or community target plugin at runtime. */
+export function registerTarget(plugin) {
+    customTargets.set(plugin.id, plugin);
+}
+/** List all registered target plugins (community overrides + built-ins). */
+export function listTargets() {
+    const byId = new Map();
+    for (const t of customTargets.values()) {
+        byId.set(t.id, t);
+    }
+    for (const t of BUILTIN_TARGETS) {
+        if (!byId.has(t.id)) {
+            byId.set(t.id, t);
+        }
+    }
+    return Array.from(byId.values());
+}
+export function getTargetById(id) {
+    return customTargets.get(id) ?? BUILTIN_TARGETS.find((t) => t.id === id);
+}
+/** Resolve the highest-confidence target plugin for a URL, query, or category. */
+export function resolveTarget(input) {
+    let bestPlugin = webSpaTargetPlugin;
+    let bestMatch = { matched: true, confidence: 0.1, reason: "Fallback web target" };
+    for (const plugin of listTargets()) {
+        const m = plugin.matches(input);
+        if (m.matched && m.confidence > bestMatch.confidence) {
+            bestPlugin = plugin;
+            bestMatch = m;
+        }
+    }
+    return { plugin: bestPlugin, match: bestMatch };
+}
+/** Run target-aware extraction on HTML/text and attach REA-style Evidence & Provenance. */
+export async function inspectTarget(input) {
+    const start = Date.now();
+    const plugin = input.targetId
+        ? (getTargetById(input.targetId) ?? webSpaTargetPlugin)
+        : resolveTarget({
+            url: input.url,
+            query: input.query,
+            contentType: input.contentType,
+            category: input.category,
+        }).plugin;
+    if (plugin.extract) {
+        const res = await plugin.extract(input);
+        res.evidence.latencyMs = Date.now() - start;
+        return res;
+    }
+    const fallback = await webSpaTargetPlugin.extract(input);
+    fallback.targetId = plugin.id;
+    fallback.targetName = plugin.name;
+    fallback.category = plugin.category;
+    fallback.evidence.targetId = plugin.id;
+    fallback.evidence.latencyMs = Date.now() - start;
+    return fallback;
+}

package/dist/core/targets/types.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Pluggable Target & Extractor Registry types for Webcrawl Anything.
+ *
+ * Modeled on Reverse Engineer Anything (REA): every target declares what it can
+ * inspect, its local/cloud prerequisites, how to match URLs or search queries,
+ * and returns structured results paired with verifiable Evidence & Provenance.
+ */
+export type TargetCategory = "web" | "documents" | "media" | "commerce" | "travel" | "real-estate" | "local" | "security" | "developer" | "academic";
+export type TargetTier = "cache" | "static" | "impit-tls" | "impersonate" | "proxy-impersonate" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
+export interface TargetEvidence {
+    /** Which target plugin handled the inspection */
+    targetId: string;
+    /** Transport or parser tier used to acquire the content */
+    tier: TargetTier;
+    /** SHA-256 of the raw content or canonical payload */
+    contentHash?: string;
+    /** End-to-end execution latency in milliseconds */
+    latencyMs?: number;
+    /** Canonical URL or upstream endpoint inspected */
+    sourceUrl?: string;
+    /** DOM selectors, JSON-LD blocks, API endpoints, or regex rules that matched */
+    matchedSignals: string[];
+    /** Explicit limitations, unverified fields, or skipped sections (REA-style honesty) */
+    limitations: string[];
+}
+export interface TargetMatch {
+    matched: boolean;
+    confidence: number;
+    reason?: string;
+    extractedParams?: Record<string, string>;
+}
+export interface TargetInspectInput {
+    url?: string;
+    query?: string;
+    html?: string;
+    markdown?: string;
+    bytes?: Uint8Array;
+    contentType?: string;
+    countryCode?: string;
+}
+export interface TargetInspectResult<T = unknown> {
+    targetId: string;
+    targetName: string;
+    category: TargetCategory;
+    title?: string;
+    markdown?: string;
+    structured?: T;
+    evidence: TargetEvidence;
+}
+export interface CrawlTargetPlugin<T = unknown> {
+    /** Unique slug (e.g. "web-spa", "pdf-office", "youtube", "shopping", "github", "arxiv", "cve") */
+    id: string;
+    /** Human-readable target name for `webcrawl targets` and README table */
+    name: string;
+    /** Category grouping */
+    category: TargetCategory;
+    /** One-line summary of what Webcrawl returns for this target (REA table column 2) */
+    returns: string;
+    /** Local/cloud requirements & guide note (REA table column 3) */
+    requirements: string;
+    /** Example URLs or queries for CLI / agent discovery */
+    examples: string[];
+    /** Fast synchronous check whether a URL or search query targets this plugin */
+    matches(input: {
+        url?: string;
+        query?: string;
+        contentType?: string;
+        category?: string;
+    }): TargetMatch;
+    /** Extract structured data + evidence from already-fetched HTML/text/bytes */
+    extract?(input: TargetInspectInput): Promise<TargetInspectResult<T>> | TargetInspectResult<T>;
+}

package/dist/core/targets/types.js

@@ -0,0 +1,8 @@
+/**
+ * Pluggable Target & Extractor Registry types for Webcrawl Anything.
+ *
+ * Modeled on Reverse Engineer Anything (REA): every target declares what it can
+ * inspect, its local/cloud prerequisites, how to match URLs or search queries,
+ * and returns structured results paired with verifiable Evidence & Provenance.
+ */
+export {};

package/dist/core/targets.d.ts

@@ -0,0 +1 @@
+export * from "./targets/index.js";

package/dist/core/targets.js

@@ -0,0 +1 @@
+export * from "./targets/index.js";

package/dist/format.js

@@ -59,7 +59,11 @@ export function formatSearch(res) {
         })
             .join("\n")}`);
     }
-    return parts.length ? parts.join("\n\n") : "No results.";
+    if (parts.length)
+        return parts.join("\n\n");
+    if (res.success === false && str(res.error))
+        return `Error: ${str(res.error)}`;
+    return "No results.";
 }
 export function formatCredits(res) {
     const c = res.credits;

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export { createServer, createWebcrawlMcpServer, resolveSchemaProfile, type McpServerOptions, type SchemaProfile, type WebcrawlLikeClient, } from "./server.js";
+export { loadLocalEngineClient, LOCAL_ENGINE_LOAD_ERROR } from "./load-local-engine.js";
 export { MAX_OUTPUT_CHARS, truncate } from "./format.js";

package/dist/index.js

@@ -1,2 +1,3 @@
 export { createServer, createWebcrawlMcpServer, resolveSchemaProfile, } from "./server.js";
+export { loadLocalEngineClient, LOCAL_ENGINE_LOAD_ERROR } from "./load-local-engine.js";
 export { MAX_OUTPUT_CHARS, truncate } from "./format.js";

package/dist/load-local-engine.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Load the zero-key local engine that ships inside this package (`dist/core`).
+ * Never falls back to webcrawl.app — callers must fail if this throws.
+ */
+import type { WebcrawlLikeClient } from "./server.js";
+export declare const LOCAL_ENGINE_LOAD_ERROR: string;
+export declare function loadLocalEngineClient(): Promise<WebcrawlLikeClient>;

package/dist/load-local-engine.js

@@ -0,0 +1,22 @@
+export const LOCAL_ENGINE_LOAD_ERROR = "webcrawl-mcp: failed to load the local engine bundled in this package (dist/core/local-engine.js). " +
+    "Reinstall @webcrawl/mcp. To use webcrawl.app cloud instead, set WEBCRAWL_API_KEY or run `webcrawl login`.";
+async function importEngine(spec) {
+    const localMod = (await import(spec));
+    if (typeof localMod.createLocalEngineClient !== "function") {
+        throw new Error(`no createLocalEngineClient export from ${spec}`);
+    }
+    return localMod.createLocalEngineClient();
+}
+export async function loadLocalEngineClient() {
+    const errors = [];
+    const packaged = new URL("./core/local-engine.js", import.meta.url).href;
+    for (const spec of [packaged, "webcrawl/local-engine"]) {
+        try {
+            return await importEngine(spec);
+        }
+        catch (e) {
+            errors.push(`${spec}: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    throw new Error(`${LOCAL_ENGINE_LOAD_ERROR}\n${errors.join("\n")}`);
+}

package/dist/server.js

@@ -66,6 +66,9 @@ export function createWebcrawlMcpServer(client, versionOrOpts = "0.1.0") {
             sites,
             scrapeContent: include_content,
         });
+        if (res.success === false && typeof res.error === "string") {
+            throw new Error(formatSearch(res) + formatCredits(res));
+        }
         return formatSearch(res) + formatCredits(res);
     }));
     server.registerTool("web_read", {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webcrawl/mcp",
-  "version": "1.4.2",
+  "version": "1.4.3",
   "mcpName": "io.github.brainfish-ai/webcrawl",
   "description": "MCP server for Webcrawl Anything: search, scrape, map, crawl, parse and monitor the web from your agent. Runs locally with no API key, or on webcrawl.app cloud.",
   "type": "module",
@@ -47,13 +47,21 @@
     }
   },
   "scripts": {
-    "build": "tsc -p tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json && tsc -p tsconfig.core.json",
     "prepack": "npm run build",
     "test": "node --conditions=development --import tsx --test test/*.test.ts"
   },
   "dependencies": {
+    "@ai-sdk/gateway": "^4.0.123",
+    "@firecrawl/pdf-inspector": "^1.25.2",
     "@modelcontextprotocol/sdk": "^1.32.2",
-    "@webcrawl/client": "^1.4.2",
+    "@vercel/sandbox": "^1.0.0",
+    "@webcrawl/client": "^1.4.3",
+    "@webcrawl/curl-impersonate": "^1.4.3",
+    "ai": "^7.0.150",
+    "impit": "^0.14.7",
+    "mammoth": "^1.13.0",
+    "officeparser": "^8.2.0",
     "zod": "^4.0.0"
   },
   "engines": {