brave-real-browser-mcp-server 2.24.4 → 2.25.0

package/dist/handlers/advanced-tools.js

@@ -586,22 +586,78 @@ export async function handleDeepAnalysis(page, args) {
 }
 /**
  * Record full network traffic - Uses response events to avoid crashes
+ * ULTRA POWERFUL: API detection, media URLs, smart categorization
  */
 export async function handleNetworkRecorder(page, args) {
     const requests = [];
     const duration = args.duration || 10000;
     let totalSize = 0;
+    const categories = {};
+    const apis = [];
+    const mediaUrls = [];
+    const seen = new Set();
+    // ============================================================
+    // SMART CATEGORIZATION HELPER
+    // ============================================================
+    const categorizeUrl = (url, resourceType) => {
+        const urlLower = url.toLowerCase();
+        // API endpoints
+        if (/\/api\/|\/v\d+\/|\.json(\?|$)|graphql/i.test(url))
+            return 'api';
+        // Media
+        if (/\.(mp4|webm|m3u8|ts|mp3|flac|ogg)/i.test(url))
+            return 'media';
+        if (resourceType === 'media' || resourceType === 'video' || resourceType === 'audio')
+            return 'media';
+        // Images
+        if (/\.(jpg|jpeg|png|gif|webp|svg|ico)/i.test(url) || resourceType === 'image')
+            return 'image';
+        // Scripts
+        if (/\.js(\?|$)/i.test(url) || resourceType === 'script')
+            return 'script';
+        // Styles
+        if (/\.css(\?|$)/i.test(url) || resourceType === 'stylesheet')
+            return 'style';
+        // Fonts
+        if (/\.(woff2?|ttf|eot|otf)/i.test(url) || resourceType === 'font')
+            return 'font';
+        // XHR/Fetch
+        if (resourceType === 'xhr' || resourceType === 'fetch')
+            return 'xhr';
+        // Documents
+        if (resourceType === 'document')
+            return 'document';
+        return 'other';
+    };
     // Response handler - safer than request interception
     const responseHandler = (response) => {
         try {
             const url = response.url();
+            // Dedup
+            if (seen.has(url))
+                return;
+            seen.add(url);
             if (args.filterUrl && !url.includes(args.filterUrl)) {
                 return;
             }
+            const resourceType = response.request()?.resourceType?.() || 'unknown';
+            const method = response.request()?.method?.() || 'GET';
+            const category = categorizeUrl(url, resourceType);
+            categories[category] = (categories[category] || 0) + 1;
+            // Collect API endpoints
+            if (category === 'api' || resourceType === 'xhr' || resourceType === 'fetch') {
+                apis.push({ url, method, type: resourceType });
+            }
+            // Collect media URLs
+            if (category === 'media' || /\.(mp4|webm|m3u8|ts|mp3)/i.test(url)) {
+                mediaUrls.push(url);
+            }
             const entry = {
                 url,
                 status: response.status(),
-                resourceType: response.request()?.resourceType?.() || 'unknown',
+                resourceType,
+                category,
+                method,
                 timestamp: Date.now(),
             };
             if (args.includeHeaders) {
@@ -612,7 +668,6 @@ export async function handleNetworkRecorder(page, args) {
                     entry.headers = {};
                 }
             }
-            // Note: Response body requires async handling, skip for stability
             requests.push(entry);
             // Track size from headers
             try {
@@ -647,6 +702,10 @@ export async function handleNetworkRecorder(page, args) {
         requests: requests.slice(0, 500),
         count: requests.length,
         totalSize,
+        categories,
+        apis: apis.length > 0 ? apis : undefined,
+        mediaUrls: mediaUrls.length > 0 ? mediaUrls : undefined,
+        message: `📡 Recorded ${requests.length} requests (${Math.round(totalSize / 1024)}KB) | APIs: ${apis.length} | Media: ${mediaUrls.length}`
     };
 }
 /**
@@ -776,6 +835,7 @@ export async function handleAdProtectionDetector(page, args) {
 }
 /**
  * Wait for dynamic AJAX loading
+ * ULTRA POWERFUL: Infinite scroll, lazy load, mutation observer
  */
 export async function handleAjaxContentWaiter(page, args) {
     const timeout = args.timeout || 30000;
@@ -783,6 +843,79 @@ export async function handleAjaxContentWaiter(page, args) {
     const startTime = Date.now();
     let content;
     let loaded = false;
+    let newElementsCount = 0;
+    let scrollDepth = 0;
+    // ============================================================
+    // 1. MUTATION OBSERVER: Track DOM changes in real-time
+    // ============================================================
+    const setupMutationObserver = async () => {
+        return await page.evaluate(() => {
+            return new Promise((resolve) => {
+                let added = 0;
+                let modified = 0;
+                const observer = new MutationObserver((mutations) => {
+                    mutations.forEach(m => {
+                        added += m.addedNodes.length;
+                        if (m.type === 'attributes' || m.type === 'characterData')
+                            modified++;
+                    });
+                });
+                observer.observe(document.body, {
+                    childList: true,
+                    subtree: true,
+                    attributes: true,
+                    characterData: true
+                });
+                // Return after 2 seconds of observation
+                setTimeout(() => {
+                    observer.disconnect();
+                    resolve({ added, modified });
+                }, 2000);
+            });
+        });
+    };
+    // ============================================================
+    // 2. INFINITE SCROLL DETECTION
+    // ============================================================
+    const handleInfiniteScroll = async () => {
+        const initialHeight = await page.evaluate(() => document.body.scrollHeight);
+        const initialCount = await page.evaluate(() => document.querySelectorAll('*').length);
+        // Scroll to bottom
+        await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+        await new Promise(r => setTimeout(r, 1000));
+        // Check if new content loaded
+        const newHeight = await page.evaluate(() => document.body.scrollHeight);
+        const newCount = await page.evaluate(() => document.querySelectorAll('*').length);
+        return {
+            scrolled: newHeight > initialHeight,
+            newElements: newCount - initialCount,
+            scrollDepth: newHeight
+        };
+    };
+    // ============================================================
+    // 3. LAZY LOAD DETECTION
+    // ============================================================
+    const detectLazyLoad = async () => {
+        return await page.evaluate(() => {
+            const lazyElements = [];
+            // Check for common lazy load patterns
+            document.querySelectorAll('[data-src], [data-lazy], [loading="lazy"], .lazy, .lazyload').forEach(el => {
+                const dataSrc = el.getAttribute('data-src') || el.getAttribute('data-lazy');
+                if (dataSrc)
+                    lazyElements.push(dataSrc);
+            });
+            // Intersection Observer based lazy images
+            document.querySelectorAll('img[data-src], img.lazy').forEach(img => {
+                const dataSrc = img.dataset.src;
+                if (dataSrc)
+                    lazyElements.push(dataSrc);
+            });
+            return lazyElements;
+        });
+    };
+    // ============================================================
+    // 4. MAIN WAITING LOGIC
+    // ============================================================
     while (Date.now() - startTime < timeout) {
         if (args.selector) {
             const element = await page.$(args.selector);
@@ -795,17 +928,35 @@ export async function handleAjaxContentWaiter(page, args) {
             }
         }
         else {
-            // Wait for network to be idle
-            await page.waitForNetworkIdle({ timeout: pollInterval }).catch(() => { });
-            loaded = true;
-            break;
+            // Smart waiting: Check for ongoing activity
+            const mutationResult = await setupMutationObserver();
+            newElementsCount = mutationResult.added;
+            if (mutationResult.added === 0 && mutationResult.modified === 0) {
+                // No DOM changes, content likely loaded
+                loaded = true;
+                break;
+            }
+        }
+        // Try infinite scroll to load more content
+        const scrollResult = await handleInfiniteScroll();
+        if (scrollResult.scrolled) {
+            scrollDepth = scrollResult.scrollDepth;
+            newElementsCount += scrollResult.newElements;
         }
         await new Promise((r) => setTimeout(r, pollInterval));
     }
+    // Detect any lazy-loaded content
+    const lazyElements = await detectLazyLoad();
     return {
         loaded,
         waitTime: Date.now() - startTime,
         content,
+        newElementsCount,
+        scrollDepth,
+        lazyElements: lazyElements.length > 0 ? lazyElements : undefined,
+        message: loaded
+            ? `✅ Content loaded in ${Date.now() - startTime}ms (${newElementsCount} new elements, scroll: ${scrollDepth}px)`
+            : `⏱️ Timeout after ${timeout}ms`
     };
 }
 /**
@@ -1002,20 +1153,124 @@ export async function handleVideoRecording(page, args, recorderState) {
 }
 /**
  * Harvest all links from page
+ * ULTRA POWERFUL: Pagination detection, smart categorization, file types
  */
 export async function handleLinkHarvester(page, args) {
     const currentUrl = new URL(page.url());
-    const allLinks = await page.evaluate((filter) => {
-        return Array.from(document.querySelectorAll('a[href]')).map((a) => ({
-            url: a.href,
-            text: a.textContent?.trim()?.substring(0, 100) || '',
-        }));
-    }, args.filter);
+    // ============================================================
+    // 1. EXTRACT ALL LINKS WITH SMART CATEGORIZATION
+    // ============================================================
+    const allLinks = await page.evaluate(() => {
+        const links = [];
+        document.querySelectorAll('a[href]').forEach((a) => {
+            const anchor = a;
+            links.push({
+                url: anchor.href,
+                text: a.textContent?.trim()?.substring(0, 100) || '',
+                attrs: {
+                    rel: anchor.rel || '',
+                    target: anchor.target || '',
+                    class: anchor.className || '',
+                    id: anchor.id || '',
+                    download: anchor.download || '',
+                }
+            });
+        });
+        return links;
+    });
+    // ============================================================
+    // 2. PAGINATION DETECTION
+    // ============================================================
+    const pagination = await page.evaluate(() => {
+        let nextPage;
+        let prevPage;
+        let totalPages;
+        // Common pagination selectors
+        const nextSelectors = [
+            'a[rel="next"]', 'a.next', 'a.pagination-next',
+            '[aria-label="Next"]', 'a:has-text("Next")', 'a:has-text(">")',
+            '.pagination a:last-child', 'a.page-link:last-child'
+        ];
+        const prevSelectors = [
+            'a[rel="prev"]', 'a.prev', 'a.pagination-prev',
+            '[aria-label="Previous"]', 'a:has-text("Prev")', 'a:has-text("<")'
+        ];
+        for (const sel of nextSelectors) {
+            try {
+                const el = document.querySelector(sel);
+                if (el?.href) {
+                    nextPage = el.href;
+                    break;
+                }
+            }
+            catch { /* invalid selector */ }
+        }
+        for (const sel of prevSelectors) {
+            try {
+                const el = document.querySelector(sel);
+                if (el?.href) {
+                    prevPage = el.href;
+                    break;
+                }
+            }
+            catch { /* invalid selector */ }
+        }
+        // Count page numbers
+        const pageNumbers = Array.from(document.querySelectorAll('.pagination a, .page-numbers a, nav a'))
+            .map(a => parseInt(a.textContent || '0', 10))
+            .filter(n => !isNaN(n) && n > 0);
+        if (pageNumbers.length > 0) {
+            totalPages = Math.max(...pageNumbers);
+        }
+        return { nextPage, prevPage, totalPages };
+    });
+    // ============================================================
+    // 3. SMART LINK CATEGORIZATION
+    // ============================================================
+    const categorizeLink = (url, text, attrs) => {
+        const urlLower = url.toLowerCase();
+        const textLower = text.toLowerCase();
+        // File downloads
+        if (/\.(pdf|doc|docx|xls|xlsx|zip|rar|7z|tar|gz)(\?.*)?$/i.test(url))
+            return 'document';
+        if (/\.(mp4|mkv|avi|mov|webm|flv)(\?.*)?$/i.test(url))
+            return 'video';
+        if (/\.(mp3|wav|flac|aac|ogg)(\?.*)?$/i.test(url))
+            return 'audio';
+        if (/\.(jpg|jpeg|png|gif|webp|svg|bmp)(\?.*)?$/i.test(url))
+            return 'image';
+        if (attrs.download)
+            return 'download';
+        // Navigation
+        if (/\/(next|page|p)\/\d+|[?&]page=\d+/i.test(url))
+            return 'pagination';
+        if (textLower.includes('next') || textLower.includes('prev'))
+            return 'pagination';
+        // Social
+        if (/facebook|twitter|instagram|linkedin|youtube|tiktok/i.test(url))
+            return 'social';
+        // Common patterns
+        if (/login|signin|sign-in/i.test(url))
+            return 'auth';
+        if (/register|signup|sign-up/i.test(url))
+            return 'auth';
+        if (/search|query|q=/i.test(url))
+            return 'search';
+        if (/contact|about|faq|help/i.test(url))
+            return 'info';
+        return 'navigation';
+    };
     const processedLinks = [];
+    const categories = {};
+    const seen = new Set();
     let internal = 0;
     let external = 0;
     for (const link of allLinks) {
         try {
+            // Dedup by URL
+            if (seen.has(link.url))
+                continue;
+            seen.add(link.url);
             const linkUrl = new URL(link.url);
             const isInternal = linkUrl.hostname === currentUrl.hostname;
             if (args.filter && !link.url.includes(args.filter) && !link.text.includes(args.filter)) {
@@ -1025,10 +1280,13 @@ export async function handleLinkHarvester(page, args) {
                 continue;
             if (!isInternal && args.includeExternal === false)
                 continue;
+            const category = categorizeLink(link.url, link.text, link.attrs);
+            categories[category] = (categories[category] || 0) + 1;
             processedLinks.push({
                 url: link.url,
                 text: link.text,
                 type: isInternal ? 'internal' : 'external',
+                category,
             });
             if (isInternal)
                 internal++;
@@ -1045,6 +1303,10 @@ export async function handleLinkHarvester(page, args) {
         links: processedLinks,
         internal,
         external,
+        pagination: (pagination.nextPage || pagination.prevPage || pagination.totalPages) ? pagination : undefined,
+        categories,
+        message: `🔗 Found ${processedLinks.length} links (${internal} internal, ${external} external)` +
+            (pagination.nextPage ? ` | Next: ${pagination.nextPage}` : '')
     };
 }
 /**
@@ -2614,3 +2876,245 @@ export async function handleStreamExtractor(page, args) {
             : 'No direct URLs found',
     };
 }
+/**
+ * Advanced web crawler with Crawlee + brave-real-launcher integration
+ * Features: URL queue, proxy rotation, rate limiting, data extraction
+ */
+export async function handleWebCrawler(page, args) {
+    // Import Crawlee dynamically to avoid load-time errors if not installed
+    let PuppeteerCrawler;
+    let RequestQueue;
+    let Configuration;
+    try {
+        const crawlee = await import('crawlee');
+        PuppeteerCrawler = crawlee.PuppeteerCrawler;
+        RequestQueue = crawlee.RequestQueue;
+        Configuration = crawlee.Configuration;
+    }
+    catch (e) {
+        return {
+            success: false,
+            crawledPages: 0,
+            results: [],
+            errors: ['Crawlee not installed. Run: npm install crawlee'],
+            message: '❌ Crawlee package not found',
+        };
+    }
+    // Import brave-real-launcher for browser launch
+    let getBravePath;
+    let braveRealPuppeteerCore;
+    try {
+        const launcher = await import('brave-real-launcher');
+        getBravePath = launcher.getBravePath;
+    }
+    catch (e) {
+        // Fallback - will use default Chromium
+    }
+    // Import brave-real-puppeteer-core for stealth features
+    try {
+        braveRealPuppeteerCore = await import('brave-real-puppeteer-core');
+    }
+    catch (e) {
+        // Will use default puppeteer
+    }
+    const results = [];
+    const errors = [];
+    const visited = new Set();
+    // Configuration
+    const maxDepth = args.maxDepth ?? 3;
+    const maxPages = args.maxPages ?? 50;
+    const concurrency = args.concurrency ?? 3;
+    const rateLimit = args.rateLimit ?? 2;
+    const retryCount = args.retryCount ?? 3;
+    const timeout = args.timeout ?? 30000;
+    // URL filtering patterns
+    const includePattern = args.includePattern ? new RegExp(args.includePattern, 'i') : null;
+    const excludePattern = args.excludePattern ? new RegExp(args.excludePattern, 'i') : null;
+    // Proxy rotation
+    let proxyIndex = 0;
+    const getNextProxy = () => {
+        if (!args.proxyList || args.proxyList.length === 0)
+            return undefined;
+        const proxy = args.proxyList[proxyIndex % args.proxyList.length];
+        proxyIndex++;
+        return proxy;
+    };
+    // Rate limiting
+    let lastRequestTime = 0;
+    const rateLimitDelay = 1000 / rateLimit;
+    const enforceRateLimit = async () => {
+        const now = Date.now();
+        const elapsed = now - lastRequestTime;
+        if (elapsed < rateLimitDelay) {
+            await new Promise(r => setTimeout(r, rateLimitDelay - elapsed));
+        }
+        lastRequestTime = Date.now();
+    };
+    try {
+        // Configure Crawlee to use memory storage (no disk)
+        Configuration.getGlobalConfig().set('persistStorage', false);
+        // Create request queue with start URLs
+        const requestQueue = await RequestQueue.open();
+        for (const url of args.startUrls) {
+            await requestQueue.addRequest({
+                url,
+                userData: { depth: 0 },
+            });
+        }
+        // Get Brave executable path if available
+        let executablePath;
+        try {
+            if (getBravePath) {
+                executablePath = getBravePath();
+            }
+        }
+        catch (e) {
+            // Use default
+        }
+        // Create crawler based on mode
+        const crawler = new PuppeteerCrawler({
+            requestQueue,
+            maxConcurrency: concurrency,
+            maxRequestRetries: retryCount,
+            requestHandlerTimeoutSecs: timeout / 1000,
+            // Use brave-real-puppeteer-core with all stealth features
+            launchContext: {
+                // Use brave-real-puppeteer-core as custom launcher for 50+ stealth features
+                launcher: braveRealPuppeteerCore || undefined,
+                launchOptions: {
+                    headless: true,
+                    executablePath,
+                    args: [
+                        '--no-sandbox',
+                        '--disable-setuid-sandbox',
+                        '--disable-blink-features=AutomationControlled',
+                        '--disable-dev-shm-usage',
+                        '--disable-accelerated-2d-canvas',
+                        '--disable-gpu',
+                    ],
+                },
+            },
+            // Browser pool configuration
+            browserPoolOptions: {
+                maxOpenPagesPerBrowser: 1,
+            },
+            // Pre-navigation hook for rate limiting and proxy
+            preNavigationHooks: [
+                async (crawlingContext) => {
+                    await enforceRateLimit();
+                    // Set custom user agent if provided
+                    if (args.userAgent) {
+                        await crawlingContext.page.setUserAgent(args.userAgent);
+                    }
+                    // Set custom headers if provided
+                    if (args.headers) {
+                        await crawlingContext.page.setExtraHTTPHeaders(args.headers);
+                    }
+                },
+            ],
+            // Main request handler
+            requestHandler: async ({ request, page: crawlerPage, enqueueLinks }) => {
+                const depth = request.userData.depth || 0;
+                const url = request.url;
+                // Skip if already visited or max pages reached
+                if (visited.has(url) || results.length >= maxPages) {
+                    return;
+                }
+                visited.add(url);
+                // URL filtering
+                if (includePattern && !includePattern.test(url))
+                    return;
+                if (excludePattern && excludePattern.test(url))
+                    return;
+                const result = {
+                    url,
+                    depth,
+                };
+                try {
+                    // Get page title
+                    result.title = await crawlerPage.title();
+                    // Extract data using selectors
+                    if (args.extractSelectors) {
+                        result.extractedData = {};
+                        for (const [key, selector] of Object.entries(args.extractSelectors)) {
+                            try {
+                                const elements = await crawlerPage.$$(selector);
+                                if (elements.length === 1) {
+                                    result.extractedData[key] = await crawlerPage.$eval(selector, (el) => el.textContent?.trim() || el.getAttribute('href') || el.getAttribute('src'));
+                                }
+                                else if (elements.length > 1) {
+                                    result.extractedData[key] = await crawlerPage.$$eval(selector, (els) => els.map(el => el.textContent?.trim() || el.getAttribute('href') || el.getAttribute('src')).filter(Boolean));
+                                }
+                            }
+                            catch (e) {
+                                // Selector not found
+                            }
+                        }
+                    }
+                    // Follow links if enabled and depth allows
+                    if (args.followLinks !== false && depth < maxDepth && results.length < maxPages) {
+                        // Get all links
+                        const pageLinks = await crawlerPage.$$eval('a[href]', (anchors) => anchors.map(a => a.href).filter(href => href.startsWith('http')));
+                        result.links = pageLinks.slice(0, 100); // Limit stored links
+                        // Filter and enqueue links
+                        const linksToEnqueue = pageLinks.filter((link) => {
+                            if (visited.has(link))
+                                return false;
+                            if (includePattern && !includePattern.test(link))
+                                return false;
+                            if (excludePattern && excludePattern.test(link))
+                                return false;
+                            return true;
+                        });
+                        // Add filtered links using Crawlee's enqueueLinks
+                        for (const link of linksToEnqueue.slice(0, 50)) {
+                            try {
+                                await requestQueue.addRequest({
+                                    url: link,
+                                    userData: { depth: depth + 1 },
+                                });
+                            }
+                            catch (e) {
+                                // Link already in queue
+                            }
+                        }
+                    }
+                    // Download media if enabled
+                    if (args.downloadMedia && args.savePath) {
+                        const mediaUrls = await crawlerPage.$$eval('img[src], video source[src], a[href$=".pdf"], a[href$=".jpg"], a[href$=".png"]', (els) => els.map(el => el.getAttribute('src') || el.getAttribute('href')).filter(Boolean));
+                        result.extractedData = result.extractedData || {};
+                        result.extractedData.mediaUrls = mediaUrls;
+                    }
+                    results.push(result);
+                }
+                catch (error) {
+                    result.error = error instanceof Error ? error.message : String(error);
+                    errors.push(`${url}: ${result.error}`);
+                    results.push(result);
+                }
+            },
+            // Failed request handler
+            failedRequestHandler: async ({ request }, error) => {
+                errors.push(`Failed: ${request.url} - ${error.message}`);
+            },
+        });
+        // Run the crawler
+        await crawler.run();
+        return {
+            success: results.length > 0,
+            crawledPages: results.length,
+            results,
+            errors,
+            message: `🕷️ Crawled ${results.length} pages (depth: ${maxDepth}, errors: ${errors.length})`,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            crawledPages: results.length,
+            results,
+            errors: [...errors, error instanceof Error ? error.message : String(error)],
+            message: `❌ Crawler error: ${error instanceof Error ? error.message : String(error)}`,
+        };
+    }
+}

package/dist/index.js

@@ -61,7 +61,9 @@ import { handleBreadcrumbNavigator, handleUrlRedirectTracer, handleSearchContent
 // Download tools
 handleFileDownloader, 
 // Enhanced streaming/download tools
-handleIframeHandler, handleStreamExtractor, } from './handlers/advanced-tools.js';
+handleIframeHandler, handleStreamExtractor, 
+// Web crawler
+handleWebCrawler, } from './handlers/advanced-tools.js';
 // State for video recording
 const recorderState = new Map();
 debug('All modules loaded successfully');
@@ -254,6 +256,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 if (!page)
                     throw new Error('Browser not initialized. Call browser_init first.');
                 return { content: [{ type: 'text', text: JSON.stringify(await handleStreamExtractor(page, args)) }] };
+            // Web Crawler (Crawlee + brave-real-launcher)
+            case TOOL_NAMES.WEB_CRAWLER:
+                if (!page)
+                    throw new Error('Browser not initialized. Call browser_init first.');
+                return { content: [{ type: 'text', text: JSON.stringify(await handleWebCrawler(page, args)) }] };
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/dist/tool-definitions.js

@@ -622,6 +622,116 @@ export const TOOLS = [
             },
         },
     },
+    // ============================================================
+    // WEB CRAWLER TOOL (Crawlee-inspired)
+    // ============================================================
+    {
+        name: 'web_crawler',
+        description: 'Advanced web crawler with Crawlee-like features: URL queue (breadth/depth-first), proxy rotation, session management, auto-retry, rate limiting, concurrency control, and data extraction. Supports both browser and HTTP modes.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                startUrls: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Initial URLs to start crawling from'
+                },
+                maxDepth: {
+                    type: 'number',
+                    description: 'Maximum crawl depth (1 = only start URLs)',
+                    default: 3
+                },
+                maxPages: {
+                    type: 'number',
+                    description: 'Maximum pages to crawl',
+                    default: 50
+                },
+                concurrency: {
+                    type: 'number',
+                    description: 'Number of concurrent requests',
+                    default: 3
+                },
+                rateLimit: {
+                    type: 'number',
+                    description: 'Maximum requests per second',
+                    default: 2
+                },
+                crawlStrategy: {
+                    type: 'string',
+                    enum: ['breadth-first', 'depth-first'],
+                    description: 'URL queue strategy',
+                    default: 'breadth-first'
+                },
+                includePattern: {
+                    type: 'string',
+                    description: 'Regex pattern for URLs to include'
+                },
+                excludePattern: {
+                    type: 'string',
+                    description: 'Regex pattern for URLs to exclude'
+                },
+                extractSelectors: {
+                    type: 'object',
+                    description: 'CSS selectors for data extraction (e.g., {"title": "h1", "links": "a[href]"})'
+                },
+                followLinks: {
+                    type: 'boolean',
+                    description: 'Follow discovered links',
+                    default: true
+                },
+                downloadMedia: {
+                    type: 'boolean',
+                    description: 'Download images/videos/files',
+                    default: false
+                },
+                savePath: {
+                    type: 'string',
+                    description: 'Path to save downloaded files'
+                },
+                proxyList: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Proxy URLs for rotation (format: protocol://host:port)'
+                },
+                retryCount: {
+                    type: 'number',
+                    description: 'Number of retries for failed requests',
+                    default: 3
+                },
+                retryDelayMs: {
+                    type: 'number',
+                    description: 'Delay between retries in ms (exponential backoff)',
+                    default: 1000
+                },
+                timeout: {
+                    type: 'number',
+                    description: 'Request timeout in ms',
+                    default: 30000
+                },
+                mode: {
+                    type: 'string',
+                    enum: ['browser', 'http'],
+                    description: 'Crawl mode (browser = Puppeteer, http = fast HTTP)',
+                    default: 'browser'
+                },
+                respectRobotsTxt: {
+                    type: 'boolean',
+                    description: 'Respect robots.txt rules',
+                    default: true
+                },
+                userAgent: {
+                    type: 'string',
+                    description: 'Custom User-Agent string'
+                },
+                headers: {
+                    type: 'object',
+                    description: 'Custom headers for all requests'
+                },
+            },
+            required: ['startUrls'],
+        },
+    },
 ];
 // Tool name constants for type safety
 export const TOOL_NAMES = {
@@ -659,6 +769,8 @@ export const TOOL_NAMES = {
     // Enhanced tools
     IFRAME_HANDLER: 'iframe_handler',
     STREAM_EXTRACTOR: 'stream_extractor',
+    // Crawler tool
+    WEB_CRAWLER: 'web_crawler',
 };
 // Tool categories for organization
 export const TOOL_CATEGORIES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brave-real-browser-mcp-server",
-  "version": "2.24.4",
+  "version": "2.25.0",
   "description": "🦁 MCP server for Brave Real Browser - NPM Workspaces Monorepo with anti-detection features, SSE streaming, and LSP compatibility",
   "type": "module",
   "main": "dist/index.js",
@@ -50,7 +50,9 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "latest",
     "@types/turndown": "latest",
-    "brave-real-browser": "^2.5.4",
+    "brave-real-browser": "^2.6.0",
+    "crawlee": "^3.15.3",
+    "puppeteer-core": "^24.35.0",
     "turndown": "latest",
     "vscode-languageserver": "^9.0.1",
     "vscode-languageserver-textdocument": "^1.0.12"