brave-real-browser-mcp-server 2.24.5 → 2.26.0

package/dist/handlers/advanced-tools.js

@@ -2876,3 +2876,369 @@ export async function handleStreamExtractor(page, args) {
             : 'No direct URLs found',
     };
 }
+/**
+ * Advanced web crawler with Crawlee + brave-real-launcher integration
+ * Features: URL queue, proxy rotation, rate limiting, data extraction
+ */
+export async function handleWebCrawler(page, args) {
+    // Import Crawlee dynamically to avoid load-time errors if not installed
+    let PuppeteerCrawler;
+    let RequestQueue;
+    let Configuration;
+    try {
+        const crawlee = await import('crawlee');
+        PuppeteerCrawler = crawlee.PuppeteerCrawler;
+        RequestQueue = crawlee.RequestQueue;
+        Configuration = crawlee.Configuration;
+    }
+    catch (e) {
+        return {
+            success: false,
+            crawledPages: 0,
+            results: [],
+            errors: ['Crawlee not installed. Run: npm install crawlee'],
+            message: '❌ Crawlee package not found',
+        };
+    }
+    // Import brave-real-launcher for browser launch
+    let getBravePath;
+    let braveRealPuppeteerCore;
+    try {
+        const launcher = await import('brave-real-launcher');
+        getBravePath = launcher.getBravePath;
+    }
+    catch (e) {
+        // Fallback - will use default Chromium
+    }
+    // Import brave-real-puppeteer-core for stealth features
+    try {
+        braveRealPuppeteerCore = await import('brave-real-puppeteer-core');
+    }
+    catch (e) {
+        // Will use default puppeteer
+    }
+    const results = [];
+    const errors = [];
+    const visited = new Set();
+    // Configuration
+    const maxDepth = args.maxDepth ?? 3;
+    const maxPages = args.maxPages ?? 50;
+    const concurrency = args.concurrency ?? 3;
+    const rateLimit = args.rateLimit ?? 2;
+    const retryCount = args.retryCount ?? 3;
+    const timeout = args.timeout ?? 30000;
+    // URL filtering patterns
+    const includePattern = args.includePattern ? new RegExp(args.includePattern, 'i') : null;
+    const excludePattern = args.excludePattern ? new RegExp(args.excludePattern, 'i') : null;
+    // Proxy rotation
+    let proxyIndex = 0;
+    const getNextProxy = () => {
+        if (!args.proxyList || args.proxyList.length === 0)
+            return undefined;
+        const proxy = args.proxyList[proxyIndex % args.proxyList.length];
+        proxyIndex++;
+        return proxy;
+    };
+    // Rate limiting
+    let lastRequestTime = 0;
+    const rateLimitDelay = 1000 / rateLimit;
+    const enforceRateLimit = async () => {
+        const now = Date.now();
+        const elapsed = now - lastRequestTime;
+        if (elapsed < rateLimitDelay) {
+            await new Promise(r => setTimeout(r, rateLimitDelay - elapsed));
+        }
+        lastRequestTime = Date.now();
+    };
+    try {
+        // Configure Crawlee to use memory storage (no disk)
+        Configuration.getGlobalConfig().set('persistStorage', false);
+        // Create request queue with start URLs
+        const requestQueue = await RequestQueue.open();
+        for (const url of args.startUrls) {
+            await requestQueue.addRequest({
+                url,
+                userData: { depth: 0 },
+            });
+        }
+        // Get Brave executable path if available
+        let executablePath;
+        try {
+            if (getBravePath) {
+                executablePath = getBravePath();
+            }
+        }
+        catch (e) {
+            // Use default
+        }
+        // Create crawler based on mode
+        const crawler = new PuppeteerCrawler({
+            requestQueue,
+            maxConcurrency: concurrency,
+            maxRequestRetries: retryCount,
+            requestHandlerTimeoutSecs: timeout / 1000,
+            // Use brave-real-puppeteer-core with all stealth features
+            launchContext: {
+                // Use brave-real-puppeteer-core as custom launcher for 50+ stealth features
+                launcher: braveRealPuppeteerCore || undefined,
+                launchOptions: {
+                    headless: true,
+                    executablePath,
+                    args: [
+                        '--no-sandbox',
+                        '--disable-setuid-sandbox',
+                        '--disable-blink-features=AutomationControlled',
+                        '--disable-dev-shm-usage',
+                        '--disable-accelerated-2d-canvas',
+                        '--disable-gpu',
+                    ],
+                },
+            },
+            // Browser pool configuration
+            browserPoolOptions: {
+                maxOpenPagesPerBrowser: 1,
+            },
+            // Pre-navigation hook for rate limiting, popup blocking, and movie streaming optimizations
+            preNavigationHooks: [
+                async (crawlingContext) => {
+                    await enforceRateLimit();
+                    const pg = crawlingContext.page;
+                    // Set custom user agent if provided
+                    if (args.userAgent) {
+                        await pg.setUserAgent(args.userAgent);
+                    }
+                    // Set custom headers if provided
+                    if (args.headers) {
+                        await pg.setExtraHTTPHeaders(args.headers);
+                    }
+                    // Block popups and overlay ads (default: true for movie streaming)
+                    if (args.blockPopups !== false) {
+                        await pg.evaluateOnNewDocument(() => {
+                            // Block window.open popups
+                            window.open = () => null;
+                            // Block alert, confirm, prompt
+                            window.alert = () => { };
+                            window.confirm = () => true;
+                            window.prompt = () => null;
+                            // Block popup via createElement
+                            const origCreate = document.createElement.bind(document);
+                            document.createElement = (tag) => {
+                                if (tag.toLowerCase() === 'a' && arguments[1]?.target === '_blank') {
+                                    return origCreate('span');
+                                }
+                                return origCreate(tag);
+                            };
+                        });
+                    }
+                    // Block overlay ads and floating elements
+                    if (args.blockOverlayAds !== false) {
+                        await pg.evaluateOnNewDocument(() => {
+                            // Remove overlay ads after DOM load
+                            const removeOverlays = () => {
+                                const selectors = [
+                                    '[class*="popup"]', '[class*="modal"]', '[class*="overlay"]',
+                                    '[id*="popup"]', '[id*="modal"]', '[id*="overlay"]',
+                                    '[class*="ad-"]', '[class*="-ad"]', '[class*="advert"]',
+                                    '[class*="banner"]', '[class*="sticky"]', '[class*="float"]',
+                                    'div[style*="position: fixed"]', 'div[style*="z-index: 9"]',
+                                ];
+                                selectors.forEach(sel => {
+                                    document.querySelectorAll(sel).forEach(el => {
+                                        const style = window.getComputedStyle(el);
+                                        if (style.position === 'fixed' || style.zIndex > '1000') {
+                                            el.style.display = 'none';
+                                        }
+                                    });
+                                });
+                            };
+                            document.addEventListener('DOMContentLoaded', removeOverlays);
+                            setInterval(removeOverlays, 2000);
+                        });
+                    }
+                },
+            ],
+            // Main request handler
+            requestHandler: async ({ request, page: crawlerPage, enqueueLinks }) => {
+                const depth = request.userData.depth || 0;
+                const url = request.url;
+                // Skip if already visited or max pages reached
+                if (visited.has(url) || results.length >= maxPages) {
+                    return;
+                }
+                visited.add(url);
+                // URL filtering
+                if (includePattern && !includePattern.test(url))
+                    return;
+                if (excludePattern && excludePattern.test(url))
+                    return;
+                const result = {
+                    url,
+                    depth,
+                };
+                try {
+                    // Get page title
+                    result.title = await crawlerPage.title();
+                    // Extract data using selectors
+                    if (args.extractSelectors) {
+                        result.extractedData = {};
+                        for (const [key, selector] of Object.entries(args.extractSelectors)) {
+                            try {
+                                const elements = await crawlerPage.$$(selector);
+                                if (elements.length === 1) {
+                                    result.extractedData[key] = await crawlerPage.$eval(selector, (el) => el.textContent?.trim() || el.getAttribute('href') || el.getAttribute('src'));
+                                }
+                                else if (elements.length > 1) {
+                                    result.extractedData[key] = await crawlerPage.$$eval(selector, (els) => els.map(el => el.textContent?.trim() || el.getAttribute('href') || el.getAttribute('src')).filter(Boolean));
+                                }
+                            }
+                            catch (e) {
+                                // Selector not found
+                            }
+                        }
+                    }
+                    // Extract video links (JWPlayer, DooPlayer, iframes, ajax sources)
+                    if (args.extractVideoLinks !== false) {
+                        result.videoLinks = await crawlerPage.evaluate(() => {
+                            const videoLinks = [];
+                            const videoPatterns = /\.(m3u8|mp4|mkv|webm|avi|mov|flv|wmv|ts)(\?|$)/i;
+                            // 1. JWPlayer detection
+                            if (window.jwplayer) {
+                                try {
+                                    const players = document.querySelectorAll('.jwplayer, [id*="jwplayer"]');
+                                    players.forEach((_, idx) => {
+                                        try {
+                                            const player = window.jwplayer(idx);
+                                            if (player && player.getPlaylistItem) {
+                                                const item = player.getPlaylistItem();
+                                                if (item?.file) {
+                                                    videoLinks.push({ url: item.file, type: item.file.includes('.m3u8') ? 'm3u8' : 'mp4', source: 'jwplayer' });
+                                                }
+                                                if (item?.sources) {
+                                                    item.sources.forEach((s) => {
+                                                        if (s.file)
+                                                            videoLinks.push({ url: s.file, type: s.type || 'mp4', source: 'jwplayer' });
+                                                    });
+                                                }
+                                            }
+                                        }
+                                        catch { }
+                                    });
+                                }
+                                catch { }
+                            }
+                            // 2. DooPlayer detection (common in movie sites)
+                            if (window.dooPlayer || document.querySelector('[id*="doo"]')) {
+                                try {
+                                    const dooConfig = window.dooPlayer?.config || window.player_config;
+                                    if (dooConfig?.source) {
+                                        videoLinks.push({ url: dooConfig.source, type: 'm3u8', source: 'dooplayer' });
+                                    }
+                                }
+                                catch { }
+                            }
+                            // 3. Iframe video sources
+                            document.querySelectorAll('iframe').forEach(iframe => {
+                                const src = iframe.src || iframe.getAttribute('data-src') || '';
+                                if (src && (src.includes('embed') || src.includes('player') || src.includes('stream'))) {
+                                    videoLinks.push({ url: src, type: 'iframe', source: 'iframe' });
+                                }
+                            });
+                            // 4. Video tags
+                            document.querySelectorAll('video source, video').forEach(el => {
+                                const src = el.getAttribute('src') || el.src;
+                                if (src && videoPatterns.test(src)) {
+                                    const ext = src.match(videoPatterns)?.[1] || 'mp4';
+                                    videoLinks.push({ url: src, type: ext, source: 'video-tag' });
+                                }
+                            });
+                            // 5. Hidden links in scripts (ajax pattern)
+                            document.querySelectorAll('script:not([src])').forEach(script => {
+                                const content = script.textContent || '';
+                                // m3u8/mp4 in script
+                                const matches = content.match(/https?:\/\/[^\s"'<>]+\.(m3u8|mp4|mkv)[^\s"'<>]*/gi);
+                                if (matches) {
+                                    matches.forEach(url => {
+                                        const ext = url.match(videoPatterns)?.[1] || 'mp4';
+                                        videoLinks.push({ url, type: ext, source: 'ajax-script' });
+                                    });
+                                }
+                            });
+                            // 6. Data attributes with video URLs
+                            document.querySelectorAll('[data-file], [data-source], [data-video], [data-stream]').forEach(el => {
+                                const url = el.getAttribute('data-file') || el.getAttribute('data-source') ||
+                                    el.getAttribute('data-video') || el.getAttribute('data-stream');
+                                if (url && (videoPatterns.test(url) || url.includes('m3u8'))) {
+                                    videoLinks.push({ url, type: url.includes('m3u8') ? 'm3u8' : 'mp4', source: 'data-attr' });
+                                }
+                            });
+                            // Deduplicate
+                            return [...new Map(videoLinks.map(v => [v.url, v])).values()];
+                        });
+                    }
+                    // Follow links if enabled and depth allows
+                    if (args.followLinks !== false && depth < maxDepth && results.length < maxPages) {
+                        // Get all links
+                        const pageLinks = await crawlerPage.$$eval('a[href]', (anchors) => anchors.map(a => a.href).filter(href => href.startsWith('http')));
+                        result.links = pageLinks.slice(0, 100); // Limit stored links
+                        // Filter and enqueue links
+                        const linksToEnqueue = pageLinks.filter((link) => {
+                            if (visited.has(link))
+                                return false;
+                            if (includePattern && !includePattern.test(link))
+                                return false;
+                            if (excludePattern && excludePattern.test(link))
+                                return false;
+                            return true;
+                        });
+                        // Add filtered links using Crawlee's enqueueLinks
+                        for (const link of linksToEnqueue.slice(0, 50)) {
+                            try {
+                                await requestQueue.addRequest({
+                                    url: link,
+                                    userData: { depth: depth + 1 },
+                                });
+                            }
+                            catch (e) {
+                                // Link already in queue
+                            }
+                        }
+                    }
+                    // Download media if enabled
+                    if (args.downloadMedia && args.savePath) {
+                        const mediaUrls = await crawlerPage.$$eval('img[src], video source[src], a[href$=".pdf"], a[href$=".jpg"], a[href$=".png"]', (els) => els.map(el => el.getAttribute('src') || el.getAttribute('href')).filter(Boolean));
+                        result.extractedData = result.extractedData || {};
+                        result.extractedData.mediaUrls = mediaUrls;
+                    }
+                    results.push(result);
+                }
+                catch (error) {
+                    result.error = error instanceof Error ? error.message : String(error);
+                    errors.push(`${url}: ${result.error}`);
+                    results.push(result);
+                }
+            },
+            // Failed request handler
+            failedRequestHandler: async ({ request }, error) => {
+                errors.push(`Failed: ${request.url} - ${error.message}`);
+            },
+        });
+        // Run the crawler
+        await crawler.run();
+        return {
+            success: results.length > 0,
+            crawledPages: results.length,
+            results,
+            errors,
+            message: `🕷️ Crawled ${results.length} pages (depth: ${maxDepth}, errors: ${errors.length})`,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            crawledPages: results.length,
+            results,
+            errors: [...errors, error instanceof Error ? error.message : String(error)],
+            message: `❌ Crawler error: ${error instanceof Error ? error.message : String(error)}`,
+        };
+    }
+}

package/dist/index.js

@@ -61,7 +61,9 @@ import { handleBreadcrumbNavigator, handleUrlRedirectTracer, handleSearchContent
 // Download tools
 handleFileDownloader, 
 // Enhanced streaming/download tools
-handleIframeHandler, handleStreamExtractor, } from './handlers/advanced-tools.js';
+handleIframeHandler, handleStreamExtractor, 
+// Web crawler
+handleWebCrawler, } from './handlers/advanced-tools.js';
 // State for video recording
 const recorderState = new Map();
 debug('All modules loaded successfully');
@@ -254,6 +256,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 if (!page)
                     throw new Error('Browser not initialized. Call browser_init first.');
                 return { content: [{ type: 'text', text: JSON.stringify(await handleStreamExtractor(page, args)) }] };
+            // Web Crawler (Crawlee + brave-real-launcher)
+            case TOOL_NAMES.WEB_CRAWLER:
+                if (!page)
+                    throw new Error('Browser not initialized. Call browser_init first.');
+                return { content: [{ type: 'text', text: JSON.stringify(await handleWebCrawler(page, args)) }] };
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/dist/tool-definitions.js

@@ -622,6 +622,127 @@ export const TOOLS = [
             },
         },
     },
+    // ============================================================
+    // WEB CRAWLER TOOL (Movie Streaming Optimized)
+    // ============================================================
+    {
+        name: 'web_crawler',
+        description: 'Advanced web crawler optimized for movie downloading and streaming websites. Features: URL queue (breadth/depth-first), proxy rotation, auto-retry, rate limiting, JavaScript popup blocking, overlay ads blocking, and video link extraction. Uses brave-real-puppeteer-core with 50+ stealth features.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                startUrls: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Initial URLs to start crawling from (movie/streaming pages)'
+                },
+                maxDepth: {
+                    type: 'number',
+                    description: 'Maximum crawl depth (1 = only start URLs)',
+                    default: 3
+                },
+                maxPages: {
+                    type: 'number',
+                    description: 'Maximum pages to crawl',
+                    default: 50
+                },
+                concurrency: {
+                    type: 'number',
+                    description: 'Number of concurrent requests',
+                    default: 3
+                },
+                rateLimit: {
+                    type: 'number',
+                    description: 'Maximum requests per second',
+                    default: 2
+                },
+                crawlStrategy: {
+                    type: 'string',
+                    enum: ['breadth-first', 'depth-first'],
+                    description: 'URL queue strategy',
+                    default: 'breadth-first'
+                },
+                includePattern: {
+                    type: 'string',
+                    description: 'Regex pattern for URLs to include'
+                },
+                excludePattern: {
+                    type: 'string',
+                    description: 'Regex pattern for URLs to exclude'
+                },
+                extractSelectors: {
+                    type: 'object',
+                    description: 'CSS selectors for data extraction (e.g., {"title": "h1", "links": "a[href]"})'
+                },
+                followLinks: {
+                    type: 'boolean',
+                    description: 'Follow discovered links',
+                    default: true
+                },
+                // Movie streaming specific options
+                blockPopups: {
+                    type: 'boolean',
+                    description: 'Block JavaScript popup ads and window.open calls',
+                    default: true
+                },
+                blockOverlayAds: {
+                    type: 'boolean',
+                    description: 'Block overlay ads, modal popups, and floating elements',
+                    default: true
+                },
+                extractVideoLinks: {
+                    type: 'boolean',
+                    description: 'Auto-extract m3u8, mp4, mkv video links from pages',
+                    default: true
+                },
+                downloadMedia: {
+                    type: 'boolean',
+                    description: 'Download video/audio files',
+                    default: false
+                },
+                savePath: {
+                    type: 'string',
+                    description: 'Path to save downloaded files'
+                },
+                proxyList: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Proxy URLs for rotation (format: protocol://host:port)'
+                },
+                retryCount: {
+                    type: 'number',
+                    description: 'Number of retries for failed requests',
+                    default: 3
+                },
+                retryDelayMs: {
+                    type: 'number',
+                    description: 'Delay between retries in ms (exponential backoff)',
+                    default: 1000
+                },
+                timeout: {
+                    type: 'number',
+                    description: 'Request timeout in ms',
+                    default: 30000
+                },
+                mode: {
+                    type: 'string',
+                    enum: ['browser', 'http'],
+                    description: 'Crawl mode (browser = Puppeteer, http = fast HTTP)',
+                    default: 'browser'
+                },
+                userAgent: {
+                    type: 'string',
+                    description: 'Custom User-Agent string'
+                },
+                headers: {
+                    type: 'object',
+                    description: 'Custom headers for all requests'
+                },
+            },
+            required: ['startUrls'],
+        },
+    },
 ];
 // Tool name constants for type safety
 export const TOOL_NAMES = {
@@ -659,6 +780,8 @@ export const TOOL_NAMES = {
     // Enhanced tools
     IFRAME_HANDLER: 'iframe_handler',
     STREAM_EXTRACTOR: 'stream_extractor',
+    // Crawler tool
+    WEB_CRAWLER: 'web_crawler',
 };
 // Tool categories for organization
 export const TOOL_CATEGORIES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brave-real-browser-mcp-server",
-  "version": "2.24.5",
+  "version": "2.26.0",
   "description": "🦁 MCP server for Brave Real Browser - NPM Workspaces Monorepo with anti-detection features, SSE streaming, and LSP compatibility",
   "type": "module",
   "main": "dist/index.js",
@@ -50,7 +50,9 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "latest",
     "@types/turndown": "latest",
-    "brave-real-browser": "^2.5.5",
+    "brave-real-browser": "^2.7.0",
+    "crawlee": "^3.15.3",
+    "puppeteer-core": "^24.35.0",
     "turndown": "latest",
     "vscode-languageserver": "^9.0.1",
     "vscode-languageserver-textdocument": "^1.0.12"