brave-real-browser-mcp-server 2.27.4 → 2.27.6

package/dist/handlers/advanced-tools.js

@@ -2876,3 +2876,135 @@ export async function handleStreamExtractor(page, args) {
             : 'No direct URLs found',
     };
 }
+/**
+ * JS Scrape - Single-call JavaScript-rendered content extraction
+ * Combines navigation, waiting, scrolling, and content extraction in one call
+ * Perfect for scraping dynamic/AJAX-loaded content
+ */
+export async function handleJsScrape(page, args) {
+    const waitForTimeout = args.waitForTimeout || 10000;
+    const returnType = args.returnType || 'html';
+    try {
+        // Step 1: Navigate to URL
+        await page.goto(args.url, {
+            waitUntil: 'domcontentloaded',
+            timeout: waitForTimeout
+        });
+        // Step 2: Wait for specific selector if provided
+        if (args.waitForSelector) {
+            try {
+                await page.waitForSelector(args.waitForSelector, {
+                    timeout: waitForTimeout,
+                    visible: true
+                });
+            }
+            catch (e) {
+                // Continue even if selector not found - page might still have content
+            }
+        }
+        // Step 3: Scroll to trigger lazy loading if requested
+        if (args.scrollToLoad !== false) {
+            await page.evaluate(async () => {
+                const scrollStep = window.innerHeight;
+                const scrollDelay = 200;
+                let totalScrolled = 0;
+                const maxScroll = document.body.scrollHeight;
+                while (totalScrolled < maxScroll && totalScrolled < 5000) {
+                    window.scrollBy(0, scrollStep);
+                    totalScrolled += scrollStep;
+                    await new Promise(r => setTimeout(r, scrollDelay));
+                }
+                // Scroll back to top
+                window.scrollTo(0, 0);
+            });
+            // Wait for any lazy-loaded content
+            await new Promise(r => setTimeout(r, 1000));
+        }
+        // Step 4: Wait additional time for JavaScript to execute
+        await new Promise(r => setTimeout(r, 500));
+        // Step 5: Extract content based on returnType
+        const title = await page.title();
+        const finalUrl = page.url();
+        let html;
+        let text;
+        let elements;
+        let elementCount = 0;
+        if (args.extractSelector) {
+            // Extract specific elements
+            const extractedData = await page.evaluate((selector, attrs) => {
+                const nodeList = document.querySelectorAll(selector);
+                const result = [];
+                nodeList.forEach((el) => {
+                    const attrObj = {};
+                    (attrs || ['href', 'src', 'alt', 'title', 'data-*']).forEach(attrName => {
+                        if (attrName.endsWith('*')) {
+                            // Handle wildcard attributes like data-*
+                            const prefix = attrName.slice(0, -1);
+                            Array.from(el.attributes).forEach(attr => {
+                                if (attr.name.startsWith(prefix)) {
+                                    attrObj[attr.name] = attr.value;
+                                }
+                            });
+                        }
+                        else {
+                            const val = el.getAttribute(attrName);
+                            if (val)
+                                attrObj[attrName] = val;
+                        }
+                    });
+                    result.push({
+                        tag: el.tagName.toLowerCase(),
+                        text: el.textContent?.trim()?.substring(0, 500) || '',
+                        html: el.outerHTML.substring(0, 2000),
+                        attributes: attrObj
+                    });
+                });
+                return result;
+            }, args.extractSelector, args.extractAttributes || []);
+            elementCount = extractedData.length;
+            if (returnType === 'elements') {
+                elements = extractedData.map(e => ({
+                    tag: e.tag,
+                    text: e.text,
+                    attributes: e.attributes
+                }));
+            }
+            else if (returnType === 'html') {
+                html = extractedData.map(e => e.html).join('\n');
+            }
+            else {
+                text = extractedData.map(e => e.text).join('\n');
+            }
+        }
+        else {
+            // Extract full page content
+            if (returnType === 'html') {
+                html = await page.evaluate(() => document.documentElement.outerHTML);
+            }
+            else {
+                text = await page.evaluate(() => document.body?.textContent || '');
+            }
+            elementCount = 1;
+        }
+        return {
+            success: true,
+            url: args.url,
+            finalUrl,
+            title,
+            html,
+            text,
+            elements,
+            elementCount
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            url: args.url,
+            finalUrl: page.url() || args.url,
+            title: '',
+            elementCount: 0,
+            error: error instanceof Error ? error.message : String(error)
+        };
+    }
+}

package/dist/index.js

@@ -61,7 +61,7 @@ import { handleBreadcrumbNavigator, handleUrlRedirectTracer, handleSearchContent
 // Download tools
 handleFileDownloader, 
 // Enhanced streaming/download tools
-handleIframeHandler, handleStreamExtractor, } from './handlers/advanced-tools.js';
+handleIframeHandler, handleStreamExtractor, handleJsScrape, } from './handlers/advanced-tools.js';
 // State for video recording
 const recorderState = new Map();
 debug('All modules loaded successfully');
@@ -256,6 +256,10 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 if (!page)
                     throw new Error('Browser not initialized. Call browser_init first.');
                 return { content: [{ type: 'text', text: JSON.stringify(await handleStreamExtractor(page, args)) }] };
+            case TOOL_NAMES.JS_SCRAPE:
+                if (!page)
+                    throw new Error('Browser not initialized. Call browser_init first.');
+                return { content: [{ type: 'text', text: JSON.stringify(await handleJsScrape(page, args)) }] };
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/dist/tool-definitions.js

@@ -761,6 +761,25 @@ export const TOOLS = [
             },
         },
     },
+    {
+        name: 'js_scrape',
+        description: 'Single-call JavaScript-rendered content extraction. Combines navigation, auto-wait, scrolling, and content extraction. Perfect for AJAX/dynamic pages that Jsoup cannot parse.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                url: { type: 'string', description: 'URL to scrape (required)' },
+                waitForSelector: { type: 'string', description: 'CSS selector to wait for before extracting content' },
+                waitForTimeout: { type: 'number', description: 'Maximum wait time in ms', default: 10000 },
+                extractSelector: { type: 'string', description: 'CSS selector for specific elements to extract (optional, extracts full page if not specified)' },
+                extractAttributes: { type: 'array', items: { type: 'string' }, description: 'Attributes to extract from elements (e.g., href, src, data-*)' },
+                returnType: { type: 'string', enum: ['html', 'text', 'elements'], description: 'Return format', default: 'html' },
+                scrollToLoad: { type: 'boolean', description: 'Scroll page to trigger lazy loading', default: true },
+                closeBrowserAfter: { type: 'boolean', description: 'Close browser after scraping', default: false },
+            },
+            required: ['url'],
+        },
+    },
 ];
 // Tool name constants for type safety
 export const TOOL_NAMES = {
@@ -799,6 +818,7 @@ export const TOOL_NAMES = {
     // Enhanced tools
     IFRAME_HANDLER: 'iframe_handler',
     STREAM_EXTRACTOR: 'stream_extractor',
+    JS_SCRAPE: 'js_scrape',
 };
 // Tool categories for organization
 export const TOOL_CATEGORIES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brave-real-browser-mcp-server",
-  "version": "2.27.4",
+  "version": "2.27.6",
   "description": "🦁 MCP server for Brave Real Browser - NPM Workspaces Monorepo with anti-detection features, SSE streaming, and LSP compatibility",
   "type": "module",
   "main": "dist/index.js",
@@ -50,7 +50,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "latest",
     "@types/turndown": "latest",
-    "brave-real-browser": "^2.8.4",
+    "brave-real-browser": "^2.8.6",
     "puppeteer-core": "^24.35.0",
     "turndown": "latest",
     "vscode-languageserver": "^9.0.1",