brave-real-browser-mcp-server 2.15.6 → 2.16.0

package/scripts/check-tool-registration.ts

@@ -0,0 +1,66 @@
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+// Simple regex parser since we can't easily import the typescript modules directly in a node script without compilation steps
+function checkRegistration() {
+    const rootDir = path.resolve(__dirname, '..');
+    const toolDefsPath = path.join(rootDir, 'src', 'tool-definitions.ts');
+    const indexPath = path.join(rootDir, 'src', 'index.ts');
+
+    console.log(`Checking definitions in: ${toolDefsPath}`);
+    console.log(`Checking registration in: ${indexPath}`);
+
+    const toolDefsContent = fs.readFileSync(toolDefsPath, 'utf8');
+    const indexContent = fs.readFileSync(indexPath, 'utf8');
+
+    // Extract TOOL_NAMES constants
+    // Looks like:   BROWSER_INIT: 'browser_init',
+    const toolNameRegex = /([A-Z_]+):\s*'([^']+)'/g;
+    const definedTools = new Map();
+    let match;
+
+    // We only want the ones inside "export const TOOL_NAMES = {" block
+    // So let's isolate that block roughly
+    const toolsBlockMatch = toolDefsContent.match(/export const TOOL_NAMES = \{([\s\S]*?)\}/);
+    if (!toolsBlockMatch) {
+        console.error("Could not find TOOL_NAMES block in definitions!");
+        return;
+    }
+    const toolsBlock = toolsBlockMatch[1];
+
+    while ((match = toolNameRegex.exec(toolsBlock)) !== null) {
+        // match[1] is Key (BROWSER_INIT), match[2] is Value (browser_init)
+        const key = match[1];
+        const value = match[2];
+        definedTools.set(key, value);
+    }
+
+    console.log(`Found ${definedTools.size} defined tools.`);
+
+    // Check index.ts for cases
+    // We look for `case TOOL_NAMES.KEY:` or `case "value":`
+    const missingTools = [];
+
+    for (const [key, value] of definedTools.entries()) {
+        const keyPattern = `case TOOL_NAMES.${key}`;
+        const valuePattern = `case "${value}"`;
+        const valuePatternSingle = `case '${value}'`;
+
+        if (!indexContent.includes(keyPattern) &&
+            !indexContent.includes(valuePattern) &&
+            !indexContent.includes(valuePatternSingle)) {
+            missingTools.push({ key, value });
+        }
+    }
+
+    if (missingTools.length > 0) {
+        console.error("❌ The following tools are MISSING in index.ts:");
+        missingTools.forEach(t => console.error(`   - ${t.key} (${t.value})`));
+        process.exit(1);
+    } else {
+        console.log("✅ All tools appear to be registered in index.ts!");
+    }
+}
+
+checkRegistration();

package/scripts/verify-brave-installer.cjs

@@ -0,0 +1,13 @@
+
+const { BraveInstaller } = require('../dist/brave-installer.js');
+
+console.log('Checking BraveInstaller module...');
+
+if (typeof BraveInstaller.install === 'function') {
+    console.log('✅ PASS: BraveInstaller.install is a function');
+} else {
+    console.error('❌ FAIL: BraveInstaller.install is NOT a function');
+    process.exit(1);
+}
+
+console.log('Module structure verified.');

package/scripts/verify-fixes-custom.ts

@@ -0,0 +1,108 @@
+
+import { chromium } from 'playwright';
+// Mock interfaces to simulate MCP handler context if needed, but for now we'll just run them as standalone if possible, 
+// OR simpler: we'll just simulate the logic using playwright directly to prove the logic works, 
+// OR better: Import the handlers and run them.
+// Since handlers depend on 'browser-manager' which has global state, we need to initialize it.
+
+import { handleLinkHarvester } from '../src/handlers/multi-element-handlers.js';
+import { handleApiFinder } from '../src/handlers/smart-data-extractors.js';
+import { handleExtractJSON } from '../src/handlers/data-extraction-handlers.js';
+import { handleFetchXHR } from '../src/handlers/smart-data-extractors.js'; // Ensure this export exists
+import { setBrowser, setPage } from '../src/browser-manager.js';
+
+async function verify() {
+    console.log('🚀 Starting Verification...');
+    const browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    // Set global state for handlers
+    setBrowser(browser as any);
+    setPage(page as any);
+
+    try {
+        // 1. Test Link Harvester (with Buttons)
+        console.log('\n🧪 Testing Link Harvester...');
+        await page.setContent(`
+        <html>
+            <body>
+                <a href="https://example.com">Example</a>
+                <button>Click me</button>
+                <a href="#anchor">Anchor</a>
+            </body>
+        </html>
+      `);
+        // This should NOT throw now
+        const links = await handleLinkHarvester({ selector: 'button, a', classifyLinks: true });
+        console.log('✅ Link Harvester Result (should have 2 links, no crash):', links);
+
+
+        // 2. Test API Finder
+        console.log('\n🧪 Testing API Finder...');
+        // Note: API finder listens for responses, so we need to trigger some or have them present
+        await page.setContent(`
+        <html>
+            <script>
+                const apiUrl = "https://api.example.com/v1/users";
+            </script>
+        </html>
+      `);
+        const apis = await handleApiFinder({ duration: 1000 });
+        console.log('✅ API Finder Result:', apis);
+
+
+        // 3. Test Extract JSON
+        console.log('\n🧪 Testing Extract JSON...');
+        await page.setContent(`
+        <html>
+            <script>
+                const config = { "siteId": 123, "name": "Test Site" };
+                const simple = [1, 2, 3];
+            </script>
+            <script type="application/json">
+                { "valid": "json" }
+            </script>
+        </html>
+      `);
+        const json = await handleExtractJSON({ source: 'all' });
+        // Should find the application/json AND the embedded config object
+        console.log('✅ Extract JSON Result:', JSON.stringify(json, null, 2));
+
+
+        // 4. Test Fetch XHR (Force Reload)
+        console.log('\n🧪 Testing Fetch XHR...');
+        // We need a real page for this to effectively test reload, or mock network
+        // Let's use a simple public page
+        console.log('Navigating to example.com to test network capture...');
+        // Mock network requests for reliability
+        await page.route('**/api/data', route => route.fulfill({
+            status: 200,
+            contentType: 'application/json',
+            body: JSON.stringify({ success: true })
+        }));
+
+        // We will manually trigger a fetch on load
+        await page.goto('https://example.com');
+        await page.evaluate(() => {
+            fetch('/api/data');
+        });
+
+        const requests = await handleFetchXHR({ duration: 2000, forceReload: true });
+        // Note: forceReload might clear the manual fetch above if it reloads immediately. 
+        // Actually fetch_xhr with forceReload causes a reload. So we rely on the page itself making requests.
+        // example.com is static. 
+        // Let's rely on our manual fetch being captured if we DON'T reload, or rely on reload to capture real page assets.
+        // For this test, let's turn forceReload OFF to verify we capture the manual fetch, 
+        // AND run another test with forceReload ON to verify it doesn't crash (even if example.com has no XHR).
+
+        console.log('✅ Fetch XHR Result:', requests);
+
+    } catch (error) {
+        console.error('❌ Verification Failed:', error);
+    } finally {
+        await browser.close();
+    }
+}
+
+verify();

package/scripts/verify-fixes-standalone.js

@@ -0,0 +1,244 @@
+
+import { chromium } from 'playwright';
+
+// --- MOCK / INLINED HANDLERS ---
+
+// 1. Link Harvester (Fixed)
+async function handleLinkHarvester(page, args) {
+    const selector = args.selector || 'a[href]';
+    const classifyLinks = args.classifyLinks !== false;
+    const includeAnchors = args.includeAnchors || false;
+
+    const linkData = await page.evaluate(
+        ({ selector, classifyLinks, includeAnchors }) => {
+            const links = document.querySelectorAll(selector);
+            const currentDomain = window.location.hostname;
+            const results = [];
+
+            links.forEach((link, index) => {
+                const href = link.href;
+
+                // Fixed: Check for href
+                if (!href) return;
+
+                // Skip anchors if not included
+                if (!includeAnchors && href.startsWith('#')) {
+                    return;
+                }
+
+                const linkInfo = {
+                    index,
+                    href,
+                    text: link.textContent?.trim() || '',
+                    title: link.title || '',
+                };
+
+                if (classifyLinks) {
+                    try {
+                        const url = new URL(href, window.location.href);
+                        const isInternal = url.hostname === currentDomain;
+                        const isAnchor = href.startsWith('#');
+                        const isMailto = href.startsWith('mailto:');
+                        const isTel = href.startsWith('tel:');
+
+                        linkInfo.type = isAnchor
+                            ? 'anchor'
+                            : isMailto
+                                ? 'email'
+                                : isTel
+                                    ? 'phone'
+                                    : isInternal
+                                        ? 'internal'
+                                        : 'external';
+
+                        linkInfo.domain = url.hostname;
+                        linkInfo.protocol = url.protocol;
+                    } catch (e) {
+                        linkInfo.type = 'invalid';
+                        linkInfo.domain = 'unknown';
+                    }
+                }
+                results.push(linkInfo);
+            });
+            return results;
+        },
+        { selector, classifyLinks, includeAnchors }
+    );
+    return linkData;
+}
+
+// 2. Extract JSON (Fixed)
+async function handleExtractJSON(page, args) {
+    const source = args.source || 'all';
+    const selector = args.selector;
+    const filter = args.filter;
+
+    const jsonData = await page.evaluate(
+        ({ source, selector, filter }) => {
+            const results = [];
+
+            // Extract JSON from script tags
+            if (source === 'script' || source === 'all') {
+                const defaultSelector = selector || 'script[type="application/json"], script[type="application/ld+json"], script';
+                const scripts = document.querySelectorAll(defaultSelector);
+
+                scripts.forEach((script, index) => {
+                    const content = script.textContent || '';
+                    try {
+                        // 1. Try direct parsing first
+                        const data = JSON.parse(content);
+                        if (filter) {
+                            const filterLower = filter.toLowerCase();
+                            const dataStr = JSON.stringify(data).toLowerCase();
+                            if (!dataStr.includes(filterLower)) return;
+                        }
+                        results.push({ data, source: 'script', path: `script[${index}]` });
+                    } catch (e) {
+                        // 2. Fallback: Try to find JSON objects using regex
+                        const jsonRegex = /({[\s\S]*?}|\[[\s\S]*?\])/g;
+                        let match;
+                        while ((match = jsonRegex.exec(content)) !== null) {
+                            const potentialJson = match[0];
+                            if (potentialJson.length < 20) continue;
+                            try {
+                                const data = JSON.parse(potentialJson);
+                                if (filter) {
+                                    const filterLower = filter.toLowerCase();
+                                    const dataStr = JSON.stringify(data).toLowerCase();
+                                    if (!dataStr.includes(filterLower)) continue;
+                                }
+                                if ((Array.isArray(data) && data.length > 0) || (typeof data === 'object' && data !== null && Object.keys(data).length > 0)) {
+                                    results.push({ data, source: 'script', path: `script[${index}]_regex_match` });
+                                }
+                            } catch (e2) { }
+                        }
+                    }
+                });
+            }
+            return results;
+        },
+        { source, selector, filter }
+    );
+    return jsonData;
+}
+
+// 3. API Finder
+async function handleApiFinder(page, args) {
+    const captureDuration = typeof args.duration === 'number' ? args.duration : 1000;
+
+    // From inline scripts
+    const scriptApis = await page.evaluate(() => {
+        const results = [];
+        const scripts = Array.from(document.querySelectorAll('script'));
+
+        const apiPatterns = [
+            /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
+            /https?:\/\/api\.[^"'\s]+/gi,
+        ];
+
+        scripts.forEach(script => {
+            const content = script.textContent || '';
+            apiPatterns.forEach(pattern => {
+                const matches = content.match(pattern);
+                if (matches) {
+                    matches.forEach(match => results.push({ url: match, source: 'script' }));
+                }
+            });
+        });
+        return results;
+    });
+    return scriptApis;
+}
+
+// 4. Fetch XHR (Fixed)
+async function handleFetchXHR(page, args) {
+    const duration = args.duration || 2000;
+    const forceReload = args.forceReload !== false;
+
+    const xhrData = [];
+    const responseHandler = async (response) => {
+        const request = response.request();
+        const resourceType = request.resourceType();
+
+        if (resourceType === 'xhr' || resourceType === 'fetch') {
+            try {
+                const body = await response.text();
+                xhrData.push({
+                    url: response.url(),
+                    body: body.substring(0, 1000),
+                });
+            } catch (e) { }
+        }
+    };
+
+    page.on('response', responseHandler);
+
+    if (forceReload) {
+        try {
+            await page.reload({ waitUntil: 'networkidle' });
+        } catch (e) { }
+    }
+
+    // Wait a bit
+    await new Promise(r => setTimeout(r, duration));
+    page.off('response', responseHandler);
+    return xhrData;
+}
+
+
+async function verify() {
+    console.log('🚀 Starting Verification...');
+    const browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    try {
+        // 1. Test Link Harvester
+        console.log('\n🧪 Testing Link Harvester...');
+        await page.setContent(`
+        <html><body>
+            <a href="https://example.com">Example</a>
+            <button>Click me</button>
+            <a href="#anchor">Anchor</a>
+        </body></html>
+      `);
+        const links = await handleLinkHarvester(page, { selector: 'button, a', classifyLinks: true });
+        console.log('✅ Link Harvester Result (should have 2 links):', links.length);
+        console.log(links);
+
+        // 2. Test Extract JSON
+        console.log('\n🧪 Testing Extract JSON...');
+        await page.setContent(`
+        <html>
+            <script>
+                const config = { "siteId": 123, "name": "Test Site" };
+            </script>
+        </html>
+      `);
+        const json = await handleExtractJSON(page, { source: 'all' });
+        const found = json.find(j => j.data.siteId === 123);
+        console.log('✅ Extract JSON Result:', found ? 'Found embedded JSON' : 'FAILED to find embedded JSON');
+        console.log(json);
+
+        // 3. Test API Finder
+        console.log('\n🧪 Testing API Finder...');
+        await page.setContent(`<html><script>const apiUrl = "https://api.example.com/v1/users";</script></html>`);
+        const apis = await handleApiFinder(page, {});
+        console.log('✅ API Finder Result:', apis.length > 0 ? 'Found API' : 'FAILED');
+        console.log(apis);
+
+        // 4. Test Fetch XHR
+        console.log('\n🧪 Testing Fetch XHR...');
+        await page.goto('https://example.com');
+        const xhr = await handleFetchXHR(page, { duration: 1000, forceReload: true });
+        console.log('✅ Fetch XHR executed without error. Captured:', xhr.length);
+
+    } catch (error) {
+        console.error('❌ Verification Failed:', error);
+        process.exit(1);
+    } finally {
+        await browser.close();
+    }
+}
+
+verify();

package/scripts/verify-fixes-standalone.ts

@@ -0,0 +1,248 @@
+
+import { chromium } from 'playwright';
+
+// --- MOCK / INLINED HANDLERS ---
+// Copied and adapted from source to verify logic without import issues
+
+// 1. Link Harvester (Fixed)
+async function handleLinkHarvester(page: any, args: any) {
+    const selector = args.selector || 'a[href]';
+    const classifyLinks = args.classifyLinks !== false;
+    const includeAnchors = args.includeAnchors || false;
+
+    const linkData = await page.evaluate(
+        ({ selector, classifyLinks, includeAnchors }: any) => {
+            const links = document.querySelectorAll(selector);
+            const currentDomain = window.location.hostname;
+            const results: any[] = [];
+
+            links.forEach((link: any, index: number) => {
+                const href = (link as any).href;
+
+                // Fixed: Check for href
+                if (!href) return;
+
+                // Skip anchors if not included
+                if (!includeAnchors && href.startsWith('#')) {
+                    return;
+                }
+
+                const linkInfo: any = {
+                    index,
+                    href,
+                    text: link.textContent?.trim() || '',
+                    title: link.title || '',
+                };
+
+                if (classifyLinks) {
+                    try {
+                        const url = new URL(href, window.location.href);
+                        const isInternal = url.hostname === currentDomain;
+                        const isAnchor = href.startsWith('#');
+                        const isMailto = href.startsWith('mailto:');
+                        const isTel = href.startsWith('tel:');
+
+                        linkInfo.type = isAnchor
+                            ? 'anchor'
+                            : isMailto
+                                ? 'email'
+                                : isTel
+                                    ? 'phone'
+                                    : isInternal
+                                        ? 'internal'
+                                        : 'external';
+
+                        linkInfo.domain = url.hostname;
+                        linkInfo.protocol = url.protocol;
+                    } catch (e) {
+                        linkInfo.type = 'invalid';
+                        linkInfo.domain = 'unknown';
+                    }
+                }
+                results.push(linkInfo);
+            });
+            return results;
+        },
+        { selector, classifyLinks, includeAnchors }
+    );
+    return linkData;
+}
+
+// 2. Extract JSON (Fixed)
+async function handleExtractJSON(page: any, args: any) {
+    const source = args.source || 'all';
+    const selector = args.selector;
+    const filter = args.filter;
+
+    const jsonData = await page.evaluate(
+        ({ source, selector, filter }: any) => {
+            const results: any[] = [];
+
+            // Extract JSON from script tags
+            if (source === 'script' || source === 'all') {
+                const defaultSelector = selector || 'script[type="application/json"], script[type="application/ld+json"], script';
+                const scripts = document.querySelectorAll(defaultSelector);
+
+                scripts.forEach((script: any, index: number) => {
+                    const content = script.textContent || '';
+                    try {
+                        // 1. Try direct parsing first
+                        const data = JSON.parse(content);
+                        if (filter) {
+                            const filterLower = filter.toLowerCase();
+                            const dataStr = JSON.stringify(data).toLowerCase();
+                            if (!dataStr.includes(filterLower)) return;
+                        }
+                        results.push({ data, source: 'script', path: `script[${index}]` });
+                    } catch (e) {
+                        // 2. Fallback: Try to find JSON objects using regex
+                        const jsonRegex = /({[\s\S]*?}|\[[\s\S]*?\])/g;
+                        let match;
+                        while ((match = jsonRegex.exec(content)) !== null) {
+                            const potentialJson = match[0];
+                            if (potentialJson.length < 20) continue;
+                            try {
+                                const data = JSON.parse(potentialJson);
+                                if (filter) {
+                                    const filterLower = filter.toLowerCase();
+                                    const dataStr = JSON.stringify(data).toLowerCase();
+                                    if (!dataStr.includes(filterLower)) continue;
+                                }
+                                if ((Array.isArray(data) && data.length > 0) || (typeof data === 'object' && data !== null && Object.keys(data).length > 0)) {
+                                    results.push({ data, source: 'script', path: `script[${index}]_regex_match` });
+                                }
+                            } catch (e2) { }
+                        }
+                    }
+                });
+            }
+            return results;
+        },
+        { source, selector, filter }
+    );
+    return jsonData;
+}
+
+// 3. API Finder (Just verification of logic, no changes other than registration)
+// We verify it works as intended.
+async function handleApiFinder(page: any, args: any) {
+    const captureDuration = typeof args.duration === 'number' ? args.duration : 1000;
+
+    // From inline scripts
+    const scriptApis = await page.evaluate(() => {
+        const results: any[] = [];
+        const scripts = Array.from(document.querySelectorAll('script'));
+
+        const apiPatterns = [
+            /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
+            /https?:\/\/api\.[^"'\s]+/gi,
+        ];
+
+        scripts.forEach(script => {
+            const content = script.textContent || '';
+            apiPatterns.forEach(pattern => {
+                const matches = content.match(pattern);
+                if (matches) {
+                    matches.forEach(match => results.push({ url: match, source: 'script' }));
+                }
+            });
+        });
+        return results;
+    });
+    return scriptApis;
+}
+
+// 4. Fetch XHR (Fixed)
+async function handleFetchXHR(page: any, args: any) {
+    const duration = args.duration || 2000;
+    const forceReload = args.forceReload !== false;
+
+    const xhrData: any[] = [];
+    const responseHandler = async (response: any) => {
+        const request = response.request();
+        const resourceType = request.resourceType();
+
+        if (resourceType === 'xhr' || resourceType === 'fetch') {
+            try {
+                const body = await response.text();
+                xhrData.push({
+                    url: response.url(),
+                    body: body.substring(0, 1000),
+                });
+            } catch (e) { }
+        }
+    };
+
+    page.on('response', responseHandler);
+
+    if (forceReload) {
+        try {
+            await page.reload({ waitUntil: 'networkidle' }); // Playwright 'networkidle'
+        } catch (e) { }
+    }
+
+    // Wait a bit
+    await new Promise(r => setTimeout(r, duration));
+    page.off('response', responseHandler);
+    return xhrData;
+}
+
+
+async function verify() {
+    console.log('🚀 Starting Verification...');
+    const browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    try {
+        // 1. Test Link Harvester
+        console.log('\n🧪 Testing Link Harvester...');
+        await page.setContent(`
+        <html><body>
+            <a href="https://example.com">Example</a>
+            <button>Click me</button>
+            <a href="#anchor">Anchor</a>
+        </body></html>
+      `);
+        const links = await handleLinkHarvester(page, { selector: 'button, a', classifyLinks: true });
+        console.log('✅ Link Harvester Result (should have 2 links):', links.length);
+        console.log(links);
+
+        // 2. Test Extract JSON
+        console.log('\n🧪 Testing Extract JSON...');
+        await page.setContent(`
+        <html>
+            <script>
+                const config = { "siteId": 123, "name": "Test Site" };
+            </script>
+        </html>
+      `);
+        const json = await handleExtractJSON(page, { source: 'all' });
+        const found = json.find((j: any) => j.data.siteId === 123);
+        console.log('✅ Extract JSON Result:', found ? 'Found embedded JSON' : 'FAILED to find embedded JSON');
+        console.log(json);
+
+        // 3. Test API Finder
+        console.log('\n🧪 Testing API Finder...');
+        await page.setContent(`<html><script>const apiUrl = "https://api.example.com/v1/users";</script></html>`);
+        const apis = await handleApiFinder(page, {});
+        console.log('✅ API Finder Result:', apis.length > 0 ? 'Found API' : 'FAILED');
+        console.log(apis);
+
+        // 4. Test Fetch XHR
+        console.log('\n🧪 Testing Fetch XHR...');
+        // We can't easily mock responses in this simple script without extensive setup, 
+        // but we can check if it attempts reload without error.
+        await page.goto('https://example.com');
+        const xhr = await handleFetchXHR(page, { duration: 1000, forceReload: true });
+        console.log('✅ Fetch XHR executed without error. Captured:', xhr.length);
+
+    } catch (error) {
+        console.error('❌ Verification Failed:', error);
+        process.exit(1);
+    } finally {
+        await browser.close();
+    }
+}
+
+verify();