brave-real-browser-mcp-server 2.17.1 → 2.17.3

package/dist/index.js

@@ -1,25 +1,14 @@
 #!/usr/bin/env node
-// Debug logging setup - Log process start
 import 'dotenv/config';
-console.error(`🔍 [DEBUG] Process starting - PID: ${process.pid}, Node: ${process.version}, Platform: ${process.platform}`);
-console.error(`🔍 [DEBUG] Working directory: ${process.cwd()}`);
-console.error(`🔍 [DEBUG] Command args: ${process.argv.join(" ")}`);
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ListPromptsRequestSchema, InitializeRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
-console.error("🔍 [DEBUG] MCP SDK imports completed successfully");
-// Import extracted modules
-console.error("🔍 [DEBUG] Loading tool definitions...");
 import { TOOLS, SERVER_INFO, CAPABILITIES, TOOL_NAMES, } from "./tool-definitions.js";
-console.error("🔍 [DEBUG] Loading system utils...");
 import { withErrorHandling } from "./system-utils.js";
 import { validateMCPResponse } from "./mcp-response-validator.js";
-console.error("🔍 [DEBUG] Loading browser manager...");
 import { closeBrowser, forceKillAllBraveProcesses } from "./browser-manager.js";
-console.error("🔍 [DEBUG] Loading core infrastructure...");
 import { setupProcessCleanup, } from "./core-infrastructure.js";
 // Import handlers
-console.error("🔍 [DEBUG] Loading handlers...");
 import { handleBrowserInit, handleBrowserClose, } from "./handlers/browser-handlers.js";
 import { handleNavigate, handleWait } from "./handlers/navigation-handlers.js";
 import { handleClick, handleType, handleSolveCaptcha, handleRandomScroll, handlePressKey, } from "./handlers/interaction-handlers.js";
@@ -53,7 +42,6 @@ import { handleVideoPlayerFinder, handleStreamDetector, handleVideoDownloadLinkF
 import { handleAdvancedVideoExtraction, handleDeobfuscateJS, handleMultiLayerRedirectTrace, handleAdProtectionDetector, } from "./handlers/advanced-extraction-handlers.js";
 console.error("🔍 [DEBUG] All modules loaded successfully");
 console.error(`🔍 [DEBUG] Server info: ${JSON.stringify(SERVER_INFO)}`);
-console.error(`🔍 [DEBUG] Available tools: ${TOOLS.length} tools loaded`);
 // Initialize MCP server
 console.error("🔍 [DEBUG] Creating MCP server instance...");
 const server = new Server(SERVER_INFO, { capabilities: CAPABILITIES });

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "brave-real-browser-mcp-server",
-  "version": "2.17.1",
+  "version": "2.17.3",
   "description": "Universal AI IDE MCP Server - Auto-detects and supports all AI IDEs (Claude Desktop, Cursor, Windsurf, Cline, Zed, VSCode, Qoder AI, etc.) with Brave browser automation",
   "type": "module",
   "main": "dist/index.js",
   "scripts": {
     "preinstall": "npm view brave-real-puppeteer-core version > .latest-version 2>&1 || echo 'Version check skipped'",
-    "postinstall": "node scripts/patch-puppeteer-screen-recorder.cjs && node scripts/update-to-latest.cjs || echo 'Auto-update skipped'",
+    "postinstall": "node scripts/patch-puppeteer-screen-recorder.cjs",
     "clean": "rimraf dist",
     "clean:cache": "npm cache clean --force",
     "fix-cache-permissions": "echo 'Run: sudo chown -R $(whoami):$(id -gn) ~/.npm' && echo 'This fixes npm cache permission issues'",

package/scripts/check-tool-registration.ts

@@ -1,66 +0,0 @@
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-// Simple regex parser since we can't easily import the typescript modules directly in a node script without compilation steps
-function checkRegistration() {
-    const rootDir = path.resolve(__dirname, '..');
-    const toolDefsPath = path.join(rootDir, 'src', 'tool-definitions.ts');
-    const indexPath = path.join(rootDir, 'src', 'index.ts');
-
-    console.log(`Checking definitions in: ${toolDefsPath}`);
-    console.log(`Checking registration in: ${indexPath}`);
-
-    const toolDefsContent = fs.readFileSync(toolDefsPath, 'utf8');
-    const indexContent = fs.readFileSync(indexPath, 'utf8');
-
-    // Extract TOOL_NAMES constants
-    // Looks like:   BROWSER_INIT: 'browser_init',
-    const toolNameRegex = /([A-Z_]+):\s*'([^']+)'/g;
-    const definedTools = new Map();
-    let match;
-
-    // We only want the ones inside "export const TOOL_NAMES = {" block
-    // So let's isolate that block roughly
-    const toolsBlockMatch = toolDefsContent.match(/export const TOOL_NAMES = \{([\s\S]*?)\}/);
-    if (!toolsBlockMatch) {
-        console.error("Could not find TOOL_NAMES block in definitions!");
-        return;
-    }
-    const toolsBlock = toolsBlockMatch[1];
-
-    while ((match = toolNameRegex.exec(toolsBlock)) !== null) {
-        // match[1] is Key (BROWSER_INIT), match[2] is Value (browser_init)
-        const key = match[1];
-        const value = match[2];
-        definedTools.set(key, value);
-    }
-
-    console.log(`Found ${definedTools.size} defined tools.`);
-
-    // Check index.ts for cases
-    // We look for `case TOOL_NAMES.KEY:` or `case "value":`
-    const missingTools = [];
-
-    for (const [key, value] of definedTools.entries()) {
-        const keyPattern = `case TOOL_NAMES.${key}`;
-        const valuePattern = `case "${value}"`;
-        const valuePatternSingle = `case '${value}'`;
-
-        if (!indexContent.includes(keyPattern) &&
-            !indexContent.includes(valuePattern) &&
-            !indexContent.includes(valuePatternSingle)) {
-            missingTools.push({ key, value });
-        }
-    }
-
-    if (missingTools.length > 0) {
-        console.error("❌ The following tools are MISSING in index.ts:");
-        missingTools.forEach(t => console.error(`   - ${t.key} (${t.value})`));
-        process.exit(1);
-    } else {
-        console.log("✅ All tools appear to be registered in index.ts!");
-    }
-}
-
-checkRegistration();

package/scripts/full-verification.ts

@@ -1,98 +0,0 @@
-
-import { handleBrowserInit, handleBrowserClose } from '../src/handlers/browser-handlers.js';
-import { handleNavigate, handleWait } from '../src/handlers/navigation-handlers.js';
-import { handleGetContent, handleFindSelector } from '../src/handlers/content-handlers.js';
-import { handleBreadcrumbNavigator } from '../src/handlers/navigation-handlers.js';
-import {
-    handleBatchElementScraper,
-    handleAttributeHarvester,
-    handleLinkHarvester,
-    handleMediaExtractor
-} from '../src/handlers/multi-element-handlers.js';
-import {
-    handleKeywordSearch,
-    handleRegexPatternMatcher,
-    handleXPathSupport,
-    handleAdvancedCSSSelectors
-} from '../src/handlers/search-filter-handlers.js';
-import { handleRandomScroll } from '../src/handlers/interaction-handlers.js';
-import { handleScrapeMetaTags, handleExtractSchema } from '../src/handlers/data-extraction-handlers.js';
-
-async function runFullVerification() {
-    console.log('🚀 Starting Comprehensive Tool Verification...');
-
-    try {
-        await handleBrowserInit({ headless: true });
-
-        // Testing on one site primarily to save time, then brief check on second
-        const url = 'https://moviesdrive.forum/';
-        console.log(`\n--------------------------------------------------`);
-        console.log(`🔍 Targeting: ${url}`);
-        console.log(`--------------------------------------------------`);
-
-        // --- Navigation & Basic ---
-        console.log(`\n[1/12] 🟢 Testing Navigation & Wait...`);
-        await handleNavigate({ url });
-        await handleWait({ type: 'timeout', value: '2000' });
-        console.log('   ✅ Navigation complete.');
-
-        // --- Interaction ---
-        console.log(`\n[2/12] 🟢 Testing Random Scroll...`);
-        await handleRandomScroll({});
-        console.log('   ✅ Scroll complete.');
-
-        // --- Content Handlers ---
-        console.log(`\n[3/12] 🟢 Testing Find Selector (Text search)...`);
-        const findRes = await handleFindSelector({ text: 'Movie' }); // Assuming "Movie" exists
-        console.log(`   Result: Found ${findRes.content[0].text.length > 50 ? 'matches' : 'no matches'} (Length: ${findRes.content[0].text.length})`);
-
-        // --- Multi-Element Handlers (The file we kept) ---
-        console.log(`\n[4/12] 🟢 Testing Batch Element Scraper...`);
-        const batchRes = await handleBatchElementScraper({ selector: 'a', maxElements: 3 });
-        console.log(`   Result: ${batchRes.content[0].text.substring(0, 100)}...`);
-
-        console.log(`\n[5/12] 🟢 Testing Attribute Harvester...`);
-        const attrRes = await handleAttributeHarvester({ selector: 'img', attributes: ['src'], maxElements: 3 });
-        console.log(`   Result: ${attrRes.content[0].text.substring(0, 100)}...`);
-
-        console.log(`\n[6/12] 🟢 Testing Media Extractor...`); // Might be empty on home page but runs logic
-        const mediaRes = await handleMediaExtractor({ types: ['video', 'iframe'] });
-        console.log(`   Result: ${mediaRes.content[0].text.substring(0, 100)}...`);
-
-        // --- Search & Filter Handlers (The file we kept) ---
-        console.log(`\n[7/12] 🟢 Testing Keyword Search...`);
-        const keyRes = await handleKeywordSearch({ keywords: ['Bollywood', 'Hollywood'] });
-        console.log(`   Result: ${keyRes.content[0].text.substring(0, 100)}...`);
-
-        console.log(`\n[8/12] 🟢 Testing Regex Pattern Matcher...`);
-        const regexRes = await handleRegexPatternMatcher({ pattern: 'https?://[^\\s"\']+' });
-        console.log(`   Result: ${regexRes.content[0].text.substring(0, 100)}...`);
-
-        console.log(`\n[9/12] 🟢 Testing XPath Support...`);
-        const xpathRes = await handleXPathSupport({ xpath: '//body//div' });
-        console.log(`   Result: ${xpathRes.content[0].text.substring(0, 100)}...`);
-
-        console.log(`\n[10/12] 🟢 Testing Advanced CSS Selectors...`);
-        const cssRes = await handleAdvancedCSSSelectors({ selector: 'div > a', operation: 'query' });
-        console.log(`   Result: ${cssRes.content[0].text.substring(0, 100)}...`);
-
-        // --- Data Extraction ---
-        console.log(`\n[11/12] 🟢 Testing Schema Extraction...`);
-        const schemaRes = await handleExtractSchema({});
-        console.log(`   Result: ${schemaRes.content[0].text.substring(0, 100)}...`);
-
-        // --- Pagination (Refactored) ---
-        console.log(`\n[12/12] 🟢 Testing Breadcrumb Navigator...`);
-        const breadRes = await handleBreadcrumbNavigator({});
-        console.log(`   Result: ${breadRes.content[0].text.substring(0, 100)}...`);
-
-        console.log('\n✅ All primary handler categories verified successfully.');
-
-    } catch (error) {
-        console.error('\n❌ Verification Failed:', error);
-    } finally {
-        await handleBrowserClose({});
-    }
-}
-
-runFullVerification();

package/scripts/live-verification.ts

@@ -1,61 +0,0 @@
-
-import { handleBrowserInit, handleBrowserClose } from '../src/handlers/browser-handlers.js';
-import { handleNavigate } from '../src/handlers/navigation-handlers.js';
-import { handleGetContent } from '../src/handlers/content-handlers.js';
-import { handleBreadcrumbNavigator } from '../src/handlers/navigation-handlers.js';
-import { handleLinkHarvester } from '../src/handlers/multi-element-handlers.js';
-import { handleScrapeMetaTags } from '../src/handlers/data-extraction-handlers.js';
-
-async function runVerification() {
-    console.log('🚀 Starting Verification on Live Sites...');
-
-    try {
-        // 1. Initialize Browser
-        console.log('\n🔵 Initializing Browser...');
-        await handleBrowserInit({ headless: true });
-
-        const sites = [
-            'https://moviesdrive.forum/',
-            'https://multimovies.golf/'
-        ];
-
-        for (const url of sites) {
-            console.log(`\n--------------------------------------------------`);
-            console.log(`🔍 Testing Site: ${url}`);
-            console.log(`--------------------------------------------------`);
-
-            // 2. Navigate
-            console.log(`\n➡️ Navigating to ${url}...`);
-            await handleNavigate({ url });
-
-            // 3. Get Content (HTML preview)
-            console.log(`\n📄 Fetching Content (Preview)...`);
-            const contentRes = await handleGetContent({ type: 'text' });
-            console.log(`   Result: ${contentRes.content[0].text.substring(0, 100)}...`);
-
-            // 4. Test Breadcrumb Navigator (Newly moved)
-            console.log(`\nnav Testing Breadcrumb Navigator...`);
-            const breadcrumbRes = await handleBreadcrumbNavigator({});
-            console.log(`   Result: ${breadcrumbRes.content[0].text.substring(0, 200)}...`);
-
-            // 5. Test Link Harvester (Existing tool)
-            console.log(`\n🔗 Testing Link Harvester (First 5 links)...`);
-            const linksRes = await handleLinkHarvester({ maxElements: 5 });
-            console.log(`   Result: ${linksRes.content[0].text.substring(0, 200)}...`);
-
-            // 6. Test Meta Tags (Data extraction)
-            console.log(`\n🏷️ Testing Meta Tag Scraper...`);
-            const metaRes = await handleScrapeMetaTags({});
-            console.log(`   Result: ${metaRes.content[0].text.substring(0, 200)}...`);
-        }
-
-    } catch (error) {
-        console.error('\n❌ Verification Failed:', error);
-    } finally {
-        // 7. Cleanup
-        console.log('\n🔴 Closing Browser...');
-        await handleBrowserClose({});
-    }
-}
-
-runVerification();

package/scripts/setup-brave.cjs

@@ -1,171 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const { execSync, exec } = require('child_process');
-const https = require('https');
-
-// Colors for console output
-const colors = {
-    reset: "\x1b[0m",
-    bright: "\x1b[1m",
-    green: "\x1b[32m",
-    yellow: "\x1b[33m",
-    red: "\x1b[31m",
-    cyan: "\x1b[36m"
-};
-
-function log(message, type = 'info') {
-    const timestamp = new Date().toISOString();
-    let color = colors.reset;
-    if (type === 'success') color = colors.green;
-    if (type === 'warn') color = colors.yellow;
-    if (type === 'error') color = colors.red;
-    if (type === 'info') color = colors.cyan;
-    console.log(`${color}[${type.toUpperCase()}] ${message}${colors.reset}`);
-}
-
-const platform = os.platform();
-
-// Define standard paths including the user's specific request
-const bravePaths = {
-    win32: [
-        'C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe', // User's requested primary path
-        'C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe',
-        path.join(os.homedir(), 'AppData', 'Local', 'BraveSoftware', 'Brave-Browser', 'Application', 'brave.exe')
-    ],
-    darwin: [
-        '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'
-    ],
-    linux: [
-        '/usr/bin/brave-browser',
-        '/usr/bin/brave',
-        '/snap/bin/brave'
-    ]
-};
-
-async function checkBraveInstallation() {
-    log("Checking for Brave Browser installation...", 'info');
-    
-    let foundPath = null;
-    const pathsToCheck = bravePaths[platform] || [];
-
-    for (const p of pathsToCheck) {
-        if (fs.existsSync(p)) {
-            foundPath = p;
-            break;
-        }
-    }
-
-    if (foundPath) {
-        log(`Brave Browser found at: ${foundPath}`, 'success');
-        updateEnvFile(foundPath);
-        return true;
-    } else {
-        log("Brave Browser not found in standard locations.", 'warn');
-        return false;
-    }
-}
-
-function updateEnvFile(bravePath) {
-    const envPath = path.join(__dirname, '..', '.env');
-    try {
-        let envContent = '';
-        if (fs.existsSync(envPath)) {
-            envContent = fs.readFileSync(envPath, 'utf8');
-        } else {
-             // Create from example if possible, or just start empty
-             const examplePath = path.join(__dirname, '..', '.env.example');
-             if(fs.existsSync(examplePath)) {
-                envContent = fs.readFileSync(examplePath, 'utf8');
-             }
-        }
-
-        // Check if BRAVE_PATH is already set
-        if (envContent.includes('BRAVE_PATH=')) {
-             // Replace existing
-             const newContent = envContent.replace(/^BRAVE_PATH=.*$/m, `BRAVE_PATH=${bravePath}`);
-             if (newContent !== envContent) {
-                 fs.writeFileSync(envPath, newContent);
-                 log("Updated .env with detected Brave path.", 'success');
-             } else {
-                 log(".env already has a BRAVE_PATH set (or format differs).", 'info');
-             }
-        } else {
-            // Append
-            fs.appendFileSync(envPath, `\nBRAVE_PATH=${bravePath}\n`);
-            log("Appended BRAVE_PATH to .env.", 'success');
-        }
-    } catch (err) {
-        log(`Failed to update .env: ${err.message}`, 'error');
-    }
-}
-
-async function installBrave() {
-    log("Attempting to install Brave Browser...", 'info');
-    
-    if (platform === 'win32') {
-        const installerUrl = 'https://laptop-updates.brave.com/latest/winx64'; // Standard stub installer
-        const installerPath = path.join(os.tmpdir(), 'BraveBrowserSetup.exe');
-        
-        try {
-            await downloadFile(installerUrl, installerPath);
-            log(`Installer downloaded to ${installerPath}`, 'success');
-            log("Running installer... Please follow the prompts if any appear.", 'info');
-            
-            // Run installer
-            // /silent /install are common flags for Chromium based browsers.
-            // But we use start to let it run interactively if needed, or silent if possible.
-            // The standard stub installer usually just runs.
-            exec(`"${installerPath}"`, (error, stdout, stderr) => {
-                if (error) {
-                    log(`Installer execution failed: ${error.message}`, 'error');
-                } else {
-                    log("Installer started. Please complete the installation.", 'success');
-                }
-            });
-
-        } catch (err) {
-            log(`Failed to download/run installer: ${err.message}`, 'error');
-            log("Please install Brave manually from https://brave.com/download/", 'warn');
-        }
-    } else if (platform === 'darwin') {
-        log("Automatic installation on macOS is complex. Opening download page...", 'info');
-        exec('open https://brave.com/download/');
-    } else if (platform === 'linux') {
-        log("Please install Brave using your package manager (apt, dnf, snap, etc.).", 'info');
-        console.log("Visit: https://brave.com/linux/");
-    } else {
-        log(`Unsupported platform for auto-install: ${platform}`, 'error');
-    }
-}
-
-function downloadFile(url, dest) {
-    return new Promise((resolve, reject) => {
-        const file = fs.createWriteStream(dest);
-        https.get(url, (response) => {
-            if (response.statusCode !== 200) {
-                 reject(new Error(`Failed to download: Status Code ${response.statusCode}`));
-                 return;
-            }
-            response.pipe(file);
-            file.on('finish', () => {
-                file.close(resolve);
-            });
-        }).on('error', (err) => {
-            fs.unlink(dest, () => {}); // Delete the file async. (But we don't check result)
-            reject(err);
-        });
-    });
-}
-
-async function main() {
-    const isInstalled = await checkBraveInstallation();
-    if (!isInstalled) {
-        await installBrave();
-    }
-}
-
-main().catch(err => {
-    console.error("Setup script failed:", err);
-    process.exit(0); 
-});

package/scripts/verify-brave-installer.cjs

@@ -1,13 +0,0 @@
-
-const { BraveInstaller } = require('../dist/brave-installer.js');
-
-console.log('Checking BraveInstaller module...');
-
-if (typeof BraveInstaller.install === 'function') {
-    console.log('✅ PASS: BraveInstaller.install is a function');
-} else {
-    console.error('❌ FAIL: BraveInstaller.install is NOT a function');
-    process.exit(1);
-}
-
-console.log('Module structure verified.');

package/scripts/verify-fixes-custom.ts

@@ -1,108 +0,0 @@
-
-import { chromium } from 'playwright';
-// Mock interfaces to simulate MCP handler context if needed, but for now we'll just run them as standalone if possible, 
-// OR simpler: we'll just simulate the logic using playwright directly to prove the logic works, 
-// OR better: Import the handlers and run them.
-// Since handlers depend on 'browser-manager' which has global state, we need to initialize it.
-
-import { handleLinkHarvester } from '../src/handlers/multi-element-handlers.js';
-import { handleApiFinder } from '../src/handlers/smart-data-extractors.js';
-import { handleExtractJSON } from '../src/handlers/data-extraction-handlers.js';
-import { handleFetchXHR } from '../src/handlers/smart-data-extractors.js'; // Ensure this export exists
-import { setBrowser, setPage } from '../src/browser-manager.js';
-
-async function verify() {
-    console.log('🚀 Starting Verification...');
-    const browser = await chromium.launch({ headless: true });
-    const context = await browser.newContext();
-    const page = await context.newPage();
-
-    // Set global state for handlers
-    setBrowser(browser as any);
-    setPage(page as any);
-
-    try {
-        // 1. Test Link Harvester (with Buttons)
-        console.log('\n🧪 Testing Link Harvester...');
-        await page.setContent(`
-        <html>
-            <body>
-                <a href="https://example.com">Example</a>
-                <button>Click me</button>
-                <a href="#anchor">Anchor</a>
-            </body>
-        </html>
-      `);
-        // This should NOT throw now
-        const links = await handleLinkHarvester({ selector: 'button, a', classifyLinks: true });
-        console.log('✅ Link Harvester Result (should have 2 links, no crash):', links);
-
-
-        // 2. Test API Finder
-        console.log('\n🧪 Testing API Finder...');
-        // Note: API finder listens for responses, so we need to trigger some or have them present
-        await page.setContent(`
-        <html>
-            <script>
-                const apiUrl = "https://api.example.com/v1/users";
-            </script>
-        </html>
-      `);
-        const apis = await handleApiFinder({ duration: 1000 });
-        console.log('✅ API Finder Result:', apis);
-
-
-        // 3. Test Extract JSON
-        console.log('\n🧪 Testing Extract JSON...');
-        await page.setContent(`
-        <html>
-            <script>
-                const config = { "siteId": 123, "name": "Test Site" };
-                const simple = [1, 2, 3];
-            </script>
-            <script type="application/json">
-                { "valid": "json" }
-            </script>
-        </html>
-      `);
-        const json = await handleExtractJSON({ source: 'all' });
-        // Should find the application/json AND the embedded config object
-        console.log('✅ Extract JSON Result:', JSON.stringify(json, null, 2));
-
-
-        // 4. Test Fetch XHR (Force Reload)
-        console.log('\n🧪 Testing Fetch XHR...');
-        // We need a real page for this to effectively test reload, or mock network
-        // Let's use a simple public page
-        console.log('Navigating to example.com to test network capture...');
-        // Mock network requests for reliability
-        await page.route('**/api/data', route => route.fulfill({
-            status: 200,
-            contentType: 'application/json',
-            body: JSON.stringify({ success: true })
-        }));
-
-        // We will manually trigger a fetch on load
-        await page.goto('https://example.com');
-        await page.evaluate(() => {
-            fetch('/api/data');
-        });
-
-        const requests = await handleFetchXHR({ duration: 2000, forceReload: true });
-        // Note: forceReload might clear the manual fetch above if it reloads immediately. 
-        // Actually fetch_xhr with forceReload causes a reload. So we rely on the page itself making requests.
-        // example.com is static. 
-        // Let's rely on our manual fetch being captured if we DON'T reload, or rely on reload to capture real page assets.
-        // For this test, let's turn forceReload OFF to verify we capture the manual fetch, 
-        // AND run another test with forceReload ON to verify it doesn't crash (even if example.com has no XHR).
-
-        console.log('✅ Fetch XHR Result:', requests);
-
-    } catch (error) {
-        console.error('❌ Verification Failed:', error);
-    } finally {
-        await browser.close();
-    }
-}
-
-verify();

package/scripts/verify-fixes-standalone.js

@@ -1,244 +0,0 @@
-
-import { chromium } from 'playwright';
-
-// --- MOCK / INLINED HANDLERS ---
-
-// 1. Link Harvester (Fixed)
-async function handleLinkHarvester(page, args) {
-    const selector = args.selector || 'a[href]';
-    const classifyLinks = args.classifyLinks !== false;
-    const includeAnchors = args.includeAnchors || false;
-
-    const linkData = await page.evaluate(
-        ({ selector, classifyLinks, includeAnchors }) => {
-            const links = document.querySelectorAll(selector);
-            const currentDomain = window.location.hostname;
-            const results = [];
-
-            links.forEach((link, index) => {
-                const href = link.href;
-
-                // Fixed: Check for href
-                if (!href) return;
-
-                // Skip anchors if not included
-                if (!includeAnchors && href.startsWith('#')) {
-                    return;
-                }
-
-                const linkInfo = {
-                    index,
-                    href,
-                    text: link.textContent?.trim() || '',
-                    title: link.title || '',
-                };
-
-                if (classifyLinks) {
-                    try {
-                        const url = new URL(href, window.location.href);
-                        const isInternal = url.hostname === currentDomain;
-                        const isAnchor = href.startsWith('#');
-                        const isMailto = href.startsWith('mailto:');
-                        const isTel = href.startsWith('tel:');
-
-                        linkInfo.type = isAnchor
-                            ? 'anchor'
-                            : isMailto
-                                ? 'email'
-                                : isTel
-                                    ? 'phone'
-                                    : isInternal
-                                        ? 'internal'
-                                        : 'external';
-
-                        linkInfo.domain = url.hostname;
-                        linkInfo.protocol = url.protocol;
-                    } catch (e) {
-                        linkInfo.type = 'invalid';
-                        linkInfo.domain = 'unknown';
-                    }
-                }
-                results.push(linkInfo);
-            });
-            return results;
-        },
-        { selector, classifyLinks, includeAnchors }
-    );
-    return linkData;
-}
-
-// 2. Extract JSON (Fixed)
-async function handleExtractJSON(page, args) {
-    const source = args.source || 'all';
-    const selector = args.selector;
-    const filter = args.filter;
-
-    const jsonData = await page.evaluate(
-        ({ source, selector, filter }) => {
-            const results = [];
-
-            // Extract JSON from script tags
-            if (source === 'script' || source === 'all') {
-                const defaultSelector = selector || 'script[type="application/json"], script[type="application/ld+json"], script';
-                const scripts = document.querySelectorAll(defaultSelector);
-
-                scripts.forEach((script, index) => {
-                    const content = script.textContent || '';
-                    try {
-                        // 1. Try direct parsing first
-                        const data = JSON.parse(content);
-                        if (filter) {
-                            const filterLower = filter.toLowerCase();
-                            const dataStr = JSON.stringify(data).toLowerCase();
-                            if (!dataStr.includes(filterLower)) return;
-                        }
-                        results.push({ data, source: 'script', path: `script[${index}]` });
-                    } catch (e) {
-                        // 2. Fallback: Try to find JSON objects using regex
-                        const jsonRegex = /({[\s\S]*?}|\[[\s\S]*?\])/g;
-                        let match;
-                        while ((match = jsonRegex.exec(content)) !== null) {
-                            const potentialJson = match[0];
-                            if (potentialJson.length < 20) continue;
-                            try {
-                                const data = JSON.parse(potentialJson);
-                                if (filter) {
-                                    const filterLower = filter.toLowerCase();
-                                    const dataStr = JSON.stringify(data).toLowerCase();
-                                    if (!dataStr.includes(filterLower)) continue;
-                                }
-                                if ((Array.isArray(data) && data.length > 0) || (typeof data === 'object' && data !== null && Object.keys(data).length > 0)) {
-                                    results.push({ data, source: 'script', path: `script[${index}]_regex_match` });
-                                }
-                            } catch (e2) { }
-                        }
-                    }
-                });
-            }
-            return results;
-        },
-        { source, selector, filter }
-    );
-    return jsonData;
-}
-
-// 3. API Finder
-async function handleApiFinder(page, args) {
-    const captureDuration = typeof args.duration === 'number' ? args.duration : 1000;
-
-    // From inline scripts
-    const scriptApis = await page.evaluate(() => {
-        const results = [];
-        const scripts = Array.from(document.querySelectorAll('script'));
-
-        const apiPatterns = [
-            /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
-            /https?:\/\/api\.[^"'\s]+/gi,
-        ];
-
-        scripts.forEach(script => {
-            const content = script.textContent || '';
-            apiPatterns.forEach(pattern => {
-                const matches = content.match(pattern);
-                if (matches) {
-                    matches.forEach(match => results.push({ url: match, source: 'script' }));
-                }
-            });
-        });
-        return results;
-    });
-    return scriptApis;
-}
-
-// 4. Fetch XHR (Fixed)
-async function handleFetchXHR(page, args) {
-    const duration = args.duration || 2000;
-    const forceReload = args.forceReload !== false;
-
-    const xhrData = [];
-    const responseHandler = async (response) => {
-        const request = response.request();
-        const resourceType = request.resourceType();
-
-        if (resourceType === 'xhr' || resourceType === 'fetch') {
-            try {
-                const body = await response.text();
-                xhrData.push({
-                    url: response.url(),
-                    body: body.substring(0, 1000),
-                });
-            } catch (e) { }
-        }
-    };
-
-    page.on('response', responseHandler);
-
-    if (forceReload) {
-        try {
-            await page.reload({ waitUntil: 'networkidle' });
-        } catch (e) { }
-    }
-
-    // Wait a bit
-    await new Promise(r => setTimeout(r, duration));
-    page.off('response', responseHandler);
-    return xhrData;
-}
-
-
-async function verify() {
-    console.log('🚀 Starting Verification...');
-    const browser = await chromium.launch({ headless: true });
-    const context = await browser.newContext();
-    const page = await context.newPage();
-
-    try {
-        // 1. Test Link Harvester
-        console.log('\n🧪 Testing Link Harvester...');
-        await page.setContent(`
-        <html><body>
-            <a href="https://example.com">Example</a>
-            <button>Click me</button>
-            <a href="#anchor">Anchor</a>
-        </body></html>
-      `);
-        const links = await handleLinkHarvester(page, { selector: 'button, a', classifyLinks: true });
-        console.log('✅ Link Harvester Result (should have 2 links):', links.length);
-        console.log(links);
-
-        // 2. Test Extract JSON
-        console.log('\n🧪 Testing Extract JSON...');
-        await page.setContent(`
-        <html>
-            <script>
-                const config = { "siteId": 123, "name": "Test Site" };
-            </script>
-        </html>
-      `);
-        const json = await handleExtractJSON(page, { source: 'all' });
-        const found = json.find(j => j.data.siteId === 123);
-        console.log('✅ Extract JSON Result:', found ? 'Found embedded JSON' : 'FAILED to find embedded JSON');
-        console.log(json);
-
-        // 3. Test API Finder
-        console.log('\n🧪 Testing API Finder...');
-        await page.setContent(`<html><script>const apiUrl = "https://api.example.com/v1/users";</script></html>`);
-        const apis = await handleApiFinder(page, {});
-        console.log('✅ API Finder Result:', apis.length > 0 ? 'Found API' : 'FAILED');
-        console.log(apis);
-
-        // 4. Test Fetch XHR
-        console.log('\n🧪 Testing Fetch XHR...');
-        await page.goto('https://example.com');
-        const xhr = await handleFetchXHR(page, { duration: 1000, forceReload: true });
-        console.log('✅ Fetch XHR executed without error. Captured:', xhr.length);
-
-    } catch (error) {
-        console.error('❌ Verification Failed:', error);
-        process.exit(1);
-    } finally {
-        await browser.close();
-    }
-}
-
-verify();

package/scripts/verify-fixes-standalone.ts

@@ -1,248 +0,0 @@
-
-import { chromium } from 'playwright';
-
-// --- MOCK / INLINED HANDLERS ---
-// Copied and adapted from source to verify logic without import issues
-
-// 1. Link Harvester (Fixed)
-async function handleLinkHarvester(page: any, args: any) {
-    const selector = args.selector || 'a[href]';
-    const classifyLinks = args.classifyLinks !== false;
-    const includeAnchors = args.includeAnchors || false;
-
-    const linkData = await page.evaluate(
-        ({ selector, classifyLinks, includeAnchors }: any) => {
-            const links = document.querySelectorAll(selector);
-            const currentDomain = window.location.hostname;
-            const results: any[] = [];
-
-            links.forEach((link: any, index: number) => {
-                const href = (link as any).href;
-
-                // Fixed: Check for href
-                if (!href) return;
-
-                // Skip anchors if not included
-                if (!includeAnchors && href.startsWith('#')) {
-                    return;
-                }
-
-                const linkInfo: any = {
-                    index,
-                    href,
-                    text: link.textContent?.trim() || '',
-                    title: link.title || '',
-                };
-
-                if (classifyLinks) {
-                    try {
-                        const url = new URL(href, window.location.href);
-                        const isInternal = url.hostname === currentDomain;
-                        const isAnchor = href.startsWith('#');
-                        const isMailto = href.startsWith('mailto:');
-                        const isTel = href.startsWith('tel:');
-
-                        linkInfo.type = isAnchor
-                            ? 'anchor'
-                            : isMailto
-                                ? 'email'
-                                : isTel
-                                    ? 'phone'
-                                    : isInternal
-                                        ? 'internal'
-                                        : 'external';
-
-                        linkInfo.domain = url.hostname;
-                        linkInfo.protocol = url.protocol;
-                    } catch (e) {
-                        linkInfo.type = 'invalid';
-                        linkInfo.domain = 'unknown';
-                    }
-                }
-                results.push(linkInfo);
-            });
-            return results;
-        },
-        { selector, classifyLinks, includeAnchors }
-    );
-    return linkData;
-}
-
-// 2. Extract JSON (Fixed)
-async function handleExtractJSON(page: any, args: any) {
-    const source = args.source || 'all';
-    const selector = args.selector;
-    const filter = args.filter;
-
-    const jsonData = await page.evaluate(
-        ({ source, selector, filter }: any) => {
-            const results: any[] = [];
-
-            // Extract JSON from script tags
-            if (source === 'script' || source === 'all') {
-                const defaultSelector = selector || 'script[type="application/json"], script[type="application/ld+json"], script';
-                const scripts = document.querySelectorAll(defaultSelector);
-
-                scripts.forEach((script: any, index: number) => {
-                    const content = script.textContent || '';
-                    try {
-                        // 1. Try direct parsing first
-                        const data = JSON.parse(content);
-                        if (filter) {
-                            const filterLower = filter.toLowerCase();
-                            const dataStr = JSON.stringify(data).toLowerCase();
-                            if (!dataStr.includes(filterLower)) return;
-                        }
-                        results.push({ data, source: 'script', path: `script[${index}]` });
-                    } catch (e) {
-                        // 2. Fallback: Try to find JSON objects using regex
-                        const jsonRegex = /({[\s\S]*?}|\[[\s\S]*?\])/g;
-                        let match;
-                        while ((match = jsonRegex.exec(content)) !== null) {
-                            const potentialJson = match[0];
-                            if (potentialJson.length < 20) continue;
-                            try {
-                                const data = JSON.parse(potentialJson);
-                                if (filter) {
-                                    const filterLower = filter.toLowerCase();
-                                    const dataStr = JSON.stringify(data).toLowerCase();
-                                    if (!dataStr.includes(filterLower)) continue;
-                                }
-                                if ((Array.isArray(data) && data.length > 0) || (typeof data === 'object' && data !== null && Object.keys(data).length > 0)) {
-                                    results.push({ data, source: 'script', path: `script[${index}]_regex_match` });
-                                }
-                            } catch (e2) { }
-                        }
-                    }
-                });
-            }
-            return results;
-        },
-        { source, selector, filter }
-    );
-    return jsonData;
-}
-
-// 3. API Finder (Just verification of logic, no changes other than registration)
-// We verify it works as intended.
-async function handleApiFinder(page: any, args: any) {
-    const captureDuration = typeof args.duration === 'number' ? args.duration : 1000;
-
-    // From inline scripts
-    const scriptApis = await page.evaluate(() => {
-        const results: any[] = [];
-        const scripts = Array.from(document.querySelectorAll('script'));
-
-        const apiPatterns = [
-            /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
-            /https?:\/\/api\.[^"'\s]+/gi,
-        ];
-
-        scripts.forEach(script => {
-            const content = script.textContent || '';
-            apiPatterns.forEach(pattern => {
-                const matches = content.match(pattern);
-                if (matches) {
-                    matches.forEach(match => results.push({ url: match, source: 'script' }));
-                }
-            });
-        });
-        return results;
-    });
-    return scriptApis;
-}
-
-// 4. Fetch XHR (Fixed)
-async function handleFetchXHR(page: any, args: any) {
-    const duration = args.duration || 2000;
-    const forceReload = args.forceReload !== false;
-
-    const xhrData: any[] = [];
-    const responseHandler = async (response: any) => {
-        const request = response.request();
-        const resourceType = request.resourceType();
-
-        if (resourceType === 'xhr' || resourceType === 'fetch') {
-            try {
-                const body = await response.text();
-                xhrData.push({
-                    url: response.url(),
-                    body: body.substring(0, 1000),
-                });
-            } catch (e) { }
-        }
-    };
-
-    page.on('response', responseHandler);
-
-    if (forceReload) {
-        try {
-            await page.reload({ waitUntil: 'networkidle' }); // Playwright 'networkidle'
-        } catch (e) { }
-    }
-
-    // Wait a bit
-    await new Promise(r => setTimeout(r, duration));
-    page.off('response', responseHandler);
-    return xhrData;
-}
-
-
-async function verify() {
-    console.log('🚀 Starting Verification...');
-    const browser = await chromium.launch({ headless: true });
-    const context = await browser.newContext();
-    const page = await context.newPage();
-
-    try {
-        // 1. Test Link Harvester
-        console.log('\n🧪 Testing Link Harvester...');
-        await page.setContent(`
-        <html><body>
-            <a href="https://example.com">Example</a>
-            <button>Click me</button>
-            <a href="#anchor">Anchor</a>
-        </body></html>
-      `);
-        const links = await handleLinkHarvester(page, { selector: 'button, a', classifyLinks: true });
-        console.log('✅ Link Harvester Result (should have 2 links):', links.length);
-        console.log(links);
-
-        // 2. Test Extract JSON
-        console.log('\n🧪 Testing Extract JSON...');
-        await page.setContent(`
-        <html>
-            <script>
-                const config = { "siteId": 123, "name": "Test Site" };
-            </script>
-        </html>
-      `);
-        const json = await handleExtractJSON(page, { source: 'all' });
-        const found = json.find((j: any) => j.data.siteId === 123);
-        console.log('✅ Extract JSON Result:', found ? 'Found embedded JSON' : 'FAILED to find embedded JSON');
-        console.log(json);
-
-        // 3. Test API Finder
-        console.log('\n🧪 Testing API Finder...');
-        await page.setContent(`<html><script>const apiUrl = "https://api.example.com/v1/users";</script></html>`);
-        const apis = await handleApiFinder(page, {});
-        console.log('✅ API Finder Result:', apis.length > 0 ? 'Found API' : 'FAILED');
-        console.log(apis);
-
-        // 4. Test Fetch XHR
-        console.log('\n🧪 Testing Fetch XHR...');
-        // We can't easily mock responses in this simple script without extensive setup, 
-        // but we can check if it attempts reload without error.
-        await page.goto('https://example.com');
-        const xhr = await handleFetchXHR(page, { duration: 1000, forceReload: true });
-        console.log('✅ Fetch XHR executed without error. Captured:', xhr.length);
-
-    } catch (error) {
-        console.error('❌ Verification Failed:', error);
-        process.exit(1);
-    } finally {
-        await browser.close();
-    }
-}
-
-verify();

package/scripts/verify-ublock.ts

@@ -1,58 +0,0 @@
-
-import { ExtensionManager } from '../src/extension-manager.js';
-import { initializeBrowser, closeBrowser } from '../src/browser-manager.js';
-import * as fs from 'fs';
-import * as path from 'path';
-
-async function verifyUBlock() {
-    console.log('🚀 Starting uBlock Origin Verification...');
-
-    // 1. Verify Download & Extraction
-    console.log('\nSTEP 1: Checking ExtensionManager...');
-    const extPath = await ExtensionManager.ensureUBlockOrigin();
-    console.log(`📂 Extension Path: ${extPath}`);
-
-    if (!extPath || !fs.existsSync(extPath)) {
-        console.error('❌ Failed: Extension path does not exist!');
-        process.exit(1);
-    }
-
-    const manifestPath = path.join(extPath, 'manifest.json');
-    if (fs.existsSync(manifestPath)) {
-        console.log('✅ Manifest.json found! Extension extracted correctly.');
-    } else {
-        console.error('❌ Failed: manifest.json missing in extension folder!');
-        process.exit(1);
-    }
-
-    // 2. Verify Browser Launch with Extension
-    console.log('\nSTEP 2: Launching Browser with uBlock...');
-    try {
-        const { browser } = await initializeBrowser({
-            headless: false, // Must be false or 'new' for extensions usually, but let's test our default config
-            autoInstall: true
-        });
-
-        // Check if arguments were passed
-        const spawnArgs = browser.process()?.spawnargs || [];
-        const loadArg = spawnArgs.find(arg => arg.includes('--load-extension'));
-
-        if (loadArg && loadArg.includes(extPath)) {
-            console.log('✅ Browser launched with --load-extension argument pointing to uBlock!');
-        } else {
-            console.error('❌ Failed: Browser launched but --load-extension arg missing or incorrect.');
-            console.error('Spawn Args:', spawnArgs);
-            await closeBrowser();
-            process.exit(1);
-        }
-
-        await closeBrowser();
-        console.log('\n✨ VERIFICATION SUCCESSFUL: uBlock Origin is integrated!');
-
-    } catch (error) {
-        console.error('❌ Browser Launch Failed:', error);
-        process.exit(1);
-    }
-}
-
-verifyUBlock();