brave-real-browser-mcp-server 2.11.0 → 2.11.2

package/dist/handlers/advanced-extraction-handlers.js

@@ -14,7 +14,7 @@ export async function handleAdvancedVideoExtraction(args) {
             requirePage: true,
         });
         const page = getCurrentPage();
-        const waitTime = args.waitTime || 10000;
+        const waitTime = args.waitTime || 20000;
         // Collect all video-related data
         const videoData = {
             directVideoUrls: [],
@@ -183,7 +183,11 @@ export async function handleAdvancedVideoExtraction(args) {
                 'button[data-download]',
                 'a[href*=".mp4"]',
                 'a[href*=".mkv"]',
-                'a[class*="download"]'
+                'a[class*="download"]',
+                'a[href*=".webm"]',
+                '.download-btn',
+                '.btn-download',
+                '[class*="download"]'
             ];
             downloadSelectors.forEach(selector => {
                 document.querySelectorAll(selector).forEach((el) => {

package/dist/handlers/api-integration-handlers.js

@@ -6,7 +6,7 @@ import { sleep } from '../system-utils.js';
  * REST API Endpoint Finder - Discover REST API endpoints
  */
 export async function handleRESTAPIEndpointFinder(args) {
-    const { url, analyzeNetworkRequests = true, scanDuration = 5000 } = args;
+    const { url, analyzeNetworkRequests = true, scanDuration = 10000 } = args;
     try {
         const page = getPageInstance();
         if (!page) {

package/dist/handlers/data-extraction-handlers.js

@@ -195,7 +195,8 @@ export async function handleExtractJSON(args) {
             const results = [];
             // Extract JSON from script tags
             if (source === 'script' || source === 'all') {
-                const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]');
+                const defaultSelector = selector || 'script[type="application/json"], script[type="application/ld+json"], script';
+                const scripts = document.querySelectorAll(defaultSelector);
                 scripts.forEach((script, index) => {
                     try {
                         const content = script.textContent || '';
@@ -346,7 +347,7 @@ export async function handleExtractSchema(args) {
         });
         const page = getCurrentPage();
         const format = args.format || 'all';
-        const schemaType = args.schemaType;
+        const schemaType = args.schemaType || ['WebPage', 'Organization', 'Product', 'BreadcrumbList'];
         const schemaData = await page.evaluate(({ format, schemaType }) => {
             const results = [];
             // Extract JSON-LD
@@ -358,7 +359,9 @@ export async function handleExtractSchema(args) {
                         // Filter by schema type if specified
                         if (schemaType) {
                             const type = data['@type'] || '';
-                            if (!type.toLowerCase().includes(schemaType.toLowerCase())) {
+                            const types = Array.isArray(schemaType) ? schemaType : [schemaType];
+                            const typeMatch = types.some(t => type.toLowerCase().includes(t.toLowerCase()));
+                            if (!typeMatch) {
                                 return;
                             }
                         }
@@ -378,8 +381,12 @@ export async function handleExtractSchema(args) {
                 const items = document.querySelectorAll('[itemscope]');
                 items.forEach((item) => {
                     const itemType = item.getAttribute('itemtype') || '';
-                    if (schemaType && !itemType.toLowerCase().includes(schemaType.toLowerCase())) {
-                        return;
+                    if (schemaType) {
+                        const types = Array.isArray(schemaType) ? schemaType : [schemaType];
+                        const typeMatch = types.some(t => itemType.toLowerCase().includes(t.toLowerCase()));
+                        if (!typeMatch) {
+                            return;
+                        }
                     }
                     const properties = {};
                     const props = item.querySelectorAll('[itemprop]');

package/dist/handlers/smart-data-extractors.js

@@ -184,7 +184,7 @@ export async function handleAjaxExtractor(args) {
             requirePage: true,
         });
         const page = getCurrentPage();
-        const duration = args.duration || 5000;
+        const duration = args.duration || 15000;
         const url = args.url;
         const requests = [];
         const requestHandler = (request) => {
@@ -223,7 +223,7 @@ export async function handleFetchXHR(args) {
             requirePage: true,
         });
         const page = getCurrentPage();
-        const duration = args.duration || 5000;
+        const duration = args.duration || 15000;
         const xhrData = [];
         const responseHandler = async (response) => {
             const request = response.request();
@@ -267,8 +267,8 @@ export async function handleNetworkRecorder(args) {
             requirePage: true,
         });
         const page = getCurrentPage();
-        const duration = args.duration || 10000;
-        const filterTypes = args.filterTypes || ['all'];
+        const duration = args.duration || 20000;
+        const filterTypes = args.filterTypes || ['video', 'xhr', 'fetch', 'media'];
         const networkActivity = [];
         const requestHandler = (request) => {
             const resourceType = request.resourceType();
@@ -317,13 +317,16 @@ export async function handleApiFinder(args) {
             requirePage: true,
         });
         const page = getCurrentPage();
-        const apis = await page.evaluate(() => {
+        const deepScan = args.deepScan !== false;
+        const apis = await page.evaluate(({ deepScan }) => {
             const results = [];
             const scripts = Array.from(document.querySelectorAll('script'));
             const apiPatterns = [
                 /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
                 /https?:\/\/api\.[^"'\s]+/gi,
                 /\/api\/v?\d*\/[^"'\s]*/gi,
+                /graphql/gi,
+                /rest\/v?\d*/gi,
             ];
             scripts.forEach(script => {
                 const content = script.textContent || '';
@@ -337,8 +340,21 @@ export async function handleApiFinder(args) {
                     }
                 });
             });
+            // Deep scan: also check HTML content for API references
+            if (deepScan) {
+                const htmlContent = document.body.innerHTML;
+                apiPatterns.forEach(pattern => {
+                    const matches = htmlContent.match(pattern);
+                    if (matches) {
+                        matches.forEach(match => results.push({
+                            url: match,
+                            source: 'html_content',
+                        }));
+                    }
+                });
+            }
             return [...new Set(results.map(r => r.url))].map(url => ({ url, source: 'script' }));
-        });
+        }, { deepScan });
         return {
             content: [{
                     type: 'text',

package/dist/optimization-utils.js

@@ -0,0 +1,479 @@
+// Optimization Utilities for All MCP Tools
+// @ts-nocheck
+import { sleep } from './system-utils.js';
+/**
+ * Tool Optimization Configuration
+ * Centralized defaults for all 113 tools
+ */
+export const TOOL_OPTIMIZATION_CONFIG = {
+    // Network Monitoring Tools (9 tools)
+    networkMonitoring: {
+        defaultDuration: 10000,
+        maxDuration: 30000,
+        monitoringTimeout: 15000,
+        retryAttempts: 3,
+        retryDelay: 2000,
+    },
+    // Video Extraction Tools (30+ tools)
+    videoExtraction: {
+        iframeDepthLimit: 5,
+        defaultTimeout: 12000,
+        hosterPatterns: [
+            'vidcloud',
+            'vidsrc',
+            'filemoon',
+            'streamtape',
+            'doodstream',
+            'mixdrop',
+            'upstream',
+            'streamwish',
+            'vidmoly',
+            'vidlox',
+            'mystream',
+            'cloudemb',
+            'embedsrc',
+            'upns.online',
+            'voe.sx',
+            'streamlare',
+            'dailymotion',
+            'youtube',
+            'vimeo',
+            'twitch',
+        ],
+        videoSelectors: [
+            'video',
+            'iframe[src*="player"]',
+            'iframe[src*="embed"]',
+            'div[class*="player"]',
+            'div[id*="player"]',
+            '[data-video-url]',
+            '[data-src*=".mp4"]',
+            '[data-src*=".m3u8"]',
+            '[data-src*=".mkv"]',
+            '[data-src*=".webm"]',
+        ],
+        downloadButtonSelectors: [
+            'a[download]',
+            'button[download]',
+            'a[href*="download"]',
+            'button[data-download]',
+            'a[href*=".mp4"]',
+            'a[href*=".mkv"]',
+            'a[href*=".webm"]',
+            'a[class*="download"]',
+            '.download-btn',
+            '.btn-download',
+            '[class*="download"]',
+            '[id*="download"]',
+        ],
+    },
+    // Data Extraction Tools (40+ tools)
+    dataExtraction: {
+        deepScanEnabled: true,
+        recursiveSearchDepth: 5,
+        deduplicationEnabled: true,
+        cachingEnabled: true,
+        cacheTTL: 300000, // 5 minutes
+        defaultTimeout: 10000,
+    },
+    // Redirect Tracing
+    redirectTracing: {
+        maxRedirects: 15,
+        redirectTimeout: 30000,
+        followIframes: true,
+        maxDepth: 5,
+    },
+    // Global Settings
+    global: {
+        defaultTimeout: 10000,
+        retryAttempts: 3,
+        retryDelay: 2000,
+        enableErrorLogging: true,
+        enableCaching: true,
+        enableDeduplication: true,
+    },
+};
+/**
+ * Result Deduplication - Remove duplicate entries
+ */
+export function deduplicateResults(results, uniqueKey = 'url') {
+    const seen = new Set();
+    return results.filter((item) => {
+        const key = item[uniqueKey] || JSON.stringify(item);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/**
+ * Deep Deduplication - For complex objects
+ */
+export function deepDeduplicateResults(results) {
+    const seen = new Set();
+    return results.filter((item) => {
+        const key = JSON.stringify(item);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/**
+ * Result Caching with TTL
+ */
+export class ResultCache {
+    cache = new Map();
+    set(key, data, ttl = 300000) {
+        this.cache.set(key, {
+            data,
+            timestamp: Date.now(),
+            ttl,
+        });
+    }
+    get(key) {
+        const cached = this.cache.get(key);
+        if (!cached)
+            return null;
+        const age = Date.now() - cached.timestamp;
+        if (age > cached.ttl) {
+            this.cache.delete(key);
+            return null;
+        }
+        return cached.data;
+    }
+    clear() {
+        this.cache.clear();
+    }
+    getStats() {
+        return {
+            size: this.cache.size,
+            entries: Array.from(this.cache.keys()),
+        };
+    }
+}
+/**
+ * Global cache instance for all tools
+ */
+export const globalCache = new ResultCache();
+/**
+ * Retry with Exponential Backoff
+ */
+export async function retryWithBackoff(fn, maxAttempts = 3, initialDelay = 1000) {
+    let lastError;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            console.error(`Attempt ${attempt}/${maxAttempts} failed:`, lastError.message);
+            if (attempt < maxAttempts) {
+                const delay = initialDelay * Math.pow(2, attempt - 1);
+                await sleep(delay);
+            }
+        }
+    }
+    throw lastError;
+}
+/**
+ * Enhanced Error Handler for Tools
+ */
+export function createErrorHandler(toolName) {
+    return {
+        handle(error, context = '') {
+            const message = error instanceof Error ? error.message : String(error);
+            console.error(`[${toolName}] Error in ${context}: ${message}`);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `❌ ${toolName} Error: ${message}`,
+                    },
+                ],
+                isError: true,
+                toolName,
+                context,
+            };
+        },
+    };
+}
+/**
+ * Tool Execution Wrapper with Timeout and Retry
+ */
+export async function executeToolWithOptimizations(toolName, toolFn, options = {}) {
+    const { timeout = TOOL_OPTIMIZATION_CONFIG.global.defaultTimeout, retryAttempts = TOOL_OPTIMIZATION_CONFIG.global.retryAttempts, retryDelay = TOOL_OPTIMIZATION_CONFIG.global.retryDelay, cacheKey, shouldCache = TOOL_OPTIMIZATION_CONFIG.global.enableCaching, } = options;
+    // Check cache first
+    if (cacheKey && shouldCache) {
+        const cached = globalCache.get(cacheKey);
+        if (cached) {
+            console.log(`[${toolName}] Cache hit for key: ${cacheKey}`);
+            return cached;
+        }
+    }
+    try {
+        // Execute with timeout
+        const result = await Promise.race([
+            toolFn(),
+            new Promise((_, reject) => setTimeout(() => reject(new Error(`${toolName} timeout after ${timeout}ms`)), timeout)),
+        ]);
+        // Cache result if enabled
+        if (cacheKey && shouldCache) {
+            globalCache.set(cacheKey, result);
+        }
+        return result;
+    }
+    catch (error) {
+        // Retry on failure for network tools
+        if (retryAttempts > 0) {
+            console.log(`[${toolName}] Retrying after error: ${error instanceof Error ? error.message : String(error)}`);
+            return retryWithBackoff(toolFn, retryAttempts, retryDelay);
+        }
+        throw error;
+    }
+}
+/**
+ * Resource Cleanup for Tools
+ */
+export async function cleanupToolResources(page, context) {
+    try {
+        // Remove all event listeners
+        if (page) {
+            await page.removeAllListeners?.();
+        }
+        // Close unnecessary tabs
+        if (context) {
+            const pages = await context.pages?.();
+            if (pages && pages.length > 1) {
+                for (const p of pages.slice(1)) {
+                    await p.close?.();
+                }
+            }
+        }
+        // Clear cache periodically
+        const cacheStats = globalCache.getStats();
+        if (cacheStats.size > 100) {
+            console.log('[Cleanup] Clearing cache - size exceeded 100 items');
+            globalCache.clear();
+        }
+    }
+    catch (error) {
+        console.error('[Cleanup] Error during cleanup:', error);
+    }
+}
+/**
+ * Video Hoster Detection Database
+ */
+export const VIDEO_HOSTERS_DB = {
+    primary: [
+        { name: 'vidcloud', patterns: ['vidcloud', 'vidnode'], embedPattern: '/embed/' },
+        { name: 'vidsrc', patterns: ['vidsrc.to', 'vidsrc.me', 'vidsrc.xyz'] },
+        { name: 'filemoon', patterns: ['filemoon.sx', 'filemoon.to'] },
+        { name: 'streamtape', patterns: ['streamtape.com', 'streamtape.to'] },
+        { name: 'doodstream', patterns: ['dood.', 'doodstream.'] },
+        { name: 'mixdrop', patterns: ['mixdrop.co', 'mixdrop.to'] },
+        { name: 'upstream', patterns: ['upstream.to', 'upstreamcdn'] },
+        { name: 'streamwish', patterns: ['streamwish.to', 'streamwish.com'] },
+    ],
+    methods: {
+        iframe: { depth: 5, timeout: 12000 },
+        directLink: { patterns: ['.m3u8', '.mp4', '.mkv', '/stream/', '.webm'] },
+        apiEndpoint: { patterns: ['/api/source', '/api/video', '/get-source', '/embed'] },
+    },
+    isKnownHoster(url) {
+        const lowerUrl = url.toLowerCase();
+        for (const hoster of this.primary) {
+            if (hoster.patterns.some((p) => lowerUrl.includes(p))) {
+                return true;
+            }
+        }
+        return false;
+    },
+    getHosterName(url) {
+        const lowerUrl = url.toLowerCase();
+        for (const hoster of this.primary) {
+            if (hoster.patterns.some((p) => lowerUrl.includes(p))) {
+                return hoster.name;
+            }
+        }
+        return null;
+    },
+};
+/**
+ * Common Selector Utilities
+ */
+export const SELECTOR_UTILS = {
+    // Link selectors
+    linkSelectors: [
+        'a[href]',
+        'a[href^="http"]',
+        'a[href^="/"]',
+        '[data-href]',
+        'button[onclick*="http"]',
+    ],
+    // Image selectors
+    imageSelectors: [
+        'img',
+        'img[src]',
+        'picture img',
+        '[style*="background-image"]',
+        'div[data-src*="image"]',
+        'div[data-background*="image"]',
+    ],
+    // Video selectors: Already defined above
+    videoSelectors: TOOL_OPTIMIZATION_CONFIG.videoExtraction.videoSelectors,
+    // Download button selectors: Already defined above
+    downloadSelectors: TOOL_OPTIMIZATION_CONFIG.videoExtraction.downloadButtonSelectors,
+    // Form selectors
+    formSelectors: ['form', 'input', 'textarea', 'select', 'button[type="submit"]'],
+    // Table selectors
+    tableSelectors: ['table', 'tbody tr', 'thead tr', '.table-row', '[role="table"]'],
+    // Pagination selectors
+    paginationSelectors: [
+        'a[rel="next"]',
+        'button[aria-label*="next"]',
+        '.pagination a',
+        '.next',
+        'a[href*="page"]',
+        'a[href*="offset"]',
+    ],
+    // Hidden element detection
+    hiddenSelectors: [
+        '[style*="display: none"]',
+        '[style*="visibility: hidden"]',
+        '[hidden]',
+        '.hidden',
+        '.d-none',
+    ],
+    // All selectors
+    getAllSelectors() {
+        return {
+            links: this.linkSelectors,
+            images: this.imageSelectors,
+            videos: this.videoSelectors,
+            downloads: this.downloadSelectors,
+            forms: this.formSelectors,
+            tables: this.tableSelectors,
+            pagination: this.paginationSelectors,
+            hidden: this.hiddenSelectors,
+        };
+    },
+};
+/**
+ * Performance Metrics Collection
+ */
+export class PerformanceMetrics {
+    metrics = new Map();
+    start(toolName) {
+        this.metrics.set(toolName, { start: Date.now() });
+    }
+    end(toolName) {
+        const metric = this.metrics.get(toolName);
+        if (metric) {
+            metric.end = Date.now();
+            metric.duration = metric.end - metric.start;
+            return metric.duration;
+        }
+        return 0;
+    }
+    getMetrics(toolName) {
+        return this.metrics.get(toolName) || null;
+    }
+    getAllMetrics() {
+        return this.metrics;
+    }
+    getAverageDuration() {
+        let totalDuration = 0;
+        let count = 0;
+        for (const metric of this.metrics.values()) {
+            if (metric.duration) {
+                totalDuration += metric.duration;
+                count++;
+            }
+        }
+        return count > 0 ? totalDuration / count : 0;
+    }
+    reset() {
+        this.metrics.clear();
+    }
+}
+/**
+ * Global metrics instance
+ */
+export const globalMetrics = new PerformanceMetrics();
+export class ToolStatusTracker {
+    tools = new Map();
+    register(toolName, category, optimizations = []) {
+        this.tools.set(toolName, {
+            name: toolName,
+            category,
+            status: 'pending',
+            optimizations,
+        });
+    }
+    updateStatus(toolName, status, performanceGain) {
+        const tool = this.tools.get(toolName);
+        if (tool) {
+            tool.status = status;
+            if (performanceGain !== undefined) {
+                tool.performanceGain = performanceGain;
+            }
+        }
+    }
+    recordExecution(toolName, duration) {
+        const tool = this.tools.get(toolName);
+        if (tool) {
+            tool.lastExecutionTime = duration;
+        }
+    }
+    getStatus(toolName) {
+        return this.tools.get(toolName) || null;
+    }
+    getAllStatus() {
+        return Array.from(this.tools.values());
+    }
+    getSummary() {
+        let optimized = 0;
+        let pending = 0;
+        let failed = 0;
+        const byCategory = new Map();
+        for (const tool of this.tools.values()) {
+            if (tool.status === 'optimized')
+                optimized++;
+            else if (tool.status === 'pending')
+                pending++;
+            else if (tool.status === 'failed')
+                failed++;
+            const count = byCategory.get(tool.category) || 0;
+            byCategory.set(tool.category, count + 1);
+        }
+        return {
+            total: this.tools.size,
+            optimized,
+            pending,
+            failed,
+            byCategory,
+        };
+    }
+}
+/**
+ * Global tool status tracker
+ */
+export const globalToolStatus = new ToolStatusTracker();
+export default {
+    TOOL_OPTIMIZATION_CONFIG,
+    deduplicateResults,
+    deepDeduplicateResults,
+    ResultCache,
+    globalCache,
+    retryWithBackoff,
+    createErrorHandler,
+    executeToolWithOptimizations,
+    cleanupToolResources,
+    VIDEO_HOSTERS_DB,
+    SELECTOR_UTILS,
+    PerformanceMetrics,
+    globalMetrics,
+    ToolStatusTracker,
+    globalToolStatus,
+};

package/dist/tool-definitions.js

@@ -942,7 +942,7 @@ export const TOOLS = [
             properties: {
                 url: { type: 'string' },
                 analyzeNetworkRequests: { type: 'boolean', default: true },
-                scanDuration: { type: 'number', default: 5000 },
+                scanDuration: { type: 'number', default: 10000 },
             },
         },
     },
@@ -1024,7 +1024,7 @@ export const TOOLS = [
         inputSchema: {
             type: 'object',
             properties: {
-                duration: { type: 'number', default: 5000 },
+                duration: { type: 'number', default: 15000 },
                 url: { type: 'string' },
             },
         },
@@ -1035,7 +1035,7 @@ export const TOOLS = [
         inputSchema: {
             type: 'object',
             properties: {
-                duration: { type: 'number', default: 5000 },
+                duration: { type: 'number', default: 15000 },
             },
         },
     },
@@ -1045,8 +1045,8 @@ export const TOOLS = [
         inputSchema: {
             type: 'object',
             properties: {
-                duration: { type: 'number', default: 10000 },
-                filterTypes: { type: 'array', items: { type: 'string' }, default: ['all'] },
+                duration: { type: 'number', default: 20000 },
+                filterTypes: { type: 'array', items: { type: 'string' }, default: ['video', 'xhr', 'fetch', 'media'] },
             },
         },
     },
@@ -1055,7 +1055,9 @@ export const TOOLS = [
         description: 'Discover API endpoints on page',
         inputSchema: {
             type: 'object',
-            properties: {},
+            properties: {
+                deepScan: { type: 'boolean', default: true },
+            },
         },
     },
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brave-real-browser-mcp-server",
-  "version": "2.11.0",
+  "version": "2.11.2",
   "description": "MCP server for brave-real-browser",
   "type": "module",
   "main": "dist/index.js",