brave-real-browser-mcp-server 2.8.0 → 2.8.1

package/dist/browser-manager.js

@@ -726,6 +726,13 @@ export function getBrowserInstance() {
 export function getPageInstance() {
     return pageInstance;
 }
+// Alias for getPageInstance for compatibility
+export function getCurrentPage() {
+    if (!pageInstance) {
+        throw new Error('No page instance available. Please initialize browser first.');
+    }
+    return pageInstance;
+}
 export function getContentPriorityConfig() {
     return contentPriorityConfig;
 }

package/dist/handlers/ai-powered-handlers.js

@@ -0,0 +1,367 @@
+// @ts-nocheck
+import { getPageInstance } from '../browser-manager.js';
+import natural from 'natural';
+import Sentiment from 'sentiment';
+import { franc } from 'franc';
+const sentiment = new Sentiment();
+const tokenizer = new natural.WordTokenizer();
+/**
+ * Smart Selector Generator - AI-powered CSS selector generation
+ */
+export async function handleSmartSelectorGenerator(args) {
+    const { url, description, context } = args;
+    try {
+        const page = getPageInstance();
+        if (!page) {
+            throw new Error('Browser not initialized. Call browser_init first.');
+        }
+        if (url && page.url() !== url) {
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+        }
+        const result = await page.evaluate((desc, ctx) => {
+            const elements = Array.from(document.querySelectorAll('*'));
+            const scores = [];
+            const keywords = desc.toLowerCase().split(/\s+/);
+            elements.forEach((el) => {
+                let score = 0;
+                const text = el.textContent?.toLowerCase() || '';
+                const tag = el.tagName.toLowerCase();
+                const className = el.className || '';
+                const id = el.id || '';
+                // Score based on text content matching
+                keywords.forEach(keyword => {
+                    if (text.includes(keyword))
+                        score += 10;
+                    if (className.includes(keyword))
+                        score += 5;
+                    if (id.includes(keyword))
+                        score += 5;
+                });
+                // Score based on context
+                if (ctx) {
+                    const contextKeywords = ctx.toLowerCase().split(/\s+/);
+                    contextKeywords.forEach(keyword => {
+                        if (text.includes(keyword))
+                            score += 3;
+                    });
+                }
+                // Prefer semantic elements
+                const semanticTags = ['button', 'input', 'a', 'nav', 'header', 'footer', 'article', 'section'];
+                if (semanticTags.includes(tag))
+                    score += 2;
+                if (score > 0) {
+                    // Generate selector
+                    let selector = tag;
+                    if (id)
+                        selector = `#${id}`;
+                    else if (className)
+                        selector = `${tag}.${className.split(' ')[0]}`;
+                    scores.push({ selector, score, text: text.substring(0, 100), element: tag });
+                }
+            });
+            // Sort by score
+            scores.sort((a, b) => b.score - a.score);
+            return {
+                bestMatch: scores[0] || null,
+                alternatives: scores.slice(1, 6),
+                totalCandidates: scores.length
+            };
+        }, description, context || '');
+        return {
+            success: true,
+            data: result,
+            description,
+            context
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * Content Classification - Classify webpage content into categories
+ */
+export async function handleContentClassification(args) {
+    const { url, categories } = args;
+    try {
+        const page = getPageInstance();
+        if (!page) {
+            throw new Error('Browser not initialized. Call browser_init first.');
+        }
+        if (url && page.url() !== url) {
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+        }
+        const content = await page.evaluate(() => {
+            return {
+                title: document.title,
+                text: document.body.innerText,
+                metaDescription: document.querySelector('meta[name="description"]')?.getAttribute('content') || '',
+                metaKeywords: document.querySelector('meta[name="keywords"]')?.getAttribute('content') || '',
+                headings: Array.from(document.querySelectorAll('h1, h2, h3')).map(h => h.textContent).join(' ')
+            };
+        });
+        const allText = [content.title, content.text, content.metaDescription, content.metaKeywords, content.headings].join(' ').toLowerCase();
+        // Define default categories if not provided
+        const defaultCategories = [
+            { name: 'E-commerce', keywords: ['shop', 'buy', 'cart', 'price', 'product', 'order', 'checkout'] },
+            { name: 'News', keywords: ['news', 'article', 'breaking', 'report', 'journalist', 'story'] },
+            { name: 'Blog', keywords: ['blog', 'post', 'author', 'comment', 'share', 'subscribe'] },
+            { name: 'Social Media', keywords: ['follow', 'like', 'share', 'post', 'friend', 'profile'] },
+            { name: 'Educational', keywords: ['learn', 'course', 'tutorial', 'education', 'study', 'lesson'] },
+            { name: 'Entertainment', keywords: ['video', 'movie', 'music', 'game', 'play', 'watch'] },
+            { name: 'Business', keywords: ['business', 'company', 'service', 'enterprise', 'solution'] },
+            { name: 'Technology', keywords: ['tech', 'software', 'app', 'code', 'developer', 'api'] }
+        ];
+        const categoriesToUse = categories || defaultCategories;
+        const scores = categoriesToUse.map((cat) => {
+            const keywords = Array.isArray(cat.keywords) ? cat.keywords : cat.keywords.split(',');
+            let score = 0;
+            keywords.forEach((keyword) => {
+                const regex = new RegExp(keyword.trim().toLowerCase(), 'g');
+                const matches = allText.match(regex);
+                score += matches ? matches.length : 0;
+            });
+            return { category: cat.name, score };
+        });
+        scores.sort((a, b) => b.score - a.score);
+        return {
+            success: true,
+            primaryCategory: scores[0],
+            allScores: scores,
+            confidence: scores[0].score / (scores.reduce((sum, s) => sum + s.score, 0) || 1)
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * Sentiment Analysis - Analyze sentiment of page content
+ */
+export async function handleSentimentAnalysis(args) {
+    const { url, selector, text } = args;
+    try {
+        let contentToAnalyze = text;
+        if (!contentToAnalyze && url) {
+            const page = getPageInstance();
+            if (!page) {
+                throw new Error('Browser not initialized. Call browser_init first.');
+            }
+            if (page.url() !== url) {
+                await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+            }
+            if (selector) {
+                contentToAnalyze = await page.evaluate((sel) => {
+                    const element = document.querySelector(sel);
+                    return element ? element.textContent : '';
+                }, selector);
+            }
+            else {
+                contentToAnalyze = await page.evaluate(() => document.body.innerText);
+            }
+        }
+        if (!contentToAnalyze) {
+            throw new Error('No content to analyze');
+        }
+        const result = sentiment.analyze(contentToAnalyze);
+        // Additional analysis using natural
+        const tokens = tokenizer.tokenize(contentToAnalyze);
+        const sentenceTokenizer = new natural.SentenceTokenizer();
+        const sentences = sentenceTokenizer.tokenize(contentToAnalyze);
+        // Classify sentiment per sentence
+        const sentenceSentiments = sentences.map(sentence => {
+            const s = sentiment.analyze(sentence);
+            return {
+                sentence: sentence.substring(0, 100),
+                score: s.score,
+                sentiment: s.score > 0 ? 'positive' : s.score < 0 ? 'negative' : 'neutral'
+            };
+        });
+        return {
+            success: true,
+            overall: {
+                score: result.score,
+                comparative: result.comparative,
+                sentiment: result.score > 0 ? 'positive' : result.score < 0 ? 'negative' : 'neutral',
+                tokens: result.tokens.length,
+                positive: result.positive,
+                negative: result.negative
+            },
+            sentences: sentenceSentiments,
+            statistics: {
+                totalSentences: sentences.length,
+                positiveSentences: sentenceSentiments.filter(s => s.sentiment === 'positive').length,
+                negativeSentences: sentenceSentiments.filter(s => s.sentiment === 'negative').length,
+                neutralSentences: sentenceSentiments.filter(s => s.sentiment === 'neutral').length
+            }
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * Summary Generator - Generate summary of page content
+ */
+export async function handleSummaryGenerator(args) {
+    const { url, maxSentences = 5, selector } = args;
+    try {
+        const page = getPageInstance();
+        if (!page) {
+            throw new Error('Browser not initialized. Call browser_init first.');
+        }
+        if (url && page.url() !== url) {
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+        }
+        let content;
+        if (selector) {
+            content = await page.evaluate((sel) => {
+                const element = document.querySelector(sel);
+                return element ? element.textContent : '';
+            }, selector);
+        }
+        else {
+            content = await page.evaluate(() => {
+                // Extract main content
+                const main = document.querySelector('main, article, .content, .post, #content');
+                if (main)
+                    return main.textContent || '';
+                return document.body.innerText;
+            });
+        }
+        if (!content) {
+            throw new Error('No content found');
+        }
+        // Use TF-IDF for extractive summarization
+        const TfIdf = natural.TfIdf;
+        const tfidf = new TfIdf();
+        const sentenceTokenizer = new natural.SentenceTokenizer();
+        const sentences = sentenceTokenizer.tokenize(content);
+        if (sentences.length === 0) {
+            throw new Error('No sentences found to summarize');
+        }
+        // Add each sentence as a document
+        sentences.forEach(sentence => {
+            tfidf.addDocument(sentence);
+        });
+        // Score each sentence
+        const sentenceScores = sentences.map((sentence, idx) => {
+            let score = 0;
+            tfidf.listTerms(idx).forEach(term => {
+                score += term.tfidf;
+            });
+            return { sentence, score, index: idx };
+        });
+        // Sort by score and take top N
+        sentenceScores.sort((a, b) => b.score - a.score);
+        const topSentences = sentenceScores.slice(0, maxSentences);
+        // Sort by original order
+        topSentences.sort((a, b) => a.index - b.index);
+        const summary = topSentences.map(s => s.sentence).join(' ');
+        return {
+            success: true,
+            summary,
+            originalLength: content.length,
+            summaryLength: summary.length,
+            compressionRatio: (summary.length / content.length * 100).toFixed(2) + '%',
+            sentenceCount: {
+                original: sentences.length,
+                summary: topSentences.length
+            },
+            topScoredSentences: topSentences.map(s => ({
+                sentence: s.sentence,
+                score: s.score.toFixed(2)
+            }))
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * Translation Support - Detect language and provide translation info
+ */
+export async function handleTranslationSupport(args) {
+    const { url, selector, text, targetLanguage = 'en' } = args;
+    try {
+        let contentToTranslate = text;
+        if (!contentToTranslate && url) {
+            const page = getPageInstance();
+            if (!page) {
+                throw new Error('Browser not initialized. Call browser_init first.');
+            }
+            if (page.url() !== url) {
+                await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+            }
+            if (selector) {
+                contentToTranslate = await page.evaluate((sel) => {
+                    const element = document.querySelector(sel);
+                    return element ? element.textContent : '';
+                }, selector);
+            }
+            else {
+                contentToTranslate = await page.evaluate(() => document.body.innerText);
+            }
+        }
+        if (!contentToTranslate) {
+            throw new Error('No content to translate');
+        }
+        // Detect language using franc
+        const detectedLang = franc(contentToTranslate, { minLength: 10 });
+        // Get language name
+        const langNames = {
+            'eng': 'English',
+            'spa': 'Spanish',
+            'fra': 'French',
+            'deu': 'German',
+            'ita': 'Italian',
+            'por': 'Portuguese',
+            'rus': 'Russian',
+            'jpn': 'Japanese',
+            'kor': 'Korean',
+            'cmn': 'Chinese (Mandarin)',
+            'ara': 'Arabic',
+            'hin': 'Hindi',
+            'und': 'Undetermined'
+        };
+        const languageName = langNames[detectedLang] || detectedLang;
+        // Extract key phrases using TF-IDF
+        const TfIdf = natural.TfIdf;
+        const tfidf = new TfIdf();
+        tfidf.addDocument(contentToTranslate);
+        const keyPhrases = tfidf.listTerms(0)
+            .slice(0, 10)
+            .map(term => term.term);
+        return {
+            success: true,
+            detectedLanguage: {
+                code: detectedLang,
+                name: languageName
+            },
+            targetLanguage,
+            needsTranslation: detectedLang !== targetLanguage && detectedLang !== 'und',
+            contentPreview: contentToTranslate.substring(0, 200),
+            contentLength: contentToTranslate.length,
+            keyPhrases,
+            translationNote: 'Use external translation API (Google Translate, DeepL) for actual translation'
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}

package/dist/handlers/api-integration-handlers.js

@@ -0,0 +1,314 @@
+// @ts-nocheck
+import { getPageInstance } from '../browser-manager.js';
+import axios from 'axios';
+/**
+ * REST API Endpoint Finder - Discover REST API endpoints
+ */
+export async function handleRESTAPIEndpointFinder(args) {
+    const { url, analyzeNetworkRequests = true, scanDuration = 5000 } = args;
+    try {
+        const page = getPageInstance();
+        if (!page) {
+            throw new Error('Browser not initialized. Call browser_init first.');
+        }
+        const apiEndpoints = [];
+        const seenUrls = new Set();
+        // Listen to network requests
+        if (analyzeNetworkRequests) {
+            const requestHandler = (request) => {
+                const requestUrl = request.url();
+                const method = request.method();
+                const resourceType = request.resourceType();
+                // Filter for API-like requests
+                if ((resourceType === 'xhr' || resourceType === 'fetch') &&
+                    !seenUrls.has(requestUrl)) {
+                    seenUrls.add(requestUrl);
+                    const headers = request.headers();
+                    const postData = request.postData();
+                    apiEndpoints.push({
+                        url: requestUrl,
+                        method,
+                        resourceType,
+                        headers: Object.keys(headers).reduce((acc, key) => {
+                            if (!key.toLowerCase().includes('cookie') && !key.toLowerCase().includes('authorization')) {
+                                acc[key] = headers[key];
+                            }
+                            return acc;
+                        }, {}),
+                        hasBody: !!postData,
+                        timestamp: new Date().toISOString()
+                    });
+                }
+            };
+            page.on('request', requestHandler);
+            // Navigate and wait
+            if (url && page.url() !== url) {
+                await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+            }
+            // Wait for additional requests
+            await page.waitForTimeout(scanDuration);
+            page.off('request', requestHandler);
+        }
+        // Also scan page content for API endpoints
+        const discoveredAPIs = await page.evaluate(() => {
+            const apis = [];
+            const scripts = Array.from(document.querySelectorAll('script'));
+            // Common API URL patterns
+            const apiPatterns = [
+                /https?:\/\/[^"'\s]+\/api\/[^"'\s]*/gi,
+                /https?:\/\/api\.[^"'\s]+/gi,
+                /\/api\/v?\d*\/[^"'\s]*/gi,
+                /graphql/gi,
+                /rest\/v?\d*/gi
+            ];
+            scripts.forEach(script => {
+                const content = script.textContent || '';
+                apiPatterns.forEach(pattern => {
+                    const matches = content.match(pattern);
+                    if (matches) {
+                        matches.forEach(match => apis.push({
+                            url: match,
+                            source: 'script_content',
+                            type: 'discovered'
+                        }));
+                    }
+                });
+            });
+            // Check for data attributes
+            const elements = Array.from(document.querySelectorAll('[data-api], [data-endpoint], [data-url]'));
+            elements.forEach(el => {
+                const apiUrl = el.getAttribute('data-api') || el.getAttribute('data-endpoint') || el.getAttribute('data-url');
+                if (apiUrl) {
+                    apis.push({
+                        url: apiUrl,
+                        source: 'data_attribute',
+                        element: el.tagName.toLowerCase()
+                    });
+                }
+            });
+            return apis;
+        });
+        return {
+            success: true,
+            networkRequests: {
+                count: apiEndpoints.length,
+                endpoints: apiEndpoints
+            },
+            discoveredInContent: {
+                count: discoveredAPIs.length,
+                endpoints: discoveredAPIs.slice(0, 20) // Limit to 20
+            },
+            summary: {
+                totalFound: apiEndpoints.length + discoveredAPIs.length,
+                uniqueNetworkAPIs: apiEndpoints.length,
+                discoveredAPIs: discoveredAPIs.length
+            }
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * Webhook Support - Set up and test webhooks
+ */
+export async function handleWebhookSupport(args) {
+    const { webhookUrl, method = 'POST', payload, headers, testMode = true } = args;
+    try {
+        if (!webhookUrl) {
+            throw new Error('Webhook URL is required');
+        }
+        const defaultHeaders = {
+            'Content-Type': 'application/json',
+            'User-Agent': 'Brave-MCP-Server/1.0',
+            ...headers
+        };
+        let response;
+        if (testMode) {
+            // Test webhook with ping
+            try {
+                response = await axios({
+                    method,
+                    url: webhookUrl,
+                    headers: defaultHeaders,
+                    data: payload || { test: true, timestamp: new Date().toISOString() },
+                    timeout: 10000
+                });
+                return {
+                    success: true,
+                    webhookUrl,
+                    method,
+                    testMode: true,
+                    response: {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: response.headers,
+                        data: response.data
+                    }
+                };
+            }
+            catch (webhookError) {
+                return {
+                    success: false,
+                    webhookUrl,
+                    testMode: true,
+                    error: webhookError.message,
+                    details: {
+                        status: webhookError.response?.status,
+                        statusText: webhookError.response?.statusText,
+                        data: webhookError.response?.data
+                    }
+                };
+            }
+        }
+        else {
+            // Production mode - set up webhook listener
+            return {
+                success: true,
+                webhookUrl,
+                method,
+                testMode: false,
+                status: 'configured',
+                note: 'Webhook configured. Send data using separate call or integrate with scraping workflow'
+            };
+        }
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}
+/**
+ * All Website API Finder - Comprehensive API discovery
+ */
+export async function handleAllWebsiteAPIFinder(args) {
+    const { url, deepScan = true, includeExternal = false } = args;
+    try {
+        const page = getPageInstance();
+        if (!page) {
+            throw new Error('Browser not initialized. Call browser_init first.');
+        }
+        if (url && page.url() !== url) {
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+        }
+        const apiDiscovery = await page.evaluate((deep, external) => {
+            const result = {
+                apis: [],
+                graphql: [],
+                rest: [],
+                websockets: [],
+                metadata: {}
+            };
+            // 1. Check for API documentation links
+            const apiLinks = Array.from(document.querySelectorAll('a[href*="api"], a[href*="docs"], a[href*="developer"]'));
+            result.documentationLinks = apiLinks.map(link => ({
+                text: link.textContent?.trim(),
+                href: link.href
+            })).slice(0, 10);
+            // 2. Scan for GraphQL
+            const graphqlIndicators = document.body.innerHTML.match(/graphql|__schema|query\s*{|mutation\s*{/gi);
+            if (graphqlIndicators) {
+                result.graphql.push({
+                    found: true,
+                    indicators: graphqlIndicators.length,
+                    possibleEndpoints: [
+                        '/graphql',
+                        '/api/graphql',
+                        '/v1/graphql'
+                    ]
+                });
+            }
+            // 3. Scan scripts for API configurations
+            const scripts = Array.from(document.querySelectorAll('script'));
+            scripts.forEach(script => {
+                const content = script.textContent || '';
+                // Look for API base URLs
+                const baseUrlPatterns = [
+                    /apiUrl\s*[:=]\s*["']([^"']+)["']/gi,
+                    /baseURL\s*[:=]\s*["']([^"']+)["']/gi,
+                    /API_BASE\s*[:=]\s*["']([^"']+)["']/gi,
+                    /endpoint\s*[:=]\s*["']([^"']+)["']/gi
+                ];
+                baseUrlPatterns.forEach(pattern => {
+                    const matches = Array.from(content.matchAll(pattern));
+                    matches.forEach(match => {
+                        if (match[1] && (external || match[1].startsWith('/'))) {
+                            result.apis.push({
+                                type: 'config',
+                                url: match[1],
+                                source: 'script'
+                            });
+                        }
+                    });
+                });
+                // Look for WebSocket connections
+                if (content.includes('WebSocket') || content.includes('ws://') || content.includes('wss://')) {
+                    const wsMatches = content.match(/wss?:\/\/[^"'\s]+/gi);
+                    if (wsMatches) {
+                        wsMatches.forEach(ws => {
+                            result.websockets.push({
+                                url: ws,
+                                protocol: ws.startsWith('wss') ? 'secure' : 'unsecure'
+                            });
+                        });
+                    }
+                }
+            });
+            // 4. Check meta tags for API info
+            const metaTags = Array.from(document.querySelectorAll('meta[name*="api"], meta[property*="api"]'));
+            result.metadata.apiMeta = metaTags.map(meta => ({
+                name: meta.getAttribute('name') || meta.getAttribute('property'),
+                content: meta.getAttribute('content')
+            }));
+            // 5. Look for REST patterns
+            const restPatterns = [
+                '/api/v1', '/api/v2', '/api/v3',
+                '/rest/v1', '/rest/v2',
+                '/api/users', '/api/products', '/api/data'
+            ];
+            restPatterns.forEach(pattern => {
+                const found = document.body.innerHTML.includes(pattern);
+                if (found) {
+                    result.rest.push({
+                        pattern,
+                        found: true
+                    });
+                }
+            });
+            // 6. Check for Swagger/OpenAPI
+            const swaggerLinks = Array.from(document.querySelectorAll('a[href*="swagger"], a[href*="openapi"], link[href*="swagger"]'));
+            if (swaggerLinks.length > 0) {
+                result.swagger = swaggerLinks.map(link => ({
+                    href: link.href || link.href
+                }));
+            }
+            return result;
+        }, deepScan, includeExternal);
+        // Deduplicate APIs
+        const uniqueAPIs = [...new Set(apiDiscovery.apis.map((api) => api.url))];
+        return {
+            success: true,
+            summary: {
+                totalAPIsFound: uniqueAPIs.length,
+                graphqlDetected: apiDiscovery.graphql.length > 0,
+                restEndpointsFound: apiDiscovery.rest.filter((r) => r.found).length,
+                websocketsFound: apiDiscovery.websockets.length,
+                documentationLinks: apiDiscovery.documentationLinks?.length || 0
+            },
+            details: apiDiscovery,
+            uniqueAPIs: uniqueAPIs.slice(0, 20),
+            recommendations: []
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+}