brave-real-browser-mcp-server 2.28.1 → 2.29.1

package/README.md

@@ -107,6 +107,115 @@ npm run dev
 
 ---
 
+## 🆕 New in v2.28.1 - Advanced Enhancements
+
+### 🌐 Advanced Navigation
+| Option | Description |
+|--------|-------------|
+| `blockResources` | Block images, fonts, CSS for faster loading |
+| `customHeaders` | Set custom HTTP headers |
+| `referrer` | Custom referrer URL |
+| `waitForSelector` | Wait for specific element after load |
+| `waitForContent` | Wait for specific text content |
+| `scrollToBottom` | Auto-scroll for lazy loading |
+| `randomDelay` | Human-like delay (100-500ms) |
+| `bypassCSP` | Bypass Content Security Policy |
+
+**Example:**
+```json
+{
+  "url": "https://example.com",
+  "blockResources": ["image", "font"],
+  "waitForSelector": ".main-content",
+  "scrollToBottom": true,
+  "randomDelay": true
+}
+```
+
+### 🚀 Parallel Scraping (js_scrape)
+| Option | Description |
+|--------|-------------|
+| `urls` | Array of multiple URLs to scrape |
+| `concurrency` | Max concurrent scrapes (1-10) |
+| `continueOnError` | Continue even if some URLs fail |
+| `delayBetween` | Delay between scrapes (ms) |
+
+**Example:**
+```json
+{
+  "urls": ["https://site1.com", "https://site2.com", "https://site3.com"],
+  "concurrency": 3,
+  "extractSelector": "article"
+}
+```
+
+### 📄 Auto-Pagination (link_harvester)
+| Option | Description |
+|--------|-------------|
+| `followPagination` | Auto-follow pagination links |
+| `maxPages` | Maximum pages to scrape (1-20) |
+| `paginationSelector` | Custom next page selector |
+| `delayBetweenPages` | Delay between pages (ms) |
+
+**Example:**
+```json
+{
+  "followPagination": true,
+  "maxPages": 10,
+  "filter": "movie"
+}
+```
+
+### 🔌 API Interceptor (network_recorder)
+| Option | Description |
+|--------|-------------|
+| `interceptMode` | `record`, `intercept`, or `mock` |
+| `blockPatterns` | URL patterns to block |
+| `mockResponses` | Fake responses for URLs |
+| `modifyHeaders` | Modify request headers |
+| `capturePayloads` | Capture POST/PUT bodies |
+
+**Example:**
+```json
+{
+  "interceptMode": "intercept",
+  "blockPatterns": ["ads", "tracking"],
+  "capturePayloads": true
+}
+```
+
+### 🛡️ Anti-Detection Enhancements
+
+| Feature | Description |
+|---------|-------------|
+| **Fingerprint Randomizer** | Canvas, Audio, Hardware randomization |
+| **Human Behavior Simulation** | Natural mouse/scroll patterns |
+| **WebGL Spoofing** | Random GPU from 8 configs (Intel, NVIDIA, AMD) |
+| **Proxy Rotation** | round-robin, random, least-used, on-error strategies |
+| **Rate Limiter** | Per-second/minute limits, domain-specific |
+| **Error Auto-Recovery** | Smart retry with different strategies |
+
+### 🔄 Internal Systems
+
+```javascript
+// Rate Limiter
+initRateLimiter({ requestsPerSecond: 5, requestsPerMinute: 100 });
+setDomainRateLimit('api.example.com', 2); // 2 req/sec for this domain
+
+// Proxy Rotation
+initProxyRotation(['proxy1:8080', 'proxy2:8080'], 'round-robin');
+rotateProxy('error'); // Rotate on error
+
+// Error Recovery
+executeWithRecovery(operation, {
+  toolName: 'navigate',
+  page: pageInstance,
+  restartBrowser: () => browser.restart()
+});
+```
+
+---
+
 ## 🌐 Unified MCP+LSP+SSE Ecosystem
 
 **एक command से सब active:**
@@ -117,7 +226,7 @@ npm run dev
 
 ### Output:
 ```
-🦁 Brave Real Browser - Unified Server v2.22.0
+🦁 Brave Real Browser - Unified Server v2.28.1
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 📡 HTTP Server: http://localhost:3000
 
@@ -185,14 +294,14 @@ brave-real-launcher
 ### Navigation
 | Tool | Description |
 |------|-------------|
-| `navigate` | Navigate to URL |
+| `navigate` | Navigate to URL with advanced options (resource blocking, custom headers, auto-scroll) |
 | `wait` | Wait for conditions |
 
 ### Content
 | Tool | Description |
 |------|-------------|
 | `get_content` | Get page HTML/text |
-| `find_element` | Find by selector/text/AI |
+| `find_element` | Find by selector/text/AI with batch mode |
 | `save_content_as_markdown` | Save as markdown file |
 
 ### Interaction
@@ -204,34 +313,32 @@ brave-real-launcher
 | `random_scroll` | Natural scrolling |
 | `solve_captcha` | Solve CAPTCHAs |
 
-### Media Extraction
+### Media & Streaming
+| Tool | Description |
+|------|-------------|
+| `stream_extractor` | Extract streams with multi-quality selector, VidSrc/Filemoon/StreamWish support |
+| `player_api_hook` | Hook into JWPlayer, Video.js, HLS.js, Plyr, Vidstack, DASH.js |
+| `iframe_handler` | Deep iframe scraping with video source extraction |
+
+### Scraping
 | Tool | Description |
 |------|-------------|
-| `media_extractor` | Extract video/audio |
-| `m3u8_parser` | Parse HLS streams |
-| `stream_extractor` | Direct download URLs |
+| `js_scrape` | JavaScript-rendered scraping with parallel URL support |
+| `link_harvester` | Harvest links with auto-pagination |
+| `network_recorder` | Record traffic with API interception |
+| `extract_json` | Extract embedded JSON with AES decryption |
+| `search_regex` | Regex search like regex101.com |
 
 ### Advanced
 | Tool | Description |
 |------|-------------|
-| `search_content` | Search patterns |
-| `extract_json` | Extract embedded JSON |
 | `scrape_meta_tags` | Meta/OG tags |
-| `deep_analysis` | Full page analysis |
-| `network_recorder` | Record traffic |
-| `api_finder` | Discover APIs |
-| `ajax_content_waiter` | Wait for AJAX |
-| `link_harvester` | Harvest links |
-| `batch_element_scraper` | Batch scrape |
-| `extract_schema` | Schema.org data |
-| `element_screenshot` | Screenshot element |
-| `breadcrumb_navigator` | Navigate breadcrumbs |
+| `deep_analysis` | Full page analysis with screenshot |
 | `redirect_tracer` | Trace redirects |
 | `progress_tracker` | Track progress |
 | `cookie_manager` | Manage cookies |
 | `file_downloader` | Download files |
-| `iframe_handler` | Handle iframes |
-| `popup_handler` | Handle popups |
+| `execute_js` | Run custom JavaScript |
 
 ---
 

package/dist/browser-manager.js

@@ -36,6 +36,350 @@ function loadEnvFile() {
     }
 }
 loadEnvFile();
+// Global proxy rotation state
+let proxyRotation = {
+    enabled: false,
+    proxies: [],
+    rotationStrategy: 'round-robin',
+    rotateAfterRequests: 50,
+    rotateOnBlock: true,
+    currentIndex: 0,
+    requestCount: 0,
+    proxyStats: new Map()
+};
+// Initialize proxy rotation
+export function initProxyRotation(proxies, strategy = 'round-robin') {
+    const parsedProxies = proxies.map(p => {
+        if (typeof p === 'string') {
+            // Parse string format: protocol://user:pass@host:port or host:port
+            const match = p.match(/^(?:(https?|socks5):\/\/)?(?:([^:]+):([^@]+)@)?([^:]+):(\d+)$/);
+            if (match) {
+                return {
+                    protocol: match[1] || 'http',
+                    username: match[2],
+                    password: match[3],
+                    host: match[4],
+                    port: parseInt(match[5], 10)
+                };
+            }
+            // Simple host:port
+            const [host, port] = p.split(':');
+            return { host, port: parseInt(port, 10), protocol: 'http' };
+        }
+        return p;
+    });
+    proxyRotation = {
+        enabled: true,
+        proxies: parsedProxies,
+        rotationStrategy: strategy,
+        rotateAfterRequests: 50,
+        rotateOnBlock: true,
+        currentIndex: 0,
+        requestCount: 0,
+        proxyStats: new Map()
+    };
+    // Initialize stats for each proxy
+    parsedProxies.forEach(p => {
+        const key = `${p.host}:${p.port}`;
+        proxyRotation.proxyStats.set(key, { uses: 0, failures: 0, lastUsed: 0 });
+    });
+    console.log(`[ProxyRotation] Initialized with ${parsedProxies.length} proxies, strategy: ${strategy}`);
+}
+// Get current proxy
+export function getCurrentProxy() {
+    if (!proxyRotation.enabled || proxyRotation.proxies.length === 0) {
+        return null;
+    }
+    return proxyRotation.proxies[proxyRotation.currentIndex];
+}
+// Get proxy as URL string
+export function getCurrentProxyUrl() {
+    const proxy = getCurrentProxy();
+    if (!proxy)
+        return null;
+    let url = `${proxy.protocol || 'http'}://`;
+    if (proxy.username && proxy.password) {
+        url += `${proxy.username}:${proxy.password}@`;
+    }
+    url += `${proxy.host}:${proxy.port}`;
+    return url;
+}
+// Rotate to next proxy
+export function rotateProxy(reason = 'scheduled') {
+    if (!proxyRotation.enabled || proxyRotation.proxies.length === 0) {
+        return null;
+    }
+    const prevProxy = getCurrentProxy();
+    const prevKey = prevProxy ? `${prevProxy.host}:${prevProxy.port}` : '';
+    switch (proxyRotation.rotationStrategy) {
+        case 'round-robin':
+            proxyRotation.currentIndex = (proxyRotation.currentIndex + 1) % proxyRotation.proxies.length;
+            break;
+        case 'random':
+            proxyRotation.currentIndex = Math.floor(Math.random() * proxyRotation.proxies.length);
+            break;
+        case 'least-used':
+            let minUses = Infinity;
+            let minIndex = 0;
+            proxyRotation.proxies.forEach((p, i) => {
+                const key = `${p.host}:${p.port}`;
+                const stats = proxyRotation.proxyStats.get(key);
+                if (stats && stats.uses < minUses) {
+                    minUses = stats.uses;
+                    minIndex = i;
+                }
+            });
+            proxyRotation.currentIndex = minIndex;
+            break;
+        case 'on-error':
+            // Only rotate on error/blocked, otherwise keep current
+            if (reason === 'error' || reason === 'blocked') {
+                proxyRotation.currentIndex = (proxyRotation.currentIndex + 1) % proxyRotation.proxies.length;
+            }
+            break;
+    }
+    proxyRotation.requestCount = 0;
+    const newProxy = getCurrentProxy();
+    const newKey = newProxy ? `${newProxy.host}:${newProxy.port}` : '';
+    // Update stats
+    if (newKey && proxyRotation.proxyStats.has(newKey)) {
+        const stats = proxyRotation.proxyStats.get(newKey);
+        stats.uses++;
+        stats.lastUsed = Date.now();
+        if (reason === 'error' || reason === 'blocked') {
+            const prevStats = proxyRotation.proxyStats.get(prevKey);
+            if (prevStats)
+                prevStats.failures++;
+        }
+    }
+    console.log(`[ProxyRotation] Rotated proxy (${reason}): ${prevKey} -> ${newKey}`);
+    return newProxy;
+}
+// Check if proxy should be rotated
+export function shouldRotateProxy() {
+    if (!proxyRotation.enabled)
+        return false;
+    proxyRotation.requestCount++;
+    if (proxyRotation.rotateAfterRequests &&
+        proxyRotation.requestCount >= proxyRotation.rotateAfterRequests) {
+        return true;
+    }
+    return false;
+}
+// Mark current proxy as blocked/failed
+export function markProxyFailed(blocked = false) {
+    if (!proxyRotation.enabled)
+        return;
+    const proxy = getCurrentProxy();
+    if (proxy) {
+        const key = `${proxy.host}:${proxy.port}`;
+        const stats = proxyRotation.proxyStats.get(key);
+        if (stats) {
+            stats.failures++;
+        }
+    }
+    if (proxyRotation.rotateOnBlock && blocked) {
+        rotateProxy('blocked');
+    }
+}
+// Get proxy stats
+export function getProxyStats() {
+    return {
+        enabled: proxyRotation.enabled,
+        currentProxy: getCurrentProxyUrl(),
+        totalProxies: proxyRotation.proxies.length,
+        currentIndex: proxyRotation.currentIndex,
+        requestCount: proxyRotation.requestCount,
+        strategy: proxyRotation.rotationStrategy,
+        stats: Object.fromEntries(proxyRotation.proxyStats)
+    };
+}
+// Global rate limiter configuration
+let rateLimiterConfig = {
+    enabled: false,
+    requestsPerSecond: 5,
+    requestsPerMinute: 100,
+    burstLimit: 10,
+    domainLimits: new Map(),
+    globalCooldown: 100,
+    retryAfter: 5000
+};
+// Global rate limiter state
+let rateLimiterState = {
+    lastRequestTime: 0,
+    requestsInSecond: 0,
+    requestsInMinute: 0,
+    secondWindowStart: Date.now(),
+    minuteWindowStart: Date.now(),
+    domainRequestTimes: new Map(),
+    isThrottled: false,
+    throttleUntil: 0
+};
+/**
+ * Initialize rate limiter with custom configuration
+ */
+export function initRateLimiter(config = {}) {
+    rateLimiterConfig = {
+        ...rateLimiterConfig,
+        enabled: true,
+        ...config
+    };
+    // Reset state
+    rateLimiterState = {
+        lastRequestTime: 0,
+        requestsInSecond: 0,
+        requestsInMinute: 0,
+        secondWindowStart: Date.now(),
+        minuteWindowStart: Date.now(),
+        domainRequestTimes: new Map(),
+        isThrottled: false,
+        throttleUntil: 0
+    };
+    console.log(`[RateLimiter] Initialized: ${rateLimiterConfig.requestsPerSecond}/sec, ${rateLimiterConfig.requestsPerMinute}/min`);
+}
+/**
+ * Set rate limit for a specific domain
+ */
+export function setDomainRateLimit(domain, requestsPerSecond, requestsPerMinute) {
+    rateLimiterConfig.domainLimits.set(domain, {
+        requestsPerSecond,
+        requestsPerMinute: requestsPerMinute || requestsPerSecond * 60
+    });
+    console.log(`[RateLimiter] Domain limit set for ${domain}: ${requestsPerSecond}/sec`);
+}
+/**
+ * Get the delay needed before making a request
+ * Returns 0 if request can be made immediately
+ */
+export function getRateLimitDelay(url) {
+    if (!rateLimiterConfig.enabled)
+        return 0;
+    const now = Date.now();
+    // Check if we're in a throttle period
+    if (rateLimiterState.isThrottled && now < rateLimiterState.throttleUntil) {
+        return rateLimiterState.throttleUntil - now;
+    }
+    else if (rateLimiterState.isThrottled) {
+        rateLimiterState.isThrottled = false;
+    }
+    // Update sliding windows
+    if (now - rateLimiterState.secondWindowStart >= 1000) {
+        rateLimiterState.requestsInSecond = 0;
+        rateLimiterState.secondWindowStart = now;
+    }
+    if (now - rateLimiterState.minuteWindowStart >= 60000) {
+        rateLimiterState.requestsInMinute = 0;
+        rateLimiterState.minuteWindowStart = now;
+    }
+    // Check global limits
+    if (rateLimiterState.requestsInSecond >= rateLimiterConfig.requestsPerSecond) {
+        return 1000 - (now - rateLimiterState.secondWindowStart);
+    }
+    if (rateLimiterState.requestsInMinute >= rateLimiterConfig.requestsPerMinute) {
+        return 60000 - (now - rateLimiterState.minuteWindowStart);
+    }
+    // Check global cooldown
+    const timeSinceLastRequest = now - rateLimiterState.lastRequestTime;
+    if (timeSinceLastRequest < rateLimiterConfig.globalCooldown) {
+        return rateLimiterConfig.globalCooldown - timeSinceLastRequest;
+    }
+    // Check domain-specific limits
+    if (url) {
+        try {
+            const domain = new URL(url).hostname;
+            const domainLimit = rateLimiterConfig.domainLimits.get(domain);
+            if (domainLimit) {
+                const domainTimes = rateLimiterState.domainRequestTimes.get(domain) || [];
+                // Clean up old timestamps
+                const recentTimes = domainTimes.filter(t => now - t < 60000);
+                rateLimiterState.domainRequestTimes.set(domain, recentTimes);
+                // Check domain limits
+                const lastSecondRequests = recentTimes.filter(t => now - t < 1000).length;
+                if (lastSecondRequests >= domainLimit.requestsPerSecond) {
+                    const oldestInSecond = recentTimes.filter(t => now - t < 1000)[0];
+                    return oldestInSecond ? 1000 - (now - oldestInSecond) : 1000;
+                }
+                const lastMinuteRequests = recentTimes.length;
+                if (lastMinuteRequests >= domainLimit.requestsPerMinute) {
+                    const oldestInMinute = recentTimes[0];
+                    return oldestInMinute ? 60000 - (now - oldestInMinute) : 60000;
+                }
+            }
+        }
+        catch {
+            // Invalid URL, ignore domain limits
+        }
+    }
+    return 0;
+}
+/**
+ * Wait for rate limit if needed, then record the request
+ */
+export async function waitForRateLimit(url) {
+    const delay = getRateLimitDelay(url);
+    if (delay > 0) {
+        console.log(`[RateLimiter] Waiting ${delay}ms before request${url ? ` to ${new URL(url).hostname}` : ''}`);
+        await new Promise(resolve => setTimeout(resolve, delay));
+    }
+    // Record the request
+    recordRequest(url);
+}
+/**
+ * Record a request for rate limiting tracking
+ */
+export function recordRequest(url) {
+    if (!rateLimiterConfig.enabled)
+        return;
+    const now = Date.now();
+    rateLimiterState.lastRequestTime = now;
+    rateLimiterState.requestsInSecond++;
+    rateLimiterState.requestsInMinute++;
+    // Record domain-specific request
+    if (url) {
+        try {
+            const domain = new URL(url).hostname;
+            const domainTimes = rateLimiterState.domainRequestTimes.get(domain) || [];
+            domainTimes.push(now);
+            rateLimiterState.domainRequestTimes.set(domain, domainTimes);
+        }
+        catch {
+            // Invalid URL
+        }
+    }
+}
+/**
+ * Signal that a rate limit response (429) was received
+ */
+export function onRateLimitHit(retryAfterMs) {
+    const delay = retryAfterMs || rateLimiterConfig.retryAfter;
+    rateLimiterState.isThrottled = true;
+    rateLimiterState.throttleUntil = Date.now() + delay;
+    console.log(`[RateLimiter] Rate limit hit! Throttling for ${delay}ms`);
+}
+/**
+ * Get current rate limiter stats
+ */
+export function getRateLimiterStats() {
+    const now = Date.now();
+    return {
+        enabled: rateLimiterConfig.enabled,
+        requestsInSecond: rateLimiterState.requestsInSecond,
+        requestsInMinute: rateLimiterState.requestsInMinute,
+        limitsPerSecond: rateLimiterConfig.requestsPerSecond,
+        limitsPerMinute: rateLimiterConfig.requestsPerMinute,
+        isThrottled: rateLimiterState.isThrottled,
+        throttleRemaining: rateLimiterState.isThrottled ? Math.max(0, rateLimiterState.throttleUntil - now) : 0,
+        domainLimits: Object.fromEntries(rateLimiterConfig.domainLimits),
+        cooldown: rateLimiterConfig.globalCooldown
+    };
+}
+/**
+ * Disable rate limiter
+ */
+export function disableRateLimiter() {
+    rateLimiterConfig.enabled = false;
+    console.log('[RateLimiter] Disabled');
+}
 // Browser error categorization
 export var BrowserErrorType;
 (function (BrowserErrorType) {