brave-real-browser-mcp-server 2.27.32 → 2.29.0

package/dist/handlers/advanced-tools.js

@@ -1707,6 +1707,7 @@ export async function handleExtractJson(page, args) {
         try {
             const key = args.decryptAES.key || "kiemtienmua911ca";
             const ivList = args.decryptAES.ivList || ["1234567890oiuytr", "0123456789abcdef"];
+            const autoIV = args.decryptAES.autoIV !== false; // Default to true for hubstream-style encryption
             let input = args.decryptAES.input || "";
             // Option: Fetch encrypted data from API URL directly (recommended for hubstream)
             if (!input && args.decryptAES.fetchUrl) {
@@ -1733,24 +1734,43 @@ export async function handleExtractJson(page, args) {
                             }
                             return bytes;
                         };
-                        for (const iv of params.ivList) {
+                        // Build IV list - if autoIV, try extracting from first 16 bytes first
+                        const ivsToTry = [];
+                        if (params.autoIV && encryptedHex.length >= 32) {
+                            // First 32 hex chars = first 16 bytes = IV (hubstream style)
+                            const ivHex = encryptedHex.substring(0, 32);
+                            ivsToTry.push({ iv: hexToBytes(ivHex), label: 'autoIV (first 16 bytes)' });
+                        }
+                        // Add manual IV list
+                        for (const ivStr of params.ivList) {
+                            ivsToTry.push({ iv: new TextEncoder().encode(ivStr), label: ivStr });
+                        }
+                        for (const { iv: ivData, label: ivLabel } of ivsToTry) {
                             try {
                                 const keyData = new TextEncoder().encode(params.key);
-                                const ivData = new TextEncoder().encode(iv);
-                                const encryptedData = hexToBytes(encryptedHex);
+                                // If using autoIV, encrypted data starts after first 32 hex chars (16 bytes)
+                                const isAutoIV = ivLabel.startsWith('autoIV');
+                                const dataHex = isAutoIV ? encryptedHex.substring(32) : encryptedHex;
+                                const encryptedData = hexToBytes(dataHex);
                                 const cryptoKey = await crypto.subtle.importKey("raw", keyData, "AES-CBC", false, ["decrypt"]);
                                 const decrypted = await crypto.subtle.decrypt({ name: "AES-CBC", iv: ivData }, cryptoKey, encryptedData);
                                 const decryptedText = new TextDecoder().decode(decrypted);
-                                // Extract source URL from decrypted JSON
+                                // Extract URLs from decrypted JSON
                                 const sourceMatch = /"source":"([^"]+)"/.exec(decryptedText);
+                                const mp4Match = /"mp4":"([^"]+)"/.exec(decryptedText);
+                                const hlsMatch = /"hls":"([^"]+)"/.exec(decryptedText);
                                 const streamUrl = sourceMatch ? sourceMatch[1].replace(/\\\//g, '/') : null;
+                                const mp4Url = mp4Match ? mp4Match[1].replace(/\\\//g, '/') : null;
+                                const hlsUrl = hlsMatch ? hlsMatch[1].replace(/\\\//g, '/') : null;
                                 return {
                                     success: true,
-                                    iv: iv,
+                                    iv: ivLabel,
                                     decrypted: decryptedText,
                                     decryptedLength: decryptedText.length,
                                     extractedStreamUrl: streamUrl,
-                                    isStreamUrl: streamUrl ? (streamUrl.includes('m3u8') || streamUrl.includes('.mp4')) : false,
+                                    extractedMp4Url: mp4Url,
+                                    extractedHlsUrl: hlsUrl,
+                                    isStreamUrl: (streamUrl && (streamUrl.includes('m3u8') || streamUrl.includes('.mp4'))) || !!mp4Url,
                                     inputLength: encryptedHex.length
                                 };
                             }
@@ -1763,7 +1783,7 @@ export async function handleExtractJson(page, args) {
                     catch (e) {
                         return { success: false, error: String(e) };
                     }
-                }, { input, key, ivList, fetchUrl: args.decryptAES.fetchUrl });
+                }, { input, key, ivList, autoIV, fetchUrl: args.decryptAES.fetchUrl });
                 decoded = {
                     source: 'decryptAES',
                     ...result,
@@ -2306,8 +2326,9 @@ export async function handleDeepAnalysis(page, args) {
     }
 }
 /**
- * Record full network traffic - Uses response events to avoid crashes
+ * Network recorder with API interception capabilities
  * ULTRA POWERFUL: API detection, media URLs, smart categorization
+ * NEW: Request interception, mocking, blocking, and header modification
  */
 export async function handleNetworkRecorder(page, args) {
     // Progress tracking
@@ -2321,7 +2342,15 @@ export async function handleNetworkRecorder(page, args) {
     const apis = [];
     const mediaUrls = [];
     const seen = new Set();
-    tracker.setProgress(10, `⏱️ Recording for ${duration}ms...`);
+    const interceptedRequests = [];
+    let blockedCount = 0;
+    let mockedCount = 0;
+    const interceptMode = args.interceptMode || 'record';
+    const blockPatterns = args.blockPatterns || [];
+    const mockResponses = args.mockResponses || [];
+    const modifyHeaders = args.modifyHeaders || [];
+    const capturePayloads = args.capturePayloads === true;
+    tracker.setProgress(10, `⏱️ Recording for ${duration}ms (mode: ${interceptMode})...`);
     // ============================================================
     // SMART CATEGORIZATION HELPER
     // ============================================================
@@ -2355,75 +2384,216 @@ export async function handleNetworkRecorder(page, args) {
             return 'document';
         return 'other';
     };
-    // Response handler - safer than request interception
-    const responseHandler = (response) => {
-        try {
-            const url = response.url();
-            // Dedup
-            if (seen.has(url))
-                return;
-            seen.add(url);
-            if (args.filterUrl && !url.includes(args.filterUrl)) {
-                return;
-            }
-            const resourceType = response.request()?.resourceType?.() || 'unknown';
-            const method = response.request()?.method?.() || 'GET';
-            const category = categorizeUrl(url, resourceType);
-            categories[category] = (categories[category] || 0) + 1;
-            // Collect API endpoints
-            if (category === 'api' || resourceType === 'xhr' || resourceType === 'fetch') {
-                apis.push({ url, method, type: resourceType });
+    // Helper to check URL against patterns
+    const matchesPattern = (url, patterns) => {
+        return patterns.some(pattern => {
+            try {
+                if (pattern.startsWith('/') && pattern.endsWith('/')) {
+                    // Regex pattern
+                    const regex = new RegExp(pattern.slice(1, -1));
+                    return regex.test(url);
+                }
+                // Simple includes check
+                return url.includes(pattern);
             }
-            // Collect media URLs
-            if (category === 'media' || /\.(mp4|webm|m3u8|ts|mp3)/i.test(url)) {
-                mediaUrls.push(url);
+            catch {
+                return url.includes(pattern);
             }
-            const entry = {
-                url,
-                status: response.status(),
-                resourceType,
-                category,
-                method,
-                timestamp: Date.now(),
-            };
-            if (args.includeHeaders) {
-                try {
-                    entry.headers = response.headers();
+        });
+    };
+    // ============================================================
+    // INTERCEPTION MODE - Uses request interception
+    // ============================================================
+    if (interceptMode === 'intercept' || interceptMode === 'mock') {
+        try {
+            await page.setRequestInterception(true);
+            const requestHandler = async (request) => {
+                const url = request.url();
+                const method = request.method();
+                const resourceType = request.resourceType();
+                const category = categorizeUrl(url, resourceType);
+                // Check if should block
+                if (blockPatterns.length > 0 && matchesPattern(url, blockPatterns)) {
+                    blockedCount++;
+                    interceptedRequests.push({
+                        url,
+                        method,
+                        action: 'blocked',
+                        timestamp: Date.now()
+                    });
+                    await request.abort();
+                    return;
                 }
-                catch (e) {
-                    entry.headers = {};
+                // Check if should mock
+                const mockConfig = mockResponses.find(m => matchesPattern(url, [m.urlPattern]));
+                if (mockConfig) {
+                    mockedCount++;
+                    interceptedRequests.push({
+                        url,
+                        method,
+                        action: 'mocked',
+                        mockResponse: mockConfig.response,
+                        timestamp: Date.now()
+                    });
+                    await request.respond({
+                        status: mockConfig.statusCode || 200,
+                        contentType: 'application/json',
+                        body: typeof mockConfig.response === 'string'
+                            ? mockConfig.response
+                            : JSON.stringify(mockConfig.response)
+                    });
+                    return;
+                }
+                // Check if should modify headers
+                const headerConfig = modifyHeaders.find(h => matchesPattern(url, [h.urlPattern]));
+                if (headerConfig) {
+                    const headers = {
+                        ...request.headers(),
+                        ...headerConfig.headers
+                    };
+                    interceptedRequests.push({
+                        url,
+                        method,
+                        action: 'headers_modified',
+                        modifiedHeaders: headerConfig.headers,
+                        timestamp: Date.now()
+                    });
+                    await request.continue({ headers });
+                    return;
+                }
+                // Continue normally but record
+                if (!seen.has(url)) {
+                    seen.add(url);
+                    categories[category] = (categories[category] || 0) + 1;
+                    const entry = {
+                        url,
+                        method,
+                        resourceType,
+                        category,
+                        timestamp: Date.now()
+                    };
+                    // Capture POST/PUT payloads
+                    if (capturePayloads && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
+                        try {
+                            entry.payload = request.postData();
+                        }
+                        catch {
+                            // Ignore
+                        }
+                    }
+                    requests.push(entry);
+                    // Collect API endpoints
+                    if (category === 'api' || resourceType === 'xhr' || resourceType === 'fetch') {
+                        apis.push({
+                            url,
+                            method,
+                            type: resourceType,
+                            payload: entry.payload
+                        });
+                    }
+                    // Collect media URLs
+                    if (category === 'media' || /\.(mp4|webm|m3u8|ts|mp3)/i.test(url)) {
+                        mediaUrls.push(url);
+                    }
                 }
+                await request.continue();
+            };
+            page.on('request', requestHandler);
+            await new Promise(r => setTimeout(r, duration));
+            page.off('request', requestHandler);
+            await page.setRequestInterception(false);
+        }
+        catch (e) {
+            // Cleanup on error
+            try {
+                await page.setRequestInterception(false);
             }
-            requests.push(entry);
-            // Track size from headers
+            catch { }
+        }
+    }
+    else {
+        // ============================================================
+        // RECORD MODE - Uses response events (safer)
+        // ============================================================
+        const responseHandler = (response) => {
             try {
-                const headers = response.headers();
-                const size = parseInt(headers['content-length'] || '0', 10);
-                totalSize += size;
+                const url = response.url();
+                // Dedup
+                if (seen.has(url))
+                    return;
+                seen.add(url);
+                if (args.filterUrl && !url.includes(args.filterUrl)) {
+                    return;
+                }
+                const resourceType = response.request()?.resourceType?.() || 'unknown';
+                const method = response.request()?.method?.() || 'GET';
+                const category = categorizeUrl(url, resourceType);
+                categories[category] = (categories[category] || 0) + 1;
+                // Collect API endpoints
+                if (category === 'api' || resourceType === 'xhr' || resourceType === 'fetch') {
+                    const apiEntry = { url, method, type: resourceType };
+                    // Capture POST data if enabled
+                    if (capturePayloads && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
+                        try {
+                            apiEntry.payload = response.request()?.postData?.();
+                        }
+                        catch { }
+                    }
+                    apis.push(apiEntry);
+                }
+                // Collect media URLs
+                if (category === 'media' || /\.(mp4|webm|m3u8|ts|mp3)/i.test(url)) {
+                    mediaUrls.push(url);
+                }
+                const entry = {
+                    url,
+                    status: response.status(),
+                    resourceType,
+                    category,
+                    method,
+                    timestamp: Date.now(),
+                };
+                if (args.includeHeaders) {
+                    try {
+                        entry.headers = response.headers();
+                    }
+                    catch (e) {
+                        entry.headers = {};
+                    }
+                }
+                requests.push(entry);
+                // Track size from headers
+                try {
+                    const headers = response.headers();
+                    const size = parseInt(headers['content-length'] || '0', 10);
+                    totalSize += size;
+                }
+                catch {
+                    // Ignore
+                }
             }
             catch {
-                // Ignore
+                // Ignore all errors in handler to prevent crash
             }
-        }
-        catch {
-            // Ignore all errors in handler to prevent crash
-        }
-    };
-    try {
-        page.on('response', responseHandler);
-        await new Promise((r) => setTimeout(r, duration));
-    }
-    catch (e) {
-        // Capture setup errors
-    }
-    finally {
+        };
         try {
-            page.off('response', responseHandler);
+            page.on('response', responseHandler);
+            await new Promise((r) => setTimeout(r, duration));
         }
         catch (e) {
-            // Ignore cleanup errors
+            // Capture setup errors
+        }
+        finally {
+            try {
+                page.off('response', responseHandler);
+            }
+            catch (e) {
+                // Ignore cleanup errors
+            }
         }
     }
+    tracker.setProgress(90, `✅ Recorded ${requests.length} requests`);
+    tracker.complete(`🎉 Network recording complete`);
     return {
         requests: requests.slice(0, 500),
         count: requests.length,
@@ -2431,7 +2601,12 @@ export async function handleNetworkRecorder(page, args) {
         categories,
         apis: apis.length > 0 ? apis : undefined,
         mediaUrls: mediaUrls.length > 0 ? mediaUrls : undefined,
-        message: `📡 Recorded ${requests.length} requests (${Math.round(totalSize / 1024)}KB) | APIs: ${apis.length} | Media: ${mediaUrls.length}`
+        blockedCount: blockedCount > 0 ? blockedCount : undefined,
+        mockedCount: mockedCount > 0 ? mockedCount : undefined,
+        interceptedRequests: interceptedRequests.length > 0 ? interceptedRequests : undefined,
+        message: `📡 Recorded ${requests.length} requests (${Math.round(totalSize / 1024)}KB) | APIs: ${apis.length} | Media: ${mediaUrls.length}` +
+            (blockedCount > 0 ? ` | Blocked: ${blockedCount}` : '') +
+            (mockedCount > 0 ? ` | Mocked: ${mockedCount}` : '')
     };
 }
 /**
@@ -2905,6 +3080,7 @@ export async function handleVideoRecording(page, args, recorderState) {
 /**
  * Harvest all links from page
  * ULTRA POWERFUL: Pagination detection, smart categorization, file types
+ * NEW: Auto-follow pagination to scrape multiple pages
  */
 export async function handleLinkHarvester(page, args) {
     // Progress tracking for real-time updates
@@ -2913,76 +3089,110 @@ export async function handleLinkHarvester(page, args) {
     tracker.start(100, '🔗 Starting link harvesting...');
     const currentUrl = new URL(page.url());
     tracker.setProgress(10, `📍 Analyzing page: ${currentUrl.hostname}`);
-    // ============================================================
-    // 1. EXTRACT ALL LINKS WITH SMART CATEGORIZATION
-    // ============================================================
-    tracker.setProgress(20, '🔍 Extracting all links from page...');
-    const allLinks = await page.evaluate(() => {
-        const links = [];
-        document.querySelectorAll('a[href]').forEach((a) => {
-            const anchor = a;
-            links.push({
-                url: anchor.href,
-                text: a.textContent?.trim()?.substring(0, 100) || '',
-                attrs: {
-                    rel: anchor.rel || '',
-                    target: anchor.target || '',
-                    class: anchor.className || '',
-                    id: anchor.id || '',
-                    download: anchor.download || '',
-                }
+    // Pagination settings
+    const followPagination = args.followPagination === true;
+    const maxPages = Math.min(args.maxPages || 5, 20); // Max 20 pages
+    const delayBetweenPages = args.delayBetweenPages || 1000;
+    const paginationSelector = args.paginationSelector;
+    // Helper function to extract links from current page
+    const extractLinksFromPage = async () => {
+        const allLinks = await page.evaluate(() => {
+            const links = [];
+            document.querySelectorAll('a[href]').forEach((a) => {
+                const anchor = a;
+                links.push({
+                    url: anchor.href,
+                    text: a.textContent?.trim()?.substring(0, 100) || '',
+                    attrs: {
+                        rel: anchor.rel || '',
+                        target: anchor.target || '',
+                        class: anchor.className || '',
+                        id: anchor.id || '',
+                        download: anchor.download || '',
+                    }
+                });
             });
+            return links;
         });
-        return links;
-    });
-    // ============================================================
-    // 2. PAGINATION DETECTION
-    // ============================================================
-    const pagination = await page.evaluate(() => {
-        let nextPage;
-        let prevPage;
-        let totalPages;
-        // Common pagination selectors
-        const nextSelectors = [
-            'a[rel="next"]', 'a.next', 'a.pagination-next',
-            '[aria-label="Next"]', 'a:has-text("Next")', 'a:has-text(">")',
-            '.pagination a:last-child', 'a.page-link:last-child'
-        ];
-        const prevSelectors = [
-            'a[rel="prev"]', 'a.prev', 'a.pagination-prev',
-            '[aria-label="Previous"]', 'a:has-text("Prev")', 'a:has-text("<")'
-        ];
-        for (const sel of nextSelectors) {
-            try {
-                const el = document.querySelector(sel);
-                if (el?.href) {
-                    nextPage = el.href;
-                    break;
+        // Pagination detection
+        const pagination = await page.evaluate((customSelector) => {
+            let nextPage;
+            let prevPage;
+            let totalPages;
+            let currentPage;
+            // Custom selector first
+            if (customSelector) {
+                try {
+                    const el = document.querySelector(customSelector);
+                    if (el?.href)
+                        nextPage = el.href;
+                }
+                catch { /* invalid selector */ }
+            }
+            // Common pagination selectors
+            const nextSelectors = [
+                'a[rel="next"]', 'a.next', 'a.pagination-next',
+                '[aria-label="Next"]', 'a.page-link.next', '.next a',
+                '.pagination a:last-child', 'a[title="Next"]',
+                'a[aria-label*="next" i]', 'button.next', '[data-testid="next"]'
+            ];
+            const prevSelectors = [
+                'a[rel="prev"]', 'a.prev', 'a.pagination-prev',
+                '[aria-label="Previous"]', 'a.page-link.prev', '.prev a'
+            ];
+            if (!nextPage) {
+                for (const sel of nextSelectors) {
+                    try {
+                        const el = document.querySelector(sel);
+                        if (el?.href) {
+                            nextPage = el.href;
+                            break;
+                        }
+                    }
+                    catch { /* invalid selector */ }
                 }
             }
-            catch { /* invalid selector */ }
-        }
-        for (const sel of prevSelectors) {
-            try {
-                const el = document.querySelector(sel);
-                if (el?.href) {
-                    prevPage = el.href;
-                    break;
+            // Text-based next detection
+            if (!nextPage) {
+                const links = Array.from(document.querySelectorAll('a'));
+                for (const link of links) {
+                    const text = link.textContent?.toLowerCase().trim() || '';
+                    if (text === 'next' || text === 'next →' || text === '>' || text === '»' || text === 'next page') {
+                        nextPage = link.href;
+                        break;
+                    }
                 }
             }
-            catch { /* invalid selector */ }
-        }
-        // Count page numbers
-        const pageNumbers = Array.from(document.querySelectorAll('.pagination a, .page-numbers a, nav a'))
-            .map(a => parseInt(a.textContent || '0', 10))
-            .filter(n => !isNaN(n) && n > 0);
-        if (pageNumbers.length > 0) {
-            totalPages = Math.max(...pageNumbers);
-        }
-        return { nextPage, prevPage, totalPages };
-    });
+            for (const sel of prevSelectors) {
+                try {
+                    const el = document.querySelector(sel);
+                    if (el?.href) {
+                        prevPage = el.href;
+                        break;
+                    }
+                }
+                catch { /* invalid selector */ }
+            }
+            // Detect current page and total pages
+            const pageNumbers = Array.from(document.querySelectorAll('.pagination a, .page-numbers a, nav a, .pager a'))
+                .map(a => ({
+                num: parseInt(a.textContent || '0', 10),
+                isActive: a.classList.contains('active') || a.classList.contains('current') ||
+                    a.getAttribute('aria-current') === 'page'
+            }))
+                .filter(p => !isNaN(p.num) && p.num > 0);
+            if (pageNumbers.length > 0) {
+                totalPages = Math.max(...pageNumbers.map(p => p.num));
+                const active = pageNumbers.find(p => p.isActive);
+                if (active)
+                    currentPage = active.num;
+            }
+            return { nextPage, prevPage, totalPages, currentPage };
+        }, paginationSelector);
+        return { links: allLinks, pagination };
+    };
     // ============================================================
-    // 3. SMART LINK CATEGORIZATION
+    // SMART LINK CATEGORIZATION
     // ============================================================
     const categorizeLink = (url, text, attrs) => {
         const urlLower = url.toLowerCase();
@@ -3017,55 +3227,107 @@ export async function handleLinkHarvester(page, args) {
             return 'info';
         return 'navigation';
     };
+    // ============================================================
+    // MAIN SCRAPING LOGIC
+    // ============================================================
     const processedLinks = [];
     const categories = {};
     const seen = new Set();
     let internal = 0;
     let external = 0;
-    for (const link of allLinks) {
-        try {
-            // Dedup by URL
-            if (seen.has(link.url))
-                continue;
-            seen.add(link.url);
-            const linkUrl = new URL(link.url);
-            const isInternal = linkUrl.hostname === currentUrl.hostname;
-            if (args.filter && !link.url.includes(args.filter) && !link.text.includes(args.filter)) {
-                continue;
-            }
-            if (isInternal && args.includeInternal === false)
-                continue;
-            if (!isInternal && args.includeExternal === false)
-                continue;
-            const category = categorizeLink(link.url, link.text, link.attrs);
-            categories[category] = (categories[category] || 0) + 1;
-            processedLinks.push({
-                url: link.url,
-                text: link.text,
-                type: isInternal ? 'internal' : 'external',
-                category,
-            });
-            if (isInternal)
-                internal++;
-            else
-                external++;
-            if (args.maxLinks && processedLinks.length >= args.maxLinks)
-                break;
+    let pagesScraped = 0;
+    let lastPagination = {};
+    const visitedPages = new Set();
+    // Process links from a page
+    const processLinks = (allLinks, pageNum) => {
+        for (const link of allLinks) {
+            try {
+                if (seen.has(link.url))
+                    continue;
+                seen.add(link.url);
+                const linkUrl = new URL(link.url);
+                const isInternal = linkUrl.hostname === currentUrl.hostname;
+                if (args.filter && !link.url.includes(args.filter) && !link.text.includes(args.filter)) {
+                    continue;
+                }
+                if (isInternal && args.includeInternal === false)
+                    continue;
+                if (!isInternal && args.includeExternal === false)
+                    continue;
+                const category = categorizeLink(link.url, link.text, link.attrs);
+                categories[category] = (categories[category] || 0) + 1;
+                processedLinks.push({
+                    url: link.url,
+                    text: link.text,
+                    type: isInternal ? 'internal' : 'external',
+                    category,
+                    page: pageNum,
+                });
+                if (isInternal)
+                    internal++;
+                else
+                    external++;
+                if (args.maxLinks && processedLinks.length >= args.maxLinks)
+                    return true; // Stop
+            }
+            catch {
+                // Invalid URL, skip
+            }
         }
-        catch {
-            // Invalid URL, skip
+        return false; // Continue
+    };
+    // Scrape first page
+    tracker.setProgress(20, '🔍 Extracting links from page 1...');
+    const firstPage = await extractLinksFromPage();
+    pagesScraped = 1;
+    visitedPages.add(page.url());
+    lastPagination = firstPage.pagination;
+    const shouldStop = processLinks(firstPage.links, 1);
+    // Follow pagination if enabled
+    if (followPagination && !shouldStop && firstPage.pagination.nextPage) {
+        let nextUrl = firstPage.pagination.nextPage;
+        while (nextUrl && pagesScraped < maxPages && !(args.maxLinks && processedLinks.length >= args.maxLinks)) {
+            // Check if we've already visited this page
+            if (visitedPages.has(nextUrl)) {
+                break;
+            }
+            visitedPages.add(nextUrl);
+            tracker.setProgress(20 + (pagesScraped / maxPages) * 60, `📄 Scraping page ${pagesScraped + 1}...`);
+            try {
+                // Navigate to next page
+                await page.goto(nextUrl, {
+                    waitUntil: 'domcontentloaded',
+                    timeout: 15000
+                });
+                // Wait for content to load
+                await new Promise(r => setTimeout(r, delayBetweenPages));
+                // Extract links from this page
+                const pageData = await extractLinksFromPage();
+                pagesScraped++;
+                lastPagination = pageData.pagination;
+                const stop = processLinks(pageData.links, pagesScraped);
+                if (stop)
+                    break;
+                // Get next page URL
+                nextUrl = pageData.pagination.nextPage || undefined;
+            }
+            catch (error) {
+                // Failed to navigate, stop pagination
+                break;
+            }
         }
     }
-    tracker.setProgress(90, `✅ Processed ${processedLinks.length} links`);
-    tracker.complete(`🎉 Link harvesting complete: ${processedLinks.length} links found`);
+    tracker.setProgress(90, `✅ Processed ${processedLinks.length} links from ${pagesScraped} pages`);
+    tracker.complete(`🎉 Link harvesting complete: ${processedLinks.length} links from ${pagesScraped} pages`);
     return {
         links: processedLinks,
         internal,
         external,
-        pagination: (pagination.nextPage || pagination.prevPage || pagination.totalPages) ? pagination : undefined,
+        pagination: (lastPagination.nextPage || lastPagination.prevPage || lastPagination.totalPages) ? lastPagination : undefined,
         categories,
-        message: `🔗 Found ${processedLinks.length} links (${internal} internal, ${external} external)` +
-            (pagination.nextPage ? ` | Next: ${pagination.nextPage}` : '')
+        pagesScraped,
+        message: `🔗 Found ${processedLinks.length} links (${internal} internal, ${external} external) from ${pagesScraped} pages` +
+            (lastPagination.nextPage && pagesScraped >= maxPages ? ` | More pages available: ${lastPagination.nextPage}` : '')
     };
 }
 /**
@@ -3440,10 +3702,13 @@ export async function handleSolveCaptchaAdvanced(page, args) {
 }
 /**
  * Parse and extract HLS/m3u8 streaming URLs
+ * ENHANCED: Segment parsing, bandwidth extraction, playlist fetching
  */
 export async function handleM3u8Parser(page, args) {
     const streams = [];
     const qualities = [];
+    const variants = [];
+    const segments = [];
     let masterPlaylist;
     // Intercept network requests to find m3u8 files
     const m3u8Urls = [];
@@ -3544,6 +3809,109 @@ export async function handleM3u8Parser(page, args) {
             streams.push(stream);
         }
     }
+    // ============================================================
+    // NEW: FETCH AND PARSE MASTER PLAYLIST FOR VARIANTS
+    // ============================================================
+    if ((args.fetchPlaylist || args.extractBandwidth) && masterPlaylist) {
+        try {
+            const playlistContent = await page.evaluate(async (url) => {
+                try {
+                    const response = await fetch(url);
+                    return await response.text();
+                }
+                catch {
+                    return null;
+                }
+            }, masterPlaylist);
+            if (playlistContent) {
+                // Parse #EXT-X-STREAM-INF lines for variants
+                const variantRegex = /#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+)(?:.*?RESOLUTION=(\d+x\d+))?[^\n]*\n([^\n]+)/g;
+                let match;
+                while ((match = variantRegex.exec(playlistContent)) !== null) {
+                    const bandwidth = parseInt(match[1], 10);
+                    const resolution = match[2] || undefined;
+                    let variantUrl = match[3].trim();
+                    // Make relative URLs absolute
+                    if (!variantUrl.startsWith('http')) {
+                        const baseUrl = masterPlaylist.substring(0, masterPlaylist.lastIndexOf('/') + 1);
+                        variantUrl = baseUrl + variantUrl;
+                    }
+                    // Determine quality from resolution or bandwidth
+                    let quality = 'unknown';
+                    if (resolution) {
+                        const height = parseInt(resolution.split('x')[1], 10);
+                        if (height >= 2160)
+                            quality = '4K';
+                        else if (height >= 1080)
+                            quality = '1080p';
+                        else if (height >= 720)
+                            quality = '720p';
+                        else if (height >= 480)
+                            quality = '480p';
+                        else if (height >= 360)
+                            quality = '360p';
+                        else
+                            quality = `${height}p`;
+                    }
+                    else if (bandwidth >= 5000000)
+                        quality = '1080p';
+                    else if (bandwidth >= 2500000)
+                        quality = '720p';
+                    else if (bandwidth >= 1000000)
+                        quality = '480p';
+                    else
+                        quality = '360p';
+                    variants.push({ quality, bandwidth, url: variantUrl, resolution });
+                }
+                // Sort variants by bandwidth (highest first)
+                variants.sort((a, b) => b.bandwidth - a.bandwidth);
+            }
+        }
+        catch (e) {
+            // Ignore fetch errors
+        }
+    }
+    // ============================================================
+    // NEW: PARSE SEGMENTS FROM MEDIA PLAYLIST
+    // ============================================================
+    if (args.parseSegments && streams.length > 0) {
+        const mediaPlaylistUrl = streams[0].url;
+        try {
+            const mediaContent = await page.evaluate(async (url) => {
+                try {
+                    const response = await fetch(url);
+                    return await response.text();
+                }
+                catch {
+                    return null;
+                }
+            }, mediaPlaylistUrl);
+            if (mediaContent) {
+                const lines = mediaContent.split('\n');
+                let segmentIndex = 0;
+                let currentDuration = 0;
+                for (let i = 0; i < lines.length; i++) {
+                    const line = lines[i].trim();
+                    // Parse duration from #EXTINF
+                    if (line.startsWith('#EXTINF:')) {
+                        currentDuration = parseFloat(line.replace('#EXTINF:', '').split(',')[0]);
+                    }
+                    // Capture segment URL
+                    else if (line && !line.startsWith('#') && (line.includes('.ts') || line.includes('.m4s'))) {
+                        let segmentUrl = line;
+                        if (!segmentUrl.startsWith('http')) {
+                            const baseUrl = mediaPlaylistUrl.substring(0, mediaPlaylistUrl.lastIndexOf('/') + 1);
+                            segmentUrl = baseUrl + segmentUrl;
+                        }
+                        segments.push({ url: segmentUrl, duration: currentDuration, index: segmentIndex++ });
+                    }
+                }
+            }
+        }
+        catch (e) {
+            // Ignore segment parsing errors
+        }
+    }
     // Filter audio if not wanted
     const filteredStreams = args.includeAudio !== false
         ? streams
@@ -3553,6 +3921,8 @@ export async function handleM3u8Parser(page, args) {
         streams: filteredStreams,
         masterPlaylist,
         qualities: [...new Set(qualities)],
+        variants: variants.length > 0 ? variants : undefined,
+        segments: segments.length > 0 ? segments : undefined,
     };
 }
 /**
@@ -4414,11 +4784,76 @@ export async function handleCloudflareBypass(page, args) {
 /**
  * Master tool: Extract direct stream/download URLs
  * ULTRA POWERFUL: Handles packed JS, JW Player, Video.js, HLS.js, obfuscated scripts
+ * ENHANCED: Multi-Quality Selector, VidSrc, Filemoon, StreamWish support
  */
 export async function handleStreamExtractor(page, args) {
     const formats = args.formats || ['mp4', 'mkv', 'm3u8', 'mp3', 'webm', 'flv', 'avi'];
     const maxRedirects = args.maxRedirects || 10;
     const directUrls = [];
+    const subtitles = [];
+    // Quality priority for auto-selection
+    const qualityPriority = {
+        '2160p': 100, '4k': 100, 'uhd': 100,
+        '1080p': 90, 'fhd': 90, 'full hd': 90,
+        '720p': 80, 'hd': 80,
+        '480p': 70, 'sd': 70,
+        '360p': 60,
+        '240p': 50,
+        '144p': 40,
+        'unknown': 10, 'auto': 10
+    };
+    // Site-specific extraction patterns
+    const sitePatterns = {
+        vidsrc: {
+            urlPattern: /vidsrc|v2\.vidsrc/i,
+            sourcePattern: [
+                /source:\s*["']([^"']+\.m3u8[^"']*)/gi,
+                /file:\s*["']([^"']+\.m3u8[^"']*)/gi
+            ]
+        },
+        filemoon: {
+            urlPattern: /filemoon|moonplayer/i,
+            sourcePattern: [
+                /sources:\s*\[\s*\{[^}]*file:\s*["']([^"']+)/gi,
+                /eval\(function\(p,a,c,k,e,[rd]\)/gi
+            ]
+        },
+        streamwish: {
+            urlPattern: /streamwish|swish/i,
+            sourcePattern: [
+                /file:\s*["']([^"']+\.m3u8[^"']*)/gi,
+                /sources:\s*\[.*?["']([^"']+\.m3u8[^"']*)/gi
+            ]
+        },
+        doodstream: {
+            urlPattern: /dood|doodstream/i,
+            sourcePattern: [
+                /\/pass_md5\/[^"']+/gi,
+                /\$.get\(['"]([^'"]+pass_md5[^'"]+)/gi
+            ]
+        },
+        mixdrop: {
+            urlPattern: /mixdrop/i,
+            sourcePattern: [
+                /MDCore\.wurl\s*=\s*["']([^"']+)/gi,
+                /wurl\s*=\s*["']([^"']+)/gi
+            ]
+        },
+        streamtape: {
+            urlPattern: /streamtape/i,
+            sourcePattern: [
+                /id=.*?&token=/gi,
+                /robotlink.*?=\s*['"]([^'"]+)/gi
+            ]
+        },
+        mp4upload: {
+            urlPattern: /mp4upload/i,
+            sourcePattern: [
+                /player\.src\(\{src:\s*["']([^"']+)/gi,
+                /src:\s*["']([^"']+\.mp4[^"']*)/gi
+            ]
+        }
+    };
     // Navigate if URL provided
     if (args.url) {
         await page.goto(args.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
@@ -4652,25 +5087,173 @@ export async function handleStreamExtractor(page, args) {
             directUrls.push({ url, format, source: 'network' });
         }
     }
+    // ============================================================
+    // NEW: SITE-SPECIFIC EXTRACTION (VidSrc, Filemoon, StreamWish, etc.)
+    // ============================================================
+    if (args.siteType && args.siteType !== 'auto') {
+        const siteConfig = sitePatterns[args.siteType];
+        if (siteConfig) {
+            const html = await page.content();
+            for (const pattern of siteConfig.sourcePattern) {
+                let match;
+                while ((match = pattern.exec(html)) !== null) {
+                    if (match[1] && !directUrls.some(d => d.url === match[1])) {
+                        const format = formats.find(f => match[1].includes(`.${f}`)) || 'm3u8';
+                        directUrls.push({ url: match[1], format, source: args.siteType });
+                    }
+                }
+            }
+        }
+    }
+    else {
+        // Auto-detect site type from URL
+        const currentUrl = page.url();
+        for (const [siteName, config] of Object.entries(sitePatterns)) {
+            if (config.urlPattern.test(currentUrl)) {
+                const html = await page.content();
+                for (const pattern of config.sourcePattern) {
+                    let match;
+                    while ((match = pattern.exec(html)) !== null) {
+                        if (match[1] && !directUrls.some(d => d.url === match[1])) {
+                            const format = formats.find(f => match[1].includes(`.${f}`)) || 'm3u8';
+                            directUrls.push({ url: match[1], format, source: siteName });
+                        }
+                    }
+                }
+                break;
+            }
+        }
+    }
+    // ============================================================
+    // NEW: EXTRACT QUALITY FROM URLs
+    // ============================================================
+    for (const item of directUrls) {
+        if (!item.quality || item.quality === 'auto') {
+            const url = item.url.toLowerCase();
+            if (url.includes('2160') || url.includes('4k') || url.includes('uhd'))
+                item.quality = '2160p';
+            else if (url.includes('1080'))
+                item.quality = '1080p';
+            else if (url.includes('720'))
+                item.quality = '720p';
+            else if (url.includes('480'))
+                item.quality = '480p';
+            else if (url.includes('360'))
+                item.quality = '360p';
+            else if (url.includes('240'))
+                item.quality = '240p';
+            else if (url.includes('144'))
+                item.quality = '144p';
+            else
+                item.quality = 'unknown';
+        }
+    }
+    // ============================================================
+    // NEW: AUTO-SELECT BEST QUALITY
+    // ============================================================
+    let bestQuality;
+    if (args.autoSelectBest || args.preferredQuality) {
+        const preferredQ = args.preferredQuality || 'highest';
+        if (preferredQ === 'highest') {
+            // Sort by quality priority (highest first)
+            const sorted = [...directUrls].sort((a, b) => {
+                const aScore = qualityPriority[a.quality?.toLowerCase() || 'unknown'] || 0;
+                const bScore = qualityPriority[b.quality?.toLowerCase() || 'unknown'] || 0;
+                return bScore - aScore;
+            });
+            if (sorted.length > 0) {
+                bestQuality = { url: sorted[0].url, format: sorted[0].format, quality: sorted[0].quality || 'unknown', source: sorted[0].source };
+            }
+        }
+        else if (preferredQ === 'lowest') {
+            // Sort by quality priority (lowest first)
+            const sorted = [...directUrls].sort((a, b) => {
+                const aScore = qualityPriority[a.quality?.toLowerCase() || 'unknown'] || 0;
+                const bScore = qualityPriority[b.quality?.toLowerCase() || 'unknown'] || 0;
+                return aScore - bScore;
+            });
+            if (sorted.length > 0) {
+                bestQuality = { url: sorted[0].url, format: sorted[0].format, quality: sorted[0].quality || 'unknown', source: sorted[0].source };
+            }
+        }
+        else {
+            // Find exact match for preferred quality
+            const exact = directUrls.find(d => d.quality?.toLowerCase() === preferredQ.toLowerCase());
+            if (exact) {
+                bestQuality = { url: exact.url, format: exact.format, quality: exact.quality || preferredQ, source: exact.source };
+            }
+            else {
+                // Fallback to highest available
+                const sorted = [...directUrls].sort((a, b) => {
+                    const aScore = qualityPriority[a.quality?.toLowerCase() || 'unknown'] || 0;
+                    const bScore = qualityPriority[b.quality?.toLowerCase() || 'unknown'] || 0;
+                    return bScore - aScore;
+                });
+                if (sorted.length > 0) {
+                    bestQuality = { url: sorted[0].url, format: sorted[0].format, quality: sorted[0].quality || 'unknown', source: sorted[0].source };
+                }
+            }
+        }
+    }
+    // ============================================================
+    // NEW: EXTRACT SUBTITLES
+    // ============================================================
+    if (args.extractSubtitles) {
+        const subData = await page.evaluate(() => {
+            const subs = [];
+            // HTML5 track elements
+            document.querySelectorAll('track[kind="subtitles"], track[kind="captions"]').forEach(track => {
+                const src = track.getAttribute('src');
+                if (src) {
+                    subs.push({
+                        url: src,
+                        language: track.getAttribute('srclang') || undefined,
+                        label: track.getAttribute('label') || undefined
+                    });
+                }
+            });
+            // VTT/SRT links
+            document.querySelectorAll('a[href*=".vtt"], a[href*=".srt"], a[href*=".ass"]').forEach(link => {
+                const href = link.href;
+                subs.push({ url: href, label: link.textContent?.trim() || undefined });
+            });
+            // Look in scripts for subtitle URLs
+            const html = document.documentElement.innerHTML;
+            const vttMatches = html.match(/https?:\/\/[^\s"']+\.vtt[^\s"']*/gi);
+            const srtMatches = html.match(/https?:\/\/[^\s"']+\.srt[^\s"']*/gi);
+            if (vttMatches)
+                vttMatches.forEach(url => subs.push({ url }));
+            if (srtMatches)
+                srtMatches.forEach(url => subs.push({ url }));
+            // Deduplicate
+            const seen = new Set();
+            return subs.filter(s => {
+                if (seen.has(s.url))
+                    return false;
+                seen.add(s.url);
+                return true;
+            });
+        });
+        subtitles.push(...subData);
+    }
     return {
         success: directUrls.length > 0,
         directUrls,
+        bestQuality,
+        subtitles: args.extractSubtitles ? subtitles : undefined,
         message: directUrls.length > 0
-            ? `🎬 Found ${directUrls.length} direct URL(s) from ${new Set(directUrls.map(d => d.source)).size} sources`
+            ? `🎬 Found ${directUrls.length} URL(s)${bestQuality ? ` | Best: ${bestQuality.quality}` : ''}${subtitles.length > 0 ? ` | ${subtitles.length} subtitles` : ''}`
             : 'No direct URLs found',
     };
 }
-/**
- * JS Scrape - Single-call JavaScript-rendered content extraction
- * Combines navigation, waiting, scrolling, and content extraction in one call
- * Perfect for scraping dynamic/AJAX-loaded content
- */
-export async function handleJsScrape(page, args) {
+// Helper function to scrape a single URL
+async function scrapeSingleUrl(page, url, args) {
+    const startTime = Date.now();
     const waitForTimeout = args.waitForTimeout || 10000;
     const returnType = args.returnType || 'html';
     try {
         // Step 1: Navigate to URL
-        await page.goto(args.url, {
+        await page.goto(url, {
             waitUntil: 'domcontentloaded',
             timeout: waitForTimeout
         });
@@ -4772,26 +5355,156 @@ export async function handleJsScrape(page, args) {
         }
         return {
             success: true,
-            url: args.url,
+            url,
             finalUrl,
             title,
             html,
             text,
             elements,
-            elementCount
+            elementCount,
+            duration: Date.now() - startTime
         };
     }
     catch (error) {
         return {
             success: false,
-            url: args.url,
-            finalUrl: page.url() || args.url,
+            url,
+            finalUrl: page.url() || url,
             title: '',
             elementCount: 0,
-            error: error instanceof Error ? error.message : String(error)
+            error: error instanceof Error ? error.message : String(error),
+            duration: Date.now() - startTime
         };
     }
 }
+/**
+ * JS Scrape - Single-call JavaScript-rendered content extraction
+ * Combines navigation, waiting, scrolling, and content extraction in one call
+ * Perfect for scraping dynamic/AJAX-loaded content
+ * NEW: Supports parallel scraping of multiple URLs with concurrency control
+ */
+export async function handleJsScrape(page, args) {
+    const startTime = Date.now();
+    // Determine URLs to scrape
+    const urlList = args.urls || (args.url ? [args.url] : []);
+    if (urlList.length === 0) {
+        return {
+            success: false,
+            error: 'No URL(s) provided. Use "url" for single URL or "urls" for multiple URLs.',
+            elementCount: 0
+        };
+    }
+    // Single URL mode - backward compatible
+    if (urlList.length === 1 && !args.urls) {
+        const result = await scrapeSingleUrl(page, urlList[0], args);
+        return result;
+    }
+    // Parallel scraping mode
+    const concurrency = Math.min(args.concurrency || 3, 10); // Max 10 concurrent
+    const continueOnError = args.continueOnError !== false;
+    const delayBetween = args.delayBetween || 500;
+    const results = [];
+    const browser = page.browser();
+    if (!browser) {
+        return {
+            success: false,
+            error: 'Browser not available for parallel scraping',
+            elementCount: 0
+        };
+    }
+    // Create a semaphore for concurrency control
+    let activeCount = 0;
+    const queue = [...urlList];
+    const errors = [];
+    // Process URLs with concurrency limit
+    const processUrl = async (url) => {
+        let newPage = null;
+        try {
+            // Create new page for each URL
+            newPage = await browser.newPage();
+            // Copy settings from original page if needed
+            await newPage.setViewport({ width: 1280, height: 720 });
+            const result = await scrapeSingleUrl(newPage, url, args);
+            return result;
+        }
+        catch (error) {
+            return {
+                success: false,
+                url,
+                finalUrl: url,
+                title: '',
+                elementCount: 0,
+                error: error instanceof Error ? error.message : String(error),
+                duration: 0
+            };
+        }
+        finally {
+            // Close the page
+            if (newPage) {
+                try {
+                    await newPage.close();
+                }
+                catch (e) {
+                    // Ignore close errors
+                }
+            }
+        }
+    };
+    // Process all URLs with concurrency control
+    const processBatch = async () => {
+        const promises = [];
+        while (queue.length > 0 || activeCount > 0) {
+            // Start new tasks up to concurrency limit
+            while (activeCount < concurrency && queue.length > 0) {
+                const url = queue.shift();
+                activeCount++;
+                const promise = (async () => {
+                    try {
+                        // Add delay between starting each scrape
+                        if (results.length > 0) {
+                            await new Promise(r => setTimeout(r, delayBetween));
+                        }
+                        const result = await processUrl(url);
+                        if (result) {
+                            results.push(result);
+                            if (!result.success && !continueOnError) {
+                                // Clear queue to stop processing
+                                queue.length = 0;
+                                errors.push(`Stopped at ${url}: ${result.error}`);
+                            }
+                        }
+                    }
+                    finally {
+                        activeCount--;
+                    }
+                })();
+                promises.push(promise);
+            }
+            // Wait for at least one to complete before continuing
+            if (promises.length > 0) {
+                await Promise.race(promises);
+            }
+            // Small delay to prevent tight loop
+            await new Promise(r => setTimeout(r, 50));
+        }
+        // Wait for all remaining promises
+        await Promise.all(promises);
+    };
+    await processBatch();
+    const successCount = results.filter(r => r.success).length;
+    const failedCount = results.filter(r => !r.success).length;
+    return {
+        success: successCount > 0,
+        isParallel: true,
+        urls: urlList,
+        results,
+        totalUrls: urlList.length,
+        successCount,
+        failedCount,
+        totalDuration: Date.now() - startTime,
+        error: errors.length > 0 ? errors.join('; ') : undefined
+    };
+}
 /**
  * Execute custom JavaScript on page
  * ULTRA POWERFUL: Execute any JS code and get results
@@ -4801,9 +5514,16 @@ export async function handleExecuteJs(page, args) {
     const waitTime = args.waitForResult || 5000;
     try {
         let result;
+        // Auto-detect if code contains await and wrap with async IIFE
+        const hasAwait = /\bawait\b/.test(args.code);
+        const isAlreadyAsync = /^\s*\(async\s+function|\(\s*async\s*\(|\(async\s*\(\)\s*=>/.test(args.code.trim());
+        // Wrap code with async IIFE if it contains await but isn't already async
+        const codeToExecute = (hasAwait && !isAlreadyAsync)
+            ? `(async () => { ${args.code} })()`
+            : args.code;
         if (args.context === 'isolated') {
             // Execute in isolated context (sandboxed)
-            result = await page.evaluate(args.code);
+            result = await page.evaluate(codeToExecute);
         }
         else {
             // Execute in page context with full access
@@ -4815,7 +5535,7 @@ export async function handleExecuteJs(page, args) {
                 catch (e) {
                     return { error: String(e) };
                 }
-            }, args.code);
+            }, codeToExecute);
         }
         // Handle async results
         if (result instanceof Promise) {