@fanboynz/network-scanner 1.0.95 → 1.0.97

package/README.md

@@ -151,7 +151,7 @@ Example:
 | `blocked`            | Array | - | Domains or regexes to block during scanning |
 | `even_blocked`       | Boolean | `false` | Add matching rules even if requests are blocked |
 | `bypass_cache`       | Boolean | `false` | Skip all caching for this site's URLs |
-
+| `window_cleanup`     | Boolean | `false` | Close extra browser windows/tabs after entire URL group completes with 16s delay |
 
 ### Redirect Handling Options
 

package/lib/browserhealth.js

@@ -5,6 +5,136 @@
 
 const { formatLogMessage, messageColors } = require('./colorize');
 
+
+// Window cleanup delay constant
+const WINDOW_CLEANUP_DELAY_MS = 16000;
+
+/**
+ * Performs group-level window cleanup after all URLs in a site group complete
+ * Closes all extra windows except the main browser window
+ * @param {import('puppeteer').Browser} browserInstance - Browser instance
+ * @param {string} groupDescription - Description of the group for logging
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<Object>} Cleanup results
+ */
+async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug) {
+  try {
+    // Wait before cleanup to allow any final operations to complete
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before cleanup for group: ${groupDescription}`));
+    }
+    await new Promise(resolve => setTimeout(resolve, WINDOW_CLEANUP_DELAY_MS));
+    
+    const allPages = await browserInstance.pages();
+    const mainPage = allPages[0]; // Always keep the first page as main
+    const extraPages = allPages.slice(1); // All other pages can be closed
+    
+    if (extraPages.length === 0) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[group_window_cleanup] No extra windows to close for group: ${groupDescription}`));
+      }
+      return { success: true, closedCount: 0, totalPages: allPages.length, estimatedMemoryFreed: 0 };
+    }
+    
+    // Estimate memory usage before closing
+    let totalEstimatedMemory = 0;
+    const pageMemoryEstimates = [];
+    
+    for (let i = 0; i < extraPages.length; i++) {
+      const page = extraPages[i];
+      let pageMemoryEstimate = 0;
+      
+      try {
+        if (!page.isClosed()) {
+          // Get page metrics if available
+          const metrics = await Promise.race([
+            page.metrics(),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('metrics timeout')), 1000))
+          ]);
+          
+          // Calculate memory estimate based on page metrics
+          if (metrics) {
+            // Puppeteer metrics provide various memory-related values
+            pageMemoryEstimate = (
+              (metrics.JSHeapUsedSize || 0) +           // JavaScript heap
+              (metrics.JSHeapTotalSize || 0) * 0.1 +    // Estimated overhead
+              (metrics.Nodes || 0) * 100 +              // DOM nodes (rough estimate)
+              (metrics.JSEventListeners || 0) * 50      // Event listeners
+            );
+          } else {
+            // Fallback: rough estimate based on page complexity
+            pageMemoryEstimate = 8 * 1024 * 1024; // 8MB default estimate per page
+          }
+        }
+      } catch (metricsErr) {
+        // Fallback estimate if metrics fail
+        pageMemoryEstimate = 8 * 1024 * 1024; // 8MB default
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[group_window_cleanup] Could not get metrics for page ${i + 1}, using default estimate: ${metricsErr.message}`));
+        }
+      }
+      
+      pageMemoryEstimates.push(pageMemoryEstimate);
+      totalEstimatedMemory += pageMemoryEstimate;
+    }
+    
+    // Close all extra pages since the entire group is complete
+    const closePromises = extraPages.map(async (page, index) => {
+      try {
+        if (!page.isClosed()) {
+          await page.close();
+          return { success: true, url: page.url() || `page-${index}`, estimatedMemory: pageMemoryEstimates[index] };
+        }
+        return { success: false, reason: 'already_closed', estimatedMemory: 0 };
+      } catch (closeErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[group_window_cleanup] Failed to close page ${index + 1}: ${closeErr.message}`));
+        }
+        return { success: false, error: closeErr.message, estimatedMemory: 0 };
+      }
+    });
+    
+    const closeResults = await Promise.all(closePromises);
+    const successfulCloses = closeResults.filter(result => result.success === true).length;
+    const actualMemoryFreed = closeResults
+      .filter(result => result.success === true)
+      .reduce((sum, result) => sum + (result.estimatedMemory || 0), 0);
+    
+    // Format memory for human readability
+    const formatMemory = (bytes) => {
+      if (bytes >= 1024 * 1024 * 1024) {
+        return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+      } else if (bytes >= 1024 * 1024) {
+        return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+      } else if (bytes >= 1024) {
+        return `${(bytes / 1024).toFixed(1)}KB`;
+      } else {
+        return `${bytes}B`;
+      }
+    };
+    
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${extraPages.length} windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Estimated memory freed: ${formatMemory(actualMemoryFreed)}`));
+    }
+    
+    return { 
+      success: true, 
+      closedCount: successfulCloses,
+      totalPages: allPages.length,
+      mainPageKept: !mainPage.isClosed(),
+      delayUsed: WINDOW_CLEANUP_DELAY_MS,
+      estimatedMemoryFreed: actualMemoryFreed,
+      estimatedMemoryFreedFormatted: formatMemory(actualMemoryFreed)
+    };
+  } catch (cleanupErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Group cleanup failed for ${groupDescription}: ${cleanupErr.message}`));
+    }
+    return { success: false, error: cleanupErr.message, estimatedMemoryFreed: 0 };
+  }
+}
+
 /**
  * Quick browser responsiveness test for use during page setup
  * Designed to catch browser degradation between operations
@@ -541,6 +671,7 @@ module.exports = {
   checkBrowserHealth,
   checkBrowserMemory,
   testBrowserConnectivity,
+  performGroupWindowCleanup,
   testNetworkCapability,
   isQuicklyResponsive,
   performHealthAssessment,

package/lib/cdp.js

@@ -27,6 +27,36 @@
 
 const { formatLogMessage } = require('./colorize');
 
+/**
+ * Creates a new page with timeout protection to prevent CDP hangs
+ * @param {import('puppeteer').Browser} browser - Browser instance
+ * @param {number} timeout - Timeout in milliseconds (default: 30000)
+ * @returns {Promise<import('puppeteer').Page>} Page instance
+ */
+async function createPageWithTimeout(browser, timeout = 30000) {
+  return Promise.race([
+    browser.newPage(),
+    new Promise((_, reject) => 
+      setTimeout(() => reject(new Error('Page creation timeout - browser may be unresponsive')), timeout)
+    )
+  ]);
+}
+
+/**
+ * Sets request interception with timeout protection
+ * @param {import('puppeteer').Page} page - Page instance
+ * @param {number} timeout - Timeout in milliseconds (default: 15000)
+ * @returns {Promise<void>}
+ */
+async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
+  return Promise.race([
+    page.setRequestInterception(true),
+    new Promise((_, reject) => 
+      setTimeout(() => reject(new Error('Request interception setup timeout')), timeout)
+    )
+  ]);
+}
+
 /**
  * Creates and manages a CDP session for network monitoring
  * 
@@ -88,8 +118,13 @@ async function createCDPSession(page, currentUrl, options = {}) {
 
   try {
     // Create CDP session using modern Puppeteer 20+ API
-    // page.target().createCDPSession() was deprecated in 19+ and removed in 20+
-    cdpSession = await page.createCDPSession();
+    // Add timeout protection for CDP session creation
+    cdpSession = await Promise.race([
+      page.createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('CDP session creation timeout')), 20000)
+      )
+    ]);
     
     // Enable network domain - required for network event monitoring  
     await cdpSession.send('Network.enable');
@@ -302,6 +337,8 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
 // - Some features may not work in --no-sandbox mode
 module.exports = {
   createCDPSession,
+  createPageWithTimeout,
+  setRequestInterceptionWithTimeout,
   validateCDPConfig,
   createEnhancedCDPSession
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.95 ===
+// === Network scanner script (nwss.js) v1.0.97 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -33,7 +33,7 @@ const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvaila
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // CDP functionality
-const { createCDPSession } = require('./lib/cdp');
+const { createCDPSession, createPageWithTimeout, setRequestInterceptionWithTimeout } = require('./lib/cdp');
 // Post-processing cleanup
 const { processResults } = require('./lib/post-processing');
 // Colorize various text when used
@@ -64,7 +64,7 @@ const TIMEOUTS = {
   EMERGENCY_RESTART_DELAY: 2000,
   BROWSER_STABILIZE_DELAY: 1000,
   CURL_HANDLER_DELAY: 3000,
-  PROTOCOL_TIMEOUT: 60000,
+  PROTOCOL_TIMEOUT: 120000,
   REDIRECT_JS_TIMEOUT: 5000
 };
 
@@ -122,10 +122,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.95'; // Script version
+const VERSION = '1.0.97'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -571,6 +571,8 @@ Advanced Options:
   dig_subdomain: true/false                    Use subdomain for dig lookup instead of root domain (default: false)
   digRecordType: "A"                          DNS record type for dig (default: A)
 
+  window_cleanup: true/false                   Close extra browser windows/tabs after entire URL group completes with 5s delay (default: false)
+
 Referrer Header Options:
   referrer_headers: "https://google.com"       Single referrer URL
   referrer_headers: ["url1", "url2"]           Random selection from array  
@@ -1465,7 +1467,7 @@ function setupFrameHandling(page, forceDebug) {
       if (browserInstance.process() && browserInstance.process().killed) {
         throw new Error('Browser process was killed - restart required');
       }
-      page = await browserInstance.newPage();
+      page = await createPageWithTimeout(browserInstance, 30000);
 
       // Enhanced page validation for Puppeteer 23.x
       if (!page || page.isClosed()) {
@@ -1779,13 +1781,8 @@ function setupFrameHandling(page, forceDebug) {
 
       // Protected request interception setup with timeout
       try {
-        // Test if network operations are responsive before enabling request interception
-        await Promise.race([
-          page.setRequestInterception(true),
-          new Promise((_, reject) => 
-            setTimeout(() => reject(new Error('Network.enable timeout')), 10000)
-          )
-        ]);
+        // Use timeout-protected request interception setup
+        await setRequestInterceptionWithTimeout(page, 15000);
         
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Request interception enabled successfully for ${currentUrl}`));
@@ -1794,13 +1791,13 @@ function setupFrameHandling(page, forceDebug) {
         if (networkErr.message.includes('timed out') || 
             networkErr.message.includes('Network.enable') || 
             networkErr.message.includes('timeout')) {
-          console.warn(formatLogMessage('warn', `Network setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
+          console.warn(formatLogMessage('warn', `Request interception setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
           return { 
             url: currentUrl, 
             rules: [], 
             success: false, 
             needsImmediateRestart: true,
-            error: 'Network.enable timeout - browser restart required'
+            error: 'Request interception timeout - browser restart required'
           };
         }
         throw networkErr; // Re-throw other errors
@@ -3235,9 +3232,21 @@ function setupFrameHandling(page, forceDebug) {
   }, 30000); // Check every 30 seconds
 
   // Process URLs in batches to maintain concurrency while allowing browser restarts
+  let siteGroupIndex = 0;
   for (let batchStart = 0; batchStart < totalUrls; batchStart += RESOURCE_CLEANUP_INTERVAL) {
     const batchEnd = Math.min(batchStart + RESOURCE_CLEANUP_INTERVAL, totalUrls);
     const currentBatch = allTasks.slice(batchStart, batchEnd);
+
+    
+    // Group tasks by their source site configuration for window cleanup
+    const tasksBySite = new Map();
+    currentBatch.forEach(task => {
+      const siteKey = `site_${sites.indexOf(task.config)}`;
+      if (!tasksBySite.has(siteKey)) {
+        tasksBySite.set(siteKey, []);
+      }
+      tasksBySite.get(siteKey).push(task);
+    });
     
     // IMPROVED: Only check health if we have indicators of problems
     let healthCheck = { shouldRestart: false, reason: null };
@@ -3391,6 +3400,25 @@ function setupFrameHandling(page, forceDebug) {
     }
 
     results.push(...batchResults);
+
+    // Perform group window cleanup for completed sites
+    for (const [siteKey, siteTasks] of tasksBySite) {
+      const siteConfig = siteTasks[0].config; // All tasks in group have same config
+      
+      if (siteConfig.window_cleanup === true) {
+        const urlCount = siteTasks.length;
+        const groupDescription = `${urlCount} URLs from site group ${++siteGroupIndex}`;
+        
+        try {
+          const groupCleanupResult = await performGroupWindowCleanup(browser, groupDescription, forceDebug);
+          if (!silentMode && groupCleanupResult.success && groupCleanupResult.closedCount > 0) {
+            console.log(`🗑️ Group cleanup: ${groupCleanupResult.closedCount} windows closed after completing ${groupDescription}`);
+          }
+        } catch (groupCleanupErr) {
+          if (forceDebug) console.log(formatLogMessage('debug', `Group window cleanup failed: ${groupCleanupErr.message}`));
+        }
+      }
+    }
     
     processedUrlCount += batchSize;
     urlsSinceLastCleanup += batchSize;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.95",
+  "version": "1.0.97",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {