@fanboynz/network-scanner 2.0.9 → 2.0.10

package/lib/browserhealth.js

@@ -970,6 +970,90 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
   }
 }
 
+/**
+ * Performs comprehensive cleanup of page resources before operations that might cause detached frames
+ * Also attempts to stop any pending navigations that might interfere
+ * Used before reloads, navigations, and other operations that can trigger frame detachment
+ * @param {import('puppeteer').Page} page - Page to clean up
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<boolean>} True if cleanup succeeded
+ */
+async function cleanupPageBeforeReload(page, forceDebug = false) {
+  try {
+    if (page.isClosed()) {
+      return false;
+    }
+
+    // First, try to stop any pending navigation
+    try {
+      await page.evaluate(() => {
+        // Stop any ongoing navigation
+        if (window.stop) {
+          window.stop();
+        }
+      });
+    } catch (e) {
+      // Page might be mid-navigation, that's ok
+    }
+
+    // Wait a bit for navigation to stop
+    await new Promise(resolve => setTimeout(resolve, 500));
+
+    // Now do the full cleanup
+    await page.evaluate(() => {
+      // Stop all media elements
+      document.querySelectorAll('video, audio').forEach(media => {
+        try { 
+          media.pause(); 
+          media.src = ''; 
+          media.load();
+        } catch(e) {}
+      });
+      
+      // Clear all timers and intervals
+      const highestId = setTimeout(() => {}, 0);
+      for (let i = highestId; i >= 0; i--) {
+        clearTimeout(i);
+        clearInterval(i);
+      }
+      
+      // Stop all animations
+      if (typeof cancelAnimationFrame !== 'undefined') {
+        const highestRAF = requestAnimationFrame(() => {});
+        for (let i = highestRAF; i >= 0; i--) {
+          cancelAnimationFrame(i);
+        }
+      }
+      
+      // Clear all iframes properly
+      document.querySelectorAll('iframe').forEach(iframe => {
+        try {
+          // Stop iframe content first
+          if (iframe.contentWindow) {
+            iframe.contentWindow.stop();
+          }
+          iframe.src = 'about:blank';
+          iframe.remove();
+        } catch(e) {}
+      });
+      
+      // Force garbage collection if available
+      if (window.gc) window.gc();
+    });
+
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', 'Page resources cleaned before reload'));
+    }
+
+    return true;
+  } catch (err) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `Page cleanup error: ${err.message}`));
+    }
+    return false;
+  }
+}
+
 module.exports = {
   checkBrowserHealth,
   checkBrowserMemory,
@@ -983,7 +1067,8 @@ module.exports = {
   monitorBrowserHealth,
   isBrowserHealthy,
   isCriticalProtocolError,
-  updatePageUsage
+  updatePageUsage,
+  cleanupPageBeforeReload
 };
 
 // Clean up tracking maps when pages are closed

package/lib/cloudflare.js

@@ -1,5 +1,6 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.3 - Fixes Cannot read properties of undefined (reading 'hasIndicators')
  * Version: 2.6.2 - Further detached Frame fixes 
  * Version: 2.6.1 - timeoutId is not defined & race condition fix
  * Version: 2.6.0 - Memory leak fixes and timeout cleanup
@@ -19,7 +20,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.2';
+const CLOUDFLARE_MODULE_VERSION = '2.6.3';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -1570,6 +1571,11 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
   
   // Quick detection first - exit early if no Cloudflare detected and no explicit config
   const quickDetection = await quickCloudflareDetection(page, forceDebug);
+  
+  // Safety check: ensure quickDetection is valid
+  if (!quickDetection) {
+    return { phishingWarning: { attempted: false, success: true }, verificationChallenge: { attempted: false, success: true }, overallSuccess: true, errors: [], quickDetectionFailed: true };
+  }
 
   // Early return structure when no Cloudflare indicators found
   // Sets attempted: false, success: true for both protection types

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.9 ===
+// === Network scanner script (nwss.js) v2.0.10 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -66,7 +66,7 @@ const TIMEOUTS = {
   EMERGENCY_RESTART_DELAY: 2000,
   BROWSER_STABILIZE_DELAY: 1000,
   CURL_HANDLER_DELAY: 3000,
-  PROTOCOL_TIMEOUT: 120000,
+  PROTOCOL_TIMEOUT: 160000,
   REDIRECT_JS_TIMEOUT: 5000
 };
 
@@ -126,10 +126,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.9'; // Script version
+const VERSION = '2.0.10'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1087,7 +1087,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Set();
+  let activeFrames = new Map(); // Use Map to track frame state
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1101,6 +1101,15 @@ function setupFrameHandling(page, forceDebug) {
   page.on('frameattached', async (frame) => {
     // Enhanced frame handling with detached frame protection
     try {
+      // Test frame accessibility first with safe method
+      let isFrameValid = false;
+      try {
+        frame.url(); // This will throw if frame is detached
+        isFrameValid = true;
+      } catch (e) {
+        return; // Frame is already detached, skip
+      }
+
       // Multiple checks for frame validity to prevent detached frame errors
       if (!frame) {
         if (forceDebug) {
@@ -1136,7 +1145,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    if (frame.parentFrame()) { // Only handle child frames, not main frame
+    // Store frame with timestamp for tracking
+    activeFrames.set(frame, Date.now());
+    
+    if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
       try {       
         if (forceDebug) {
           console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
@@ -1979,7 +1991,17 @@ function setupFrameHandling(page, forceDebug) {
       }
 
       // --- Apply all fingerprint spoofing (user agent, Brave, fingerprint protection) ---
-      await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      try {
+        await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      } catch (fingerprintErr) {
+        if (fingerprintErr.message.includes('Session closed') || 
+            fingerprintErr.message.includes('Protocol error') ||
+            fingerprintErr.message.includes('addScriptToEvaluateOnNewDocument')) {
+          console.warn(`[fingerprint protection failed] ${currentUrl}: ${fingerprintErr.message}`);
+        } else {
+          throw fingerprintErr;
+        }
+      }
 
       const regexes = Array.isArray(siteConfig.filterRegex)
         ? siteConfig.filterRegex.map(r => new RegExp(r.replace(/^\/(.*)\/$/, '$1')))
@@ -3082,6 +3104,18 @@ function setupFrameHandling(page, forceDebug) {
       // Page finished initial loading - mark as idle
       updatePageUsage(page, false);
       } catch (err) {
+        // Handle detached frame errors during navigation
+        if (err.message.includes('Navigating frame was detached') || 
+            err.message.includes('Attempted to use detached')) {
+          // Silent handling - this is expected for iframe-heavy sites
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Frame detachment during navigation (expected): ${currentUrl}`));
+          }
+          // Continue with partial success - don't fail completely
+          currentPageUrl = currentUrl;
+          siteCounter++;
+          // Skip to post-navigation processing
+        } else {
         // Enhanced error handling for redirect timeouts using redirect module
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
         
@@ -3095,6 +3129,7 @@ function setupFrameHandling(page, forceDebug) {
           throw err;
         }
       }
+      }
 
       if (interactEnabled && !disableInteract) {
         if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
@@ -3141,13 +3176,41 @@ function setupFrameHandling(page, forceDebug) {
       }
       
       for (let i = 1; i <= totalReloads; i++) {
-        // Clear any stale frame references before reload
+  // Check browser health before attempting reload
+  try {
+    const browserHealthy = await isQuicklyResponsive(browser, 2000);
+    if (!browserHealthy) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Browser unresponsive before reload #${i}, skipping remaining reloads`));
+      }
+      console.warn(`Browser unresponsive before reload #${i}, skipping remaining reloads`);
+      break;
+    }
+  } catch (healthErr) {
+    console.warn(`Browser health check failed before reload #${i}: ${healthErr.message}`);
+    break;
+  }
+        // Check if page is still valid before attempting reload
+        let pageStillValid = false;
         try {
-          await page.evaluate(() => {
-            // Force cleanup of any pending frame operations
-            if (window.requestIdleCallback) window.requestIdleCallback(() => {});
-          });
-        } catch (cleanupErr) { /* ignore */ }
+    // Add timeout to page validity check
+    await Promise.race([
+      page.evaluate(() => true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Page validity check timeout')), 3000)
+      )
+    ]);
+          pageStillValid = true;
+        } catch (validityCheck) {
+          console.warn(`Page invalid before reload #${i}, skipping remaining reloads`);
+          break;
+        }
+        
+        // Use comprehensive cleanup from browserhealth module
+        await cleanupPageBeforeReload(page, forceDebug);
+        
+        // Add stabilization delay after cleanup
+        await fastTimeout(1000);
 
         if (siteConfig.clear_sitedata === true) {
           try {
@@ -3159,8 +3222,11 @@ function setupFrameHandling(page, forceDebug) {
         }
         
       let reloadSuccess = false;
+
+  // Skip force reload if browser seems unhealthy
+  const skipForceReload = i > 2; // After 2 attempts, skip force reload
       
-      if (useForceReload && !reloadSuccess) {
+      if (useForceReload && !reloadSuccess && !skipForceReload) {
         // Attempt force reload: disable cache, reload, re-enable cache
           try {
           // Timeout-protected cache disable
@@ -3169,28 +3235,23 @@ function setupFrameHandling(page, forceDebug) {
             new Promise((_, reject) => setTimeout(() => reject(new Error('Cache disable timeout')), 8000))
           ]);
           
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
+            // Use networkidle2 for force reload to better detect when page is actually loaded
+            await page.reload({ waitUntil: 'networkidle2', timeout: Math.min(timeout, 15000) });
           
           // Timeout-protected cache enable
           await Promise.race([
             page.setCacheEnabled(true),
             new Promise((_, reject) => setTimeout(() => reject(new Error('Cache enable timeout')), 8000))
           ]);
-          
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
-          
-          await Promise.race([
-            page.setCacheEnabled(true),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('setCacheEnabled(true) timeout')), 5000)
-            )
-          ]);
-          
+         
           reloadSuccess = true;
             if (forceDebug) console.log(formatLogMessage('debug', `Force reload #${i} completed for ${currentUrl}`));
 
           } catch (forceReloadErr) {
-          console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            // Don't warn for timeouts on problematic sites, just fall back silently
+            if (forceDebug || !forceReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            }
           reloadSuccess = false; // Ensure we try standard reload
         }
       }
@@ -3198,24 +3259,39 @@ function setupFrameHandling(page, forceDebug) {
       // Fallback to standard reload if force reload failed or wasn't attempted
       if (!reloadSuccess) {
         try {
-          // Reduced timeout for faster failure detection
-          const standardReloadTimeout = Math.min(timeout, 8000); // Reduced from 15000ms
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: standardReloadTimeout });
+            const canReload = await page.evaluate(() => {
+              return !!(document && document.body);
+            }).catch(() => false);
+            
+            if (!canReload) {
+              throw new Error('Page document invalid for reload');
+            }
+
+            // Use networkidle2 with reasonable timeout
+      // Use simpler reload for problematic pages
+      const reloadOptions = i > 1 
+        ? { waitUntil: 'domcontentloaded', timeout: 10000 }  // Simpler after failures
+        : { waitUntil: 'networkidle2', timeout: 15000 };     // Full wait first time
+      
+      await page.reload(reloadOptions);
+            
           if (forceDebug) console.log(formatLogMessage('debug', `Standard reload #${i} completed for ${currentUrl}`));
         } catch (standardReloadErr) {
-          console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            // Only warn for non-timeout errors
+            if (!standardReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            } else if (forceDebug) {
+              console.log(formatLogMessage('debug', `Reload #${i} timed out for ${currentUrl}, continuing anyway`));
+            }
           
           // Check if this is a persistent failure that should skip remaining reloads
-          const isPersistentFailure = standardReloadErr.message.includes('Navigation timeout') || 
+            const isPersistentFailure = standardReloadErr.message.includes('detached Frame') ||
+                                     standardReloadErr.message.includes('Attempted to use detached') ||
+                                     standardReloadErr.message.includes('Navigating frame was detached') ||
+                                     standardReloadErr.message.includes('document invalid') ||
                                      standardReloadErr.message.includes('net::ERR_') ||
                                      standardReloadErr.message.includes('Protocol error') ||
-                                     standardReloadErr.message.includes('Page crashed') ||
-                                     // CDP and injection failures
-                                     standardReloadErr.constructor.name === 'ProtocolError' ||
-                                     standardReloadErr.name === 'ProtocolError' ||
-                                     standardReloadErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
-                                     standardReloadErr.message.includes('Runtime.callFunctionOn timed out') ||
-                                     standardReloadErr.message.includes('CDP injection timeout');
+                                       standardReloadErr.message.includes('Page crashed');
           
           if (isPersistentFailure) {
             const remainingReloads = totalReloads - i;
@@ -3225,16 +3301,16 @@ function setupFrameHandling(page, forceDebug) {
             // Break out of reload loop to move to next URL faster
             break;
           }
+            // For navigation timeouts, we can continue - the page might still be partially loaded
+            // Don't break the loop for simple timeouts
           }
-        } else {
-          // Regular reload
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
         }
 
       // Only add delay if we're continuing with more reloads
       if (i < totalReloads) {
-
-        await fastTimeout(delayMs);
+    // Reduce delay for problematic sites
+    const adjustedDelay = i > 1 ? Math.min(delayMs, 2000) : delayMs;
+    await fastTimeout(adjustedDelay);
       }
     }
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.9",
+  "version": "2.0.10",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {