@fanboynz/network-scanner 2.0.9 → 2.0.11

package/lib/browserhealth.js

@@ -970,6 +970,90 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
   }
 }
 
+/**
+ * Performs comprehensive cleanup of page resources before operations that might cause detached frames
+ * Also attempts to stop any pending navigations that might interfere
+ * Used before reloads, navigations, and other operations that can trigger frame detachment
+ * @param {import('puppeteer').Page} page - Page to clean up
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<boolean>} True if cleanup succeeded
+ */
+async function cleanupPageBeforeReload(page, forceDebug = false) {
+  try {
+    if (page.isClosed()) {
+      return false;
+    }
+
+    // First, try to stop any pending navigation
+    try {
+      await page.evaluate(() => {
+        // Stop any ongoing navigation
+        if (window.stop) {
+          window.stop();
+        }
+      });
+    } catch (e) {
+      // Page might be mid-navigation, that's ok
+    }
+
+    // Wait a bit for navigation to stop
+    await new Promise(resolve => setTimeout(resolve, 500));
+
+    // Now do the full cleanup
+    await page.evaluate(() => {
+      // Stop all media elements
+      document.querySelectorAll('video, audio').forEach(media => {
+        try { 
+          media.pause(); 
+          media.src = ''; 
+          media.load();
+        } catch(e) {}
+      });
+      
+      // Clear all timers and intervals
+      const highestId = setTimeout(() => {}, 0);
+      for (let i = highestId; i >= 0; i--) {
+        clearTimeout(i);
+        clearInterval(i);
+      }
+      
+      // Stop all animations
+      if (typeof cancelAnimationFrame !== 'undefined') {
+        const highestRAF = requestAnimationFrame(() => {});
+        for (let i = highestRAF; i >= 0; i--) {
+          cancelAnimationFrame(i);
+        }
+      }
+      
+      // Clear all iframes properly
+      document.querySelectorAll('iframe').forEach(iframe => {
+        try {
+          // Stop iframe content first
+          if (iframe.contentWindow) {
+            iframe.contentWindow.stop();
+          }
+          iframe.src = 'about:blank';
+          iframe.remove();
+        } catch(e) {}
+      });
+      
+      // Force garbage collection if available
+      if (window.gc) window.gc();
+    });
+
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', 'Page resources cleaned before reload'));
+    }
+
+    return true;
+  } catch (err) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `Page cleanup error: ${err.message}`));
+    }
+    return false;
+  }
+}
+
 module.exports = {
   checkBrowserHealth,
   checkBrowserMemory,
@@ -983,7 +1067,8 @@ module.exports = {
   monitorBrowserHealth,
   isBrowserHealthy,
   isCriticalProtocolError,
-  updatePageUsage
+  updatePageUsage,
+  cleanupPageBeforeReload
 };
 
 // Clean up tracking maps when pages are closed

package/lib/cloudflare.js

@@ -1,5 +1,6 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.3 - Fixes Cannot read properties of undefined (reading 'hasIndicators')
  * Version: 2.6.2 - Further detached Frame fixes 
  * Version: 2.6.1 - timeoutId is not defined & race condition fix
  * Version: 2.6.0 - Memory leak fixes and timeout cleanup
@@ -19,7 +20,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.2';
+const CLOUDFLARE_MODULE_VERSION = '2.6.3';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -1570,6 +1571,11 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
   
   // Quick detection first - exit early if no Cloudflare detected and no explicit config
   const quickDetection = await quickCloudflareDetection(page, forceDebug);
+  
+  // Safety check: ensure quickDetection is valid
+  if (!quickDetection) {
+    return { phishingWarning: { attempted: false, success: true }, verificationChallenge: { attempted: false, success: true }, overallSuccess: true, errors: [], quickDetectionFailed: true };
+  }
 
   // Early return structure when no Cloudflare indicators found
   // Sets attempted: false, success: true for both protection types

package/lib/fingerprint.js

@@ -173,6 +173,23 @@ function generateCSIData() {
   };
 }
 
+/**
+ * Validates page state before script injection to avoid timeouts
+ */
+async function validatePageForInjection(page, currentUrl, forceDebug) {
+  try {
+    if (page.isClosed()) return false;
+    await Promise.race([
+      page.evaluate(() => document.readyState),
+      new Promise((_, reject) => setTimeout(() => reject(new Error('Page unresponsive')), 3000))
+    ]);
+    return true;
+  } catch (validationErr) {
+    if (forceDebug) console.log(`[debug] Skipping fingerprint protection - page unresponsive: ${currentUrl}`);
+    return false;
+  }
+}
+
 /**
  * Creates mock fingerprinting objects
  */
@@ -251,6 +268,9 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
 
   if (forceDebug) console.log(`[debug] User agent spoofing: ${siteConfig.userAgent}`);
 
+  // Validate page state before injection
+  if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
+
   const selectedUserAgents = USER_AGENT_COLLECTIONS[siteConfig.userAgent.toLowerCase()];
   const ua = selectedUserAgents ? selectedUserAgents[Math.floor(Math.random() * selectedUserAgents.length)] : null;
   
@@ -660,6 +680,12 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
 
       }, ua, forceDebug);
     } catch (stealthErr) {
+      if (stealthErr.message.includes('Session closed') || 
+          stealthErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+          stealthErr.message.includes('Target closed')) {
+        if (forceDebug) console.log(`[debug] Page closed during stealth injection: ${currentUrl}`);
+        return;
+      }
       console.warn(`[stealth protection failed] ${currentUrl}: ${stealthErr.message}`);
     }
   }
@@ -672,8 +698,23 @@ async function applyBraveSpoofing(page, siteConfig, forceDebug, currentUrl) {
   if (!siteConfig.isBrave) return;
 
   if (forceDebug) console.log(`[debug] Brave spoofing enabled for ${currentUrl}`);
+
+  // Validate page state before injection
+  if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
   
-  await page.evaluateOnNewDocument((debugEnabled) => {
+  try {
+    await page.evaluateOnNewDocument((debugEnabled) => {
+      // ... existing Brave spoofing code ...
+    }, forceDebug);
+  } catch (braveErr) {
+    if (braveErr.message.includes('Session closed') || 
+        braveErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+        braveErr.message.includes('Target closed')) {
+      if (forceDebug) console.log(`[debug] Page closed during Brave injection: ${currentUrl}`);
+      return;
+    }
+    if (forceDebug) console.log(`[debug] Brave spoofing failed: ${currentUrl} - ${braveErr.message}`);
+  }
     try {
       Object.defineProperty(navigator, 'brave', {
         get: () => ({
@@ -695,7 +736,6 @@ async function applyBraveSpoofing(page, siteConfig, forceDebug, currentUrl) {
     } catch (err) {
       if (debugEnabled) console.log(`[fingerprint] Brave spoofing error: ${err.message}`);
     }
-  }, forceDebug);
 }
 
 /**
@@ -707,6 +747,9 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
 
   if (forceDebug) console.log(`[debug] Fingerprint protection enabled for ${currentUrl}`);
   
+  // Validate page state before injection
+  if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
+  
   const spoof = fingerprintSetting === 'random' ? getRandomFingerprint() : {
     deviceMemory: 8, 
     hardwareConcurrency: 4,
@@ -796,6 +839,12 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
       
     }, { spoof, debugEnabled: forceDebug });
   } catch (err) {
+    if (err.message.includes('Session closed') || 
+        err.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+        err.message.includes('Target closed')) {
+      if (forceDebug) console.log(`[debug] Page closed during fingerprint injection: ${currentUrl}`);
+      return;
+    }
     console.warn(`[fingerprint protection failed] ${currentUrl}: ${err.message}`);
   }
 }

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.9 ===
+// === Network scanner script (nwss.js) v2.0.11 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -55,19 +55,20 @@ function fastTimeout(ms) {
 
 // --- Configuration Constants ---
 const TIMEOUTS = {
-  DEFAULT_PAGE: 30000,
-  DEFAULT_NAVIGATION: 25000,
+  DEFAULT_PAGE: 35000,                // Standard page load timeout (35s)
+  DEFAULT_NAVIGATION: 25000,          // Navigation operation timeout
   DEFAULT_NAVIGATION_REDUCED: 20000,  // Reduced timeout for faster failures
-  DEFAULT_PAGE_REDUCED: 15000,        // Reduced page timeout
-  FRAME_LOAD_WAIT: 2000,
-  NETWORK_IDLE: 2000,
-  NETWORK_IDLE_MAX: 10000,
-  FAST_SITE_THRESHOLD: 15000,
-  EMERGENCY_RESTART_DELAY: 2000,
-  BROWSER_STABILIZE_DELAY: 1000,
-  CURL_HANDLER_DELAY: 3000,
-  PROTOCOL_TIMEOUT: 120000,
-  REDIRECT_JS_TIMEOUT: 5000
+  DEFAULT_PAGE_REDUCED: 15000,        // Faster page timeout for quick failures
+  FRAME_LOAD_WAIT: 2000,              // Wait time for iframes to load 
+  DEFAULT_DELAY: 6000,                // Default delay: after page load
+  NETWORK_IDLE: 2000,                 // Network idle detection time
+  NETWORK_IDLE_MAX: 10000,            // Maximum network idle wait time
+  FAST_SITE_THRESHOLD: 15000,         // Threshold for "fast site" optimizations
+  EMERGENCY_RESTART_DELAY: 2000,      // Delay after emergency browser restart
+  BROWSER_STABILIZE_DELAY: 1000,      // Browser stabilization after restart
+  CURL_HANDLER_DELAY: 3000,           // Wait for async curl operations
+  PROTOCOL_TIMEOUT: 180000,           // Chrome DevTools Protocol timeout
+  REDIRECT_JS_TIMEOUT: 5000           // JavaScript redirect detection timeout
 };
 
 const CACHE_LIMITS = {
@@ -126,10 +127,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.9'; // Script version
+const VERSION = '2.0.11'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1087,7 +1088,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Set();
+  let activeFrames = new Map(); // Use Map to track frame state
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1101,6 +1102,15 @@ function setupFrameHandling(page, forceDebug) {
   page.on('frameattached', async (frame) => {
     // Enhanced frame handling with detached frame protection
     try {
+      // Test frame accessibility first with safe method
+      let isFrameValid = false;
+      try {
+        frame.url(); // This will throw if frame is detached
+        isFrameValid = true;
+      } catch (e) {
+        return; // Frame is already detached, skip
+      }
+
       // Multiple checks for frame validity to prevent detached frame errors
       if (!frame) {
         if (forceDebug) {
@@ -1136,7 +1146,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    if (frame.parentFrame()) { // Only handle child frames, not main frame
+    // Store frame with timestamp for tracking
+    activeFrames.set(frame, Date.now());
+    
+    if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
       try {       
         if (forceDebug) {
           console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
@@ -1979,7 +1992,17 @@ function setupFrameHandling(page, forceDebug) {
       }
 
       // --- Apply all fingerprint spoofing (user agent, Brave, fingerprint protection) ---
-      await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      try {
+        await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      } catch (fingerprintErr) {
+        if (fingerprintErr.message.includes('Session closed') || 
+            fingerprintErr.message.includes('Protocol error') ||
+            fingerprintErr.message.includes('addScriptToEvaluateOnNewDocument')) {
+          console.warn(`[fingerprint protection failed] ${currentUrl}: ${fingerprintErr.message}`);
+        } else {
+          throw fingerprintErr;
+        }
+      }
 
       const regexes = Array.isArray(siteConfig.filterRegex)
         ? siteConfig.filterRegex.map(r => new RegExp(r.replace(/^\/(.*)\/$/, '$1')))
@@ -3082,6 +3105,18 @@ function setupFrameHandling(page, forceDebug) {
       // Page finished initial loading - mark as idle
       updatePageUsage(page, false);
       } catch (err) {
+        // Handle detached frame errors during navigation
+        if (err.message.includes('Navigating frame was detached') || 
+            err.message.includes('Attempted to use detached')) {
+          // Silent handling - this is expected for iframe-heavy sites
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Frame detachment during navigation (expected): ${currentUrl}`));
+          }
+          // Continue with partial success - don't fail completely
+          currentPageUrl = currentUrl;
+          siteCounter++;
+          // Skip to post-navigation processing
+        } else {
         // Enhanced error handling for redirect timeouts using redirect module
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
         
@@ -3095,6 +3130,7 @@ function setupFrameHandling(page, forceDebug) {
           throw err;
         }
       }
+      }
 
       if (interactEnabled && !disableInteract) {
         if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
@@ -3105,7 +3141,7 @@ function setupFrameHandling(page, forceDebug) {
         await performPageInteraction(page, currentUrl, interactionConfig, forceDebug);
       }
 
-      const delayMs = siteConfig.delay || 4000;
+      const delayMs = DEFAULT_DELAY;
       
       // Optimized delays for Puppeteer 23.x performance
       const isFastSite = timeout <= TIMEOUTS.FAST_SITE_THRESHOLD;
@@ -3141,13 +3177,41 @@ function setupFrameHandling(page, forceDebug) {
       }
       
       for (let i = 1; i <= totalReloads; i++) {
-        // Clear any stale frame references before reload
+  // Check browser health before attempting reload
+  try {
+    const browserHealthy = await isQuicklyResponsive(browser, 2000);
+    if (!browserHealthy) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Browser unresponsive before reload #${i}, skipping remaining reloads`));
+      }
+      console.warn(`Browser unresponsive before reload #${i}, skipping remaining reloads`);
+      break;
+    }
+  } catch (healthErr) {
+    console.warn(`Browser health check failed before reload #${i}: ${healthErr.message}`);
+    break;
+  }
+        // Check if page is still valid before attempting reload
+        let pageStillValid = false;
         try {
-          await page.evaluate(() => {
-            // Force cleanup of any pending frame operations
-            if (window.requestIdleCallback) window.requestIdleCallback(() => {});
-          });
-        } catch (cleanupErr) { /* ignore */ }
+    // Add timeout to page validity check
+    await Promise.race([
+      page.evaluate(() => true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Page validity check timeout')), 3000)
+      )
+    ]);
+          pageStillValid = true;
+        } catch (validityCheck) {
+          console.warn(`Page invalid before reload #${i}, skipping remaining reloads`);
+          break;
+        }
+        
+        // Use comprehensive cleanup from browserhealth module
+        await cleanupPageBeforeReload(page, forceDebug);
+        
+        // Add stabilization delay after cleanup
+        await fastTimeout(1000);
 
         if (siteConfig.clear_sitedata === true) {
           try {
@@ -3159,8 +3223,11 @@ function setupFrameHandling(page, forceDebug) {
         }
         
       let reloadSuccess = false;
+
+  // Skip force reload if browser seems unhealthy
+  const skipForceReload = i > 2; // After 2 attempts, skip force reload
       
-      if (useForceReload && !reloadSuccess) {
+      if (useForceReload && !reloadSuccess && !skipForceReload) {
         // Attempt force reload: disable cache, reload, re-enable cache
           try {
           // Timeout-protected cache disable
@@ -3169,28 +3236,23 @@ function setupFrameHandling(page, forceDebug) {
             new Promise((_, reject) => setTimeout(() => reject(new Error('Cache disable timeout')), 8000))
           ]);
           
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
+            // Use networkidle2 for force reload to better detect when page is actually loaded
+            await page.reload({ waitUntil: 'networkidle2', timeout: Math.min(timeout, 15000) });
           
           // Timeout-protected cache enable
           await Promise.race([
             page.setCacheEnabled(true),
             new Promise((_, reject) => setTimeout(() => reject(new Error('Cache enable timeout')), 8000))
           ]);
-          
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
-          
-          await Promise.race([
-            page.setCacheEnabled(true),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('setCacheEnabled(true) timeout')), 5000)
-            )
-          ]);
-          
+         
           reloadSuccess = true;
             if (forceDebug) console.log(formatLogMessage('debug', `Force reload #${i} completed for ${currentUrl}`));
 
           } catch (forceReloadErr) {
-          console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            // Don't warn for timeouts on problematic sites, just fall back silently
+            if (forceDebug || !forceReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            }
           reloadSuccess = false; // Ensure we try standard reload
         }
       }
@@ -3198,24 +3260,39 @@ function setupFrameHandling(page, forceDebug) {
       // Fallback to standard reload if force reload failed or wasn't attempted
       if (!reloadSuccess) {
         try {
-          // Reduced timeout for faster failure detection
-          const standardReloadTimeout = Math.min(timeout, 8000); // Reduced from 15000ms
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: standardReloadTimeout });
+            const canReload = await page.evaluate(() => {
+              return !!(document && document.body);
+            }).catch(() => false);
+            
+            if (!canReload) {
+              throw new Error('Page document invalid for reload');
+            }
+
+            // Use networkidle2 with reasonable timeout
+      // Use simpler reload for problematic pages
+      const reloadOptions = i > 1 
+        ? { waitUntil: 'domcontentloaded', timeout: 10000 }  // Simpler after failures
+        : { waitUntil: 'networkidle2', timeout: 15000 };     // Full wait first time
+      
+      await page.reload(reloadOptions);
+            
           if (forceDebug) console.log(formatLogMessage('debug', `Standard reload #${i} completed for ${currentUrl}`));
         } catch (standardReloadErr) {
-          console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            // Only warn for non-timeout errors
+            if (!standardReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            } else if (forceDebug) {
+              console.log(formatLogMessage('debug', `Reload #${i} timed out for ${currentUrl}, continuing anyway`));
+            }
           
           // Check if this is a persistent failure that should skip remaining reloads
-          const isPersistentFailure = standardReloadErr.message.includes('Navigation timeout') || 
+            const isPersistentFailure = standardReloadErr.message.includes('detached Frame') ||
+                                     standardReloadErr.message.includes('Attempted to use detached') ||
+                                     standardReloadErr.message.includes('Navigating frame was detached') ||
+                                     standardReloadErr.message.includes('document invalid') ||
                                      standardReloadErr.message.includes('net::ERR_') ||
                                      standardReloadErr.message.includes('Protocol error') ||
-                                     standardReloadErr.message.includes('Page crashed') ||
-                                     // CDP and injection failures
-                                     standardReloadErr.constructor.name === 'ProtocolError' ||
-                                     standardReloadErr.name === 'ProtocolError' ||
-                                     standardReloadErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
-                                     standardReloadErr.message.includes('Runtime.callFunctionOn timed out') ||
-                                     standardReloadErr.message.includes('CDP injection timeout');
+                                       standardReloadErr.message.includes('Page crashed');
           
           if (isPersistentFailure) {
             const remainingReloads = totalReloads - i;
@@ -3225,16 +3302,16 @@ function setupFrameHandling(page, forceDebug) {
             // Break out of reload loop to move to next URL faster
             break;
           }
+            // For navigation timeouts, we can continue - the page might still be partially loaded
+            // Don't break the loop for simple timeouts
           }
-        } else {
-          // Regular reload
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
         }
 
       // Only add delay if we're continuing with more reloads
       if (i < totalReloads) {
-
-        await fastTimeout(delayMs);
+    // Reduce delay for problematic sites
+    const adjustedDelay = i > 1 ? Math.min(DEFAULT_DELAY, 2000) : DEFAULT_DELAY;
+    await fastTimeout(adjustedDelay);
       }
     }
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.9",
+  "version": "2.0.11",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {