@fanboynz/network-scanner 2.0.8 → 2.0.10

package/README.md

@@ -248,6 +248,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `isBrave`            | Boolean | `false` | Spoof Brave browser detection |
 | `evaluateOnNewDocument` | Boolean | `false` | Inject fetch/XHR interceptor in page |
 | `cdp`                | Boolean | `false` | Enable CDP logging for this site |
+| `cdp_specific`       | Array | - | Enable CDP logging only for specific domains in the URL list |
 | `css_blocked`        | Array | - | CSS selectors to hide elements |
 | `source`             | Boolean | `false` | Save page source HTML after load |
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |

package/lib/browserhealth.js

@@ -970,6 +970,90 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
   }
 }
 
+/**
+ * Performs comprehensive cleanup of page resources before operations that might cause detached frames
+ * Also attempts to stop any pending navigations that might interfere
+ * Used before reloads, navigations, and other operations that can trigger frame detachment
+ * @param {import('puppeteer').Page} page - Page to clean up
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<boolean>} True if cleanup succeeded
+ */
+async function cleanupPageBeforeReload(page, forceDebug = false) {
+  try {
+    if (page.isClosed()) {
+      return false;
+    }
+
+    // First, try to stop any pending navigation
+    try {
+      await page.evaluate(() => {
+        // Stop any ongoing navigation
+        if (window.stop) {
+          window.stop();
+        }
+      });
+    } catch (e) {
+      // Page might be mid-navigation, that's ok
+    }
+
+    // Wait a bit for navigation to stop
+    await new Promise(resolve => setTimeout(resolve, 500));
+
+    // Now do the full cleanup
+    await page.evaluate(() => {
+      // Stop all media elements
+      document.querySelectorAll('video, audio').forEach(media => {
+        try { 
+          media.pause(); 
+          media.src = ''; 
+          media.load();
+        } catch(e) {}
+      });
+      
+      // Clear all timers and intervals
+      const highestId = setTimeout(() => {}, 0);
+      for (let i = highestId; i >= 0; i--) {
+        clearTimeout(i);
+        clearInterval(i);
+      }
+      
+      // Stop all animations
+      if (typeof cancelAnimationFrame !== 'undefined') {
+        const highestRAF = requestAnimationFrame(() => {});
+        for (let i = highestRAF; i >= 0; i--) {
+          cancelAnimationFrame(i);
+        }
+      }
+      
+      // Clear all iframes properly
+      document.querySelectorAll('iframe').forEach(iframe => {
+        try {
+          // Stop iframe content first
+          if (iframe.contentWindow) {
+            iframe.contentWindow.stop();
+          }
+          iframe.src = 'about:blank';
+          iframe.remove();
+        } catch(e) {}
+      });
+      
+      // Force garbage collection if available
+      if (window.gc) window.gc();
+    });
+
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', 'Page resources cleaned before reload'));
+    }
+
+    return true;
+  } catch (err) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `Page cleanup error: ${err.message}`));
+    }
+    return false;
+  }
+}
+
 module.exports = {
   checkBrowserHealth,
   checkBrowserMemory,
@@ -983,7 +1067,8 @@ module.exports = {
   monitorBrowserHealth,
   isBrowserHealthy,
   isCriticalProtocolError,
-  updatePageUsage
+  updatePageUsage,
+  cleanupPageBeforeReload
 };
 
 // Clean up tracking maps when pages are closed

package/lib/cdp.js

@@ -116,6 +116,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
  * @param {boolean} options.enableCDP - Global CDP flag (from --cdp command line)
  * @param {boolean} options.siteSpecificCDP - Site-specific CDP flag (from config)
  * @param {boolean} options.forceDebug - Debug logging flag
+ * @param {string} options.currentUrl - Current URL for domain-specific CDP decisions
  * @returns {Promise<object>} CDP session object with cleanup method
  */
 async function createCDPSession(page, currentUrl, options = {}) {
@@ -132,10 +133,14 @@ async function createCDPSession(page, currentUrl, options = {}) {
 
   // Log which CDP mode is being used
   if (forceDebug) {
+    const urlHostname = (() => {
+      try { return new URL(currentUrl).hostname; } catch { return 'unknown'; }
+    })();
+
     if (enableCDP) {
-      console.log(formatLogMessage('debug', `CDP logging globally enabled by --cdp, applying to page: ${currentUrl}`));
+      console.log(formatLogMessage('debug', `[cdp] Global CDP enabled by --cdp flag for ${urlHostname}`));
     } else if (siteSpecificCDP === true) {
-      console.log(formatLogMessage('debug', `CDP logging enabled for page ${currentUrl} via site-specific 'cdp: true' config.`));
+      console.log(formatLogMessage('debug', `[cdp] Site-specific CDP enabled for ${urlHostname} (via cdp: true or cdp_specific domain match)`));
     }
   }
 
@@ -163,14 +168,22 @@ async function createCDPSession(page, currentUrl, options = {}) {
       // Extract hostname for logging context (handles URL parsing errors gracefully)
       let hostnameForLog = 'unknown-host';
       try {
-        hostnameForLog = new URL(currentUrl).hostname;
+        const currentHostname = new URL(currentUrl).hostname;
+        const requestHostname = new URL(requestUrl).hostname;
+        // Show both hostnames if different (cross-domain requests)
+        if (currentHostname !== requestHostname) {
+          hostnameForLog = `${currentHostname}?${requestHostname}`;
+        } else {
+          hostnameForLog = currentHostname;
+        }
       } catch (_) { 
         // Ignore URL parsing errors for logging context
       }
       
-      // Log the request with context - customize this for your needs
-      // Format: [cdp][hostname] METHOD url (initiator: type)
-      console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
+ // Log the request with context only if debug mode is enabled
+ if (forceDebug) {
+   console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
+ }
     });
 
     if (forceDebug) {
@@ -200,6 +213,15 @@ async function createCDPSession(page, currentUrl, options = {}) {
   } catch (cdpErr) {
     cdpSession = null; // Reset on failure
     
+    // Enhanced error context for CDP domain-specific debugging
+    const urlContext = (() => {
+      try {
+        return new URL(currentUrl).hostname;
+      } catch {
+        return currentUrl.substring(0, 50) + '...';
+      }
+    })();
+
     // Categorize CDP errors for proper handling
     // Enhanced error handling for Puppeteer 20+ error patterns
     if (cdpErr.message.includes('Network.enable timed out') || 
@@ -236,9 +258,10 @@ async function createCDPSession(page, currentUrl, options = {}) {
  *
  * @param {object} siteConfig - Site configuration object
  * @param {boolean} globalCDP - Global CDP flag
+ * @param {Array} cdpSpecificDomains - Array of domains for cdp_specific feature
  * @returns {object} Validation result with recommendations
  */
-function validateCDPConfig(siteConfig, globalCDP) {
+function validateCDPConfig(siteConfig, globalCDP, cdpSpecificDomains = []) {
   const warnings = [];
   const recommendations = [];
   
@@ -247,8 +270,25 @@ function validateCDPConfig(siteConfig, globalCDP) {
     warnings.push('Site-specific CDP disabled but global CDP is enabled - global setting will override');
   }
   
+  // Validate cdp_specific configuration
+  if (siteConfig.cdp_specific) {
+    if (!Array.isArray(siteConfig.cdp_specific)) {
+      warnings.push('cdp_specific must be an array of domain strings');
+    } else if (siteConfig.cdp_specific.length === 0) {
+      warnings.push('cdp_specific is empty - no domains will have CDP enabled');
+    } else {
+      // Validate domain format
+      const invalidDomains = siteConfig.cdp_specific.filter(domain => {
+        return typeof domain !== 'string' || domain.trim() === '';
+      });
+      if (invalidDomains.length > 0) {
+        warnings.push(`cdp_specific contains invalid domains: ${invalidDomains.join(', ')}`);
+      }
+    }
+  }
+  
   // Performance recommendations
-  if (globalCDP || siteConfig.cdp === true) {
+  if (globalCDP || siteConfig.cdp === true || (siteConfig.cdp_specific && siteConfig.cdp_specific.length > 0)) {
     recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
     
     if (siteConfig.timeout && siteConfig.timeout < 30000) {

package/lib/cloudflare.js

@@ -1,5 +1,6 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.3 - Fixes Cannot read properties of undefined (reading 'hasIndicators')
  * Version: 2.6.2 - Further detached Frame fixes 
  * Version: 2.6.1 - timeoutId is not defined & race condition fix
  * Version: 2.6.0 - Memory leak fixes and timeout cleanup
@@ -19,7 +20,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.2';
+const CLOUDFLARE_MODULE_VERSION = '2.6.3';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -1570,6 +1571,11 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
   
   // Quick detection first - exit early if no Cloudflare detected and no explicit config
   const quickDetection = await quickCloudflareDetection(page, forceDebug);
+  
+  // Safety check: ensure quickDetection is valid
+  if (!quickDetection) {
+    return { phishingWarning: { attempted: false, success: true }, verificationChallenge: { attempted: false, success: true }, overallSuccess: true, errors: [], quickDetectionFailed: true };
+  }
 
   // Early return structure when no Cloudflare indicators found
   // Sets attempted: false, success: true for both protection types

package/nwss.1

@@ -282,7 +282,6 @@ Array of CSS selectors to hide elements on the page.
 .B userAgent
 Spoof User-Agent: \fB"chrome"\fR, \fB"chrome_mac"\fR, \fB"chrome_linux"\fR, \fB"firefox"\fR, \fB"firefox_mac"\fR, \fB"firefox_linux"\fR, or \fB"safari"\fR.
 
-
 .TP
 .B interact
 Boolean. Simulate mouse movements and clicks.
@@ -442,6 +441,10 @@ Boolean. Inject Fetch/XHR interceptor scripts into page context.
 .B cdp
 Boolean. Enable Chrome DevTools Protocol logging for this specific site.
 
+.TP
+.B cdp_specific
+Array of domain names. Enable Chrome DevTools Protocol logging only for URLs matching these specific domains within a multi-URL site configuration. Takes precedence over \fBcdp: false\fR but is ignored if \fBcdp: true\fR is set. Supports exact hostname matching and subdomain matching (e.g., "example.com" matches both "example.com" and "subdomain.example.com"). Useful for selective debugging of network requests on specific domains while avoiding CDP overhead on others.
+
 .TP
 .B source
 Boolean. Save page source HTML after loading.
@@ -880,6 +883,26 @@ node nwss.js --ignore-cache --debug -o rules.txt
 }
 .EE
 
+.SS Selective CDP logging for specific domains:
+.EX
+{
+  "url": [
+    "https://site1.com/page1",
+    "https://debug-target.com/page2", 
+    "https://site2.com/page3"
+  ],
+  "filterRegex": "\\\\.(space|website)\\\\b",
+  "cdp_specific": ["debug-target.com"],
+  "resourceTypes": ["script", "fetch"],
+  "comments": [
+    "CDP enabled only for debug-target.com",
+    "Other URLs run without CDP overhead"
+  ]
+}
+.EE
+
+Note: If \fBcdp: true\fR is also set, \fBcdp_specific\fR is ignored and CDP is enabled for all URLs.
+
 .SS FlowProxy protection handling:
 .EX
 {

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.8 ===
+// === Network scanner script (nwss.js) v2.0.10 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -66,7 +66,7 @@ const TIMEOUTS = {
   EMERGENCY_RESTART_DELAY: 2000,
   BROWSER_STABILIZE_DELAY: 1000,
   CURL_HANDLER_DELAY: 3000,
-  PROTOCOL_TIMEOUT: 120000,
+  PROTOCOL_TIMEOUT: 160000,
   REDIRECT_JS_TIMEOUT: 5000
 };
 
@@ -126,10 +126,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.8'; // Script version
+const VERSION = '2.0.10'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -555,6 +555,7 @@ FlowProxy Protection Options:
 Advanced Options:
   evaluateOnNewDocument: true/false           Inject fetch/XHR interceptor in page (for this site)
   cdp: true/false                            Enable CDP logging for this site Inject fetch/XHR interceptor in page
+  cdp_specific: ["domain1.com", "domain2.com"] Enable CDP logging only for specific domains in the URL list
   interact_duration: <milliseconds>           Duration of interaction simulation (default: 2000)
   interact_scrolling: true/false              Enable scrolling simulation (default: true)
   interact_clicks: true/false                 Enable element clicking simulation (default: false)
@@ -923,6 +924,30 @@ function safeGetDomain(url, getFullHostname = false) {
   }
 }
 
+/**
+ * Checks if a URL matches any domain in the cdp_specific list
+ * @param {string} url - The URL to check
+ * @param {Array} cdpSpecificList - Array of domains that should have CDP enabled
+ * @returns {boolean} True if URL matches a domain in the list
+ */
+function shouldEnableCDPForUrl(url, cdpSpecificList) {
+  if (!cdpSpecificList || !Array.isArray(cdpSpecificList) || cdpSpecificList.length === 0) {
+    return false;
+  }
+  
+  try {
+    const urlHostname = new URL(url).hostname;
+    return cdpSpecificList.some(domain => {
+      // Remove protocol if present and clean domain
+      const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
+      // Match exact hostname or subdomain
+      return urlHostname === cleanDomain || urlHostname.endsWith('.' + cleanDomain);
+    });
+  } catch (urlErr) {
+    return false;
+  }
+}
+
 /**
  * Outputs dry run results to console with formatted display
  * If outputFile is specified, also captures output for file writing
@@ -1062,7 +1087,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Set();
+  let activeFrames = new Map(); // Use Map to track frame state
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1076,6 +1101,15 @@ function setupFrameHandling(page, forceDebug) {
   page.on('frameattached', async (frame) => {
     // Enhanced frame handling with detached frame protection
     try {
+      // Test frame accessibility first with safe method
+      let isFrameValid = false;
+      try {
+        frame.url(); // This will throw if frame is detached
+        isFrameValid = true;
+      } catch (e) {
+        return; // Frame is already detached, skip
+      }
+
       // Multiple checks for frame validity to prevent detached frame errors
       if (!frame) {
         if (forceDebug) {
@@ -1111,7 +1145,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    if (frame.parentFrame()) { // Only handle child frames, not main frame
+    // Store frame with timestamp for tracking
+    activeFrames.set(frame, Date.now());
+    
+    if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
       try {       
         if (forceDebug) {
           console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
@@ -1475,6 +1512,24 @@ function setupFrameHandling(page, forceDebug) {
       return { url: currentUrl, rules: [], success: false, skipped: true };
     }
 
+    // Determine CDP enablement based on cdp_specific or traditional cdp setting
+    let shouldEnableCDPForThisUrl = false;
+    if (siteConfig.cdp === true) {
+      // If cdp: true is set, enable CDP for all URLs and ignore cdp_specific
+      shouldEnableCDPForThisUrl = true;
+      if (forceDebug && siteConfig.cdp_specific) {
+        console.log(formatLogMessage('debug', `CDP enabled for all URLs via cdp: true - ignoring cdp_specific for ${currentUrl}`));
+      }
+    } else if (siteConfig.cdp_specific && Array.isArray(siteConfig.cdp_specific)) {
+      // Only use cdp_specific if cdp is not explicitly set to true
+      shouldEnableCDPForThisUrl = shouldEnableCDPForUrl(currentUrl, siteConfig.cdp_specific);
+      if (forceDebug && shouldEnableCDPForThisUrl) {
+        console.log(formatLogMessage('debug', `CDP enabled for ${currentUrl} via cdp_specific domain match`));
+      }
+    } else {
+      shouldEnableCDPForThisUrl = false;
+    }
+
     let page = null;
     let cdpSession = null;
     let cdpSessionManager = null;
@@ -1884,7 +1939,7 @@ function setupFrameHandling(page, forceDebug) {
       try {
         cdpSessionManager = await createCDPSession(page, currentUrl, {
           enableCDP,
-          siteSpecificCDP: siteConfig.cdp,
+          siteSpecificCDP: shouldEnableCDPForThisUrl,
           forceDebug
         });
       } catch (cdpErr) {
@@ -1936,7 +1991,17 @@ function setupFrameHandling(page, forceDebug) {
       }
 
       // --- Apply all fingerprint spoofing (user agent, Brave, fingerprint protection) ---
-      await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      try {
+        await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+      } catch (fingerprintErr) {
+        if (fingerprintErr.message.includes('Session closed') || 
+            fingerprintErr.message.includes('Protocol error') ||
+            fingerprintErr.message.includes('addScriptToEvaluateOnNewDocument')) {
+          console.warn(`[fingerprint protection failed] ${currentUrl}: ${fingerprintErr.message}`);
+        } else {
+          throw fingerprintErr;
+        }
+      }
 
       const regexes = Array.isArray(siteConfig.filterRegex)
         ? siteConfig.filterRegex.map(r => new RegExp(r.replace(/^\/(.*)\/$/, '$1')))
@@ -3039,6 +3104,18 @@ function setupFrameHandling(page, forceDebug) {
       // Page finished initial loading - mark as idle
       updatePageUsage(page, false);
       } catch (err) {
+        // Handle detached frame errors during navigation
+        if (err.message.includes('Navigating frame was detached') || 
+            err.message.includes('Attempted to use detached')) {
+          // Silent handling - this is expected for iframe-heavy sites
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Frame detachment during navigation (expected): ${currentUrl}`));
+          }
+          // Continue with partial success - don't fail completely
+          currentPageUrl = currentUrl;
+          siteCounter++;
+          // Skip to post-navigation processing
+        } else {
         // Enhanced error handling for redirect timeouts using redirect module
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
         
@@ -3052,6 +3129,7 @@ function setupFrameHandling(page, forceDebug) {
           throw err;
         }
       }
+      }
 
       if (interactEnabled && !disableInteract) {
         if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
@@ -3098,13 +3176,41 @@ function setupFrameHandling(page, forceDebug) {
       }
       
       for (let i = 1; i <= totalReloads; i++) {
-        // Clear any stale frame references before reload
+  // Check browser health before attempting reload
+  try {
+    const browserHealthy = await isQuicklyResponsive(browser, 2000);
+    if (!browserHealthy) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Browser unresponsive before reload #${i}, skipping remaining reloads`));
+      }
+      console.warn(`Browser unresponsive before reload #${i}, skipping remaining reloads`);
+      break;
+    }
+  } catch (healthErr) {
+    console.warn(`Browser health check failed before reload #${i}: ${healthErr.message}`);
+    break;
+  }
+        // Check if page is still valid before attempting reload
+        let pageStillValid = false;
         try {
-          await page.evaluate(() => {
-            // Force cleanup of any pending frame operations
-            if (window.requestIdleCallback) window.requestIdleCallback(() => {});
-          });
-        } catch (cleanupErr) { /* ignore */ }
+    // Add timeout to page validity check
+    await Promise.race([
+      page.evaluate(() => true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Page validity check timeout')), 3000)
+      )
+    ]);
+          pageStillValid = true;
+        } catch (validityCheck) {
+          console.warn(`Page invalid before reload #${i}, skipping remaining reloads`);
+          break;
+        }
+        
+        // Use comprehensive cleanup from browserhealth module
+        await cleanupPageBeforeReload(page, forceDebug);
+        
+        // Add stabilization delay after cleanup
+        await fastTimeout(1000);
 
         if (siteConfig.clear_sitedata === true) {
           try {
@@ -3116,32 +3222,36 @@ function setupFrameHandling(page, forceDebug) {
         }
         
       let reloadSuccess = false;
+
+  // Skip force reload if browser seems unhealthy
+  const skipForceReload = i > 2; // After 2 attempts, skip force reload
       
-      if (useForceReload && !reloadSuccess) {
+      if (useForceReload && !reloadSuccess && !skipForceReload) {
         // Attempt force reload: disable cache, reload, re-enable cache
           try {
-          // Add timeout protection for setCacheEnabled operations
+          // Timeout-protected cache disable
           await Promise.race([
             page.setCacheEnabled(false),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('setCacheEnabled(false) timeout')), 5000)
-            )
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache disable timeout')), 8000))
           ]);
           
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
+            // Use networkidle2 for force reload to better detect when page is actually loaded
+            await page.reload({ waitUntil: 'networkidle2', timeout: Math.min(timeout, 15000) });
           
+          // Timeout-protected cache enable
           await Promise.race([
             page.setCacheEnabled(true),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('setCacheEnabled(true) timeout')), 5000)
-            )
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache enable timeout')), 8000))
           ]);
-          
+         
           reloadSuccess = true;
             if (forceDebug) console.log(formatLogMessage('debug', `Force reload #${i} completed for ${currentUrl}`));
 
           } catch (forceReloadErr) {
-          console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            // Don't warn for timeouts on problematic sites, just fall back silently
+            if (forceDebug || !forceReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+            }
           reloadSuccess = false; // Ensure we try standard reload
         }
       }
@@ -3149,24 +3259,39 @@ function setupFrameHandling(page, forceDebug) {
       // Fallback to standard reload if force reload failed or wasn't attempted
       if (!reloadSuccess) {
         try {
-          // Reduced timeout for faster failure detection
-          const standardReloadTimeout = Math.min(timeout, 8000); // Reduced from 15000ms
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: standardReloadTimeout });
+            const canReload = await page.evaluate(() => {
+              return !!(document && document.body);
+            }).catch(() => false);
+            
+            if (!canReload) {
+              throw new Error('Page document invalid for reload');
+            }
+
+            // Use networkidle2 with reasonable timeout
+      // Use simpler reload for problematic pages
+      const reloadOptions = i > 1 
+        ? { waitUntil: 'domcontentloaded', timeout: 10000 }  // Simpler after failures
+        : { waitUntil: 'networkidle2', timeout: 15000 };     // Full wait first time
+      
+      await page.reload(reloadOptions);
+            
           if (forceDebug) console.log(formatLogMessage('debug', `Standard reload #${i} completed for ${currentUrl}`));
         } catch (standardReloadErr) {
-          console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            // Only warn for non-timeout errors
+            if (!standardReloadErr.message.includes('timeout')) {
+              console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+            } else if (forceDebug) {
+              console.log(formatLogMessage('debug', `Reload #${i} timed out for ${currentUrl}, continuing anyway`));
+            }
           
           // Check if this is a persistent failure that should skip remaining reloads
-          const isPersistentFailure = standardReloadErr.message.includes('Navigation timeout') || 
+            const isPersistentFailure = standardReloadErr.message.includes('detached Frame') ||
+                                     standardReloadErr.message.includes('Attempted to use detached') ||
+                                     standardReloadErr.message.includes('Navigating frame was detached') ||
+                                     standardReloadErr.message.includes('document invalid') ||
                                      standardReloadErr.message.includes('net::ERR_') ||
                                      standardReloadErr.message.includes('Protocol error') ||
-                                     standardReloadErr.message.includes('Page crashed') ||
-                                     // CDP and injection failures
-                                     standardReloadErr.constructor.name === 'ProtocolError' ||
-                                     standardReloadErr.name === 'ProtocolError' ||
-                                     standardReloadErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
-                                     standardReloadErr.message.includes('Runtime.callFunctionOn timed out') ||
-                                     standardReloadErr.message.includes('CDP injection timeout');
+                                       standardReloadErr.message.includes('Page crashed');
           
           if (isPersistentFailure) {
             const remainingReloads = totalReloads - i;
@@ -3176,16 +3301,16 @@ function setupFrameHandling(page, forceDebug) {
             // Break out of reload loop to move to next URL faster
             break;
           }
+            // For navigation timeouts, we can continue - the page might still be partially loaded
+            // Don't break the loop for simple timeouts
           }
-        } else {
-          // Regular reload
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
         }
 
       // Only add delay if we're continuing with more reloads
       if (i < totalReloads) {
-
-        await fastTimeout(delayMs);
+    // Reduce delay for problematic sites
+    const adjustedDelay = i > 1 ? Math.min(delayMs, 2000) : delayMs;
+    await fastTimeout(adjustedDelay);
       }
     }
     
@@ -3349,7 +3474,7 @@ function setupFrameHandling(page, forceDebug) {
     urlsToProcess.forEach(url => {
       allTasks.push({
         url,
-        config: site,
+        config: { ...site, _originalUrl: url }, // Preserve original URL for CDP domain checking
         taskId: allTasks.length // For tracking
       });
     });
@@ -3867,4 +3992,4 @@ function setupFrameHandling(page, forceDebug) {
   if (forceDebug) console.log(formatLogMessage('debug', `About to exit process...`));
   process.exit(0);
   
-})();
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.8",
+  "version": "2.0.10",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {