@fanboynz/network-scanner 2.0.4 → 2.0.6

package/lib/browserhealth.js

@@ -9,7 +9,7 @@ const { formatLogMessage, messageColors } = require('./colorize');
 // Window cleanup delay constant
 const WINDOW_CLEANUP_DELAY_MS = 15000;
 // window_clean REALTIME
-const REALTIME_CLEANUP_BUFFER_MS = 15000; // Additional buffer time after site delay. Delay increased to fix missing hits.
+const REALTIME_CLEANUP_BUFFER_MS = 35000; // Additional buffer time after site delay (increased for Cloudflare)
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default number of pages to keep
 const REALTIME_CLEANUP_MIN_PAGES = 3; // Minimum pages before cleanup kicks in
 
@@ -270,14 +270,14 @@ function updatePageUsage(page, isProcessing = false) {
 
 /**
  * Performs realtime window cleanup - removes oldest pages when threshold is exceeded
- * Waits for site delay + 3 seconds before cleanup to ensure delayed requests are captured
+ * Waits for site delay + buffer before cleanup, with extended buffer for Cloudflare sites
  * @param {import('puppeteer').Browser} browserInstance - Browser instance
  * @param {number} threshold - Maximum number of pages to keep (default: 8)
  * @param {boolean} forceDebug - Debug logging flag
- * @param {number} siteDelay - Current site's delay value in milliseconds (default: 4000)
+ * @param {number} totalDelay - Total delay including site delay and appropriate buffer (default: 4000 + 15000)
  * @returns {Promise<Object>} Cleanup results
  */
-async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIME_CLEANUP_THRESHOLD, forceDebug, siteDelay = 4000) {
+async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIME_CLEANUP_THRESHOLD, forceDebug, totalDelay = 19000) {
   try {
     const allPages = await browserInstance.pages();
     
@@ -289,11 +289,11 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
       return { success: true, closedCount: 0, totalPages: allPages.length, reason: 'below_threshold' };
     }
 
-    // Calculate cleanup delay: site delay + 3 second buffer
-    const cleanupDelay = siteDelay + REALTIME_CLEANUP_BUFFER_MS;
+    // Use the provided total delay (already includes appropriate buffer)
+    const cleanupDelay = totalDelay;
     
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[realtime_cleanup] Waiting ${cleanupDelay}ms (site delay: ${siteDelay}ms + ${REALTIME_CLEANUP_BUFFER_MS}ms buffer) before cleanup (threshold: ${threshold})`));
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Waiting ${cleanupDelay}ms before cleanup (threshold: ${threshold})`));
     }
     await new Promise(resolve => setTimeout(resolve, cleanupDelay));
     
@@ -999,4 +999,4 @@ if (originalPageClose) {
     }
     return originalPageClose.apply(this, args);
   };
-}
+}

package/lib/cdp.js

@@ -49,12 +49,37 @@ async function createPageWithTimeout(browser, timeout = 30000) {
  * @returns {Promise<void>}
  */
 async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
-  return Promise.race([
-    page.setRequestInterception(true),
-    new Promise((_, reject) => 
-      setTimeout(() => reject(new Error('Request interception setup timeout')), timeout)
-    )
-  ]);
+  try {
+    await Promise.race([
+      page.setRequestInterception(true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Request interception timeout - first attempt')), timeout)
+      )
+    ]);
+  } catch (firstError) {
+    // Check for immediate critical failures
+    if (firstError.message.includes('Target closed') || 
+        firstError.message.includes('Session closed') ||
+        firstError.message.includes('Browser has been closed')) {
+      throw new Error('CRITICAL_BROWSER_ERROR: ' + firstError.message);
+    }
+    
+    // Retry with extended timeout
+    try {
+      await Promise.race([
+        page.setRequestInterception(true),
+        new Promise((_, reject) => 
+          setTimeout(() => reject(new Error('Request interception timeout - retry failed')), timeout * 2)
+        )
+      ]);
+    } catch (retryError) {
+      if (retryError.message.includes('Network.enable timed out') || 
+          retryError.message.includes('ProtocolError')) {
+        throw new Error('CRITICAL_NETWORK_ERROR: ' + retryError.message);
+      }
+      throw retryError;
+    }
+  }
 }
 
 /**

package/lib/cloudflare.js

@@ -1,5 +1,6 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.2 - Further detached Frame fixes 
  * Version: 2.6.1 - timeoutId is not defined & race condition fix
  * Version: 2.6.0 - Memory leak fixes and timeout cleanup
  * Version: 2.5.0 - Fix Frame Lifecycle issue, Timing and Race condition
@@ -26,11 +27,11 @@ const CLOUDFLARE_MODULE_VERSION = '2.6.1';
  * All values tuned for maximum scanning speed while maintaining functionality
  */
 const TIMEOUTS = {
-  PAGE_EVALUATION: 8000,           // Standard page evaluation timeout
-  PAGE_EVALUATION_SAFE: 10000,     // Safe page evaluation with extra buffer
+  PAGE_EVALUATION: 12000,           // Standard page evaluation timeout
+  PAGE_EVALUATION_SAFE: 12000,     // Safe page evaluation with extra buffer
   PHISHING_CLICK: 3000,           // Timeout for clicking phishing continue button
   PHISHING_NAVIGATION: 8000,       // Wait for navigation after phishing bypass
-  JS_CHALLENGE_BUFFER: 30000,     // JS challenge with safety buffer
+  JS_CHALLENGE_BUFFER: 26000,     // JS challenge with safety buffer
   TURNSTILE_COMPLETION: 20000,    // Turnstile completion check
   TURNSTILE_COMPLETION_BUFFER: 25000, // Turnstile completion with buffer
   CLICK_TIMEOUT: 5000,            // Standard click operation timeout
@@ -50,13 +51,13 @@ const TIMEOUTS = {
 
 // Fast timeout constants - optimized for speed
 const FAST_TIMEOUTS = {
-  QUICK_DETECTION: 2000,           // Fast Cloudflare detection
+  QUICK_DETECTION: 4000,           // Fast Cloudflare detection
   PHISHING_WAIT: 1000,            // Fast phishing check
   CHALLENGE_WAIT: 500,            // Fast challenge detection
   ELEMENT_INTERACTION_DELAY: 250, // Fast element interactions
-  SELECTOR_WAIT: 1500,            // Fast selector waits
+  SELECTOR_WAIT: 3000,            // Fast selector waits
   TURNSTILE_OPERATION: 6000,      // Fast Turnstile operations
-  JS_CHALLENGE: 22000,            // Fast JS challenge completion
+  JS_CHALLENGE: 19000,            // Fast JS challenge completion
   CHALLENGE_SOLVING: 30000,       // Fast overall challenge solving
   CHALLENGE_COMPLETION: 8000      // Fast completion check
 };
@@ -323,19 +324,64 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
     let timeoutId = null;
     
     try {
-      // Validate page state before evaluation
+      // Multi-layered page state validation
       if (page.isClosed()) {
-        throw new Error('Page is closed');
+        throw new Error('Page is closed or invalid');
       }
       
+      // Check if page is still navigating or has valid context
+      let currentUrl;
+      try {
+        currentUrl = await page.url();
+        if (!currentUrl || currentUrl === 'about:blank') {
+          throw new Error('Page URL is invalid or blank');
+        }
+      } catch (urlError) {
+        throw new Error('Page URL access failed - likely detached');
+      }
+
+      // Quick execution context validation with timeout
+      const contextValid = await Promise.race([
+        page.evaluate(() => {
+          try {
+            // Quick context validation
+            if (typeof window === 'undefined' || !document) {
+              return false;
+            }
+            // Check if document is ready for interaction
+            if (document.readyState === 'uninitialized') {
+              return false;
+            }
+            return true;
+          } catch (e) {
+            return false;
+          }
+        }),
+        new Promise((_, reject) => {
+          setTimeout(() => reject(new Error('Context validation timeout')), 3500);
+        })
+      ]).catch(() => false);
+
+      if (!contextValid) {
+        throw new Error('Page execution context is invalid');
+      }
 
       const result = await Promise.race([
         page.evaluate(() => {
-          // Validate execution context inside evaluation to prevent race
-          if (typeof window === 'undefined' || !document) {
-            throw new Error('Execution context invalid');
+          // Additional runtime validation inside evaluation
+          try {
+            if (typeof window === 'undefined' || !document) {
+              throw new Error('Execution context invalid during evaluation');
+            }
+            return func();
+          } catch (evalError) {
+            // Return error info instead of throwing to avoid unhandled promise rejections
+            return {
+              __evaluation_error: true,
+              message: evalError.message,
+              type: 'evaluation_error'
+            };
           }
-          return (func)();
         }),
         new Promise((_, reject) => {
           timeoutId = setTimeout(() => reject(new Error('Page evaluation timeout')), timeout);
@@ -346,7 +392,12 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
       if (timeoutId) {
         clearTimeout(timeoutId);
       }
-      
+
+      // Check if evaluation returned an error
+      if (result && result.__evaluation_error) {
+        throw new Error(`Evaluation failed: ${result.message}`);
+      }
+
       if (forceDebug && attempt > 1) {
         console.log(formatLogMessage('cloudflare', `Page evaluation succeeded on attempt ${attempt}`));
       }
@@ -370,23 +421,19 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
         if (forceDebug) {
           console.warn(formatLogMessage('cloudflare', `Detached frame detected on attempt ${attempt}/${maxRetries} - using longer delay`));
         }
-        // For detached frames, wait longer and don't retry as aggressively
-        await new Promise(resolve => setTimeout(resolve, 2000)); // Longer delay
+        // For detached frames, wait longer
+        await new Promise(resolve => setTimeout(resolve, 3000)); // Longer delay
+        
+        // For detached frames, only retry once more
+        if (attempt >= 2) {
+          break;
+        }
         continue;
       }
       
       // Don't retry if error type is not retryable or if it's the last attempt
       if (!RETRY_CONFIG.retryableErrors.includes(errorType) || attempt === maxRetries) {
-        return {
-          isChallengePresent: false,
-          isPhishingWarning: false,
-          isTurnstile: false,
-          isJSChallenge: false,
-          isChallengeCompleted: false,
-          error: error.message,
-          errorType: errorType,
-          attempts: attempt
-        };
+        break;
       }
       
       // Wait before retrying with exponential backoff
@@ -394,19 +441,7 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
     }
   }
   
-  // If all retries failed due to detached frames, return safe defaults
-  if (lastError?.message.includes('detached Frame') || lastError?.message.includes('Attempted to use detached')) {
-    return {
-      isChallengePresent: false,
-      isPhishingWarning: false,
-      isTurnstile: false,
-      isJSChallenge: false,
-      isChallengeCompleted: false,
-      error: 'Frame detached - skipping evaluation',
-      errorType: ERROR_TYPES.DETACHED_FRAME,
-      attempts: maxRetries
-    };
-  }
+  // Return safe defaults if all retries failed
 
   return {
     isChallengePresent: false,
@@ -1845,4 +1880,4 @@ module.exports = {
   detectChallengeLoop,
   // Memory management
   cleanup
-};
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.4 ===
+// === Network scanner script (nwss.js) v2.0.6 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -127,7 +127,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.4'; // Script version
+const VERSION = '2.0.6'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1541,10 +1541,17 @@ function setupFrameHandling(page, forceDebug) {
           ? siteConfig.window_cleanup_threshold 
           : REALTIME_CLEANUP_THRESHOLD;
         
-        // Get the site's delay value for cleanup timing
+        // Calculate appropriate delay based on site configuration
         const siteDelay = siteConfig.delay || 4000;
+        const hasCloudflareConfig = siteConfig.cloudflare_bypass || siteConfig.cloudflare_phish;
+        const bufferTime = hasCloudflareConfig ? 23000 : REALTIME_CLEANUP_BUFFER_MS; // 23s for Cloudflare, 15s for normal
+        const totalDelay = siteDelay + bufferTime;
         
-        const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, siteDelay);
+        if (forceDebug && hasCloudflareConfig) {
+          console.log(formatLogMessage('debug', `[realtime_cleanup] Using extended delay for Cloudflare site: ${totalDelay}ms (${siteDelay}ms + ${bufferTime}ms CF buffer)`));
+        }
+        
+        const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, totalDelay);
         if (realtimeResult.success && realtimeResult.closedCount > 0 && forceDebug) {
           console.log(formatLogMessage('debug', `[realtime_cleanup] Cleaned ${realtimeResult.closedCount} old pages, ${realtimeResult.remainingPages} remaining`));
         }
@@ -1896,7 +1903,10 @@ function setupFrameHandling(page, forceDebug) {
           console.log(formatLogMessage('debug', `Request interception enabled successfully for ${currentUrl}`));
         }
       } catch (networkErr) {
-        if (networkErr.message.includes('timed out') || 
+        if (networkErr.message.includes('CRITICAL_NETWORK_ERROR') || 
+            networkErr.message.includes('CRITICAL_BROWSER_ERROR') ||
+            networkErr.message.includes('ProtocolError') ||
+            networkErr.message.includes('timed out') || 
             networkErr.message.includes('Network.enable') || 
             networkErr.message.includes('timeout')) {
           console.warn(formatLogMessage('warn', `Request interception setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.4",
+  "version": "2.0.6",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {