@fanboynz/network-scanner 2.0.3 → 2.0.5

package/lib/browserhealth.js

@@ -9,7 +9,7 @@ const { formatLogMessage, messageColors } = require('./colorize');
 // Window cleanup delay constant
 const WINDOW_CLEANUP_DELAY_MS = 15000;
 // window_clean REALTIME
-const REALTIME_CLEANUP_BUFFER_MS = 15000; // Additional buffer time after site delay. Delay increased to fix missing hits.
+const REALTIME_CLEANUP_BUFFER_MS = 35000; // Additional buffer time after site delay (increased for Cloudflare)
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default number of pages to keep
 const REALTIME_CLEANUP_MIN_PAGES = 3; // Minimum pages before cleanup kicks in
 
@@ -270,14 +270,14 @@ function updatePageUsage(page, isProcessing = false) {
 
 /**
  * Performs realtime window cleanup - removes oldest pages when threshold is exceeded
- * Waits for site delay + 3 seconds before cleanup to ensure delayed requests are captured
+ * Waits for site delay + buffer before cleanup, with extended buffer for Cloudflare sites
  * @param {import('puppeteer').Browser} browserInstance - Browser instance
  * @param {number} threshold - Maximum number of pages to keep (default: 8)
  * @param {boolean} forceDebug - Debug logging flag
- * @param {number} siteDelay - Current site's delay value in milliseconds (default: 4000)
+ * @param {number} totalDelay - Total delay including site delay and appropriate buffer (default: 4000 + 15000)
  * @returns {Promise<Object>} Cleanup results
  */
-async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIME_CLEANUP_THRESHOLD, forceDebug, siteDelay = 4000) {
+async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIME_CLEANUP_THRESHOLD, forceDebug, totalDelay = 19000) {
   try {
     const allPages = await browserInstance.pages();
     
@@ -289,11 +289,11 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
       return { success: true, closedCount: 0, totalPages: allPages.length, reason: 'below_threshold' };
     }
 
-    // Calculate cleanup delay: site delay + 3 second buffer
-    const cleanupDelay = siteDelay + REALTIME_CLEANUP_BUFFER_MS;
+    // Use the provided total delay (already includes appropriate buffer)
+    const cleanupDelay = totalDelay;
     
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[realtime_cleanup] Waiting ${cleanupDelay}ms (site delay: ${siteDelay}ms + ${REALTIME_CLEANUP_BUFFER_MS}ms buffer) before cleanup (threshold: ${threshold})`));
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Waiting ${cleanupDelay}ms before cleanup (threshold: ${threshold})`));
     }
     await new Promise(resolve => setTimeout(resolve, cleanupDelay));
     
@@ -999,4 +999,4 @@ if (originalPageClose) {
     }
     return originalPageClose.apply(this, args);
   };
-}
+}

package/lib/cloudflare.js

@@ -1,5 +1,7 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.2 - Further detached Frame fixes 
+ * Version: 2.6.1 - timeoutId is not defined & race condition fix
  * Version: 2.6.0 - Memory leak fixes and timeout cleanup
  * Version: 2.5.0 - Fix Frame Lifecycle issue, Timing and Race condition
  * Version: 2.4.1 - Bump timeout values
@@ -17,7 +19,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.0';
+const CLOUDFLARE_MODULE_VERSION = '2.6.1';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -25,11 +27,11 @@ const CLOUDFLARE_MODULE_VERSION = '2.6.0';
  * All values tuned for maximum scanning speed while maintaining functionality
  */
 const TIMEOUTS = {
-  PAGE_EVALUATION: 8000,           // Standard page evaluation timeout
-  PAGE_EVALUATION_SAFE: 10000,     // Safe page evaluation with extra buffer
+  PAGE_EVALUATION: 12000,           // Standard page evaluation timeout
+  PAGE_EVALUATION_SAFE: 12000,     // Safe page evaluation with extra buffer
   PHISHING_CLICK: 3000,           // Timeout for clicking phishing continue button
   PHISHING_NAVIGATION: 8000,       // Wait for navigation after phishing bypass
-  JS_CHALLENGE_BUFFER: 30000,     // JS challenge with safety buffer
+  JS_CHALLENGE_BUFFER: 26000,     // JS challenge with safety buffer
   TURNSTILE_COMPLETION: 20000,    // Turnstile completion check
   TURNSTILE_COMPLETION_BUFFER: 25000, // Turnstile completion with buffer
   CLICK_TIMEOUT: 5000,            // Standard click operation timeout
@@ -49,13 +51,13 @@ const TIMEOUTS = {
 
 // Fast timeout constants - optimized for speed
 const FAST_TIMEOUTS = {
-  QUICK_DETECTION: 2000,           // Fast Cloudflare detection
+  QUICK_DETECTION: 4000,           // Fast Cloudflare detection
   PHISHING_WAIT: 1000,            // Fast phishing check
   CHALLENGE_WAIT: 500,            // Fast challenge detection
   ELEMENT_INTERACTION_DELAY: 250, // Fast element interactions
-  SELECTOR_WAIT: 1500,            // Fast selector waits
+  SELECTOR_WAIT: 3000,            // Fast selector waits
   TURNSTILE_OPERATION: 6000,      // Fast Turnstile operations
-  JS_CHALLENGE: 22000,            // Fast JS challenge completion
+  JS_CHALLENGE: 19000,            // Fast JS challenge completion
   CHALLENGE_SOLVING: 30000,       // Fast overall challenge solving
   CHALLENGE_COMPLETION: 8000      // Fast completion check
 };
@@ -319,21 +321,68 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
   let lastError = null;
   
   for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    let timeoutId = null;
+    
     try {
-      // Validate page/frame state before evaluation
+      // Multi-layered page state validation
       if (page.isClosed()) {
-        throw new Error('Page is closed');
+        throw new Error('Page is closed or invalid');
       }
       
-      // Check if main frame is still attached
-      const mainFrame = page.mainFrame();
-      if (mainFrame.isDetached()) {
-        throw new Error('Main frame is detached');
+      // Check if page is still navigating or has valid context
+      let currentUrl;
+      try {
+        currentUrl = await page.url();
+        if (!currentUrl || currentUrl === 'about:blank') {
+          throw new Error('Page URL is invalid or blank');
+        }
+      } catch (urlError) {
+        throw new Error('Page URL access failed - likely detached');
+      }
+
+      // Quick execution context validation with timeout
+      const contextValid = await Promise.race([
+        page.evaluate(() => {
+          try {
+            // Quick context validation
+            if (typeof window === 'undefined' || !document) {
+              return false;
+            }
+            // Check if document is ready for interaction
+            if (document.readyState === 'uninitialized') {
+              return false;
+            }
+            return true;
+          } catch (e) {
+            return false;
+          }
+        }),
+        new Promise((_, reject) => {
+          setTimeout(() => reject(new Error('Context validation timeout')), 3500);
+        })
+      ]).catch(() => false);
+
+      if (!contextValid) {
+        throw new Error('Page execution context is invalid');
       }
 
-      let timeoutId = null;
       const result = await Promise.race([
-        page.evaluate(func),
+        page.evaluate(() => {
+          // Additional runtime validation inside evaluation
+          try {
+            if (typeof window === 'undefined' || !document) {
+              throw new Error('Execution context invalid during evaluation');
+            }
+            return func();
+          } catch (evalError) {
+            // Return error info instead of throwing to avoid unhandled promise rejections
+            return {
+              __evaluation_error: true,
+              message: evalError.message,
+              type: 'evaluation_error'
+            };
+          }
+        }),
         new Promise((_, reject) => {
           timeoutId = setTimeout(() => reject(new Error('Page evaluation timeout')), timeout);
         })
@@ -342,9 +391,13 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
       // Clear timeout if evaluation completed first 
       if (timeoutId) {
         clearTimeout(timeoutId);
-        timeoutId = null;
       }
-      
+
+      // Check if evaluation returned an error
+      if (result && result.__evaluation_error) {
+        throw new Error(`Evaluation failed: ${result.message}`);
+      }
+
       if (forceDebug && attempt > 1) {
         console.log(formatLogMessage('cloudflare', `Page evaluation succeeded on attempt ${attempt}`));
       }
@@ -354,7 +407,6 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
       // Ensure timeout is cleared on any error
       if (timeoutId) {
         clearTimeout(timeoutId);
-        timeoutId = null;
       }
 
       lastError = error;
@@ -369,23 +421,19 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
         if (forceDebug) {
           console.warn(formatLogMessage('cloudflare', `Detached frame detected on attempt ${attempt}/${maxRetries} - using longer delay`));
         }
-        // For detached frames, wait longer and don't retry as aggressively
-        await new Promise(resolve => setTimeout(resolve, 2000)); // Longer delay
+        // For detached frames, wait longer
+        await new Promise(resolve => setTimeout(resolve, 3000)); // Longer delay
+        
+        // For detached frames, only retry once more
+        if (attempt >= 2) {
+          break;
+        }
         continue;
       }
       
       // Don't retry if error type is not retryable or if it's the last attempt
       if (!RETRY_CONFIG.retryableErrors.includes(errorType) || attempt === maxRetries) {
-        return {
-          isChallengePresent: false,
-          isPhishingWarning: false,
-          isTurnstile: false,
-          isJSChallenge: false,
-          isChallengeCompleted: false,
-          error: error.message,
-          errorType: errorType,
-          attempts: attempt
-        };
+        break;
       }
       
       // Wait before retrying with exponential backoff
@@ -393,19 +441,7 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
     }
   }
   
-  // If all retries failed due to detached frames, return safe defaults
-  if (lastError?.message.includes('detached Frame') || lastError?.message.includes('Attempted to use detached')) {
-    return {
-      isChallengePresent: false,
-      isPhishingWarning: false,
-      isTurnstile: false,
-      isJSChallenge: false,
-      isChallengeCompleted: false,
-      error: 'Frame detached - skipping evaluation',
-      errorType: ERROR_TYPES.DETACHED_FRAME,
-      attempts: maxRetries
-    };
-  }
+  // Return safe defaults if all retries failed
 
   return {
     isChallengePresent: false,
@@ -427,8 +463,7 @@ async function safeClick(page, selector, timeout = TIMEOUTS.CLICK_TIMEOUT) {
     return await Promise.race([
       page.click(selector, { timeout: timeout }),
       new Promise((_, reject) => {
-        const timerId = setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER);
-        // Timer will be cleared when promise resolves/rejects
+        setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER);
       })
     ]);
   } catch (error) {
@@ -444,8 +479,7 @@ async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT
     return await Promise.race([
       page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
       new Promise((_, reject) => {
-        const timerId = setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER);
-        // Timer will be cleared when promise resolves/rejects
+        setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER);
       })
     ]);
   } catch (error) {
@@ -963,13 +997,14 @@ async function attemptChallengeSolveWithTimeout(page, currentUrl, challengeInfo,
   const result = {
     success: false,
     error: null,
-    method: null,
-    _timeoutId: null
+    method: null
   };
+  
+  let timeoutId = null;
 
   try {
     const timeoutPromise = new Promise((_, reject) => {
-      result._timeoutId = setTimeout(() => reject(new Error('Challenge solving timeout')), FAST_TIMEOUTS.CHALLENGE_SOLVING);
+      timeoutId = setTimeout(() => reject(new Error('Challenge solving timeout')), FAST_TIMEOUTS.CHALLENGE_SOLVING);
     });
     // Reduced timeout for challenge solving
     const finalResult = await Promise.race([
@@ -977,15 +1012,15 @@ async function attemptChallengeSolveWithTimeout(page, currentUrl, challengeInfo,
       timeoutPromise
     ]);
     // Clear timeout if operation completed first
-    if (result._timeoutId) {
-      clearTimeout(result._timeoutId);
+    if (timeoutId) {
+      clearTimeout(timeoutId);
     }
     return finalResult;
 
   } catch (error) {
     // Clear timeout on error
-    if (result._timeoutId) {
-      clearTimeout(result._timeoutId);
+    if (timeoutId) {
+      clearTimeout(timeoutId);
     }
     result.error = `Challenge solving timed out: ${error.message}`;
     if (forceDebug) console.log(formatLogMessage('cloudflare', `Challenge solving timeout for ${currentUrl}`));
@@ -1205,15 +1240,16 @@ async function handleEmbeddedIframeChallenge(page, forceDebug = false) {
 async function waitForJSChallengeCompletion(page, forceDebug = false) {
   const result = {
     success: false,
-    error: null,
-    _timeoutId: null
+    error: null
   };
 
+  let timeoutId = null;
+
   try {
     if (forceDebug) console.log(formatLogMessage('cloudflare', `Waiting for JS challenge completion`));
 
     const timeoutPromise = new Promise((_, reject) => {
-      result._timeoutId = setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER);
+      timeoutId = setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER);
     });
     
     // Reduced timeout for JS challenge completion
@@ -1231,16 +1267,16 @@ async function waitForJSChallengeCompletion(page, forceDebug = false) {
     ]);
 
     // Clear timeout if completion detected first
-    if (result._timeoutId) {
-      clearTimeout(result._timeoutId);
+    if (timeoutId) {
+      clearTimeout(timeoutId);
     }
     
     result.success = true;
     if (forceDebug) console.log(formatLogMessage('cloudflare', `JS challenge completed automatically`));
   } catch (error) {
     // Clear timeout on error
-    if (result._timeoutId) {
-      clearTimeout(result._timeoutId);
+    if (timeoutId) {
+      clearTimeout(timeoutId);
     }
 
     result.error = `JS challenge timeout: ${error.message}`;

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.3 ===
+// === Network scanner script (nwss.js) v2.0.5 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -127,7 +127,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.3'; // Script version
+const VERSION = '2.0.5'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1541,10 +1541,17 @@ function setupFrameHandling(page, forceDebug) {
           ? siteConfig.window_cleanup_threshold 
           : REALTIME_CLEANUP_THRESHOLD;
         
-        // Get the site's delay value for cleanup timing
+        // Calculate appropriate delay based on site configuration
         const siteDelay = siteConfig.delay || 4000;
+        const hasCloudflareConfig = siteConfig.cloudflare_bypass || siteConfig.cloudflare_phish;
+        const bufferTime = hasCloudflareConfig ? 23000 : REALTIME_CLEANUP_BUFFER_MS; // 23s for Cloudflare, 15s for normal
+        const totalDelay = siteDelay + bufferTime;
         
-        const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, siteDelay);
+        if (forceDebug && hasCloudflareConfig) {
+          console.log(formatLogMessage('debug', `[realtime_cleanup] Using extended delay for Cloudflare site: ${totalDelay}ms (${siteDelay}ms + ${bufferTime}ms CF buffer)`));
+        }
+        
+        const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, totalDelay);
         if (realtimeResult.success && realtimeResult.closedCount > 0 && forceDebug) {
           console.log(formatLogMessage('debug', `[realtime_cleanup] Cleaned ${realtimeResult.closedCount} old pages, ${realtimeResult.remainingPages} remaining`));
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {