@fanboynz/network-scanner 2.0.1 → 2.0.3

package/lib/browserhealth.js

@@ -211,6 +211,14 @@ async function isPageSafeToClose(page, forceDebug) {
       return true; // Already closed
     }
 
+  // EXTRA SAFETY: Never close pages that might be in injection process
+  try {
+    const url = page.url();
+    if (url && url !== 'about:blank' && Date.now() - (pageCreationTracker.get(page) || 0) < 30000) {
+      return false; // Don't close recently created pages (within 30 seconds)
+    }
+  } catch (err) { /* ignore */ }
+
     const usage = pageUsageTracker.get(page);
     if (!usage) {
       // No usage data - assume safe if page exists for a while
@@ -991,4 +999,4 @@ if (originalPageClose) {
     }
     return originalPageClose.apply(this, args);
   };
-}
+}

package/lib/cloudflare.js

@@ -1,5 +1,7 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
+ * Version: 2.6.0 - Memory leak fixes and timeout cleanup
+ * Version: 2.5.0 - Fix Frame Lifecycle issue, Timing and Race condition
  * Version: 2.4.1 - Bump timeout values
  * Version: 2.4.0 - Fix possible endless loops with retry logic and loop detection
  * Version: 2.3.1 - Colorize CF
@@ -15,7 +17,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.4.0';
+const CLOUDFLARE_MODULE_VERSION = '2.6.0';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -67,6 +69,7 @@ const ERROR_TYPES = {
   ELEMENT_NOT_FOUND: 'element_not_found',
   EVALUATION_FAILED: 'evaluation_failed',
   NAVIGATION_FAILED: 'navigation_failed',
+  DETACHED_FRAME: 'detached_frame',
   UNKNOWN: 'unknown'
 };
 
@@ -118,7 +121,7 @@ const RETRY_CONFIG = {
   baseDelay: 1000,
   maxDelay: 8000,
   backoffMultiplier: 2,
-  retryableErrors: [ERROR_TYPES.NETWORK, ERROR_TYPES.TIMEOUT, ERROR_TYPES.ELEMENT_NOT_FOUND]
+  retryableErrors: [ERROR_TYPES.NETWORK, ERROR_TYPES.TIMEOUT, ERROR_TYPES.ELEMENT_NOT_FOUND, ERROR_TYPES.DETACHED_FRAME]
 };
 
 /**
@@ -131,6 +134,8 @@ class CloudflareDetectionCache {
     this.ttl = ttl;
     this.hits = 0;
     this.misses = 0;
+    // Prevent memory buildup in long-running processes
+    this.cleanupInterval = setInterval(() => this.cleanupExpired(), ttl / 10);
   }
 
   getCacheKey(url) {
@@ -173,6 +178,20 @@ class CloudflareDetectionCache {
     }
   }
 
+  cleanupExpired() {
+    const now = Date.now();
+    for (const [key, value] of this.cache.entries()) {
+      if (now - value.timestamp >= this.ttl) {
+        this.cache.delete(key);
+      }
+    }
+  }
+
+  destroy() {
+    if (this.cleanupInterval) clearInterval(this.cleanupInterval);
+    this.clear();
+  }
+
   clear() {
     this.cache.clear();
     this.hits = 0;
@@ -260,6 +279,10 @@ async function waitForTimeout(page, timeout) {
  */
 function categorizeError(error) {
   const errorMessage = error.message || '';
+
+  if (errorMessage.includes('detached Frame') || errorMessage.includes('Attempted to use detached')) {
+    return ERROR_TYPES.DETACHED_FRAME;
+  }
   
   if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
     return ERROR_TYPES.TIMEOUT;
@@ -297,12 +320,30 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
   
   for (let attempt = 1; attempt <= maxRetries; attempt++) {
     try {
+      // Validate page/frame state before evaluation
+      if (page.isClosed()) {
+        throw new Error('Page is closed');
+      }
+      
+      // Check if main frame is still attached
+      const mainFrame = page.mainFrame();
+      if (mainFrame.isDetached()) {
+        throw new Error('Main frame is detached');
+      }
+
+      let timeoutId = null;
       const result = await Promise.race([
         page.evaluate(func),
-        new Promise((_, reject) => 
-          setTimeout(() => reject(new Error('Page evaluation timeout')), timeout)
-        )
+        new Promise((_, reject) => {
+          timeoutId = setTimeout(() => reject(new Error('Page evaluation timeout')), timeout);
+        })
       ]);
+
+      // Clear timeout if evaluation completed first 
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+        timeoutId = null;
+      }
       
       if (forceDebug && attempt > 1) {
         console.log(formatLogMessage('cloudflare', `Page evaluation succeeded on attempt ${attempt}`));
@@ -310,12 +351,28 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
       
       return result;
     } catch (error) {
+      // Ensure timeout is cleared on any error
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+        timeoutId = null;
+      }
+
       lastError = error;
       const errorType = categorizeError(error);
       
       if (forceDebug) {
         console.warn(formatLogMessage('cloudflare', `Page evaluation failed (attempt ${attempt}/${maxRetries}): ${error.message} [${errorType}]`));
       }
+
+      // Handle detached frame errors specifically
+      if (errorType === ERROR_TYPES.DETACHED_FRAME) {
+        if (forceDebug) {
+          console.warn(formatLogMessage('cloudflare', `Detached frame detected on attempt ${attempt}/${maxRetries} - using longer delay`));
+        }
+        // For detached frames, wait longer and don't retry as aggressively
+        await new Promise(resolve => setTimeout(resolve, 2000)); // Longer delay
+        continue;
+      }
       
       // Don't retry if error type is not retryable or if it's the last attempt
       if (!RETRY_CONFIG.retryableErrors.includes(errorType) || attempt === maxRetries) {
@@ -336,6 +393,20 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
     }
   }
   
+  // If all retries failed due to detached frames, return safe defaults
+  if (lastError?.message.includes('detached Frame') || lastError?.message.includes('Attempted to use detached')) {
+    return {
+      isChallengePresent: false,
+      isPhishingWarning: false,
+      isTurnstile: false,
+      isJSChallenge: false,
+      isChallengeCompleted: false,
+      error: 'Frame detached - skipping evaluation',
+      errorType: ERROR_TYPES.DETACHED_FRAME,
+      attempts: maxRetries
+    };
+  }
+
   return {
     isChallengePresent: false,
     isPhishingWarning: false,
@@ -355,9 +426,10 @@ async function safeClick(page, selector, timeout = TIMEOUTS.CLICK_TIMEOUT) {
   try {
     return await Promise.race([
       page.click(selector, { timeout: timeout }),
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER)
-      )
+      new Promise((_, reject) => {
+        const timerId = setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER);
+        // Timer will be cleared when promise resolves/rejects
+      })
     ]);
   } catch (error) {
     throw new Error(`Click failed: ${error.message}`);
@@ -371,9 +443,10 @@ async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT
   try {
     return await Promise.race([
       page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER)
-      )
+      new Promise((_, reject) => {
+        const timerId = setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER);
+        // Timer will be cleared when promise resolves/rejects
+      })
     ]);
   } catch (error) {
     console.warn(formatLogMessage('cloudflare', `Navigation wait failed: ${error.message}`));
@@ -890,18 +963,30 @@ async function attemptChallengeSolveWithTimeout(page, currentUrl, challengeInfo,
   const result = {
     success: false,
     error: null,
-    method: null
+    method: null,
+    _timeoutId: null
   };
 
   try {
+    const timeoutPromise = new Promise((_, reject) => {
+      result._timeoutId = setTimeout(() => reject(new Error('Challenge solving timeout')), FAST_TIMEOUTS.CHALLENGE_SOLVING);
+    });
     // Reduced timeout for challenge solving
-    return await Promise.race([
+    const finalResult = await Promise.race([
       attemptChallengeSolve(page, currentUrl, challengeInfo, forceDebug),
-      new Promise((_, reject) =>
-        setTimeout(() => reject(new Error('Challenge solving timeout')), FAST_TIMEOUTS.CHALLENGE_SOLVING)
-      )
+      timeoutPromise
     ]);
+    // Clear timeout if operation completed first
+    if (result._timeoutId) {
+      clearTimeout(result._timeoutId);
+    }
+    return finalResult;
+
   } catch (error) {
+    // Clear timeout on error
+    if (result._timeoutId) {
+      clearTimeout(result._timeoutId);
+    }
     result.error = `Challenge solving timed out: ${error.message}`;
     if (forceDebug) console.log(formatLogMessage('cloudflare', `Challenge solving timeout for ${currentUrl}`));
     return result;
@@ -1120,11 +1205,16 @@ async function handleEmbeddedIframeChallenge(page, forceDebug = false) {
 async function waitForJSChallengeCompletion(page, forceDebug = false) {
   const result = {
     success: false,
-    error: null
+    error: null,
+    _timeoutId: null
   };
 
   try {
     if (forceDebug) console.log(formatLogMessage('cloudflare', `Waiting for JS challenge completion`));
+
+    const timeoutPromise = new Promise((_, reject) => {
+      result._timeoutId = setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER);
+    });
     
     // Reduced timeout for JS challenge completion
     await Promise.race([
@@ -1137,14 +1227,22 @@ async function waitForJSChallengeCompletion(page, forceDebug = false) {
         },
         { timeout: FAST_TIMEOUTS.JS_CHALLENGE }
       ),
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER)
-      )
+      timeoutPromise
     ]);
+
+    // Clear timeout if completion detected first
+    if (result._timeoutId) {
+      clearTimeout(result._timeoutId);
+    }
     
     result.success = true;
     if (forceDebug) console.log(formatLogMessage('cloudflare', `JS challenge completed automatically`));
   } catch (error) {
+    // Clear timeout on error
+    if (result._timeoutId) {
+      clearTimeout(result._timeoutId);
+    }
+
     result.error = `JS challenge timeout: ${error.message}`;
     if (forceDebug) console.log(formatLogMessage('cloudflare', `JS challenge wait failed: ${error.message}`));
   }
@@ -1209,7 +1307,6 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
       
       const checkboxSelectors = [
         'input[type="checkbox"].ctp-checkbox',
-        'input[type="checkbox"]',
         '.ctp-checkbox-label',
         '.ctp-checkbox'
       ];
@@ -1712,6 +1809,15 @@ function clearDetectionCache() {
   detectionCache.clear();
 }
 
+/**
+ * Cleanup function to prevent memory leaks in long-running processes
+ */
+function cleanup() {
+  if (detectionCache) {
+    detectionCache.destroy();
+  }
+}
+
 module.exports = {
   analyzeCloudflareChallenge,
   handlePhishingWarning,
@@ -1735,5 +1841,7 @@ module.exports = {
   ERROR_TYPES,
   RETRY_CONFIG,
   getRetryConfig,
-  detectChallengeLoop
+  detectChallengeLoop,
+  // Memory management
+  cleanup
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.0 ===
+// === Network scanner script (nwss.js) v2.0.3 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -127,7 +127,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.0'; // Script version
+const VERSION = '2.0.3'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1059,17 +1059,48 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 }
 
 function setupFrameHandling(page, forceDebug) {
+  // Track active frames and clear on navigation to prevent detached frame access
+  let activeFrames = new Set();
+  
+  // Clear frame tracking on navigation to prevent stale references
+  page.on('framenavigated', (frame) => {
+    if (frame === page.mainFrame()) {
+      // Main frame navigated - clear all tracked frames
+      activeFrames.clear();
+    }
+  });
+
   // Handle frame creation with error suppression
   page.on('frameattached', async (frame) => {
-    // Enhanced frame handling for Puppeteer 23.x
+    // Enhanced frame handling with detached frame protection
     try {
-      // Check if frame is valid before processing
-      if (frame.isDetached()) {
+      // Multiple checks for frame validity to prevent detached frame errors
+      if (!frame) {
         if (forceDebug) {
-          console.log(formatLogMessage('debug', `Skipping detached frame`));
+          console.log(formatLogMessage('debug', `Skipping null frame`));
         }
         return;
       }
+      
+      // Enhanced frame validation with multiple safety checks
+      let frameUrl;
+      try {
+        // Test frame accessibility first
+        frameUrl = frame.url();
+        
+        // Check if frame is detached (if method exists)
+        if (frame.isDetached && frame.isDetached()) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Skipping detached frame`));
+          }
+          return;
+        }
+      } catch (frameAccessError) {
+        // Frame is not accessible (likely detached)
+        return;
+      }
+      
+      activeFrames.add(frame);
     } catch (detachError) {
       // Frame state checking can throw in 23.x, handle gracefully
       if (forceDebug) {
@@ -1079,9 +1110,7 @@ function setupFrameHandling(page, forceDebug) {
     }
 
     if (frame.parentFrame()) { // Only handle child frames, not main frame
-      try {
-        const frameUrl = frame.url();
-        
+      try {       
         if (forceDebug) {
           console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
         }
@@ -1139,7 +1168,17 @@ function setupFrameHandling(page, forceDebug) {
   });
   // Handle frame navigations (keep this for monitoring)
   page.on('framenavigated', (frame) => {
-    const frameUrl = frame.url();
+    let frameUrl;
+
+    // Skip if frame is not in our active set
+    if (!activeFrames.has(frame)) return;
+
+    try {
+      frameUrl = frame.url();
+    } catch (urlErr) {
+      // Frame likely detached during navigation
+      return;
+    }
     if (forceDebug &&
         frameUrl &&
         frameUrl !== 'about:blank' &&
@@ -1154,8 +1193,18 @@ function setupFrameHandling(page, forceDebug) {
 
   // Optional: Handle frame detachment for cleanup
   page.on('framedetached', (frame) => {
+    // Remove from active tracking
+    activeFrames.delete(frame);
+    
+    // Skip logging if we can't access frame URL
+    let frameUrl;
     if (forceDebug) {
-      const frameUrl = frame.url();
+      try {
+        frameUrl = frame.url();
+      } catch (urlErr) {
+        // Frame already detached, can't get URL
+        return;
+      }
       if (frameUrl &&
           frameUrl !== 'about:blank' &&
           frameUrl !== 'about:srcdoc' &&
@@ -1583,6 +1632,11 @@ function setupFrameHandling(page, forceDebug) {
       const shouldInjectEvalForPage = siteConfig.evaluateOnNewDocument === true || globalEvalOnDoc;
       let evalOnDocSuccess = false; // Track injection success for fallback logic
       
+      // PREVENT realtime cleanup during injection to avoid "Session closed" errors
+      if (shouldInjectEvalForPage && siteConfig.window_cleanup === "realtime") {
+          updatePageUsage(page, true); // Mark page as actively processing BEFORE injection
+      }
+
       if (shouldInjectEvalForPage) {
           if (forceDebug) {
               if (globalEvalOnDoc) {
@@ -1595,8 +1649,13 @@ function setupFrameHandling(page, forceDebug) {
           // Strategy 1: Try full injection with health check
           let browserResponsive = false;
           try {
+              // Check if browser is still connected before attempting health check
+              if (!browserInstance.isConnected()) {
+                  throw new Error('Browser not connected');
+              }
+
               await Promise.race([
-                  browserInstance.version(), // Quick responsiveness test
+                  browserInstance.pages(), // Simple existence check that doesn't require active session
                   new Promise((_, reject) => 
                       setTimeout(() => reject(new Error('Browser health check timeout')), 3000)
                   )
@@ -1616,6 +1675,13 @@ function setupFrameHandling(page, forceDebug) {
                   await Promise.race([
                       // Main injection with all safety checks
                       page.evaluateOnNewDocument(() => {
+                          // Prevent duplicate injections
+                          if (window.__nwss_injection_applied) {
+                              console.log('[evalOnDoc] Already injected, skipping');
+                              return;
+                          }
+                          window.__nwss_injection_applied = true;
+
                           // Wrap everything in try-catch to prevent page crashes
                           try {
                               // Add timeout check within the injection
@@ -1719,7 +1785,18 @@ function setupFrameHandling(page, forceDebug) {
                   // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
                   try {
                       await Promise.race([
-                          page.evaluateOnNewDocument(() => {
+                          (async () => {
+                              // Validate page state before minimal injection
+                              if (!page || page.isClosed()) {
+                                  throw new Error('Page is closed');
+                              }
+                              
+                              const pageUrl = await page.url().catch(() => 'about:blank');
+                              if (pageUrl === 'about:blank') {
+                                  throw new Error('Cannot inject on about:blank');
+                              }
+                              
+                              return page.evaluateOnNewDocument(() => {
                               // Minimal injection - just fetch monitoring
                               if (window.fetch) {
                                   const originalFetch = window.fetch;
@@ -1732,7 +1809,8 @@ function setupFrameHandling(page, forceDebug) {
                                       }
                                   };
                               }
-                          }),
+                              });
+                          })(),
                           new Promise((_, reject) => 
                               setTimeout(() => reject(new Error('Minimal injection timeout')), 3000)
                           )
@@ -1760,6 +1838,10 @@ function setupFrameHandling(page, forceDebug) {
           if (!evalOnDocSuccess) {
               console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
           }
+      // Allow realtime cleanup to proceed after injection completes
+      if (shouldInjectEvalForPage && siteConfig.window_cleanup === "realtime") {
+          updatePageUsage(page, false); // Mark page as idle after injection
+      }
       }
       // --- END: evaluateOnNewDocument for Fetch/XHR Interception ---
 
@@ -2161,8 +2243,15 @@ function setupFrameHandling(page, forceDebug) {
         // This prevents redirect destinations from being marked as third-party
         const isFirstParty = checkedRootDomain && firstPartyDomains.has(checkedRootDomain);
         
-        // Block infinite iframe loops
-        const frameUrl = request.frame() ? request.frame().url() : '';
+        // Block infinite iframe loops - safely access frame URL
+        const frameUrl = (() => {
+          try {
+            const frame = request.frame();
+            return frame ? frame.url() : '';
+          } catch (err) {
+            return '';
+          }
+        })();
         if (frameUrl && frameUrl.includes('creative.dmzjmp.com') && 
             request.url().includes('go.dmzjmp.com/api/models')) {
           if (forceDebug) {
@@ -2174,8 +2263,18 @@ function setupFrameHandling(page, forceDebug) {
 
         // Enhanced debug logging to show which frame the request came from
         if (forceDebug) {
-          const frameUrl = request.frame() ? request.frame().url() : 'unknown-frame';
-          const isMainFrame = request.frame() === page.mainFrame();
+          let frameUrl = 'unknown-frame';
+          let isMainFrame = false;
+          
+          try {
+            const frame = request.frame();
+            if (frame) {
+              frameUrl = frame.url();
+              isMainFrame = frame === page.mainFrame();
+            }
+          } catch (frameErr) {
+            frameUrl = 'detached-frame';
+          }
           console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[frame: ${isMainFrame ? 'main' : 'iframe'}] ${frameUrl} → ${request.url()}`));
         }
 
@@ -2997,6 +3096,14 @@ function setupFrameHandling(page, forceDebug) {
       }
       
       for (let i = 1; i <= totalReloads; i++) {
+        // Clear any stale frame references before reload
+        try {
+          await page.evaluate(() => {
+            // Force cleanup of any pending frame operations
+            if (window.requestIdleCallback) window.requestIdleCallback(() => {});
+          });
+        } catch (cleanupErr) { /* ignore */ }
+
         if (siteConfig.clear_sitedata === true) {
           try {
             let reloadClearSession = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.1",
+  "version": "2.0.3",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {