@fanboynz/network-scanner 2.0.16 → 2.0.18

package/lib/browserhealth.js

@@ -9,9 +9,9 @@ const { formatLogMessage, messageColors } = require('./colorize');
 // Window cleanup delay constant
 const WINDOW_CLEANUP_DELAY_MS = 15000;
 // window_clean REALTIME
-const REALTIME_CLEANUP_BUFFER_MS = 35000; // Additional buffer time after site delay (increased for Cloudflare)
-const REALTIME_CLEANUP_THRESHOLD = 8; // Default number of pages to keep
-const REALTIME_CLEANUP_MIN_PAGES = 3; // Minimum pages before cleanup kicks in
+const REALTIME_CLEANUP_BUFFER_MS = 25000; // Additional buffer time after site delay (increased for Cloudflare)
+const REALTIME_CLEANUP_THRESHOLD = 12; // Default number of pages to keep
+const REALTIME_CLEANUP_MIN_PAGES = 6; // Minimum pages before cleanup kicks in
 
 // Track page creation order for realtime cleanup
 const pageCreationTracker = new Map(); // Maps page -> creation timestamp
@@ -19,7 +19,7 @@ let pageCreationCounter = 0;
 
 // Track page usage for realtime cleanup safety
 const pageUsageTracker = new Map(); // Maps page -> { lastActivity: timestamp, isProcessing: boolean }
-const PAGE_IDLE_THRESHOLD = 15000; // 15 seconds of inactivity before considering page safe to clean
+const PAGE_IDLE_THRESHOLD = 25000; // 25 seconds of inactivity before considering page safe to clean
 
 /**
  * Performs group-level window cleanup after all URLs in a site group complete
@@ -299,6 +299,29 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
     
     const allPagesAfterDelay = await browserInstance.pages();
     
+    // Also check for and close any popup contexts
+    try {
+      const contexts = await browserInstance.browserContexts();
+      for (const context of contexts) {
+        if (context.isIncognito && context !== browserInstance.defaultBrowserContext()) {
+          const contextPages = await context.pages();
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[realtime_cleanup] Found ${contextPages.length} pages in popup context`));
+          }
+          // Close popup context pages
+          for (const page of contextPages) {
+            if (!page.isClosed()) {
+              await page.close();
+            }
+          }
+        }
+      }
+    } catch (contextErr) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[realtime_cleanup] Context cleanup error: ${contextErr.message}`));
+      }
+    }
+    
     // Find main Puppeteer page (usually about:blank)
     let mainPage = allPagesAfterDelay.find(page => {
       const url = page.url();

package/lib/fingerprint.js

@@ -1333,6 +1333,18 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
  */
 async function simulateHumanBehavior(page, forceDebug) {
   try {
+    // Validate page state before injection
+    if (!page || page.isClosed()) {
+      if (forceDebug) console.log(`[debug] Human behavior simulation skipped - page closed`);
+      return;
+    }
+    
+    // Check if browser is still connected
+    if (!page.browser().isConnected()) {
+      if (forceDebug) console.log(`[debug] Human behavior simulation skipped - browser disconnected`);
+      return;
+    }
+
     await page.evaluateOnNewDocument((debugEnabled) => {
       
       try {

package/lib/nettools.js

@@ -418,15 +418,17 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
     
     // Add delay between retry attempts to prevent rate limiting
     if (attemptCount > 1) {
-      if (debugMode) {
-        if (logFunc) {
-          logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`);
-        } else {
-          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`));
+      if (whoisDelay > 0) {
+        if (debugMode) {
+          if (logFunc) {
+            logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`);
+          } else {
+            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`));
+          }
         }
+        await new Promise(resolve => setTimeout(resolve, whoisDelay));
       }
       
-      await new Promise(resolve => setTimeout(resolve, whoisDelay));
     } else if (whoisDelay > 0) {
       // Add initial delay on first attempt if configured
       if (debugMode) {
@@ -437,6 +439,13 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
         }
       }
       await new Promise(resolve => setTimeout(resolve, whoisDelay));
+    } else if (debugMode) {
+      // Log when delay is skipped due to whoisDelay being 0
+      if (logFunc) {
+        logFunc(`${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`);
+      } else {
+        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`));
+      }
    }
     
     try {

package/lib/redirect.js

@@ -45,6 +45,22 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
   // Monitor JavaScript redirects by intercepting location changes
   const jsRedirectDetector = async () => {
     try {
+      // Validate page state before injection
+      if (!page || page.isClosed()) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', 'JS redirect detector skipped - page closed'));
+        }
+        return;
+      }
+      
+      // Check if browser is still connected
+      if (!page.browser().isConnected()) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', 'JS redirect detector skipped - browser disconnected'));
+        }
+        return;
+      }
+
       await page.evaluateOnNewDocument(() => {
         // Store original location methods
         const originalReplace = window.location.replace;
@@ -121,7 +137,14 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
         }
       });
     } catch (jsErr) {
-      if (forceDebug) {
+      if (jsErr.message.includes('Session closed') || 
+          jsErr.message.includes('Target closed') ||
+          jsErr.message.includes('Protocol error')) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', 'JS redirect detector skipped - session ended'));
+        }
+        return;
+      } else if (forceDebug) {
         console.log(formatLogMessage('debug', `Failed to inject JS redirect detector: ${jsErr.message}`));
       }
     }

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.16 ===
+// === Network scanner script (nwss.js) v2.0.18 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -130,7 +130,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.16'; // Script version
+const VERSION = '2.0.18'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -2555,7 +2555,7 @@ function setupFrameHandling(page, forceDebug) {
               const netToolsHandler = createNetToolsHandler({
                 whoisTerms,
                 whoisOrTerms,
-                whoisDelay: siteConfig.whois_delay || whois_delay,
+                whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
                 whoisServer,
                 whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
                 debugLogFile,
@@ -2662,7 +2662,7 @@ function setupFrameHandling(page, forceDebug) {
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
-               whoisDelay: siteConfig.whois_delay || whois_delay, // Site-specific or global fallback
+               whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay, // Site-specific or global fallback
 	       whoisServer, // Pass whois server configuration
                whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
                debugLogFile, // Pass debug log file for whois error logging

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.16",
+  "version": "2.0.18",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {