@fanboynz/network-scanner 2.0.55 → 2.0.57

package/nwss.js

@@ -20,7 +20,8 @@ const {
   handleCloudflareProtection,
   getCacheStats,
   clearDetectionCache,
-  parallelChallengeDetection
+  parallelChallengeDetection,
+  cleanup: cleanupCloudflareCache
 } = require('./lib/cloudflare');
 // FP Bypass
 const { handleFlowProxyProtection, getFlowProxyTimeouts } = require('./lib/flowproxy');
@@ -84,8 +85,8 @@ const TIMEOUTS = Object.freeze({
 });
 
 const CACHE_LIMITS = Object.freeze({
-  DISK_CACHE_SIZE: 52428800, // 50MB
-  MEDIA_CACHE_SIZE: 52428800, // 50MB
+  DISK_CACHE_SIZE: 1, // Effectively disabled — forcereload clears cache between loads
+  MEDIA_CACHE_SIZE: 1, // Effectively disabled — no media caching needed for scanning
   DEFAULT_CACHE_PATH: '.cache',
   DEFAULT_MAX_SIZE: 5000
 });
@@ -150,7 +151,7 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload, purgeStaleTrackers } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
 const VERSION = '2.0.33'; // Script version
@@ -1020,11 +1021,28 @@ function flushLogBuffers() {
   for (const [filePath, entries] of _logBuffers) {
     if (entries.length > 0) {
       try {
-        fs.appendFileSync(filePath, entries.join(''));
+        const data = entries.join('');
+        entries.length = 0; // Clear buffer immediately
+        fs.writeFile(filePath, data, { flag: 'a' }, (err) => {
+          if (err) {
+            console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
+          }
+        });
       } catch (err) {
         console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
       }
-      entries.length = 0; // Clear buffer
+    }
+  }
+}
+
+// Synchronous flush for exit handlers — guarantees data is written before process exits
+function flushLogBuffersSync() {
+  for (const [filePath, entries] of _logBuffers) {
+    if (entries.length > 0) {
+      try {
+        fs.appendFileSync(filePath, entries.join(''));
+      } catch (err) { /* best effort on exit */ }
+      entries.length = 0;
     }
   }
 }
@@ -1426,12 +1444,19 @@ function setupFrameHandling(page, forceDebug) {
         '--use-mock-keychain',
         '--disable-client-side-phishing-detection',
         '--enable-features=NetworkService',
-        // Disk space controls - 50MB cache limits
-        `--disk-cache-size=${CACHE_LIMITS.DISK_CACHE_SIZE}`, // 50MB disk cache
-        `--media-cache-size=${CACHE_LIMITS.MEDIA_CACHE_SIZE}`, // 50MB media cache
+        // Disk space controls - minimal cache for scanning workloads
+        `--disk-cache-size=${CACHE_LIMITS.DISK_CACHE_SIZE}`,
+        `--media-cache-size=${CACHE_LIMITS.MEDIA_CACHE_SIZE}`,
         '--disable-application-cache',
         '--disable-offline-load-stale-cache',
         '--disable-background-downloads',
+        // DISK I/O REDUCTION: Eliminate unnecessary Chrome disk writes
+        '--disable-breakpad',          // No crash dump files
+        '--disable-component-update',  // No component update downloads
+        '--disable-logging',           // No Chrome internal log files
+        '--log-level=3',               // Fatal errors only (suppresses verbose disk logging)
+        '--no-service-autorun',        // No background service disk activity
+        '--disable-domain-reliability', // No reliability monitor disk writes
         // PERFORMANCE: Enhanced Puppeteer 23.x optimizations
         '--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor',
         '--disable-features=TranslateUI,BlinkGenPropertyTrees,Translate',
@@ -1466,6 +1491,12 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-background-timer-throttling',
         '--disable-features=site-per-process', // Better for single-site scanning
         '--no-zygote', // Better process isolation
+        // PERFORMANCE: Process and memory reduction for high concurrency
+        '--renderer-process-limit=10',  // Cap renderer processes (default: unlimited)
+        '--disable-accelerated-2d-canvas', // Software canvas only (we spoof it anyway)
+        '--disable-hang-monitor',      // Remove per-renderer hang check overhead
+        '--disable-features=PaintHolding', // Don't hold frames in renderer memory
+        '--js-flags=--max-old-space-size=512', // Cap V8 heap per renderer to 512MB
         ...extraArgs,
         ],
         // Optimized timeouts for Puppeteer 23.x performance
@@ -1517,7 +1548,7 @@ function setupFrameHandling(page, forceDebug) {
   // Set up cleanup on process termination
   process.on('SIGINT', async () => {
     if (forceDebug) console.log(formatLogMessage('debug', 'SIGINT received, performing cleanup...'));
-    flushLogBuffers();
+    flushLogBuffersSync();
     if (_logFlushTimer) clearInterval(_logFlushTimer);
     await performEmergencyCleanup();
     process.exit(0);
@@ -1525,7 +1556,7 @@ function setupFrameHandling(page, forceDebug) {
 
   process.on('SIGTERM', async () => {
     if (forceDebug) console.log(formatLogMessage('debug', 'SIGTERM received, performing cleanup...'));
-    flushLogBuffers();
+    flushLogBuffersSync();
     if (_logFlushTimer) clearInterval(_logFlushTimer);
     await performEmergencyCleanup();
     process.exit(0);
@@ -1556,6 +1587,8 @@ function setupFrameHandling(page, forceDebug) {
     }
     wgDisconnectAll(forceDebug);
     ovpnDisconnectAll(forceDebug);
+    cleanupCloudflareCache();
+    purgeStaleTrackers();
   }
  
   let siteCounter = 0;
@@ -3219,7 +3252,9 @@ function setupFrameHandling(page, forceDebug) {
         siteCounter++;
 
         // Enhanced Cloudflare handling with parallel detection
-        if (siteConfig.cloudflare_parallel_detection !== false) { // Enable by default
+        // Only run parallel detection if cloudflare handling is explicitly configured
+        const hasCloudflareConfig = siteConfig.cloudflare_bypass || siteConfig.cloudflare_phish;
+        if (hasCloudflareConfig && siteConfig.cloudflare_parallel_detection !== false) {
           try {
             const parallelResult = await parallelChallengeDetection(page, forceDebug);
             if (parallelResult.hasAnyChallenge && forceDebug) {
@@ -4073,6 +4108,7 @@ function setupFrameHandling(page, forceDebug) {
       
       // Reset cleanup counter and add delay
       urlsSinceLastCleanup = 0;
+      purgeStaleTrackers();
       await fastTimeout(TIMEOUTS.BROWSER_STABILIZE_DELAY);
     }
 
@@ -4125,6 +4161,7 @@ function setupFrameHandling(page, forceDebug) {
       browser = await createBrowser(proxyArgs);
       currentProxyKey = batchProxyKey;
       urlsSinceLastCleanup = 0;
+      purgeStaleTrackers();
       await fastTimeout(TIMEOUTS.BROWSER_STABILIZE_DELAY);
     }
     
@@ -4156,6 +4193,7 @@ function setupFrameHandling(page, forceDebug) {
        const timeoutProxyArgs = currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : [];
        browser = await createBrowser(timeoutProxyArgs);
        urlsSinceLastCleanup = 0;
+       purgeStaleTrackers();
      } catch (restartErr) {
        throw restartErr;
      }
@@ -4274,6 +4312,7 @@ function setupFrameHandling(page, forceDebug) {
         }
         browser = await createBrowser(currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : []);
         urlsSinceLastCleanup = 0; // Reset counter
+        purgeStaleTrackers();
         await fastTimeout(TIMEOUTS.EMERGENCY_RESTART_DELAY); // Give browser time to stabilize
       } catch (emergencyRestartErr) {
         if (forceDebug) console.log(formatLogMessage('debug', `Emergency restart failed: ${emergencyRestartErr.message}`));
@@ -4286,6 +4325,7 @@ function setupFrameHandling(page, forceDebug) {
         await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false, cleanTempFiles: true });
         browser = await createBrowser(currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : []);
         urlsSinceLastCleanup = 0;
+        purgeStaleTrackers();
         forceRestartFlag = false; // Reset flag
         await fastTimeout(TIMEOUTS.EMERGENCY_RESTART_DELAY);
         if (forceDebug) console.log(formatLogMessage('debug', `Emergency hang detection restart completed`));
@@ -4429,7 +4469,7 @@ function setupFrameHandling(page, forceDebug) {
   }
   
   // Flush any remaining buffered log entries before compression/exit
-  flushLogBuffers();
+  flushLogBuffersSync();
   if (_logFlushTimer) {
     clearInterval(_logFlushTimer);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.55",
+  "version": "2.0.57",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {
@@ -17,7 +17,8 @@
   },
   "overrides": {
     "tar-fs": "3.1.1",
-    "ws": "8.18.3"
+    "ws": "8.18.3",
+    "yauzl": ">=3.2.1"
   },
   "keywords": [
     "puppeteer",