@fanboynz/network-scanner 2.0.34 → 2.0.36

package/lib/browserexit.js

@@ -3,6 +3,10 @@
  * Provides graceful and forced browser closure functionality with comprehensive temp file cleanup
  */
 
+
+const fs = require('fs');
+const { execSync } = require('child_process');
+
 // Constants for temp file cleanup
 const CHROME_TEMP_PATHS = [
   '/tmp',
@@ -13,8 +17,7 @@ const CHROME_TEMP_PATHS = [
 const CHROME_TEMP_PATTERNS = [
   '.com.google.Chrome.*',        // Google Chrome temp files
   '.org.chromium.Chromium.*',
-  'puppeteer-*',
-  '.com.google.Chrome.*'         // Ensure Google Chrome pattern is included
+  'puppeteer-*'
 ];
 
 /**
@@ -33,7 +36,6 @@ async function cleanupChromeTempFiles(options = {}) {
   } = options;
 
   try {
-    const { execSync } = require('child_process');
 
     // Base cleanup commands for standard temp directories
     const cleanupCommands = [
@@ -56,17 +58,17 @@ async function cleanupChromeTempFiles(options = {}) {
     
     for (const command of cleanupCommands) {
       try {
-        // Get file count before cleanup for reporting
-        const listCommand = command.replace('rm -rf', 'ls -1d').replace(' 2>/dev/null || true', ' 2>/dev/null | wc -l || echo 0');
-        const fileCount = parseInt(execSync(listCommand, { stdio: 'pipe' }).toString().trim()) || 0;
+        // Extract glob pattern and count matches before deletion
+        const globPattern = command.match(/rm -rf ([^ ]+)/)?.[1];
+        if (!globPattern) continue;
+        const fileCount = parseInt(execSync(`ls -1d ${globPattern} 2>/dev/null | wc -l || echo 0`, { stdio: 'pipe' }).toString().trim()) || 0;
         
         if (fileCount > 0) {
           execSync(command, { stdio: 'ignore' });
           totalCleaned += fileCount;
           
           if (forceDebug) {
-            const pathPattern = command.match(/rm -rf ([^ ]+)/)?.[1] || 'unknown';
-            console.log(`[debug] [temp-cleanup] Cleaned ${fileCount} items from ${pathPattern}`);
+            console.log(`[debug] [temp-cleanup] Cleaned ${fileCount} items from ${globPattern}`);
           }
         }
       } catch (cmdErr) {
@@ -102,7 +104,6 @@ async function comprehensiveChromeTempCleanup(options = {}) {
   const { forceDebug = false, verbose = false } = options;
   
   try {
-    const { execSync } = require('child_process');
     let totalCleaned = 0;
     
     if (verbose && !forceDebug) {
@@ -112,8 +113,7 @@ async function comprehensiveChromeTempCleanup(options = {}) {
     for (const basePath of CHROME_TEMP_PATHS) {
       // Check if the base path exists before trying to clean it
       try {
-        const pathExists = execSync(`test -d "${basePath}" && echo "exists" || echo "missing"`, { stdio: 'pipe' })
-          .toString().trim() === 'exists';
+        const pathExists = fs.existsSync(basePath);
         
         if (!pathExists) {
           if (forceDebug) {
@@ -149,7 +149,7 @@ async function comprehensiveChromeTempCleanup(options = {}) {
     if (verbose && totalCleaned > 0) {
       console.log(`[temp-cleanup] ? Removed ${totalCleaned} temporary file(s)/folder(s)`);
     } else if (verbose && totalCleaned === 0) {
-      console.log(`[temp-cleanup] ??  No temporary files found to remove`);
+      console.log(`[temp-cleanup] ? Clean - no remaining temporary files`);
     } else if (forceDebug) {
       console.log(`[debug] [temp-cleanup] Comprehensive cleanup completed (${totalCleaned} items)`);
     }
@@ -179,7 +179,6 @@ async function cleanupUserDataDir(userDataDir, forceDebug = false) {
   }
 
   try {
-    const fs = require('fs');
     
     if (!fs.existsSync(userDataDir)) {
       if (forceDebug) {
@@ -281,9 +280,7 @@ async function forceBrowserKill(browser, forceDebug = false) {
     const mainPid = browserProcess.pid;
     if (forceDebug) console.log(`[debug] [browser] Main Chrome PID: ${mainPid}`);
 
-    // Find and kill ALL related Chrome processes
-    const { execSync } = require('child_process');
-     
+    // Find and kill ALL related Chrome processes    
     
     try {
       // Find all Chrome processes with puppeteer in command line
@@ -390,9 +387,7 @@ async function forceBrowserKill(browser, forceDebug = false) {
  * @returns {Promise<void>}
  */
 async function killAllPuppeteerChrome(forceDebug = false) {
-  try {
-    const { execSync } = require('child_process');
-    
+  try {    
     if (forceDebug) console.log(`[debug] [browser] Nuclear option: killing all puppeteer Chrome processes...`);
     
     try {
@@ -439,7 +434,7 @@ async function handleBrowserExit(browser, options = {}) {
   
   const results = {
     browserClosed: false,
-    tempFilescleaned: 0,
+    tempFilesCleanedCount: 0,
     userDataCleaned: false,
     success: false,
     errors: []

package/nwss.js

@@ -694,6 +694,11 @@ const {
   ...otherGlobalConfig 
 } = config;
 
+// Pre-compile global blocked regexes ONCE (used in every processUrl call)
+const globalBlockedRegexes = Array.isArray(globalBlocked)
+  ? globalBlocked.map(pattern => new RegExp(pattern))
+  : [];
+
 // Apply global configuration overrides with validation
 // Priority: Command line args > config.json > defaults
 const MAX_CONCURRENT_SITES = (() => {
@@ -943,6 +948,39 @@ if (dumpUrls) {
   }
 }
 
+// --- Buffered Log Writer ---
+// Avoids blocking I/O on every intercepted request in debug/dumpurls mode
+const _logBuffers = new Map();  // filePath -> string[]
+const LOG_FLUSH_INTERVAL = 2000; // Flush every 2 seconds
+let _logFlushTimer = null;
+
+function bufferedLogWrite(filePath, entry) {
+  if (!filePath) return;
+  if (!_logBuffers.has(filePath)) {
+    _logBuffers.set(filePath, []);
+  }
+  _logBuffers.get(filePath).push(entry);
+}
+
+function flushLogBuffers() {
+  for (const [filePath, entries] of _logBuffers) {
+    if (entries.length > 0) {
+      try {
+        fs.appendFileSync(filePath, entries.join(''));
+      } catch (err) {
+        console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
+      }
+      entries.length = 0; // Clear buffer
+    }
+  }
+}
+
+// Start periodic flush if any logging is enabled
+if (forceDebug || dumpUrls) {
+  _logFlushTimer = setInterval(flushLogBuffers, LOG_FLUSH_INTERVAL);
+  _logFlushTimer.unref(); // Don't keep process alive just for flushing
+}
+
 // Log comments if debug mode is enabled and comments exist
 if (forceDebug && globalComments) {
   const commentList = Array.isArray(globalComments) ? globalComments : [globalComments];
@@ -1047,15 +1085,21 @@ function shouldBypassCacheForUrl(url, siteConfig) {
   return siteConfig.bypass_cache === true;
 }
 
-// ability to use widcards in ignoreDomains
+// ability to use wildcards in ignoreDomains
+// Cache compiled wildcard regexes to avoid recompilation on every request
+const _wildcardRegexCache = new Map();
 function matchesIgnoreDomain(domain, ignorePatterns) {
   return ignorePatterns.some(pattern => {
     if (pattern.includes('*')) {
-      // Convert wildcard pattern to regex
-      const regexPattern = pattern
-        .replace(/\./g, '\\.')  // Escape dots
-        .replace(/\*/g, '.*');  // Convert * to .*
-      return new RegExp(`^${regexPattern}$`).test(domain);
+      let compiled = _wildcardRegexCache.get(pattern);
+      if (!compiled) {
+        const regexPattern = pattern
+          .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')  // Escape all regex specials including *
+          .replace(/\\\*/g, '.*');                    // Convert escaped \* back to .*
+        compiled = new RegExp(`^${regexPattern}$`);
+        _wildcardRegexCache.set(pattern, compiled);
+      }
+      return compiled.test(domain);
     }
     return domain.endsWith(pattern);
   });
@@ -1065,14 +1109,6 @@ function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
   let activeFrames = new Set(); // Use Set to track frame references
   
-  // Clear frame tracking on navigation to prevent stale references
-  page.on('framenavigated', (frame) => {
-    if (frame === page.mainFrame()) {
-      // Main frame navigated - clear all tracked frames
-      activeFrames.clear();
-    }
-  });
-
   // Handle frame creation with error suppression
   page.on('frameattached', async (frame) => {
     // Enhanced frame handling with detached frame protection
@@ -1182,10 +1218,16 @@ function setupFrameHandling(page, forceDebug) {
       }
     }
   });
-  // Handle frame navigations (keep this for monitoring)
+  // Handle frame navigations - clear stale tracking and monitor activity
   page.on('framenavigated', (frame) => {
 
-    // Skip if frame is not in our active set
+    // Main frame navigated - clear all tracked frames to prevent stale references
+    if (frame === page.mainFrame()) {
+      activeFrames.clear();
+      return;
+    }
+
+    // Skip child frames not in our active set
     if (!activeFrames.has(frame)) return;
 
     let frameUrl;
@@ -1399,12 +1441,16 @@ function setupFrameHandling(page, forceDebug) {
   // Set up cleanup on process termination
   process.on('SIGINT', async () => {
     if (forceDebug) console.log(formatLogMessage('debug', 'SIGINT received, performing cleanup...'));
+    flushLogBuffers();
+    if (_logFlushTimer) clearInterval(_logFlushTimer);
     await performEmergencyCleanup();
     process.exit(0);
   });
 
   process.on('SIGTERM', async () => {
     if (forceDebug) console.log(formatLogMessage('debug', 'SIGTERM received, performing cleanup...'));
+    flushLogBuffers();
+    if (_logFlushTimer) clearInterval(_logFlushTimer);
     await performEmergencyCleanup();
     process.exit(0);
   });
@@ -2195,11 +2241,10 @@ function setupFrameHandling(page, forceDebug) {
         ? siteConfig.blocked.map(pattern => new RegExp(pattern))
         : [];
 		
-      // Add global blocked patterns
-      const globalBlockedRegexes = Array.isArray(globalBlocked)
-        ? globalBlocked.map(pattern => new RegExp(pattern))
-        : [];
-      const allBlockedRegexes = [...blockedRegexes, ...globalBlockedRegexes];
+      // Combine site-specific with pre-compiled global blocked patterns
+      const allBlockedRegexes = blockedRegexes.length > 0
+        ? [...blockedRegexes, ...globalBlockedRegexes]
+        : globalBlockedRegexes; // Avoid spread when no site-specific patterns
 
       /**
        * Helper function to add domain to matched collection
@@ -2329,12 +2374,14 @@ function setupFrameHandling(page, forceDebug) {
       // - URL matching against blocklists (`blockedRegexes`).
       // - URL matching against filter patterns (`regexes`) for domain extraction.
       // - Global `ignoreDomains` list.
+      // Pre-compute values that are constant for this URL
+      const simplifiedCurrentUrl = getRootDomain(currentUrl);
+
       page.on('request', request => {
         const checkedUrl = request.url();
-        const checkedHostname = safeGetDomain(checkedUrl, true);
-        const checkedRootDomain = safeGetDomain(checkedUrl, false); // Root domain for first-party detection
+        const fullSubdomain = safeGetDomain(checkedUrl, true);  // Full hostname for cache
+        const checkedRootDomain = safeGetDomain(checkedUrl, false);
         // Check against ALL first-party domains (original + all redirects)
-        // This prevents redirect destinations from being marked as third-party
         const isFirstParty = checkedRootDomain && firstPartyDomains.has(checkedRootDomain);
         
         // Block infinite iframe loops - safely access frame URL
@@ -2347,9 +2394,9 @@ function setupFrameHandling(page, forceDebug) {
           }
         })();
         if (frameUrl && frameUrl.includes('creative.dmzjmp.com') && 
-            request.url().includes('go.dmzjmp.com/api/models')) {
+            checkedUrl.includes('go.dmzjmp.com/api/models')) {
           if (forceDebug) {
-            console.log(formatLogMessage('debug', `Blocking potential infinite iframe loop: ${request.url()}`));
+            console.log(formatLogMessage('debug', `Blocking potential infinite iframe loop: ${checkedUrl}`));
           }
           request.abort();
           return;
@@ -2357,19 +2404,19 @@ function setupFrameHandling(page, forceDebug) {
 
         // Enhanced debug logging to show which frame the request came from
         if (forceDebug) {
-          let frameUrl = 'unknown-frame';
+          let debugFrameUrl = 'unknown-frame';
           let isMainFrame = false;
           
           try {
             const frame = request.frame();
             if (frame) {
-              frameUrl = frame.url();
+              debugFrameUrl = frame.url();
               isMainFrame = frame === page.mainFrame();
             }
           } catch (frameErr) {
-            frameUrl = 'detached-frame';
+            debugFrameUrl = 'detached-frame';
           }
-          console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[frame: ${isMainFrame ? 'main' : 'iframe'}] ${frameUrl} → ${request.url()}`));
+          console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[frame: ${isMainFrame ? 'main' : 'iframe'}] ${debugFrameUrl} → ${checkedUrl}`));
         }
 
         // Apply adblock rules BEFORE expensive regex checks for better performance
@@ -2395,46 +2442,36 @@ function setupFrameHandling(page, forceDebug) {
 
         // Show --debug output and the url while its scanning
         if (forceDebug) {
-          const simplifiedUrl = getRootDomain(currentUrl);
           const timestamp = new Date().toISOString();
-          const logEntry = `${timestamp} [debug req][${simplifiedUrl}] ${request.url()}`;
+          const logEntry = `${timestamp} [debug req][${simplifiedCurrentUrl}] ${checkedUrl}\n`;
 
           // Output to console
-          console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[${simplifiedUrl}] ${request.url()}`));
+          console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[${simplifiedCurrentUrl}] ${checkedUrl}`));
 
-          // Output to file
-          if (debugLogFile) {
-            try {
-              fs.appendFileSync(debugLogFile, logEntry + '\n');
-            } catch (logErr) {
-              console.warn(formatLogMessage('warn', `Failed to write to debug log file: ${logErr.message}`));
-            }
-          }
+          // Output to file (buffered)
+          bufferedLogWrite(debugLogFile, logEntry);
         }
-        const reqUrl = request.url();
+        const reqUrl = checkedUrl;
         
-        // ALWAYS extract the FULL subdomain for cache checking to preserve unique subdomains
-        const fullSubdomain = safeGetDomain(reqUrl, true); // Always get full subdomain for cache
         const reqDomain = safeGetDomain(reqUrl, perSiteSubDomains); // Output domain based on config
 
         if (allBlockedRegexes.some(re => re.test(reqUrl))) {
          if (forceDebug) {
-           // Find which specific pattern matched for debug logging
-            const allPatterns = [...(siteConfig.blocked || []), ...globalBlocked];
-            const matchedPattern = allPatterns.find(pattern => new RegExp(pattern).test(reqUrl));
-            const patternSource = siteConfig.blocked && siteConfig.blocked.includes(matchedPattern) ? 'site' : 'global';
-            const simplifiedUrl = getRootDomain(currentUrl);
-            console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked]')}[${simplifiedUrl}] ${reqUrl} blocked by ${patternSource} pattern: ${matchedPattern}`));
-            
-            // Also log to file if debug logging is enabled
-            if (debugLogFile) {
-              try {
-                const timestamp = new Date().toISOString();
-                fs.appendFileSync(debugLogFile, `${timestamp} [blocked][${simplifiedUrl}] ${reqUrl} (${patternSource} pattern: ${matchedPattern})\n`);
-              } catch (logErr) {
-                console.warn(formatLogMessage('warn', `Failed to write blocked domain to debug log: ${logErr.message}`));
+           // Find which specific pattern matched using already-compiled regexes
+            let matchedPattern = '(unknown)';
+            let patternSource = 'global';
+            for (let i = 0; i < allBlockedRegexes.length; i++) {
+              if (allBlockedRegexes[i].test(reqUrl)) {
+                matchedPattern = allBlockedRegexes[i].source;
+                patternSource = i < blockedRegexes.length ? 'site' : 'global';
+                break;
               }
             }
+            console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked]')}[${simplifiedCurrentUrl}] ${reqUrl} blocked by ${patternSource} pattern: ${matchedPattern}`));
+            
+            // Also log to file (buffered)
+            const timestamp = new Date().toISOString();
+            bufferedLogWrite(debugLogFile, `${timestamp} [blocked][${simplifiedCurrentUrl}] ${reqUrl} (${patternSource} pattern: ${matchedPattern})\n`);
           }
           
           // NEW: Check if even_blocked is enabled and this URL matches filter regex
@@ -2461,15 +2498,14 @@ function setupFrameHandling(page, forceDebug) {
                       addMatchedDomain(reqDomain, resourceType, fullSubdomain);
                     }
                     
-                    const simplifiedUrl = getRootDomain(currentUrl);
                     if (siteConfig.verbose === 1) {
                       const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
-                      console.log(formatLogMessage('match', `[${simplifiedUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
+                      console.log(formatLogMessage('match', `[${simplifiedCurrentUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
                     }
                     if (dumpUrls) {
                       const timestamp = new Date().toISOString();
                       const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
-                      fs.appendFileSync(matchedUrlsLogFile, `${timestamp} [match][${simplifiedUrl}] ${reqUrl} (resourceType: ${resourceType})${resourceInfo} [BLOCKED BUT ADDED]\n`);
+                      bufferedLogWrite(matchedUrlsLogFile, `${timestamp} [match][${simplifiedCurrentUrl}] ${reqUrl} (resourceType: ${resourceType})${resourceInfo} [BLOCKED BUT ADDED]\n`);
                     }
                     break; // Only match once per URL
                   }
@@ -2625,15 +2661,14 @@ function setupFrameHandling(page, forceDebug) {
              } else {
                addMatchedDomain(reqDomain, resourceType);
              }
-             const simplifiedUrl = getRootDomain(currentUrl);
              if (siteConfig.verbose === 1) {
                const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
-              console.log(formatLogMessage('match', `[${simplifiedUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
+              console.log(formatLogMessage('match', `[${simplifiedCurrentUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
              }
              if (dumpUrls) {
                const timestamp = new Date().toISOString();
                const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
-               fs.appendFileSync(matchedUrlsLogFile, `${timestamp} [match][${simplifiedUrl}] ${reqUrl} (resourceType: ${resourceType})${resourceInfo}\n`);
+               bufferedLogWrite(matchedUrlsLogFile, `${timestamp} [match][${simplifiedCurrentUrl}] ${reqUrl} (resourceType: ${resourceType})${resourceInfo}\n`);
              }
             } else if (hasNetTools && !hasSearchString && !hasSearchStringAnd) {
              // If nettools are configured (whois/dig), perform checks on the domain
@@ -4117,6 +4152,12 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
   
+  // Flush any remaining buffered log entries before compression/exit
+  flushLogBuffers();
+  if (_logFlushTimer) {
+    clearInterval(_logFlushTimer);
+  }
+
   // Compress log files if --compress-logs is enabled
   if (compressLogs && dumpUrls && !dryRunMode) {
     // Collect all existing log files for compression

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.34",
+  "version": "2.0.36",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {