@fanboynz/network-scanner 1.0.40 → 1.0.42

package/lib/nettools.js

@@ -725,9 +725,23 @@ function createNetToolsHandler(config) {
   const hasDig = digTerms && Array.isArray(digTerms) && digTerms.length > 0;
   const hasDigOr = digOrTerms && Array.isArray(digOrTerms) && digOrTerms.length > 0;
   
-  // Add deduplication cache for nettools lookups
-  const processedDomains = new Set();
-  
+  // Add separate deduplication caches for different lookup types
+  const processedWhoisDomains = new Set();
+  const processedDigDomains = new Set();
+  // Add whois resolution caching to avoid redundant whois lookups
+  const whoisResultCache = new Map();
+  const WHOIS_CACHE_TTL = 900000; // 15 minutes cache TTL (whois data changes less frequently)
+  const MAX_CACHE_SIZE = 400; // Larger cache for whois due to longer TTL
+  // Size          Memory
+  // 100          ~900KB
+  // 200          1.8MB
+  // 300          2.6MB
+  // 400          3.4MB
+  // 500          4.2MB  
+  // Add DNS resolution caching to avoid redundant dig lookups
+  const digResultCache = new Map();
+  const DIG_CACHE_TTL = 300000; // 5 minutes cache TTL
+  const DIG_MAX_CACHE_SIZE = 400; // Smaller cache for dig due to shorter TTL
   
   return async function handleNetToolsCheck(domain, originalDomain) {
     // Helper function to log to BOTH console and debug file
@@ -761,18 +775,25 @@ function createNetToolsHandler(config) {
       }
     }
     
-    // Skip if we've already processed this domain
-    if (processedDomains.has(domain)) {
+    // Determine which domain will be used for dig lookup
+    const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
+    
+    // Check if we need to perform any lookups
+    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(domain);
+    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDomain);
+    
+    // Skip if we don't need to perform any lookups
+    if (!needsWhoisLookup && !needsDigLookup) {
       if (forceDebug) {
-        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookup for ${domain}`);
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${!needsWhoisLookup}, dig: ${!needsDigLookup})`);
       }
       return;
     }
     
-    // Mark domain as being processed
-    processedDomains.add(domain);
+
     if (forceDebug) {
-      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing new domain: ${domain} (${processedDomains.size} total processed)`);
+      const totalProcessed = processedWhoisDomains.size + processedDigDomains.size;
+      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup}, dig: ${needsDigLookup}) (${totalProcessed} total processed)`);
     }
 
       // Log site-specific whois delay if different from default
@@ -816,6 +837,7 @@ function createNetToolsHandler(config) {
         logToConsoleAndFile(`${messageColors.highlight('[nettools]')} digSubdomain setting: ${digSubdomain}`);
         logToConsoleAndFile(`${messageColors.highlight('[nettools]')} domain parameter: ${domain}`);
         logToConsoleAndFile(`${messageColors.highlight('[nettools]')} originalDomain parameter: ${originalDomain}`);
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Final digDomain will be: ${digDomain}`);
         if (whoisServer) {
           const serverInfo = Array.isArray(whoisServer) 
             ? `randomized from [${whoisServer.join(', ')}]` 
@@ -824,13 +846,6 @@ function createNetToolsHandler(config) {
         }
       }
       
-      // Determine which domain to use for dig lookup
-      const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
-      
-      if (forceDebug) {
-        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Final digDomain will be: ${digDomain}`);
-      }
-      
       // Enhanced dry run logging
       if (dryRunCallback && forceDebug) {
         logToConsoleAndFile(`${messageColors.highlight('[nettools-dryrun]')} Processing ${domain} (original: ${originalDomain})`);
@@ -863,15 +878,47 @@ function createNetToolsHandler(config) {
       }
       
       // Perform whois lookup if either whois or whois-or is configured
-      if (hasWhois || hasWhoisOr) {
+      if (needsWhoisLookup) {
+        // Mark whois domain as being processed
+        processedWhoisDomains.add(domain);
+        
+        // Check whois cache first - cache key includes server for accuracy
         const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
+        const whoisCacheKey = `${domain}-${selectedServer || 'default'}`;
+        const now = Date.now();
+        let whoisResult = null;
         
-        if (forceDebug) {
-          const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
-          logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing whois lookup for ${domain}${serverInfo}`);
+        if (whoisResultCache.has(whoisCacheKey)) {
+          const cachedEntry = whoisResultCache.get(whoisCacheKey);
+          if (now - cachedEntry.timestamp < WHOIS_CACHE_TTL) {
+            if (forceDebug) {
+              const age = Math.round((now - cachedEntry.timestamp) / 1000);
+              const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
+            }
+            whoisResult = { 
+              ...cachedEntry.result,
+              // Add cache metadata
+              fromCache: true,
+              cacheAge: now - cachedEntry.timestamp,
+              originalTimestamp: cachedEntry.timestamp
+            };
+          } else {
+            // Cache expired, remove it
+            whoisResultCache.delete(whoisCacheKey);
+            if (forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${domain}, performing fresh lookup`);
+            }
+          }
         }
-               
-        try {
+        
+        // Perform fresh lookup if not cached
+        if (!whoisResult) {
+          if (forceDebug) {
+            const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
+            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
+          }
+        
           // Configure retry options based on site config or use defaults
           const retryOptions = {
             maxRetries: siteConfig.whois_max_retries || 2,
@@ -881,7 +928,47 @@ function createNetToolsHandler(config) {
             retryOnError: siteConfig.whois_retry_on_error === true // Default false
           };
           
-          const whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+          try {
+            whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+            
+            // Cache successful results (and certain types of failures)
+            if (whoisResult.success || 
+                (whoisResult.error && !whoisResult.isTimeout && 
+                 !whoisResult.error.toLowerCase().includes('connection') &&
+                 !whoisResult.error.toLowerCase().includes('network'))) {
+              
+              whoisResultCache.set(whoisCacheKey, {
+                result: whoisResult,
+                timestamp: now
+              });
+              
+              if (forceDebug) {
+                const cacheType = whoisResult.success ? 'successful' : 'failed';
+                const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
+                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${domain}${serverInfo}`);
+              }
+            }
+          } catch (whoisError) {
+            // Handle exceptions from whois lookup
+            if (forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
+              if (whoisError.stack) {
+                logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
+              }
+            }
+            
+            // Log whois exceptions in dry run mode
+            if (dryRunCallback && forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+            }
+            // Continue with dig if configured
+            whoisResult = null; // Ensure we don't process a null result
+          }
+        }
+        
+        // Process whois result (whether from cache or fresh lookup)
+        if (whoisResult) {
           
           if (whoisResult.success) {
             // Check AND terms if configured
@@ -891,6 +978,7 @@ function createNetToolsHandler(config) {
                 dryRunCallback(domain, 'whois', 'AND logic', whoisTerms.join(', '), 'All terms found in whois data', {
                   server: whoisResult.whoisServer || 'default',
                   duration: whoisResult.duration,
+                  fromCache: whoisResult.fromCache || false,
                   retryAttempts: whoisResult.retryInfo?.totalAttempts || 1
                 });
               }
@@ -908,6 +996,7 @@ function createNetToolsHandler(config) {
                 dryRunCallback(domain, 'whois', 'OR logic', matchedTerm, 'Term found in whois data', {
                   server: whoisResult.whoisServer || 'default',
                   duration: whoisResult.duration,
+                  fromCache: whoisResult.fromCache || false,
                   retryAttempts: whoisResult.retryInfo?.totalAttempts || 1
                 });
               }
@@ -920,7 +1009,9 @@ function createNetToolsHandler(config) {
             if (forceDebug) {
               const serverUsed = whoisResult.whoisServer ? ` (server: ${whoisResult.whoisServer})` : ' (default server)';
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} in ${whoisResult.duration}ms${retryInfo}`);             
+              const cacheInfo = whoisResult.fromCache ? ` [CACHED - ${Math.round(whoisResult.cacheAge / 1000)}s old]` : '';
+              const duration = whoisResult.fromCache ? `cached in 0ms` : `in ${whoisResult.duration}ms`;
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
               
               if (whoisResult.retryInfo && whoisResult.retryInfo.retriedAfterFailure) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Success after retry - servers attempted: [${whoisResult.retryInfo.serversAttempted.map(s => s || 'default').join(', ')}]`);
@@ -996,25 +1087,29 @@ function createNetToolsHandler(config) {
             }
             // Don't return early - continue with dig if configured
           }
-        } catch (whoisError) {
-          if (forceDebug) {
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
-            if (whoisError.stack) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
+        }
+        
+        // Periodic whois cache cleanup to prevent memory leaks
+        if (whoisResultCache.size > MAX_CACHE_SIZE) {
+          const now = Date.now();
+          let cleanedCount = 0;
+          for (const [key, entry] of whoisResultCache.entries()) {
+            if (now - entry.timestamp > WHOIS_CACHE_TTL) {
+              whoisResultCache.delete(key);
+              cleanedCount++;
             }
           }
-          
-          // Log whois exceptions in dry run mode
-          if (dryRunCallback && forceDebug) {
-            logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+          if (forceDebug && cleanedCount > 0) {
+            logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cleaned ${cleanedCount} expired entries, cache size: ${whoisResultCache.size}`);
           }
-          // Continue with dig if configured
         }
       }
       
       // Perform dig lookup if configured
-      if (hasDig || hasDigOr) {
+      if (needsDigLookup) {
+        // Mark dig domain as being processed
+        processedDigDomains.add(digDomain);
+        
         if (forceDebug) {
           const digTypes = [];
           if (hasDig) digTypes.push('dig-and');
@@ -1023,7 +1118,37 @@ function createNetToolsHandler(config) {
         }
         
         try {
+          // Check dig cache first to avoid redundant dig operations
+          const digCacheKey = `${digDomain}-${digRecordType}`;
+          const now = Date.now();
+          let digResult = null;
+          
+          if (digResultCache.has(digCacheKey)) {
+            const cachedEntry = digResultCache.get(digCacheKey);
+            if (now - cachedEntry.timestamp < DIG_CACHE_TTL) {
+              if (forceDebug) {
+                logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Using cached result for ${digDomain} (${digRecordType}) [age: ${Math.round((now - cachedEntry.timestamp) / 1000)}s]`);
+              }
+              digResult = cachedEntry.result;
+            } else {
+              // Cache expired, remove it
+              digResultCache.delete(digCacheKey);
+            }
+          }
+          
+          if (!digResult) {
           const digResult = await digLookup(digDomain, digRecordType, 5000); // 5 second timeout for dig
+            
+            // Cache the result for future use
+            digResultCache.set(digCacheKey, {
+              result: digResult,
+              timestamp: now
+            });
+            
+            if (forceDebug && digResult.success) {
+              logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cached new result for ${digDomain} (${digRecordType})`);
+            }
+          }
           
           if (digResult.success) {
             // Check AND terms if configured
@@ -1083,6 +1208,21 @@ function createNetToolsHandler(config) {
             logToConsoleAndFile(`${messageColors.highlight('[dig-dryrun]')} Exception: ${digError.message}`);
           }
         }
+        
+        // Periodic dig cache cleanup to prevent memory leaks
+        if (digResultCache.size > DIG_MAX_CACHE_SIZE) {
+          const now = Date.now();
+          let cleanedCount = 0;
+          for (const [key, entry] of digResultCache.entries()) {
+            if (now - entry.timestamp > DIG_CACHE_TTL) {
+              digResultCache.delete(key);
+              cleanedCount++;
+            }
+          }
+          if (forceDebug && cleanedCount > 0) {
+            logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cleaned ${cleanedCount} expired entries, cache size: ${digResultCache.size}`);
+          }
+        }
       }
       
       // Domain matches if any of these conditions are true:

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.40 ===
+// === Network scanner script (nwss.js) v1.0.42 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -33,7 +33,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.40'; // Script version
+const VERSION = '1.0.42'; // Script version
 
 // get startTime
 const startTime = Date.now();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.40",
+  "version": "1.0.42",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {