@fanboynz/network-scanner 2.0.59 → 2.0.60

package/lib/compare.js

@@ -1,6 +1,23 @@
 const fs = require('fs');
 const path = require('path');
 
+// Pre-compiled regexes for rule normalization (shared across functions)
+const RE_ADBLOCK_PREFIX = /^\|\|?/;
+const RE_LOCALHOST_127 = /^127\.0\.0\.1\s+/;
+const RE_LOCALHOST_000 = /^0\.0\.0\.0\s+/;
+const RE_CARET_SUFFIX = /\^.*$/;
+const RE_DOLLAR_SUFFIX = /\$.*$/;
+
+function normalizeRuleInline(rule) {
+  return rule
+    .replace(RE_ADBLOCK_PREFIX, '')
+    .replace(RE_LOCALHOST_127, '')
+    .replace(RE_LOCALHOST_000, '')
+    .replace(RE_CARET_SUFFIX, '')
+    .replace(RE_DOLLAR_SUFFIX, '')
+    .trim();
+}
+
 /**
  * Loads rules from a comparison file and returns them as a Set for fast lookup
  * @param {string} compareFilePath - Path to the file containing existing rules
@@ -17,23 +34,7 @@ function loadComparisonRules(compareFilePath, forceDebug = false) {
     const rules = new Set();
     
     for (const line of lines) {
-      // Normalize the rule by removing different prefixes/formats
-      let normalizedRule = line;
-      
-      // Remove adblock prefixes (||, |, etc.)
-      normalizedRule = normalizedRule.replace(/^\|\|/, '');
-      normalizedRule = normalizedRule.replace(/^\|/, '');
-      
-      // Remove localhost prefixes
-      normalizedRule = normalizedRule.replace(/^127\.0\.0\.1\s+/, '');
-      normalizedRule = normalizedRule.replace(/^0\.0\.0\.0\s+/, '');
-      
-      // Remove adblock suffixes and modifiers
-      normalizedRule = normalizedRule.replace(/\^.*$/, ''); // Remove ^ and everything after
-      normalizedRule = normalizedRule.replace(/\$.*$/, ''); // Remove $ and everything after
-      
-      // Clean up and add to set
-      normalizedRule = normalizedRule.trim();
+      const normalizedRule = normalizeRuleInline(line);
       if (normalizedRule) {
         rules.add(normalizedRule);
       }
@@ -55,21 +56,7 @@ function loadComparisonRules(compareFilePath, forceDebug = false) {
  * @returns {string} Normalized rule
  */
 function normalizeRule(rule) {
-  let normalized = rule;
-  
-  // Remove adblock prefixes
-  normalized = normalized.replace(/^\|\|/, '');
-  normalized = normalized.replace(/^\|/, '');
-  
-  // Remove localhost prefixes
-  normalized = normalized.replace(/^127\.0\.0\.1\s+/, '');
-  normalized = normalized.replace(/^0\.0\.0\.0\s+/, '');
-  
-  // Remove adblock suffixes and modifiers
-  normalized = normalized.replace(/\^.*$/, '');
-  normalized = normalized.replace(/\$.*$/, '');
-  
-  return normalized.trim();
+  return normalizeRuleInline(rule);
 }
 
 /**

package/lib/domain-cache.js

@@ -148,14 +148,16 @@ class DomainCache {
    */
   clearOldestEntries(count) {
     if (count <= 0) return;
-    
-    const entries = Array.from(this.cache);
-    const toRemove = entries.slice(0, count);
-    
-    toRemove.forEach(domain => this.cache.delete(domain));
-    
+
+    let removed = 0;
+    for (const domain of this.cache) {
+      if (removed >= count) break;
+      this.cache.delete(domain);
+      removed++;
+    }
+
     if (this.enableLogging) {
-      console.log(formatLogMessage('debug', `${this.logPrefix} Cleared ${toRemove.length} old entries, cache size now: ${this.cache.size}`));
+      console.log(formatLogMessage('debug', `${this.logPrefix} Cleared ${removed} old entries, cache size now: ${this.cache.size}`));
     }
   }
 

package/lib/grep.js

@@ -41,23 +41,19 @@ function grepContent(content, searchPatterns, options = {}) {
   }
  
   try {
-    
     const allMatches = [];
     let firstMatch = null;
-    
+
+    // Build common args once outside the loop
+    const baseArgs = ['--text', '--color=never'];
+    if (ignoreCase) baseArgs.push('-i');
+    if (wholeWord) baseArgs.push('-w');
+    if (!regex) baseArgs.push('-F');
+
     for (const pattern of searchPatterns) {
       if (!pattern || pattern.trim().length === 0) continue;
-      
-      const grepArgs = [
-        '--text', // Treat file as text
-        '--color=never', // Disable color output
-      ];
-      
-      if (ignoreCase) grepArgs.push('-i');
-      if (wholeWord) grepArgs.push('-w');
-      if (!regex) grepArgs.push('-F'); // Fixed strings (literal)
-      
-      grepArgs.push(pattern);
+
+      const grepArgs = [...baseArgs, pattern];
       
       try {
         const result = spawnSync('grep', grepArgs, {

package/lib/nettools.js

@@ -5,8 +5,11 @@
 
 const { exec, execSync } = require('child_process');
 const util = require('util');
+const fs = require('fs');
+const path = require('path');
 const { formatLogMessage, messageColors } = require('./colorize');
 const execPromise = util.promisify(exec);
+const ANSI_REGEX = /\x1b\[[0-9;]*m/g;
 
 // Cycling index for whois server rotation
 let whoisServerCycleIndex = 0;
@@ -16,14 +19,121 @@ let whoisServerCycleIndex = 0;
 // DNS records don't change based on what terms you're searching for,
 // so we cache the raw dig output and let each handler check its own terms against it
 const globalDigResultCache = new Map();
-const GLOBAL_DIG_CACHE_TTL = 300000; // 5 minutes
-const GLOBAL_DIG_CACHE_MAX = 500;
+const GLOBAL_DIG_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
+const GLOBAL_DIG_CACHE_MAX = 1000;
 
 // Global whois result cache — shared across ALL handler instances and processUrl calls
 // Whois data is per root domain and doesn't change based on search terms
 const globalWhoisResultCache = new Map();
-const GLOBAL_WHOIS_CACHE_TTL = 900000; // 15 minutes (whois data changes less frequently)
-const GLOBAL_WHOIS_CACHE_MAX = 500;
+const GLOBAL_WHOIS_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
+const GLOBAL_WHOIS_CACHE_MAX = 1000;
+
+// Persistent disk cache file paths
+const DIG_CACHE_FILE = path.join(__dirname, '..', '.digcache');
+const WHOIS_CACHE_FILE = path.join(__dirname, '..', '.whoiscache');
+
+/**
+ * Load persistent cache from disk into in-memory Map
+ * Skips expired entries and enforces max size
+ * @param {string} filePath - Path to cache file
+ * @param {Map} cache - In-memory cache Map to populate
+ * @param {number} ttl - TTL in milliseconds
+ * @param {number} maxSize - Maximum cache entries
+ */
+function loadDiskCache(filePath, cache, ttl, maxSize) {
+  try {
+    if (!fs.existsSync(filePath)) return;
+    const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+    const now = Date.now();
+    let loaded = 0;
+    for (const [key, entry] of Object.entries(data)) {
+      if (loaded >= maxSize) break;
+      if (now - entry.timestamp < ttl) {
+        cache.set(key, entry);
+        loaded++;
+      }
+    }
+  } catch {
+    // Corrupt or unreadable cache file — delete and start fresh
+    try { fs.unlinkSync(filePath); } catch {}
+  }
+}
+
+/**
+ * Save in-memory cache to disk, evicting oldest entries if over max size
+ * @param {string} filePath - Path to cache file
+ * @param {Map} cache - In-memory cache Map to persist
+ * @param {number} ttl - TTL in milliseconds
+ * @param {number} maxSize - Maximum cache entries
+ */
+function saveDiskCache(filePath, cache, ttl, maxSize) {
+  try {
+    const now = Date.now();
+    const entries = {};
+    let count = 0;
+
+    // Collect valid entries, skip expired
+    for (const [key, entry] of cache) {
+      if (now - entry.timestamp < ttl) {
+        entries[key] = entry;
+        count++;
+      }
+    }
+
+    // If over max, keep only the newest entries
+    if (count > maxSize) {
+      const sorted = Object.entries(entries)
+        .sort((a, b) => b[1].timestamp - a[1].timestamp)
+        .slice(0, maxSize);
+      const trimmed = {};
+      for (const [key, entry] of sorted) {
+        trimmed[key] = entry;
+      }
+      fs.writeFileSync(filePath, JSON.stringify(trimmed, null, 2));
+    } else {
+      fs.writeFileSync(filePath, JSON.stringify(entries, null, 2));
+    }
+  } catch {
+    // Disk write failed — non-fatal, in-memory cache still works
+  }
+}
+
+// Track in-flight lookups to prevent duplicate concurrent requests
+const pendingDigLookups = new Map();
+const pendingWhoisLookups = new Map();
+
+// DNS cache statistics
+const dnsCacheStats = { digHits: 0, digMisses: 0, whoisHits: 0, whoisMisses: 0, freshDig: [], freshWhois: [] };
+
+/**
+ * Get DNS cache statistics for end-of-scan reporting
+ * @returns {Object} Cache hit/miss counts and fresh domain lists
+ */
+function getDnsCacheStats() {
+  return { ...dnsCacheStats };
+}
+
+// Disk cache is opt-in via --dns-cache flag
+let diskCacheEnabled = false;
+
+/**
+ * Enable persistent disk caching for dig/whois results
+ * Call this when --dns-cache flag is set
+ */
+function enableDiskCache() {
+  diskCacheEnabled = true;
+  loadDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
+  loadDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
+
+  // Save caches to disk once on process exit instead of per-lookup
+  const flushCaches = () => {
+    saveDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
+    saveDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
+  };
+  process.on('exit', flushCaches);
+  process.on('SIGINT', () => { flushCaches(); process.exit(0); });
+  process.on('SIGTERM', () => { flushCaches(); process.exit(0); });
+}
 
 /**
  * Strips ANSI color codes from a string for clean file logging
@@ -32,7 +142,8 @@ const GLOBAL_WHOIS_CACHE_MAX = 500;
  */
 function stripAnsiColors(text) {
   // Remove ANSI escape sequences (color codes)
-  return text.replace(/\x1b\[[0-9;]*m/g, '');
+  ANSI_REGEX.lastIndex = 0;
+  return text.replace(ANSI_REGEX, '');
 }
 
 /**
@@ -1008,6 +1119,7 @@ function createNetToolsHandler(config) {
               cacheAge: now - cachedEntry.timestamp,
               originalTimestamp: cachedEntry.timestamp
             });
+            dnsCacheStats.whoisHits++;
           } else {
             // Cache expired, remove it
             globalWhoisResultCache.delete(whoisCacheKey);
@@ -1019,34 +1131,43 @@ function createNetToolsHandler(config) {
         
         // Perform fresh lookup if not cached
         if (!whoisResult) {
-          if (forceDebug) {
-            const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
-          }
-        
-          // Configure retry options based on site config or use defaults
-          const retryOptions = {
-            maxRetries: siteConfig.whois_max_retries || 3,
-            timeoutMultiplier: siteConfig.whois_timeout_multiplier || 1.5,
-            useFallbackServers: siteConfig.whois_use_fallback !== false, // Default true
-            retryOnTimeout: siteConfig.whois_retry_on_timeout !== false, // Default true
-            retryOnError: siteConfig.whois_retry_on_error === true // Default false
-          };
-          
-          try {
-            whoisResult = await whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
-            
-            // Cache successful results (and certain types of failures)
-            if (whoisResult.success || 
-                (whoisResult.error && !whoisResult.isTimeout && 
-                 !whoisResult.error.toLowerCase().includes('connection') &&
-                 !whoisResult.error.toLowerCase().includes('network'))) {
-              
-              globalWhoisResultCache.set(whoisCacheKey, {
-                result: whoisResult,
+          // Deduplicate concurrent lookups — wait for in-flight request instead of starting a new one
+          if (pendingWhoisLookups.has(whoisCacheKey)) {
+            whoisResult = await pendingWhoisLookups.get(whoisCacheKey);
+          } else {
+            if (forceDebug) {
+              const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
+            }
+
+            // Configure retry options based on site config or use defaults
+            const retryOptions = {
+              maxRetries: siteConfig.whois_max_retries || 3,
+              timeoutMultiplier: siteConfig.whois_timeout_multiplier || 1.5,
+              useFallbackServers: siteConfig.whois_use_fallback !== false, // Default true
+              retryOnTimeout: siteConfig.whois_retry_on_timeout !== false, // Default true
+              retryOnError: siteConfig.whois_retry_on_error === true // Default false
+            };
+
+            try {
+              const lookupPromise = whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+              pendingWhoisLookups.set(whoisCacheKey, lookupPromise);
+              whoisResult = await lookupPromise;
+              pendingWhoisLookups.delete(whoisCacheKey);
+
+              // Cache successful results (and certain types of failures)
+              if (whoisResult.success ||
+                  (whoisResult.error && !whoisResult.isTimeout &&
+                   !whoisResult.error.toLowerCase().includes('connection') &&
+                   !whoisResult.error.toLowerCase().includes('network'))) {
+
+                globalWhoisResultCache.set(whoisCacheKey, {
+                  result: whoisResult,
                 timestamp: now
               });
-              
+              dnsCacheStats.whoisMisses++;
+              dnsCacheStats.freshWhois.push(whoisRootDomain);
+
               if (forceDebug) {
                 const cacheType = whoisResult.success ? 'successful' : 'failed';
                 const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
@@ -1070,6 +1191,7 @@ function createNetToolsHandler(config) {
             // Continue with dig if configured
             whoisResult = null; // Ensure we don't process a null result
           }
+          }
         }
         
         // Process whois result (whether from cache or fresh lookup)
@@ -1235,6 +1357,7 @@ function createNetToolsHandler(config) {
                 logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Using cached result for ${digDomain} (${digRecordType}) [age: ${Math.round((now - cachedEntry.timestamp) / 1000)}s]`);
               }
               digResult = cachedEntry.result;
+              dnsCacheStats.digHits++;
             } else {
               // Cache expired, remove it
               globalDigResultCache.delete(digCacheKey);
@@ -1242,16 +1365,26 @@ function createNetToolsHandler(config) {
           }
           
           if (!digResult) {
-          digResult = await digLookup(digDomain, digRecordType, 5000); // 5 second timeout for dig
-            
-            // Cache the result for future use
-            globalDigResultCache.set(digCacheKey, {
-              result: digResult,
-              timestamp: now
-            });
-            
-            if (forceDebug && digResult.success) {
-              logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cached new result for ${digDomain} (${digRecordType})`);
+            // Deduplicate concurrent lookups — wait for in-flight request instead of starting a new one
+            if (pendingDigLookups.has(digCacheKey)) {
+              digResult = await pendingDigLookups.get(digCacheKey);
+            } else {
+              const lookupPromise = digLookup(digDomain, digRecordType, 5000);
+              pendingDigLookups.set(digCacheKey, lookupPromise);
+              digResult = await lookupPromise;
+              pendingDigLookups.delete(digCacheKey);
+
+              // Cache the result for future use
+              globalDigResultCache.set(digCacheKey, {
+                result: digResult,
+                timestamp: now
+              });
+              dnsCacheStats.digMisses++;
+              dnsCacheStats.freshDig.push(`${digDomain} (${digRecordType})`);
+
+              if (forceDebug && digResult.success) {
+                logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cached new result for ${digDomain} (${digRecordType})`);
+              }
             }
           }
           
@@ -1435,5 +1568,7 @@ module.exports = {
   selectWhoisServer,
   getCommonWhoisServers,
   suggestWhoisServers,
-  execWithTimeout // Export for testing
+  execWithTimeout, // Export for testing
+  enableDiskCache,
+  getDnsCacheStats
 };

package/lib/output.js

@@ -5,8 +5,17 @@ const { getTotalDomainsSkipped } = require('./domain-cache');
 const { loadComparisonRules, filterUniqueRules } = require('./compare');
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
 
-// Cache for compiled wildcard regex patterns in matchesIgnoreDomain
+// Cache for compiled wildcard regex patterns in matchesIgnoreDomain (capped to prevent memory leak)
 const wildcardRegexCache = new Map();
+const WILDCARD_CACHE_MAX = 500;
+
+// Hoisted resource type map — avoid recreating per call
+const RESOURCE_TYPE_TO_ADBLOCK = {
+  'script': 'script', 'xhr': 'xmlhttprequest', 'fetch': 'xmlhttprequest',
+  'stylesheet': 'stylesheet', 'image': 'image', 'font': 'font',
+  'document': 'document', 'subdocument': 'subdocument', 'iframe': 'subdocument',
+  'websocket': 'websocket', 'media': 'media', 'ping': 'ping', 'other': null
+};
 
 /**
  * Check if domain matches any ignore patterns (supports wildcards)
@@ -23,18 +32,9 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
     if (pattern.includes('*')) {
       // Enhanced wildcard pattern handling
       if (pattern.startsWith('*.')) {
-        // Pattern: *.example.com
-        const wildcardDomain = pattern.substring(2); // Remove "*."
-        const wildcardRoot = extractDomainFromRule(`||${wildcardDomain}^`) || wildcardDomain;
-        const domainRoot = extractDomainFromRule(`||${domain}^`) || domain;
-        
-        // Use basic root domain comparison for output filtering
-        const getSimpleRoot = (d) => {
-          const parts = d.split('.');
-          return parts.length >= 2 ? parts.slice(-2).join('.') : d;
-        };
-        
-        return getSimpleRoot(domainRoot) === getSimpleRoot(wildcardRoot);
+        // Pattern: *.example.com — match exact or any subdomain
+        const suffix = pattern.substring(2);
+        return domain === suffix || domain.endsWith('.' + suffix);
       } else if (pattern.endsWith('.*')) {
         // Pattern: example.*
         const baseDomain = pattern.slice(0, -2); // Remove ".*"
@@ -42,6 +42,9 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
       } else {
         // Complex wildcard pattern (cached)
         if (!wildcardRegexCache.has(pattern)) {
+          if (wildcardRegexCache.size >= WILDCARD_CACHE_MAX) {
+            wildcardRegexCache.delete(wildcardRegexCache.keys().next().value);
+          }
           const regexPattern = pattern
             .replace(/\./g, '\\.')  // Escape dots
             .replace(/\*/g, '.*');  // Convert * to .*
@@ -153,23 +156,7 @@ function formatDomain(domain, options = {}) {
  * @returns {string|null} Adblock filter modifier, or null if should be ignored
  */
 function mapResourceTypeToAdblockModifier(resourceType) {
-  const typeMap = {
-    'script': 'script',
-    'xhr': 'xmlhttprequest', 
-    'fetch': 'xmlhttprequest',
-    'stylesheet': 'stylesheet',
-    'image': 'image',
-    'font': 'font',
-    'document': 'document',
-    'subdocument': 'subdocument',
-    'iframe': 'subdocument',
-    'websocket': 'websocket',
-    'media': 'media',
-    'ping': 'ping',
-    'other': null  // Ignore 'other' type - return null
-  };
-  
-  return typeMap[resourceType] || null; // Return null for unknown types too
+  return RESOURCE_TYPE_TO_ADBLOCK[resourceType] || null;
 }
 
 /**