@fanboynz/network-scanner 2.0.31 → 2.0.32

package/lib/domain-cache.js

@@ -10,18 +10,24 @@ const { formatLogMessage } = require('./colorize');
  */
 class DomainCache {
   constructor(options = {}) {
+    // V8 Optimization: Initialize all properties in constructor for stable hidden class
     this.cache = new Set();
+    
+    // V8 Optimization: Use consistent object shape (no dynamic property addition)
     this.stats = {
       totalDetected: 0,
       totalSkipped: 0,
       cacheHits: 0,
       cacheMisses: 0
     };
-    this.options = {
-      enableLogging: options.enableLogging || false,
-      logPrefix: options.logPrefix || '[domain-cache]',
-      maxCacheSize: options.maxCacheSize || 10000 // Prevent memory leaks
-    };
+    
+    // V8 Optimization: Store options directly instead of nested object for faster property access
+    this.enableLogging = options.enableLogging || false;
+    this.logPrefix = options.logPrefix || '[domain-cache]';
+    this.maxCacheSize = options.maxCacheSize || 10000; // Prevent memory leaks
+    
+    // V8 Optimization: Pre-calculate 90% target to avoid repeated Math.floor
+    this.targetCacheSize = Math.floor(this.maxCacheSize * 0.9);
   }
 
   /**
@@ -40,14 +46,14 @@ class DomainCache {
       this.stats.totalSkipped++;
       this.stats.cacheHits++;
       
-      if (this.options.enableLogging) {
-        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache HIT: ${domain} (skipped)`));
+      if (this.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.logPrefix} Cache HIT: ${domain} (skipped)`));
       }
     } else {
       this.stats.cacheMisses++;
       
-      if (this.options.enableLogging) {
-        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache MISS: ${domain} (processing)`));
+      if (this.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.logPrefix} Cache MISS: ${domain} (processing)`));
       }
     }
     
@@ -63,25 +69,77 @@ class DomainCache {
       return false;
     }
 
-    // Prevent cache from growing too large
-    if (this.cache.size >= this.options.maxCacheSize) {
-      this.clearOldestEntries(Math.floor(this.options.maxCacheSize * 0.1)); // Remove 10% of entries
-    }
-
     const wasNew = !this.cache.has(domain);
     this.cache.add(domain);
     
     if (wasNew) {
       this.stats.totalDetected++;
       
-      if (this.options.enableLogging) {
-        console.log(formatLogMessage('debug', `${this.options.logPrefix} Marked as detected: ${domain} (cache size: ${this.cache.size})`));
+      if (this.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.logPrefix} Marked as detected: ${domain} (cache size: ${this.cache.size})`));
+      }
+    }
+    
+    // Check size AFTER adding to prevent race where multiple threads see same size
+    // and all trigger cleanup before any adds complete
+    // V8 Optimization: Use pre-calculated targetCacheSize
+    if (this.cache.size > this.maxCacheSize) {
+      const toRemove = this.cache.size - this.targetCacheSize;
+      if (toRemove > 0) {
+        this.clearOldestEntries(toRemove);
       }
     }
     
     return wasNew;
   }
 
+  /**
+   * Atomically check if domain was detected and mark it if new (race-condition free)
+   * This method combines isDomainAlreadyDetected + markDomainAsDetected in one atomic operation
+   * @param {string} domain - Domain to check and potentially mark
+   * @returns {boolean} True if domain was ALREADY detected (should skip), false if NEW (should process)
+   */
+  checkAndMark(domain) {
+    if (!domain || typeof domain !== 'string') {
+      return false;
+    }
+
+    const wasAlreadyDetected = this.cache.has(domain);
+    
+    if (wasAlreadyDetected) {
+      // Domain already exists - update skip stats and return true (should skip)
+      this.stats.totalSkipped++;
+      this.stats.cacheHits++;
+      
+      if (this.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.logPrefix} Cache HIT: ${domain} (skipped)`));
+      }
+      return true; // Already detected, should skip
+    }
+    
+    // Domain is NEW - mark it as detected
+    this.stats.cacheMisses++;
+    
+    this.cache.add(domain);
+    this.stats.totalDetected++;
+    
+    if (this.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.logPrefix} Cache MISS: ${domain} (processing and marked, cache size: ${this.cache.size})`));
+    }
+    
+    // Check size AFTER adding to prevent race where multiple threads see same size
+    // and all trigger cleanup before any adds complete
+    // V8 Optimization: Use pre-calculated targetCacheSize
+    if (this.cache.size > this.maxCacheSize) {
+      const toRemove = this.cache.size - this.targetCacheSize;
+      if (toRemove > 0) {
+        this.clearOldestEntries(toRemove);
+      }
+    }
+    
+    return false; // New domain, should process
+  }
+
   /**
    * Clear oldest entries from cache (basic LRU simulation)
    * Note: Set doesn't maintain insertion order in all Node.js versions,
@@ -96,8 +154,8 @@ class DomainCache {
     
     toRemove.forEach(domain => this.cache.delete(domain));
     
-    if (this.options.enableLogging) {
-      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cleared ${toRemove.length} old entries, cache size now: ${this.cache.size}`));
+    if (this.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.logPrefix} Cleared ${toRemove.length} old entries, cache size now: ${this.cache.size}`));
     }
   }
 
@@ -128,8 +186,8 @@ class DomainCache {
       cacheMisses: 0
     };
     
-    if (this.options.enableLogging) {
-      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache cleared (${previousSize} entries removed)`));
+    if (this.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.logPrefix} Cache cleared (${previousSize} entries removed)`));
     }
   }
 
@@ -158,8 +216,8 @@ class DomainCache {
   removeDomain(domain) {
     const wasRemoved = this.cache.delete(domain);
     
-    if (wasRemoved && this.options.enableLogging) {
-      console.log(formatLogMessage('debug', `${this.options.logPrefix} Removed from cache: ${domain}`));
+    if (wasRemoved && this.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.logPrefix} Removed from cache: ${domain}`));
     }
     
     return wasRemoved;
@@ -193,6 +251,7 @@ class DomainCache {
     return {
       isDomainAlreadyDetected: this.isDomainAlreadyDetected.bind(this),
       markDomainAsDetected: this.markDomainAsDetected.bind(this),
+      checkAndMark: this.checkAndMark.bind(this),
       getSkippedCount: () => this.stats.totalSkipped,
       getCacheSize: () => this.cache.size,
       getStats: this.getStats.bind(this)
@@ -261,6 +320,16 @@ function markDomainAsDetected(domain) {
   cache.markDomainAsDetected(domain);
 }
 
+/**
+ * Atomically check and mark a domain (race-condition free)
+ * @param {string} domain - Domain to check and mark
+ * @returns {boolean} True if already detected (skip), false if new (process)
+ */
+function checkAndMark(domain) {
+  const cache = getGlobalDomainCache();
+  return cache.checkAndMark(domain);
+}
+
 /**
  * Get total domains skipped (legacy wrapper)
  * @returns {number} Number of domains skipped
@@ -291,6 +360,7 @@ module.exports = {
   // Legacy wrapper functions for backward compatibility
   isDomainAlreadyDetected,
   markDomainAsDetected,
+  checkAndMark,
   getTotalDomainsSkipped,
   getDetectedDomainsCount
 };

package/lib/referrer.js

@@ -1,6 +1,16 @@
 // === Referrer Header Generation Module ===
 // This module handles generation of referrer headers for different traffic simulation modes
 
+/**
+ * Performance utility: Get random element from array
+ * Reduces code duplication and improves readability
+ * @param {Array} array - Array to select from
+ * @returns {*} Random element from array
+ */
+function getRandomElement(array) {
+  return array[Math.floor(Math.random() * array.length)];
+}
+
 /**
  * Referrer URL collections for different modes
  */
@@ -69,7 +79,7 @@ const REFERRER_COLLECTIONS = Object.freeze({
  */
 function generateSearchTerm(customTerms, context = null) {
   if (customTerms && customTerms.length > 0) {
-    return customTerms[Math.floor(Math.random() * customTerms.length)];
+    return getRandomElement(customTerms);
   }
   
   // Use context-specific terms if available
@@ -80,7 +90,7 @@ function generateSearchTerm(customTerms, context = null) {
     termCollection = REFERRER_COLLECTIONS.TECH_TERMS;
   }
   
-  return termCollection[Math.floor(Math.random() * termCollection.length)];
+  return getRandomElement(termCollection);
 }
 
 /**
@@ -91,9 +101,8 @@ function generateSearchTerm(customTerms, context = null) {
  * @returns {string} Generated search engine referrer URL
  */
 function generateSearchReferrer(searchTerms, context, forceDebug) {
-  const randomEngine = REFERRER_COLLECTIONS.SEARCH_ENGINES[
-    Math.floor(Math.random() * REFERRER_COLLECTIONS.SEARCH_ENGINES.length)
-  ];
+  const randomEngine = getRandomElement(REFERRER_COLLECTIONS.SEARCH_ENGINES);
+  
   const searchTerm = generateSearchTerm(searchTerms, context);
   const referrerUrl = randomEngine + encodeURIComponent(searchTerm);
   
@@ -110,9 +119,7 @@ function generateSearchReferrer(searchTerms, context, forceDebug) {
  * @returns {string} Generated social media referrer URL
  */
 function generateSocialMediaReferrer(forceDebug) {
-  const randomSocial = REFERRER_COLLECTIONS.SOCIAL_MEDIA[
-    Math.floor(Math.random() * REFERRER_COLLECTIONS.SOCIAL_MEDIA.length)
-  ];
+  const randomSocial = getRandomElement(REFERRER_COLLECTIONS.SOCIAL_MEDIA);
   
   if (forceDebug) {
     console.log(`[debug] Generated social media referrer: ${randomSocial}`);
@@ -127,9 +134,7 @@ function generateSocialMediaReferrer(forceDebug) {
  * @returns {string} Generated news site referrer URL
  */
 function generateNewsReferrer(forceDebug) {
-  const randomNews = REFERRER_COLLECTIONS.NEWS_SITES[
-    Math.floor(Math.random() * REFERRER_COLLECTIONS.NEWS_SITES.length)
-  ];
+  const randomNews = getRandomElement(REFERRER_COLLECTIONS.NEWS_SITES);
   
   if (forceDebug) {
     console.log(`[debug] Generated news referrer: ${randomNews}`);
@@ -155,33 +160,47 @@ function isValidUrl(url) {
  * @returns {boolean} True if referrer should be disabled for this URL
  */
 function shouldDisableReferrer(targetUrl, disableList, forceDebug = false) {
-  if (!disableList || !Array.isArray(disableList) || disableList.length === 0) {
+  // Fast path: early return for empty/invalid inputs
+  if (!disableList?.length || !targetUrl || typeof targetUrl !== 'string') {
     return false;
   }
   
-  if (!targetUrl || typeof targetUrl !== 'string') {
-    return false;
+  // Parse target URL once (performance optimization)
+  let targetHostname = null;
+  let targetUrlParsed = false;
+  
+  try {
+    targetHostname = new URL(targetUrl).hostname;
+    targetUrlParsed = true;
+  } catch (e) {
+    // Invalid URL - can only do string matching
+    targetUrlParsed = false;
   }
   
   for (const disablePattern of disableList) {
     if (typeof disablePattern !== 'string') continue;
     
-    // Exact URL match
+    // Fast check: Exact URL match (no parsing needed)
     if (targetUrl === disablePattern) {
-      if (forceDebug) console.log(`[debug] Referrer disabled for exact URL match: ${targetUrl}`);
+      if (forceDebug) console.log(`[debug] Referrer disabled for exact match: ${targetUrl}`);
       return true;
     }
     
-    // Domain/hostname match
-    try {
-      const targetHostname = new URL(targetUrl).hostname;
-      const disableHostname = new URL(disablePattern).hostname;
-      if (targetHostname === disableHostname) {
-        if (forceDebug) console.log(`[debug] Referrer disabled for domain match: ${targetHostname}`);
-        return true;
+    // Domain/hostname match (use cached parsed URL)
+    if (targetUrlParsed) {
+      try {
+        const disableHostname = new URL(disablePattern).hostname;
+        if (targetHostname === disableHostname) {
+          if (forceDebug) console.log(`[debug] Referrer disabled for domain match: ${targetHostname}`);
+          return true;
+        }
+      } catch (e) {
+        // disablePattern is not a valid URL, try substring match below
       }
-    } catch (e) {
-      // If pattern is not a valid URL, try simple string matching
+    }
+    
+    // Fallback: Simple substring match (for patterns like 'example.com')
+    if (!targetUrlParsed || disablePattern.includes('/') === false) {
       if (targetUrl.includes(disablePattern)) {
         if (forceDebug) console.log(`[debug] Referrer disabled for pattern match: ${disablePattern} in ${targetUrl}`);
         return true;
@@ -218,7 +237,7 @@ function generateReferrerUrl(referrerConfig, forceDebug = false) {
         return '';
       }
       
-      const randomUrl = referrerConfig[Math.floor(Math.random() * referrerConfig.length)];
+      const randomUrl = getRandomElement(referrerConfig);
       const url = isValidUrl(randomUrl) ? randomUrl : '';
       
       if (forceDebug) {
@@ -261,7 +280,7 @@ function generateReferrerUrl(referrerConfig, forceDebug = false) {
         case 'mixed': {
           // Randomly choose between different referrer types
           const modes = ['random_search', 'social_media', 'news_sites'];
-          const randomMode = modes[Math.floor(Math.random() * modes.length)];
+          const randomMode = getRandomElement(modes);
           
           if (forceDebug) console.log(`[debug] Mixed mode selected: ${randomMode}`);
           
@@ -335,14 +354,26 @@ function validateReferrerConfig(referrerConfig) {
   if (Array.isArray(referrerConfig)) {
     if (referrerConfig.length === 0) {
       result.warnings.push('Empty referrer array will result in no referrer');
-    } else {
-      referrerConfig.forEach((url, index) => {
+      return result;
+    }
+    
+    // Fast validation: check only first and last items if array is large
+    const itemsToCheck = referrerConfig.length > 10 
+      ? [referrerConfig[0], referrerConfig[referrerConfig.length - 1]]
+      : referrerConfig;
+    
+    itemsToCheck.forEach((url, index) => {
         if (!isValidUrl(url)) {
-          result.errors.push(`Array item ${index} is not a valid HTTP/HTTPS URL: ${url}`);
+          const actualIndex = itemsToCheck === referrerConfig ? index : (index === 0 ? 0 : referrerConfig.length - 1);
+          result.errors.push(`Array item ${actualIndex} is not a valid HTTP/HTTPS URL: ${url}`);
           result.isValid = false;
         }
       });
+
+    if (referrerConfig.length > 10 && itemsToCheck.length < referrerConfig.length) {
+      result.warnings.push(`Large array (${referrerConfig.length} items): only validated first and last items for performance`);
     }
+    
     return result;
   }
   

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.31 ===
+// === Network scanner script (nwss.js) v2.0.32 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -145,7 +145,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.31'; // Script version
+const VERSION = '2.0.32'; // Script version
 
 // get startTime
 const startTime = Date.now();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.31",
+  "version": "2.0.32",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {