@fanboynz/network-scanner 1.0.42 → 1.0.44

package/lib/cloudflare.js

@@ -9,6 +9,38 @@
  */
 const CLOUDFLARE_MODULE_VERSION = '2.1.0';
 
+/**
+ * Timeout constants for various operations (in milliseconds)
+ */
+const TIMEOUTS = {
+  QUICK_DETECTION: 3000,           // Quick Cloudflare detection check
+  PAGE_EVALUATION: 8000,           // Standard page evaluation timeout
+  PAGE_EVALUATION_SAFE: 10000,     // Safe page evaluation with extra buffer
+  CHALLENGE_COMPLETION: 3000,      // Challenge completion check
+  PHISHING_WAIT: 2000,            // Wait before checking phishing warning
+  PHISHING_CLICK: 3000,           // Timeout for clicking phishing continue button
+  PHISHING_NAVIGATION: 8000,       // Wait for navigation after phishing bypass
+  CHALLENGE_WAIT: 1000,           // Wait before checking verification challenge
+  CHALLENGE_SOLVING: 20000,        // Overall challenge solving timeout
+  JS_CHALLENGE: 15000,            // JS challenge completion wait
+  JS_CHALLENGE_BUFFER: 18000,     // JS challenge with safety buffer
+  TURNSTILE_OPERATION: 8000,      // Turnstile iframe operations
+  TURNSTILE_COMPLETION: 12000,    // Turnstile completion check
+  TURNSTILE_COMPLETION_BUFFER: 15000, // Turnstile completion with buffer
+  SELECTOR_WAIT: 2000,            // Wait for selector to appear
+  SELECTOR_WAIT_BUFFER: 2500,     // Selector wait with safety buffer
+  ELEMENT_INTERACTION_DELAY: 500, // Delay before element interactions
+  CLICK_TIMEOUT: 5000,            // Standard click operation timeout
+  CLICK_TIMEOUT_BUFFER: 1000,     // Click timeout safety buffer
+  NAVIGATION_TIMEOUT: 15000,      // Standard navigation timeout
+  NAVIGATION_TIMEOUT_BUFFER: 2000, // Navigation timeout safety buffer
+  FALLBACK_TIMEOUT: 5000,         // Fallback timeout for failed operations
+  ADAPTIVE_TIMEOUT_WITH_INDICATORS: 25000,    // Adaptive timeout when indicators found + explicit config
+  ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS: 20000, // Adaptive timeout with explicit config only
+  ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS: 15000,   // Adaptive timeout for auto-detected with indicators
+  ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000 // Adaptive timeout for auto-detected without indicators
+};
+
 /**
  * Gets module version information
  * @returns {object} Version information object
@@ -29,26 +61,26 @@ async function waitForTimeout(page, timeout) {
     if (typeof page.waitForTimeout === 'function') {
       await Promise.race([
         page.waitForTimeout(timeout),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('waitForTimeout exceeded')), timeout + 5000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('waitForTimeout exceeded')), timeout + TIMEOUTS.FALLBACK_TIMEOUT))
       ]);
     } else if (typeof page.waitFor === 'function') {
       await Promise.race([
         page.waitFor(timeout),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('waitFor exceeded')), timeout + 5000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('waitFor exceeded')), timeout + TIMEOUTS.FALLBACK_TIMEOUT))
       ]);
     } else {
       await new Promise(resolve => setTimeout(resolve, timeout));
     }
   } catch (error) {
     // If all else fails, use setTimeout
-    await new Promise(resolve => setTimeout(resolve, Math.min(timeout, 5000)));
+    await new Promise(resolve => setTimeout(resolve, Math.min(timeout, TIMEOUTS.FALLBACK_TIMEOUT)));
   }
 }
 
 /**
  * Safe page evaluation with timeout protection
  */
-async function safePageEvaluate(page, func, timeout = 10000) {
+async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE) {
   try {
     return await Promise.race([
       page.evaluate(func),
@@ -72,12 +104,12 @@ async function safePageEvaluate(page, func, timeout = 10000) {
 /**
  * Safe element clicking with timeout protection
  */
-async function safeClick(page, selector, timeout = 5000) {
+async function safeClick(page, selector, timeout = TIMEOUTS.CLICK_TIMEOUT) {
   try {
     return await Promise.race([
       page.click(selector, { timeout: timeout }),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Click timeout')), timeout + 1000)
+        setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER)
       )
     ]);
   } catch (error) {
@@ -88,12 +120,12 @@ async function safeClick(page, selector, timeout = 5000) {
 /**
  * Safe navigation waiting with timeout protection
  */
-async function safeWaitForNavigation(page, timeout = 15000) {
+async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT) {
   try {
     return await Promise.race([
       page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Navigation timeout')), timeout + 2000)
+        setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER)
       )
     ]);
   } catch (error) {
@@ -141,7 +173,7 @@ async function quickCloudflareDetection(page, forceDebug = false) {
         url,
         bodySnippet: bodyText.substring(0, 200)
       };
-    }, 3000); // Quick 3-second timeout
+    }, TIMEOUTS.QUICK_DETECTION);
     
     if (forceDebug && quickCheck.hasIndicators) {
       console.log(`[debug][cloudflare] Quick detection found Cloudflare indicators on ${quickCheck.url}`);
@@ -232,7 +264,7 @@ async function analyzeCloudflareChallenge(page) {
         url: window.location.href,
         bodySnippet: bodyText.substring(0, 200)
       };
-    }, 8000); // Reduced from 10 to 8 seconds
+    }, TIMEOUTS.PAGE_EVALUATION);
   } catch (error) {
     return {
       isChallengePresent: false,
@@ -247,6 +279,17 @@ async function analyzeCloudflareChallenge(page) {
 
 /**
  * Handles Cloudflare phishing warnings with timeout protection and enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<Object>} Phishing warning result:
+ * {
+ *   success: boolean,    // True if no warning found OR successfully bypassed
+ *   attempted: boolean,  // True if warning was detected and bypass attempted
+ *   error: string|null,  // Error message if bypass failed
+ *   details: object|null // Analysis details from analyzeCloudflareChallenge()
+ * }
  */
 async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
   const result = {
@@ -260,7 +303,7 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
     if (forceDebug) console.log(`[debug][cloudflare] Checking for phishing warning on ${currentUrl}`);
     
     // Shorter wait with timeout protection
-    await waitForTimeout(page, 2000);
+    await waitForTimeout(page, TIMEOUTS.PHISHING_WAIT);
 
     const challengeInfo = await analyzeCloudflareChallenge(page);
     
@@ -277,8 +320,8 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
 
       try {
         // Use safe click with shorter timeout
-        await safeClick(page, 'a[href*="continue"]', 3000);
-        await safeWaitForNavigation(page, 8000);
+        await safeClick(page, 'a[href*="continue"]', TIMEOUTS.PHISHING_CLICK);
+        await safeWaitForNavigation(page, TIMEOUTS.PHISHING_NAVIGATION);
         
         result.success = true;
         if (forceDebug) console.log(`[debug][cloudflare] Successfully bypassed phishing warning for ${currentUrl}`);
@@ -300,6 +343,19 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
 
 /**
  * Attempts to solve Cloudflare challenges with timeout protection and enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag  
+ * @returns {Promise<Object>} Challenge verification result:
+ * {
+ *   success: boolean,        // True if no challenge found OR successfully solved
+ *   attempted: boolean,      // True if challenge was detected and solving attempted
+ *   error: string|null,      // Error message if solving failed
+ *   requiresHuman: boolean,  // True if CAPTCHA detected (requires manual intervention)
+ *   method: string|null,     // Method that succeeded: 'js_challenge_wait', 'turnstile', 'legacy_checkbox'
+ *   details: object|null     // Analysis details from analyzeCloudflareChallenge()
+ * }
  */
 async function handleVerificationChallenge(page, currentUrl, forceDebug = false) {
   const result = {
@@ -315,7 +371,7 @@ async function handleVerificationChallenge(page, currentUrl, forceDebug = false)
     if (forceDebug) console.log(`[debug][cloudflare] Checking for verification challenge on ${currentUrl}`);
     
     // Reduced wait time
-    await waitForTimeout(page, 1000);
+    await waitForTimeout(page, TIMEOUTS.CHALLENGE_WAIT);
 
     const challengeInfo = await analyzeCloudflareChallenge(page);
     result.details = challengeInfo;
@@ -381,7 +437,7 @@ async function attemptChallengeSolveWithTimeout(page, currentUrl, challengeInfo,
     return await Promise.race([
       attemptChallengeSolve(page, currentUrl, challengeInfo, forceDebug),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Challenge solving timeout')), 20000)
+        setTimeout(() => reject(new Error('Challenge solving timeout')), TIMEOUTS.CHALLENGE_SOLVING)
       )
     ]);
   } catch (error) {
@@ -480,10 +536,10 @@ async function waitForJSChallengeCompletion(page, forceDebug = false) {
                  !document.querySelector('.cf-challenge-running') &&
                  !document.querySelector('[data-cf-challenge]');
         },
-        { timeout: 15000 }
+        { timeout: TIMEOUTS.JS_CHALLENGE }
       ),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('JS challenge timeout')), 18000)
+        setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER)
       )
     ]);
     
@@ -508,7 +564,7 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
 
   try {
     // Reduced timeout for Turnstile operations
-    const turnstileTimeout = 8000; // Reduced from 10 to 8 seconds
+    const turnstileTimeout = TIMEOUTS.TURNSTILE_OPERATION;
     
     const turnstileSelectors = [
       'iframe[src*="challenges.cloudflare.com"]',
@@ -520,8 +576,8 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
     for (const selector of turnstileSelectors) {
       try {
         await Promise.race([
-          page.waitForSelector(selector, { timeout: 2000 }),
-          new Promise((_, reject) => setTimeout(() => reject(new Error('Selector timeout')), 2500))
+          page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Selector timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
         ]);
         
         const frames = await page.frames();
@@ -554,11 +610,11 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
       for (const selector of checkboxSelectors) {
         try {
           await Promise.race([
-            turnstileFrame.waitForSelector(selector, { timeout: 2000 }),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Checkbox timeout')), 2500))
+            turnstileFrame.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Checkbox timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
           ]);
           
-          await waitForTimeout(page, 500);
+          await waitForTimeout(page, TIMEOUTS.ELEMENT_INTERACTION_DELAY);
           await turnstileFrame.click(selector);
           
           if (forceDebug) console.log(`[debug][cloudflare] Clicked Turnstile checkbox: ${selector}`);
@@ -576,9 +632,9 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
             const responseInput = document.querySelector('input[name="cf-turnstile-response"]');
             return responseInput && responseInput.value && responseInput.value.length > 0;
           },
-          { timeout: 12000 }
+          { timeout: TIMEOUTS.TURNSTILE_COMPLETION }
         ),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('Turnstile completion timeout')), 15000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('Turnstile completion timeout')), TIMEOUTS.TURNSTILE_COMPLETION_BUFFER))
       ]);
       
       if (forceDebug) console.log(`[debug][cloudflare] Turnstile response token generated successfully`);
@@ -596,11 +652,11 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
       for (const selector of containerSelectors) {
         try {
           await Promise.race([
-            page.waitForSelector(selector, { timeout: 2000 }),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Container timeout')), 2500))
+            page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Container timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
           ]);
           
-          await waitForTimeout(page, 500);
+          await waitForTimeout(page, TIMEOUTS.ELEMENT_INTERACTION_DELAY);
           await page.click(selector);
           
           if (forceDebug) console.log(`[debug][cloudflare] Clicked Turnstile container: ${selector}`);
@@ -652,8 +708,8 @@ async function handleLegacyCheckbox(page, forceDebug = false) {
     for (const selector of legacySelectors) {
       try {
         await Promise.race([
-          page.waitForSelector(selector, { timeout: 2000 }),
-          new Promise((_, reject) => setTimeout(() => reject(new Error('Legacy selector timeout')), 2500))
+          page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Legacy selector timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
         ]);
         
         const checkbox = await page.$(selector);
@@ -705,7 +761,7 @@ async function checkChallengeCompletion(page) {
       return (noChallengeRunning && noChallengeContainer && noChallengePage) ||
              hasClearanceCookie ||
              hasTurnstileResponse;
-    }, 3000); // Reduced timeout
+    }, TIMEOUTS.CHALLENGE_COMPLETION);
     
     return { isCompleted };
   } catch (error) {
@@ -715,6 +771,43 @@ async function checkChallengeCompletion(page) {
 
 /**
  * Main function to handle all Cloudflare challenges with smart detection and adaptive timeouts
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {Object} siteConfig - Configuration object with cloudflare_phish and cloudflare_bypass flags
+ * @param {boolean} forceDebug - Enable debug logging
+ * 
+ * @returns {Promise<Object>} Result object with the following structure:
+ * {
+ *   phishingWarning: {
+ *     attempted: boolean,     // Whether phishing bypass was attempted
+ *     success: boolean,       // Whether bypass succeeded (true if no warning or successfully bypassed)
+ *     error: string|null,     // Error message if bypass failed
+ *     details: object|null    // Challenge analysis details from analyzeCloudflareChallenge()
+ *   },
+ *   verificationChallenge: {
+ *     attempted: boolean,     // Whether challenge bypass was attempted
+ *     success: boolean,       // Whether challenge was solved (true if no challenge or successfully solved)
+ *     error: string|null,     // Error message if solving failed
+ *     requiresHuman: boolean, // True if CAPTCHA detected - requires manual intervention
+ *     method: string|null,    // Successful method used: 'js_challenge_wait', 'turnstile', 'legacy_checkbox'
+ *     details: object|null    // Challenge analysis details from analyzeCloudflareChallenge()
+ *   },
+ *   overallSuccess: boolean,  // True if no critical failures occurred (challenges may be unsolved but didn't error)
+ *   errors: string[],         // Array of error messages from failed operations
+ *   skippedNoIndicators: boolean, // True if processing was skipped due to no Cloudflare indicators detected
+ *   timedOut: boolean         // True if adaptive timeout was reached (processing continued anyway)
+ * }
+ * 
+ * @example
+ * const result = await handleCloudflareProtection(page, url, {cloudflare_bypass: true}, false);
+ * if (result.verificationChallenge.requiresHuman) {
+ *   console.log('Manual CAPTCHA solving required');
+ * } else if (!result.overallSuccess) {
+ *   console.error('Critical errors:', result.errors);
+ * } else if (result.verificationChallenge.attempted && result.verificationChallenge.success) {
+ *   console.log(`Challenge solved using: ${result.verificationChallenge.method}`);
+ * }
  */
 async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug = false) {
   if (forceDebug) {
@@ -722,7 +815,10 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
   }
   // Quick detection first - exit early if no Cloudflare detected and no explicit config
   const quickDetection = await quickCloudflareDetection(page, forceDebug);
-  
+
+  // Early return structure when no Cloudflare indicators found
+  // Sets attempted: false, success: true for both protection types
+ 
   // Only proceed if we have indicators OR explicit config enables Cloudflare handling
   if (!quickDetection.hasIndicators && !siteConfig.cloudflare_phish && !siteConfig.cloudflare_bypass) {
     if (forceDebug) console.log(`[debug][cloudflare] No Cloudflare indicators found and no explicit config, skipping protection handling for ${currentUrl}`);
@@ -736,6 +832,9 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
     };
   }
 
+  // Standard return structure for all processing paths
+  // Individual handlers update their respective sections
+  // overallSuccess becomes false if any critical errors occur
   const result = {
     phishingWarning: { attempted: false, success: false },
     verificationChallenge: { attempted: false, success: false },
@@ -748,10 +847,10 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
     let adaptiveTimeout;
     if (siteConfig.cloudflare_phish || siteConfig.cloudflare_bypass) {
       // Explicit config - give more time
-      adaptiveTimeout = quickDetection.hasIndicators ? 25000 : 20000;
+      adaptiveTimeout = quickDetection.hasIndicators ? TIMEOUTS.ADAPTIVE_TIMEOUT_WITH_INDICATORS : TIMEOUTS.ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS;
     } else {
       // Auto-detected only - shorter timeout
-      adaptiveTimeout = quickDetection.hasIndicators ? 15000 : 10000;
+      adaptiveTimeout = quickDetection.hasIndicators ? TIMEOUTS.ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS : TIMEOUTS.ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS;
     }
 
     if (forceDebug) {
@@ -783,6 +882,12 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
 
 /**
  * Performs the actual Cloudflare handling with enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance  
+ * @param {string} currentUrl - URL being processed
+ * @param {Object} siteConfig - Configuration flags
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<Object>} Same structure as handleCloudflareProtection()
  */
 async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebug = false) {
   const result = {
@@ -794,6 +899,8 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
 
   if (forceDebug) console.log(`[debug][cloudflare] Starting Cloudflare protection handling for ${currentUrl}`);
 
+  // Handle phishing warnings first - updates result.phishingWarning
+  // Only runs if siteConfig.cloudflare_phish === true
   // Handle phishing warnings if enabled
   if (siteConfig.cloudflare_phish === true) {
     if (forceDebug) console.log(`[debug][cloudflare] Phishing warning bypass enabled for ${currentUrl}`);
@@ -812,6 +919,9 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
     console.log(`[debug][cloudflare] Phishing warning bypass disabled for ${currentUrl}`);
   }
 
+  // Handle verification challenges second - updates result.verificationChallenge  
+  // Only runs if siteConfig.cloudflare_bypass === true
+  // Sets requiresHuman: true if CAPTCHA detected (no bypass attempted)
   // Handle verification challenges if enabled
   if (siteConfig.cloudflare_bypass === true) {
     if (forceDebug) console.log(`[debug][cloudflare] Challenge bypass enabled for ${currentUrl}`);

package/lib/domain-cache.js

@@ -0,0 +1,296 @@
+/**
+ * Domain Cache Module - Tracks detected domains to prevent duplicate processing
+ * Provides performance optimization by skipping already detected domains
+ */
+
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * Domain detection cache class for tracking processed domains
+ */
+class DomainCache {
+  constructor(options = {}) {
+    this.cache = new Set();
+    this.stats = {
+      totalDetected: 0,
+      totalSkipped: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    };
+    this.options = {
+      enableLogging: options.enableLogging || false,
+      logPrefix: options.logPrefix || '[domain-cache]',
+      maxCacheSize: options.maxCacheSize || 10000 // Prevent memory leaks
+    };
+  }
+
+  /**
+   * Check if a domain was already detected in a previous scan
+   * @param {string} domain - Domain to check
+   * @returns {boolean} True if domain was already detected
+   */
+  isDomainAlreadyDetected(domain) {
+    if (!domain || typeof domain !== 'string') {
+      return false;
+    }
+
+    const isDetected = this.cache.has(domain);
+    
+    if (isDetected) {
+      this.stats.totalSkipped++;
+      this.stats.cacheHits++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache HIT: ${domain} (skipped)`));
+      }
+    } else {
+      this.stats.cacheMisses++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache MISS: ${domain} (processing)`));
+      }
+    }
+    
+    return isDetected;
+  }
+
+  /**
+   * Mark a domain as detected for future reference
+   * @param {string} domain - Domain to mark as detected
+   */
+  markDomainAsDetected(domain) {
+    if (!domain || typeof domain !== 'string') {
+      return false;
+    }
+
+    // Prevent cache from growing too large
+    if (this.cache.size >= this.options.maxCacheSize) {
+      this.clearOldestEntries(Math.floor(this.options.maxCacheSize * 0.1)); // Remove 10% of entries
+    }
+
+    const wasNew = !this.cache.has(domain);
+    this.cache.add(domain);
+    
+    if (wasNew) {
+      this.stats.totalDetected++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Marked as detected: ${domain} (cache size: ${this.cache.size})`));
+      }
+    }
+    
+    return wasNew;
+  }
+
+  /**
+   * Clear oldest entries from cache (basic LRU simulation)
+   * Note: Set doesn't maintain insertion order in all Node.js versions,
+   * so this is a simple implementation that clears a portion of the cache
+   * @param {number} count - Number of entries to remove
+   */
+  clearOldestEntries(count) {
+    if (count <= 0) return;
+    
+    const entries = Array.from(this.cache);
+    const toRemove = entries.slice(0, count);
+    
+    toRemove.forEach(domain => this.cache.delete(domain));
+    
+    if (this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cleared ${toRemove.length} old entries, cache size now: ${this.cache.size}`));
+    }
+  }
+
+  /**
+   * Get cache statistics
+   * @returns {object} Cache statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      cacheSize: this.cache.size,
+      hitRate: this.stats.cacheHits > 0 ? 
+        (this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses) * 100).toFixed(2) + '%' : 
+        '0%'
+    };
+  }
+
+  /**
+   * Clear all cached domains
+   */
+  clear() {
+    const previousSize = this.cache.size;
+    this.cache.clear();
+    this.stats = {
+      totalDetected: 0,
+      totalSkipped: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    };
+    
+    if (this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache cleared (${previousSize} entries removed)`));
+    }
+  }
+
+  /**
+   * Get all cached domains (for debugging)
+   * @returns {Array<string>} Array of cached domains
+   */
+  getAllCachedDomains() {
+    return Array.from(this.cache);
+  }
+
+  /**
+   * Check if cache contains a specific domain (without updating stats)
+   * @param {string} domain - Domain to check
+   * @returns {boolean} True if domain exists in cache
+   */
+  has(domain) {
+    return this.cache.has(domain);
+  }
+
+  /**
+   * Remove a specific domain from cache
+   * @param {string} domain - Domain to remove
+   * @returns {boolean} True if domain was removed, false if it wasn't in cache
+   */
+  removeDomain(domain) {
+    const wasRemoved = this.cache.delete(domain);
+    
+    if (wasRemoved && this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Removed from cache: ${domain}`));
+    }
+    
+    return wasRemoved;
+  }
+
+  /**
+   * Add multiple domains to cache at once
+   * @param {Array<string>} domains - Array of domains to add
+   * @returns {number} Number of domains actually added (excludes duplicates)
+   */
+  markMultipleDomainsAsDetected(domains) {
+    if (!Array.isArray(domains)) {
+      return 0;
+    }
+
+    let addedCount = 0;
+    domains.forEach(domain => {
+      if (this.markDomainAsDetected(domain)) {
+        addedCount++;
+      }
+    });
+
+    return addedCount;
+  }
+
+  /**
+   * Create bound helper functions for easy integration with existing code
+   * @returns {object} Object with bound helper functions
+   */
+  createHelpers() {
+    return {
+      isDomainAlreadyDetected: this.isDomainAlreadyDetected.bind(this),
+      markDomainAsDetected: this.markDomainAsDetected.bind(this),
+      getSkippedCount: () => this.stats.totalSkipped,
+      getCacheSize: () => this.cache.size,
+      getStats: this.getStats.bind(this)
+    };
+  }
+}
+
+/**
+ * Create a global domain cache instance (singleton pattern)
+ */
+let globalDomainCache = null;
+
+/**
+ * Get or create the global domain cache instance
+ * @param {object} options - Cache options
+ * @returns {DomainCache} Global cache instance
+ */
+function getGlobalDomainCache(options = {}) {
+  if (!globalDomainCache) {
+    globalDomainCache = new DomainCache(options);
+  }
+  return globalDomainCache;
+}
+
+/**
+ * Create helper functions that use the global cache
+ * @param {object} options - Cache options (only used if global cache doesn't exist)
+ * @returns {object} Helper functions bound to global cache
+ */
+function createGlobalHelpers(options = {}) {
+  const cache = getGlobalDomainCache(options);
+  return cache.createHelpers();
+}
+
+/**
+ * Reset the global cache (useful for testing or manual resets)
+ */
+function resetGlobalCache() {
+  if (globalDomainCache) {
+    globalDomainCache.clear();
+  }
+  globalDomainCache = null;
+}
+
+/**
+ * Legacy wrapper functions for backward compatibility
+ * These match the original function signatures from nwss.js
+ */
+
+/**
+ * Check if a domain was already detected (legacy wrapper)
+ * @param {string} domain - Domain to check
+ * @returns {boolean} True if domain was already detected
+ */
+function isDomainAlreadyDetected(domain) {
+  const cache = getGlobalDomainCache();
+  return cache.isDomainAlreadyDetected(domain);
+}
+
+/**
+ * Mark a domain as detected (legacy wrapper)
+ * @param {string} domain - Domain to mark as detected
+ */
+function markDomainAsDetected(domain) {
+  const cache = getGlobalDomainCache();
+  cache.markDomainAsDetected(domain);
+}
+
+/**
+ * Get total domains skipped (legacy wrapper)
+ * @returns {number} Number of domains skipped
+ */
+function getTotalDomainsSkipped() {
+  const cache = getGlobalDomainCache();
+  return cache.stats.totalSkipped;
+}
+
+/**
+ * Get detected domains cache size (legacy wrapper)
+ * @returns {number} Size of the detected domains cache
+ */
+function getDetectedDomainsCount() {
+  const cache = getGlobalDomainCache();
+  return cache.cache.size;
+}
+
+module.exports = {
+  // Main class
+  DomainCache,
+  
+  // Global cache functions
+  getGlobalDomainCache,
+  createGlobalHelpers,
+  resetGlobalCache,
+  
+  // Legacy wrapper functions for backward compatibility
+  isDomainAlreadyDetected,
+  markDomainAsDetected,
+  getTotalDomainsSkipped,
+  getDetectedDomainsCount
+};

package/lib/nettools.js

@@ -711,7 +711,7 @@ function createNetToolsHandler(config) {
     dryRunCallback = null,
     matchedDomains,
     addMatchedDomain,
-    currentUrl,
+    isDomainAlreadyDetected,
     getRootDomain,
     siteConfig,
     dumpUrls,
@@ -743,8 +743,19 @@ function createNetToolsHandler(config) {
   const DIG_CACHE_TTL = 300000; // 5 minutes cache TTL
   const DIG_MAX_CACHE_SIZE = 400; // Smaller cache for dig due to shorter TTL
   
-  return async function handleNetToolsCheck(domain, originalDomain) {
+  return async function handleNetToolsCheck(domain, fullSubdomain) {
+    // Use fullSubdomain parameter instead of originalDomain to maintain consistency
+    // with the domain cache fix approach
+    const originalDomain = fullSubdomain;
     // Helper function to log to BOTH console and debug file
+
+    // Check if domain was already detected (skip expensive operations)
+    if (typeof isDomainAlreadyDetected === 'function' && isDomainAlreadyDetected(fullSubdomain)) {
+      if (forceDebug) {
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping already detected subdomain: ${fullSubdomain} (output domain: ${domain})`);
+      }
+      return;
+    }
     
     // NOTE: The logToConsoleAndFile function needs to be declared INSIDE this function
     // so it has access to the closure variables (forceDebug, debugLogFile, fs) from the 
@@ -1267,13 +1278,13 @@ function createNetToolsHandler(config) {
           // No need to add to matched domains
         } else {
           if (typeof addMatchedDomain === 'function') {
-            addMatchedDomain(domain);
+            addMatchedDomain(domain, null, fullSubdomain);
           } else {
             matchedDomains.add(domain);
           }
         }
         
-        const simplifiedUrl = currentUrl ? getRootDomain(currentUrl) : 'unknown';
+        const simplifiedUrl = config.currentUrl ? getRootDomain(config.currentUrl) : 'unknown';
         
         if (siteConfig.verbose === 1) {
           const matchType = [];

package/lib/output.js

@@ -1,5 +1,7 @@
 const fs = require('fs');
 const path = require('path');
+// Import domain cache functions for statistics
+const { getTotalDomainsSkipped } = require('./domain-cache');
 const { loadComparisonRules, filterUniqueRules } = require('./compare');
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
 
@@ -426,7 +428,6 @@ function writeOutput(lines, outputFile = null, silentMode = false) {
  * Main output handler that combines all output operations
  * @param {Array} results - Processing results from scanner
  * @param {object} config - Output configuration
- * @param {string[]} config.ignoreDomains - Domains to filter out from final output
  * @returns {object} Output statistics and file paths
  */
 function handleOutput(results, config = {}) {
@@ -440,7 +441,8 @@ function handleOutput(results, config = {}) {
     dumpUrls = false,
     adblockRulesLogFile = null,
     forceDebug = false,
-    ignoreDomains = []
+    ignoreDomains = [],
+    totalDomainsSkipped = null  // Allow override or get from cache
   } = config;
   
   // Handle append mode
@@ -572,7 +574,11 @@ function handleOutput(results, config = {}) {
   if (dumpUrls && adblockRulesLogFile) {
     logSuccess = writeOutput(outputLinesWithTitles, adblockRulesLogFile, silentMode);
   }
-  
+
+  // Get domain skip statistics from cache if not provided
+  const finalTotalDomainsSkipped = totalDomainsSkipped !== null ? 
+    totalDomainsSkipped : getTotalDomainsSkipped();
+
   return {
     success: mainSuccess && logSuccess,
     outputFile,
@@ -582,6 +588,8 @@ function handleOutput(results, config = {}) {
     filteredOutCount,
     totalLines: filteredOutputLines.length,
     outputLines: outputFile ? null : filteredOutputLines // Only return lines if not written to file
+    // Note: totalDomainsSkipped statistic is now available via getTotalDomainsSkipped() 
+    // and doesn't need to be passed through the output handler
   };
 }
 

package/lib/searchstring.js

@@ -250,11 +250,19 @@ function createCurlHandler(config) {
   } = config;
 
   return async function curlHandler(requestUrl) {
-    const respDomain = perSiteSubDomains ? (new URL(requestUrl)).hostname : getRootDomain(requestUrl);
     
     // Only process URLs that match our regex patterns
     const matchesRegex = regexes.some(re => re.test(requestUrl));
     if (!matchesRegex) return;
+
+    // Extract domain and check if already detected (skip expensive operations)
+    const reqDomain = perSiteSubDomains ? (new URL(requestUrl)).hostname : getRootDomain(requestUrl);
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(reqDomain)) {
+      if (forceDebug) {
+        console.log(`[debug][curl] Skipping already detected domain: ${reqDomain}`);
+      }
+      return;
+    }
     
     // Check if this is a first-party request (same domain as the URL being scanned)
     const currentUrlHostname = new URL(currentUrl).hostname;
@@ -283,11 +291,11 @@ function createCurlHandler(config) {
       
       // If NO searchstring is defined, match immediately (like browser behavior)
       if (!hasSearchString && !hasSearchStringAnd) {
-        if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
+        if (!reqDomain || matchesIgnoreDomain(reqDomain, ignoreDomains)) {
           return;
         }
         
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, resourceType);
+        addDomainToCollection(matchedDomains, addMatchedDomain, reqDomain, resourceType);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
@@ -317,11 +325,11 @@ function createCurlHandler(config) {
       const { found, matchedString, logicType } = searchContent(content, searchStrings, searchStringsAnd, '');
       
       if (found) {
-        if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
+        if (!reqDomain || matchesIgnoreDomain(reqDomain, ignoreDomains)) {
           return;
         }
         
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, resourceType);
+        addDomainToCollection(matchedDomains, addMatchedDomain, reqDomain, resourceType);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
@@ -387,6 +395,14 @@ function createResponseHandler(config) {
     const matchesRegex = regexes.some(re => re.test(respUrl));
     if (!matchesRegex) return;
     
+    // Extract domain and check if already detected (skip expensive operations)
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(respDomain)) {
+      if (forceDebug) {
+        console.log(`[debug] Skipping response analysis for already detected domain: ${respDomain}`);
+      }
+      return;
+    }
+
     // Check if this is a first-party response (same domain as the URL being scanned)
     const currentUrlHostname = new URL(currentUrl).hostname;
     const responseHostname = new URL(respUrl).hostname;

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.42 ===
+// === Network scanner script (nwss.js) v1.0.44 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -27,17 +27,23 @@ const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvaila
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // Colorize various text when used
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./lib/colorize');
+// Domain detection cache for performance optimization
+const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.42'; // Script version
+const VERSION = '1.0.44'; // Script version
 
 // get startTime
 const startTime = Date.now();
 
+// Initialize domain cache helpers with debug logging if enabled
+const domainCacheOptions = { enableLogging: false }; // Set to true for cache debug logs
+const { isDomainAlreadyDetected, markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
+
 // --- Command-Line Argument Parsing ---
 const args = process.argv.slice(2);
 
@@ -1475,9 +1481,12 @@ function setupFrameHandling(page, forceDebug) {
       /**
        * Helper function to add domain to matched collection
        * @param {string} domain - Domain to add
+       * @param {string} fullSubdomain - Full subdomain for cache tracking
        * @param {string} resourceType - Resource type (for --adblock-rules mode)
        */
-      function addMatchedDomain(domain, resourceType = null) {
+      function addMatchedDomain(domain, resourceType = null, fullSubdomain = null) {
+       // Use fullSubdomain for cache tracking if provided, otherwise fall back to domain
+       const cacheKey = fullSubdomain || domain;
        // Check if we should ignore similar domains
        const ignoreSimilarEnabled = siteConfig.ignore_similar !== undefined ? siteConfig.ignore_similar : ignore_similar;
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
@@ -1517,6 +1526,9 @@ function setupFrameHandling(page, forceDebug) {
           return; // Skip adding this domain
         }
       }
+
+      // Mark full subdomain as detected for future reference
+      markDomainAsDetected(cacheKey);
       
         if (matchedDomains instanceof Map) {
           if (!matchedDomains.has(domain)) {
@@ -1583,6 +1595,10 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
         const reqUrl = request.url();
+        
+        // ALWAYS extract the FULL subdomain for cache checking to preserve unique subdomains
+        const fullSubdomain = safeGetDomain(reqUrl, true); // Always get full subdomain for cache
+        const reqDomain = safeGetDomain(reqUrl, perSiteSubDomains); // Output domain based on config
 
         if (allBlockedRegexes.some(re => re.test(reqUrl))) {
          if (forceDebug) {
@@ -1606,7 +1622,7 @@ function setupFrameHandling(page, forceDebug) {
           
           // NEW: Check if even_blocked is enabled and this URL matches filter regex
           if (evenBlocked) {
-            const reqDomain = safeGetDomain(reqUrl, perSiteSubDomains);
+            // reqDomain already defined above
             if (reqDomain && !matchesIgnoreDomain(reqDomain, ignoreDomains)) {
               for (const re of regexes) {
                 if (re.test(reqUrl)) {
@@ -1625,7 +1641,7 @@ function setupFrameHandling(page, forceDebug) {
                         wasBlocked: true
                       });
                     } else {
-                      addMatchedDomain(reqDomain, resourceType);
+                      addMatchedDomain(reqDomain, resourceType, fullSubdomain);
                     }
                     
                     const simplifiedUrl = getRootDomain(currentUrl);
@@ -1649,8 +1665,7 @@ function setupFrameHandling(page, forceDebug) {
           return;
         }
 
-        const reqDomain = safeGetDomain(reqUrl, perSiteSubDomains);
-
+      
         if (!reqDomain) {
           if (forceDebug) {
             console.log(formatLogMessage('debug', `Skipping request with unparseable URL: ${reqUrl}`));
@@ -1659,8 +1674,8 @@ function setupFrameHandling(page, forceDebug) {
           return;
         }
 
-        // Skip matching if this domain is one of the redirect intermediaries
-        if (redirectDomainsToExclude && redirectDomainsToExclude.includes(reqDomain)) {
+      // Skip matching if this full subdomain is one of the redirect intermediaries
+      if (redirectDomainsToExclude && redirectDomainsToExclude.includes(fullSubdomain)) {
           if (forceDebug) {
             console.log(formatLogMessage('debug', `Skipping redirect intermediary domain: ${reqDomain}`));
           }
@@ -1699,9 +1714,9 @@ function setupFrameHandling(page, forceDebug) {
            }
 
            // Check ignoreDomains AFTER regex match but BEFORE domain processing
-           if (matchesIgnoreDomain(reqDomain, ignoreDomains)) {
+           if (matchesIgnoreDomain(fullSubdomain, ignoreDomains)) {
              if (forceDebug) {
-               console.log(formatLogMessage('debug', `Ignoring domain ${reqDomain} (matches ignoreDomains pattern)`));
+               console.log(formatLogMessage('debug', `Ignoring domain ${fullSubdomain} (matches ignoreDomains pattern)`));
              }
             break; // Skip this URL - domain is in ignore list
           }
@@ -1734,6 +1749,14 @@ function setupFrameHandling(page, forceDebug) {
              }
             } else if (hasNetTools && !hasSearchString && !hasSearchStringAnd) {
              // If nettools are configured (whois/dig), perform checks on the domain
+             // Skip nettools check if full subdomain was already detected
+             if (isDomainAlreadyDetected(fullSubdomain)) {
+               if (forceDebug) {
+                 console.log(formatLogMessage('debug', `Skipping nettools check for already detected subdomain: ${fullSubdomain}`));
+               }
+               break; // Skip to next URL
+             }
+             
              if (forceDebug) {
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${re} and resourceType ${resourceType}, queued for nettools check`));
              }
@@ -1767,6 +1790,7 @@ function setupFrameHandling(page, forceDebug) {
                dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
                matchedDomains,
                addMatchedDomain,
+               isDomainAlreadyDetected,
                currentUrl,
                getRootDomain,
                siteConfig,
@@ -1777,10 +1801,17 @@ function setupFrameHandling(page, forceDebug) {
              });
              
              // Execute nettools check asynchronously
-            const originalDomain = (new URL(reqUrl)).hostname;
+            const originalDomain = fullSubdomain; // Use full subdomain for nettools
             setImmediate(() => netToolsHandler(reqDomain, originalDomain));
            } else {
              // If searchstring or searchstring_and IS defined (with or without nettools), queue for content checking
+             // Skip searchstring check if full subdomain was already detected
+             if (isDomainAlreadyDetected(fullSubdomain)) {
+               if (forceDebug) {
+                 console.log(formatLogMessage('debug', `Skipping searchstring check for already detected subdomain: ${fullSubdomain}`));
+               }
+               break; // Skip to next URL
+             }
              if (forceDebug) {
                const searchType = hasSearchStringAnd ? 'searchstring_and' : 'searchstring';
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${re} and resourceType ${resourceType}, queued for ${searchType} content search`));
@@ -1808,6 +1839,7 @@ function setupFrameHandling(page, forceDebug) {
                    regexes,
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
+                   isDomainAlreadyDetected,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -1838,6 +1870,7 @@ function setupFrameHandling(page, forceDebug) {
                    regexes,
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
+                   isDomainAlreadyDetected,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -1876,6 +1909,7 @@ function setupFrameHandling(page, forceDebug) {
          regexes,
          matchedDomains,
          addMatchedDomain, // Pass the helper function
+         isDomainAlreadyDetected,
          currentUrl,
          perSiteSubDomains,
          ignoreDomains,
@@ -2462,6 +2496,8 @@ function setupFrameHandling(page, forceDebug) {
   const totalMatches = results.reduce((sum, r) => sum + (r.rules ? r.rules.length : 0), 0);
 
   // Debug: Show output format being used
+  const totalDomainsSkipped = getTotalDomainsSkipped();
+  const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
       localhostMode,
@@ -2476,6 +2512,7 @@ function setupFrameHandling(page, forceDebug) {
     };
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
+     console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
   }
   
   // Compress log files if --compress-logs is enabled
@@ -2567,6 +2604,9 @@ function setupFrameHandling(page, forceDebug) {
     } else if (outputResult.totalRules > 0 && dryRunMode) {
       console.log(messageColors.success('Found') + ` ${outputResult.totalRules} total matches across all URLs`);
     }
+    if (totalDomainsSkipped > 0) {
+      console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.42",
+  "version": "1.0.44",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {