@fanboynz/network-scanner 1.0.41 → 1.0.43

package/lib/cloudflare.js

@@ -9,6 +9,38 @@
  */
 const CLOUDFLARE_MODULE_VERSION = '2.1.0';
 
+/**
+ * Timeout constants for various operations (in milliseconds)
+ */
+const TIMEOUTS = {
+  QUICK_DETECTION: 3000,           // Quick Cloudflare detection check
+  PAGE_EVALUATION: 8000,           // Standard page evaluation timeout
+  PAGE_EVALUATION_SAFE: 10000,     // Safe page evaluation with extra buffer
+  CHALLENGE_COMPLETION: 3000,      // Challenge completion check
+  PHISHING_WAIT: 2000,            // Wait before checking phishing warning
+  PHISHING_CLICK: 3000,           // Timeout for clicking phishing continue button
+  PHISHING_NAVIGATION: 8000,       // Wait for navigation after phishing bypass
+  CHALLENGE_WAIT: 1000,           // Wait before checking verification challenge
+  CHALLENGE_SOLVING: 20000,        // Overall challenge solving timeout
+  JS_CHALLENGE: 15000,            // JS challenge completion wait
+  JS_CHALLENGE_BUFFER: 18000,     // JS challenge with safety buffer
+  TURNSTILE_OPERATION: 8000,      // Turnstile iframe operations
+  TURNSTILE_COMPLETION: 12000,    // Turnstile completion check
+  TURNSTILE_COMPLETION_BUFFER: 15000, // Turnstile completion with buffer
+  SELECTOR_WAIT: 2000,            // Wait for selector to appear
+  SELECTOR_WAIT_BUFFER: 2500,     // Selector wait with safety buffer
+  ELEMENT_INTERACTION_DELAY: 500, // Delay before element interactions
+  CLICK_TIMEOUT: 5000,            // Standard click operation timeout
+  CLICK_TIMEOUT_BUFFER: 1000,     // Click timeout safety buffer
+  NAVIGATION_TIMEOUT: 15000,      // Standard navigation timeout
+  NAVIGATION_TIMEOUT_BUFFER: 2000, // Navigation timeout safety buffer
+  FALLBACK_TIMEOUT: 5000,         // Fallback timeout for failed operations
+  ADAPTIVE_TIMEOUT_WITH_INDICATORS: 25000,    // Adaptive timeout when indicators found + explicit config
+  ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS: 20000, // Adaptive timeout with explicit config only
+  ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS: 15000,   // Adaptive timeout for auto-detected with indicators
+  ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000 // Adaptive timeout for auto-detected without indicators
+};
+
 /**
  * Gets module version information
  * @returns {object} Version information object
@@ -29,26 +61,26 @@ async function waitForTimeout(page, timeout) {
     if (typeof page.waitForTimeout === 'function') {
       await Promise.race([
         page.waitForTimeout(timeout),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('waitForTimeout exceeded')), timeout + 5000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('waitForTimeout exceeded')), timeout + TIMEOUTS.FALLBACK_TIMEOUT))
       ]);
     } else if (typeof page.waitFor === 'function') {
       await Promise.race([
         page.waitFor(timeout),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('waitFor exceeded')), timeout + 5000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('waitFor exceeded')), timeout + TIMEOUTS.FALLBACK_TIMEOUT))
       ]);
     } else {
       await new Promise(resolve => setTimeout(resolve, timeout));
     }
   } catch (error) {
     // If all else fails, use setTimeout
-    await new Promise(resolve => setTimeout(resolve, Math.min(timeout, 5000)));
+    await new Promise(resolve => setTimeout(resolve, Math.min(timeout, TIMEOUTS.FALLBACK_TIMEOUT)));
   }
 }
 
 /**
  * Safe page evaluation with timeout protection
  */
-async function safePageEvaluate(page, func, timeout = 10000) {
+async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE) {
   try {
     return await Promise.race([
       page.evaluate(func),
@@ -72,12 +104,12 @@ async function safePageEvaluate(page, func, timeout = 10000) {
 /**
  * Safe element clicking with timeout protection
  */
-async function safeClick(page, selector, timeout = 5000) {
+async function safeClick(page, selector, timeout = TIMEOUTS.CLICK_TIMEOUT) {
   try {
     return await Promise.race([
       page.click(selector, { timeout: timeout }),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Click timeout')), timeout + 1000)
+        setTimeout(() => reject(new Error('Click timeout')), timeout + TIMEOUTS.CLICK_TIMEOUT_BUFFER)
       )
     ]);
   } catch (error) {
@@ -88,12 +120,12 @@ async function safeClick(page, selector, timeout = 5000) {
 /**
  * Safe navigation waiting with timeout protection
  */
-async function safeWaitForNavigation(page, timeout = 15000) {
+async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT) {
   try {
     return await Promise.race([
       page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Navigation timeout')), timeout + 2000)
+        setTimeout(() => reject(new Error('Navigation timeout')), timeout + TIMEOUTS.NAVIGATION_TIMEOUT_BUFFER)
       )
     ]);
   } catch (error) {
@@ -141,7 +173,7 @@ async function quickCloudflareDetection(page, forceDebug = false) {
         url,
         bodySnippet: bodyText.substring(0, 200)
       };
-    }, 3000); // Quick 3-second timeout
+    }, TIMEOUTS.QUICK_DETECTION);
     
     if (forceDebug && quickCheck.hasIndicators) {
       console.log(`[debug][cloudflare] Quick detection found Cloudflare indicators on ${quickCheck.url}`);
@@ -232,7 +264,7 @@ async function analyzeCloudflareChallenge(page) {
         url: window.location.href,
         bodySnippet: bodyText.substring(0, 200)
       };
-    }, 8000); // Reduced from 10 to 8 seconds
+    }, TIMEOUTS.PAGE_EVALUATION);
   } catch (error) {
     return {
       isChallengePresent: false,
@@ -247,6 +279,17 @@ async function analyzeCloudflareChallenge(page) {
 
 /**
  * Handles Cloudflare phishing warnings with timeout protection and enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<Object>} Phishing warning result:
+ * {
+ *   success: boolean,    // True if no warning found OR successfully bypassed
+ *   attempted: boolean,  // True if warning was detected and bypass attempted
+ *   error: string|null,  // Error message if bypass failed
+ *   details: object|null // Analysis details from analyzeCloudflareChallenge()
+ * }
  */
 async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
   const result = {
@@ -260,7 +303,7 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
     if (forceDebug) console.log(`[debug][cloudflare] Checking for phishing warning on ${currentUrl}`);
     
     // Shorter wait with timeout protection
-    await waitForTimeout(page, 2000);
+    await waitForTimeout(page, TIMEOUTS.PHISHING_WAIT);
 
     const challengeInfo = await analyzeCloudflareChallenge(page);
     
@@ -277,8 +320,8 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
 
       try {
         // Use safe click with shorter timeout
-        await safeClick(page, 'a[href*="continue"]', 3000);
-        await safeWaitForNavigation(page, 8000);
+        await safeClick(page, 'a[href*="continue"]', TIMEOUTS.PHISHING_CLICK);
+        await safeWaitForNavigation(page, TIMEOUTS.PHISHING_NAVIGATION);
         
         result.success = true;
         if (forceDebug) console.log(`[debug][cloudflare] Successfully bypassed phishing warning for ${currentUrl}`);
@@ -300,6 +343,19 @@ async function handlePhishingWarning(page, currentUrl, forceDebug = false) {
 
 /**
  * Attempts to solve Cloudflare challenges with timeout protection and enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag  
+ * @returns {Promise<Object>} Challenge verification result:
+ * {
+ *   success: boolean,        // True if no challenge found OR successfully solved
+ *   attempted: boolean,      // True if challenge was detected and solving attempted
+ *   error: string|null,      // Error message if solving failed
+ *   requiresHuman: boolean,  // True if CAPTCHA detected (requires manual intervention)
+ *   method: string|null,     // Method that succeeded: 'js_challenge_wait', 'turnstile', 'legacy_checkbox'
+ *   details: object|null     // Analysis details from analyzeCloudflareChallenge()
+ * }
  */
 async function handleVerificationChallenge(page, currentUrl, forceDebug = false) {
   const result = {
@@ -315,7 +371,7 @@ async function handleVerificationChallenge(page, currentUrl, forceDebug = false)
     if (forceDebug) console.log(`[debug][cloudflare] Checking for verification challenge on ${currentUrl}`);
     
     // Reduced wait time
-    await waitForTimeout(page, 1000);
+    await waitForTimeout(page, TIMEOUTS.CHALLENGE_WAIT);
 
     const challengeInfo = await analyzeCloudflareChallenge(page);
     result.details = challengeInfo;
@@ -381,7 +437,7 @@ async function attemptChallengeSolveWithTimeout(page, currentUrl, challengeInfo,
     return await Promise.race([
       attemptChallengeSolve(page, currentUrl, challengeInfo, forceDebug),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Challenge solving timeout')), 20000)
+        setTimeout(() => reject(new Error('Challenge solving timeout')), TIMEOUTS.CHALLENGE_SOLVING)
       )
     ]);
   } catch (error) {
@@ -480,10 +536,10 @@ async function waitForJSChallengeCompletion(page, forceDebug = false) {
                  !document.querySelector('.cf-challenge-running') &&
                  !document.querySelector('[data-cf-challenge]');
         },
-        { timeout: 15000 }
+        { timeout: TIMEOUTS.JS_CHALLENGE }
       ),
       new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('JS challenge timeout')), 18000)
+        setTimeout(() => reject(new Error('JS challenge timeout')), TIMEOUTS.JS_CHALLENGE_BUFFER)
       )
     ]);
     
@@ -508,7 +564,7 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
 
   try {
     // Reduced timeout for Turnstile operations
-    const turnstileTimeout = 8000; // Reduced from 10 to 8 seconds
+    const turnstileTimeout = TIMEOUTS.TURNSTILE_OPERATION;
     
     const turnstileSelectors = [
       'iframe[src*="challenges.cloudflare.com"]',
@@ -520,8 +576,8 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
     for (const selector of turnstileSelectors) {
       try {
         await Promise.race([
-          page.waitForSelector(selector, { timeout: 2000 }),
-          new Promise((_, reject) => setTimeout(() => reject(new Error('Selector timeout')), 2500))
+          page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Selector timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
         ]);
         
         const frames = await page.frames();
@@ -554,11 +610,11 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
       for (const selector of checkboxSelectors) {
         try {
           await Promise.race([
-            turnstileFrame.waitForSelector(selector, { timeout: 2000 }),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Checkbox timeout')), 2500))
+            turnstileFrame.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Checkbox timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
           ]);
           
-          await waitForTimeout(page, 500);
+          await waitForTimeout(page, TIMEOUTS.ELEMENT_INTERACTION_DELAY);
           await turnstileFrame.click(selector);
           
           if (forceDebug) console.log(`[debug][cloudflare] Clicked Turnstile checkbox: ${selector}`);
@@ -576,9 +632,9 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
             const responseInput = document.querySelector('input[name="cf-turnstile-response"]');
             return responseInput && responseInput.value && responseInput.value.length > 0;
           },
-          { timeout: 12000 }
+          { timeout: TIMEOUTS.TURNSTILE_COMPLETION }
         ),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('Turnstile completion timeout')), 15000))
+        new Promise((_, reject) => setTimeout(() => reject(new Error('Turnstile completion timeout')), TIMEOUTS.TURNSTILE_COMPLETION_BUFFER))
       ]);
       
       if (forceDebug) console.log(`[debug][cloudflare] Turnstile response token generated successfully`);
@@ -596,11 +652,11 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
       for (const selector of containerSelectors) {
         try {
           await Promise.race([
-            page.waitForSelector(selector, { timeout: 2000 }),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Container timeout')), 2500))
+            page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Container timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
           ]);
           
-          await waitForTimeout(page, 500);
+          await waitForTimeout(page, TIMEOUTS.ELEMENT_INTERACTION_DELAY);
           await page.click(selector);
           
           if (forceDebug) console.log(`[debug][cloudflare] Clicked Turnstile container: ${selector}`);
@@ -652,8 +708,8 @@ async function handleLegacyCheckbox(page, forceDebug = false) {
     for (const selector of legacySelectors) {
       try {
         await Promise.race([
-          page.waitForSelector(selector, { timeout: 2000 }),
-          new Promise((_, reject) => setTimeout(() => reject(new Error('Legacy selector timeout')), 2500))
+          page.waitForSelector(selector, { timeout: TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Legacy selector timeout')), TIMEOUTS.SELECTOR_WAIT_BUFFER))
         ]);
         
         const checkbox = await page.$(selector);
@@ -705,7 +761,7 @@ async function checkChallengeCompletion(page) {
       return (noChallengeRunning && noChallengeContainer && noChallengePage) ||
              hasClearanceCookie ||
              hasTurnstileResponse;
-    }, 3000); // Reduced timeout
+    }, TIMEOUTS.CHALLENGE_COMPLETION);
     
     return { isCompleted };
   } catch (error) {
@@ -715,6 +771,43 @@ async function checkChallengeCompletion(page) {
 
 /**
  * Main function to handle all Cloudflare challenges with smart detection and adaptive timeouts
+ * 
+ * @param {Object} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {Object} siteConfig - Configuration object with cloudflare_phish and cloudflare_bypass flags
+ * @param {boolean} forceDebug - Enable debug logging
+ * 
+ * @returns {Promise<Object>} Result object with the following structure:
+ * {
+ *   phishingWarning: {
+ *     attempted: boolean,     // Whether phishing bypass was attempted
+ *     success: boolean,       // Whether bypass succeeded (true if no warning or successfully bypassed)
+ *     error: string|null,     // Error message if bypass failed
+ *     details: object|null    // Challenge analysis details from analyzeCloudflareChallenge()
+ *   },
+ *   verificationChallenge: {
+ *     attempted: boolean,     // Whether challenge bypass was attempted
+ *     success: boolean,       // Whether challenge was solved (true if no challenge or successfully solved)
+ *     error: string|null,     // Error message if solving failed
+ *     requiresHuman: boolean, // True if CAPTCHA detected - requires manual intervention
+ *     method: string|null,    // Successful method used: 'js_challenge_wait', 'turnstile', 'legacy_checkbox'
+ *     details: object|null    // Challenge analysis details from analyzeCloudflareChallenge()
+ *   },
+ *   overallSuccess: boolean,  // True if no critical failures occurred (challenges may be unsolved but didn't error)
+ *   errors: string[],         // Array of error messages from failed operations
+ *   skippedNoIndicators: boolean, // True if processing was skipped due to no Cloudflare indicators detected
+ *   timedOut: boolean         // True if adaptive timeout was reached (processing continued anyway)
+ * }
+ * 
+ * @example
+ * const result = await handleCloudflareProtection(page, url, {cloudflare_bypass: true}, false);
+ * if (result.verificationChallenge.requiresHuman) {
+ *   console.log('Manual CAPTCHA solving required');
+ * } else if (!result.overallSuccess) {
+ *   console.error('Critical errors:', result.errors);
+ * } else if (result.verificationChallenge.attempted && result.verificationChallenge.success) {
+ *   console.log(`Challenge solved using: ${result.verificationChallenge.method}`);
+ * }
  */
 async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug = false) {
   if (forceDebug) {
@@ -722,7 +815,10 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
   }
   // Quick detection first - exit early if no Cloudflare detected and no explicit config
   const quickDetection = await quickCloudflareDetection(page, forceDebug);
-  
+
+  // Early return structure when no Cloudflare indicators found
+  // Sets attempted: false, success: true for both protection types
+ 
   // Only proceed if we have indicators OR explicit config enables Cloudflare handling
   if (!quickDetection.hasIndicators && !siteConfig.cloudflare_phish && !siteConfig.cloudflare_bypass) {
     if (forceDebug) console.log(`[debug][cloudflare] No Cloudflare indicators found and no explicit config, skipping protection handling for ${currentUrl}`);
@@ -736,6 +832,9 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
     };
   }
 
+  // Standard return structure for all processing paths
+  // Individual handlers update their respective sections
+  // overallSuccess becomes false if any critical errors occur
   const result = {
     phishingWarning: { attempted: false, success: false },
     verificationChallenge: { attempted: false, success: false },
@@ -748,10 +847,10 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
     let adaptiveTimeout;
     if (siteConfig.cloudflare_phish || siteConfig.cloudflare_bypass) {
       // Explicit config - give more time
-      adaptiveTimeout = quickDetection.hasIndicators ? 25000 : 20000;
+      adaptiveTimeout = quickDetection.hasIndicators ? TIMEOUTS.ADAPTIVE_TIMEOUT_WITH_INDICATORS : TIMEOUTS.ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS;
     } else {
       // Auto-detected only - shorter timeout
-      adaptiveTimeout = quickDetection.hasIndicators ? 15000 : 10000;
+      adaptiveTimeout = quickDetection.hasIndicators ? TIMEOUTS.ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS : TIMEOUTS.ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS;
     }
 
     if (forceDebug) {
@@ -783,6 +882,12 @@ async function handleCloudflareProtection(page, currentUrl, siteConfig, forceDeb
 
 /**
  * Performs the actual Cloudflare handling with enhanced debug logging
+ * 
+ * @param {Object} page - Puppeteer page instance  
+ * @param {string} currentUrl - URL being processed
+ * @param {Object} siteConfig - Configuration flags
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<Object>} Same structure as handleCloudflareProtection()
  */
 async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebug = false) {
   const result = {
@@ -794,6 +899,8 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
 
   if (forceDebug) console.log(`[debug][cloudflare] Starting Cloudflare protection handling for ${currentUrl}`);
 
+  // Handle phishing warnings first - updates result.phishingWarning
+  // Only runs if siteConfig.cloudflare_phish === true
   // Handle phishing warnings if enabled
   if (siteConfig.cloudflare_phish === true) {
     if (forceDebug) console.log(`[debug][cloudflare] Phishing warning bypass enabled for ${currentUrl}`);
@@ -812,6 +919,9 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
     console.log(`[debug][cloudflare] Phishing warning bypass disabled for ${currentUrl}`);
   }
 
+  // Handle verification challenges second - updates result.verificationChallenge  
+  // Only runs if siteConfig.cloudflare_bypass === true
+  // Sets requiresHuman: true if CAPTCHA detected (no bypass attempted)
   // Handle verification challenges if enabled
   if (siteConfig.cloudflare_bypass === true) {
     if (forceDebug) console.log(`[debug][cloudflare] Challenge bypass enabled for ${currentUrl}`);

package/lib/domain-cache.js

@@ -0,0 +1,296 @@
+/**
+ * Domain Cache Module - Tracks detected domains to prevent duplicate processing
+ * Provides performance optimization by skipping already detected domains
+ */
+
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * Domain detection cache class for tracking processed domains
+ */
+class DomainCache {
+  constructor(options = {}) {
+    this.cache = new Set();
+    this.stats = {
+      totalDetected: 0,
+      totalSkipped: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    };
+    this.options = {
+      enableLogging: options.enableLogging || false,
+      logPrefix: options.logPrefix || '[domain-cache]',
+      maxCacheSize: options.maxCacheSize || 10000 // Prevent memory leaks
+    };
+  }
+
+  /**
+   * Check if a domain was already detected in a previous scan
+   * @param {string} domain - Domain to check
+   * @returns {boolean} True if domain was already detected
+   */
+  isDomainAlreadyDetected(domain) {
+    if (!domain || typeof domain !== 'string') {
+      return false;
+    }
+
+    const isDetected = this.cache.has(domain);
+    
+    if (isDetected) {
+      this.stats.totalSkipped++;
+      this.stats.cacheHits++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache HIT: ${domain} (skipped)`));
+      }
+    } else {
+      this.stats.cacheMisses++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache MISS: ${domain} (processing)`));
+      }
+    }
+    
+    return isDetected;
+  }
+
+  /**
+   * Mark a domain as detected for future reference
+   * @param {string} domain - Domain to mark as detected
+   */
+  markDomainAsDetected(domain) {
+    if (!domain || typeof domain !== 'string') {
+      return false;
+    }
+
+    // Prevent cache from growing too large
+    if (this.cache.size >= this.options.maxCacheSize) {
+      this.clearOldestEntries(Math.floor(this.options.maxCacheSize * 0.1)); // Remove 10% of entries
+    }
+
+    const wasNew = !this.cache.has(domain);
+    this.cache.add(domain);
+    
+    if (wasNew) {
+      this.stats.totalDetected++;
+      
+      if (this.options.enableLogging) {
+        console.log(formatLogMessage('debug', `${this.options.logPrefix} Marked as detected: ${domain} (cache size: ${this.cache.size})`));
+      }
+    }
+    
+    return wasNew;
+  }
+
+  /**
+   * Clear oldest entries from cache (basic LRU simulation)
+   * Note: Set doesn't maintain insertion order in all Node.js versions,
+   * so this is a simple implementation that clears a portion of the cache
+   * @param {number} count - Number of entries to remove
+   */
+  clearOldestEntries(count) {
+    if (count <= 0) return;
+    
+    const entries = Array.from(this.cache);
+    const toRemove = entries.slice(0, count);
+    
+    toRemove.forEach(domain => this.cache.delete(domain));
+    
+    if (this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cleared ${toRemove.length} old entries, cache size now: ${this.cache.size}`));
+    }
+  }
+
+  /**
+   * Get cache statistics
+   * @returns {object} Cache statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      cacheSize: this.cache.size,
+      hitRate: this.stats.cacheHits > 0 ? 
+        (this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses) * 100).toFixed(2) + '%' : 
+        '0%'
+    };
+  }
+
+  /**
+   * Clear all cached domains
+   */
+  clear() {
+    const previousSize = this.cache.size;
+    this.cache.clear();
+    this.stats = {
+      totalDetected: 0,
+      totalSkipped: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    };
+    
+    if (this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Cache cleared (${previousSize} entries removed)`));
+    }
+  }
+
+  /**
+   * Get all cached domains (for debugging)
+   * @returns {Array<string>} Array of cached domains
+   */
+  getAllCachedDomains() {
+    return Array.from(this.cache);
+  }
+
+  /**
+   * Check if cache contains a specific domain (without updating stats)
+   * @param {string} domain - Domain to check
+   * @returns {boolean} True if domain exists in cache
+   */
+  has(domain) {
+    return this.cache.has(domain);
+  }
+
+  /**
+   * Remove a specific domain from cache
+   * @param {string} domain - Domain to remove
+   * @returns {boolean} True if domain was removed, false if it wasn't in cache
+   */
+  removeDomain(domain) {
+    const wasRemoved = this.cache.delete(domain);
+    
+    if (wasRemoved && this.options.enableLogging) {
+      console.log(formatLogMessage('debug', `${this.options.logPrefix} Removed from cache: ${domain}`));
+    }
+    
+    return wasRemoved;
+  }
+
+  /**
+   * Add multiple domains to cache at once
+   * @param {Array<string>} domains - Array of domains to add
+   * @returns {number} Number of domains actually added (excludes duplicates)
+   */
+  markMultipleDomainsAsDetected(domains) {
+    if (!Array.isArray(domains)) {
+      return 0;
+    }
+
+    let addedCount = 0;
+    domains.forEach(domain => {
+      if (this.markDomainAsDetected(domain)) {
+        addedCount++;
+      }
+    });
+
+    return addedCount;
+  }
+
+  /**
+   * Create bound helper functions for easy integration with existing code
+   * @returns {object} Object with bound helper functions
+   */
+  createHelpers() {
+    return {
+      isDomainAlreadyDetected: this.isDomainAlreadyDetected.bind(this),
+      markDomainAsDetected: this.markDomainAsDetected.bind(this),
+      getSkippedCount: () => this.stats.totalSkipped,
+      getCacheSize: () => this.cache.size,
+      getStats: this.getStats.bind(this)
+    };
+  }
+}
+
+/**
+ * Create a global domain cache instance (singleton pattern)
+ */
+let globalDomainCache = null;
+
+/**
+ * Get or create the global domain cache instance
+ * @param {object} options - Cache options
+ * @returns {DomainCache} Global cache instance
+ */
+function getGlobalDomainCache(options = {}) {
+  if (!globalDomainCache) {
+    globalDomainCache = new DomainCache(options);
+  }
+  return globalDomainCache;
+}
+
+/**
+ * Create helper functions that use the global cache
+ * @param {object} options - Cache options (only used if global cache doesn't exist)
+ * @returns {object} Helper functions bound to global cache
+ */
+function createGlobalHelpers(options = {}) {
+  const cache = getGlobalDomainCache(options);
+  return cache.createHelpers();
+}
+
+/**
+ * Reset the global cache (useful for testing or manual resets)
+ */
+function resetGlobalCache() {
+  if (globalDomainCache) {
+    globalDomainCache.clear();
+  }
+  globalDomainCache = null;
+}
+
+/**
+ * Legacy wrapper functions for backward compatibility
+ * These match the original function signatures from nwss.js
+ */
+
+/**
+ * Check if a domain was already detected (legacy wrapper)
+ * @param {string} domain - Domain to check
+ * @returns {boolean} True if domain was already detected
+ */
+function isDomainAlreadyDetected(domain) {
+  const cache = getGlobalDomainCache();
+  return cache.isDomainAlreadyDetected(domain);
+}
+
+/**
+ * Mark a domain as detected (legacy wrapper)
+ * @param {string} domain - Domain to mark as detected
+ */
+function markDomainAsDetected(domain) {
+  const cache = getGlobalDomainCache();
+  cache.markDomainAsDetected(domain);
+}
+
+/**
+ * Get total domains skipped (legacy wrapper)
+ * @returns {number} Number of domains skipped
+ */
+function getTotalDomainsSkipped() {
+  const cache = getGlobalDomainCache();
+  return cache.stats.totalSkipped;
+}
+
+/**
+ * Get detected domains cache size (legacy wrapper)
+ * @returns {number} Size of the detected domains cache
+ */
+function getDetectedDomainsCount() {
+  const cache = getGlobalDomainCache();
+  return cache.cache.size;
+}
+
+module.exports = {
+  // Main class
+  DomainCache,
+  
+  // Global cache functions
+  getGlobalDomainCache,
+  createGlobalHelpers,
+  resetGlobalCache,
+  
+  // Legacy wrapper functions for backward compatibility
+  isDomainAlreadyDetected,
+  markDomainAsDetected,
+  getTotalDomainsSkipped,
+  getDetectedDomainsCount
+};

package/lib/nettools.js

@@ -728,10 +728,31 @@ function createNetToolsHandler(config) {
   // Add separate deduplication caches for different lookup types
   const processedWhoisDomains = new Set();
   const processedDigDomains = new Set();
-  
+  // Add whois resolution caching to avoid redundant whois lookups
+  const whoisResultCache = new Map();
+  const WHOIS_CACHE_TTL = 900000; // 15 minutes cache TTL (whois data changes less frequently)
+  const MAX_CACHE_SIZE = 400; // Larger cache for whois due to longer TTL
+  // Size          Memory
+  // 100          ~900KB
+  // 200          1.8MB
+  // 300          2.6MB
+  // 400          3.4MB
+  // 500          4.2MB  
+  // Add DNS resolution caching to avoid redundant dig lookups
+  const digResultCache = new Map();
+  const DIG_CACHE_TTL = 300000; // 5 minutes cache TTL
+  const DIG_MAX_CACHE_SIZE = 400; // Smaller cache for dig due to shorter TTL
   
   return async function handleNetToolsCheck(domain, originalDomain) {
     // Helper function to log to BOTH console and debug file
+
+    // Check if domain was already detected (skip expensive operations)
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(domain)) {
+      if (forceDebug) {
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping already detected domain: ${domain}`);
+      }
+      return;
+    }
     
     // NOTE: The logToConsoleAndFile function needs to be declared INSIDE this function
     // so it has access to the closure variables (forceDebug, debugLogFile, fs) from the 
@@ -869,14 +890,43 @@ function createNetToolsHandler(config) {
         // Mark whois domain as being processed
         processedWhoisDomains.add(domain);
         
+        // Check whois cache first - cache key includes server for accuracy
         const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
+        const whoisCacheKey = `${domain}-${selectedServer || 'default'}`;
+        const now = Date.now();
+        let whoisResult = null;
         
-        if (forceDebug) {
-          const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
-          logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing whois lookup for ${domain}${serverInfo}`);
+        if (whoisResultCache.has(whoisCacheKey)) {
+          const cachedEntry = whoisResultCache.get(whoisCacheKey);
+          if (now - cachedEntry.timestamp < WHOIS_CACHE_TTL) {
+            if (forceDebug) {
+              const age = Math.round((now - cachedEntry.timestamp) / 1000);
+              const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
+            }
+            whoisResult = { 
+              ...cachedEntry.result,
+              // Add cache metadata
+              fromCache: true,
+              cacheAge: now - cachedEntry.timestamp,
+              originalTimestamp: cachedEntry.timestamp
+            };
+          } else {
+            // Cache expired, remove it
+            whoisResultCache.delete(whoisCacheKey);
+            if (forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${domain}, performing fresh lookup`);
+            }
+          }
         }
-               
-        try {
+        
+        // Perform fresh lookup if not cached
+        if (!whoisResult) {
+          if (forceDebug) {
+            const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
+            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
+          }
+        
           // Configure retry options based on site config or use defaults
           const retryOptions = {
             maxRetries: siteConfig.whois_max_retries || 2,
@@ -886,7 +936,47 @@ function createNetToolsHandler(config) {
             retryOnError: siteConfig.whois_retry_on_error === true // Default false
           };
           
-          const whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+          try {
+            whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+            
+            // Cache successful results (and certain types of failures)
+            if (whoisResult.success || 
+                (whoisResult.error && !whoisResult.isTimeout && 
+                 !whoisResult.error.toLowerCase().includes('connection') &&
+                 !whoisResult.error.toLowerCase().includes('network'))) {
+              
+              whoisResultCache.set(whoisCacheKey, {
+                result: whoisResult,
+                timestamp: now
+              });
+              
+              if (forceDebug) {
+                const cacheType = whoisResult.success ? 'successful' : 'failed';
+                const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
+                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${domain}${serverInfo}`);
+              }
+            }
+          } catch (whoisError) {
+            // Handle exceptions from whois lookup
+            if (forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
+              if (whoisError.stack) {
+                logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
+              }
+            }
+            
+            // Log whois exceptions in dry run mode
+            if (dryRunCallback && forceDebug) {
+              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+            }
+            // Continue with dig if configured
+            whoisResult = null; // Ensure we don't process a null result
+          }
+        }
+        
+        // Process whois result (whether from cache or fresh lookup)
+        if (whoisResult) {
           
           if (whoisResult.success) {
             // Check AND terms if configured
@@ -896,6 +986,7 @@ function createNetToolsHandler(config) {
                 dryRunCallback(domain, 'whois', 'AND logic', whoisTerms.join(', '), 'All terms found in whois data', {
                   server: whoisResult.whoisServer || 'default',
                   duration: whoisResult.duration,
+                  fromCache: whoisResult.fromCache || false,
                   retryAttempts: whoisResult.retryInfo?.totalAttempts || 1
                 });
               }
@@ -913,6 +1004,7 @@ function createNetToolsHandler(config) {
                 dryRunCallback(domain, 'whois', 'OR logic', matchedTerm, 'Term found in whois data', {
                   server: whoisResult.whoisServer || 'default',
                   duration: whoisResult.duration,
+                  fromCache: whoisResult.fromCache || false,
                   retryAttempts: whoisResult.retryInfo?.totalAttempts || 1
                 });
               }
@@ -925,7 +1017,9 @@ function createNetToolsHandler(config) {
             if (forceDebug) {
               const serverUsed = whoisResult.whoisServer ? ` (server: ${whoisResult.whoisServer})` : ' (default server)';
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} in ${whoisResult.duration}ms${retryInfo}`);             
+              const cacheInfo = whoisResult.fromCache ? ` [CACHED - ${Math.round(whoisResult.cacheAge / 1000)}s old]` : '';
+              const duration = whoisResult.fromCache ? `cached in 0ms` : `in ${whoisResult.duration}ms`;
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
               
               if (whoisResult.retryInfo && whoisResult.retryInfo.retriedAfterFailure) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Success after retry - servers attempted: [${whoisResult.retryInfo.serversAttempted.map(s => s || 'default').join(', ')}]`);
@@ -1001,20 +1095,21 @@ function createNetToolsHandler(config) {
             }
             // Don't return early - continue with dig if configured
           }
-        } catch (whoisError) {
-          if (forceDebug) {
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
-            if (whoisError.stack) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
+        }
+        
+        // Periodic whois cache cleanup to prevent memory leaks
+        if (whoisResultCache.size > MAX_CACHE_SIZE) {
+          const now = Date.now();
+          let cleanedCount = 0;
+          for (const [key, entry] of whoisResultCache.entries()) {
+            if (now - entry.timestamp > WHOIS_CACHE_TTL) {
+              whoisResultCache.delete(key);
+              cleanedCount++;
             }
           }
-          
-          // Log whois exceptions in dry run mode
-          if (dryRunCallback && forceDebug) {
-            logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+          if (forceDebug && cleanedCount > 0) {
+            logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cleaned ${cleanedCount} expired entries, cache size: ${whoisResultCache.size}`);
           }
-          // Continue with dig if configured
         }
       }
       
@@ -1031,7 +1126,37 @@ function createNetToolsHandler(config) {
         }
         
         try {
+          // Check dig cache first to avoid redundant dig operations
+          const digCacheKey = `${digDomain}-${digRecordType}`;
+          const now = Date.now();
+          let digResult = null;
+          
+          if (digResultCache.has(digCacheKey)) {
+            const cachedEntry = digResultCache.get(digCacheKey);
+            if (now - cachedEntry.timestamp < DIG_CACHE_TTL) {
+              if (forceDebug) {
+                logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Using cached result for ${digDomain} (${digRecordType}) [age: ${Math.round((now - cachedEntry.timestamp) / 1000)}s]`);
+              }
+              digResult = cachedEntry.result;
+            } else {
+              // Cache expired, remove it
+              digResultCache.delete(digCacheKey);
+            }
+          }
+          
+          if (!digResult) {
           const digResult = await digLookup(digDomain, digRecordType, 5000); // 5 second timeout for dig
+            
+            // Cache the result for future use
+            digResultCache.set(digCacheKey, {
+              result: digResult,
+              timestamp: now
+            });
+            
+            if (forceDebug && digResult.success) {
+              logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cached new result for ${digDomain} (${digRecordType})`);
+            }
+          }
           
           if (digResult.success) {
             // Check AND terms if configured
@@ -1091,6 +1216,21 @@ function createNetToolsHandler(config) {
             logToConsoleAndFile(`${messageColors.highlight('[dig-dryrun]')} Exception: ${digError.message}`);
           }
         }
+        
+        // Periodic dig cache cleanup to prevent memory leaks
+        if (digResultCache.size > DIG_MAX_CACHE_SIZE) {
+          const now = Date.now();
+          let cleanedCount = 0;
+          for (const [key, entry] of digResultCache.entries()) {
+            if (now - entry.timestamp > DIG_CACHE_TTL) {
+              digResultCache.delete(key);
+              cleanedCount++;
+            }
+          }
+          if (forceDebug && cleanedCount > 0) {
+            logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cleaned ${cleanedCount} expired entries, cache size: ${digResultCache.size}`);
+          }
+        }
       }
       
       // Domain matches if any of these conditions are true:

package/lib/output.js

@@ -1,5 +1,7 @@
 const fs = require('fs');
 const path = require('path');
+// Import domain cache functions for statistics
+const { getTotalDomainsSkipped } = require('./domain-cache');
 const { loadComparisonRules, filterUniqueRules } = require('./compare');
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
 
@@ -426,7 +428,6 @@ function writeOutput(lines, outputFile = null, silentMode = false) {
  * Main output handler that combines all output operations
  * @param {Array} results - Processing results from scanner
  * @param {object} config - Output configuration
- * @param {string[]} config.ignoreDomains - Domains to filter out from final output
  * @returns {object} Output statistics and file paths
  */
 function handleOutput(results, config = {}) {
@@ -440,7 +441,8 @@ function handleOutput(results, config = {}) {
     dumpUrls = false,
     adblockRulesLogFile = null,
     forceDebug = false,
-    ignoreDomains = []
+    ignoreDomains = [],
+    totalDomainsSkipped = null  // Allow override or get from cache
   } = config;
   
   // Handle append mode
@@ -572,7 +574,11 @@ function handleOutput(results, config = {}) {
   if (dumpUrls && adblockRulesLogFile) {
     logSuccess = writeOutput(outputLinesWithTitles, adblockRulesLogFile, silentMode);
   }
-  
+
+  // Get domain skip statistics from cache if not provided
+  const finalTotalDomainsSkipped = totalDomainsSkipped !== null ? 
+    totalDomainsSkipped : getTotalDomainsSkipped();
+
   return {
     success: mainSuccess && logSuccess,
     outputFile,
@@ -582,6 +588,8 @@ function handleOutput(results, config = {}) {
     filteredOutCount,
     totalLines: filteredOutputLines.length,
     outputLines: outputFile ? null : filteredOutputLines // Only return lines if not written to file
+    // Note: totalDomainsSkipped statistic is now available via getTotalDomainsSkipped() 
+    // and doesn't need to be passed through the output handler
   };
 }
 

package/lib/searchstring.js

@@ -250,11 +250,19 @@ function createCurlHandler(config) {
   } = config;
 
   return async function curlHandler(requestUrl) {
-    const respDomain = perSiteSubDomains ? (new URL(requestUrl)).hostname : getRootDomain(requestUrl);
     
     // Only process URLs that match our regex patterns
     const matchesRegex = regexes.some(re => re.test(requestUrl));
     if (!matchesRegex) return;
+
+    // Extract domain and check if already detected (skip expensive operations)
+    const reqDomain = perSiteSubDomains ? (new URL(requestUrl)).hostname : getRootDomain(requestUrl);
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(reqDomain)) {
+      if (forceDebug) {
+        console.log(`[debug][curl] Skipping already detected domain: ${reqDomain}`);
+      }
+      return;
+    }
     
     // Check if this is a first-party request (same domain as the URL being scanned)
     const currentUrlHostname = new URL(currentUrl).hostname;
@@ -283,11 +291,11 @@ function createCurlHandler(config) {
       
       // If NO searchstring is defined, match immediately (like browser behavior)
       if (!hasSearchString && !hasSearchStringAnd) {
-        if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
+        if (!reqDomain || matchesIgnoreDomain(reqDomain, ignoreDomains)) {
           return;
         }
         
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, resourceType);
+        addDomainToCollection(matchedDomains, addMatchedDomain, reqDomain, resourceType);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
@@ -317,11 +325,11 @@ function createCurlHandler(config) {
       const { found, matchedString, logicType } = searchContent(content, searchStrings, searchStringsAnd, '');
       
       if (found) {
-        if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
+        if (!reqDomain || matchesIgnoreDomain(reqDomain, ignoreDomains)) {
           return;
         }
         
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, resourceType);
+        addDomainToCollection(matchedDomains, addMatchedDomain, reqDomain, resourceType);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
@@ -387,6 +395,14 @@ function createResponseHandler(config) {
     const matchesRegex = regexes.some(re => re.test(respUrl));
     if (!matchesRegex) return;
     
+    // Extract domain and check if already detected (skip expensive operations)
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(respDomain)) {
+      if (forceDebug) {
+        console.log(`[debug] Skipping response analysis for already detected domain: ${respDomain}`);
+      }
+      return;
+    }
+
     // Check if this is a first-party response (same domain as the URL being scanned)
     const currentUrlHostname = new URL(currentUrl).hostname;
     const responseHostname = new URL(respUrl).hostname;

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.41 ===
+// === Network scanner script (nwss.js) v1.0.43 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -27,17 +27,23 @@ const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvaila
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // Colorize various text when used
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./lib/colorize');
+// Domain detection cache for performance optimization
+const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.41'; // Script version
+const VERSION = '1.0.43'; // Script version
 
 // get startTime
 const startTime = Date.now();
 
+// Initialize domain cache helpers with debug logging if enabled
+const domainCacheOptions = { enableLogging: false }; // Set to true for cache debug logs
+const { isDomainAlreadyDetected, markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
+
 // --- Command-Line Argument Parsing ---
 const args = process.argv.slice(2);
 
@@ -1517,6 +1523,9 @@ function setupFrameHandling(page, forceDebug) {
           return; // Skip adding this domain
         }
       }
+
+      // Mark domain as detected for future reference
+      markDomainAsDetected(domain);
       
         if (matchedDomains instanceof Map) {
           if (!matchedDomains.has(domain)) {
@@ -1734,6 +1743,14 @@ function setupFrameHandling(page, forceDebug) {
              }
             } else if (hasNetTools && !hasSearchString && !hasSearchStringAnd) {
              // If nettools are configured (whois/dig), perform checks on the domain
+             // Skip nettools check if domain was already detected
+             if (isDomainAlreadyDetected(reqDomain)) {
+               if (forceDebug) {
+                 console.log(formatLogMessage('debug', `Skipping nettools check for already detected domain: ${reqDomain}`));
+               }
+               break; // Skip to next URL
+             }
+             
              if (forceDebug) {
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${re} and resourceType ${resourceType}, queued for nettools check`));
              }
@@ -1767,6 +1784,7 @@ function setupFrameHandling(page, forceDebug) {
                dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
                matchedDomains,
                addMatchedDomain,
+               isDomainAlreadyDetected,
                currentUrl,
                getRootDomain,
                siteConfig,
@@ -1781,6 +1799,13 @@ function setupFrameHandling(page, forceDebug) {
             setImmediate(() => netToolsHandler(reqDomain, originalDomain));
            } else {
              // If searchstring or searchstring_and IS defined (with or without nettools), queue for content checking
+             // Skip searchstring check if domain was already detected
+             if (isDomainAlreadyDetected(reqDomain)) {
+               if (forceDebug) {
+                 console.log(formatLogMessage('debug', `Skipping searchstring check for already detected domain: ${reqDomain}`));
+               }
+               break; // Skip to next URL
+             }
              if (forceDebug) {
                const searchType = hasSearchStringAnd ? 'searchstring_and' : 'searchstring';
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${re} and resourceType ${resourceType}, queued for ${searchType} content search`));
@@ -1808,6 +1833,7 @@ function setupFrameHandling(page, forceDebug) {
                    regexes,
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
+                   isDomainAlreadyDetected,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -1838,6 +1864,7 @@ function setupFrameHandling(page, forceDebug) {
                    regexes,
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
+                   isDomainAlreadyDetected,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -1876,6 +1903,7 @@ function setupFrameHandling(page, forceDebug) {
          regexes,
          matchedDomains,
          addMatchedDomain, // Pass the helper function
+         isDomainAlreadyDetected,
          currentUrl,
          perSiteSubDomains,
          ignoreDomains,
@@ -2462,6 +2490,8 @@ function setupFrameHandling(page, forceDebug) {
   const totalMatches = results.reduce((sum, r) => sum + (r.rules ? r.rules.length : 0), 0);
 
   // Debug: Show output format being used
+  const totalDomainsSkipped = getTotalDomainsSkipped();
+  const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
       localhostMode,
@@ -2476,6 +2506,7 @@ function setupFrameHandling(page, forceDebug) {
     };
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
+     console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
   }
   
   // Compress log files if --compress-logs is enabled
@@ -2567,6 +2598,9 @@ function setupFrameHandling(page, forceDebug) {
     } else if (outputResult.totalRules > 0 && dryRunMode) {
       console.log(messageColors.success('Found') + ` ${outputResult.totalRules} total matches across all URLs`);
     }
+    if (totalDomainsSkipped > 0) {
+      console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.41",
+  "version": "1.0.43",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {