@fanboynz/network-scanner 1.0.57 → 1.0.58

package/lib/smart-cache.js

@@ -14,6 +14,11 @@ const { formatLogMessage } = require('./colorize');
  */
 class SmartCache {
   constructor(options = {}) {
+    // Calculate dynamic values first
+    const concurrency = options.concurrency || 6;
+    const optimalHeapLimit = this._calculateOptimalHeapLimit(concurrency);
+    const checkInterval = this._calculateCheckInterval(concurrency);
+
     this.options = {
       maxSize: options.maxSize || 5000,
       ttl: options.ttl || 1000 * 60 * 60, // 1 hour default
@@ -24,8 +29,18 @@ class SmartCache {
       persistencePath: options.persistencePath || '.cache',
       forceDebug: options.forceDebug || false,
       autoSave: options.autoSave !== false,
-      autoSaveInterval: options.autoSaveInterval || 60000 // 1 minute
+      autoSaveInterval: options.autoSaveInterval || 60000, // 1 minute
+      maxHeapUsage: options.maxHeapUsage || optimalHeapLimit,
+      memoryCheckInterval: options.memoryCheckInterval || checkInterval,
+      concurrency: concurrency,
+      aggressiveMode: options.aggressiveMode || false
     };
+
+    // Add save debouncing
+    this.lastSaveTime = 0;
+    this.saveInProgress = false;
+    this.saveTimeout = null;
+    this.pendingSave = false;
     
     // Initialize cache layers
     this._initializeCaches();
@@ -42,6 +57,34 @@ class SmartCache {
     if (this.options.enablePersistence && this.options.autoSave) {
       this._setupAutoSave();
     }
+    
+    // Set up memory monitoring
+    this.memoryCheckInterval = setInterval(() => {
+      this._checkMemoryPressure();
+    }, this.options.memoryCheckInterval);
+  }
+  
+  /**
+   * Calculate optimal heap limit based on concurrency
+   * @private
+   */
+  _calculateOptimalHeapLimit(concurrency) {
+    // Base cache needs: 100MB
+    // Per concurrent connection: ~75MB average
+    // Safety margin: 50%
+    const baseCacheMemory = 100 * 1024 * 1024; // 100MB
+    const perConnectionMemory = 75 * 1024 * 1024; // 75MB
+    const totalEstimated = baseCacheMemory + (concurrency * perConnectionMemory);
+    return Math.round(totalEstimated * 0.4); // Cache should use max 40% of estimated total
+  }
+  
+  /**
+   * Calculate check interval based on concurrency
+   * @private
+   */
+  _calculateCheckInterval(concurrency) {
+    // Higher concurrency = more frequent checks
+    return Math.max(5000, 30000 - (concurrency * 1000)); // 5s min, scales down with concurrency
   }
   
   /**
@@ -57,29 +100,43 @@ class SmartCache {
       updateAgeOnHas: false
     });
     
-    // Pattern matching results cache
+    // Pattern matching results cache - reduce size for high concurrency
+    const patternCacheSize = this.options.concurrency > 10 ? 500 : 1000;
     this.patternCache = new LRUCache({
-      max: 1000,
+      max: patternCacheSize,
       ttl: this.options.ttl * 2 // Patterns are more stable
     });
     
-    // Response content cache for searchstring operations
+    // Response content cache - aggressive limits for high concurrency
+    const responseCacheSize = this.options.concurrency > 10 ? 50 : 200;
+    const responseCacheMemory = this.options.concurrency > 10 ? 20 * 1024 * 1024 : 50 * 1024 * 1024;
     this.responseCache = new LRUCache({
-      max: 200,
+      max: responseCacheSize,
       ttl: 1000 * 60 * 30, // 30 minutes for response content
-      maxSize: 50 * 1024 * 1024, // 50MB max cache size
+      maxSize: responseCacheMemory,
       sizeCalculation: (value) => value.length
     });
     
+    // Disable response cache entirely for very high concurrency
+    if (this.options.concurrency > 15 || this.options.aggressiveMode) {
+      this.options.enableResponseCache = false;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Response cache disabled for high concurrency (${this.options.concurrency})`
+        ));
+      }
+    }
+
     // WHOIS/DNS results cache
     this.netToolsCache = new LRUCache({
       max: 500,
       ttl: 1000 * 60 * 60 * 24 // 24 hours for WHOIS/DNS
     });
     
-    // Similarity cache for expensive string comparisons
+    // Similarity cache - reduce for high concurrency
+    const similarityCacheSize = this.options.concurrency > 10 ? 1000 : 2000;
     this.similarityCache = new LRUCache({
-      max: 2000,
+      max: similarityCacheSize,
       ttl: this.options.ttl
     });
     
@@ -107,6 +164,9 @@ class SmartCache {
       regexCacheHits: 0,
       persistenceLoads: 0,
       persistenceSaves: 0,
+      memoryPressureEvents: 0,
+      memoryWarnings: 0,
+      responseCacheSkips: 0,
       startTime: Date.now()
     };
   }
@@ -274,6 +334,21 @@ class SmartCache {
   cacheResponse(url, content) {
     if (!this.options.enableResponseCache) return;
     
+    // Skip response caching entirely for very high concurrency
+    if (this.options.concurrency > 12) {
+      this.stats.responseCacheSkips++;
+      return;
+    }
+    
+    // Check memory before caching large content
+    const memUsage = process.memoryUsage();
+    const threshold = this.options.concurrency > 10 ? 0.7 : 0.8; // Lower threshold for high concurrency
+    if (memUsage.heapUsed > this.options.maxHeapUsage * threshold) {
+      this.stats.responseCacheSkips++;
+      this._logMemorySkip('response cache');
+      return;
+    }
+    
     // Only cache if content is reasonable size
     if (content && content.length < 5 * 1024 * 1024) { // 5MB limit per response
       this.responseCache.set(url, content);
@@ -350,7 +425,85 @@ class SmartCache {
     this.stats.similarityMisses++;
     return null;
   }
+
+   /**
+   * Monitor memory usage and proactively manage caches
+   * @private
+   */
+  _checkMemoryPressure() {
+    const memUsage = process.memoryUsage();
+    const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
+    const maxHeapMB = Math.round(this.options.maxHeapUsage / 1024 / 1024);
+    const usagePercent = (memUsage.heapUsed / this.options.maxHeapUsage) * 100;
+    
+    // Adjust thresholds based on concurrency
+    const criticalThreshold = this.options.concurrency > 10 ? 0.85 : 1.0;
+    const warningThreshold = this.options.concurrency > 10 ? 0.70 : 0.85;
+    const infoThreshold = this.options.concurrency > 10 ? 0.60 : 0.75;
+    
+    // Critical threshold - aggressive cleanup
+    if (memUsage.heapUsed > this.options.maxHeapUsage * criticalThreshold) {
+      this._performMemoryCleanup('critical', heapUsedMB, maxHeapMB);
+      return true;
+    }
+    
+    // Warning threshold - moderate cleanup
+    if (memUsage.heapUsed > this.options.maxHeapUsage * warningThreshold) {
+      this._performMemoryCleanup('warning', heapUsedMB, maxHeapMB);
+      return true;
+    }
+    
+    // Info threshold - log only
+    if (memUsage.heapUsed > this.options.maxHeapUsage * infoThreshold) {
+      this.stats.memoryWarnings++;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Memory info: ${heapUsedMB}MB/${maxHeapMB}MB (${usagePercent.toFixed(1)}%)`
+        ));
+      }
+    }
+    
+    return false;
+  }
   
+  /**
+   * Perform memory cleanup based on severity
+   * @private
+   */
+  _performMemoryCleanup(level, heapUsedMB, maxHeapMB) {
+    this.stats.memoryPressureEvents++;
+    
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Memory ${level}: ${heapUsedMB}MB/${maxHeapMB}MB, performing cleanup...`
+      ));
+    }
+    
+    if (level === 'critical' || this.options.concurrency > 12) {
+      // Aggressive cleanup - clear volatile caches
+      this.responseCache.clear();
+      this.patternCache.clear();
+      this.similarityCache.clear();
+      
+      // For very high concurrency, also trim domain cache
+      if (this.options.concurrency > 15) {
+        const currentSize = this.domainCache.size;
+        this.domainCache.clear();
+        if (this.options.forceDebug) {
+          console.log(formatLogMessage('debug', `[SmartCache] Cleared ${currentSize} domain cache entries`));
+        }
+      }
+    } else if (level === 'warning') {
+      // Moderate cleanup - clear largest cache
+      this.responseCache.clear();
+    }
+    
+    // Force garbage collection if available
+    if (global.gc) {
+      global.gc();
+    }
+  }  
+
   /**
    * Get cache statistics
    * @returns {Object} Statistics object
@@ -364,6 +517,9 @@ class SmartCache {
       (this.stats.responseHits + this.stats.responseMisses) || 0;
     const netToolsHitRate = this.stats.netToolsHits / 
       (this.stats.netToolsHits + this.stats.netToolsMisses) || 0;
+      
+    
+    const memUsage = process.memoryUsage();
     
     return {
       ...this.stats,
@@ -380,7 +536,11 @@ class SmartCache {
       regexCacheSize: this.regexCache.size,
       totalCacheEntries: this.domainCache.size + this.patternCache.size + 
         this.responseCache.size + this.netToolsCache.size + 
-        this.similarityCache.size + this.regexCache.size
+        this.similarityCache.size + this.regexCache.size,
+      memoryUsageMB: Math.round(memUsage.heapUsed / 1024 / 1024),
+      memoryMaxMB: Math.round(this.options.maxHeapUsage / 1024 / 1024),
+      memoryUsagePercent: ((memUsage.heapUsed / this.options.maxHeapUsage) * 100).toFixed(1) + '%',
+      responseCacheMemoryMB: Math.round((this.responseCache.calculatedSize || 0) / 1024 / 1024)
     };
   }
   
@@ -401,6 +561,18 @@ class SmartCache {
     }
   }
   
+   /**
+   * Helper method to log memory-related cache skips
+   * @private
+   */
+  _logMemorySkip(operation) {
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Skipping ${operation} due to memory pressure`
+      ));
+    }
+  }  
+  
   /**
    * Load persistent cache from disk
    * @private
@@ -464,6 +636,34 @@ class SmartCache {
    */
   savePersistentCache() {
     if (!this.options.enablePersistence) return;
+
+    // Prevent concurrent saves
+    if (this.saveInProgress) {
+      this.pendingSave = true;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', '[SmartCache] Save in progress, marking pending...'));
+      }
+      return;
+    }
+    
+    // Debounce saves - don't save more than once every 10 seconds
+    const now = Date.now();
+    if (now - this.lastSaveTime < 10000) {
+      // Schedule a delayed save if none is pending
+      if (!this.saveTimeout && !this.pendingSave) {
+        this.pendingSave = true;
+        this.saveTimeout = setTimeout(() => {
+          this.saveTimeout = null;
+          if (this.pendingSave) {
+            this.pendingSave = false;
+            this.savePersistentCache();
+          }
+        }, 10000 - (now - this.lastSaveTime));
+      }
+      return;
+    }
+    this.saveInProgress = true;
+    this.lastSaveTime = now;
     
     const cacheDir = this.options.persistencePath;
     const cacheFile = path.join(cacheDir, 'smart-cache.json');
@@ -475,7 +675,7 @@ class SmartCache {
       }
       
       const data = {
-        timestamp: Date.now(),
+        timestamp: now,
         domainCache: Array.from(this.domainCache.entries()),
         netToolsCache: Array.from(this.netToolsCache.entries()),
         stats: this.stats
@@ -495,6 +695,14 @@ class SmartCache {
           `[SmartCache] Failed to save cache: ${err.message}`
         ));
       }
+    } finally {
+      this.saveInProgress = false;
+      
+      // Process any pending saves
+      if (this.pendingSave && !this.saveTimeout) {
+        this.pendingSave = false;
+        setTimeout(() => this.savePersistentCache(), 1000);
+      }
     }
   }
   
@@ -512,9 +720,16 @@ class SmartCache {
    * Clean up resources
    */
   destroy() {
+    if (this.memoryCheckInterval) {
+      clearInterval(this.memoryCheckInterval);
+    }
     if (this.autoSaveInterval) {
       clearInterval(this.autoSaveInterval);
     }
+    if (this.saveTimeout) {
+      clearTimeout(this.saveTimeout);
+      this.saveTimeout = null;
+    }
     
     // Save cache one last time
     if (this.options.enablePersistence) {
@@ -523,6 +738,92 @@ class SmartCache {
     
     this.clear();
   }
+  
+  /**
+   * Clear persistent cache files and directories
+   * @param {Object} options - Clear options
+   * @param {boolean} options.silent - Suppress console output
+   * @param {boolean} options.forceDebug - Enable debug logging
+   * @returns {Object} Clear operation results
+   */
+  static clearPersistentCache(options = {}) {
+    const { silent = false, forceDebug = false, cachePath = '.cache' } = options;
+    
+    const cachePaths = [
+      cachePath,
+      path.join(cachePath, 'smart-cache.json'),
+      // Add other potential cache files here if needed
+    ];
+    
+    let clearedItems = 0;
+    let totalSize = 0;
+    const clearedFiles = [];
+    const errors = [];
+    
+    if (!silent) {
+      console.log(`\n???  Clearing cache...`);
+    }
+    
+    for (const currentCachePath of cachePaths) {
+      if (fs.existsSync(currentCachePath)) {
+        try {
+          const stats = fs.statSync(currentCachePath);
+          if (stats.isDirectory()) {
+            // Calculate total size of directory contents
+            const files = fs.readdirSync(currentCachePath);
+            for (const file of files) {
+              const filePath = path.join(currentCachePath, file);
+              if (fs.existsSync(filePath)) {
+                totalSize += fs.statSync(filePath).size;
+              }
+            }
+            fs.rmSync(currentCachePath, { recursive: true, force: true });
+            clearedItems++;
+            clearedFiles.push({ type: 'directory', path: currentCachePath, size: totalSize });
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Cleared cache directory: ${currentCachePath}`));
+            }
+          } else {
+            totalSize += stats.size;
+            fs.unlinkSync(currentCachePath);
+            clearedItems++;
+            clearedFiles.push({ type: 'file', path: currentCachePath, size: stats.size });
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Cleared cache file: ${currentCachePath}`));
+            }
+          }
+        } catch (clearErr) {
+          errors.push({ path: currentCachePath, error: clearErr.message });
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Failed to clear ${currentCachePath}: ${clearErr.message}`));
+          }
+        }
+      }
+    }
+    
+    const result = {
+      success: errors.length === 0,
+      clearedItems,
+      totalSize,
+      sizeMB: (totalSize / 1024 / 1024).toFixed(2),
+      clearedFiles,
+      errors
+    };
+    
+    if (!silent) {
+      if (clearedItems > 0) {
+        console.log(`? Cache cleared: ${clearedItems} item(s), ${result.sizeMB}MB freed`);
+      } else {
+        console.log(`??  No cache files found to clear`);
+      }
+      
+      if (errors.length > 0) {
+        console.warn(`??  ${errors.length} error(s) occurred during cache clearing`);
+      }
+    }
+    
+    return result;
+  }
 }
 
 /**
@@ -541,11 +842,16 @@ function createSmartCache(config = {}) {
     persistencePath: config.cache_path || '.cache',
     forceDebug: config.forceDebug || false,
     autoSave: config.cache_autosave !== false,
-    autoSaveInterval: (config.cache_autosave_minutes || 1) * 60 * 1000
+    autoSaveInterval: (config.cache_autosave_minutes || 1) * 60 * 1000,
+    maxHeapUsage: config.cache_max_heap_mb ? config.cache_max_heap_mb * 1024 * 1024 : undefined,
+    memoryCheckInterval: (config.cache_memory_check_seconds || 30) * 1000,
+    concurrency: config.max_concurrent_sites || 6,
+    aggressiveMode: config.cache_aggressive_mode === true
   });
 }
 
 module.exports = {
   SmartCache,
-  createSmartCache
+  createSmartCache,
+  clearPersistentCache: SmartCache.clearPersistentCache
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.57 ===
+// === Network scanner script (nwss.js) v1.0.58 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -34,13 +34,14 @@ const { performPageInteraction, createInteractionConfig } = require('./lib/inter
 // Domain detection cache for performance optimization
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
+const { clearPersistentCache } = require('./lib/smart-cache');
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.57'; // Script version
+const VERSION = '1.0.58'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -102,6 +103,8 @@ const validateConfig = args.includes('--validate-config');
 const validateRules = args.includes('--validate-rules');
 const testValidation = args.includes('--test-validation');
 let cleanRules = args.includes('--clean-rules');
+const clearCache = args.includes('--clear-cache');
+const ignoreCache = args.includes('--ignore-cache');
 
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
@@ -224,6 +227,15 @@ if (args.includes('--version')) {
   process.exit(0);
 }
 
+// Handle --clear-cache before config loading (uses default cache path)
+if (clearCache && !dryRunMode) {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: '.cache' // Default path, will be updated after config loads if needed
+  });
+}
+
 // Handle validation-only operations before main help
 if (testValidation) {
   console.log(`\n${messageColors.processing('Running domain validation tests...')}`);
@@ -360,6 +372,8 @@ Validation Options:
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
   --test-validation              Run domain validation tests and exit
+  --clear-cache                  Clear persistent cache before scanning (improves fresh start performance)
+  --ignore-cache                 Bypass all smart caching functionality during scanning
   
 Global config.json options:
   ignoreDomains: ["domain.com", "*.ads.com"]     Domains to completely ignore (supports wildcards)
@@ -551,15 +565,40 @@ const RESOURCE_CLEANUP_INTERVAL = (() => {
   return 180;
 })();
 
-// Initialize smart cache system AFTER config is loaded
+// Perform cache clear after config is loaded for custom cache paths
+if (clearCache && dryRunMode) {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: config.cache_path || '.cache'
+  });
+}
+
+// Also clear for custom cache paths in normal mode if not already cleared
+if (clearCache && !dryRunMode && config.cache_path && config.cache_path !== '.cache') {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: config.cache_path
+  });
+}
+
+// Initialize smart cache system AFTER config is loaded (unless --ignore-cache is used)
+if (ignoreCache) {
+  smartCache = null;
+  if (forceDebug) console.log(formatLogMessage('debug', 'Smart cache disabled by --ignore-cache flag'));
+} else {
 smartCache = createSmartCache({
   ...config,
   forceDebug,
-  cache_persistence: config.cache_persistence !== false, // Enable by default
-  cache_autosave: config.cache_autosave !== false,
+  max_concurrent_sites: MAX_CONCURRENT_SITES,  // Pass concurrency info
+  cache_aggressive_mode: MAX_CONCURRENT_SITES > 12,  // Auto-enable for high concurrency
+  cache_persistence: false, // Disable persistence completely
+  cache_autosave: false, // Disable auto-save completely
   cache_autosave_minutes: config.cache_autosave_minutes || 1,
   cache_max_size: config.cache_max_size || 5000
 });
+}
 
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
@@ -1503,7 +1542,7 @@ function setupFrameHandling(page, forceDebug) {
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
        const ignoreSimilarIgnoredDomains = siteConfig.ignore_similar_ignored_domains !== undefined ? siteConfig.ignore_similar_ignored_domains : ignore_similar_ignored_domains;
        
-       // Use smart cache's similarity cache for performance
+       // Use smart cache's similarity cache for performance (if cache is enabled)
        if (ignoreSimilarEnabled && smartCache) {
          const existingDomains = matchedDomains instanceof Map 
            ? Array.from(matchedDomains.keys()).filter(key => !['dryRunMatches', 'dryRunNetTools', 'dryRunSearchString'].includes(key))
@@ -1522,14 +1561,14 @@ function setupFrameHandling(page, forceDebug) {
            // If no cached similarity exists, calculate and cache it
            if (cachedSimilarity === null) {
              const similarity = calculateSimilarity(domain, existingDomain);
-             if (smartCache) {
+             if (smartCache && !ignoreCache) {
                smartCache.cacheSimilarity(domain, existingDomain, similarity);
              }
            }
          }
        }
 
-       // Check smart cache first
+       // Check smart cache first (if cache is enabled)
        const context = {
          filterRegex: siteConfig.filterRegex,
          searchString: siteConfig.searchstring,
@@ -1581,7 +1620,7 @@ function setupFrameHandling(page, forceDebug) {
       // Mark full subdomain as detected for future reference
       markDomainAsDetected(cacheKey);
       
-      // Also mark in smart cache with context
+      // Also mark in smart cache with context (if cache is enabled)
       if (smartCache) {
         smartCache.markDomainProcessed(domain, context, { resourceType, fullSubdomain });
       }
@@ -1831,7 +1870,7 @@ function setupFrameHandling(page, forceDebug) {
              }
              
              // Create and execute nettools handler
-             // Check smart cache for nettools results
+             // Check smart cache for nettools results (if cache is enabled)
              const cachedWhois = smartCache ? smartCache.getCachedNetTools(reqDomain, 'whois') : null;
              const cachedDig = smartCache ? smartCache.getCachedNetTools(reqDomain, 'dig', digRecordType) : null;
              
@@ -1839,7 +1878,7 @@ function setupFrameHandling(page, forceDebug) {
                console.log(formatLogMessage('debug', `[SmartCache] Using cached nettools results for ${reqDomain}`));
              }
              
-             // Create nettools handler with cache callbacks
+             // Create nettools handler with cache callbacks (if cache is enabled)
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
@@ -1857,7 +1896,7 @@ function setupFrameHandling(page, forceDebug) {
                matchedDomains,
                addMatchedDomain,
                isDomainAlreadyDetected,
-               // Add cache callbacks if smart cache is available
+               // Add cache callbacks if smart cache is available and caching is enabled
                onWhoisResult: smartCache ? (domain, result) => {
                  smartCache.cacheNetTools(domain, 'whois', result);
                } : undefined,
@@ -1905,7 +1944,7 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check response cache first if smart cache is available
+             // Check response cache first if smart cache is available and caching is enabled
              const cachedContent = smartCache ? smartCache.getCachedResponse(reqUrl) : null;
              
              if (cachedContent && forceDebug) {
@@ -1922,7 +1961,7 @@ function setupFrameHandling(page, forceDebug) {
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
                    isDomainAlreadyDetected,
-                   onContentFetched: smartCache ? (url, content) => {
+                   onContentFetched: smartCache && !ignoreCache ? (url, content) => {
                      smartCache.cacheResponse(url, content);
                    } : undefined,
                    currentUrl,
@@ -2587,7 +2626,7 @@ function setupFrameHandling(page, forceDebug) {
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
      console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
-     // Log smart cache statistics
+     // Log smart cache statistics (if cache is enabled)
     if (smartCache) {
     const cacheStats = smartCache.getStats();  
     console.log(formatLogMessage('debug', '=== Smart Cache Statistics ==='));
@@ -2677,7 +2716,7 @@ function setupFrameHandling(page, forceDebug) {
   const seconds = totalSeconds % 60;
 
   // Final summary report with timing and success statistics
-  // Clean up smart cache
+  // Clean up smart cache (if it exists)
   if (smartCache) {
     smartCache.destroy();
   }
@@ -2699,6 +2738,9 @@ function setupFrameHandling(page, forceDebug) {
     if (totalDomainsSkipped > 0) {
       console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
     }
+    if (ignoreCache && forceDebug) {
+      console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.57",
+  "version": "1.0.58",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {