@fanboynz/network-scanner 1.0.55 → 1.0.57

package/.github/workflows/npm-publish.yml

@@ -2,18 +2,18 @@ name: Publish to NPM
 on:
   push:
     branches: [ main, master ]
-    paths-ignore:
-      - 'README.md'
-      - 'docs/**'
-      - '.gitignore'
 
 jobs:
   publish:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write  # This is key!
+      
     steps:
       - uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
           
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -21,22 +21,22 @@ jobs:
           node-version: '18'
           registry-url: 'https://registry.npmjs.org'
           
-      - name: Install dependencies
-        run: npm ci
-        
-      - name: Run linting
-        run: npm run lint
-        
+      - run: npm ci
+      - run: npm run lint
+      
       - name: Configure git
         run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
           
-      - name: Version bump and publish
+      - name: Version and publish
         run: |
           npm version patch
           npm publish
-          git push --follow-tags
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          
+      - name: Push changes
+        run: git push --follow-tags
+        env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/lib/smart-cache.js

@@ -0,0 +1,551 @@
+/**
+ * Smart Cache Module - Intelligent multi-layer caching system for network scanner
+ * Provides context-aware caching for domains, patterns, responses, and network tools
+ */
+
+const { LRUCache } = require('lru-cache');
+const fs = require('fs');
+const path = require('path');
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * SmartCache - Intelligent caching system with multiple cache layers
+ * @class
+ */
+class SmartCache {
+  constructor(options = {}) {
+    this.options = {
+      maxSize: options.maxSize || 5000,
+      ttl: options.ttl || 1000 * 60 * 60, // 1 hour default
+      enablePatternCache: options.enablePatternCache !== false,
+      enableResponseCache: options.enableResponseCache !== false,
+      enableWhoisCache: options.enableWhoisCache !== false,
+      enablePersistence: options.enablePersistence === true,
+      persistencePath: options.persistencePath || '.cache',
+      forceDebug: options.forceDebug || false,
+      autoSave: options.autoSave !== false,
+      autoSaveInterval: options.autoSaveInterval || 60000 // 1 minute
+    };
+    
+    // Initialize cache layers
+    this._initializeCaches();
+    
+    // Initialize statistics
+    this._initializeStats();
+    
+    // Load persistent cache if enabled
+    if (this.options.enablePersistence) {
+      this._loadPersistentCache();
+    }
+    
+    // Set up auto-save if enabled
+    if (this.options.enablePersistence && this.options.autoSave) {
+      this._setupAutoSave();
+    }
+  }
+  
+  /**
+   * Initialize all cache layers
+   * @private
+   */
+  _initializeCaches() {
+    // Domain detection cache with TTL
+    this.domainCache = new LRUCache({
+      max: this.options.maxSize,
+      ttl: this.options.ttl,
+      updateAgeOnGet: true,
+      updateAgeOnHas: false
+    });
+    
+    // Pattern matching results cache
+    this.patternCache = new LRUCache({
+      max: 1000,
+      ttl: this.options.ttl * 2 // Patterns are more stable
+    });
+    
+    // Response content cache for searchstring operations
+    this.responseCache = new LRUCache({
+      max: 200,
+      ttl: 1000 * 60 * 30, // 30 minutes for response content
+      maxSize: 50 * 1024 * 1024, // 50MB max cache size
+      sizeCalculation: (value) => value.length
+    });
+    
+    // WHOIS/DNS results cache
+    this.netToolsCache = new LRUCache({
+      max: 500,
+      ttl: 1000 * 60 * 60 * 24 // 24 hours for WHOIS/DNS
+    });
+    
+    // Similarity cache for expensive string comparisons
+    this.similarityCache = new LRUCache({
+      max: 2000,
+      ttl: this.options.ttl
+    });
+    
+    // Regex compilation cache
+    this.regexCache = new Map();
+  }
+  
+  /**
+   * Initialize statistics tracking
+   * @private
+   */
+  _initializeStats() {
+    this.stats = {
+      hits: 0,
+      misses: 0,
+      patternHits: 0,
+      patternMisses: 0,
+      responseHits: 0,
+      responseMisses: 0,
+      netToolsHits: 0,
+      netToolsMisses: 0,
+      similarityHits: 0,
+      similarityMisses: 0,
+      regexCompilations: 0,
+      regexCacheHits: 0,
+      persistenceLoads: 0,
+      persistenceSaves: 0,
+      startTime: Date.now()
+    };
+  }
+  
+  /**
+   * Check if domain should be skipped based on smart caching
+   * @param {string} domain - Domain to check
+   * @param {Object} context - Processing context
+   * @returns {boolean} True if domain should be skipped
+   */
+  shouldSkipDomain(domain, context = {}) {
+    const cacheKey = this._generateCacheKey(domain, context);
+    
+    if (this.domainCache.has(cacheKey)) {
+      this.stats.hits++;
+      if (this.options.forceDebug) {
+        const cached = this.domainCache.get(cacheKey);
+        const age = Date.now() - cached.timestamp;
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Cache hit for ${domain} (age: ${Math.round(age/1000)}s, context: ${JSON.stringify(context)})`
+        ));
+      }
+      return true;
+    }
+    
+    this.stats.misses++;
+    return false;
+  }
+  
+  /**
+   * Mark domain as processed with context
+   * @param {string} domain - Domain to mark
+   * @param {Object} context - Processing context
+   * @param {Object} metadata - Additional metadata to store
+   */
+  markDomainProcessed(domain, context = {}, metadata = {}) {
+    const cacheKey = this._generateCacheKey(domain, context);
+    this.domainCache.set(cacheKey, {
+      timestamp: Date.now(),
+      metadata,
+      context,
+      domain
+    });
+    
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Marked ${domain} as processed (context: ${JSON.stringify(context)})`
+      ));
+    }
+  }
+  
+  /**
+   * Generate cache key with context awareness
+   * @param {string} domain - Domain
+   * @param {Object} context - Context object
+   * @returns {string} Cache key
+   * @private
+   */
+  _generateCacheKey(domain, context) {
+    const { filterRegex, searchString, resourceType, nettools } = context;
+    const components = [
+      domain,
+      filterRegex || '',
+      searchString || '',
+      resourceType || '',
+      nettools ? 'nt' : ''
+    ].filter(Boolean);
+    
+    return components.join(':');
+  }
+  
+  /**
+   * Get or compile regex pattern with caching
+   * @param {string} pattern - Regex pattern string
+   * @returns {RegExp} Compiled regex
+   */
+  getCompiledRegex(pattern) {
+    if (!this.regexCache.has(pattern)) {
+      this.stats.regexCompilations++;
+      try {
+        const regex = new RegExp(pattern.replace(/^\/(.*)\/$/, '$1'));
+        this.regexCache.set(pattern, regex);
+      } catch (err) {
+        if (this.options.forceDebug) {
+          console.log(formatLogMessage('debug', 
+            `[SmartCache] Failed to compile regex: ${pattern}`
+          ));
+        }
+        return null;
+      }
+    } else {
+      this.stats.regexCacheHits++;
+    }
+    
+    return this.regexCache.get(pattern);
+  }
+  
+  /**
+   * Check pattern matching cache
+   * @param {string} url - URL to check
+   * @param {string} pattern - Regex pattern
+   * @returns {boolean|null} Cached result or null if not cached
+   */
+  getCachedPatternMatch(url, pattern) {
+    if (!this.options.enablePatternCache) return null;
+    
+    const cacheKey = `${url}:${pattern}`;
+    const cached = this.patternCache.get(cacheKey);
+    
+    if (cached !== undefined) {
+      this.stats.patternHits++;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Pattern cache hit for ${url.substring(0, 50)}...`
+        ));
+      }
+      return cached;
+    }
+    
+    this.stats.patternMisses++;
+    return null;
+  }
+  
+  /**
+   * Cache pattern matching result
+   * @param {string} url - URL
+   * @param {string} pattern - Regex pattern
+   * @param {boolean} result - Match result
+   */
+  cachePatternMatch(url, pattern, result) {
+    if (!this.options.enablePatternCache) return;
+    
+    const cacheKey = `${url}:${pattern}`;
+    this.patternCache.set(cacheKey, result);
+  }
+  
+  /**
+   * Get cached response content
+   * @param {string} url - URL
+   * @returns {string|null} Cached content or null
+   */
+  getCachedResponse(url) {
+    if (!this.options.enableResponseCache) return null;
+    
+    const cached = this.responseCache.get(url);
+    if (cached) {
+      this.stats.responseHits++;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Response cache hit for ${url.substring(0, 50)}...`
+        ));
+      }
+      return cached;
+    }
+    
+    this.stats.responseMisses++;
+    return null;
+  }
+  
+  /**
+   * Cache response content
+   * @param {string} url - URL
+   * @param {string} content - Response content
+   */
+  cacheResponse(url, content) {
+    if (!this.options.enableResponseCache) return;
+    
+    // Only cache if content is reasonable size
+    if (content && content.length < 5 * 1024 * 1024) { // 5MB limit per response
+      this.responseCache.set(url, content);
+    }
+  }
+  
+  /**
+   * Get cached WHOIS/DNS results
+   * @param {string} domain - Domain
+   * @param {string} tool - Tool name (whois/dig)
+   * @param {string} recordType - Record type for dig
+   * @returns {Object|null} Cached result or null
+   */
+  getCachedNetTools(domain, tool, recordType = null) {
+    if (!this.options.enableWhoisCache) return null;
+    
+    const cacheKey = `${tool}:${domain}${recordType ? ':' + recordType : ''}`;
+    const cached = this.netToolsCache.get(cacheKey);
+    
+    if (cached) {
+      this.stats.netToolsHits++;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] ${tool.toUpperCase()} cache hit for ${domain}`
+        ));
+      }
+      return cached;
+    }
+    
+    this.stats.netToolsMisses++;
+    return null;
+  }
+  
+  /**
+   * Cache WHOIS/DNS results
+   * @param {string} domain - Domain
+   * @param {string} tool - Tool name
+   * @param {Object} result - Result to cache
+   * @param {string} recordType - Record type for dig
+   */
+  cacheNetTools(domain, tool, result, recordType = null) {
+    if (!this.options.enableWhoisCache) return;
+    
+    const cacheKey = `${tool}:${domain}${recordType ? ':' + recordType : ''}`;
+    this.netToolsCache.set(cacheKey, result);
+  }
+  
+  /**
+   * Cache similarity comparison result
+   * @param {string} domain1 - First domain
+   * @param {string} domain2 - Second domain
+   * @param {number} similarity - Similarity score
+   */
+  cacheSimilarity(domain1, domain2, similarity) {
+    const key = [domain1, domain2].sort().join('|');
+    this.similarityCache.set(key, similarity);
+  }
+  
+  /**
+   * Get cached similarity score
+   * @param {string} domain1 - First domain
+   * @param {string} domain2 - Second domain
+   * @returns {number|null} Cached similarity or null
+   */
+  getCachedSimilarity(domain1, domain2) {
+    const key = [domain1, domain2].sort().join('|');
+    const cached = this.similarityCache.get(key);
+    
+    if (cached !== undefined) {
+      this.stats.similarityHits++;
+      return cached;
+    }
+    
+    this.stats.similarityMisses++;
+    return null;
+  }
+  
+  /**
+   * Get cache statistics
+   * @returns {Object} Statistics object
+   */
+  getStats() {
+    const runtime = Date.now() - this.stats.startTime;
+    const hitRate = this.stats.hits / (this.stats.hits + this.stats.misses) || 0;
+    const patternHitRate = this.stats.patternHits / 
+      (this.stats.patternHits + this.stats.patternMisses) || 0;
+    const responseHitRate = this.stats.responseHits / 
+      (this.stats.responseHits + this.stats.responseMisses) || 0;
+    const netToolsHitRate = this.stats.netToolsHits / 
+      (this.stats.netToolsHits + this.stats.netToolsMisses) || 0;
+    
+    return {
+      ...this.stats,
+      runtime: Math.round(runtime / 1000), // seconds
+      hitRate: (hitRate * 100).toFixed(2) + '%',
+      patternHitRate: (patternHitRate * 100).toFixed(2) + '%',
+      responseHitRate: (responseHitRate * 100).toFixed(2) + '%',
+      netToolsHitRate: (netToolsHitRate * 100).toFixed(2) + '%',
+      domainCacheSize: this.domainCache.size,
+      patternCacheSize: this.patternCache.size,
+      responseCacheSize: this.responseCache.size,
+      netToolsCacheSize: this.netToolsCache.size,
+      similarityCacheSize: this.similarityCache.size,
+      regexCacheSize: this.regexCache.size,
+      totalCacheEntries: this.domainCache.size + this.patternCache.size + 
+        this.responseCache.size + this.netToolsCache.size + 
+        this.similarityCache.size + this.regexCache.size
+    };
+  }
+  
+  /**
+   * Clear all caches
+   */
+  clear() {
+    this.domainCache.clear();
+    this.patternCache.clear();
+    this.responseCache.clear();
+    this.netToolsCache.clear();
+    this.similarityCache.clear();
+    this.regexCache.clear();
+    this._initializeStats();
+    
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', '[SmartCache] All caches cleared'));
+    }
+  }
+  
+  /**
+   * Load persistent cache from disk
+   * @private
+   */
+  _loadPersistentCache() {
+    const cacheFile = path.join(this.options.persistencePath, 'smart-cache.json');
+    
+    if (!fs.existsSync(cacheFile)) {
+      return;
+    }
+    
+    try {
+      const data = JSON.parse(fs.readFileSync(cacheFile, 'utf8'));
+      const now = Date.now();
+      
+      // Validate cache age
+      if (data.timestamp && now - data.timestamp > 24 * 60 * 60 * 1000) {
+        if (this.options.forceDebug) {
+          console.log(formatLogMessage('debug', 
+            '[SmartCache] Persistent cache too old, ignoring'
+          ));
+        }
+        return;
+      }
+      
+      // Load domain cache
+      if (data.domainCache && Array.isArray(data.domainCache)) {
+        data.domainCache.forEach(([key, value]) => {
+          // Only load if not expired
+          if (now - value.timestamp < this.options.ttl) {
+            this.domainCache.set(key, value);
+          }
+        });
+      }
+      
+      // Load nettools cache
+      if (data.netToolsCache && Array.isArray(data.netToolsCache)) {
+        data.netToolsCache.forEach(([key, value]) => {
+          this.netToolsCache.set(key, value);
+        });
+      }
+      
+      this.stats.persistenceLoads++;
+      
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Loaded persistent cache: ${this.domainCache.size} domains, ${this.netToolsCache.size} nettools`
+        ));
+      }
+    } catch (err) {
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Failed to load persistent cache: ${err.message}`
+        ));
+      }
+    }
+  }
+  
+  /**
+   * Save cache to disk
+   */
+  savePersistentCache() {
+    if (!this.options.enablePersistence) return;
+    
+    const cacheDir = this.options.persistencePath;
+    const cacheFile = path.join(cacheDir, 'smart-cache.json');
+    
+    try {
+      // Create cache directory if it doesn't exist
+      if (!fs.existsSync(cacheDir)) {
+        fs.mkdirSync(cacheDir, { recursive: true });
+      }
+      
+      const data = {
+        timestamp: Date.now(),
+        domainCache: Array.from(this.domainCache.entries()),
+        netToolsCache: Array.from(this.netToolsCache.entries()),
+        stats: this.stats
+      };
+      
+      fs.writeFileSync(cacheFile, JSON.stringify(data, null, 2));
+      this.stats.persistenceSaves++;
+      
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Saved cache to disk: ${cacheFile}`
+        ));
+      }
+    } catch (err) {
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Failed to save cache: ${err.message}`
+        ));
+      }
+    }
+  }
+  
+  /**
+   * Set up auto-save interval
+   * @private
+   */
+  _setupAutoSave() {
+    this.autoSaveInterval = setInterval(() => {
+      this.savePersistentCache();
+    }, this.options.autoSaveInterval);
+  }
+  
+  /**
+   * Clean up resources
+   */
+  destroy() {
+    if (this.autoSaveInterval) {
+      clearInterval(this.autoSaveInterval);
+    }
+    
+    // Save cache one last time
+    if (this.options.enablePersistence) {
+      this.savePersistentCache();
+    }
+    
+    this.clear();
+  }
+}
+
+/**
+ * Factory function to create SmartCache instance with config
+ * @param {Object} config - Configuration object
+ * @returns {SmartCache} SmartCache instance
+ */
+function createSmartCache(config = {}) {
+  return new SmartCache({
+    maxSize: config.cache_max_size,
+    ttl: (config.cache_ttl_minutes || 60) * 60 * 1000,
+    enablePatternCache: config.cache_patterns !== false,
+    enableResponseCache: config.cache_responses !== false,
+    enableWhoisCache: config.cache_nettools !== false,
+    enablePersistence: config.cache_persistence === true,
+    persistencePath: config.cache_path || '.cache',
+    forceDebug: config.forceDebug || false,
+    autoSave: config.cache_autosave !== false,
+    autoSaveInterval: (config.cache_autosave_minutes || 1) * 60 * 1000
+  });
+}
+
+module.exports = {
+  SmartCache,
+  createSmartCache
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.52 ===
+// === Network scanner script (nwss.js) v1.0.57 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -18,7 +18,7 @@ const { handleCloudflareProtection } = require('./lib/cloudflare');
 // FP Bypass
 const { handleFlowProxyProtection, getFlowProxyTimeouts } = require('./lib/flowproxy');
 // ignore_similar rules
-const { shouldIgnoreSimilarDomain } = require('./lib/ignore_similar');
+const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore_similar');
 // Graceful exit
 const { handleBrowserExit, cleanupChromeTempFiles } = require('./lib/browserexit');
 // Whois & Dig
@@ -33,13 +33,14 @@ const { colorize, colors, messageColors, tags, formatLogMessage } = require('./l
 const { performPageInteraction, createInteractionConfig } = require('./lib/interaction');
 // Domain detection cache for performance optimization
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
+const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.52'; // Script version
+const VERSION = '1.0.57'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -48,6 +49,9 @@ const startTime = Date.now();
 const domainCacheOptions = { enableLogging: false }; // Set to true for cache debug logs
 const { isDomainAlreadyDetected, markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
 
+// Smart cache will be initialized after config is loaded
+let smartCache = null;
+
 // --- Command-Line Argument Parsing ---
 const args = process.argv.slice(2);
 
@@ -547,6 +551,16 @@ const RESOURCE_CLEANUP_INTERVAL = (() => {
   return 180;
 })();
 
+// Initialize smart cache system AFTER config is loaded
+smartCache = createSmartCache({
+  ...config,
+  forceDebug,
+  cache_persistence: config.cache_persistence !== false, // Enable by default
+  cache_autosave: config.cache_autosave !== false,
+  cache_autosave_minutes: config.cache_autosave_minutes || 1,
+  cache_max_size: config.cache_max_size || 5000
+});
+
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
   const filesToClean = cleanRulesFile ? [cleanRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -949,6 +963,10 @@ function setupFrameHandling(page, forceDebug) {
 // --- Main Asynchronous IIFE (Immediately Invoked Function Expression) ---
 // This is the main entry point and execution block for the network scanner script.
 (async () => {
+
+  // Declare userDataDir in outer scope for cleanup access
+  let userDataDir = null;
+  
   /**
    * Creates a new browser instance with consistent configuration
    * Uses system Chrome and temporary directories to minimize disk usage
@@ -957,7 +975,7 @@ function setupFrameHandling(page, forceDebug) {
   async function createBrowser() {
     // Create temporary user data directory that we can fully control and clean up
     const tempUserDataDir = `/tmp/puppeteer-${Date.now()}-${Math.random().toString(36).substring(7)}`;
-    let userDataDir = tempUserDataDir; // Store for cleanup tracking
+    userDataDir = tempUserDataDir; // Store for cleanup tracking (use outer scope variable)
 
     // Try to find system Chrome installation to avoid Puppeteer downloads
     const systemChromePaths = [
@@ -1485,6 +1503,46 @@ function setupFrameHandling(page, forceDebug) {
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
        const ignoreSimilarIgnoredDomains = siteConfig.ignore_similar_ignored_domains !== undefined ? siteConfig.ignore_similar_ignored_domains : ignore_similar_ignored_domains;
        
+       // Use smart cache's similarity cache for performance
+       if (ignoreSimilarEnabled && smartCache) {
+         const existingDomains = matchedDomains instanceof Map 
+           ? Array.from(matchedDomains.keys()).filter(key => !['dryRunMatches', 'dryRunNetTools', 'dryRunSearchString'].includes(key))
+           : Array.from(matchedDomains);
+           
+         // Check cached similarity scores first
+         for (const existingDomain of existingDomains) {
+           const cachedSimilarity = smartCache.getCachedSimilarity(domain, existingDomain);
+           if (cachedSimilarity !== null && cachedSimilarity >= similarityThreshold) {
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `[SmartCache] Used cached similarity: ${domain} ~= ${existingDomain} (${cachedSimilarity}%)`));
+             }
+             return; // Skip adding this domain
+           }
+           
+           // If no cached similarity exists, calculate and cache it
+           if (cachedSimilarity === null) {
+             const similarity = calculateSimilarity(domain, existingDomain);
+             if (smartCache) {
+               smartCache.cacheSimilarity(domain, existingDomain, similarity);
+             }
+           }
+         }
+       }
+
+       // Check smart cache first
+       const context = {
+         filterRegex: siteConfig.filterRegex,
+         searchString: siteConfig.searchstring,
+         resourceType: resourceType
+       };
+       
+       if (smartCache && smartCache.shouldSkipDomain(domain, context)) {
+         if (forceDebug) {
+           console.log(formatLogMessage('debug', `[SmartCache] Skipping cached domain: ${domain}`));
+         }
+         return; // Skip adding this domain
+       }
+
        if (ignoreSimilarEnabled) {
          const existingDomains = matchedDomains instanceof Map 
            ? Array.from(matchedDomains.keys()).filter(key => !['dryRunMatches', 'dryRunNetTools', 'dryRunSearchString'].includes(key))
@@ -1523,6 +1581,11 @@ function setupFrameHandling(page, forceDebug) {
       // Mark full subdomain as detected for future reference
       markDomainAsDetected(cacheKey);
       
+      // Also mark in smart cache with context
+      if (smartCache) {
+        smartCache.markDomainProcessed(domain, context, { resourceType, fullSubdomain });
+      }
+
         if (matchedDomains instanceof Map) {
           if (!matchedDomains.has(domain)) {
             matchedDomains.set(domain, new Set());
@@ -1716,7 +1779,8 @@ function setupFrameHandling(page, forceDebug) {
  
             // REMOVED: Check if this URL matches any blocked patterns - if so, skip detection but still continue browser blocking
             // This check is no longer needed here since even_blocked handles it above
-            
+
+           
            // If NO searchstring AND NO nettools are defined, match immediately (existing behavior)
            if (!hasSearchString && !hasSearchStringAnd && !hasNetTools) {
              if (dryRunMode) {
@@ -1767,6 +1831,15 @@ function setupFrameHandling(page, forceDebug) {
              }
              
              // Create and execute nettools handler
+             // Check smart cache for nettools results
+             const cachedWhois = smartCache ? smartCache.getCachedNetTools(reqDomain, 'whois') : null;
+             const cachedDig = smartCache ? smartCache.getCachedNetTools(reqDomain, 'dig', digRecordType) : null;
+             
+             if ((cachedWhois || cachedDig) && forceDebug) {
+               console.log(formatLogMessage('debug', `[SmartCache] Using cached nettools results for ${reqDomain}`));
+             }
+             
+             // Create nettools handler with cache callbacks
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
@@ -1784,6 +1857,15 @@ function setupFrameHandling(page, forceDebug) {
                matchedDomains,
                addMatchedDomain,
                isDomainAlreadyDetected,
+               // Add cache callbacks if smart cache is available
+               onWhoisResult: smartCache ? (domain, result) => {
+                 smartCache.cacheNetTools(domain, 'whois', result);
+               } : undefined,
+               onDigResult: smartCache ? (domain, result, recordType) => {
+                 smartCache.cacheNetTools(domain, 'dig', result, recordType);
+               } : undefined,
+               cachedWhois,
+               cachedDig,
                currentUrl,
                getRootDomain,
                siteConfig,
@@ -1823,6 +1905,13 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
+             // Check response cache first if smart cache is available
+             const cachedContent = smartCache ? smartCache.getCachedResponse(reqUrl) : null;
+             
+             if (cachedContent && forceDebug) {
+               console.log(formatLogMessage('debug', `[SmartCache] Using cached response content for ${reqUrl.substring(0, 50)}...`));
+               // Process cached content instead of fetching
+             } else {
              try {
                // Use grep handler if both grep and searchstring/searchstring_and are enabled
                if (useGrep && (hasSearchString || hasSearchStringAnd)) {
@@ -1833,6 +1922,9 @@ function setupFrameHandling(page, forceDebug) {
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
                    isDomainAlreadyDetected,
+                   onContentFetched: smartCache ? (url, content) => {
+                     smartCache.cacheResponse(url, content);
+                   } : undefined,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -1885,6 +1977,7 @@ function setupFrameHandling(page, forceDebug) {
                  console.log(formatLogMessage('debug', `Curl handler failed for ${reqUrl}: ${curlErr.message}`));
                }
              }
+             }
            }
 
           break;
@@ -2494,6 +2587,19 @@ function setupFrameHandling(page, forceDebug) {
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
      console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
+     // Log smart cache statistics
+    if (smartCache) {
+    const cacheStats = smartCache.getStats();  
+    console.log(formatLogMessage('debug', '=== Smart Cache Statistics ==='));
+    console.log(formatLogMessage('debug', `Runtime: ${cacheStats.runtime}s, Total entries: ${cacheStats.totalCacheEntries}`));
+    console.log(formatLogMessage('debug', `Hit Rates - Domain: ${cacheStats.hitRate}, Pattern: ${cacheStats.patternHitRate}`));
+    console.log(formatLogMessage('debug', `Response: ${cacheStats.responseHitRate}, NetTools: ${cacheStats.netToolsHitRate}`));
+    console.log(formatLogMessage('debug', `Regex compilations saved: ${cacheStats.regexCacheHits}`));
+    console.log(formatLogMessage('debug', `Similarity cache hits: ${cacheStats.similarityHits}`));
+    if (config.cache_persistence) {
+      console.log(formatLogMessage('debug', `Persistence - Loads: ${cacheStats.persistenceLoads}, Saves: ${cacheStats.persistenceSaves}`));
+    }
+    }
   }
   
   // Compress log files if --compress-logs is enabled
@@ -2571,6 +2677,11 @@ function setupFrameHandling(page, forceDebug) {
   const seconds = totalSeconds % 60;
 
   // Final summary report with timing and success statistics
+  // Clean up smart cache
+  if (smartCache) {
+    smartCache.destroy();
+  }
+ 
   if (!silentMode) {
     if (pagesWithMatches > outputResult.successfulPageLoads) {
       console.log(`\n${messageColors.success(dryRunMode ? 'Dry run completed.' : 'Scan completed.')} ${outputResult.successfulPageLoads} of ${totalUrls} URLs loaded successfully, ${pagesWithMatches} had matches in ${messageColors.timing(`${hours}h ${minutes}m ${seconds}s`)}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.55",
+  "version": "1.0.57",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {