@fanboynz/network-scanner 1.0.57 → 1.0.59

package/.github/workflows/npm-publish.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
           registry-url: 'https://registry.npmjs.org'
           
       - run: npm ci

package/lib/searchstring.js

@@ -4,6 +4,13 @@
 const fs = require('fs');
 const { spawnSync } = require('child_process');
 
+// Configuration constants for search logic
+const SEARCH_CONFIG = {
+  MAX_CONTENT_SIZE: 50 * 1024 * 1024, // 50MB max content size
+  MAX_SEARCH_STRING_LENGTH: 1000,
+  XML_ENTITY_TIMEOUT: 5000 // 5 second timeout for XML processing
+};
+
 /**
  * Parses searchstring configuration into a normalized format
  * @param {string|Array<string>|undefined} searchstring - The searchstring config value (OR logic)
@@ -83,6 +90,8 @@ async function downloadWithCurl(url, userAgent = '', timeout = 30) {
         '--max-time', timeout.toString(),
         '--max-redirs', '5',
         '--fail-with-body', // Return body even on HTTP errors
+        '--max-filesize', '52428800', // 50MB limit
+        '--range', '0-52428799', // Limit download size
         '--compressed', // Accept compressed responses
       ];
 
@@ -105,7 +114,8 @@ async function downloadWithCurl(url, userAgent = '', timeout = 30) {
       const result = spawnSync('curl', curlArgs, { 
         encoding: 'utf8',
         timeout: timeout * 1000,
-        maxBuffer: 10 * 1024 * 1024 // 10MB max buffer
+        maxBuffer: 10 * 1024 * 1024, // 10MB max buffer
+        killSignal: 'SIGTERM'
       });
       
       if (result.error) {
@@ -123,6 +133,113 @@ async function downloadWithCurl(url, userAgent = '', timeout = 30) {
   });
 }
 
+/**
+ * Downloads content with retry logic for transient failures
+ * @param {string} url - The URL to download
+ * @param {string} userAgent - User agent string to use
+ * @param {number} timeout - Timeout in seconds
+ * @param {number} retries - Number of retry attempts (default: 2)
+ * @returns {Promise<string>} The downloaded content
+ */
+async function downloadWithRetry(url, userAgent = '', timeout = 30, retries = 2) {
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      return await downloadWithCurl(url, userAgent, timeout);
+    } catch (err) {
+      // Don't retry on final attempt
+      if (attempt === retries) throw err;
+      
+      // Only retry on specific transient errors
+      const shouldRetry = err.message.includes('timeout') || 
+                         err.message.includes('Connection refused') ||
+                         err.message.includes('502') || 
+                         err.message.includes('503') ||
+                         err.message.includes('Connection reset');
+      
+      if (!shouldRetry) throw err;
+      
+      // Exponential backoff: 1s, 2s, 4s...
+      await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt)));
+    }
+  }
+}
+
+/**
+ * Safely decodes XML entities with timeout protection
+ * @param {string} content - Content to decode
+ * @returns {string} Decoded content or original if processing fails
+ */
+function safeDecodeXmlEntities(content) {
+  const startTime = Date.now();
+  
+  try {
+    let decoded = content
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&amp;/g, '&')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .replace(/&apos;/g, "'");
+    
+    // Check timeout before expensive regex operations
+    if (Date.now() - startTime > SEARCH_CONFIG.XML_ENTITY_TIMEOUT) {
+      console.warn('[warn] XML entity decoding timeout, using partial result');
+      return decoded;
+    }
+    
+    // Decode numeric entities (decimal)
+    decoded = decoded.replace(/&#(\d+);/g, (match, dec) => {
+      const num = parseInt(dec, 10);
+      // Validate range for safety (valid Unicode range)
+      if (num >= 0 && num <= 0x10FFFF) {
+        return String.fromCharCode(num);
+      }
+      return match; // Keep original if invalid
+    });
+    
+    // Check timeout again
+    if (Date.now() - startTime > SEARCH_CONFIG.XML_ENTITY_TIMEOUT) {
+      console.warn('[warn] XML entity decoding timeout, using partial result');
+      return decoded;
+    }
+    
+    // Decode numeric entities (hexadecimal)
+    decoded = decoded.replace(/&#x([0-9a-f]+);/gi, (match, hex) => {
+      const num = parseInt(hex, 16);
+      // Validate range for safety (valid Unicode range)
+      if (num >= 0 && num <= 0x10FFFF) {
+        return String.fromCharCode(num);
+      }
+      return match; // Keep original if invalid
+    });
+    
+    return decoded;
+  } catch (xmlErr) {
+    console.warn(`[warn] XML entity decoding failed: ${xmlErr.message}`);
+    return content; // Return original content if decoding fails
+  }
+}
+
+/**
+ * Safely strips XML/HTML tags with size limits
+ * @param {string} content - Content to strip tags from
+ * @returns {string} Content with tags removed
+ */
+function safeStripTags(content) {
+  try {
+    // Limit content size for tag stripping to prevent excessive memory usage
+    const limitedContent = content.length > SEARCH_CONFIG.MAX_CONTENT_SIZE 
+      ? content.substring(0, SEARCH_CONFIG.MAX_CONTENT_SIZE)
+      : content;
+    
+    // Replace tags with spaces to preserve word boundaries
+    return limitedContent.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ');
+  } catch (stripErr) {
+    console.warn(`[warn] Tag stripping failed: ${stripErr.message}`);
+    return content;
+  }
+}
+
 /**
  * Checks if response content contains any of the search strings (OR logic)
  * or all of the AND search strings (AND logic)
@@ -131,58 +248,118 @@ async function downloadWithCurl(url, userAgent = '', timeout = 30) {
  * @param {Array<string>} searchStrings - Array of strings to search for (OR logic)
  * @param {Array<string>} searchStringsAnd - Array of strings that must all be present (AND logic)
  * @param {string} contentType - Content type for specialized handling
+ * @param {string} url - URL for debugging context (optional)
  * @returns {object} Object with found boolean, matchedString/matchedStrings, allMatches array, and logic type
  */
-function searchContent(content, searchStrings, searchStringsAnd = [], contentType = '') {
+function searchContent(content, searchStrings, searchStringsAnd = [], contentType = '', url = '') {
+  // Input validation
+  if (!content || typeof content !== 'string') {
+    return { 
+      found: false, 
+      matchedString: null, 
+      matchedStrings: [], 
+      allMatches: [], 
+      logicType: 'NONE',
+      error: 'Invalid or empty content'
+    };
+  }
+  
+  // Size check and truncation with warning
+  const originalLength = content.length;
+  if (originalLength > SEARCH_CONFIG.MAX_CONTENT_SIZE) {
+    content = content.substring(0, SEARCH_CONFIG.MAX_CONTENT_SIZE);
+    console.warn(`[warn] Content truncated from ${originalLength} to ${SEARCH_CONFIG.MAX_CONTENT_SIZE} chars for ${url || 'unknown URL'}`);
+  }
   let searchableContent = content;
+
+  const isXmlContent = contentType.toLowerCase().includes('xml') || 
+                      contentType.toLowerCase().includes('html');
   
-  // For XML content, also search decoded entities and stripped tags for better matching
-  if (contentType.includes('xml')) {
-    // Decode common XML entities
-    const decodedContent = content
-      .replace(/&lt;/g, '<')
-      .replace(/&gt;/g, '>')
-      .replace(/&amp;/g, '&')
-      .replace(/&quot;/g, '"')
-      .replace(/&#39;/g, "'");
-    
-    // Create version with XML tags stripped for text content search
-    const strippedContent = decodedContent.replace(/<[^>]*>/g, ' ');
-    
-    // Search in: original + decoded + stripped content
-    searchableContent = content + '\n' + decodedContent + '\n' + strippedContent;
+  if (isXmlContent) {
+    try {
+      // Safely decode XML entities
+      const decodedContent = safeDecodeXmlEntities(content);
+      
+      // Safely strip tags to extract text content
+      const strippedContent = safeStripTags(decodedContent);
+      
+      // Search in: original + decoded + stripped content
+      // Use newlines as separators to prevent false matches across content types
+      searchableContent = [content, decodedContent, strippedContent].join('\n');
+      
+    } catch (xmlProcessingErr) {
+      console.warn(`[warn] XML processing failed for ${url || 'unknown URL'}: ${xmlProcessingErr.message}`);
+      // Fall back to original content
+      searchableContent = content;
+    }
+  }
+  
+  // Input validation for search strings
+  const validSearchStrings = searchStrings.filter(str => 
+    str && typeof str === 'string' && str.length > 0 && str.length <= SEARCH_CONFIG.MAX_SEARCH_STRING_LENGTH
+  );
+  const validSearchStringsAnd = searchStringsAnd.filter(str => 
+    str && typeof str === 'string' && str.length > 0 && str.length <= SEARCH_CONFIG.MAX_SEARCH_STRING_LENGTH
+  );
+  
+  // Warn about filtered search strings
+  if (validSearchStrings.length !== searchStrings.length) {
+    console.warn(`[warn] Filtered ${searchStrings.length - validSearchStrings.length} invalid search strings`);
+  }
+  if (validSearchStringsAnd.length !== searchStringsAnd.length) {
+    console.warn(`[warn] Filtered ${searchStringsAnd.length - validSearchStringsAnd.length} invalid AND search strings`);
   }
   
-  // Check AND logic first (more restrictive)
-  if (searchStringsAnd && searchStringsAnd.length > 0) {
-    const lowerContent = searchableContent.toLowerCase();
+  // Early return if no valid search strings
+  if (validSearchStrings.length === 0 && validSearchStringsAnd.length === 0) {
+    return {
+      found: false,
+      matchedString: null,
+      matchedStrings: [],
+      allMatches: [],
+      logicType: 'NONE',
+      error: 'No valid search strings provided'
+    };
+  }
+  
+  // Pre-compute lowercase content once for better performance
+  const lowerContent = searchableContent.toLowerCase();
+  
+  // Check AND logic first (more restrictive) - ALL strings must be present
+  if (validSearchStringsAnd && validSearchStringsAnd.length > 0) {
     const foundAndStrings = [];
     
-    for (const searchStr of searchStringsAnd) {
-      if (lowerContent.includes(searchStr.toLowerCase())) {
+    for (const searchStr of validSearchStringsAnd) {
+      const lowerSearchStr = searchStr.toLowerCase();
+      if (lowerContent.includes(lowerSearchStr)) {
         foundAndStrings.push(searchStr);
+      } else {
+        // Early exit if any AND string is not found
+        break;
       }
     }
     
-    // AND logic: ALL strings must be found
-    if (foundAndStrings.length === searchStringsAnd.length) {
+    // AND logic: ALL valid strings must be found
+    if (foundAndStrings.length === validSearchStringsAnd.length) {
       return {
         found: true,
-        matchedString: foundAndStrings.join(' AND '), // Show all matched strings
+        matchedString: foundAndStrings.join(' AND '),
         matchedStrings: foundAndStrings,
         allMatches: foundAndStrings,
-        logicType: 'AND'
+        logicType: 'AND',
+        contentSize: originalLength,
+        searchableSize: searchableContent.length
       };
     }
   }
   
-  // Fall back to OR logic if AND logic didn't match or wasn't specified
-  const lowerContent = searchableContent.toLowerCase();
+  // OR logic: ANY string can match
   const allMatches = [];
   let firstMatch = null;
   
-  for (const searchStr of searchStrings) {
-    if (lowerContent.includes(searchStr.toLowerCase())) {
+  for (const searchStr of validSearchStrings) {
+    const lowerSearchStr = searchStr.toLowerCase();
+    if (lowerContent.includes(lowerSearchStr)) {
       allMatches.push(searchStr);
       if (!firstMatch) {
         firstMatch = searchStr;
@@ -195,7 +372,10 @@ function searchContent(content, searchStrings, searchStringsAnd = [], contentTyp
     matchedString: firstMatch,
     matchedStrings: allMatches,
     allMatches: allMatches,
-    logicType: 'OR'
+    logicType: validSearchStrings.length > 0 ? 'OR' : 'NONE',
+    contentSize: originalLength,
+    searchableSize: searchableContent.length,
+    processedAsXml: isXmlContent
   };
 }
 
@@ -206,6 +386,9 @@ function searchContent(content, searchStrings, searchStringsAnd = [], contentTyp
  */
 function shouldAnalyzeContentType(contentType) {
   if (!contentType) return false;
+
+  // Normalize content type (remove charset and other parameters)
+  const normalizedType = contentType.toLowerCase().split(';')[0].trim();
   
   const textTypes = [
     'text/',                    // text/html, text/plain, text/xml, etc.
@@ -216,10 +399,16 @@ function shouldAnalyzeContentType(contentType) {
     'application/soap+xml',     // SOAP XML
     'application/rss+xml',      // RSS feeds
     'application/atom+xml',     // Atom feeds
-    'application/xhtml+xml'     // XHTML
+    'application/xhtml+xml',    // XHTML
+    'application/ld+json',      // JSON-LD structured data
+    'application/manifest+json', // Web App Manifest
+    'application/feed+xml',     // Generic XML feeds
+    'application/vnd.api+json', // JSON API specification
+    'application/hal+json',     // HAL (Hypertext Application Language)
+    'application/problem+json'  // Problem Details for HTTP APIs
   ];
   
-  return textTypes.some(type => contentType.includes(type));
+  return textTypes.some(type => normalizedType.startsWith(type));
 }
 
 /**
@@ -319,10 +508,10 @@ function createCurlHandler(config) {
       }
       
       // If searchstring IS defined, download and search content
-      const content = await downloadWithCurl(requestUrl, userAgent, 30);
+      const content = await downloadWithRetry(requestUrl, userAgent, 30);
       
       // Check if content contains search strings (OR or AND logic)
-      const { found, matchedString, logicType } = searchContent(content, searchStrings, searchStringsAnd, '');
+      const { found, matchedString, logicType, error } = searchContent(content, searchStrings, searchStringsAnd, '', requestUrl);
       
       if (found) {
         if (!reqDomain || matchesIgnoreDomain(reqDomain, ignoreDomains)) {
@@ -352,6 +541,9 @@ function createCurlHandler(config) {
       } else if (forceDebug) {
         const partyType = isFirstParty ? 'first-party' : 'third-party';
         console.log(`[debug][curl] ${requestUrl} (${partyType}) matched regex but no searchstring found`);
+        if (error) {
+          console.log(`[debug][curl] Search error: ${error}`);
+        }
       }
       
     } catch (err) {
@@ -442,7 +634,7 @@ function createResponseHandler(config) {
       const content = await response.text();
       
       // Check if content contains search strings (OR or AND logic)
-      const { found, matchedString, logicType } = searchContent(content, searchStrings, searchStringsAnd, contentType);
+      const { found, matchedString, logicType, error } = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
       
       if (found) {
         if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
@@ -471,6 +663,9 @@ function createResponseHandler(config) {
       } else if (forceDebug) {
         const partyType = isFirstParty ? 'first-party' : 'third-party';
         console.log(`[debug] ${respUrl} (${partyType}) matched regex but no searchstring found`);
+        if (error) {
+          console.log(`[debug] Search error: ${error}`);
+        }
       }
       
     } catch (err) {
@@ -541,9 +736,40 @@ function validateSearchString(searchstring, searchstringAnd) {
   }
   
   // Check that both searchstring and searchstring_and aren't defined simultaneously
-  if (searchstring && searchstringAnd) {
+  if ((searchstring !== undefined && searchstring !== null) && 
+      (searchstringAnd !== undefined && searchstringAnd !== null)) {
     return { isValid: false, error: 'Cannot use both searchstring (OR) and searchstring_and (AND) simultaneously. Choose one logic type.' };
   }
+
+  // Additional validation for search string length limits
+  const validateStringLength = (str, fieldName) => {
+    if (str.length > SEARCH_CONFIG.MAX_SEARCH_STRING_LENGTH) {
+      return { isValid: false, error: `${fieldName} too long (max ${SEARCH_CONFIG.MAX_SEARCH_STRING_LENGTH} chars)` };
+    }
+    return { isValid: true };
+  };
+  
+  // Validate search string lengths
+  if (typeof searchstring === 'string') {
+    const lengthCheck = validateStringLength(searchstring, 'searchstring');
+    if (!lengthCheck.isValid) return lengthCheck;
+  } else if (Array.isArray(searchstring)) {
+    for (let i = 0; i < searchstring.length; i++) {
+      const lengthCheck = validateStringLength(searchstring[i], `searchstring[${i}]`);
+      if (!lengthCheck.isValid) return lengthCheck;
+    }
+  }
+  
+  // Validate AND search string lengths
+  if (typeof searchstringAnd === 'string') {
+    const lengthCheck = validateStringLength(searchstringAnd, 'searchstring_and');
+    if (!lengthCheck.isValid) return lengthCheck;
+  } else if (Array.isArray(searchstringAnd)) {
+    for (let i = 0; i < searchstringAnd.length; i++) {
+      const lengthCheck = validateStringLength(searchstringAnd[i], `searchstring_and[${i}]`);
+      if (!lengthCheck.isValid) return lengthCheck;
+    }
+  }
   
   return { isValid: false, error: 'searchstring must be string or array of strings' };
 }
@@ -567,11 +793,13 @@ function getSearchStats(matchedDomains, searchStrings) {
 module.exports = {
   parseSearchStrings,
   searchContent,
+  safeDecodeXmlEntities,
   shouldAnalyzeContentType,
   createResponseHandler,
   createCurlHandler,
   downloadWithCurl,
   validateSearchString,
   getSearchStats,
-  addDomainToCollection
+  addDomainToCollection,
+  downloadWithRetry
 };

package/lib/smart-cache.js

@@ -14,6 +14,11 @@ const { formatLogMessage } = require('./colorize');
  */
 class SmartCache {
   constructor(options = {}) {
+    // Calculate dynamic values first
+    const concurrency = options.concurrency || 6;
+    const optimalHeapLimit = this._calculateOptimalHeapLimit(concurrency);
+    const checkInterval = this._calculateCheckInterval(concurrency);
+
     this.options = {
       maxSize: options.maxSize || 5000,
       ttl: options.ttl || 1000 * 60 * 60, // 1 hour default
@@ -24,8 +29,18 @@ class SmartCache {
       persistencePath: options.persistencePath || '.cache',
       forceDebug: options.forceDebug || false,
       autoSave: options.autoSave !== false,
-      autoSaveInterval: options.autoSaveInterval || 60000 // 1 minute
+      autoSaveInterval: options.autoSaveInterval || 60000, // 1 minute
+      maxHeapUsage: options.maxHeapUsage || optimalHeapLimit,
+      memoryCheckInterval: options.memoryCheckInterval || checkInterval,
+      concurrency: concurrency,
+      aggressiveMode: options.aggressiveMode || false
     };
+
+    // Add save debouncing
+    this.lastSaveTime = 0;
+    this.saveInProgress = false;
+    this.saveTimeout = null;
+    this.pendingSave = false;
     
     // Initialize cache layers
     this._initializeCaches();
@@ -42,6 +57,34 @@ class SmartCache {
     if (this.options.enablePersistence && this.options.autoSave) {
       this._setupAutoSave();
     }
+    
+    // Set up memory monitoring
+    this.memoryCheckInterval = setInterval(() => {
+      this._checkMemoryPressure();
+    }, this.options.memoryCheckInterval);
+  }
+  
+  /**
+   * Calculate optimal heap limit based on concurrency
+   * @private
+   */
+  _calculateOptimalHeapLimit(concurrency) {
+    // Base cache needs: 100MB
+    // Per concurrent connection: ~75MB average
+    // Safety margin: 50%
+    const baseCacheMemory = 100 * 1024 * 1024; // 100MB
+    const perConnectionMemory = 75 * 1024 * 1024; // 75MB
+    const totalEstimated = baseCacheMemory + (concurrency * perConnectionMemory);
+    return Math.round(totalEstimated * 0.4); // Cache should use max 40% of estimated total
+  }
+  
+  /**
+   * Calculate check interval based on concurrency
+   * @private
+   */
+  _calculateCheckInterval(concurrency) {
+    // Higher concurrency = more frequent checks
+    return Math.max(5000, 30000 - (concurrency * 1000)); // 5s min, scales down with concurrency
   }
   
   /**
@@ -57,29 +100,43 @@ class SmartCache {
       updateAgeOnHas: false
     });
     
-    // Pattern matching results cache
+    // Pattern matching results cache - reduce size for high concurrency
+    const patternCacheSize = this.options.concurrency > 10 ? 500 : 1000;
     this.patternCache = new LRUCache({
-      max: 1000,
+      max: patternCacheSize,
       ttl: this.options.ttl * 2 // Patterns are more stable
     });
     
-    // Response content cache for searchstring operations
+    // Response content cache - aggressive limits for high concurrency
+    const responseCacheSize = this.options.concurrency > 10 ? 50 : 200;
+    const responseCacheMemory = this.options.concurrency > 10 ? 20 * 1024 * 1024 : 50 * 1024 * 1024;
     this.responseCache = new LRUCache({
-      max: 200,
+      max: responseCacheSize,
       ttl: 1000 * 60 * 30, // 30 minutes for response content
-      maxSize: 50 * 1024 * 1024, // 50MB max cache size
+      maxSize: responseCacheMemory,
       sizeCalculation: (value) => value.length
     });
     
+    // Disable response cache entirely for very high concurrency
+    if (this.options.concurrency > 15 || this.options.aggressiveMode) {
+      this.options.enableResponseCache = false;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Response cache disabled for high concurrency (${this.options.concurrency})`
+        ));
+      }
+    }
+
     // WHOIS/DNS results cache
     this.netToolsCache = new LRUCache({
       max: 500,
       ttl: 1000 * 60 * 60 * 24 // 24 hours for WHOIS/DNS
     });
     
-    // Similarity cache for expensive string comparisons
+    // Similarity cache - reduce for high concurrency
+    const similarityCacheSize = this.options.concurrency > 10 ? 1000 : 2000;
     this.similarityCache = new LRUCache({
-      max: 2000,
+      max: similarityCacheSize,
       ttl: this.options.ttl
     });
     
@@ -107,6 +164,9 @@ class SmartCache {
       regexCacheHits: 0,
       persistenceLoads: 0,
       persistenceSaves: 0,
+      memoryPressureEvents: 0,
+      memoryWarnings: 0,
+      responseCacheSkips: 0,
       startTime: Date.now()
     };
   }
@@ -274,6 +334,21 @@ class SmartCache {
   cacheResponse(url, content) {
     if (!this.options.enableResponseCache) return;
     
+    // Skip response caching entirely for very high concurrency
+    if (this.options.concurrency > 12) {
+      this.stats.responseCacheSkips++;
+      return;
+    }
+    
+    // Check memory before caching large content
+    const memUsage = process.memoryUsage();
+    const threshold = this.options.concurrency > 10 ? 0.7 : 0.8; // Lower threshold for high concurrency
+    if (memUsage.heapUsed > this.options.maxHeapUsage * threshold) {
+      this.stats.responseCacheSkips++;
+      this._logMemorySkip('response cache');
+      return;
+    }
+    
     // Only cache if content is reasonable size
     if (content && content.length < 5 * 1024 * 1024) { // 5MB limit per response
       this.responseCache.set(url, content);
@@ -350,7 +425,85 @@ class SmartCache {
     this.stats.similarityMisses++;
     return null;
   }
+
+   /**
+   * Monitor memory usage and proactively manage caches
+   * @private
+   */
+  _checkMemoryPressure() {
+    const memUsage = process.memoryUsage();
+    const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
+    const maxHeapMB = Math.round(this.options.maxHeapUsage / 1024 / 1024);
+    const usagePercent = (memUsage.heapUsed / this.options.maxHeapUsage) * 100;
+    
+    // Adjust thresholds based on concurrency
+    const criticalThreshold = this.options.concurrency > 10 ? 0.85 : 1.0;
+    const warningThreshold = this.options.concurrency > 10 ? 0.70 : 0.85;
+    const infoThreshold = this.options.concurrency > 10 ? 0.60 : 0.75;
+    
+    // Critical threshold - aggressive cleanup
+    if (memUsage.heapUsed > this.options.maxHeapUsage * criticalThreshold) {
+      this._performMemoryCleanup('critical', heapUsedMB, maxHeapMB);
+      return true;
+    }
+    
+    // Warning threshold - moderate cleanup
+    if (memUsage.heapUsed > this.options.maxHeapUsage * warningThreshold) {
+      this._performMemoryCleanup('warning', heapUsedMB, maxHeapMB);
+      return true;
+    }
+    
+    // Info threshold - log only
+    if (memUsage.heapUsed > this.options.maxHeapUsage * infoThreshold) {
+      this.stats.memoryWarnings++;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Memory info: ${heapUsedMB}MB/${maxHeapMB}MB (${usagePercent.toFixed(1)}%)`
+        ));
+      }
+    }
+    
+    return false;
+  }
   
+  /**
+   * Perform memory cleanup based on severity
+   * @private
+   */
+  _performMemoryCleanup(level, heapUsedMB, maxHeapMB) {
+    this.stats.memoryPressureEvents++;
+    
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Memory ${level}: ${heapUsedMB}MB/${maxHeapMB}MB, performing cleanup...`
+      ));
+    }
+    
+    if (level === 'critical' || this.options.concurrency > 12) {
+      // Aggressive cleanup - clear volatile caches
+      this.responseCache.clear();
+      this.patternCache.clear();
+      this.similarityCache.clear();
+      
+      // For very high concurrency, also trim domain cache
+      if (this.options.concurrency > 15) {
+        const currentSize = this.domainCache.size;
+        this.domainCache.clear();
+        if (this.options.forceDebug) {
+          console.log(formatLogMessage('debug', `[SmartCache] Cleared ${currentSize} domain cache entries`));
+        }
+      }
+    } else if (level === 'warning') {
+      // Moderate cleanup - clear largest cache
+      this.responseCache.clear();
+    }
+    
+    // Force garbage collection if available
+    if (global.gc) {
+      global.gc();
+    }
+  }  
+
   /**
    * Get cache statistics
    * @returns {Object} Statistics object
@@ -364,6 +517,9 @@ class SmartCache {
       (this.stats.responseHits + this.stats.responseMisses) || 0;
     const netToolsHitRate = this.stats.netToolsHits / 
       (this.stats.netToolsHits + this.stats.netToolsMisses) || 0;
+      
+    
+    const memUsage = process.memoryUsage();
     
     return {
       ...this.stats,
@@ -380,7 +536,11 @@ class SmartCache {
       regexCacheSize: this.regexCache.size,
       totalCacheEntries: this.domainCache.size + this.patternCache.size + 
         this.responseCache.size + this.netToolsCache.size + 
-        this.similarityCache.size + this.regexCache.size
+        this.similarityCache.size + this.regexCache.size,
+      memoryUsageMB: Math.round(memUsage.heapUsed / 1024 / 1024),
+      memoryMaxMB: Math.round(this.options.maxHeapUsage / 1024 / 1024),
+      memoryUsagePercent: ((memUsage.heapUsed / this.options.maxHeapUsage) * 100).toFixed(1) + '%',
+      responseCacheMemoryMB: Math.round((this.responseCache.calculatedSize || 0) / 1024 / 1024)
     };
   }
   
@@ -401,6 +561,18 @@ class SmartCache {
     }
   }
   
+   /**
+   * Helper method to log memory-related cache skips
+   * @private
+   */
+  _logMemorySkip(operation) {
+    if (this.options.forceDebug) {
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Skipping ${operation} due to memory pressure`
+      ));
+    }
+  }  
+  
   /**
    * Load persistent cache from disk
    * @private
@@ -464,6 +636,34 @@ class SmartCache {
    */
   savePersistentCache() {
     if (!this.options.enablePersistence) return;
+
+    // Prevent concurrent saves
+    if (this.saveInProgress) {
+      this.pendingSave = true;
+      if (this.options.forceDebug) {
+        console.log(formatLogMessage('debug', '[SmartCache] Save in progress, marking pending...'));
+      }
+      return;
+    }
+    
+    // Debounce saves - don't save more than once every 10 seconds
+    const now = Date.now();
+    if (now - this.lastSaveTime < 10000) {
+      // Schedule a delayed save if none is pending
+      if (!this.saveTimeout && !this.pendingSave) {
+        this.pendingSave = true;
+        this.saveTimeout = setTimeout(() => {
+          this.saveTimeout = null;
+          if (this.pendingSave) {
+            this.pendingSave = false;
+            this.savePersistentCache();
+          }
+        }, 10000 - (now - this.lastSaveTime));
+      }
+      return;
+    }
+    this.saveInProgress = true;
+    this.lastSaveTime = now;
     
     const cacheDir = this.options.persistencePath;
     const cacheFile = path.join(cacheDir, 'smart-cache.json');
@@ -475,7 +675,7 @@ class SmartCache {
       }
       
       const data = {
-        timestamp: Date.now(),
+        timestamp: now,
         domainCache: Array.from(this.domainCache.entries()),
         netToolsCache: Array.from(this.netToolsCache.entries()),
         stats: this.stats
@@ -495,6 +695,14 @@ class SmartCache {
           `[SmartCache] Failed to save cache: ${err.message}`
         ));
       }
+    } finally {
+      this.saveInProgress = false;
+      
+      // Process any pending saves
+      if (this.pendingSave && !this.saveTimeout) {
+        this.pendingSave = false;
+        setTimeout(() => this.savePersistentCache(), 1000);
+      }
     }
   }
   
@@ -512,9 +720,16 @@ class SmartCache {
    * Clean up resources
    */
   destroy() {
+    if (this.memoryCheckInterval) {
+      clearInterval(this.memoryCheckInterval);
+    }
     if (this.autoSaveInterval) {
       clearInterval(this.autoSaveInterval);
     }
+    if (this.saveTimeout) {
+      clearTimeout(this.saveTimeout);
+      this.saveTimeout = null;
+    }
     
     // Save cache one last time
     if (this.options.enablePersistence) {
@@ -523,6 +738,92 @@ class SmartCache {
     
     this.clear();
   }
+  
+  /**
+   * Clear persistent cache files and directories
+   * @param {Object} options - Clear options
+   * @param {boolean} options.silent - Suppress console output
+   * @param {boolean} options.forceDebug - Enable debug logging
+   * @returns {Object} Clear operation results
+   */
+  static clearPersistentCache(options = {}) {
+    const { silent = false, forceDebug = false, cachePath = '.cache' } = options;
+    
+    const cachePaths = [
+      cachePath,
+      path.join(cachePath, 'smart-cache.json'),
+      // Add other potential cache files here if needed
+    ];
+    
+    let clearedItems = 0;
+    let totalSize = 0;
+    const clearedFiles = [];
+    const errors = [];
+    
+    if (!silent) {
+      console.log(`\n???  Clearing cache...`);
+    }
+    
+    for (const currentCachePath of cachePaths) {
+      if (fs.existsSync(currentCachePath)) {
+        try {
+          const stats = fs.statSync(currentCachePath);
+          if (stats.isDirectory()) {
+            // Calculate total size of directory contents
+            const files = fs.readdirSync(currentCachePath);
+            for (const file of files) {
+              const filePath = path.join(currentCachePath, file);
+              if (fs.existsSync(filePath)) {
+                totalSize += fs.statSync(filePath).size;
+              }
+            }
+            fs.rmSync(currentCachePath, { recursive: true, force: true });
+            clearedItems++;
+            clearedFiles.push({ type: 'directory', path: currentCachePath, size: totalSize });
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Cleared cache directory: ${currentCachePath}`));
+            }
+          } else {
+            totalSize += stats.size;
+            fs.unlinkSync(currentCachePath);
+            clearedItems++;
+            clearedFiles.push({ type: 'file', path: currentCachePath, size: stats.size });
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Cleared cache file: ${currentCachePath}`));
+            }
+          }
+        } catch (clearErr) {
+          errors.push({ path: currentCachePath, error: clearErr.message });
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Failed to clear ${currentCachePath}: ${clearErr.message}`));
+          }
+        }
+      }
+    }
+    
+    const result = {
+      success: errors.length === 0,
+      clearedItems,
+      totalSize,
+      sizeMB: (totalSize / 1024 / 1024).toFixed(2),
+      clearedFiles,
+      errors
+    };
+    
+    if (!silent) {
+      if (clearedItems > 0) {
+        console.log(`? Cache cleared: ${clearedItems} item(s), ${result.sizeMB}MB freed`);
+      } else {
+        console.log(`??  No cache files found to clear`);
+      }
+      
+      if (errors.length > 0) {
+        console.warn(`??  ${errors.length} error(s) occurred during cache clearing`);
+      }
+    }
+    
+    return result;
+  }
 }
 
 /**
@@ -541,11 +842,16 @@ function createSmartCache(config = {}) {
     persistencePath: config.cache_path || '.cache',
     forceDebug: config.forceDebug || false,
     autoSave: config.cache_autosave !== false,
-    autoSaveInterval: (config.cache_autosave_minutes || 1) * 60 * 1000
+    autoSaveInterval: (config.cache_autosave_minutes || 1) * 60 * 1000,
+    maxHeapUsage: config.cache_max_heap_mb ? config.cache_max_heap_mb * 1024 * 1024 : undefined,
+    memoryCheckInterval: (config.cache_memory_check_seconds || 30) * 1000,
+    concurrency: config.max_concurrent_sites || 6,
+    aggressiveMode: config.cache_aggressive_mode === true
   });
 }
 
 module.exports = {
   SmartCache,
-  createSmartCache
+  createSmartCache,
+  clearPersistentCache: SmartCache.clearPersistentCache
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.57 ===
+// === Network scanner script (nwss.js) v1.0.60 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -34,13 +34,14 @@ const { performPageInteraction, createInteractionConfig } = require('./lib/inter
 // Domain detection cache for performance optimization
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
+const { clearPersistentCache } = require('./lib/smart-cache');
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.57'; // Script version
+const VERSION = '1.0.60'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -102,6 +103,8 @@ const validateConfig = args.includes('--validate-config');
 const validateRules = args.includes('--validate-rules');
 const testValidation = args.includes('--test-validation');
 let cleanRules = args.includes('--clean-rules');
+const clearCache = args.includes('--clear-cache');
+const ignoreCache = args.includes('--ignore-cache');
 
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
@@ -224,6 +227,15 @@ if (args.includes('--version')) {
   process.exit(0);
 }
 
+// Handle --clear-cache before config loading (uses default cache path)
+if (clearCache && !dryRunMode) {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: '.cache' // Default path, will be updated after config loads if needed
+  });
+}
+
 // Handle validation-only operations before main help
 if (testValidation) {
   console.log(`\n${messageColors.processing('Running domain validation tests...')}`);
@@ -360,6 +372,8 @@ Validation Options:
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
   --test-validation              Run domain validation tests and exit
+  --clear-cache                  Clear persistent cache before scanning (improves fresh start performance)
+  --ignore-cache                 Bypass all smart caching functionality during scanning
   
 Global config.json options:
   ignoreDomains: ["domain.com", "*.ads.com"]     Domains to completely ignore (supports wildcards)
@@ -551,15 +565,40 @@ const RESOURCE_CLEANUP_INTERVAL = (() => {
   return 180;
 })();
 
-// Initialize smart cache system AFTER config is loaded
+// Perform cache clear after config is loaded for custom cache paths
+if (clearCache && dryRunMode) {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: config.cache_path || '.cache'
+  });
+}
+
+// Also clear for custom cache paths in normal mode if not already cleared
+if (clearCache && !dryRunMode && config.cache_path && config.cache_path !== '.cache') {
+  clearPersistentCache({
+    silent: silentMode,
+    forceDebug,
+    cachePath: config.cache_path
+  });
+}
+
+// Initialize smart cache system AFTER config is loaded (unless --ignore-cache is used)
+if (ignoreCache) {
+  smartCache = null;
+  if (forceDebug) console.log(formatLogMessage('debug', 'Smart cache disabled by --ignore-cache flag'));
+} else {
 smartCache = createSmartCache({
   ...config,
   forceDebug,
-  cache_persistence: config.cache_persistence !== false, // Enable by default
-  cache_autosave: config.cache_autosave !== false,
+  max_concurrent_sites: MAX_CONCURRENT_SITES,  // Pass concurrency info
+  cache_aggressive_mode: MAX_CONCURRENT_SITES > 12,  // Auto-enable for high concurrency
+  cache_persistence: false, // Disable persistence completely
+  cache_autosave: false, // Disable auto-save completely
   cache_autosave_minutes: config.cache_autosave_minutes || 1,
   cache_max_size: config.cache_max_size || 5000
 });
+}
 
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
@@ -1503,7 +1542,7 @@ function setupFrameHandling(page, forceDebug) {
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
        const ignoreSimilarIgnoredDomains = siteConfig.ignore_similar_ignored_domains !== undefined ? siteConfig.ignore_similar_ignored_domains : ignore_similar_ignored_domains;
        
-       // Use smart cache's similarity cache for performance
+       // Use smart cache's similarity cache for performance (if cache is enabled)
        if (ignoreSimilarEnabled && smartCache) {
          const existingDomains = matchedDomains instanceof Map 
            ? Array.from(matchedDomains.keys()).filter(key => !['dryRunMatches', 'dryRunNetTools', 'dryRunSearchString'].includes(key))
@@ -1522,14 +1561,14 @@ function setupFrameHandling(page, forceDebug) {
            // If no cached similarity exists, calculate and cache it
            if (cachedSimilarity === null) {
              const similarity = calculateSimilarity(domain, existingDomain);
-             if (smartCache) {
+             if (smartCache && !ignoreCache) {
                smartCache.cacheSimilarity(domain, existingDomain, similarity);
              }
            }
          }
        }
 
-       // Check smart cache first
+       // Check smart cache first (if cache is enabled)
        const context = {
          filterRegex: siteConfig.filterRegex,
          searchString: siteConfig.searchstring,
@@ -1581,7 +1620,7 @@ function setupFrameHandling(page, forceDebug) {
       // Mark full subdomain as detected for future reference
       markDomainAsDetected(cacheKey);
       
-      // Also mark in smart cache with context
+      // Also mark in smart cache with context (if cache is enabled)
       if (smartCache) {
         smartCache.markDomainProcessed(domain, context, { resourceType, fullSubdomain });
       }
@@ -1831,7 +1870,7 @@ function setupFrameHandling(page, forceDebug) {
              }
              
              // Create and execute nettools handler
-             // Check smart cache for nettools results
+             // Check smart cache for nettools results (if cache is enabled)
              const cachedWhois = smartCache ? smartCache.getCachedNetTools(reqDomain, 'whois') : null;
              const cachedDig = smartCache ? smartCache.getCachedNetTools(reqDomain, 'dig', digRecordType) : null;
              
@@ -1839,7 +1878,7 @@ function setupFrameHandling(page, forceDebug) {
                console.log(formatLogMessage('debug', `[SmartCache] Using cached nettools results for ${reqDomain}`));
              }
              
-             // Create nettools handler with cache callbacks
+             // Create nettools handler with cache callbacks (if cache is enabled)
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
@@ -1857,7 +1896,7 @@ function setupFrameHandling(page, forceDebug) {
                matchedDomains,
                addMatchedDomain,
                isDomainAlreadyDetected,
-               // Add cache callbacks if smart cache is available
+               // Add cache callbacks if smart cache is available and caching is enabled
                onWhoisResult: smartCache ? (domain, result) => {
                  smartCache.cacheNetTools(domain, 'whois', result);
                } : undefined,
@@ -1905,7 +1944,7 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check response cache first if smart cache is available
+             // Check response cache first if smart cache is available and caching is enabled
              const cachedContent = smartCache ? smartCache.getCachedResponse(reqUrl) : null;
              
              if (cachedContent && forceDebug) {
@@ -1922,7 +1961,7 @@ function setupFrameHandling(page, forceDebug) {
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
                    isDomainAlreadyDetected,
-                   onContentFetched: smartCache ? (url, content) => {
+                   onContentFetched: smartCache && !ignoreCache ? (url, content) => {
                      smartCache.cacheResponse(url, content);
                    } : undefined,
                    currentUrl,
@@ -2587,7 +2626,7 @@ function setupFrameHandling(page, forceDebug) {
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
      console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
-     // Log smart cache statistics
+     // Log smart cache statistics (if cache is enabled)
     if (smartCache) {
     const cacheStats = smartCache.getStats();  
     console.log(formatLogMessage('debug', '=== Smart Cache Statistics ==='));
@@ -2677,7 +2716,7 @@ function setupFrameHandling(page, forceDebug) {
   const seconds = totalSeconds % 60;
 
   // Final summary report with timing and success statistics
-  // Clean up smart cache
+  // Clean up smart cache (if it exists)
   if (smartCache) {
     smartCache.destroy();
   }
@@ -2699,6 +2738,9 @@ function setupFrameHandling(page, forceDebug) {
     if (totalDomainsSkipped > 0) {
       console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
     }
+    if (ignoreCache && forceDebug) {
+      console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.57",
+  "version": "1.0.59",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {
@@ -10,9 +10,10 @@
     "lint": "eslint *.js lib/*.js"
   },
   "dependencies": {
+    "lru-cache": "^10.4.3",
     "p-limit": "^4.0.0",
     "psl": "^1.15.0",
-    "puppeteer": "^23.10.0"
+    "puppeteer": "^23.11.1"
   },
   "keywords": [
     "puppeteer",
@@ -29,7 +30,7 @@
   "author": "FanboyNZ",
   "license": "GPL-3.0",
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "repository": {
     "type": "git",