@fanboynz/network-scanner 2.0.55 → 2.0.57

package/lib/smart-cache.js

@@ -8,12 +8,15 @@ const fs = require('fs');
 const path = require('path');
 const { formatLogMessage } = require('./colorize');
 
+// Shared frozen empty object -- avoids allocating new {} for every default param
+const EMPTY = Object.freeze({});
+
 /**
  * SmartCache - Intelligent caching system with multiple cache layers
  * @class
  */
 class SmartCache {
-  constructor(options = {}) {
+  constructor(options = EMPTY) {
     // Calculate dynamic values first
     const concurrency = options.concurrency || 6;
     const optimalHeapLimit = this._calculateOptimalHeapLimit(concurrency);
@@ -44,16 +47,33 @@ class SmartCache {
     this.saveInProgress = false;
     this.saveTimeout = null;
     this.pendingSave = false;
-    
-    // Initialize cache layers
+
+    // Cache hot-path flags BEFORE _initializeCaches uses them
+    this._debug = this.options.forceDebug;
+    this._highConcurrency = this.options.concurrency > 10;
+    this._criticalThreshold = this._highConcurrency ? 0.85 : 1.0;
+    this._warningThreshold = this._highConcurrency ? 0.70 : 0.85;
+    this._infoThreshold = this._highConcurrency ? 0.60 : 0.75;
+    
+    // Initialize cache layers (may disable responseCache for high concurrency)
     this._initializeCaches();
+
+    // Cache enable flags AFTER _initializeCaches which may modify options
+    this._enablePattern = this.options.enablePatternCache;
+    this._enableResponse = this.options.enableResponseCache;
+    this._enableWhois = this.options.enableWhoisCache;
+    this._enableRequest = this.options.enableRequestCache;
     
     // Initialize statistics
     this._initializeStats();
 
+    // Cached memory usage (updated by _checkMemoryPressure interval)
+    this._lastHeapUsed = 0;
+    this._memoryPressure = false;
+
     
     // NEW: Clear request cache
-    if (this.options.enableRequestCache) {
+    if (this._enableRequest) {
       this.clearRequestCache();
     }
     
@@ -110,15 +130,15 @@ class SmartCache {
     });
     
     // Pattern matching results cache - reduce size for high concurrency
-    const patternCacheSize = this.options.concurrency > 10 ? 500 : 1000;
+    const patternCacheSize = this._highConcurrency ? 500 : 1000;
     this.patternCache = new LRUCache({
       max: patternCacheSize,
       ttl: this.options.ttl * 2 // Patterns are more stable
     });
     
     // Response content cache - aggressive limits for high concurrency
-    const responseCacheSize = this.options.concurrency > 10 ? 50 : 200;
-    const responseCacheMemory = this.options.concurrency > 10 ? 20 * 1024 * 1024 : 50 * 1024 * 1024;
+    const responseCacheSize = this._highConcurrency ? 50 : 200;
+    const responseCacheMemory = this._highConcurrency ? 20 * 1024 * 1024 : 50 * 1024 * 1024;
     this.responseCache = new LRUCache({
       max: responseCacheSize,
       ttl: 1000 * 60 * 30, // 30 minutes for response content
@@ -127,21 +147,24 @@ class SmartCache {
     });
 
     // NEW: Request-level cache for --cache-requests feature
-    if (this.options.enableRequestCache) {
+    if (this._enableRequest) {
       this.requestCache = new LRUCache({
         max: this.options.requestCacheMaxSize,
         ttl: 1000 * 60 * 15, // 15 minutes for request cache (shorter than response cache)
         maxSize: this.options.requestCacheMaxMemory,
         sizeCalculation: (value) => {
-          let size = 0;
-          if (value.headers) size += JSON.stringify(value.headers).length;
+          let size = 100; // Base overhead for object shell + metadata fields
+          if (value.headers) {
+            // Estimate header size without JSON.stringify (each header ~50 bytes avg)
+            const keys = Object.keys(value.headers);
+            size += keys.length * 50;
+          }
           if (value.body) size += value.body.length;
-          if (value.status) size += 4;
           return size;
         }
       });
       
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', `[SmartCache] Request cache initialized: ${this.options.requestCacheMaxSize} entries`));
       }
     }
@@ -149,7 +172,7 @@ class SmartCache {
     // Disable response cache entirely for very high concurrency
     if (this.options.concurrency > 15 || this.options.aggressiveMode) {
       this.options.enableResponseCache = false;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Response cache disabled for high concurrency (${this.options.concurrency})`
         ));
@@ -163,14 +186,17 @@ class SmartCache {
     });
     
     // Similarity cache - reduce for high concurrency
-    const similarityCacheSize = this.options.concurrency > 10 ? 1000 : 2000;
+    const similarityCacheSize = this._highConcurrency ? 1000 : 2000;
     this.similarityCache = new LRUCache({
       max: similarityCacheSize,
       ttl: this.options.ttl
     });
     
-    // Regex compilation cache
-    this.regexCache = new Map();
+    // Regex compilation cache (bounded to prevent unbounded growth)
+    this.regexCache = new LRUCache({ max: 500 });
+    
+    // Precompile the regex-stripping pattern (used in getCompiledRegex)
+    this._regexStripPattern = /^\/(.*)\/$/;
   }
   
   /**
@@ -210,16 +236,16 @@ class SmartCache {
    * @param {Object} context - Processing context
    * @returns {boolean} True if domain should be skipped
    */
-  shouldSkipDomain(domain, context = {}) {
+  shouldSkipDomain(domain, context = EMPTY) {
     const cacheKey = this._generateCacheKey(domain, context);
+    const cached = this.domainCache.get(cacheKey);
     
-    if (this.domainCache.has(cacheKey)) {
+    if (cached !== undefined) {
       this.stats.hits++;
-      if (this.options.forceDebug) {
-        const cached = this.domainCache.get(cacheKey);
+      if (this._debug) {
         const age = Date.now() - cached.timestamp;
         console.log(formatLogMessage('debug', 
-          `[SmartCache] Cache hit for ${domain} (age: ${Math.round(age/1000)}s, context: ${JSON.stringify(context)})`
+          `[SmartCache] Cache hit for ${domain} (age: ${Math.round(age/1000)}s)`
         ));
       }
       return true;
@@ -235,18 +261,19 @@ class SmartCache {
    * @param {Object} context - Processing context
    * @param {Object} metadata - Additional metadata to store
  */
-  markDomainProcessed(domain, context = {}, metadata = {}) {
+  markDomainProcessed(domain, context = EMPTY, metadata = EMPTY) {
     const cacheKey = this._generateCacheKey(domain, context);
-    this.domainCache.set(cacheKey, {
-      timestamp: Date.now(),
-      metadata,
-      context,
-      domain
-    });
-    
-    if (this.options.forceDebug) {
+    // Consistent property order ensures V8 reuses the same hidden class
+    const entry = { timestamp: 0, domain: '', context: null, metadata: null };
+    entry.timestamp = Date.now();
+    entry.domain = domain;
+    entry.context = context;
+    entry.metadata = metadata;
+    this.domainCache.set(cacheKey, entry);
+    
+    if (this._debug) {
       console.log(formatLogMessage('debug', 
-        `[SmartCache] Marked ${domain} as processed (context: ${JSON.stringify(context)})`
+        `[SmartCache] Marked ${domain} as processed`
       ));
     }
   }
@@ -258,20 +285,20 @@ class SmartCache {
    * @private
    */
   _normalizeUrl(url) {
-    try {
-      const parsedUrl = new URL(url);
-      // Remove fragment but preserve query params and path
-      parsedUrl.hash = '';
-      // Normalize trailing slashes: /folder/ becomes /folder
-      let pathname = parsedUrl.pathname;
-      if (pathname.length > 1 && pathname.endsWith('/')) {
-        pathname = pathname.slice(0, -1);
+    // Fast path: strip fragment with indexOf (avoids new URL() for most URLs)
+    const hashIdx = url.indexOf('#');
+    let normalized = hashIdx !== -1 ? url.substring(0, hashIdx) : url;
+    
+    // Strip trailing slash (but not root path '/')
+    if (normalized.length > 1 && normalized.charCodeAt(normalized.length - 1) === 47) { // '/'
+      // Make sure we're not stripping query string trailing slash
+      const qIdx = normalized.indexOf('?');
+      if (qIdx === -1 || normalized.length - 1 < qIdx) {
+        normalized = normalized.slice(0, -1);
       }
-      parsedUrl.pathname = pathname;
-      return parsedUrl.toString();
-    } catch (err) {
-      return url; // Return original if URL parsing fails
     }
+    
+    return normalized;
   }
   
 
@@ -284,15 +311,12 @@ class SmartCache {
    */
   _generateCacheKey(domain, context) {
     const { filterRegex, searchString, resourceType, nettools } = context;
-    const components = [
-      domain,
-      filterRegex || '',
-      searchString || '',
-      resourceType || '',
-      nettools ? 'nt' : ''
-    ].filter(Boolean);
-    
-    return components.join(':');
+    let key = domain;
+    if (filterRegex) key += ':' + filterRegex;
+    if (searchString) key += ':' + searchString;
+    if (resourceType) key += ':' + resourceType;
+    if (nettools) key += ':nt';
+    return key;
   }
   
   /**
@@ -302,7 +326,7 @@ class SmartCache {
    * @returns {boolean} True if should bypass cache
    * @private
    */
-  _shouldBypassCache(url, siteConfig = {}) {
+  _shouldBypassCache(url, siteConfig = EMPTY) {
     if (!siteConfig.bypass_cache) return false;
     
     const bypassPatterns = Array.isArray(siteConfig.bypass_cache) 
@@ -310,12 +334,8 @@ class SmartCache {
       : [siteConfig.bypass_cache];
     
     return bypassPatterns.some(pattern => {
-      try {
-        const regex = new RegExp(pattern);
-        return regex.test(url);
-      } catch (regexErr) {
-        return pattern === url; // Exact string match fallback
-      }
+      const regex = this.getCompiledRegex(pattern);
+      return regex ? regex.test(url) : pattern === url;
     });
   }
   
@@ -326,23 +346,20 @@ class SmartCache {
    * @returns {string} Cache key
    * @private
    */
-  _generateRequestCacheKey(url, options = {}) {
+  _generateRequestCacheKey(url, options = EMPTY, normalizedUrl = null) {
     const method = options.method || 'GET';
-    const normalizedUrl = this._normalizeUrl(url);
+    const normUrl = normalizedUrl || this._normalizeUrl(url);
     const headers = options.headers || {};
-    
-    // Include relevant headers that might affect the response
-    const relevantHeaders = {};
     const importantHeaders = ['accept', 'accept-encoding', 'accept-language', 'user-agent'];
     
+    // Build header portion of key by direct concat (avoids JSON.stringify + intermediate array)
+    let headerKey = '';
     for (const header of importantHeaders) {
-      if (headers[header]) {
-        relevantHeaders[header] = headers[header];
-      }
+      const val = headers[header];
+      if (val) headerKey += header + '=' + val + '&';
     }
     
-    const cacheKey = [method, normalizedUrl, JSON.stringify(relevantHeaders)].join('|');
-    return cacheKey;
+    return method + '|' + normUrl + '|' + headerKey;
   }
   
   /**
@@ -351,8 +368,8 @@ class SmartCache {
    * @param {Object} options - Request options
    * @returns {Object|null} Cached request result or null
    */
-  getCachedRequest(url, options = {}) {
-    if (!this.options.enableRequestCache) return null;
+  getCachedRequest(url, options = EMPTY) {
+    if (!this._enableRequest) return null;
     
     // Check bypass_cache setting
     if (options.siteConfig && this._shouldBypassCache(url, options.siteConfig)) {
@@ -364,16 +381,15 @@ class SmartCache {
     
     if (cached) {
       this.stats.requestCacheHits++;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Request cache hit for ${url.substring(0, 50)}... (${cached.status || 'unknown status'})`
         ));
       }
-      return {
-        ...cached,
-        fromCache: true,
-        cacheAge: Date.now() - cached.timestamp
-      };
+      // Set directly on cached object (avoids spread allocation per hit)
+      cached.fromCache = true;
+      cached.cacheAge = Date.now() - cached.timestamp;
+      return cached;
     }
     
     this.stats.requestCacheMisses++;
@@ -386,8 +402,8 @@ class SmartCache {
    * @param {Object} options - Request options
    * @param {Object} result - Request result (status, headers, body)
    */
-  cacheRequest(url, options = {}, result = {}) {
-    if (!this.options.enableRequestCache) return;
+  cacheRequest(url, options = EMPTY, result = EMPTY) {
+    if (!this._enableRequest) return;
     
     // Check bypass_cache setting
     if (options.siteConfig && this._shouldBypassCache(url, options.siteConfig)) {
@@ -400,9 +416,8 @@ class SmartCache {
       return;
     }
     
-    const memUsage = process.memoryUsage();
-    const threshold = this.options.concurrency > 10 ? 0.75 : 0.85;
-    if (memUsage.heapUsed > this.options.maxHeapUsage * threshold) {
+    // Use cached memory pressure flag (updated by periodic _checkMemoryPressure)
+    if (this._memoryPressure) {
       this.stats.requestCacheSkips++;
       this._logMemorySkip('request cache');
       return;
@@ -413,39 +428,34 @@ class SmartCache {
       return;
     }
     
-    const cacheKey = this._generateRequestCacheKey(url, options);
+    // Check size limit BEFORE building cache value object (avoids wasted allocation)
+    const method = options.method || 'GET';
+    const isHeadRequest = method.toUpperCase() === 'HEAD';
+    if (!isHeadRequest && result.body && result.body.length >= 10485760) {
+      return; // Over 10MB, skip
+    }
+
     const normalizedUrl = this._normalizeUrl(url);
-    const cacheValue = {
+    const cacheKey = this._generateRequestCacheKey(url, options, normalizedUrl);
+    
+    this.requestCache.set(cacheKey, {
       timestamp: Date.now(),
       status: result.status,
       statusText: result.statusText,
       headers: result.headers,
       body: result.body,
-      url: normalizedUrl, // Store normalized URL
-      originalUrl: url,    // Keep original for reference
-      requestOptions: {
-        method: options.method || 'GET',
-        headers: options.headers || {}
-      }
-    };
-    
-    // HEAD requests have no body, so always cache them
-    // For other requests, only cache if body is reasonable size
-    const isHeadRequest = (options.method || 'GET').toUpperCase() === 'HEAD';
-    const shouldCache = isHeadRequest || (!result.body || result.body.length < 10 * 1024 * 1024);
+      url: normalizedUrl,
+      originalUrl: url,
+      requestOptions: { method: method, headers: options.headers || {} }
+    });
     
-    if (shouldCache) {
-      this.requestCache.set(cacheKey, cacheValue);
-      
-      if (this.options.forceDebug) {
-        const bodySize = result.body ? result.body.length : 0;
-        console.log(formatLogMessage('debug', 
-          `[SmartCache] Cached request: ${normalizedUrl.substring(0, 50)}... (${result.status || 'unknown'}, ${Math.round(bodySize / 1024)}KB)`
-        ));
-        // Log path distinction if different from original
-        if (normalizedUrl !== url) {
-          console.log(formatLogMessage('debug', `[SmartCache] URL normalized: ${url} -> ${normalizedUrl}`));
-        }
+    if (this._debug) {
+      const bodySize = result.body ? result.body.length : 0;
+      console.log(formatLogMessage('debug', 
+        `[SmartCache] Cached request: ${normalizedUrl.substring(0, 50)}... (${result.status || 'unknown'}, ${Math.round(bodySize / 1024)}KB)`
+      ));
+      if (normalizedUrl !== url) {
+        console.log(formatLogMessage('debug', `[SmartCache] URL normalized: ${url} -> ${normalizedUrl}`));
       }
     }
   }
@@ -455,13 +465,13 @@ class SmartCache {
    * Useful when switching between different JSON configs
    */
   clearRequestCache() {
-    if (!this.options.enableRequestCache || !this.requestCache) return;
+    if (!this._enableRequest || !this.requestCache) return;
     
     const clearedCount = this.requestCache.size;
     this.requestCache.clear();
     this.stats.requestCacheClears++;
     
-    if (this.options.forceDebug) {
+    if (this._debug) {
       console.log(formatLogMessage('debug', 
         `[SmartCache] Cleared request cache: ${clearedCount} entries removed`
       ));
@@ -474,7 +484,7 @@ class SmartCache {
    * Get request cache statistics
    */
   getRequestCacheStats() {
-    if (!this.options.enableRequestCache || !this.requestCache) {
+    if (!this._enableRequest || !this.requestCache) {
       return { enabled: false, size: 0, hitRate: '0%', memoryUsage: 0 };
     }
     
@@ -501,25 +511,31 @@ class SmartCache {
    * @returns {Object} Enhanced statistics including path distribution
    */
   getEnhancedRequestCacheStats() {
-    if (!this.options.enableRequestCache || !this.requestCache) {
+    if (!this._enableRequest || !this.requestCache) {
       return this.getRequestCacheStats();
     }
     
     const basicStats = this.getRequestCacheStats();
     
-    // Analyze path distribution in cache
-    const pathDistribution = new Map();
+    // Analyze path distribution using string split (avoids new URL() per entry)
+    const pathDistribution = {};
     for (const [key, value] of this.requestCache.entries()) {
-      try {
-        const url = new URL(value.url || value.originalUrl);
-        const pathKey = `${url.hostname}${url.pathname}`;
-        pathDistribution.set(pathKey, (pathDistribution.get(pathKey) || 0) + 1);
-      } catch (urlErr) {
-        // Skip invalid URLs
-      }
+      const rawUrl = value.url || value.originalUrl;
+      if (!rawUrl) continue;
+      // Extract hostname+path: skip protocol, find first '/' after '://', take up to '?'
+      const protoEnd = rawUrl.indexOf('://');
+      if (protoEnd === -1) continue;
+      const pathStart = rawUrl.indexOf('/', protoEnd + 3);
+      const qIdx = rawUrl.indexOf('?', pathStart);
+      const pathKey = pathStart !== -1 
+        ? rawUrl.substring(protoEnd + 3, qIdx !== -1 ? qIdx : rawUrl.length)
+        : rawUrl.substring(protoEnd + 3);
+      pathDistribution[pathKey] = (pathDistribution[pathKey] || 0) + 1;
     }
     
-    return { ...basicStats, pathDistribution: Object.fromEntries(pathDistribution) };
+    // Assign directly instead of spread
+    basicStats.pathDistribution = pathDistribution;
+    return basicStats;
   }
 
   /**
@@ -528,24 +544,25 @@ class SmartCache {
    * @returns {RegExp} Compiled regex
    */
   getCompiledRegex(pattern) {
-    if (!this.regexCache.has(pattern)) {
-      this.stats.regexCompilations++;
-      try {
-        const regex = new RegExp(pattern.replace(/^\/(.*)\/$/, '$1'));
-        this.regexCache.set(pattern, regex);
-      } catch (err) {
-        if (this.options.forceDebug) {
-          console.log(formatLogMessage('debug', 
-            `[SmartCache] Failed to compile regex: ${pattern}`
-          ));
-        }
-        return null;
-      }
-    } else {
+    const cached = this.regexCache.get(pattern);
+    if (cached !== undefined) {
       this.stats.regexCacheHits++;
+      return cached;
     }
     
-    return this.regexCache.get(pattern);
+    this.stats.regexCompilations++;
+    try {
+      const regex = new RegExp(pattern.replace(this._regexStripPattern, '$1'));
+      this.regexCache.set(pattern, regex);
+      return regex;
+    } catch (err) {
+      if (this._debug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Failed to compile regex: ${pattern}`
+        ));
+      }
+      return null;
+    }
   }
   
   /**
@@ -555,14 +572,14 @@ class SmartCache {
    * @returns {boolean|null} Cached result or null if not cached
    */
   getCachedPatternMatch(url, pattern) {
-    if (!this.options.enablePatternCache) return null;
+    if (!this._enablePattern) return null;
     
-    const cacheKey = `${url}:${pattern}`;
+    const cacheKey = url + ':' + pattern;
     const cached = this.patternCache.get(cacheKey);
     
     if (cached !== undefined) {
       this.stats.patternHits++;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Pattern cache hit for ${url.substring(0, 50)}...`
         ));
@@ -581,9 +598,9 @@ class SmartCache {
    * @param {boolean} result - Match result
    */
   cachePatternMatch(url, pattern, result) {
-    if (!this.options.enablePatternCache) return;
+    if (!this._enablePattern) return;
     
-    const cacheKey = `${url}:${pattern}`;
+    const cacheKey = url + ':' + pattern;
     this.patternCache.set(cacheKey, result);
   }
   
@@ -593,7 +610,7 @@ class SmartCache {
    * @returns {string|null} Cached content or null
    */
   getCachedResponse(url) {
-    if (!this.options.enableResponseCache) return null;
+    if (!this._enableResponse) return null;
     
     // Note: Response cache doesn't have direct access to siteConfig
     // bypass_cache primarily affects request cache, not response cache
@@ -602,7 +619,7 @@ class SmartCache {
     const cached = this.responseCache.get(url);
     if (cached) {
       this.stats.responseHits++;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Response cache hit for ${url.substring(0, 50)}...`
         ));
@@ -620,18 +637,16 @@ class SmartCache {
    * @param {string} content - Response content
    */
   cacheResponse(url, content) {
-    if (!this.options.enableResponseCache) return;
+    if (!this._enableResponse) return;
     
     // Skip response caching entirely for very high concurrency
-    if (this.options.concurrency > 12) {
+    if (this._highConcurrency) {
       this.stats.responseCacheSkips++;
       return;
     }
     
-    // Check memory before caching large content
-    const memUsage = process.memoryUsage();
-    const threshold = this.options.concurrency > 10 ? 0.7 : 0.8; // Lower threshold for high concurrency
-    if (memUsage.heapUsed > this.options.maxHeapUsage * threshold) {
+    // Check memory before caching large content (use cached flag from periodic check)
+    if (this._memoryPressure) {
       this.stats.responseCacheSkips++;
       this._logMemorySkip('response cache');
       return;
@@ -651,14 +666,14 @@ class SmartCache {
    * @returns {Object|null} Cached result or null
    */
   getCachedNetTools(domain, tool, recordType = null) {
-    if (!this.options.enableWhoisCache) return null;
+    if (!this._enableWhois) return null;
     
-    const cacheKey = `${tool}:${domain}${recordType ? ':' + recordType : ''}`;
+    const cacheKey = recordType ? tool + ':' + domain + ':' + recordType : tool + ':' + domain;
     const cached = this.netToolsCache.get(cacheKey);
     
     if (cached) {
       this.stats.netToolsHits++;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] ${tool.toUpperCase()} cache hit for ${domain}`
         ));
@@ -678,9 +693,9 @@ class SmartCache {
    * @param {string} recordType - Record type for dig
    */
   cacheNetTools(domain, tool, result, recordType = null) {
-    if (!this.options.enableWhoisCache) return;
+    if (!this._enableWhois) return;
     
-    const cacheKey = `${tool}:${domain}${recordType ? ':' + recordType : ''}`;
+    const cacheKey = recordType ? tool + ':' + domain + ':' + recordType : tool + ':' + domain;
     this.netToolsCache.set(cacheKey, result);
   }
   
@@ -691,7 +706,8 @@ class SmartCache {
    * @param {number} similarity - Similarity score
    */
   cacheSimilarity(domain1, domain2, similarity) {
-    const key = [domain1, domain2].sort().join('|');
+    // Consistent key without array allocation: alphabetically smaller domain first
+    const key = domain1 < domain2 ? domain1 + '|' + domain2 : domain2 + '|' + domain1;
     this.similarityCache.set(key, similarity);
   }
   
@@ -702,7 +718,7 @@ class SmartCache {
    * @returns {number|null} Cached similarity or null
    */
   getCachedSimilarity(domain1, domain2) {
-    const key = [domain1, domain2].sort().join('|');
+    const key = domain1 < domain2 ? domain1 + '|' + domain2 : domain2 + '|' + domain1;
     const cached = this.similarityCache.get(key);
     
     if (cached !== undefined) {
@@ -720,31 +736,32 @@ class SmartCache {
    */
   _checkMemoryPressure() {
     const memUsage = process.memoryUsage();
-    const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
-    const maxHeapMB = Math.round(this.options.maxHeapUsage / 1024 / 1024);
-    const usagePercent = (memUsage.heapUsed / this.options.maxHeapUsage) * 100;
-    
-    // Adjust thresholds based on concurrency
-    const criticalThreshold = this.options.concurrency > 10 ? 0.85 : 1.0;
-    const warningThreshold = this.options.concurrency > 10 ? 0.70 : 0.85;
-    const infoThreshold = this.options.concurrency > 10 ? 0.60 : 0.75;
+    this._lastHeapUsed = memUsage.heapUsed;
+    const maxHeap = this.options.maxHeapUsage;
+    const heapUsedMB = Math.round(memUsage.heapUsed / 1048576);
+    const maxHeapMB = Math.round(maxHeap / 1048576);
+    const usagePercent = (memUsage.heapUsed / maxHeap) * 100;
     
     // Critical threshold - aggressive cleanup
-    if (memUsage.heapUsed > this.options.maxHeapUsage * criticalThreshold) {
+    if (memUsage.heapUsed > maxHeap * this._criticalThreshold) {
+      this._memoryPressure = true;
       this._performMemoryCleanup('critical', heapUsedMB, maxHeapMB);
       return true;
     }
     
     // Warning threshold - moderate cleanup
-    if (memUsage.heapUsed > this.options.maxHeapUsage * warningThreshold) {
+    if (memUsage.heapUsed > maxHeap * this._warningThreshold) {
+      this._memoryPressure = true;
       this._performMemoryCleanup('warning', heapUsedMB, maxHeapMB);
       return true;
     }
     
+    this._memoryPressure = false;
+    
     // Info threshold - log only
-    if (memUsage.heapUsed > this.options.maxHeapUsage * infoThreshold) {
+    if (memUsage.heapUsed > maxHeap * this._infoThreshold) {
       this.stats.memoryWarnings++;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Memory info: ${heapUsedMB}MB/${maxHeapMB}MB (${usagePercent.toFixed(1)}%)`
         ));
@@ -761,26 +778,26 @@ class SmartCache {
   _performMemoryCleanup(level, heapUsedMB, maxHeapMB) {
     this.stats.memoryPressureEvents++;
     
-    if (this.options.forceDebug) {
+    if (this._debug) {
       console.log(formatLogMessage('debug', 
         `[SmartCache] Memory ${level}: ${heapUsedMB}MB/${maxHeapMB}MB, performing cleanup...`
       ));
     }
     
-    if (level === 'critical' || this.options.concurrency > 12) {
+    if (level === 'critical' || this._highConcurrency) {
       // Aggressive cleanup - clear volatile caches
       this.responseCache.clear();
       this.patternCache.clear();
       this.similarityCache.clear();
 
       // NEW: Clear request cache during critical cleanup
-      if (this.options.enableRequestCache) this.clearRequestCache();
+      if (this._enableRequest) this.clearRequestCache();
       
       // For very high concurrency, also trim domain cache
       if (this.options.concurrency > 15) {
         const currentSize = this.domainCache.size;
         this.domainCache.clear();
-        if (this.options.forceDebug) {
+        if (this._debug) {
           console.log(formatLogMessage('debug', `[SmartCache] Cleared ${currentSize} domain cache entries`));
         }
       }
@@ -789,7 +806,7 @@ class SmartCache {
       this.responseCache.clear();
 
       // NEW: Clear request cache during warning cleanup if it's large
-      if (this.options.enableRequestCache && this.requestCache.size > this.options.requestCacheMaxSize * 0.8) {
+      if (this._enableRequest && this.requestCache.size > this.options.requestCacheMaxSize * 0.8) {
         this.clearRequestCache();
       }
     }
@@ -805,24 +822,41 @@ class SmartCache {
    * @returns {Object} Statistics object
    */
   getStats() {
-    const runtime = Date.now() - this.stats.startTime;
-    const hitRate = this.stats.hits / (this.stats.hits + this.stats.misses) || 0;
-    const patternHitRate = this.stats.patternHits / 
-      (this.stats.patternHits + this.stats.patternMisses) || 0;
-    const responseHitRate = this.stats.responseHits / 
-      (this.stats.responseHits + this.stats.responseMisses) || 0;
-    const netToolsHitRate = this.stats.netToolsHits / 
-      (this.stats.netToolsHits + this.stats.netToolsMisses) || 0;
-      
-    // NEW: Request cache hit rate
-    const requestHitRate = this.stats.requestCacheHits / 
-      (this.stats.requestCacheHits + this.stats.requestCacheMisses) || 0;
+    const s = this.stats;
+    const runtime = Date.now() - s.startTime;
+    const hitRate = s.hits / (s.hits + s.misses) || 0;
+    const patternHitRate = s.patternHits / (s.patternHits + s.patternMisses) || 0;
+    const responseHitRate = s.responseHits / (s.responseHits + s.responseMisses) || 0;
+    const netToolsHitRate = s.netToolsHits / (s.netToolsHits + s.netToolsMisses) || 0;
+    const requestHitRate = s.requestCacheHits / (s.requestCacheHits + s.requestCacheMisses) || 0;
 
-    const memUsage = process.memoryUsage();
+    // Use cached heap value from periodic _checkMemoryPressure (avoids syscall)
+    const heapUsed = this._lastHeapUsed;
+    const maxHeap = this.options.maxHeapUsage;
     
     return {
-      ...this.stats,
-      runtime: Math.round(runtime / 1000), // seconds
+      hits: s.hits,
+      misses: s.misses,
+      patternHits: s.patternHits,
+      patternMisses: s.patternMisses,
+      responseHits: s.responseHits,
+      responseMisses: s.responseMisses,
+      netToolsHits: s.netToolsHits,
+      netToolsMisses: s.netToolsMisses,
+      similarityHits: s.similarityHits,
+      similarityMisses: s.similarityMisses,
+      regexCompilations: s.regexCompilations,
+      regexCacheHits: s.regexCacheHits,
+      persistenceLoads: s.persistenceLoads,
+      persistenceSaves: s.persistenceSaves,
+      memoryPressureEvents: s.memoryPressureEvents,
+      memoryWarnings: s.memoryWarnings,
+      responseCacheSkips: s.responseCacheSkips,
+      requestCacheHits: s.requestCacheHits,
+      requestCacheMisses: s.requestCacheMisses,
+      requestCacheSkips: s.requestCacheSkips,
+      requestCacheClears: s.requestCacheClears,
+      runtime: Math.round(runtime / 1000),
       hitRate: (hitRate * 100).toFixed(2) + '%',
       patternHitRate: (patternHitRate * 100).toFixed(2) + '%',
       responseHitRate: (responseHitRate * 100).toFixed(2) + '%',
@@ -833,19 +867,18 @@ class SmartCache {
       netToolsCacheSize: this.netToolsCache.size,
       similarityCacheSize: this.similarityCache.size,
       regexCacheSize: this.regexCache.size,
-      // NEW: Request cache statistics
-      requestHitRate: this.options.enableRequestCache ? 
+      requestHitRate: this._enableRequest ? 
         (requestHitRate * 100).toFixed(2) + '%' : '0% (disabled)',
-      requestCacheSize: this.options.enableRequestCache ? this.requestCache.size : 0,
-      requestCacheMemoryMB: this.options.enableRequestCache ? 
-        Math.round((this.requestCache.calculatedSize || 0) / 1024 / 1024) : 0,
+      requestCacheSize: this._enableRequest ? this.requestCache.size : 0,
+      requestCacheMemoryMB: this._enableRequest ? 
+        Math.round((this.requestCache.calculatedSize || 0) / 1048576) : 0,
       totalCacheEntries: this.domainCache.size + this.patternCache.size + 
         this.responseCache.size + this.netToolsCache.size + 
-        this.similarityCache.size + this.regexCache.size + (this.options.enableRequestCache ? this.requestCache.size : 0),
-      memoryUsageMB: Math.round(memUsage.heapUsed / 1024 / 1024),
-      memoryMaxMB: Math.round(this.options.maxHeapUsage / 1024 / 1024),
-      memoryUsagePercent: ((memUsage.heapUsed / this.options.maxHeapUsage) * 100).toFixed(1) + '%',
-      responseCacheMemoryMB: Math.round((this.responseCache.calculatedSize || 0) / 1024 / 1024)
+        this.similarityCache.size + this.regexCache.size + (this._enableRequest ? this.requestCache.size : 0),
+      memoryUsageMB: Math.round(heapUsed / 1048576),
+      memoryMaxMB: Math.round(maxHeap / 1048576),
+      memoryUsagePercent: ((heapUsed / maxHeap) * 100).toFixed(1) + '%',
+      responseCacheMemoryMB: Math.round((this.responseCache.calculatedSize || 0) / 1048576)
     };
   }
   
@@ -859,12 +892,12 @@ class SmartCache {
     this.netToolsCache.clear();
     this.similarityCache.clear();
     this.regexCache.clear();
-    if (this.options.enableRequestCache && this.requestCache) {
+    if (this._enableRequest && this.requestCache) {
       this.requestCache.clear();
     }
     this._initializeStats();
     
-    if (this.options.forceDebug) {
+    if (this._debug) {
       console.log(formatLogMessage('debug', '[SmartCache] All caches cleared'));
     }
   }
@@ -874,7 +907,7 @@ class SmartCache {
    * @private
    */
   _logMemorySkip(operation) {
-    if (this.options.forceDebug) {
+    if (this._debug) {
       console.log(formatLogMessage('debug', 
         `[SmartCache] Skipping ${operation} due to memory pressure`
       ));
@@ -888,17 +921,21 @@ class SmartCache {
   _loadPersistentCache() {
     const cacheFile = path.join(this.options.persistencePath, 'smart-cache.json');
     
-    if (!fs.existsSync(cacheFile)) {
+    let raw;
+    try {
+      raw = fs.readFileSync(cacheFile, 'utf8');
+    } catch (readErr) {
+      // File doesn't exist or unreadable -- nothing to load
       return;
     }
     
     try {
-      const data = JSON.parse(fs.readFileSync(cacheFile, 'utf8'));
+      const data = JSON.parse(raw);
       const now = Date.now();
       
       // Validate cache age
       if (data.timestamp && now - data.timestamp > 24 * 60 * 60 * 1000) {
-        if (this.options.forceDebug) {
+        if (this._debug) {
           console.log(formatLogMessage('debug', 
             '[SmartCache] Persistent cache too old, ignoring'
           ));
@@ -908,30 +945,31 @@ class SmartCache {
       
       // Load domain cache
       if (data.domainCache && Array.isArray(data.domainCache)) {
-        data.domainCache.forEach(([key, value]) => {
-          // Only load if not expired
-          if (now - value.timestamp < this.options.ttl) {
-            this.domainCache.set(key, value);
+        const entries = data.domainCache;
+        for (let i = 0; i < entries.length; i++) {
+          if (now - entries[i][1].timestamp < this.options.ttl) {
+            this.domainCache.set(entries[i][0], entries[i][1]);
           }
-        });
+        }
       }
       
       // Load nettools cache
       if (data.netToolsCache && Array.isArray(data.netToolsCache)) {
-        data.netToolsCache.forEach(([key, value]) => {
-          this.netToolsCache.set(key, value);
-        });
+        const entries = data.netToolsCache;
+        for (let i = 0; i < entries.length; i++) {
+          this.netToolsCache.set(entries[i][0], entries[i][1]);
+        }
       }
       
       this.stats.persistenceLoads++;
       
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Loaded persistent cache: ${this.domainCache.size} domains, ${this.netToolsCache.size} nettools`
         ));
       }
     } catch (err) {
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Failed to load persistent cache: ${err.message}`
         ));
@@ -948,7 +986,7 @@ class SmartCache {
     // Prevent concurrent saves
     if (this.saveInProgress) {
       this.pendingSave = true;
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', '[SmartCache] Save in progress, marking pending...'));
       }
       return;
@@ -975,12 +1013,11 @@ class SmartCache {
     
     const cacheDir = this.options.persistencePath;
     const cacheFile = path.join(cacheDir, 'smart-cache.json');
+    const tmpFile = cacheFile + '.tmp';
     
     try {
-      // Create cache directory if it doesn't exist
-      if (!fs.existsSync(cacheDir)) {
-        fs.mkdirSync(cacheDir, { recursive: true });
-      }
+      // recursive:true is a no-op if dir exists -- no need for existsSync check
+      fs.mkdirSync(cacheDir, { recursive: true });
       
       const data = {
         timestamp: now,
@@ -989,21 +1026,52 @@ class SmartCache {
         stats: this.stats
       };
       
-      fs.writeFileSync(cacheFile, JSON.stringify(data, null, 2));
-      this.stats.persistenceSaves++;
+      // Async write to temp file, then atomic rename (no pretty-print -- saves ~30% serialization time)
+      const jsonStr = JSON.stringify(data);
+      fs.writeFile(tmpFile, jsonStr, (writeErr) => {
+        if (writeErr) {
+          if (this._debug) {
+            console.log(formatLogMessage('debug', 
+              `[SmartCache] Failed to write cache temp file: ${writeErr.message}`
+            ));
+          }
+          this.saveInProgress = false;
+          return;
+        }
+        
+        fs.rename(tmpFile, cacheFile, (renameErr) => {
+          if (renameErr) {
+            if (this._debug) {
+              console.log(formatLogMessage('debug', 
+                `[SmartCache] Failed to rename cache file: ${renameErr.message}`
+              ));
+            }
+          } else {
+            this.stats.persistenceSaves++;
+            if (this._debug) {
+              console.log(formatLogMessage('debug', 
+                `[SmartCache] Saved cache to disk: ${cacheFile}`
+              ));
+            }
+          }
+          
+          this.saveInProgress = false;
+          
+          // Process any pending saves
+          if (this.pendingSave && !this.saveTimeout) {
+            this.pendingSave = false;
+            setTimeout(() => this.savePersistentCache(), 1000);
+          }
+        });
+      });
+      return; // Async -- don't fall through to finally
       
-      if (this.options.forceDebug) {
-        console.log(formatLogMessage('debug', 
-          `[SmartCache] Saved cache to disk: ${cacheFile}`
-        ));
-      }
     } catch (err) {
-      if (this.options.forceDebug) {
+      if (this._debug) {
         console.log(formatLogMessage('debug', 
           `[SmartCache] Failed to save cache: ${err.message}`
         ));
       }
-    } finally {
       this.saveInProgress = false;
       
       // Process any pending saves
@@ -1054,15 +1122,9 @@ class SmartCache {
    * @param {boolean} options.forceDebug - Enable debug logging
    * @returns {Object} Clear operation results
    */
-  static clearPersistentCache(options = {}) {
+  static clearPersistentCache(options = EMPTY) {
     const { silent = false, forceDebug = false, cachePath = '.cache' } = options;
     
-    const cachePaths = [
-      cachePath,
-      path.join(cachePath, 'smart-cache.json'),
-      // Add other potential cache files here if needed
-    ];
-    
     let clearedItems = 0;
     let totalSize = 0;
     const clearedFiles = [];
@@ -1072,40 +1134,58 @@ class SmartCache {
       console.log(`\n???  Clearing cache...`);
     }
     
-    for (const currentCachePath of cachePaths) {
-      if (fs.existsSync(currentCachePath)) {
+    // Try the directory first -- rmSync recursive handles all files inside
+    const cacheFile = path.join(cachePath, 'smart-cache.json');
+    let dirHandled = false;
+    
+    try {
+      const stats = fs.statSync(cachePath);
+      if (stats.isDirectory()) {
+        // Sum file sizes (readdirSync proves they exist -- no existsSync needed)
         try {
-          const stats = fs.statSync(currentCachePath);
-          if (stats.isDirectory()) {
-            // Calculate total size of directory contents
-            const files = fs.readdirSync(currentCachePath);
-            for (const file of files) {
-              const filePath = path.join(currentCachePath, file);
-              if (fs.existsSync(filePath)) {
-                totalSize += fs.statSync(filePath).size;
-              }
-            }
-            fs.rmSync(currentCachePath, { recursive: true, force: true });
-            clearedItems++;
-            clearedFiles.push({ type: 'directory', path: currentCachePath, size: totalSize });
-            if (forceDebug) {
-              console.log(formatLogMessage('debug', `Cleared cache directory: ${currentCachePath}`));
-            }
-          } else {
-            totalSize += stats.size;
-            fs.unlinkSync(currentCachePath);
-            clearedItems++;
-            clearedFiles.push({ type: 'file', path: currentCachePath, size: stats.size });
-            if (forceDebug) {
-              console.log(formatLogMessage('debug', `Cleared cache file: ${currentCachePath}`));
-            }
-          }
-        } catch (clearErr) {
-          errors.push({ path: currentCachePath, error: clearErr.message });
-          if (forceDebug) {
-            console.log(formatLogMessage('debug', `Failed to clear ${currentCachePath}: ${clearErr.message}`));
+          const files = fs.readdirSync(cachePath);
+          for (const file of files) {
+            try {
+              totalSize += fs.statSync(path.join(cachePath, file)).size;
+            } catch (e) { /* file disappeared between readdir and stat */ }
           }
+        } catch (e) { /* empty dir or read error */ }
+        fs.rmSync(cachePath, { recursive: true, force: true });
+        dirHandled = true;
+        clearedItems++;
+        clearedFiles.push({ type: 'directory', path: cachePath, size: totalSize });
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Cleared cache directory: ${cachePath}`));
         }
+      } else {
+        totalSize += stats.size;
+        fs.unlinkSync(cachePath);
+        dirHandled = true;
+        clearedItems++;
+        clearedFiles.push({ type: 'file', path: cachePath, size: stats.size });
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Cleared cache file: ${cachePath}`));
+        }
+      }
+    } catch (clearErr) {
+      if (clearErr.code !== 'ENOENT') {
+        errors.push({ path: cachePath, error: clearErr.message });
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Failed to clear ${cachePath}: ${clearErr.message}`));
+        }
+      }
+    }
+    
+    // Fallback: if directory was missing or failed, try the json file directly (orphaned file safety net)
+    if (!dirHandled) {
+      try {
+        const fileStats = fs.statSync(cacheFile);
+        totalSize += fileStats.size;
+        fs.unlinkSync(cacheFile);
+        clearedItems++;
+        clearedFiles.push({ type: 'file', path: cacheFile, size: fileStats.size });
+      } catch (e) {
+        // ENOENT = nothing to clean, any other error is ignorable
       }
     }