@fanboynz/network-scanner 2.0.66 → 3.0.1

package/lib/adblock.js

@@ -4,6 +4,10 @@
 
 const fs = require('fs');
 const psl = require('psl');
+const { formatLogMessage, messageColors } = require('./colorize');
+// Subsystem tag matches the project convention (other modules use
+// flowproxy/cloudflare/curl/grep/etc. precomputed tags).
+const ADBLOCK_TAG = messageColors.processing('[adblock]');
 
 // Hoisted constants — avoid recreating per rule (~80K times for EasyList)
 const COSMETIC_OPTIONS = new Set(['generichide', 'elemhide', 'specifichide', 'genericblock']);
@@ -15,32 +19,41 @@ const PARSE_TYPE_MAP = {
 };
 
 /**
- * Simple LRU cache for URL parsing results
- * Prevents memory leaks with fixed size limit
+ * FIFO cache with a hard size cap. Evicts the OLDEST-inserted entry on
+ * overflow (not the least-recently-used) — this is FIFO, not LRU, despite
+ * the original "Simple LRU cache" naming. Proper LRU would require
+ * deleting-and-reinserting on every .get() hit to bubble accessed keys
+ * to the end, which adds per-hit Map.delete + Map.set overhead. In hot
+ * paths (32k-entry result cache hit thousands of times per scan), that
+ * overhead is real; FIFO with a generous cap is the simpler trade.
+ *
+ * If access-pattern shifts become a real problem (a few "always hot"
+ * URLs evicted while never-used entries survive), implement move-on-get
+ * here — the API stays compatible.
  */
-class URLCache {
+class FIFOCache {
   constructor(maxSize = 1000) {
     this.cache = new Map();
     this.maxSize = maxSize;
   }
-  
+
   get(url) {
     return this.cache.get(url);
   }
-  
+
   set(url, value) {
-    // LRU eviction: if at max size, delete oldest entry
+    // FIFO eviction: drop the oldest-inserted entry when full.
     if (this.cache.size >= this.maxSize) {
       const firstKey = this.cache.keys().next().value;
       this.cache.delete(firstKey);
     }
     this.cache.set(url, value);
   }
-  
+
   clear() {
     this.cache.clear();
   }
-  
+
   getStats() {
     return {
       size: this.cache.size,
@@ -56,10 +69,12 @@ class URLCache {
  * @returns {Object} Rule matcher with matching functions
  */
 function parseAdblockRules(filePath, options = {}) {
-  const {
-    enableLogging = false,
-    caseSensitive = false
-  } = options;
+  // caseSensitive option removed — it was destructured here and threaded
+  // through to createMatcher, then never actually read. All hostname/
+  // pattern comparisons are case-insensitive (lowercased explicitly or
+  // via /i regex flag), so the option would have been a substantial
+  // refactor to honor. Removed rather than left as a documented lie.
+  const { enableLogging = false } = options;
 
   let fileContent;
   try {
@@ -175,27 +190,27 @@ function parseAdblockRules(filePath, options = {}) {
     } catch (err) {
       rules.stats.invalid++;
       if (enableLogging) {
-        console.log(`[Adblock] Failed to parse rule: ${line} - ${err.message}`);
+        console.log(formatLogMessage('warn', `${ADBLOCK_TAG} Failed to parse rule: ${line} - ${err.message}`));
       }
     }
   }
 
   if (enableLogging) {
-    console.log(`[Adblock] Loaded ${rules.stats.total} rules:`);
-    console.log(`  - Domain rules: ${rules.stats.domain}`);
-    console.log(`    • Exact matches (Map): ${rules.stats.domainMapEntries}`);
-    console.log(`    • Wildcard patterns (Array): ${rules.domainRules.length}`);
-    console.log(`  - Third-party rules: ${rules.stats.thirdParty}`);
-    console.log(`  - First-party rules: ${rules.stats.firstParty}`);
-    console.log(`  - Path rules: ${rules.stats.path}`);
-    console.log(`  - Script rules: ${rules.stats.script}`);
-    console.log(`  - Regex rules: ${rules.stats.regex}`);
-    console.log(`  - Whitelist rules: ${rules.stats.whitelist}`);
-    console.log(`  - Comments/Element hiding: ${rules.stats.comments + rules.stats.elementHiding}`);
-    console.log(`  - Invalid rules: ${rules.stats.invalid}`);
+    console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Loaded ${rules.stats.total} rules:`));
+    console.log(formatLogMessage('debug', `  - Domain rules: ${rules.stats.domain}`));
+    console.log(formatLogMessage('debug', `    • Exact matches (Map): ${rules.stats.domainMapEntries}`));
+    console.log(formatLogMessage('debug', `    • Wildcard patterns (Array): ${rules.domainRules.length}`));
+    console.log(formatLogMessage('debug', `  - Third-party rules: ${rules.stats.thirdParty}`));
+    console.log(formatLogMessage('debug', `  - First-party rules: ${rules.stats.firstParty}`));
+    console.log(formatLogMessage('debug', `  - Path rules: ${rules.stats.path}`));
+    console.log(formatLogMessage('debug', `  - Script rules: ${rules.stats.script}`));
+    console.log(formatLogMessage('debug', `  - Regex rules: ${rules.stats.regex}`));
+    console.log(formatLogMessage('debug', `  - Whitelist rules: ${rules.stats.whitelist}`));
+    console.log(formatLogMessage('debug', `  - Comments/Element hiding: ${rules.stats.comments + rules.stats.elementHiding}`));
+    console.log(formatLogMessage('debug', `  - Invalid rules: ${rules.stats.invalid}`));
   }
 
-  return createMatcher(rules, { enableLogging, caseSensitive });
+  return createMatcher(rules, { enableLogging });
 }
 
 /**
@@ -229,12 +244,13 @@ function parseRule(rule, isWhitelist, enableLogging = false) {
     const options = optionsStr.split(',');
     const parsedOptions = {};
 
+    // No cosmetic-option guard here — the line-level filter at lines
+    // ~126-135 already drops any rule containing $generichide/$elemhide/
+    // etc. before parseRule is ever called. The previous in-loop
+    // COSMETIC_OPTIONS.has() guard was unreachable.
     for (const opt of options) {
       const [key, value] = opt.split('=');
-      const trimmedKey = key.trim();
-      if (!COSMETIC_OPTIONS.has(trimmedKey)) {
-        parsedOptions[trimmedKey] = value ? value.trim() : true;
-      }
+      parsedOptions[key.trim()] = value ? value.trim() : true;
     }
 
     // Check for third-party option
@@ -359,10 +375,12 @@ function createDomainMatcher(domain) {
 }
 
 /**
- * Shared regex cache — deduplicates identical compiled patterns across rules
- * Large lists (EasyList ~80K rules) often have thousands of duplicate patterns
+ * Shared regex cache — deduplicates identical compiled patterns across rules.
+ * Large lists (EasyList ~80K rules) often have thousands of duplicate
+ * patterns. Bounded via FIFOCache (was a bare Map with no size cap; for
+ * giant lists with many unique patterns it could grow unbounded).
  */
-const _regexCache = new Map();
+const _regexCache = new FIFOCache(20000);
 
 /**
  * Creates a pattern matcher for path/wildcard rules
@@ -370,8 +388,12 @@ const _regexCache = new Map();
  * @returns {Function} Matcher function
  */
 function createPatternMatcher(pattern) {
-  // Check cache for already-compiled identical pattern
-  const cached = _regexCache.get(pattern);
+  // Capture the ORIGINAL pattern as the cache key — the local `pattern`
+  // var gets mutated by the anchor-strip below, so previously the get()
+  // used the pre-strip key but set() stored under the post-strip key,
+  // making every anchored pattern miss its own cache on the next lookup.
+  const cacheKey = pattern;
+  const cached = _regexCache.get(cacheKey);
   if (cached) return cached;
 
   // Convert adblock pattern to regex
@@ -401,7 +423,7 @@ function createPatternMatcher(pattern) {
 
   const regex = new RegExp(regexPattern, 'i');
   const matcher = (url) => regex.test(url);
-  _regexCache.set(pattern, matcher);
+  _regexCache.set(cacheKey, matcher);
   return matcher;
 }
 
@@ -412,15 +434,21 @@ function createPatternMatcher(pattern) {
  * @returns {Object} Matcher with shouldBlock function
  */
 function createMatcher(rules, options = {}) {
-  const { enableLogging = false, caseSensitive = false } = options;
+  const { enableLogging = false } = options;
   
-  const urlCache = new URLCache(16000);
-  let cacheHits = 0;
-  let cacheMisses = 0;
+  const urlCache = new FIFOCache(16000);
+  // Per-cache counters split out. Previously ONE counter pair received
+  // hits/misses from THREE caches: the result cache, the request-URL
+  // parse cache, AND the source-URL parse cache — getStats().hitRate
+  // was the average of three different cache behaviors and not
+  // interpretable for diagnosing performance.
+  let resultCacheHits = 0, resultCacheMisses = 0;
+  let urlCacheHits = 0, urlCacheMisses = 0;
+  let sourceCacheHits = 0, sourceCacheMisses = 0;
   const hasPartyRules = rules.thirdPartyRules.length > 0 || rules.firstPartyRules.length > 0;
-  // Result cache with LRU eviction — evicts oldest entries one at a time
-  // instead of clearing everything when full
-  const resultCache = new URLCache(32000);
+  // Result cache uses FIFO eviction (see FIFOCache class comment) —
+  // evicts oldest entries one at a time instead of clearing everything.
+  const resultCache = new FIFOCache(32000);
 
   function resultCacheGet(url, sourceUrl, resourceType) {
     return resultCache.get(url + '\0' + sourceUrl + '\0' + resourceType);
@@ -445,30 +473,30 @@ function createMatcher(rules, options = {}) {
         // Check result cache — same URL+source+type always produces same result
         const cachedResult = resultCacheGet(url, sourceUrl, resourceType);
         if (cachedResult) {
-          cacheHits++;
+          resultCacheHits++;
           return cachedResult;
         }
-        cacheMisses++;
+        resultCacheMisses++;
 
         // OPTIMIZATION: Check cache first for URL parsing (60% faster)
         let cachedData = urlCache.get(url);
         let hostname, lowerHostname;
-        
+
         if (cachedData) {
           hostname = cachedData.hostname;
           lowerHostname = cachedData.lowerHostname;
-          cacheHits++;
+          urlCacheHits++;
         } else {
           // Parse URL and cache result
           const urlObj = new URL(url);
           hostname = urlObj.hostname;
           lowerHostname = hostname.toLowerCase();
-          
+
           urlCache.set(url, {
             hostname,
             lowerHostname
           });
-          cacheMisses++;
+          urlCacheMisses++;
         }
         
         // Lazy parent domain computation — only built when exact Map lookup misses
@@ -491,19 +519,19 @@ function createMatcher(rules, options = {}) {
           
           if (cachedSourceData) {
             sourceDomain = cachedSourceData.lowerHostname;
-            cacheHits++;
+            sourceCacheHits++;
           } else {
             // Parse and cache sourceUrl
             try {
               const sourceUrlObj = new URL(sourceUrl);
               sourceDomain = sourceUrlObj.hostname.toLowerCase();
-              
+
               // Cache sourceUrl parsing result (same as request URLs)
               urlCache.set(sourceUrl, {
                 hostname: sourceUrlObj.hostname,
                 lowerHostname: sourceDomain
               });
-              cacheMisses++;
+              sourceCacheMisses++;
             } catch (err) {
             // Invalid sourceUrl, leave as null
             }
@@ -522,7 +550,7 @@ function createMatcher(rules, options = {}) {
         if (rule) {
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Whitelisted: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Whitelisted: ${url} (${rule.raw || rule.pattern})`));
             }
             const r = { blocked: false, rule: rule.raw || rule.pattern, reason: 'whitelisted' };
             resultCacheSet(url, sourceUrl, resourceType, r);
@@ -537,7 +565,7 @@ function createMatcher(rules, options = {}) {
           if (rule) {
             if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
               if (enableLogging) {
-                console.log(`[Adblock] Whitelisted: ${url} (${rule.raw || rule.pattern})`);
+                console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Whitelisted: ${url} (${rule.raw || rule.pattern})`));
               }
               const r = { blocked: false, rule: rule.raw || rule.pattern, reason: 'whitelisted' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -552,7 +580,7 @@ function createMatcher(rules, options = {}) {
           const rule = rules.whitelist[i];
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Whitelisted: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Whitelisted: ${url} (${rule.raw || rule.pattern})`));
             }
               const r = { blocked: false, rule: rule.raw || rule.pattern, reason: 'whitelisted' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -567,7 +595,7 @@ function createMatcher(rules, options = {}) {
         if (rule) {
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Blocked domain: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked domain: ${url} (${rule.raw || rule.pattern})`));
             }
             const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'domain_rule' };
             resultCacheSet(url, sourceUrl, resourceType, r);
@@ -581,7 +609,7 @@ function createMatcher(rules, options = {}) {
           if (rule) {
             if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
               if (enableLogging) {
-                console.log(`[Adblock] Blocked domain: ${url} (${rule.raw || rule.pattern})`);
+                console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked domain: ${url} (${rule.raw || rule.pattern})`));
               }
               const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'domain_rule' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -596,7 +624,7 @@ function createMatcher(rules, options = {}) {
           const rule = rules.domainRules[i];
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Blocked domain: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked domain: ${url} (${rule.raw || rule.pattern})`));
             }
             const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'domain_rule' };
             resultCacheSet(url, sourceUrl, resourceType, r);
@@ -611,7 +639,7 @@ function createMatcher(rules, options = {}) {
             const rule = rules.thirdPartyRules[i];
             if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
               if (enableLogging) {
-                console.log(`[Adblock] Blocked third-party: ${url} (${rule.raw || rule.pattern})`);
+                console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked third-party: ${url} (${rule.raw || rule.pattern})`));
               }
               const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'third_party_rule' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -627,7 +655,7 @@ function createMatcher(rules, options = {}) {
             const rule = rules.firstPartyRules[i];
             if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
               if (enableLogging) {
-                console.log(`[Adblock] Blocked first-party: ${url} (${rule.raw || rule.pattern})`);
+                console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked first-party: ${url} (${rule.raw || rule.pattern})`));
               }
               const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'first_party_rule' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -643,7 +671,7 @@ function createMatcher(rules, options = {}) {
             const rule = rules.scriptRules[i];
            if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
               if (enableLogging) {
-                console.log(`[Adblock] Blocked script: ${url} (${rule.raw || rule.pattern})`);
+                console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked script: ${url} (${rule.raw || rule.pattern})`));
               }
               const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'script_rule' };
               resultCacheSet(url, sourceUrl, resourceType, r);
@@ -658,7 +686,7 @@ function createMatcher(rules, options = {}) {
           const rule = rules.pathRules[i];
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Blocked path: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked path: ${url} (${rule.raw || rule.pattern})`));
             }
             const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'path_rule' };
             resultCacheSet(url, sourceUrl, resourceType, r);
@@ -672,7 +700,7 @@ function createMatcher(rules, options = {}) {
           const rule = rules.regexRules[i];
           if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
             if (enableLogging) {
-              console.log(`[Adblock] Blocked regex: ${url} (${rule.raw || rule.pattern})`);
+              console.log(formatLogMessage('debug', `${ADBLOCK_TAG} Blocked regex: ${url} (${rule.raw || rule.pattern})`));
             }
             const r = { blocked: true, rule: rule.raw || rule.pattern, reason: 'regex_rule' };
             resultCacheSet(url, sourceUrl, resourceType, r);
@@ -687,7 +715,7 @@ function createMatcher(rules, options = {}) {
 
       } catch (err) {
         if (enableLogging) {
-          console.log(`[Adblock] Error checking ${url}: ${err.message}`);
+          console.log(formatLogMessage('warn', `${ADBLOCK_TAG} Error checking ${url}: ${err.message}`));
         }
         // On error, allow request
         return { 
@@ -703,19 +731,34 @@ function createMatcher(rules, options = {}) {
      * @returns {Object} Statistics object
      */
     getStats() {
-      const hitRate = cacheHits + cacheMisses > 0 
-        ? ((cacheHits / (cacheHits + cacheMisses)) * 100).toFixed(1) + '%'
-        : '0%';
-      
-      return { 
+      // Per-cache hit rates instead of one lumped figure. The previous
+      // single `hitRate` averaged three different cache behaviors and
+      // wasn't actionable for diagnosing performance.
+      const rate = (hits, misses) =>
+        hits + misses > 0 ? ((hits / (hits + misses)) * 100).toFixed(1) + '%' : '0%';
+
+      return {
         ...rules.stats,
         cache: {
-          hits: cacheHits,
-          misses: cacheMisses,
-          hitRate: hitRate,
-          urlCacheSize: urlCache.cache.size,
-          resultCacheSize: resultCache.cache.size,
-          maxSize: urlCache.maxSize
+          result: {
+            hits: resultCacheHits,
+            misses: resultCacheMisses,
+            hitRate: rate(resultCacheHits, resultCacheMisses),
+            size: resultCache.cache.size,
+            maxSize: resultCache.maxSize
+          },
+          url: {
+            hits: urlCacheHits,
+            misses: urlCacheMisses,
+            hitRate: rate(urlCacheHits, urlCacheMisses),
+            size: urlCache.cache.size,
+            maxSize: urlCache.maxSize
+          },
+          source: {
+            hits: sourceCacheHits,
+            misses: sourceCacheMisses,
+            hitRate: rate(sourceCacheHits, sourceCacheMisses)
+          }
         }
       };
     }
@@ -849,24 +892,26 @@ function matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDoma
 }
 
 /**
- * Extract base domain from hostname using Public Suffix List
- * Correctly handles multi-part TLDs like .co.uk, .com.au, .com.br
+ * Extract base domain from hostname using Public Suffix List.
+ * Correctly handles multi-part TLDs like .co.uk, .com.au, .com.br.
  * @param {string} hostname - Full hostname
  * @returns {string} Base domain
  */
-const _baseDomainCache = new Map();
+// FIFOCache (not the previous clear-all-when-full Map). The old code
+// nuked all 10000 cached entries on overflow, forcing the next 10000
+// hosts to re-pay the psl.parse cost. FIFO evicts one entry at a time.
+const _baseDomainCache = new FIFOCache(10000);
 function getBaseDomain(hostname) {
   const cached = _baseDomainCache.get(hostname);
   if (cached) return cached;
   const parsed = psl.parse(hostname);
   const result = (parsed && parsed.domain) ? parsed.domain : hostname;
-  // Cap cache size
-  if (_baseDomainCache.size > 10000) _baseDomainCache.clear();
   _baseDomainCache.set(hostname, result);
   return result;
 }
 
+// Public surface. getBaseDomain stays internal — used by createMatcher's
+// third-party check but no external caller ever imported it.
 module.exports = {
-  parseAdblockRules,
-  getBaseDomain
+  parseAdblockRules
 };