@fanboynz/network-scanner 2.0.66 → 3.0.0

package/lib/ignore_similar.js

@@ -1,69 +1,42 @@
-const { formatLogMessage } = require('./colorize');
+const psl = require('psl');
+const { formatLogMessage, messageColors } = require('./colorize');
+const IGNORE_SIMILAR_TAG = messageColors.processing('[ignore_similar]');
 
-// Precompiled regex (avoids recompilation per getBaseDomainName call)
+// Strip protocol before handing to psl.parse, which expects a bare
+// hostname per Public Suffix List semantics. psl handles 'www.' as a
+// subdomain naturally (no need for a separate strip).
 const REGEX_PROTOCOL = /^https?:\/\//;
-const REGEX_WWW = /^www\./;
-
-// Multi-part TLD lookup (module-level Set, O(1) instead of per-call array + O(n) .includes)
-const MULTI_PART_TLDS = new Set([
-  'co.uk', 'co.nz', 'com.au', 'co.za', 'co.in', 'co.jp', 'co.kr',
-  'com.br', 'com.mx', 'com.ar', 'com.co', 'com.pe', 'com.ve',
-  'co.th', 'co.id', 'co.il', 'co.ke', 'co.tz', 'co.zw', 'co.bw',
-  'com.sg', 'com.my', 'com.hk', 'com.tw', 'com.ph', 'com.vn',
-  'co.cr', 'co.ug', 'co.zm', 'co.ao', 'co.mz', 'co.ls',
-  'org.uk', 'me.uk', 'ltd.uk', 'plc.uk', 'gov.uk', 'ac.uk', 'sch.uk',
-  'com.de', 'org.de', 'com.fr', 'org.fr', 'com.es', 'org.es',
-  'com.it', 'org.it', 'com.pl', 'org.pl', 'com.nl', 'org.nl',
-  'com.ru', 'org.ru', 'com.ua', 'org.ua', 'com.tr', 'org.tr',
-  'or.jp', 'ne.jp', 'ac.jp', 'ed.jp', 'go.jp',
-  'or.kr', 'ne.kr', 'com.cn', 'org.cn', 'net.cn', 'edu.cn', 'gov.cn',
-  'org.in', 'net.in', 'org.au', 'net.au', 'edu.au', 'gov.au',
-  'org.nz', 'net.nz', 'org.il', 'net.il', 'org.za', 'net.za',
-  'org.br', 'net.br', 'edu.br', 'gov.br', 'org.ar', 'org.mx',
-  'org.co', 'org.pe', 'com.cl', 'org.cl', 'com.uy', 'org.uy',
-  'org.ve', 'com.do', 'org.do', 'com.pr', 'org.pr',
-  'com.gt', 'org.gt', 'com.pa', 'org.pa', 'com.sv', 'org.sv',
-  'com.ni', 'org.ni', 'com.hn', 'org.hn', 'org.cr',
-  'com.eg', 'org.eg', 'or.ke'
-]);
-
-// 3-part TLD lookup
-const THREE_PART_TLDS = new Set(['com.au.com', 'co.uk.com']);
 
 /**
- * Extracts the base domain name without TLD for similarity comparison
+ * Extracts the base domain name (sld) without TLD for similarity comparison.
+ *
+ * Uses the project's `psl` dependency — the canonical Public Suffix List
+ * parser, maintained against the live Mozilla list. Replaces a hand-curated
+ * ~80-entry MULTI_PART_TLDS Set that went stale as PSL changed, plus a
+ * THREE_PART_TLDS set that only listed two entries (both vanity domains
+ * 'com.au.com'/'co.uk.com', not real public suffixes). The rest of the
+ * codebase already uses psl (nwss.js, lib/post-processing.js, etc.) — this
+ * brings ignore_similar in line.
+ *
  * @param {string} domain - The domain to process
- * @returns {string} The base domain name
+ * @returns {string} The base domain name (sld), e.g. 'example' for
+ *   'www.example.co.uk'. Returns '' for invalid input; falls back to
+ *   second-to-last token for hostnames psl can't parse (IPs, single-token
+ *   hosts, unlisted TLDs).
  */
 function getBaseDomainName(domain) {
   if (!domain || typeof domain !== 'string') {
     return '';
   }
-  
-  domain = domain.replace(REGEX_PROTOCOL, '');
-  domain = domain.replace(REGEX_WWW, '');
-  
-  const parts = domain.split('.');
-  if (parts.length < 2) {
-    return domain;
+  const hostname = domain.replace(REGEX_PROTOCOL, '');
+  const parsed = psl.parse(hostname);
+  if (parsed && parsed.sld) {
+    return parsed.sld;
   }
-  
-  // Check multi-part TLD (O(1) Set lookup instead of O(n) array scan)
-  const lastTwoParts = parts[parts.length - 2] + '.' + parts[parts.length - 1];
-  
-  if (MULTI_PART_TLDS.has(lastTwoParts)) {
-    return parts.length >= 3 ? parts[parts.length - 3] : parts[0];
-  }
-  
-  // Handle rare 3-part TLDs
-  if (parts.length >= 4) {
-    const lastThreeParts = parts[parts.length - 3] + '.' + lastTwoParts;
-    if (THREE_PART_TLDS.has(lastThreeParts)) {
-      return parts[parts.length - 4];
-    }
-  }
-  
-  return parts[parts.length - 2];
+  // Fallback for IPs / single-token / unparseable: best-effort
+  // second-to-last token (the old behavior's default branch).
+  const parts = hostname.split('.');
+  return parts.length >= 2 ? parts[parts.length - 2] : hostname;
 }
 
 /**
@@ -75,12 +48,13 @@ function getBaseDomainName(domain) {
 function calculateSimilarity(domain1, domain2) {
   if (domain1 === domain2) return 100;
   if (!domain1 || !domain2) return 0;
-  
-  const longer = domain1.length > domain2.length ? domain1 : domain2;
+
+  // Both inputs are non-empty different strings at this point — the
+  // `''` cases are handled by the two guards above. (Used to have an
+  // `if (longer.length === 0) return 100` here but it was unreachable.)
+  const longer  = domain1.length > domain2.length ? domain1 : domain2;
   const shorter = domain1.length > domain2.length ? domain2 : domain1;
-  
-  if (longer.length === 0) return 100;
-  
+
   const distance = levenshteinDistance(longer, shorter);
   return Math.round(((longer.length - distance) / longer.length) * 100);
 }
@@ -93,26 +67,29 @@ function calculateSimilarity(domain1, domain2) {
  * @returns {number} Edit distance
  */
 function levenshteinDistance(str1, str2) {
-  const m = str1.length;
-  const n = str2.length;
-  
-  // Ensure we iterate over the shorter dimension for row arrays
-  if (m < n) return levenshteinDistance(str2, str1);
-  
-  // Two rows instead of full matrix
+  // Ensure str1 is the longer one so the inner-loop dimension (n)
+  // stays small. Inline swap instead of recursive re-entry — the old
+  // `if (m < n) return levenshteinDistance(str2, str1)` paid a stack
+  // frame + re-validation for what's really just a variable rename.
+  let a = str1, b = str2;
+  if (a.length < b.length) { const t = a; a = b; b = t; }
+  const m = a.length;
+  const n = b.length;
+
+  // Two rows instead of full matrix — O(n) space instead of O(m*n).
   let prevRow = new Array(n + 1);
   let currRow = new Array(n + 1);
-  
+
   for (let j = 0; j <= n; j++) {
     prevRow[j] = j;
   }
-  
+
   for (let i = 1; i <= m; i++) {
     currRow[0] = i;
-    const ch1 = str1[i - 1];
-    
+    const ch1 = a[i - 1];
+
     for (let j = 1; j <= n; j++) {
-      if (ch1 === str2[j - 1]) {
+      if (ch1 === b[j - 1]) {
         currRow[j] = prevRow[j - 1];
       } else {
         const sub = prevRow[j - 1];
@@ -121,13 +98,13 @@ function levenshteinDistance(str1, str2) {
         currRow[j] = (sub < ins ? (sub < del ? sub : del) : (ins < del ? ins : del)) + 1;
       }
     }
-    
+
     // Swap rows
     const temp = prevRow;
     prevRow = currRow;
     currRow = temp;
   }
-  
+
   return prevRow[n];
 }
 
@@ -165,18 +142,28 @@ function shouldIgnoreSimilarDomain(newDomain, existingDomains, options = {}) {
     if (!existingDomain || existingDomain === newDomain) {
       continue;
     }
-    
+
     const existingBaseDomain = getBaseDomainName(existingDomain);
-    if (!existingBaseDomain || existingBaseDomain === newBaseDomain) {
+    if (!existingBaseDomain) {
       continue;
     }
-    
+
+    // BEHAVIOR NOTE: identical base names (e.g. google.com vs google.net)
+    // now count as 100% similar — calculateSimilarity returns 100 for
+    // matching strings, which is above any reasonable threshold. The old
+    // `existingBaseDomain === newBaseDomain` skip silently exempted
+    // same-base-different-TLD pairs, defeating the dedup purpose for the
+    // most common variant case (brand registrations across multiple TLDs).
+    // Both call sites in nwss.js (matched-dedup at ~2833, ignoreDomains
+    // expansion at ~2849) want this stricter behavior. Set a lower
+    // threshold or disable ignore_similar entirely if you actually want
+    // to keep brand variants.
     const similarity = calculateSimilarity(newBaseDomain, existingBaseDomain);
     
     if (similarity >= threshold) {
       if (forceDebug) {
         console.log(formatLogMessage('debug', 
-          `[ignore_similar] ${newDomain} (${newBaseDomain}) is ${similarity}% similar to ${existingDomain} (${existingBaseDomain}) - ignoring`
+          `${IGNORE_SIMILAR_TAG} ${newDomain} (${newBaseDomain}) is ${similarity}% similar to ${existingDomain} (${existingBaseDomain}) - ignoring`
         ));
       }
       
@@ -194,52 +181,11 @@ function shouldIgnoreSimilarDomain(newDomain, existingDomains, options = {}) {
   return { shouldIgnore: false, reason: 'no similar domains found' };
 }
 
-/**
- * Utility function: Filters out similar domains from a collection
- * @param {Array} domains - Array of domains to filter
- * @param {object} options - Filtering options
- * @returns {object} Result with filtered domains and removed domains
- */
-function filterSimilarDomains(domains, options = {}) {
-  const {
-    enabled = true,
-    threshold = 80,
-    forceDebug = false
-  } = options;
-  
-  if (!enabled || !Array.isArray(domains)) {
-    return { filtered: domains, removed: [] };
-  }
-  
-  const filtered = [];
-  const removed = [];
-  
-  for (const domain of domains) {
-    const result = shouldIgnoreSimilarDomain(domain, filtered, { enabled, threshold, forceDebug });
-    
-    if (result.shouldIgnore) {
-      removed.push({
-        domain,
-        reason: result.reason,
-        similarTo: result.similarDomain
-      });
-    } else {
-      filtered.push(domain);
-    }
-  }
-  
-  if (forceDebug && removed.length > 0) {
-    console.log(formatLogMessage('debug', 
-      `[ignore_similar] Filtered out ${removed.length} similar domains`
-    ));
-  }
-  
-  return { filtered, removed };
-}
-
+// Public surface used by nwss.js. getBaseDomainName + (deleted)
+// filterSimilarDomains had zero external callers — getBaseDomainName
+// stays as an internal helper, filterSimilarDomains is gone entirely
+// (no internal callers either).
 module.exports = {
-  getBaseDomainName,
   calculateSimilarity,
-  shouldIgnoreSimilarDomain,
-  filterSimilarDomains
+  shouldIgnoreSimilarDomain
 };