@fanboynz/network-scanner 2.0.66 → 3.0.0

package/lib/redirect.js

@@ -165,8 +165,14 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
     // Inject JavaScript redirect detection
     await jsRedirectDetector();
 
-    if (forceDebug && Object.keys(gotoOptions).length > 0) {
-      console.log(formatLogMessage('debug', `Using goto options: ${JSON.stringify(gotoOptions)}`));
+    if (forceDebug) {
+      // Avoid Object.keys allocation just to check emptiness — a for...in
+      // early-exit on the first own key is enough.
+      let hasOpts = false;
+      for (const _k in gotoOptions) { hasOpts = true; break; }
+      if (hasOpts) {
+        console.log(formatLogMessage('debug', `Using goto options: ${JSON.stringify(gotoOptions)}`));
+      }
     }
 
     // Initial navigation. Puppeteer's page.goto returns the response for the
@@ -184,7 +190,7 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
       } catch (_) { /* response disposed or detached — fine, stays null */ }
     }
 
-    if (response && response.url() !== currentUrl) {
+    if (response && response.url() !== currentUrl && !response.url().startsWith('chrome-error://')) {
       // Check redirect limit before adding
       if (redirectChain.length >= maxRedirects) {
         if (forceDebug) {
@@ -192,12 +198,12 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
         }
         finalUrl = currentUrl; // Keep original URL
       } else {
-      finalUrl = response.url();
-      redirected = true;
-      if (!redirectChain.includes(finalUrl)) redirectChain.push(finalUrl);
+        finalUrl = response.url();
+        redirected = true;
+        if (!redirectChain.includes(finalUrl)) redirectChain.push(finalUrl);
       }
       if (forceDebug) {
-        console.log(formatLogMessage('debug', `HTTP redirect detected: ${currentUrl} ? ${finalUrl}`));
+        console.log(formatLogMessage('debug', `HTTP redirect detected: ${currentUrl} -> ${finalUrl}`));
       }
     }
 
@@ -223,9 +229,11 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
           };
         });
         
-        // Check if URL changed (either through JS redirect or automatic redirect)
+        // Check if URL changed (either through JS redirect or automatic redirect).
+        // Skip chrome-error://* — it's Puppeteer's landing page on DNS/connection
+        // failure and adding it to the chain produces bogus intermediate hops.
         const currentPageUrl = page.url();
-        if (currentPageUrl && currentPageUrl !== finalUrl && !redirectChain.includes(currentPageUrl)) {
+        if (currentPageUrl && currentPageUrl !== finalUrl && !currentPageUrl.startsWith('chrome-error://') && !redirectChain.includes(currentPageUrl)) {
           // Check redirect limit before adding
           if (redirectChain.length >= maxRedirects) {
             if (forceDebug) {
@@ -275,21 +283,23 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
       await detectCommonJSRedirects(page, forceDebug, formatLogMessage);
     }
 
-    // Final URL check
+    // Final URL check. Same chrome-error://* skip as the earlier branches —
+    // a navigation that ended in a chrome-error landing shouldn't be treated
+    // as the "final" URL of a successful redirect chain.
     const finalPageUrl = page.url();
-    if (finalPageUrl && finalPageUrl !== finalUrl) {
+    if (finalPageUrl && finalPageUrl !== finalUrl && !finalPageUrl.startsWith('chrome-error://')) {
       // Check redirect limit before final update
       if (redirectChain.length >= maxRedirects) {
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Maximum redirects (${maxRedirects}) reached, keeping current finalUrl`));
         }
       } else {
-      finalUrl = finalPageUrl;
-      redirected = true;
-      if (!redirectChain.includes(finalUrl)) {
-        redirectChain.push(finalUrl);
+        finalUrl = finalPageUrl;
+        redirected = true;
+        if (!redirectChain.includes(finalUrl)) {
+          redirectChain.push(finalUrl);
+        }
       }
-     }
     }
 
   } finally {
@@ -298,21 +308,19 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
 
   // Log redirect summary
   if (redirected && forceDebug) {
-    console.log(formatLogMessage('debug', `Redirect chain: ${redirectChain.join(' ? ')}`));
+    console.log(formatLogMessage('debug', `Redirect chain: ${redirectChain.join(' -> ')}`));
   }
 
-  // Extract redirect domains to exclude from matching
-  let redirectDomains = [];
+  // Extract intermediate redirect domains (exclude the final entry). Single
+  // loop instead of slice().map().filter() — three array allocations down to
+  // one push-loop. redirectChain is bounded at maxRedirects (default 10).
+  const redirectDomains = [];
   if (redirected && redirectChain.length > 1) {
-    // Get all intermediate domains (exclude the final domain)
-    const intermediateDomains = redirectChain.slice(0, -1).map(url => {
+    for (let i = 0; i < redirectChain.length - 1; i++) {
       try {
-        return new URL(url).hostname;
-      } catch {
-        return null;
-      }
-    }).filter(Boolean);
-    redirectDomains = intermediateDomains;
+        redirectDomains.push(new URL(redirectChain[i]).hostname);
+      } catch (_) { /* skip malformed entries */ }
+    }
   }
 
   return { finalUrl, redirected, redirectChain, originalUrl: currentUrl, redirectDomains, httpStatus, cfRay };
@@ -410,13 +418,21 @@ async function handleRedirectTimeout(page, originalUrl, error, safeGetDomain, fo
   
   try {
     const currentPageUrl = page.url();
-    if (currentPageUrl && currentPageUrl !== 'about:blank' && currentPageUrl !== originalUrl) {
+    // Skip chrome-error://* the same way navigateWithRedirectHandling does:
+    // a DNS/connection-failure landing isn't a "partial redirect recovery",
+    // and safeGetDomain('chrome-error://chromewebdata/') returns
+    // 'chromewebdata', which would otherwise differ from the original
+    // domain and falsely report success here.
+    if (currentPageUrl
+        && currentPageUrl !== 'about:blank'
+        && !currentPageUrl.startsWith('chrome-error://')
+        && currentPageUrl !== originalUrl) {
       const originalDomain = safeGetDomain(originalUrl);
       const currentDomain = safeGetDomain(currentPageUrl);
-      
+
       if (originalDomain !== currentDomain) {
         if (forceDebug) {
-          console.log(formatLogMessage('debug', `Partial redirect timeout recovered: ${originalDomain} ? ${currentDomain}`));
+          console.log(formatLogMessage('debug', `Partial redirect timeout recovered: ${originalDomain} -> ${currentDomain}`));
         }
         return { success: true, finalUrl: currentPageUrl, redirected: true };
       }

package/lib/referrer.js

@@ -1,6 +1,12 @@
 // === Referrer Header Generation Module ===
 // This module handles generation of referrer headers for different traffic simulation modes
 
+const { formatLogMessage, messageColors } = require('./colorize');
+
+// Precomputed colored '[referrer]' subsystem prefix — matches the project
+// convention used by other modules (flowproxy/cloudflare/smart-cache/etc.).
+const REFERRER_TAG = messageColors.processing('[referrer]');
+
 /**
  * Performance utility: Get random element from array
  * Reduces code duplication and improves readability
@@ -78,7 +84,10 @@ const REFERRER_COLLECTIONS = Object.freeze({
  * @returns {string} Selected search term
  */
 function generateSearchTerm(customTerms, context = null) {
-  if (customTerms && customTerms.length > 0) {
+  // Array.isArray guard belt-and-braces: validateReferrerConfig now blocks
+  // non-array search_terms at config load, but a direct internal caller
+  // could still pass a non-array and trigger a TypeError on .length.
+  if (Array.isArray(customTerms) && customTerms.length > 0) {
     return getRandomElement(customTerms);
   }
   
@@ -107,7 +116,7 @@ function generateSearchReferrer(searchTerms, context, forceDebug) {
   const referrerUrl = randomEngine + encodeURIComponent(searchTerm);
   
   if (forceDebug) {
-    console.log(`[debug] Generated search referrer: ${referrerUrl} (engine: ${randomEngine.split('//')[1].split('/')[0]}, term: "${searchTerm}")`);
+    console.log(formatLogMessage('debug', `${REFERRER_TAG} Generated search referrer: ${referrerUrl} (engine: ${randomEngine.split('//')[1].split('/')[0]}, term: "${searchTerm}")`));
   }
   
   return referrerUrl;
@@ -122,7 +131,7 @@ function generateSocialMediaReferrer(forceDebug) {
   const randomSocial = getRandomElement(REFERRER_COLLECTIONS.SOCIAL_MEDIA);
   
   if (forceDebug) {
-    console.log(`[debug] Generated social media referrer: ${randomSocial}`);
+    console.log(formatLogMessage('debug', `${REFERRER_TAG} Generated social media referrer: ${randomSocial}`));
   }
   
   return randomSocial;
@@ -137,7 +146,7 @@ function generateNewsReferrer(forceDebug) {
   const randomNews = getRandomElement(REFERRER_COLLECTIONS.NEWS_SITES);
   
   if (forceDebug) {
-    console.log(`[debug] Generated news referrer: ${randomNews}`);
+    console.log(formatLogMessage('debug', `${REFERRER_TAG} Generated news referrer: ${randomNews}`));
   }
   
   return randomNews;
@@ -179,35 +188,44 @@ function shouldDisableReferrer(targetUrl, disableList, forceDebug = false) {
   
   for (const disablePattern of disableList) {
     if (typeof disablePattern !== 'string') continue;
-    
+
     // Fast check: Exact URL match (no parsing needed)
     if (targetUrl === disablePattern) {
-      if (forceDebug) console.log(`[debug] Referrer disabled for exact match: ${targetUrl}`);
+      if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Referrer disabled for exact match: ${targetUrl}`));
       return true;
     }
-    
-    // Domain/hostname match (use cached parsed URL)
-    if (targetUrlParsed) {
-      try {
-        const disableHostname = new URL(disablePattern).hostname;
-        if (targetHostname === disableHostname) {
-          if (forceDebug) console.log(`[debug] Referrer disabled for domain match: ${targetHostname}`);
-          return true;
-        }
-      } catch (e) {
-        // disablePattern is not a valid URL, try substring match below
-      }
+
+    // Resolve pattern to a hostname — full URL patterns ('https://example.com')
+    // and bare-hostname patterns ('example.com') both end up running through
+    // the same suffix-match logic so they behave identically. Previously
+    // full-URL patterns only did exact hostname equality (no subdomain
+    // match), while bare-hostname patterns did suffix match — same user
+    // intent, different result depending on string form.
+    let patternHostname = null;
+    if (disablePattern.includes('/')) {
+      try { patternHostname = new URL(disablePattern).hostname; } catch (_) { /* fall through */ }
+    } else {
+      patternHostname = disablePattern;
     }
-    
-    // Fallback: Simple substring match (for patterns like 'example.com')
-    if (!targetUrlParsed || disablePattern.includes('/') === false) {
+
+    if (targetUrlParsed && patternHostname) {
+      const p = patternHostname.toLowerCase();
+      const h = targetHostname.toLowerCase();
+      if (h === p || h.endsWith('.' + p)) {
+        if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Referrer disabled for hostname match: ${p} matches ${h}`));
+        return true;
+      }
+    } else if (!targetUrlParsed) {
+      // Pathological fallback: target URL didn't parse. Substring match
+      // as last resort. Shouldn't fire in practice — we only call this
+      // on URLs we're about to navigate to.
       if (targetUrl.includes(disablePattern)) {
-        if (forceDebug) console.log(`[debug] Referrer disabled for pattern match: ${disablePattern} in ${targetUrl}`);
+        if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Referrer disabled for pattern match (unparseable target): ${disablePattern}`));
         return true;
       }
     }
   }
-  
+
   return false;
 }
 
@@ -218,94 +236,94 @@ function shouldDisableReferrer(targetUrl, disableList, forceDebug = false) {
  * @returns {string} Generated referrer URL or empty string
  */
 function generateReferrerUrl(referrerConfig, forceDebug = false) {
-  try {
-    // Handle simple string URLs
-    if (typeof referrerConfig === 'string') {
-      const url = isValidUrl(referrerConfig) ? referrerConfig : '';
-      if (forceDebug && url) {
-        console.log(`[debug] Using direct referrer URL: ${url}`);
-      } else if (forceDebug && !url) {
-        console.log(`[debug] Invalid referrer URL provided: ${referrerConfig}`);
-      }
-      return url;
+  // No top-level try/catch — nothing in here throws synchronously.
+  // typeof / Array.isArray / object property access / string concat /
+  // console.log / the helper functions are all non-throwing. The old
+  // try/catch was unreachable defensive scaffolding.
+
+  // Handle simple string URLs
+  if (typeof referrerConfig === 'string') {
+    const url = isValidUrl(referrerConfig) ? referrerConfig : '';
+    if (forceDebug && url) {
+      console.log(formatLogMessage('debug', `${REFERRER_TAG} Using direct referrer URL: ${url}`));
+    } else if (forceDebug && !url) {
+      console.log(formatLogMessage('debug', `${REFERRER_TAG} Invalid referrer URL provided: ${referrerConfig}`));
     }
-    
-    // Handle arrays - pick random URL
-    if (Array.isArray(referrerConfig)) {
-      if (referrerConfig.length === 0) {
-        if (forceDebug) console.log(`[debug] Empty referrer array provided`);
-        return '';
+    return url;
+  }
+
+  // Handle arrays - pick random URL
+  if (Array.isArray(referrerConfig)) {
+    if (referrerConfig.length === 0) {
+      if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Empty referrer array provided`));
+      return '';
+    }
+
+    const randomUrl = getRandomElement(referrerConfig);
+    const url = isValidUrl(randomUrl) ? randomUrl : '';
+
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `${REFERRER_TAG} Selected referrer from array (${referrerConfig.length} options): ${url || 'invalid URL'}`));
+    }
+
+    return url;
+  }
+
+  // Handle object modes
+  if (typeof referrerConfig === 'object' && referrerConfig !== null && referrerConfig.mode) {
+    switch (referrerConfig.mode) {
+      case 'random_search': {
+        const searchTerms = referrerConfig.search_terms;
+        const context = referrerConfig.context; // Optional context hint
+        return generateSearchReferrer(searchTerms, context, forceDebug);
       }
-      
-      const randomUrl = getRandomElement(referrerConfig);
-      const url = isValidUrl(randomUrl) ? randomUrl : '';
-      
-      if (forceDebug) {
-        console.log(`[debug] Selected referrer from array (${referrerConfig.length} options): ${url || 'invalid URL'}`);
+
+      case 'social_media': {
+        return generateSocialMediaReferrer(forceDebug);
       }
-      
-      return url;
-    }
-    
-    // Handle object modes
-    if (typeof referrerConfig === 'object' && referrerConfig !== null && referrerConfig.mode) {
-      switch (referrerConfig.mode) {
-        case 'random_search': {
-          const searchTerms = referrerConfig.search_terms;
-          const context = referrerConfig.context; // Optional context hint
-          return generateSearchReferrer(searchTerms, context, forceDebug);
-        }
-        
-        case 'social_media': {
-          return generateSocialMediaReferrer(forceDebug);
-        }
-        
-        case 'news_sites': {
-          return generateNewsReferrer(forceDebug);
-        }
-        
-        case 'direct_navigation': {
-          if (forceDebug) console.log(`[debug] Using direct navigation (no referrer)`);
-          return '';
-        }
-        
-        case 'custom': {
-          const url = isValidUrl(referrerConfig.url) ? referrerConfig.url : '';
-          if (forceDebug) {
-            console.log(`[debug] Using custom referrer URL: ${url || 'invalid URL provided'}`);
-          }
-          return url;
-        }
-        
-        case 'mixed': {
-          // Randomly choose between different referrer types
-          const modes = ['random_search', 'social_media', 'news_sites'];
-          const randomMode = getRandomElement(modes);
-          
-          if (forceDebug) console.log(`[debug] Mixed mode selected: ${randomMode}`);
-          
-          const mixedConfig = { mode: randomMode };
-          if (randomMode === 'random_search' && referrerConfig.search_terms) {
-            mixedConfig.search_terms = referrerConfig.search_terms;
-            mixedConfig.context = referrerConfig.context;
-          }
-          
-          return generateReferrerUrl(mixedConfig, forceDebug);
+
+      case 'news_sites': {
+        return generateNewsReferrer(forceDebug);
+      }
+
+      case 'direct_navigation': {
+        if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Using direct navigation (no referrer)`));
+        return '';
+      }
+
+      case 'custom': {
+        const url = isValidUrl(referrerConfig.url) ? referrerConfig.url : '';
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `${REFERRER_TAG} Using custom referrer URL: ${url || 'invalid URL provided'}`));
         }
-        
-        default: {
-          if (forceDebug) console.log(`[debug] Unknown referrer mode: ${referrerConfig.mode}`);
-          return '';
+        return url;
+      }
+
+      case 'mixed': {
+        // Randomly choose between different referrer types
+        const modes = ['random_search', 'social_media', 'news_sites'];
+        const randomMode = getRandomElement(modes);
+
+        if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Mixed mode selected: ${randomMode}`));
+
+        const mixedConfig = { mode: randomMode };
+        if (randomMode === 'random_search' && referrerConfig.search_terms) {
+          mixedConfig.search_terms = referrerConfig.search_terms;
+          mixedConfig.context = referrerConfig.context;
         }
+
+        return generateReferrerUrl(mixedConfig, forceDebug);
+      }
+
+      default: {
+        if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Unknown referrer mode: ${referrerConfig.mode}`));
+        return '';
       }
     }
-    
-    if (forceDebug) console.log(`[debug] Invalid referrer configuration type: ${typeof referrerConfig}`);
-    return '';
-  } catch (err) {
-    if (forceDebug) console.log(`[debug] Referrer generation failed: ${err.message}`);
-    return '';
   }
+
+  if (forceDebug) console.log(formatLogMessage('debug', `${REFERRER_TAG} Invalid referrer configuration type: ${typeof referrerConfig}`));
+  return '';
 }
 
 /**
@@ -350,30 +368,24 @@ function validateReferrerConfig(referrerConfig) {
     return result;
   }
   
-  // Validate arrays
+  // Validate arrays — every item gets checked. The old code spot-checked
+  // only first and last when length > 10 "for performance", but config
+  // validation runs ONCE at startup and referrer arrays are tiny; the
+  // perf savings were imaginary, the correctness gap (items 2..N-1 never
+  // validated, typo'd URLs slipping through) was real.
   if (Array.isArray(referrerConfig)) {
     if (referrerConfig.length === 0) {
       result.warnings.push('Empty referrer array will result in no referrer');
       return result;
     }
-    
-    // Fast validation: check only first and last items if array is large
-    const itemsToCheck = referrerConfig.length > 10 
-      ? [referrerConfig[0], referrerConfig[referrerConfig.length - 1]]
-      : referrerConfig;
-    
-    itemsToCheck.forEach((url, index) => {
-        if (!isValidUrl(url)) {
-          const actualIndex = itemsToCheck === referrerConfig ? index : (index === 0 ? 0 : referrerConfig.length - 1);
-          result.errors.push(`Array item ${actualIndex} is not a valid HTTP/HTTPS URL: ${url}`);
-          result.isValid = false;
-        }
-      });
 
-    if (referrerConfig.length > 10 && itemsToCheck.length < referrerConfig.length) {
-      result.warnings.push(`Large array (${referrerConfig.length} items): only validated first and last items for performance`);
+    for (let i = 0; i < referrerConfig.length; i++) {
+      if (!isValidUrl(referrerConfig[i])) {
+        result.errors.push(`Array item ${i} is not a valid HTTP/HTTPS URL: ${referrerConfig[i]}`);
+        result.isValid = false;
+      }
     }
-    
+
     return result;
   }
   
@@ -406,11 +418,25 @@ function validateReferrerConfig(referrerConfig) {
         break;
         
       case 'random_search':
-        if (referrerConfig.search_terms && !Array.isArray(referrerConfig.search_terms)) {
-          result.warnings.push('search_terms should be an array of strings');
-        }
-        if (referrerConfig.search_terms && referrerConfig.search_terms.length === 0) {
-          result.warnings.push('Empty search_terms array will use default terms');
+        // Upgrade from warning to error: generateSearchTerm reads
+        // customTerms.length, which throws TypeError on a non-array
+        // (e.g. a single string the user expected to be auto-wrapped).
+        // Letting this slip past validation produces a runtime crash
+        // mid-scan instead of a clean config-load failure.
+        if (referrerConfig.search_terms !== undefined && !Array.isArray(referrerConfig.search_terms)) {
+          result.isValid = false;
+          result.errors.push(`search_terms must be an array of strings (got ${typeof referrerConfig.search_terms})`);
+        } else if (Array.isArray(referrerConfig.search_terms)) {
+          if (referrerConfig.search_terms.length === 0) {
+            result.warnings.push('Empty search_terms array will use default terms');
+          } else {
+            for (let i = 0; i < referrerConfig.search_terms.length; i++) {
+              if (typeof referrerConfig.search_terms[i] !== 'string') {
+                result.isValid = false;
+                result.errors.push(`search_terms[${i}] must be a string (got ${typeof referrerConfig.search_terms[i]})`);
+              }
+            }
+          }
         }
         break;
     }
@@ -464,47 +490,14 @@ function validateReferrerDisable(referrerDisable) {
   return result;
 }
 
-/**
- * Gets available referrer modes and their descriptions
- * @returns {Object} Object containing mode descriptions
- */
-function getReferrerModes() {
-  return {
-    'random_search': 'Generate random search engine referrers with customizable search terms',
-    'social_media': 'Use random social media platform referrers',
-    'news_sites': 'Use random news website referrers',
-    'direct_navigation': 'No referrer (simulates direct URL entry)',
-    'custom': 'Use a specific custom referrer URL',
-    'mixed': 'Randomly mix different referrer types for varied traffic simulation'
-  };
-}
-
-/**
- * Gets statistics about available referrer collections
- * @returns {Object} Statistics about referrer collections
- */
-function getReferrerStats() {
-  return {
-    searchEngines: REFERRER_COLLECTIONS.SEARCH_ENGINES.length,
-    socialMedia: REFERRER_COLLECTIONS.SOCIAL_MEDIA.length,
-    newsSites: REFERRER_COLLECTIONS.NEWS_SITES.length,
-    defaultSearchTerms: REFERRER_COLLECTIONS.DEFAULT_SEARCH_TERMS.length,
-    ecommerceTerms: REFERRER_COLLECTIONS.ECOMMERCE_TERMS.length,
-    techTerms: REFERRER_COLLECTIONS.TECH_TERMS.length
-  };
-}
-
+// Public surface used by nwss.js. Internal helpers
+// (generateReferrerUrl, shouldDisableReferrer, generateSearch/Social/News
+// Referrer, isValidUrl, REFERRER_COLLECTIONS) stay module-private —
+// the old export list included nine names no caller imported, plus two
+// dead helper functions (getReferrerModes, getReferrerStats) that have
+// been removed entirely.
 module.exports = {
-  generateReferrerUrl,
   getReferrerForUrl,
-  shouldDisableReferrer,
   validateReferrerConfig,
-  validateReferrerDisable,
-  getReferrerModes,
-  getReferrerStats,
-  generateSearchReferrer,
-  generateSocialMediaReferrer,
-  generateNewsReferrer,
-  isValidUrl,
-  REFERRER_COLLECTIONS
+  validateReferrerDisable
 };