@fanboynz/network-scanner 2.0.30 → 2.0.31

package/lib/browserexit.js

@@ -211,14 +211,33 @@ async function cleanupUserDataDir(userDataDir, forceDebug = false) {
  * @returns {Promise<void>}
  */
 async function gracefulBrowserCleanup(browser, forceDebug = false) {
+  // FIX: Check browser connection before operations
+  if (!browser || !browser.isConnected()) {
+    if (forceDebug) console.log(`[debug] [browser] Browser not connected, skipping cleanup`);
+    return;
+  }
   if (forceDebug) console.log(`[debug] [browser] Getting all browser pages...`);
-  const pages = await browser.pages();
+  let pages;
+  try {
+    pages = await browser.pages();
+  } catch (pagesErr) {
+    if (forceDebug) console.log(`[debug] [browser] Failed to get pages: ${pagesErr.message}`);
+    return;
+  }
   if (forceDebug) console.log(`[debug] [browser] Found ${pages.length} pages to close`);
   
   await Promise.all(pages.map(async (page) => {
     if (!page.isClosed()) {
       try {
-        if (forceDebug) console.log(`[debug] [browser] Closing page: ${page.url()}`);
+        // FIX: Wrap page.url() in try-catch to handle race condition
+        let pageUrl = 'unknown';
+        try {
+          pageUrl = page.url();
+        } catch (urlErr) {
+          // Page closed between check and url call
+        }
+        
+        if (forceDebug) console.log(`[debug] [browser] Closing page: ${pageUrl}`);
         await page.close();
         if (forceDebug) console.log(`[debug] [browser] Page closed successfully`);
       } catch (err) {
@@ -229,8 +248,18 @@ async function gracefulBrowserCleanup(browser, forceDebug = false) {
   }));
   
   if (forceDebug) console.log(`[debug] [browser] All pages closed, closing browser...`);
-  await browser.close();
-  if (forceDebug) console.log(`[debug] [browser] Browser closed successfully`);
+  
+  // FIX: Check browser is still connected before closing
+  try {
+    if (browser.isConnected()) {
+      await browser.close();
+      if (forceDebug) console.log(`[debug] [browser] Browser closed successfully`);
+    } else {
+      if (forceDebug) console.log(`[debug] [browser] Browser already disconnected`);
+    }
+  } catch (closeErr) {
+    if (forceDebug) console.log(`[debug] [browser] Browser close failed: ${closeErr.message}`);
+  }
 }
 
 /**

package/lib/browserhealth.js

@@ -492,6 +492,10 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
  */
 async function isPageFromPreviousScan(page, forceDebug) {
   try {
+    // FIX: Check page state first before any operations
+    if (page.isClosed()) {
+      return true; // Closed pages should be cleaned up
+    }
     // Cache page.url() for all checks in this function
     const pageUrl = page.url();
     
@@ -524,9 +528,13 @@ async function isPageFromPreviousScan(page, forceDebug) {
     return false; // Conservative - don't close unless we're sure
   } catch (err) {
     if (forceDebug) {
-      // Cache URL for error logging
-      const pageUrl = page.url();
-      console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${pageUrl}: ${err.message}`));
+      try {
+        // Cache URL for error logging - wrap in try-catch as page might be closed
+        const pageUrl = page.url();
+        console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${pageUrl}: ${err.message}`));
+      } catch (urlErr) {
+        console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page: ${err.message}`));
+      }
     }
     return false; // Conservative - don't close if we can't evaluate
   }
@@ -1104,16 +1112,25 @@ async function cleanupPageBeforeReload(page, forceDebug = false) {
     // Wait a bit for navigation to stop
     await new Promise(resolve => setTimeout(resolve, 500));
 
+    // FIX: Check if page is still open after delay before cleanup
+    if (page.isClosed()) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', 'Page closed during cleanup delay'));
+      }
+      return false;
+    }
+
     // Now do the full cleanup
-    await page.evaluate(() => {
-      // Stop all media elements
-      document.querySelectorAll('video, audio').forEach(media => {
-        try { 
-          media.pause(); 
-          media.src = ''; 
-          media.load();
-        } catch(e) {}
-      });
+    try {
+      await page.evaluate(() => {
+        // Stop all media elements
+        document.querySelectorAll('video, audio').forEach(media => {
+          try { 
+            media.pause(); 
+            media.src = ''; 
+            media.load();
+          } catch(e) {}
+        });
       
       // Clear all timers and intervals
       const highestId = setTimeout(() => {}, 0);
@@ -1144,7 +1161,15 @@ async function cleanupPageBeforeReload(page, forceDebug = false) {
       
       // Force garbage collection if available
       if (window.gc) window.gc();
-    });
+      });
+
+    } catch (evalErr) {
+      // Page closed during cleanup
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Page cleanup evaluation failed: ${evalErr.message}`));
+      }
+      return false;
+    }
 
     if (forceDebug) {
       console.log(formatLogMessage('debug', 'Page resources cleaned before reload'));

package/lib/fingerprint.js

@@ -268,14 +268,19 @@ function generateCSIData() {
  */
 async function validatePageForInjection(page, currentUrl, forceDebug) {
   try {
-    if (page.isClosed()) return false;
+    if (!page || page.isClosed()) return false;
+    
+    if (!page.browser().isConnected()) {
+      if (forceDebug) console.log(`[debug] Page validation failed - browser disconnected: ${currentUrl}`);
+      return false;
+    }
     await Promise.race([
       page.evaluate(() => document.readyState || 'loading'),
-      new Promise((_, reject) => setTimeout(() => reject(new Error('Page unresponsive')), 3000))
+      new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 1500))
     ]);
     return true;
   } catch (validationErr) {
-    if (forceDebug) console.log(`[debug] Skipping fingerprint protection - page unresponsive: ${currentUrl}`);
+    if (forceDebug) console.log(`[debug] Page validation failed - ${validationErr.message}: ${currentUrl}`);
     return false;
   }
 }
@@ -362,7 +367,13 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
   const ua = USER_AGENT_COLLECTIONS.get(siteConfig.userAgent.toLowerCase());
   
   if (ua) {
-    await page.setUserAgent(ua);
+    // FIX: Wrap setUserAgent in try-catch to handle race condition
+    try {
+      await page.setUserAgent(ua);
+    } catch (uaErr) {
+      if (forceDebug) console.log(`[debug] Could not set user agent - page closed: ${currentUrl}`);
+      return;
+    }
     
     if (forceDebug) console.log(`[debug] Applying stealth protection for ${currentUrl}`);
     
@@ -1317,6 +1328,20 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         }, 'enhanced mouse/pointer spoofing');
 
         safeExecute(() => {
+          // Filter DevTools/automation traces from console.debug
+          const originalConsoleDebug = console.debug;
+          console.debug = function(...args) {
+            const message = args.join(' ');
+            if (typeof message === 'string' && (
+                message.includes('DevTools') ||
+                message.includes('Runtime.evaluate') ||
+                message.includes('Page.addScriptToEvaluateOnNewDocument') ||
+                message.includes('Protocol error'))) {
+              return; // Silently drop DevTools-related debug messages
+            }
+            return originalConsoleDebug.apply(this, args);
+          };
+
           const originalConsoleError = console.error;
           console.error = function(...args) {
             const message = args.join(' ');
@@ -1331,6 +1356,22 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
             return originalConsoleError.apply(this, arguments);
           };
         }, 'console error suppression');
+        
+        // Hide source URL indicators (data: URLs reveal script injection)
+        safeExecute(() => {
+          const originalLocation = window.location;
+          Object.defineProperty(window, 'location', {
+            value: new Proxy(originalLocation, {
+              get: function(target, prop) {
+                if (prop === 'href' && target[prop] && target[prop].includes('data:')) {
+                  return 'about:blank';
+                }
+                return target[prop];
+              }
+            }),
+            configurable: false
+          });
+        }, 'location URL masking');
 
       }, ua, forceDebug);
     } catch (stealthErr) {
@@ -1422,8 +1463,15 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
   
   // Validate page state before injection
   if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
-  
- const currentUserAgent = await page.evaluate(() => navigator.userAgent);
+
+  // FIX: Wrap page.evaluate in try-catch to handle race condition 
+  let currentUserAgent;
+  try {
+    currentUserAgent = await page.evaluate(() => navigator.userAgent);
+  } catch (evalErr) {
+    if (forceDebug) console.log(`[debug] Could not get user agent - page closed: ${currentUrl}`);
+    return;
+  }
   
  const spoof = fingerprintSetting === 'random' ? generateRealisticFingerprint(currentUserAgent) : {
     deviceMemory: 8, 

package/lib/grep.js

@@ -3,6 +3,7 @@
 
 const fs = require('fs');
 const { spawnSync } = require('child_process');
+const crypto = require('crypto');
 const path = require('path');
 const os = require('os');
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
@@ -21,23 +22,48 @@ const GREP_DEFAULTS = {
   GREP_NOT_FOUND_STATUS: 1,
   CURL_SUCCESS_STATUS: 0,
   VERSION_LINE_INDEX: 0,
-  RANDOM_STRING_LENGTH: 9
+  RANDOM_STRING_LENGTH: 9,
+  TEMP_DIR_PREFIX: 'grep_search_'
 };
 
 /**
- * Creates a temporary file with content for grep processing
+ * Creates a temporary directory and file with content for grep processing
+ * Uses mkdtempSync to avoid race conditions from filename collisions
  * @param {string} content - The content to write to temp file
  * @param {string} prefix - Prefix for temp filename
- * @returns {string} Path to the created temporary file
+ * @returns {object} Object containing tempDir and tempFile paths
  */
 function createTempFile(content, prefix = 'scanner_grep') {
   const tempDir = os.tmpdir();
-  const tempFile = path.join(tempDir, `${prefix}_${Date.now()}_${Math.random().toString(36).substr(2, GREP_DEFAULTS.RANDOM_STRING_LENGTH)}.tmp`);
+  
+  // Create a unique temporary directory to avoid race conditions
+  const uniqueTempDir = fs.mkdtempSync(path.join(tempDir, GREP_DEFAULTS.TEMP_DIR_PREFIX));
+  
+  // Use cryptographically secure random ID for additional uniqueness
+  const uniqueId = crypto.randomBytes(8).toString('hex');
+  const tempFile = path.join(uniqueTempDir, `${prefix}_${uniqueId}.tmp`);
   
   try {
-    fs.writeFileSync(tempFile, content, 'utf8');
-    return tempFile;
+    // Write atomically with error handling
+    fs.writeFileSync(tempFile, content, { 
+      encoding: 'utf8',
+      mode: 0o600 // Restrict permissions for security
+    });
+    
+    return { tempDir: uniqueTempDir, tempFile };
   } catch (error) {
+    // Clean up temp directory on write failure
+    try {
+      if (fs.existsSync(tempFile)) {
+        fs.unlinkSync(tempFile);
+      }
+      if (fs.existsSync(uniqueTempDir)) {
+        fs.rmdirSync(uniqueTempDir);
+      }
+    } catch (cleanupErr) {
+      // Ignore cleanup errors, report original error
+    }
+
     throw new Error(`Failed to create temp file: ${error.message}`);
   }
 }
@@ -64,8 +90,10 @@ async function grepContent(content, searchPatterns, options = {}) {
   let tempFile = null;
   
   try {
-    // Create temporary file with content
-    tempFile = createTempFile(content, 'grep_search');
+    // Create temporary directory and file with content
+    const tempResult = createTempFile(content, 'grep_search');
+    tempDir = tempResult.tempDir;
+    tempFile = tempResult.tempFile;
     
     const allMatches = [];
     let firstMatch = null;
@@ -119,7 +147,7 @@ async function grepContent(content, searchPatterns, options = {}) {
   } catch (error) {
     throw new Error(`Grep search failed: ${error.message}`);
   } finally {
-    // Clean up temporary file
+    // Clean up temporary file and directory
     if (tempFile) {
       try {
         fs.unlinkSync(tempFile);
@@ -127,6 +155,13 @@ async function grepContent(content, searchPatterns, options = {}) {
         console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp file ${tempFile}: ${cleanupErr.message}`));
       }
     }
+    if (tempDir) {
+      try {
+        fs.rmdirSync(tempDir);
+      } catch (cleanupErr) {
+        console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp directory ${tempDir}: ${cleanupErr.message}`));
+      }
+    }
   }
 }
 

package/lib/referrer.js

@@ -0,0 +1,479 @@
+// === Referrer Header Generation Module ===
+// This module handles generation of referrer headers for different traffic simulation modes
+
+/**
+ * Referrer URL collections for different modes
+ */
+const REFERRER_COLLECTIONS = Object.freeze({
+  SEARCH_ENGINES: [
+    'https://www.google.com/search?q=',
+    'https://www.bing.com/search?q=',
+    'https://duckduckgo.com/?q=',
+    'https://search.yahoo.com/search?p=',
+    'https://yandex.com/search/?text=',
+    'https://www.baidu.com/s?wd=',
+    'https://www.startpage.com/sp/search?query=',
+    'https://search.brave.com/search?q='
+  ],
+  
+  SOCIAL_MEDIA: [
+    'https://www.facebook.com/',
+    'https://twitter.com/',
+    'https://www.linkedin.com/',
+    'https://www.reddit.com/',
+    'https://www.instagram.com/',
+    'https://www.pinterest.com/',
+    'https://www.tiktok.com/',
+    'https://www.youtube.com/',
+    'https://discord.com/channels/',
+    'https://t.me/',
+    'https://www.snapchat.com/',
+    'https://www.tumblr.com/',
+    'https://www.threads.net/',
+    'https://mastodon.social/'
+  ],
+  
+  NEWS_SITES: [
+    'https://news.google.com/',
+    'https://www.reddit.com/r/news/',
+    'https://news.ycombinator.com/',
+    'https://www.bbc.com/news',
+    'https://www.cnn.com/',
+    'https://techcrunch.com/',
+    'https://www.theverge.com/'
+  ],
+  
+  DEFAULT_SEARCH_TERMS: [
+    'reviews', 'deals', 'discount', 'price', 'buy', 'shop', 'store',
+    'compare', 'best', 'top', 'guide', 'how to', 'tutorial', 'tips',
+    'news', 'update', 'latest', 'new', 'trending', 'popular', 'cheap',
+    'free', 'download', 'online', 'service', 'product', 'website'
+  ],
+  
+  ECOMMERCE_TERMS: [
+    'buy online', 'shopping', 'store', 'sale', 'discount', 'coupon',
+    'free shipping', 'best price', 'deals', 'outlet', 'marketplace'
+  ],
+  
+  TECH_TERMS: [
+    'software', 'app', 'download', 'tutorial', 'guide', 'review',
+    'comparison', 'features', 'specs', 'performance', 'benchmark'
+  ]
+});
+
+/**
+ * Generates a random search term based on context or defaults
+ * @param {Array} customTerms - Custom search terms provided by user
+ * @param {string} context - Context hint for term selection (e.g., 'ecommerce', 'tech')
+ * @returns {string} Selected search term
+ */
+function generateSearchTerm(customTerms, context = null) {
+  if (customTerms && customTerms.length > 0) {
+    return customTerms[Math.floor(Math.random() * customTerms.length)];
+  }
+  
+  // Use context-specific terms if available
+  let termCollection = REFERRER_COLLECTIONS.DEFAULT_SEARCH_TERMS;
+  if (context === 'ecommerce') {
+    termCollection = REFERRER_COLLECTIONS.ECOMMERCE_TERMS;
+  } else if (context === 'tech') {
+    termCollection = REFERRER_COLLECTIONS.TECH_TERMS;
+  }
+  
+  return termCollection[Math.floor(Math.random() * termCollection.length)];
+}
+
+/**
+ * Generates a search engine referrer URL
+ * @param {Array} searchTerms - Custom search terms
+ * @param {string} context - Context for term selection
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {string} Generated search engine referrer URL
+ */
+function generateSearchReferrer(searchTerms, context, forceDebug) {
+  const randomEngine = REFERRER_COLLECTIONS.SEARCH_ENGINES[
+    Math.floor(Math.random() * REFERRER_COLLECTIONS.SEARCH_ENGINES.length)
+  ];
+  const searchTerm = generateSearchTerm(searchTerms, context);
+  const referrerUrl = randomEngine + encodeURIComponent(searchTerm);
+  
+  if (forceDebug) {
+    console.log(`[debug] Generated search referrer: ${referrerUrl} (engine: ${randomEngine.split('//')[1].split('/')[0]}, term: "${searchTerm}")`);
+  }
+  
+  return referrerUrl;
+}
+
+/**
+ * Generates a social media referrer URL
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {string} Generated social media referrer URL
+ */
+function generateSocialMediaReferrer(forceDebug) {
+  const randomSocial = REFERRER_COLLECTIONS.SOCIAL_MEDIA[
+    Math.floor(Math.random() * REFERRER_COLLECTIONS.SOCIAL_MEDIA.length)
+  ];
+  
+  if (forceDebug) {
+    console.log(`[debug] Generated social media referrer: ${randomSocial}`);
+  }
+  
+  return randomSocial;
+}
+
+/**
+ * Generates a news site referrer URL
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {string} Generated news site referrer URL
+ */
+function generateNewsReferrer(forceDebug) {
+  const randomNews = REFERRER_COLLECTIONS.NEWS_SITES[
+    Math.floor(Math.random() * REFERRER_COLLECTIONS.NEWS_SITES.length)
+  ];
+  
+  if (forceDebug) {
+    console.log(`[debug] Generated news referrer: ${randomNews}`);
+  }
+  
+  return randomNews;
+}
+
+/**
+ * Validates a URL string
+ * @param {string} url - URL to validate
+ * @returns {boolean} True if valid HTTP/HTTPS URL
+ */
+function isValidUrl(url) {
+  return typeof url === 'string' && (url.startsWith('http://') || url.startsWith('https://'));
+}
+
+/**
+ * Checks if a URL should have its referrer disabled
+ * @param {string} targetUrl - The URL being visited
+ * @param {Array} disableList - Array of URLs/patterns that should have no referrer
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {boolean} True if referrer should be disabled for this URL
+ */
+function shouldDisableReferrer(targetUrl, disableList, forceDebug = false) {
+  if (!disableList || !Array.isArray(disableList) || disableList.length === 0) {
+    return false;
+  }
+  
+  if (!targetUrl || typeof targetUrl !== 'string') {
+    return false;
+  }
+  
+  for (const disablePattern of disableList) {
+    if (typeof disablePattern !== 'string') continue;
+    
+    // Exact URL match
+    if (targetUrl === disablePattern) {
+      if (forceDebug) console.log(`[debug] Referrer disabled for exact URL match: ${targetUrl}`);
+      return true;
+    }
+    
+    // Domain/hostname match
+    try {
+      const targetHostname = new URL(targetUrl).hostname;
+      const disableHostname = new URL(disablePattern).hostname;
+      if (targetHostname === disableHostname) {
+        if (forceDebug) console.log(`[debug] Referrer disabled for domain match: ${targetHostname}`);
+        return true;
+      }
+    } catch (e) {
+      // If pattern is not a valid URL, try simple string matching
+      if (targetUrl.includes(disablePattern)) {
+        if (forceDebug) console.log(`[debug] Referrer disabled for pattern match: ${disablePattern} in ${targetUrl}`);
+        return true;
+      }
+    }
+  }
+  
+  return false;
+}
+
+/**
+ * Generates a referrer URL based on the specified mode and options
+ * @param {Object|string|Array} referrerConfig - Referrer configuration
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {string} Generated referrer URL or empty string
+ */
+function generateReferrerUrl(referrerConfig, forceDebug = false) {
+  try {
+    // Handle simple string URLs
+    if (typeof referrerConfig === 'string') {
+      const url = isValidUrl(referrerConfig) ? referrerConfig : '';
+      if (forceDebug && url) {
+        console.log(`[debug] Using direct referrer URL: ${url}`);
+      } else if (forceDebug && !url) {
+        console.log(`[debug] Invalid referrer URL provided: ${referrerConfig}`);
+      }
+      return url;
+    }
+    
+    // Handle arrays - pick random URL
+    if (Array.isArray(referrerConfig)) {
+      if (referrerConfig.length === 0) {
+        if (forceDebug) console.log(`[debug] Empty referrer array provided`);
+        return '';
+      }
+      
+      const randomUrl = referrerConfig[Math.floor(Math.random() * referrerConfig.length)];
+      const url = isValidUrl(randomUrl) ? randomUrl : '';
+      
+      if (forceDebug) {
+        console.log(`[debug] Selected referrer from array (${referrerConfig.length} options): ${url || 'invalid URL'}`);
+      }
+      
+      return url;
+    }
+    
+    // Handle object modes
+    if (typeof referrerConfig === 'object' && referrerConfig !== null && referrerConfig.mode) {
+      switch (referrerConfig.mode) {
+        case 'random_search': {
+          const searchTerms = referrerConfig.search_terms;
+          const context = referrerConfig.context; // Optional context hint
+          return generateSearchReferrer(searchTerms, context, forceDebug);
+        }
+        
+        case 'social_media': {
+          return generateSocialMediaReferrer(forceDebug);
+        }
+        
+        case 'news_sites': {
+          return generateNewsReferrer(forceDebug);
+        }
+        
+        case 'direct_navigation': {
+          if (forceDebug) console.log(`[debug] Using direct navigation (no referrer)`);
+          return '';
+        }
+        
+        case 'custom': {
+          const url = isValidUrl(referrerConfig.url) ? referrerConfig.url : '';
+          if (forceDebug) {
+            console.log(`[debug] Using custom referrer URL: ${url || 'invalid URL provided'}`);
+          }
+          return url;
+        }
+        
+        case 'mixed': {
+          // Randomly choose between different referrer types
+          const modes = ['random_search', 'social_media', 'news_sites'];
+          const randomMode = modes[Math.floor(Math.random() * modes.length)];
+          
+          if (forceDebug) console.log(`[debug] Mixed mode selected: ${randomMode}`);
+          
+          const mixedConfig = { mode: randomMode };
+          if (randomMode === 'random_search' && referrerConfig.search_terms) {
+            mixedConfig.search_terms = referrerConfig.search_terms;
+            mixedConfig.context = referrerConfig.context;
+          }
+          
+          return generateReferrerUrl(mixedConfig, forceDebug);
+        }
+        
+        default: {
+          if (forceDebug) console.log(`[debug] Unknown referrer mode: ${referrerConfig.mode}`);
+          return '';
+        }
+      }
+    }
+    
+    if (forceDebug) console.log(`[debug] Invalid referrer configuration type: ${typeof referrerConfig}`);
+    return '';
+  } catch (err) {
+    if (forceDebug) console.log(`[debug] Referrer generation failed: ${err.message}`);
+    return '';
+  }
+}
+
+/**
+ * Main function to determine referrer for a specific URL
+ * Handles both referrer generation and referrer_disable functionality
+ * @param {string} targetUrl - The URL being visited
+ * @param {Object|string|Array} referrerConfig - Referrer configuration
+ * @param {Array} referrerDisable - Array of URLs that should have no referrer
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {string} Generated referrer URL or empty string if disabled/none
+ */
+function getReferrerForUrl(targetUrl, referrerConfig, referrerDisable, forceDebug = false) {
+  // Check if referrer should be disabled for this specific URL
+  if (shouldDisableReferrer(targetUrl, referrerDisable, forceDebug)) {
+    return '';
+  }
+  
+  // Generate referrer normally if not disabled
+  return generateReferrerUrl(referrerConfig, forceDebug);
+}
+
+/**
+ * Validates referrer configuration
+ * @param {Object|string|Array} referrerConfig - Referrer configuration to validate
+ * @returns {Object} Validation result with isValid flag and error messages
+ */
+function validateReferrerConfig(referrerConfig) {
+  const result = { isValid: true, errors: [], warnings: [] };
+  
+  if (!referrerConfig) {
+    result.isValid = false;
+    result.errors.push('Referrer configuration is required');
+    return result;
+  }
+  
+  // Validate string URLs
+  if (typeof referrerConfig === 'string') {
+    if (!isValidUrl(referrerConfig)) {
+      result.isValid = false;
+      result.errors.push('String referrer must be a valid HTTP/HTTPS URL');
+    }
+    return result;
+  }
+  
+  // Validate arrays
+  if (Array.isArray(referrerConfig)) {
+    if (referrerConfig.length === 0) {
+      result.warnings.push('Empty referrer array will result in no referrer');
+    } else {
+      referrerConfig.forEach((url, index) => {
+        if (!isValidUrl(url)) {
+          result.errors.push(`Array item ${index} is not a valid HTTP/HTTPS URL: ${url}`);
+          result.isValid = false;
+        }
+      });
+    }
+    return result;
+  }
+  
+  // Validate object modes
+  if (typeof referrerConfig === 'object') {
+    const validModes = ['random_search', 'social_media', 'news_sites', 'direct_navigation', 'custom', 'mixed'];
+    
+    if (!referrerConfig.mode) {
+      result.isValid = false;
+      result.errors.push('Object referrer configuration must have a "mode" property');
+      return result;
+    }
+    
+    if (!validModes.includes(referrerConfig.mode)) {
+      result.isValid = false;
+      result.errors.push(`Invalid referrer mode: ${referrerConfig.mode}. Valid modes: ${validModes.join(', ')}`);
+      return result;
+    }
+    
+    // Mode-specific validation
+    switch (referrerConfig.mode) {
+      case 'custom':
+        if (!referrerConfig.url) {
+          result.isValid = false;
+          result.errors.push('Custom mode requires a "url" property');
+        } else if (!isValidUrl(referrerConfig.url)) {
+          result.isValid = false;
+          result.errors.push('Custom mode URL must be a valid HTTP/HTTPS URL');
+        }
+        break;
+        
+      case 'random_search':
+        if (referrerConfig.search_terms && !Array.isArray(referrerConfig.search_terms)) {
+          result.warnings.push('search_terms should be an array of strings');
+        }
+        if (referrerConfig.search_terms && referrerConfig.search_terms.length === 0) {
+          result.warnings.push('Empty search_terms array will use default terms');
+        }
+        break;
+    }
+    
+    return result;
+  }
+  
+  result.isValid = false;
+  result.errors.push('Referrer configuration must be a string, array, or object');
+  return result;
+}
+
+/**
+ * Validates referrer_disable configuration
+ * @param {Array} referrerDisable - Array of URLs/patterns to disable referrer for
+ * @returns {Object} Validation result with isValid flag and error messages
+ */
+function validateReferrerDisable(referrerDisable) {
+  const result = { isValid: true, errors: [], warnings: [] };
+  
+  if (!referrerDisable) {
+    return result; // referrer_disable is optional
+  }
+  
+  if (!Array.isArray(referrerDisable)) {
+    result.isValid = false;
+    result.errors.push('referrer_disable must be an array of URLs/patterns');
+    return result;
+  }
+  
+  if (referrerDisable.length === 0) {
+    result.warnings.push('Empty referrer_disable array has no effect');
+    return result;
+  }
+  
+  referrerDisable.forEach((pattern, index) => {
+    if (typeof pattern !== 'string') {
+      result.errors.push(`referrer_disable item ${index} must be a string (got ${typeof pattern})`);
+      result.isValid = false;
+    } else if (pattern.trim() === '') {
+      result.warnings.push(`referrer_disable item ${index} is empty string`);
+    } else if (!pattern.includes('.') && !pattern.includes('/')) {
+      result.warnings.push(`referrer_disable item ${index} "${pattern}" might be too broad - consider using full URLs or hostnames`);
+    }
+  });
+  
+  if (referrerDisable.length > 100) {
+    result.warnings.push('Large referrer_disable list (>100 items) may impact performance');
+  }
+  
+  return result;
+}
+
+/**
+ * Gets available referrer modes and their descriptions
+ * @returns {Object} Object containing mode descriptions
+ */
+function getReferrerModes() {
+  return {
+    'random_search': 'Generate random search engine referrers with customizable search terms',
+    'social_media': 'Use random social media platform referrers',
+    'news_sites': 'Use random news website referrers',
+    'direct_navigation': 'No referrer (simulates direct URL entry)',
+    'custom': 'Use a specific custom referrer URL',
+    'mixed': 'Randomly mix different referrer types for varied traffic simulation'
+  };
+}
+
+/**
+ * Gets statistics about available referrer collections
+ * @returns {Object} Statistics about referrer collections
+ */
+function getReferrerStats() {
+  return {
+    searchEngines: REFERRER_COLLECTIONS.SEARCH_ENGINES.length,
+    socialMedia: REFERRER_COLLECTIONS.SOCIAL_MEDIA.length,
+    newsSites: REFERRER_COLLECTIONS.NEWS_SITES.length,
+    defaultSearchTerms: REFERRER_COLLECTIONS.DEFAULT_SEARCH_TERMS.length,
+    ecommerceTerms: REFERRER_COLLECTIONS.ECOMMERCE_TERMS.length,
+    techTerms: REFERRER_COLLECTIONS.TECH_TERMS.length
+  };
+}
+
+module.exports = {
+  generateReferrerUrl,
+  getReferrerForUrl,
+  shouldDisableReferrer,
+  validateReferrerConfig,
+  validateReferrerDisable,
+  getReferrerModes,
+  getReferrerStats,
+  generateSearchReferrer,
+  generateSocialMediaReferrer,
+  generateNewsReferrer,
+  isValidUrl,
+  REFERRER_COLLECTIONS
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.29 ===
+// === Network scanner script (nwss.js) v2.0.31 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -48,6 +48,8 @@ const { clearPersistentCache } = require('./lib/smart-cache');
 const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
 const { clearSiteData } = require('./lib/clear_sitedata');
+// Referrer header generation
+const { getReferrerForUrl, validateReferrerConfig, validateReferrerDisable } = require('./lib/referrer');
 
 // Fast setTimeout helper for Puppeteer 22.x compatibility
 // Uses standard Promise constructor for better performance than node:timers/promises
@@ -143,7 +145,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.29'; // Script version
+const VERSION = '2.0.31'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -368,9 +370,22 @@ if (validateConfig) {
     // Validate referrer_headers format
     for (const site of sites) {
        if (site.referrer_headers && typeof site.referrer_headers === 'object' && !Array.isArray(site.referrer_headers)) {
-         const validModes = ['random_search', 'social_media', 'direct_navigation', 'custom'];
-         if (site.referrer_headers.mode && !validModes.includes(site.referrer_headers.mode)) {
-           console.warn(`⚠ Invalid referrer_headers mode: ${site.referrer_headers.mode}. Valid modes: ${validModes.join(', ')}`);
+         const validation = validateReferrerConfig(site.referrer_headers);
+         if (!validation.isValid) {
+           console.warn(`⚠ Invalid referrer_headers configuration: ${validation.errors.join(', ')}`);
+         }
+         if (validation.warnings.length > 0) {
+           console.warn(`⚠ Referrer warnings: ${validation.warnings.join(', ')}`);
+         }
+       }
+       // Validate referrer_disable format
+       if (site.referrer_disable) {
+         const disableValidation = validateReferrerDisable(site.referrer_disable);
+         if (!disableValidation.isValid) {
+           console.warn(`⚠ Invalid referrer_disable configuration: ${disableValidation.errors.join(', ')}`);
+         }
+         if (disableValidation.warnings.length > 0) {
+           console.warn(`⚠ Referrer disable warnings: ${disableValidation.warnings.join(', ')}`);
          }
        }
     }
@@ -551,6 +566,7 @@ Redirect Handling Options:
   bypass_cache: true/false                     Skip all caching for this site's URLs (default: false)
   referrer_headers: "url" or ["url1", "url2"] Set referrer header for realistic traffic sources
   custom_headers: {"Header": "value"}         Add custom HTTP headers to requests
+  referrer_disable: ["url1", "url2"]         Disable referrer headers for specific URLs
 
 Cloudflare Protection Options:
   cloudflare_phish: true/false                 Auto-click through Cloudflare phishing warnings (default: false)
@@ -606,6 +622,10 @@ Referrer Header Options:
   referrer_headers: {"mode": "random_search", "search_terms": ["term1"]} Smart search engine traffic
   referrer_headers: {"mode": "social_media"}   Random social media referrers
   referrer_headers: {"mode": "direct_navigation"} No referrer (direct access)
+  referrer_headers: {"mode": "news_sites"}     Random news website referrers
+  referrer_headers: {"mode": "custom", "url": "https://example.com"} Custom referrer URL
+  referrer_headers: {"mode": "mixed"}          Mixed referrer types for varied traffic
+  referrer_disable: ["https://example.com/no-ref", "sensitive-site.com"] Disable referrer for specific URLs
   custom_headers: {"Header": "Value"}          Additional HTTP headers
 `);
   process.exit(0);
@@ -1808,7 +1828,15 @@ function setupFrameHandling(page, forceDebug) {
                                   throw new Error('Page is closed');
                               }
                               
-                              const pageUrl = await page.url().catch(() => 'about:blank');
+                              // FIX: Properly wrap page.url() in try-catch to handle race condition
+                              let pageUrl;
+                              try {
+                                  pageUrl = await page.url();
+                              } catch (urlErr) {
+                                  // Page closed between isClosed check and url call
+                                  throw new Error('Page closed while getting URL');
+                              }
+
                               if (pageUrl === 'about:blank') {
                                   throw new Error('Cannot inject on about:blank');
                               }
@@ -2848,20 +2876,23 @@ function setupFrameHandling(page, forceDebug) {
       // --- Runtime CSS Element Blocking (Fallback) ---
       // Apply CSS blocking after page load as a fallback in case evaluateOnNewDocument didn't work
       if (cssBlockedSelectors && Array.isArray(cssBlockedSelectors) && cssBlockedSelectors.length > 0) {
-        try {
-          await page.evaluate((selectors) => {
-            const existingStyle = document.querySelector('#css-blocker-runtime');
-            if (!existingStyle) {
-              const style = document.createElement('style');
-              style.id = 'css-blocker-runtime';
-              style.type = 'text/css';
-              const cssRules = selectors.map(selector => `${selector} { display: none !important; visibility: hidden !important; }`).join('\n');
-              style.innerHTML = cssRules;
-              document.head.appendChild(style);
-            }
-          }, cssBlockedSelectors);
-        } catch (cssRuntimeErr) {
-          console.warn(formatLogMessage('warn', `[css_blocked] Failed to apply runtime CSS blocking for ${currentUrl}: ${cssRuntimeErr.message}`));
+        // FIX: Check page state before evaluation
+        if (page && !page.isClosed()) {
+          try {
+            await page.evaluate((selectors) => {
+              const existingStyle = document.querySelector('#css-blocker-runtime');
+              if (!existingStyle) {
+                const style = document.createElement('style');
+                style.id = 'css-blocker-runtime';
+                style.type = 'text/css';
+                const cssRules = selectors.map(selector => `${selector} { display: none !important; visibility: hidden !important; }`).join('\n');
+                style.innerHTML = cssRules;
+                document.head.appendChild(style);
+              }
+            }, cssBlockedSelectors);
+          } catch (cssRuntimeErr) {
+            console.warn(formatLogMessage('warn', `[css_blocked] Failed to apply runtime CSS blocking for ${currentUrl}: ${cssRuntimeErr.message}`));
+          }
         }
       }
 
@@ -2885,11 +2916,8 @@ function setupFrameHandling(page, forceDebug) {
           timeout: Math.min(timeout, TIMEOUTS.DEFAULT_PAGE), // Cap at default page timeout
           // Puppeteer 23.x: Fixed referrer header handling
           ...(siteConfig.referrer_headers && (() => {
-            const referrerUrl = Array.isArray(siteConfig.referrer_headers) 
-              ? siteConfig.referrer_headers[0] 
-              : siteConfig.referrer_headers;
-            // Ensure referrer is a valid string URL, not an object
-            return typeof referrerUrl === 'string' && referrerUrl.startsWith('http') 
+            const referrerUrl = getReferrerForUrl(currentUrl, siteConfig.referrer_headers, siteConfig.referrer_disable, forceDebug);
+            return referrerUrl 
               ? { referer: referrerUrl } 
               : {};
           })())
@@ -3069,7 +3097,15 @@ function setupFrameHandling(page, forceDebug) {
         }
 
         console.log(formatLogMessage('info', `${messageColors.loaded('Loaded:')} (${siteCounter}/${totalUrls}) ${currentUrl}`));
-        await page.evaluate(() => { console.log('Safe to evaluate on loaded page.'); });
+
+        // FIX: Check page state before evaluation
+        if (page && !page.isClosed()) {
+          try {
+            await page.evaluate(() => { console.log('Safe to evaluate on loaded page.'); });
+          } catch (evalErr) {
+            // Page closed during evaluation - safe to ignore
+          }
+        }
         
         // Mark page as processing frames
         updatePageUsage(page, true);
@@ -3146,10 +3182,19 @@ function setupFrameHandling(page, forceDebug) {
       const networkIdleTimeout = Math.min(timeout / 2, TIMEOUTS.NETWORK_IDLE_MAX);  // Balanced: 10s timeout
       const actualDelay = Math.min(delayMs, TIMEOUTS.NETWORK_IDLE);  // Balanced: 2s delay for stability
       
-      await page.waitForNetworkIdle({ 
-        idleTime: networkIdleTime, 
-        timeout: networkIdleTimeout 
-      });
+      // FIX: Check page state before waiting for network idle
+      if (page && !page.isClosed()) {
+        try {
+          await page.waitForNetworkIdle({ 
+            idleTime: networkIdleTime, 
+            timeout: networkIdleTimeout 
+          });
+        } catch (networkIdleErr) {
+          // Page closed or network idle timeout - continue anyway
+          if (forceDebug) console.log(formatLogMessage('debug', `Network idle wait failed: ${networkIdleErr.message}`));
+        }
+      }
+
       // Use fast timeout helper for Puppeteer 23.x compatibility with better performance
       await fastTimeout(actualDelay);
 
@@ -3330,6 +3375,11 @@ function setupFrameHandling(page, forceDebug) {
       // Fallback to standard reload if force reload failed or wasn't attempted
       if (!reloadSuccess) {
         try {
+            // FIX: Check page state before reload validation
+            if (page.isClosed()) {
+              throw new Error('Page closed before reload check');
+            }
+
             const canReload = await page.evaluate(() => {
               return !!(document && document.body);
             }).catch(() => false);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.30",
+  "version": "2.0.31",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {