@fanboynz/network-scanner 2.0.28 → 2.0.30

package/.clauderc

@@ -0,0 +1,30 @@
+{
+  "description": "Network scanner that monitors website requests and generates blocking rules. Uses Puppeteer to load sites, intercepts network traffic, matches patterns, and outputs rules in various formats (adblock, dnsmasq, hosts file, etc.).",
+  
+  "conventions": [
+    "Store modular functionality in ./lib/ directory with focused, single-purpose modules",
+    "Use messageColors and formatLogMessage from ./lib/colorize for consistent console output",
+    "Implement timeout protection for all Puppeteer operations using Promise.race patterns",
+    "Handle browser lifecycle with comprehensive cleanup in try-finally blocks",
+    "Validate all external tool availability before use (grep, curl, whois, dig)",
+    "Use forceDebug flag for detailed logging, silentMode for minimal output"
+  ],
+  
+  "files": {
+    "important": [
+      "nwss.js",
+      "config.json",
+      "lib/*.js",
+      "*.md",
+      "nwss.1"
+    ],
+    "ignore": [
+      "node_modules/**",
+      "logs/**",
+      "sources/**",
+      ".cache/**",
+      "*.log",
+      "*.gz"
+    ]
+  }
+}

package/lib/fingerprint.js

@@ -59,7 +59,7 @@ const PRECOMPILED_MOCKS = Object.freeze({
       postMessage: () => {},
       disconnect: () => {}
     }),
-    getManifest: () => Object.freeze({ name: "Chrome", version: "141.0.0.0" }),
+    getManifest: () => Object.freeze({ name: "Chrome", version: "142.0.0.0" }),
     getURL: (path) => `chrome-extension://invalid/${path}`,
     id: undefined
   }),
@@ -89,22 +89,15 @@ const BUILT_IN_PROPERTIES = new Set([
 ]);
 
 // User agent collections with latest versions
-const USER_AGENT_COLLECTIONS = {
-  chrome: [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
-  ],
-  firefox: [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.0) Gecko/20100101 Firefox/143.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",
-    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:143.0) Gecko/20100101 Firefox/143.0"
-  ],
-  safari: [
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15"
-  ]
-};
+const USER_AGENT_COLLECTIONS = Object.freeze(new Map([
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
+]));
 
 // Timezone configuration with offsets
 const TIMEZONE_CONFIG = {
@@ -366,8 +359,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
   // Validate page state before injection
   if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
 
-  const selectedUserAgents = USER_AGENT_COLLECTIONS[siteConfig.userAgent.toLowerCase()];
-  const ua = selectedUserAgents ? selectedUserAgents[Math.floor(Math.random() * selectedUserAgents.length)] : null;
+  const ua = USER_AGENT_COLLECTIONS.get(siteConfig.userAgent.toLowerCase());
   
   if (ua) {
     await page.setUserAgent(ua);
@@ -512,7 +504,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
             'phantomjs', '_Selenium_IDE_Recorder', 'callSelenium', '_selenium',
             '__phantomas', '__selenium_evaluate', '__driver_unwrapped',
             'webdriver-evaluate', '__webdriverFunc', 'driver-evaluate', '__driver-evaluate', '__selenium-evaluate',
-            'spawn', 'emit', 'Buffer', 'domAutomation', 'domAutomationController'
+            'spawn', 'emit', 'Buffer', 'domAutomation', 'domAutomationController',
+            'cdc_adoQpoasnfa76pfcZLmcfl_JSON', 'cdc_adoQpoasnfa76pfcZLmcfl_Object',
+            'cdc_adoQpoasnfa76pfcZLmcfl_Proxy', 'cdc_adoQpoasnfa76pfcZLmcfl_Reflect',
+            '$cdc_asdjflasutopfhvcZLmcfl_', '$chrome_asyncScriptInfo', '__$webdriverAsyncExecutor'
           ];
           
           automationProps.forEach(prop => {
@@ -528,16 +523,45 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         // Simulate Chrome runtime
         //
         safeExecute(() => {
-          if (!window.chrome?.runtime) {
+          if (!window.chrome) {
             window.chrome = {
               runtime: {
                 onConnect: { addListener: () => {}, removeListener: () => {} },
                 onMessage: { addListener: () => {}, removeListener: () => {} },
                 sendMessage: () => {},
-                connect: () => ({ onMessage: { addListener: () => {}, removeListener: () => {} }, postMessage: () => {}, disconnect: () => {} }),
-                getManifest: () => ({ name: "Chrome", version: "141.0.0.0" }),
+                connect: () => ({ 
+                  onMessage: { addListener: () => {}, removeListener: () => {} }, 
+                  postMessage: () => {}, 
+                  disconnect: () => {} 
+                }),
+                getManifest: () => ({ 
+                  name: "Chrome", 
+                  version: "142.0.0.0",
+                  manifest_version: 3,
+                  description: "Chrome Browser"
+                }),
                 getURL: (path) => `chrome-extension://invalid/${path}`,
-                id: undefined
+                id: undefined,
+                getPlatformInfo: (callback) => callback({
+                  os: navigator.platform.includes('Win') ? 'win' : 
+                      navigator.platform.includes('Mac') ? 'mac' : 'linux',
+                  arch: 'x86-64',
+                  nacl_arch: 'x86-64'
+                })
+              },
+              storage: {
+                local: {
+                  get: (keys, callback) => callback && callback({}),
+                  set: (items, callback) => callback && callback(),
+                  remove: (keys, callback) => callback && callback(),
+                  clear: (callback) => callback && callback()
+                },
+                sync: {
+                  get: (keys, callback) => callback && callback({}),
+                  set: (items, callback) => callback && callback(),
+                  remove: (keys, callback) => callback && callback(),
+                  clear: (callback) => callback && callback()
+                }
               },
               loadTimes: () => {
                 const now = performance.now();
@@ -555,9 +579,56 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
                 startE: Date.now() - Math.random() * 2000
               })
             };
+            
+            // Make chrome object non-enumerable to match real Chrome
+            Object.defineProperty(window, 'chrome', {
+              value: window.chrome,
+              writable: false,
+              enumerable: false,
+              configurable: true
+            });
+            
+            // Add Chrome-specific globals that Cloudflare might check
+            if (!window.external) {
+              window.external = {
+                AddSearchProvider: () => {},
+                IsSearchProviderInstalled: () => 0
+              };
+            }
+            
+            // Ensure chrome.runtime appears as a native object
+            Object.defineProperty(window.chrome.runtime, 'toString', {
+              value: () => '[object Object]'
+            });
           }
         }, 'Chrome runtime simulation');
 
+        // Add realistic Chrome browser behavior
+        //
+        safeExecute(() => {
+          // Remove Puppeteer-specific properties not covered in main automation cleanup
+          delete window.__puppeteer_evaluation_script__;
+          delete window.__runtime;
+          delete window._asyncToGenerator;
+          delete window.__puppeteer;
+          delete window.__cdp;
+          delete window.__REACT_DEVTOOLS_GLOBAL_HOOK__;
+          
+          // Simulate Chrome's performance observer
+          if (window.PerformanceObserver) {
+            try {
+              const observer = new PerformanceObserver(() => {});
+              observer.observe({entryTypes: ['navigation']});
+            } catch(e) {
+              // Silently ignore if PerformanceObserver fails
+            }
+          }
+          
+          // Ensure external object exists (don't overwrite chrome object)
+          window.external = window.external || {};
+          
+        }, 'realistic Chrome behavior');
+
         // Spoof plugins based on user agent
         //
         safeExecute(() => {
@@ -874,29 +945,33 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
           if (navigator.permissions?.query) {
             const originalQuery = navigator.permissions.query;
             navigator.permissions.query = function(descriptor) {
-              // More realistic permission states based on permission type
+
               const permissionName = descriptor.name || descriptor;
-              let state = 'prompt'; // Default state
+              // Realistic Chrome permission defaults
+              const chromeDefaults = {
+                'notifications': 'default',
+                'geolocation': 'prompt', 
+                'camera': 'prompt',
+                'microphone': 'prompt',
+                'persistent-storage': 'granted',
+                'background-sync': 'granted',
+                'midi': 'prompt',
+                'push': 'prompt',
+                'accelerometer': 'granted',
+                'gyroscope': 'granted',
+                'magnetometer': 'granted'
+              };
               
-              // Common permissions that are usually granted
-              if (['notifications', 'persistent-storage'].includes(permissionName)) {
-                state = Math.random() > 0.3 ? 'granted' : 'prompt';
-              }
-              // Privacy-sensitive permissions that are usually denied/prompt
-              else if (['camera', 'microphone', 'geolocation'].includes(permissionName)) {
-                state = Math.random() > 0.8 ? 'granted' : 'prompt';
-              }
-              // Other permissions random
-              else {
-                state = Math.random() > 0.5 ? 'granted' : 'prompt';
-              }
+              const state = chromeDefaults[permissionName] || 'prompt';
               
               return Promise.resolve({ 
-                state: state,
+                state,
                 onchange: null
               });
-            };
+              }
+
           }
+
           // Block permission prompts from actually appearing
           if (window.Notification && Notification.requestPermission) {
             Notification.requestPermission = () => Promise.resolve('default');
@@ -995,12 +1070,28 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         // Font fingerprinting protection
         //
         safeExecute(() => {
-          // Standardize available fonts to common system fonts
-          const standardFonts = [
-            'Arial', 'Helvetica', 'Times New Roman', 'Times', 'Courier New', 'Courier',
-            'Verdana', 'Georgia', 'Palatino', 'Garamond', 'Bookman', 'Comic Sans MS',
-            'Trebuchet MS', 'Arial Black', 'Impact', 'Tahoma', 'Lucida Console'
-          ];
+          // OS-specific font profiles for better realism
+          const getOSFonts = (userAgent) => {
+            if (userAgent.includes('Windows') || userAgent.includes('Win')) {
+              return ['Arial', 'Times New Roman', 'Courier New', 'Verdana', 'Tahoma', 'Trebuchet MS', 'Georgia', 'Impact', 'Comic Sans MS', 'Segoe UI'];
+            } else if (userAgent.includes('Macintosh') || userAgent.includes('Mac OS X')) {
+              return ['Arial', 'Times New Roman', 'Courier New', 'Helvetica', 'Times', 'Courier', 'Verdana', 'Georgia', 'Palatino', 'San Francisco'];
+            } else {
+              return ['Arial', 'Times New Roman', 'Courier New', 'Liberation Sans', 'Liberation Serif', 'DejaVu Sans', 'Ubuntu'];
+            }
+          };
+          const standardFonts = getOSFonts(navigator.userAgent);
+          
+          // CRITICAL: Block font enumeration
+          if (document.fonts) {
+            Object.defineProperty(document.fonts, 'values', {
+              value: () => standardFonts.map(font => ({ family: font, style: 'normal', weight: '400' }))[Symbol.iterator]()
+            });
+            Object.defineProperty(document.fonts, 'forEach', {
+              value: (callback) => standardFonts.forEach((font, i) => callback({ family: font, style: 'normal', weight: '400' }, i))
+            });
+            Object.defineProperty(document.fonts, 'size', { get: () => standardFonts.length });
+          }
           
           // Intercept font availability detection
           if (document.fonts && document.fonts.check) {
@@ -1637,4 +1728,4 @@ module.exports = {
   applyTimezoneSpoofing,
   DEFAULT_PLATFORM,
   DEFAULT_TIMEZONE
-};
+};

package/lib/nettools.js

@@ -779,6 +779,8 @@ function createNetToolsHandler(config) {
     isDomainAlreadyDetected,
     getRootDomain,
     siteConfig,
+    processedWhoisDomains = new Set(), // Accept global sets, fallback to new for backward compatibility
+    processedDigDomains = new Set(),
     dumpUrls,
     matchedUrlsLogFile,
     forceDebug,
@@ -790,9 +792,22 @@ function createNetToolsHandler(config) {
   const hasDig = digTerms && Array.isArray(digTerms) && digTerms.length > 0;
   const hasDigOr = digOrTerms && Array.isArray(digOrTerms) && digOrTerms.length > 0;
   
-  // Add separate deduplication caches for different lookup types
-  const processedWhoisDomains = new Set();
-  const processedDigDomains = new Set();
+  // Create config-aware cache keys for deduplication
+  // Whois: Only include search terms + server (domain registry data is consistent across subdomains)
+  const whoisConfigKey = JSON.stringify({
+    terms: whoisTerms || [],
+    orTerms: whoisOrTerms || [],
+    server: whoisServer || 'default',
+    serverMode: whoisServerMode || 'random'
+  });
+  // Dig: Include all config (DNS records can vary by specific subdomain)
+  const digConfigKey = JSON.stringify({
+    terms: digTerms || [],
+    orTerms: digOrTerms || [],
+    recordType: digRecordType,
+    subdomain: digSubdomain
+  });
+
   // Add whois resolution caching to avoid redundant whois lookups
   const whoisResultCache = new Map();
   const WHOIS_CACHE_TTL = 900000; // 15 minutes cache TTL (whois data changes less frequently)
@@ -853,15 +868,24 @@ function createNetToolsHandler(config) {
     
     // Determine which domain will be used for dig lookup
     const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
+
+    // For whois: use root domain only (whois data is consistent for entire domain)
+    const whoisRootDomain = getRootDomain ? getRootDomain(`http://${domain}`) : domain;
     
-    // Check if we need to perform any lookups
-    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(domain);
-    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDomain);
+    // Check if we need to perform any lookups with appropriate deduplication
+    // Whois: root domain + config (whois data same for sub.example.com and example.com)
+    const whoisDedupeKey = `${whoisRootDomain}:${whoisConfigKey}`;
+    // Dig: specific subdomain + config (DNS records can differ between subdomains)
+    const digDedupeKey = `${digDomain}:${digConfigKey}`;
+    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(whoisDedupeKey);
+    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDedupeKey);
     
     // Skip if we don't need to perform any lookups
     if (!needsWhoisLookup && !needsDigLookup) {
       if (forceDebug) {
-        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${!needsWhoisLookup}, dig: ${!needsDigLookup})`);
+        const whoisSkipped = (hasWhois || hasWhoisOr) ? `cached(${whoisRootDomain})` : 'n/a';
+        const digSkipped = (hasDig || hasDigOr) ? `cached(${digDomain})` : 'n/a';
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${whoisSkipped}, dig: ${digSkipped})`);
       }
       return;
     }
@@ -869,7 +893,7 @@ function createNetToolsHandler(config) {
 
     if (forceDebug) {
       const totalProcessed = processedWhoisDomains.size + processedDigDomains.size;
-      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup}, dig: ${needsDigLookup}) (${totalProcessed} total processed)`);
+      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup ? whoisRootDomain : 'skip'}, dig: ${needsDigLookup ? digDomain : 'skip'}) (${totalProcessed} total processed)`);
     }
 
       // Log site-specific whois delay if different from default
@@ -955,12 +979,12 @@ function createNetToolsHandler(config) {
       
       // Perform whois lookup if either whois or whois-or is configured
       if (needsWhoisLookup) {
-        // Mark whois domain as being processed
-        processedWhoisDomains.add(domain);
+        // Mark whois root domain+config as being processed
+        processedWhoisDomains.add(whoisDedupeKey);
         
         // Check whois cache first - cache key includes server for accuracy
         const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
-        const whoisCacheKey = `${domain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
+        const whoisCacheKey = `${whoisRootDomain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
         const now = Date.now();
         let whoisResult = null;
         
@@ -970,7 +994,7 @@ function createNetToolsHandler(config) {
             if (forceDebug) {
               const age = Math.round((now - cachedEntry.timestamp) / 1000);
               const serverInfo = (selectedServer && selectedServer !== '') ? ` (server: ${selectedServer})` : ' (default server)';
-              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${whoisRootDomain}${serverInfo} [age: ${age}s]`);
             }
             // V8 Optimized: Object.assign is faster than spread for object merging
             whoisResult = Object.assign({}, cachedEntry.result, {
@@ -982,7 +1006,7 @@ function createNetToolsHandler(config) {
             // Cache expired, remove it
             whoisResultCache.delete(whoisCacheKey);
             if (forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${domain}, performing fresh lookup`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${whoisRootDomain}, performing fresh lookup`);
             }
           }
         }
@@ -991,7 +1015,7 @@ function createNetToolsHandler(config) {
         if (!whoisResult) {
           if (forceDebug) {
             const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
+            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
           }
         
           // Configure retry options based on site config or use defaults
@@ -1004,7 +1028,7 @@ function createNetToolsHandler(config) {
           };
           
           try {
-            whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+            whoisResult = await whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
             
             // Cache successful results (and certain types of failures)
             if (whoisResult.success || 
@@ -1020,13 +1044,13 @@ function createNetToolsHandler(config) {
               if (forceDebug) {
                 const cacheType = whoisResult.success ? 'successful' : 'failed';
                 const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
-                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${domain}${serverInfo}`);
+                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${whoisRootDomain}${serverInfo}`);
               }
             }
           } catch (whoisError) {
             // Handle exceptions from whois lookup
             if (forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${whoisRootDomain}: ${whoisError.message}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
               if (whoisError.stack) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
@@ -1035,7 +1059,7 @@ function createNetToolsHandler(config) {
             
             // Log whois exceptions in dry run mode
             if (dryRunCallback && forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception for ${whoisRootDomain}: ${whoisError.message}`);
             }
             // Continue with dig if configured
             whoisResult = null; // Ensure we don't process a null result
@@ -1086,7 +1110,7 @@ function createNetToolsHandler(config) {
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
               const cacheInfo = whoisResult.fromCache ? ` [CACHED - ${Math.round(whoisResult.cacheAge / 1000)}s old]` : '';
               const duration = whoisResult.fromCache ? `cached in 0ms` : `in ${whoisResult.duration}ms`;
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${whoisRootDomain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
               
               if (whoisResult.retryInfo && whoisResult.retryInfo.retriedAfterFailure) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Success after retry - servers attempted: [${whoisResult.retryInfo.serversAttempted.map(s => s || 'default').join(', ')}]`);
@@ -1099,7 +1123,7 @@ function createNetToolsHandler(config) {
               const errorContext = whoisResult.isTimeout ? 'TIMEOUT' : 'ERROR';
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
               
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} ${errorContext}: Lookup failed for ${domain}${serverUsed} after ${whoisResult.duration}ms${retryInfo}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} ${errorContext}: Lookup failed for ${whoisRootDomain}${serverUsed} after ${whoisResult.duration}ms${retryInfo}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Command executed: ${whoisResult.command || 'unknown'}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Error details: ${whoisResult.error}`);
               
@@ -1182,8 +1206,8 @@ function createNetToolsHandler(config) {
       
       // Perform dig lookup if configured
       if (needsDigLookup) {
-        // Mark dig domain as being processed
-        processedDigDomains.add(digDomain);
+        // Mark dig domain+config as being processed (includes specific subdomain)
+        processedDigDomains.add(digDedupeKey);
         
         if (forceDebug) {
           const digTypes = [];

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.27 ===
+// === Network scanner script (nwss.js) v2.0.29 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -89,12 +89,12 @@ const CONCURRENCY_LIMITS = Object.freeze({
 
 // V8 Optimization: Use Map for user agent lookups instead of object
 const USER_AGENTS = Object.freeze(new Map([
-  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
-  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
-  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0"],
   ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
 ]));
 
@@ -143,7 +143,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.27'; // Script version
+const VERSION = '2.0.29'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -747,6 +747,12 @@ function safeMarkDomainProcessed(domain, context, metadata) {
   }
 }
 
+// Global deduplication for nettools to avoid redundant lookups across all URLs
+// Whois: keyed by root domain only (whois data is consistent for entire domain)
+// Dig: keyed by specific subdomain+config (DNS records can vary by subdomain)
+const globalProcessedWhoisDomains = new Set();
+const globalProcessedDigDomains = new Set();
+
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
   const filesToClean = cleanRulesFile ? [cleanRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -1245,6 +1251,16 @@ function setupFrameHandling(page, forceDebug) {
       // Auto-detect best headless mode based on Puppeteer version
       headless: headlessMode,
       args: [
+        // CRITICAL: Remove automation detection markers
+        '--disable-blink-features=AutomationControlled',
+        '--no-first-run',
+        '--disable-default-apps',
+        '--disable-component-extensions-with-background-pages',
+        // HIGH IMPACT: Normal Chrome behavior simulation
+        '--password-store=basic',
+        '--use-mock-keychain',
+        '--disable-client-side-phishing-detection',
+        '--enable-features=NetworkService',
         // Disk space controls - 50MB cache limits
         '--disable-features=VizDisplayCompositor',
         `--disk-cache-size=${CACHE_LIMITS.DISK_CACHE_SIZE}`, // 50MB disk cache
@@ -1260,10 +1276,7 @@ function setupFrameHandling(page, forceDebug) {
         '--aggressive-cache-discard',
         '--memory-pressure-off',
         '--max_old_space_size=2048',
-        '--no-first-run',
         '--disable-prompt-on-repost',  // Fixes form popup on page reload
-        '--disable-default-apps',
-        '--disable-component-extensions-with-background-pages',
         '--disable-background-networking',
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -1290,14 +1303,13 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-backgrounding-occluded-windows',
         '--disable-background-timer-throttling',
         '--disable-features=site-per-process', // Better for single-site scanning
-        '--disable-blink-features=AutomationControlled', // Avoid detection
         '--no-zygote', // Better process isolation
         ],
         // Optimized timeouts for Puppeteer 23.x performance
         protocolTimeout: TIMEOUTS.PROTOCOL_TIMEOUT,
         slowMo: 0, // No artificial delays
         defaultViewport: null, // Use system default viewport
-        ignoreDefaultArgs: ['--enable-automation'] // Avoid automation detection
+        ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=AutomationControlled'] // Avoid automation detection
     });
     
     // Store the user data directory on the browser object for cleanup
@@ -1953,14 +1965,14 @@ function setupFrameHandling(page, forceDebug) {
           }
                     
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Chromium";v="141", "Not=A?Brand";v="24", "Google Chrome";v="141"',
+            'Sec-CH-UA': '"Chromium";v="142", "Not=A?Brand";v="24", "Google Chrome";v="142"',
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"141.0.7390.55"',
-            'Sec-CH-UA-Full-Version-List': '"Chromium";v="141.0.7390.55", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="141.0.7390.55"'
+            'Sec-CH-UA-Full-Version': '"142.0.7444,69"',
+            'Sec-CH-UA-Full-Version-List': '"Chromium";v="142.0.7444,69", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="142.0.7444,69"'
           });
         }
       } catch (fingerprintErr) {
@@ -2485,6 +2497,8 @@ function setupFrameHandling(page, forceDebug) {
               const netToolsHandler = createNetToolsHandler({
                 whoisTerms,
                 whoisOrTerms,
+               processedWhoisDomains: globalProcessedWhoisDomains,
+               processedDigDomains: globalProcessedDigDomains,
                 whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
                 whoisServer,
                 whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
@@ -2591,6 +2605,8 @@ function setupFrameHandling(page, forceDebug) {
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
+               processedWhoisDomains: globalProcessedWhoisDomains,
+               processedDigDomains: globalProcessedDigDomains,
                whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay, // Site-specific or global fallback
 	       whoisServer, // Pass whois server configuration
                whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
@@ -3544,21 +3560,39 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
 
-  // Hang detection for debugging concurrency issues
-  let currentBatchInfo = { batchStart: 0, batchSize: 0 };
-  const hangDetectionInterval = setInterval(() => {
-  // Only show hang detection messages in debug mode
-  if (forceDebug) {
-    const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
-    const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
-    console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
-    console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
-    }
-  }, 30000); // Check every 30 seconds
+ // Enhanced hang detection with browser restart recovery
+ let currentBatchInfo = { batchStart: 0, batchSize: 0 };
+ let lastProcessedCount = 0;
+ let hangCheckCount = 0;
+ let forceRestartFlag = false; // Flag to trigger restart on next iteration
+ 
+ const hangDetectionInterval = setInterval(() => {
+   if (forceDebug) {
+     const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
+     const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
+     console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
+     console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
+     
+     // Check progress and trigger browser restart if hung
+     if (processedUrlCount === lastProcessedCount) {
+       hangCheckCount++;
+       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
+       if (hangCheckCount >= 5) {
+         console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
+         forceRestartFlag = true; // Set flag instead of exiting
+         hangCheckCount = 0; // Reset counter for next cycle
+       }
+     } else {
+       hangCheckCount = 0;
+     }
+     lastProcessedCount = processedUrlCount;
+   }
+ }, 30000);
 
-  // Process URLs in batches to maintain concurrency while allowing browser restarts
-  let siteGroupIndex = 0;
-  for (let batchStart = 0; batchStart < totalUrls; batchStart += RESOURCE_CLEANUP_INTERVAL) {
+ // Process URLs in batches with exception handling
+ let siteGroupIndex = 0;
+ try {
+   for (let batchStart = 0; batchStart < totalUrls; batchStart += RESOURCE_CLEANUP_INTERVAL) {
     const batchEnd = Math.min(batchStart + RESOURCE_CLEANUP_INTERVAL, totalUrls);
     const currentBatch = allTasks.slice(batchStart, batchEnd);
 
@@ -3587,14 +3621,22 @@ function setupFrameHandling(page, forceDebug) {
         hasCriticalErrors ||
         urlsSinceLastCleanup > RESOURCE_CLEANUP_INTERVAL * 0.9  // Very close to cleanup limit
     )) {
-      healthCheck = await monitorBrowserHealth(browser, {}, {
-        siteIndex: Math.floor(batchStart / RESOURCE_CLEANUP_INTERVAL),
-        totalSites: Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL),
-        urlsSinceCleanup: urlsSinceLastCleanup,
-        cleanupInterval: RESOURCE_CLEANUP_INTERVAL,
-        forceDebug,
-        silentMode
-      });
+     try {
+       healthCheck = await Promise.race([
+         monitorBrowserHealth(browser, {}, {
+           siteIndex: Math.floor(batchStart / RESOURCE_CLEANUP_INTERVAL),
+           totalSites: Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL),
+           urlsSinceCleanup: urlsSinceLastCleanup,
+           cleanupInterval: RESOURCE_CLEANUP_INTERVAL,
+           forceDebug,
+           silentMode
+         }),
+         new Promise((_, reject) => setTimeout(() => reject(new Error('Health check timeout')), 30000))
+       ]);
+     } catch (healthError) {
+       console.log(formatLogMessage('warn', `[HEALTH CHECK] Timeout, assuming restart needed`));
+       healthCheck = { shouldRestart: true, reason: 'Health check timeout' };
+     }
     } else if (forceDebug && urlsSinceLastCleanup > 10) {
       console.log(formatLogMessage('debug', `Skipping health check: failure rate ${Math.round(recentFailureRate * 100)}%, critical errors: ${hasCriticalErrors ? 'yes' : 'no'}`));
     }
@@ -3612,11 +3654,13 @@ function setupFrameHandling(page, forceDebug) {
       !healthCheck.reason?.includes('Scheduled cleanup') && 
       (healthCheck.reason?.includes('Critical') || healthCheck.reason?.includes('disconnected'));
     
-    // Restart browser if we've processed enough URLs, health check suggests it, and this isn't the last site
-    if ((wouldExceedLimit || shouldRestartFromHealth || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8 && isNotLastBatch) {
-      
+    // Restart browser if we've processed enough URLs, health check suggests it, hang detected, and this isn't the last site
+    if ((wouldExceedLimit || shouldRestartFromHealth || forceRestartFlag || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8 && isNotLastBatch) {      
       let restartReason = 'Unknown';
-      if (shouldRestartFromHealth) {
+      if (forceRestartFlag) {
+        restartReason = 'Emergency restart due to 2.5-minute hang detection';
+        forceRestartFlag = false; // Reset the flag
+      } else if (shouldRestartFromHealth) {
         restartReason = healthCheck.reason;
       } else if (hasHighFailureRate) {
         restartReason = `High failure rate: ${Math.round(recentFailureRate * 100)}% in recent batch`;
@@ -3685,9 +3729,34 @@ function setupFrameHandling(page, forceDebug) {
       console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
-    // Create tasks with current browser instance and process them with TRUE concurrency
-    const batchTasks = currentBatch.map(task => originalLimit(() => processUrl(task.url, task.config, browser)));
-    const batchResults = await Promise.all(batchTasks);
+ // Create tasks with timeout protection
+ const batchTasks = currentBatch.map(task => originalLimit(() => processUrl(task.url, task.config, browser)));
+ 
+ let batchResults;
+ try {
+   batchResults = await Promise.race([
+     Promise.all(batchTasks),
+     new Promise((_, reject) => 
+       setTimeout(() => reject(new Error('Batch timeout')), 600000) // 10 min timeout
+     )
+   ]);
+ } catch (timeoutError) {
+   if (timeoutError.message.includes('timeout')) {
+     console.log(formatLogMessage('error', `[TIMEOUT] Batch hung. Restarting browser.`));
+     try {
+       await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false });
+       browser = await createBrowser();
+       urlsSinceLastCleanup = 0;
+     } catch (restartErr) {
+       throw restartErr;
+     }
+     batchResults = currentBatch.map(task => ({
+       success: false, error: 'Batch timeout', needsImmediateRestart: true, url: task.url
+     }));
+   } else {
+     throw timeoutError;
+   }
+ }
 
     // IMPROVED: Much more conservative emergency restart logic
     const criticalRestartCount = batchResults.filter(r => r.needsImmediateRestart).length;
@@ -3801,6 +3870,26 @@ function setupFrameHandling(page, forceDebug) {
         if (forceDebug) console.log(formatLogMessage('debug', `Emergency restart failed: ${emergencyRestartErr.message}`));
       }
     }
+    // Handle hang detection flag if it's still set (e.g., on last batch where normal restart wouldn't trigger)
+    if (forceRestartFlag && batchEnd < totalUrls) {
+      console.log(`\n${messageColors.fileOp('🔄 Emergency hang detection restart:')} Browser appears hung, forcing restart`);
+      try {
+        await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false, cleanTempFiles: true });
+        browser = await createBrowser();
+        urlsSinceLastCleanup = 0;
+        forceRestartFlag = false; // Reset flag
+        await fastTimeout(TIMEOUTS.EMERGENCY_RESTART_DELAY);
+        if (forceDebug) console.log(formatLogMessage('debug', `Emergency hang detection restart completed`));
+      } catch (hangRestartErr) {
+        if (forceDebug) console.log(formatLogMessage('debug', `Hang detection restart failed: ${hangRestartErr.message}`));
+        // Continue anyway - better to try processing remaining URLs than exit
+      }
+    }
+  }
+ } catch (processingError) {
+   console.log(formatLogMessage('error', `Critical error: ${processingError.message}`));
+   clearInterval(hangDetectionInterval);
+   throw processingError;
   }
 
   // Clear hang detection interval

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.28",
+  "version": "2.0.30",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {