@fanboynz/network-scanner 2.0.28 → 2.0.29

package/.clauderc

@@ -0,0 +1,30 @@
+{
+  "description": "Network scanner that monitors website requests and generates blocking rules. Uses Puppeteer to load sites, intercepts network traffic, matches patterns, and outputs rules in various formats (adblock, dnsmasq, hosts file, etc.).",
+  
+  "conventions": [
+    "Store modular functionality in ./lib/ directory with focused, single-purpose modules",
+    "Use messageColors and formatLogMessage from ./lib/colorize for consistent console output",
+    "Implement timeout protection for all Puppeteer operations using Promise.race patterns",
+    "Handle browser lifecycle with comprehensive cleanup in try-finally blocks",
+    "Validate all external tool availability before use (grep, curl, whois, dig)",
+    "Use forceDebug flag for detailed logging, silentMode for minimal output"
+  ],
+  
+  "files": {
+    "important": [
+      "nwss.js",
+      "config.json",
+      "lib/*.js",
+      "*.md",
+      "nwss.1"
+    ],
+    "ignore": [
+      "node_modules/**",
+      "logs/**",
+      "sources/**",
+      ".cache/**",
+      "*.log",
+      "*.gz"
+    ]
+  }
+}

package/lib/fingerprint.js

@@ -96,9 +96,9 @@ const USER_AGENT_COLLECTIONS = {
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
   ],
   firefox: [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.0) Gecko/20100101 Firefox/143.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",
-    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:143.0) Gecko/20100101 Firefox/143.0"
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0"
   ],
   safari: [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15",
@@ -528,16 +528,45 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         // Simulate Chrome runtime
         //
         safeExecute(() => {
-          if (!window.chrome?.runtime) {
+          if (!window.chrome) {
             window.chrome = {
               runtime: {
                 onConnect: { addListener: () => {}, removeListener: () => {} },
                 onMessage: { addListener: () => {}, removeListener: () => {} },
                 sendMessage: () => {},
-                connect: () => ({ onMessage: { addListener: () => {}, removeListener: () => {} }, postMessage: () => {}, disconnect: () => {} }),
-                getManifest: () => ({ name: "Chrome", version: "141.0.0.0" }),
+                connect: () => ({ 
+                  onMessage: { addListener: () => {}, removeListener: () => {} }, 
+                  postMessage: () => {}, 
+                  disconnect: () => {} 
+                }),
+                getManifest: () => ({ 
+                  name: "Chrome", 
+                  version: "141.0.0.0",
+                  manifest_version: 3,
+                  description: "Chrome Browser"
+                }),
                 getURL: (path) => `chrome-extension://invalid/${path}`,
-                id: undefined
+                id: undefined,
+                getPlatformInfo: (callback) => callback({
+                  os: navigator.platform.includes('Win') ? 'win' : 
+                      navigator.platform.includes('Mac') ? 'mac' : 'linux',
+                  arch: 'x86-64',
+                  nacl_arch: 'x86-64'
+                })
+              },
+              storage: {
+                local: {
+                  get: (keys, callback) => callback && callback({}),
+                  set: (items, callback) => callback && callback(),
+                  remove: (keys, callback) => callback && callback(),
+                  clear: (callback) => callback && callback()
+                },
+                sync: {
+                  get: (keys, callback) => callback && callback({}),
+                  set: (items, callback) => callback && callback(),
+                  remove: (keys, callback) => callback && callback(),
+                  clear: (callback) => callback && callback()
+                }
               },
               loadTimes: () => {
                 const now = performance.now();
@@ -555,9 +584,56 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
                 startE: Date.now() - Math.random() * 2000
               })
             };
+            
+            // Make chrome object non-enumerable to match real Chrome
+            Object.defineProperty(window, 'chrome', {
+              value: window.chrome,
+              writable: false,
+              enumerable: false,
+              configurable: true
+            });
+            
+            // Add Chrome-specific globals that Cloudflare might check
+            if (!window.external) {
+              window.external = {
+                AddSearchProvider: () => {},
+                IsSearchProviderInstalled: () => 0
+              };
+            }
+            
+            // Ensure chrome.runtime appears as a native object
+            Object.defineProperty(window.chrome.runtime, 'toString', {
+              value: () => '[object Object]'
+            });
           }
         }, 'Chrome runtime simulation');
 
+        // Add realistic Chrome browser behavior
+        //
+        safeExecute(() => {
+          // Remove Puppeteer-specific properties not covered in main automation cleanup
+          delete window.__puppeteer_evaluation_script__;
+          delete window.__runtime;
+          delete window._asyncToGenerator;
+          delete window.__puppeteer;
+          delete window.__cdp;
+          delete window.__REACT_DEVTOOLS_GLOBAL_HOOK__;
+          
+          // Simulate Chrome's performance observer
+          if (window.PerformanceObserver) {
+            try {
+              const observer = new PerformanceObserver(() => {});
+              observer.observe({entryTypes: ['navigation']});
+            } catch(e) {
+              // Silently ignore if PerformanceObserver fails
+            }
+          }
+          
+          // Ensure external object exists (don't overwrite chrome object)
+          window.external = window.external || {};
+          
+        }, 'realistic Chrome behavior');
+
         // Spoof plugins based on user agent
         //
         safeExecute(() => {
@@ -874,29 +950,33 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
           if (navigator.permissions?.query) {
             const originalQuery = navigator.permissions.query;
             navigator.permissions.query = function(descriptor) {
-              // More realistic permission states based on permission type
+
               const permissionName = descriptor.name || descriptor;
-              let state = 'prompt'; // Default state
+              // Realistic Chrome permission defaults
+              const chromeDefaults = {
+                'notifications': 'default',
+                'geolocation': 'prompt', 
+                'camera': 'prompt',
+                'microphone': 'prompt',
+                'persistent-storage': 'granted',
+                'background-sync': 'granted',
+                'midi': 'prompt',
+                'push': 'prompt',
+                'accelerometer': 'granted',
+                'gyroscope': 'granted',
+                'magnetometer': 'granted'
+              };
               
-              // Common permissions that are usually granted
-              if (['notifications', 'persistent-storage'].includes(permissionName)) {
-                state = Math.random() > 0.3 ? 'granted' : 'prompt';
-              }
-              // Privacy-sensitive permissions that are usually denied/prompt
-              else if (['camera', 'microphone', 'geolocation'].includes(permissionName)) {
-                state = Math.random() > 0.8 ? 'granted' : 'prompt';
-              }
-              // Other permissions random
-              else {
-                state = Math.random() > 0.5 ? 'granted' : 'prompt';
-              }
+              const state = chromeDefaults[permissionName] || 'prompt';
               
               return Promise.resolve({ 
-                state: state,
+                state,
                 onchange: null
               });
-            };
+              }
+
           }
+
           // Block permission prompts from actually appearing
           if (window.Notification && Notification.requestPermission) {
             Notification.requestPermission = () => Promise.resolve('default');
@@ -1637,4 +1717,4 @@ module.exports = {
   applyTimezoneSpoofing,
   DEFAULT_PLATFORM,
   DEFAULT_TIMEZONE
-};
+};

package/lib/nettools.js

@@ -779,6 +779,8 @@ function createNetToolsHandler(config) {
     isDomainAlreadyDetected,
     getRootDomain,
     siteConfig,
+    processedWhoisDomains = new Set(), // Accept global sets, fallback to new for backward compatibility
+    processedDigDomains = new Set(),
     dumpUrls,
     matchedUrlsLogFile,
     forceDebug,
@@ -790,9 +792,22 @@ function createNetToolsHandler(config) {
   const hasDig = digTerms && Array.isArray(digTerms) && digTerms.length > 0;
   const hasDigOr = digOrTerms && Array.isArray(digOrTerms) && digOrTerms.length > 0;
   
-  // Add separate deduplication caches for different lookup types
-  const processedWhoisDomains = new Set();
-  const processedDigDomains = new Set();
+  // Create config-aware cache keys for deduplication
+  // Whois: Only include search terms + server (domain registry data is consistent across subdomains)
+  const whoisConfigKey = JSON.stringify({
+    terms: whoisTerms || [],
+    orTerms: whoisOrTerms || [],
+    server: whoisServer || 'default',
+    serverMode: whoisServerMode || 'random'
+  });
+  // Dig: Include all config (DNS records can vary by specific subdomain)
+  const digConfigKey = JSON.stringify({
+    terms: digTerms || [],
+    orTerms: digOrTerms || [],
+    recordType: digRecordType,
+    subdomain: digSubdomain
+  });
+
   // Add whois resolution caching to avoid redundant whois lookups
   const whoisResultCache = new Map();
   const WHOIS_CACHE_TTL = 900000; // 15 minutes cache TTL (whois data changes less frequently)
@@ -853,15 +868,24 @@ function createNetToolsHandler(config) {
     
     // Determine which domain will be used for dig lookup
     const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
+
+    // For whois: use root domain only (whois data is consistent for entire domain)
+    const whoisRootDomain = getRootDomain ? getRootDomain(`http://${domain}`) : domain;
     
-    // Check if we need to perform any lookups
-    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(domain);
-    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDomain);
+    // Check if we need to perform any lookups with appropriate deduplication
+    // Whois: root domain + config (whois data same for sub.example.com and example.com)
+    const whoisDedupeKey = `${whoisRootDomain}:${whoisConfigKey}`;
+    // Dig: specific subdomain + config (DNS records can differ between subdomains)
+    const digDedupeKey = `${digDomain}:${digConfigKey}`;
+    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(whoisDedupeKey);
+    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDedupeKey);
     
     // Skip if we don't need to perform any lookups
     if (!needsWhoisLookup && !needsDigLookup) {
       if (forceDebug) {
-        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${!needsWhoisLookup}, dig: ${!needsDigLookup})`);
+        const whoisSkipped = (hasWhois || hasWhoisOr) ? `cached(${whoisRootDomain})` : 'n/a';
+        const digSkipped = (hasDig || hasDigOr) ? `cached(${digDomain})` : 'n/a';
+        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${whoisSkipped}, dig: ${digSkipped})`);
       }
       return;
     }
@@ -869,7 +893,7 @@ function createNetToolsHandler(config) {
 
     if (forceDebug) {
       const totalProcessed = processedWhoisDomains.size + processedDigDomains.size;
-      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup}, dig: ${needsDigLookup}) (${totalProcessed} total processed)`);
+      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup ? whoisRootDomain : 'skip'}, dig: ${needsDigLookup ? digDomain : 'skip'}) (${totalProcessed} total processed)`);
     }
 
       // Log site-specific whois delay if different from default
@@ -955,12 +979,12 @@ function createNetToolsHandler(config) {
       
       // Perform whois lookup if either whois or whois-or is configured
       if (needsWhoisLookup) {
-        // Mark whois domain as being processed
-        processedWhoisDomains.add(domain);
+        // Mark whois root domain+config as being processed
+        processedWhoisDomains.add(whoisDedupeKey);
         
         // Check whois cache first - cache key includes server for accuracy
         const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
-        const whoisCacheKey = `${domain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
+        const whoisCacheKey = `${whoisRootDomain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
         const now = Date.now();
         let whoisResult = null;
         
@@ -970,7 +994,7 @@ function createNetToolsHandler(config) {
             if (forceDebug) {
               const age = Math.round((now - cachedEntry.timestamp) / 1000);
               const serverInfo = (selectedServer && selectedServer !== '') ? ` (server: ${selectedServer})` : ' (default server)';
-              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${whoisRootDomain}${serverInfo} [age: ${age}s]`);
             }
             // V8 Optimized: Object.assign is faster than spread for object merging
             whoisResult = Object.assign({}, cachedEntry.result, {
@@ -982,7 +1006,7 @@ function createNetToolsHandler(config) {
             // Cache expired, remove it
             whoisResultCache.delete(whoisCacheKey);
             if (forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${domain}, performing fresh lookup`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${whoisRootDomain}, performing fresh lookup`);
             }
           }
         }
@@ -991,7 +1015,7 @@ function createNetToolsHandler(config) {
         if (!whoisResult) {
           if (forceDebug) {
             const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
-            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
+            logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
           }
         
           // Configure retry options based on site config or use defaults
@@ -1004,7 +1028,7 @@ function createNetToolsHandler(config) {
           };
           
           try {
-            whoisResult = await whoisLookupWithRetry(domain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
+            whoisResult = await whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
             
             // Cache successful results (and certain types of failures)
             if (whoisResult.success || 
@@ -1020,13 +1044,13 @@ function createNetToolsHandler(config) {
               if (forceDebug) {
                 const cacheType = whoisResult.success ? 'successful' : 'failed';
                 const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
-                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${domain}${serverInfo}`);
+                logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cached ${cacheType} result for ${whoisRootDomain}${serverInfo}`);
               }
             }
           } catch (whoisError) {
             // Handle exceptions from whois lookup
             if (forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${domain}: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception during lookup for ${whoisRootDomain}: ${whoisError.message}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Exception type: ${whoisError.constructor.name}`);
               if (whoisError.stack) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Stack trace: ${whoisError.stack.split('\n').slice(0, 3).join(' -> ')}`);
@@ -1035,7 +1059,7 @@ function createNetToolsHandler(config) {
             
             // Log whois exceptions in dry run mode
             if (dryRunCallback && forceDebug) {
-              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception: ${whoisError.message}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Exception for ${whoisRootDomain}: ${whoisError.message}`);
             }
             // Continue with dig if configured
             whoisResult = null; // Ensure we don't process a null result
@@ -1086,7 +1110,7 @@ function createNetToolsHandler(config) {
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
               const cacheInfo = whoisResult.fromCache ? ` [CACHED - ${Math.round(whoisResult.cacheAge / 1000)}s old]` : '';
               const duration = whoisResult.fromCache ? `cached in 0ms` : `in ${whoisResult.duration}ms`;
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${domain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Lookup completed for ${whoisRootDomain}${serverUsed} ${duration}${retryInfo}${cacheInfo}`);            
               
               if (whoisResult.retryInfo && whoisResult.retryInfo.retriedAfterFailure) {
                 logToConsoleAndFile(`${messageColors.highlight('[whois]')} Success after retry - servers attempted: [${whoisResult.retryInfo.serversAttempted.map(s => s || 'default').join(', ')}]`);
@@ -1099,7 +1123,7 @@ function createNetToolsHandler(config) {
               const errorContext = whoisResult.isTimeout ? 'TIMEOUT' : 'ERROR';
               const retryInfo = whoisResult.retryInfo ? ` [${whoisResult.retryInfo.totalAttempts}/${whoisResult.retryInfo.maxAttempts} attempts]` : '';
               
-              logToConsoleAndFile(`${messageColors.highlight('[whois]')} ${errorContext}: Lookup failed for ${domain}${serverUsed} after ${whoisResult.duration}ms${retryInfo}`);
+              logToConsoleAndFile(`${messageColors.highlight('[whois]')} ${errorContext}: Lookup failed for ${whoisRootDomain}${serverUsed} after ${whoisResult.duration}ms${retryInfo}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Command executed: ${whoisResult.command || 'unknown'}`);
               logToConsoleAndFile(`${messageColors.highlight('[whois]')} Error details: ${whoisResult.error}`);
               
@@ -1182,8 +1206,8 @@ function createNetToolsHandler(config) {
       
       // Perform dig lookup if configured
       if (needsDigLookup) {
-        // Mark dig domain as being processed
-        processedDigDomains.add(digDomain);
+        // Mark dig domain+config as being processed (includes specific subdomain)
+        processedDigDomains.add(digDedupeKey);
         
         if (forceDebug) {
           const digTypes = [];

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.27 ===
+// === Network scanner script (nwss.js) v2.0.28 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -92,9 +92,9 @@ const USER_AGENTS = Object.freeze(new Map([
   ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
   ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
   ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0"],
   ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
 ]));
 
@@ -143,7 +143,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.27'; // Script version
+const VERSION = '2.0.28'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -747,6 +747,12 @@ function safeMarkDomainProcessed(domain, context, metadata) {
   }
 }
 
+// Global deduplication for nettools to avoid redundant lookups across all URLs
+// Whois: keyed by root domain only (whois data is consistent for entire domain)
+// Dig: keyed by specific subdomain+config (DNS records can vary by subdomain)
+const globalProcessedWhoisDomains = new Set();
+const globalProcessedDigDomains = new Set();
+
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
   const filesToClean = cleanRulesFile ? [cleanRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -1245,6 +1251,16 @@ function setupFrameHandling(page, forceDebug) {
       // Auto-detect best headless mode based on Puppeteer version
       headless: headlessMode,
       args: [
+        // CRITICAL: Remove automation detection markers
+        '--disable-blink-features=AutomationControlled',
+        '--no-first-run',
+        '--disable-default-apps',
+        '--disable-component-extensions-with-background-pages',
+        // HIGH IMPACT: Normal Chrome behavior simulation
+        '--password-store=basic',
+        '--use-mock-keychain',
+        '--disable-client-side-phishing-detection',
+        '--enable-features=NetworkService',
         // Disk space controls - 50MB cache limits
         '--disable-features=VizDisplayCompositor',
         `--disk-cache-size=${CACHE_LIMITS.DISK_CACHE_SIZE}`, // 50MB disk cache
@@ -1260,10 +1276,7 @@ function setupFrameHandling(page, forceDebug) {
         '--aggressive-cache-discard',
         '--memory-pressure-off',
         '--max_old_space_size=2048',
-        '--no-first-run',
         '--disable-prompt-on-repost',  // Fixes form popup on page reload
-        '--disable-default-apps',
-        '--disable-component-extensions-with-background-pages',
         '--disable-background-networking',
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -1290,14 +1303,13 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-backgrounding-occluded-windows',
         '--disable-background-timer-throttling',
         '--disable-features=site-per-process', // Better for single-site scanning
-        '--disable-blink-features=AutomationControlled', // Avoid detection
         '--no-zygote', // Better process isolation
         ],
         // Optimized timeouts for Puppeteer 23.x performance
         protocolTimeout: TIMEOUTS.PROTOCOL_TIMEOUT,
         slowMo: 0, // No artificial delays
         defaultViewport: null, // Use system default viewport
-        ignoreDefaultArgs: ['--enable-automation'] // Avoid automation detection
+        ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=AutomationControlled'] // Avoid automation detection
     });
     
     // Store the user data directory on the browser object for cleanup
@@ -2485,6 +2497,8 @@ function setupFrameHandling(page, forceDebug) {
               const netToolsHandler = createNetToolsHandler({
                 whoisTerms,
                 whoisOrTerms,
+               processedWhoisDomains: globalProcessedWhoisDomains,
+               processedDigDomains: globalProcessedDigDomains,
                 whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
                 whoisServer,
                 whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
@@ -2591,6 +2605,8 @@ function setupFrameHandling(page, forceDebug) {
              const netToolsHandler = createNetToolsHandler({
                whoisTerms,
                whoisOrTerms,
+               processedWhoisDomains: globalProcessedWhoisDomains,
+               processedDigDomains: globalProcessedDigDomains,
                whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay, // Site-specific or global fallback
 	       whoisServer, // Pass whois server configuration
                whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.28",
+  "version": "2.0.29",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {