@fanboynz/network-scanner 2.0.59 → 2.0.61

package/nwss.js

@@ -2,7 +2,8 @@
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
-const usePuppeteerCore = process.argv.includes('--use-puppeteer-core');
+const useObscura = process.argv.includes('--use-obscura');
+const usePuppeteerCore = process.argv.includes('--use-puppeteer-core') || useObscura;
 const puppeteer = usePuppeteerCore ? require('puppeteer-core') : require('puppeteer');
 const fs = require('fs');
 const os = require('os');
@@ -32,7 +33,7 @@ const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore
 // Graceful exit
 const { handleBrowserExit, cleanupChromeTempFiles } = require('./lib/browserexit');
 // Whois & Dig
-const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability } = require('./lib/nettools');
+const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats } = require('./lib/nettools');
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // CDP functionality
@@ -104,12 +105,12 @@ const CONCURRENCY_LIMITS = Object.freeze({
 
 // V8 Optimization: Use Map for user agent lookups instead of object
 const USER_AGENTS = Object.freeze(new Map([
-  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0"],
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0"],
   ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
 ]));
 
@@ -177,6 +178,82 @@ if (args.length === 0) {
   args.push('--help');
 }
 
+// --- .nwssconfig support: inject per-config settings into args ---
+const NWSSCONFIG_PATH = path.join(__dirname, '.nwssconfig');
+if (fs.existsSync(NWSSCONFIG_PATH)) {
+  try {
+    const nwssConfig = JSON.parse(fs.readFileSync(NWSSCONFIG_PATH, 'utf-8'));
+    // Find which config file is being used (--custom-json <file> or positional .json arg)
+    const customJsonIdx = args.findIndex(arg => arg === '--custom-json');
+    const configFilename = (customJsonIdx !== -1 && args[customJsonIdx + 1])
+      ? args[customJsonIdx + 1]
+      : args.find(a => a.endsWith('.json') && !a.startsWith('--'));
+
+    if (configFilename && nwssConfig.configs && nwssConfig.configs[configFilename]) {
+      const settings = nwssConfig.configs[configFilename];
+      const originalArgs = args.join(' ');
+
+      // Map settings keys to CLI flags — only inject if not already in args
+      const settingsMap = {
+        output: ['-o', '--output'],
+        max_concurrent: ['--max-concurrent'],
+        dns_cache: ['--dns-cache'],
+        cache_requests: ['--cache-requests'],
+        dumpurls: ['--dumpurls'],
+        remove_tempfiles: ['--remove-tempfiles'],
+        color: ['--color'],
+        remove_dupes: ['--remove-dupes', '--remove-dubes'],
+        'remove-dupes': ['--remove-dupes', '--remove-dubes'],
+        'remove-dubes': ['--remove-dupes', '--remove-dubes'],
+        compress_logs: ['--compress-logs'],
+        debug: ['--debug'],
+        silent: ['--silent'],
+        verbose: ['--verbose'],
+        headful: ['--headful'],
+        keep_open: ['--keep-open'],
+        dry_run: ['--dry-run'],
+        titles: ['--titles'],
+        sub_domains: ['--sub-domains'],
+        no_interact: ['--no-interact'],
+        ghost_cursor: ['--ghost-cursor'],
+        plain: ['--plain'],
+        cdp: ['--cdp'],
+        dnsmasq: ['--dnsmasq'],
+        unbound: ['--unbound'],
+        privoxy: ['--privoxy'],
+        pihole: ['--pihole'],
+        eval_on_doc: ['--eval-on-doc'],
+        use_puppeteer_core: ['--use-puppeteer-core'],
+        ignore_cache: ['--ignore-cache'],
+        clear_cache: ['--clear-cache'],
+        block_ads: ['--block-ads'],
+        compare: ['--compare'],
+        localhost: ['--localhost'],
+        append: ['--append']
+      };
+
+      for (const [key, flags] of Object.entries(settingsMap)) {
+        // Support both underscore and hyphen variants (e.g. dns_cache or dns-cache)
+        const value = settings[key] !== undefined ? settings[key]
+          : settings[key.replace(/_/g, '-')] !== undefined ? settings[key.replace(/_/g, '-')]
+          : settings[key.replace(/-/g, '_')] !== undefined ? settings[key.replace(/-/g, '_')]
+          : undefined;
+        if (value === undefined) continue;
+        // Skip if any variant of the flag is already in CLI args
+        if (flags.some(f => originalArgs.includes(f))) continue;
+
+        if (typeof value === 'boolean') {
+          if (value) args.push(flags[flags.length - 1]);
+        } else if (typeof value === 'string' || typeof value === 'number') {
+          args.push(flags[flags.length - 1], String(value));
+        }
+      }
+    }
+  } catch (e) {
+    console.error(`Warning: Failed to parse .nwssconfig: ${e.message}`);
+  }
+}
+
 const headfulMode = args.includes('--headful');
 const SOURCES_FOLDER = 'sources';
 
@@ -208,6 +285,12 @@ if (localhostIndex !== -1) {
   localhostIP = args[localhostIndex].includes('=') ? args[localhostIndex].split('=')[1] : '127.0.0.1';
 }
 const keepBrowserOpen = args.includes('--keep-open');
+const loadExtensionPaths = [];
+args.forEach((arg, idx) => {
+  if (arg === '--load-extension' && args[idx + 1] && !args[idx + 1].startsWith('--')) {
+    loadExtensionPaths.push(path.resolve(args[idx + 1]));
+  }
+});
 const disableInteract = args.includes('--no-interact');
 const globalGhostCursor = args.includes('--ghost-cursor');
 const plainOutput = args.includes('--plain');
@@ -229,6 +312,8 @@ let cleanRules = args.includes('--clean-rules');
 const clearCache = args.includes('--clear-cache');
 const ignoreCache = args.includes('--ignore-cache');
 const cacheRequests = args.includes('--cache-requests');
+const dnsCacheMode = args.includes('--dns-cache');
+if (dnsCacheMode) enableDiskCache();
 
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
@@ -499,22 +584,38 @@ if (validateRules || validateRulesFile) {
   }
 }
 
-// Parse --block-ads argument for request-level ad blocking
+// Parse --block-ads argument for request-level ad blocking (supports comma-separated lists)
 const blockAdsIndex = args.findIndex(arg => arg.startsWith('--block-ads'));
 if (blockAdsIndex !== -1) {
-  const rulesFile = args[blockAdsIndex].includes('=') 
-    ? args[blockAdsIndex].split('=')[1] 
+  const rulesArg = args[blockAdsIndex].includes('=')
+    ? args[blockAdsIndex].split('=')[1]
     : args[blockAdsIndex + 1];
-  
-  if (!rulesFile || !fs.existsSync(rulesFile)) {
-    console.log(`Error: Adblock rules file not found: ${rulesFile || '(not specified)'}`);
+
+  if (!rulesArg) {
+    console.log('Error: No adblock rules file specified');
     process.exit(1);
   }
-  
+
+  const rulesFiles = rulesArg.split(',').map(f => f.trim()).filter(f => f);
+  for (const file of rulesFiles) {
+    if (!fs.existsSync(file)) {
+      console.log(`Error: Adblock rules file not found: ${file}`);
+      process.exit(1);
+    }
+  }
+
+  // Concatenate multiple lists into a single temp file for the parser
+  let rulesFile = rulesFiles[0];
+  if (rulesFiles.length > 1) {
+    rulesFile = path.join(os.tmpdir(), `nwss-adblock-combined-${Date.now()}.txt`);
+    const combined = rulesFiles.map(f => fs.readFileSync(f, 'utf-8')).join('\n');
+    fs.writeFileSync(rulesFile, combined);
+  }
+
   adblockEnabled = true;
   adblockMatcher = parseAdblockRules(rulesFile, { enableLogging: forceDebug });
   const stats = adblockMatcher.getStats();
-  if (!silentMode) console.log(messageColors.success(`Adblock enabled: Loaded ${stats.total} blocking rules from ${rulesFile}`));
+  if (!silentMode) console.log(messageColors.success(`Adblock enabled: Loaded ${stats.total} blocking rules from ${rulesFiles.length} list${rulesFiles.length > 1 ? 's' : ''}`));
 }
 
 if (args.includes('--help') || args.includes('-h')) {
@@ -541,6 +642,12 @@ Request Blocking:
   --block-ads=<file>             Block ads/trackers using EasyList format rules (||domain.com^, /ads/*, etc)
                                  Works at request-level for maximum performance
 
+Per-config settings file (.nwssconfig):
+  Place a .nwssconfig file in the project root to define per-config settings.
+  When a config filename matches a key in .nwssconfig, those settings are used.
+  CLI flags merge with and override .nwssconfig settings.
+  See README.md for format details.
+
 General Options:
   --verbose                      Force verbose mode globally
   --debug                        Force debug mode globally
@@ -556,6 +663,9 @@ General Options:
   --headful                      Launch browser with GUI (not headless)
   --keep-open                    Keep browser open after scan completes (use with --headful)
   --use-puppeteer-core           Use puppeteer-core with system Chrome instead of bundled Chromium
+  --use-obscura                  Connect to running Obscura CDP server (ws://127.0.0.1:9222 or OBSCURA_WS env)
+                                 Skips fingerprint injection — Obscura provides built-in stealth
+  --load-extension <path>        Load unpacked Chrome extension from directory
   --cdp                          Enable Chrome DevTools Protocol logging (now per-page if enabled)
   --remove-dupes                 Remove duplicate domains from output (only with -o)
   --eval-on-doc                 Globally enable evaluateOnNewDocument() for Fetch/XHR interception
@@ -567,6 +677,7 @@ General Options:
 
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
+  --dns-cache                    Persist dig/whois results to disk between runs (3hr/4hr TTL)
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -583,6 +694,7 @@ Global config.json options:
   ignore_similar_ignored_domains: true/false      Ignore domains similar to ignoreDomains list (default: true)
   max_concurrent_sites: 8                        Maximum concurrent site processing (1-50, default: 8)
   resource_cleanup_interval: 80                  Browser restart interval in URLs processed (1-1000, default: 80)
+  disable_ad_tagging: true/false                 Disable Chrome AdTagging to prevent ad frame throttling (default: true)
 
 Per-site config.json options:
   url: "site" or ["site1", "site2"]          Single URL or list of URLs
@@ -748,7 +860,8 @@ const {
   whois_server_mode = 'random', 
   ignore_similar = true, 
   ignore_similar_threshold = 80, 
-  ignore_similar_ignored_domains = true, 
+  ignore_similar_ignored_domains = true,
+  disable_ad_tagging = true,
   max_concurrent_sites = 6,
   resource_cleanup_interval = 80,
   comments: globalComments, 
@@ -760,6 +873,23 @@ const globalBlockedRegexes = Array.isArray(globalBlocked)
   ? globalBlocked.map(pattern => new RegExp(pattern))
   : [];
 
+// Cache compiled regexes by pattern string — avoids recompiling same patterns across URLs
+const _compiledRegexCache = new Map();
+function getCompiledRegex(pattern) {
+  let compiled = _compiledRegexCache.get(pattern);
+  if (!compiled) {
+    compiled = new RegExp(pattern.replace(/^\/(.*)\/$/, '$1'));
+    if (_compiledRegexCache.size > 2000) _compiledRegexCache.clear();
+    _compiledRegexCache.set(pattern, compiled);
+  }
+  return compiled;
+}
+function getCompiledRegexes(patterns) {
+  if (!patterns) return [];
+  const arr = Array.isArray(patterns) ? patterns : [patterns];
+  return arr.map(p => getCompiledRegex(p));
+}
+
 // Pre-split ignoreDomains into exact Set (O(1) lookup) and wildcard array
 const _ignoreDomainsExact = new Set();
 const _ignoreDomainsWildcard = [];
@@ -1088,12 +1218,19 @@ if (forceDebug && globalComments) {
  * @param {string} url - The URL string to parse.
  * @returns {string} The root domain, or the original hostname if parsing fails (e.g., for IP addresses or invalid URLs), or an empty string on error.
  */
+const _rootDomainCache = new Map();
 function getRootDomain(url) {
+  const cached = _rootDomainCache.get(url);
+  if (cached !== undefined) return cached;
   try {
     const { hostname } = new URL(url);
     const parsed = psl.parse(hostname);
-    return parsed.domain || hostname;
+    const result = parsed.domain || hostname;
+    if (_rootDomainCache.size > 5000) _rootDomainCache.clear();
+    _rootDomainCache.set(url, result);
+    return result;
   } catch {
+    _rootDomainCache.set(url, '');
     return '';
   }
 }
@@ -1388,6 +1525,23 @@ function setupFrameHandling(page, forceDebug) {
    * @returns {Promise<import('puppeteer').Browser>} Browser instance
    */
   async function createBrowser(extraArgs = []) {
+    // Obscura mode: connect to a running Obscura CDP server instead of launching Chrome
+    if (useObscura) {
+      const obscuraEndpoint = process.env.OBSCURA_WS || 'ws://127.0.0.1:9222/devtools/browser';
+      if (forceDebug) console.log(formatLogMessage('debug', `Connecting to Obscura at ${obscuraEndpoint}`));
+      try {
+        const browser = await puppeteer.connect({ browserWSEndpoint: obscuraEndpoint });
+        if (!silentMode) console.log(messageColors.success(`Connected to Obscura CDP at ${obscuraEndpoint}`));
+        browser._nwssUserDataDir = null; // No temp dir to clean
+        browser._nwssIsObscura = true;
+        return browser;
+      } catch (err) {
+        console.error(formatLogMessage('error', `Failed to connect to Obscura: ${err.message}`));
+        console.error(formatLogMessage('error', `Start Obscura first: obscura serve --port 9222 --stealth`));
+        process.exit(1);
+      }
+    }
+
     // Create temporary user data directory that we can fully control and clean up
     const tempUserDataDir = path.join(os.tmpdir(), `puppeteer-${Date.now()}-${Math.random().toString(36).substring(7)}`);
     userDataDir = tempUserDataDir; // Store for cleanup tracking (use outer scope variable)
@@ -1460,7 +1614,7 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-blink-features=AutomationControlled',
         '--no-first-run',
         '--disable-default-apps',
-        '--disable-component-extensions-with-background-pages',
+        ...(keepBrowserOpen ? [] : ['--disable-component-extensions-with-background-pages']),
         // HIGH IMPACT: Normal Chrome behavior simulation
         '--password-store=basic',
         '--use-mock-keychain',
@@ -1474,30 +1628,29 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-background-downloads',
         // DISK I/O REDUCTION: Eliminate unnecessary Chrome disk writes
         '--disable-breakpad',          // No crash dump files
-        '--disable-component-update',  // No component update downloads
+        ...(keepBrowserOpen ? [] : ['--disable-component-update']),  // No component update downloads
         '--disable-logging',           // No Chrome internal log files
         '--log-level=3',               // Fatal errors only (suppresses verbose disk logging)
         '--no-service-autorun',        // No background service disk activity
         '--disable-domain-reliability', // No reliability monitor disk writes
-        // PERFORMANCE: Enhanced Puppeteer 23.x optimizations
-        '--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor',
-        '--disable-features=TranslateUI,BlinkGenPropertyTrees,Translate',
-        '--disable-features=BackForwardCache,AcceptCHFrame',
+        // PERFORMANCE: Disable non-essential Chrome features in a single flag
+        // IMPORTANT: Chrome only reads the LAST --disable-features flag, so combine all into one
+        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding${disable_ad_tagging ? ',AdTagging' : ''}`,
         '--disable-ipc-flooding-protection',
         '--aggressive-cache-discard',
         '--memory-pressure-off',
         '--max_old_space_size=2048',   // V8 heap limit
         '--disable-prompt-on-repost',  // Fixes form popup on page reload
-        '--disable-background-networking',
+        ...(keepBrowserOpen ? [] : ['--disable-background-networking']),
         '--no-sandbox',
         '--disable-setuid-sandbox',
-        '--disable-features=SafeBrowsing',
         '--disable-dev-shm-usage',
-        '--disable-sync',
+        ...(keepBrowserOpen ? [] : ['--disable-sync']),
         '--mute-audio',
         '--disable-translate',
         '--window-size=1920,1080',
-        '--disable-extensions',
+        ...(keepBrowserOpen ? [] : ['--disable-extensions', '--disable-component-update']),
+        ...(loadExtensionPaths.length ? [`--load-extension=${loadExtensionPaths.join(',')}`, '--enable-extensions'] : []),
         '--no-default-browser-check',
         '--safebrowsing-disable-auto-update',
         '--ignore-ssl-errors',
@@ -1506,18 +1659,15 @@ function setupFrameHandling(page, forceDebug) {
         '--ignore-certificate-errors-ca-list',
         '--disable-web-security',
         '--allow-running-insecure-content',
-        '--disable-features=HttpsFirstBalancedModeAutoEnable',
         // Puppeteer 23.x: Enhanced performance and stability args
         '--disable-renderer-backgrounding',
         '--disable-backgrounding-occluded-windows',
         '--disable-background-timer-throttling',
-        '--disable-features=site-per-process', // Better for single-site scanning
         '--no-zygote', // Better process isolation
         // PERFORMANCE: Process and memory reduction for high concurrency
         '--renderer-process-limit=10',  // Cap renderer processes (default: unlimited)
         '--disable-accelerated-2d-canvas', // Software canvas only (we spoof it anyway)
         '--disable-hang-monitor',      // Remove per-renderer hang check overhead
-        '--disable-features=PaintHolding', // Don't hold frames in renderer memory
         '--js-flags=--max-old-space-size=512', // Cap V8 heap per renderer to 512MB
         ...extraArgs,
         ],
@@ -2207,11 +2357,16 @@ function setupFrameHandling(page, forceDebug) {
       }
 
       // --- Apply all fingerprint spoofing (user agent, Brave, fingerprint protection) ---
+      // Skip when using Obscura — it has built-in stealth that conflicts with our injection
       try {
-        await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+        if (!useObscura) {
+          await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+        } else if (forceDebug) {
+          console.log(formatLogMessage('debug', `Skipping fingerprint injection — Obscura provides built-in stealth`));
+        }
         
-        // Client Hints protection for Chrome user agents
-        if (siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
+        // Client Hints protection for Chrome user agents (skipped under Obscura — it sets its own)
+        if (!useObscura && siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
           const userAgentKey = siteConfig.userAgent.toLowerCase();
           let platform = 'Windows';
           let platformVersion = '15.0.0';
@@ -2228,14 +2383,14 @@ function setupFrameHandling(page, forceDebug) {
           }
                     
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Not:A-Brand";v="99", "Google Chrome";v="145", "Chromium";v="145"',
+            'Sec-CH-UA': '"Not:A-Brand";v="99", "Google Chrome";v="146", "Chromium";v="146"',
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"145.0.7632.160"',
-            'Sec-CH-UA-Full-Version-List': '"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="145.0.7632.160", "Chromium";v="145.0.7632.160"'
+            'Sec-CH-UA-Full-Version': '"146.0.0.0"',
+            'Sec-CH-UA-Full-Version-List': '"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="146.0.0.0", "Chromium";v="146.0.0.0"'
           });
         }
       } catch (fingerprintErr) {
@@ -2248,11 +2403,7 @@ function setupFrameHandling(page, forceDebug) {
         }
       }
 
-      const regexes = Array.isArray(siteConfig.filterRegex)
-        ? siteConfig.filterRegex.map(r => new RegExp(r.replace(/^\/(.*)\/$/, '$1')))
-        : siteConfig.filterRegex
-          ? [new RegExp(siteConfig.filterRegex.replace(/^\/(.*)\/$/, '$1'))]
-          : [];
+      const regexes = getCompiledRegexes(siteConfig.filterRegex);
 
       // NEW: Get regex_and setting (defaults to false for backward compatibility)
       const useRegexAnd = siteConfig.regex_and === true;
@@ -2400,7 +2551,7 @@ function setupFrameHandling(page, forceDebug) {
   }
 
       const blockedRegexes = Array.isArray(siteConfig.blocked)
-        ? siteConfig.blocked.map(pattern => new RegExp(pattern))
+        ? siteConfig.blocked.map(pattern => getCompiledRegex(pattern))
         : [];
 		
       // Combine site-specific with pre-compiled global blocked patterns
@@ -3177,7 +3328,23 @@ function setupFrameHandling(page, forceDebug) {
           ? { ...defaultGotoOptions, ...siteConfig.goto_options } : defaultGotoOptions;
 
         // Enhanced navigation with redirect handling - passes existing gotoOptions
-        const navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOptions, forceDebug, formatLogMessage);
+        let navigationResult;
+        try {
+          navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOptions, forceDebug, formatLogMessage);
+        } catch (navErr) {
+          // Only retry on genuine timeouts, not chrome-error:// redirects
+          let pageUrl = '';
+          try { if (!page.isClosed()) pageUrl = page.url(); } catch {}
+          const isPopupFailure = navErr.message.includes('chrome-error://') || navErr.message.includes('invalid URL') ||
+            pageUrl.startsWith('chrome-error://') || pageUrl === 'about:blank';
+          if ((navErr.message.includes('timeout') || navErr.message.includes('Timeout')) && !isPopupFailure) {
+            if (forceDebug) console.log(formatLogMessage('debug', `Navigation timeout, retrying with waitUntil:networkidle2 for ${currentUrl}`));
+            const fallbackOptions = { ...gotoOptions, waitUntil: 'networkidle2', timeout: Math.min(timeout, 10000) };
+            navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, fallbackOptions, forceDebug, formatLogMessage);
+          } else {
+            throw navErr;
+          }
+        }
         
         const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains } = navigationResult;
         
@@ -3233,7 +3400,8 @@ function setupFrameHandling(page, forceDebug) {
           }
           
           if (originalDomain !== finalDomain) {
-            if (!silentMode) {
+            const isPopupRedirect = !finalUrl || finalUrl === 'about:blank' || finalUrl.startsWith('chrome-error://');
+            if (!silentMode && !isPopupRedirect) {
               console.log(`🔄 Redirect detected: ${originalDomain} → ${finalDomain}`);
             }
             
@@ -3260,13 +3428,11 @@ function setupFrameHandling(page, forceDebug) {
                 }
               }
             } else {
-              // Invalid final URL - don't update currentUrl, treat as failed redirect
-              console.warn(`⚠ Redirect to invalid URL ignored: ${originalDomain} → ${finalUrl}`);
+              // Invalid final URL (ad popup redirect) - continue with original URL
               if (forceDebug) {
-                console.log(formatLogMessage('debug', `Redirect chain ended with invalid URL, keeping original: ${originalUrl}`));
+                console.log(formatLogMessage('debug', `Popup redirect ignored: ${originalDomain} → ${finalUrl}, keeping original: ${originalUrl}`));
               }
-              // Keep processing with the original URL or throw an error
-              throw new Error(`Redirect resulted in invalid URL: ${finalUrl}`);
+              // Continue with original URL — requests captured before the redirect are still valid
             }
           }
         }
@@ -3407,7 +3573,10 @@ function setupFrameHandling(page, forceDebug) {
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
         
         if (timeoutResult.success) {
-          console.log(`⚠ Partial redirect timeout recovered: ${safeGetDomain(currentUrl)} → ${safeGetDomain(timeoutResult.finalUrl)}`);
+          const isPopupRedirect = timeoutResult.finalUrl && (timeoutResult.finalUrl === 'about:blank' || timeoutResult.finalUrl.startsWith('chrome-error://'));
+          if (!isPopupRedirect) {
+            console.log(`⚠ Partial redirect timeout recovered: ${safeGetDomain(currentUrl)} → ${safeGetDomain(timeoutResult.finalUrl)}`);
+          }
           currentUrl = timeoutResult.finalUrl; // Use the partial redirect URL
           siteCounter++;
           // Continue processing with the redirected URL instead of throwing error
@@ -4036,6 +4205,10 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
 
+ // Track domain timeout counts — skip domain after 3 failures
+ const domainTimeoutCounts = new Map();
+ const DOMAIN_TIMEOUT_THRESHOLD = 3;
+
  // Enhanced hang detection with browser restart recovery
  let currentBatchInfo = { batchStart: 0, batchSize: 0 };
  let lastProcessedCount = 0;
@@ -4261,8 +4434,17 @@ function setupFrameHandling(page, forceDebug) {
       console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
- // Create tasks with timeout protection
- const batchTasks = currentBatch.map(task => originalLimit(() => processUrl(task.url, task.config, browser)));
+ // Create tasks with timeout protection — skip domains that repeatedly timed out
+ const batchTasks = currentBatch.map(task => originalLimit(() => {
+   try {
+     const taskDomain = new URL(task.url).hostname;
+     if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
+       if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
+       return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+     }
+   } catch {}
+   return processUrl(task.url, task.config, browser);
+ }));
  
  let batchResults;
  try {
@@ -4292,6 +4474,16 @@ function setupFrameHandling(page, forceDebug) {
    }
  }
 
+    // Track domain timeout counts — skip after threshold
+    for (const result of batchResults) {
+      if (!result.success && !result.skipped && result.error && result.error.includes('timeout')) {
+        try {
+          const domain = new URL(result.url).hostname;
+          domainTimeoutCounts.set(domain, (domainTimeoutCounts.get(domain) || 0) + 1);
+        } catch {}
+      }
+    }
+
     // IMPROVED: Much more conservative emergency restart logic
     const criticalRestartCount = batchResults.filter(r => r.needsImmediateRestart).length;
     // Require either:
@@ -4601,9 +4793,19 @@ function setupFrameHandling(page, forceDebug) {
   // Keep browser open if --keep-open flag is set (useful with --headful for inspection)
   if (keepBrowserOpen && !launchHeadless) {
     console.log(messageColors.info('Browser kept open.') + ' Close the browser window or press Ctrl+C to exit.');
+    const cleanup = async () => {
+      try {
+        if (browser.isConnected()) await browser.close();
+      } catch {}
+      process.exit(0);
+    };
+    process.on('SIGINT', cleanup);
+    process.on('SIGTERM', cleanup);
     await new Promise((resolve) => {
       browser.on('disconnected', resolve);
     });
+    process.removeListener('SIGINT', cleanup);
+    process.removeListener('SIGTERM', cleanup);
   }
 
   // Perform comprehensive final cleanup using enhanced browserexit module
@@ -4617,15 +4819,23 @@ function setupFrameHandling(page, forceDebug) {
     if (forceDebug) console.log(formatLogMessage('debug', `Browser connection check failed: ${connErr.message}`));
   }
 
-  const cleanupResult = await handleBrowserExit(browser, {
-    forceDebug,
-    timeout: 10000,
-    exitOnFailure: true,
-    cleanTempFiles: true,
-    comprehensiveCleanup: removeTempFiles,  // Use --remove-tempfiles flag
-    userDataDir: browser._nwssUserDataDir,
-    verbose: !silentMode && removeTempFiles  // Show verbose output only if removing temp files and not silent
-  });
+  // Obscura: just disconnect, don't kill — we don't own the browser process
+  let cleanupResult;
+  if (browser._nwssIsObscura) {
+    try { await browser.disconnect(); } catch {}
+    cleanupResult = { success: true, browserClosed: true, tempFilesCleanedCount: 0, userDataCleaned: false, errors: [] };
+    if (forceDebug) console.log(formatLogMessage('debug', `Disconnected from Obscura (process left running)`));
+  } else {
+    cleanupResult = await handleBrowserExit(browser, {
+      forceDebug,
+      timeout: 10000,
+      exitOnFailure: true,
+      cleanTempFiles: true,
+      comprehensiveCleanup: removeTempFiles,
+      userDataDir: browser._nwssUserDataDir,
+      verbose: !silentMode && removeTempFiles
+    });
+  }
 
   if (forceDebug) {
     console.log(formatLogMessage('debug', `Final cleanup results: ${cleanupResult.success ? 'success' : 'failed'}`));
@@ -4680,6 +4890,24 @@ function setupFrameHandling(page, forceDebug) {
     if (ignoreCache && forceDebug) {
       console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);
     }
+    // DNS cache statistics
+    const dnsStats = getDnsCacheStats();
+    if (dnsStats.digHits + dnsStats.digMisses > 0 || dnsStats.whoisHits + dnsStats.whoisMisses > 0) {
+      const parts = [];
+      if (dnsStats.digHits + dnsStats.digMisses > 0) {
+        parts.push(`${messageColors.success(dnsStats.digHits)} dig cached, ${messageColors.timing(dnsStats.digMisses)} fresh`);
+      }
+      if (dnsStats.whoisHits + dnsStats.whoisMisses > 0) {
+        parts.push(`${messageColors.success(dnsStats.whoisHits)} whois cached, ${messageColors.timing(dnsStats.whoisMisses)} fresh`);
+      }
+      console.log(messageColors.info('DNS cache:') + ` ${parts.join(' | ')}`);
+      if (dnsStats.freshDig.length > 0) {
+        console.log(messageColors.info('  Fresh dig:') + ` ${dnsStats.freshDig.join(', ')}`);
+      }
+      if (dnsStats.freshWhois.length > 0) {
+        console.log(messageColors.info('  Fresh whois:') + ` ${dnsStats.freshWhois.join(', ')}`);
+      }
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.59",
+  "version": "2.0.61",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {