@fanboynz/network-scanner 2.0.60 → 2.0.62

package/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to the Network Scanner (nwss.js) project.
 
+## [2.0.61] - 2026-03-17
+
+### Added
+- `.nwssconfig` file for per-config-file CLI settings — define output, concurrency, flags per JSON config
+- `--no-color` / `--no-colour` flag to disable colors (colors now enabled by default)
+- Navigation timeout fallback — retries with `waitUntil: networkidle2` on timeout, 10s cap
+- Skip domains after 3 consecutive timeouts in the same scan to avoid wasting time on down sites
+- Fingerprint cache capped at 500 entries with LRU eviction
+
+### Fixed
+- `chrome-error://` popup redirects no longer throw errors — continue processing captured requests
+- Suppressed noisy `about:blank` and `chrome-error://` redirect warnings (visible with `--debug` only)
+- Fallback retry skipped for `chrome-error://` redirects (instant failure, not genuine timeout)
+- Page URL checked before fallback retry to detect already-failed state
+- `.nwssconfig` keys support both hyphens and underscores (`dns-cache` and `dns_cache` both work)
+
+### Improved
+- Colors enabled by default — no need for `--color` flag or `color: true` in `.nwssconfig`
+- Chrome UA bumped to 146, Firefox UA bumped to 148
+- Sec-CH-UA headers updated to match Chrome 146
+
 ## [2.0.60] - 2026-03-16
 
 ### Added

package/README.md

@@ -29,7 +29,7 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 |:---------------------------|:------------|
 | `-o, --output <file>`       | Output file for rules. If omitted, prints to console |
 | `--compare <file>`          | Remove rules that already exist in this file before output |
-| `--color, --colour`         | Enable colored console output for status messages |
+| `--no-color, --no-colour`   | Disable colored console output (colors enabled by default) |
 | `--append`                  | Append new rules to output file instead of overwriting (requires `-o`) |
 
 ### Output Format Options
@@ -64,6 +64,9 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 | `--headful`                 | Launch browser with GUI (not headless) |
 | `--keep-open`               | Keep browser and tabs open after scan completes (use with `--headful` for debugging) |
 | `--use-puppeteer-core`      | Use `puppeteer-core` with system Chrome instead of bundled Chromium |
+| `--load-extension <path>`   | Load unpacked Chrome extension from directory (can be used multiple times) |
+| `--dns-cache`               | Persist dig/whois results to disk between runs (14hr TTL, `.digcache`/`.whoiscache`) |
+| `--block-ads=<files>`       | Block ads using EasyList format rules (comma-separated: `easylist.txt,easyprivacy.txt`) |
 | `--cdp`                     | Enable Chrome DevTools Protocol logging (now per-page if enabled) |
 | `--remove-dupes`            | Remove duplicate domains from output (only with `-o`) |
 | `--dry-run`                 | Console output only: show matching regex, titles, whois/dig/searchstring results, and adblock rules |
@@ -404,6 +407,53 @@ If a proxy fails mid-scan, Chromium's error code is detected and diagnosed:
 
 Detected error codes: `ERR_PROXY_CONNECTION_FAILED`, `ERR_SOCKS_CONNECTION_FAILED`, `ERR_TUNNEL_CONNECTION_FAILED`, `ERR_PROXY_AUTH_UNSUPPORTED`, `ERR_PROXY_AUTH_REQUESTED`, `ERR_SOCKS_CONNECTION_HOST_UNREACHABLE`, `ERR_PROXY_CERTIFICATE_INVALID`, `ERR_NO_SUPPORTED_PROXIES`.
 
+---
+
+## .nwssconfig — Per-Config Settings
+
+Create a `.nwssconfig` file in the project root to define CLI settings per config file. When a config filename matches a key, those settings are automatically applied. CLI flags merge with and override `.nwssconfig` settings.
+
+```json
+{
+  "configs": {
+    "config-clean1.json": {
+      "output": "outputfile.txt",
+      "max_concurrent": 30,
+      "dns_cache": true,
+      "cache_requests": true,
+      "dumpurls": true,
+      "remove_tempfiles": true,
+      "color": true
+    },
+    "config-clean2.json": {
+      "output": "outputfile.txt",
+      "max_concurrent": 15,
+      "dns_cache": true,
+      "cache_requests": true,
+      "dumpurls": true,
+      "remove_tempfiles": true,
+      "color": true,
+      "debug": true,
+      "block_ads": "easylist.txt,easyprivacy.txt"
+    }
+  }
+}
+```
+
+**Usage:**
+
+```bash
+node nwss.js config-clean1.json                    # uses .nwssconfig settings
+node nwss.js config-clean2.json --debug             # .nwssconfig + debug override
+node nwss.js config-other.json --max-concurrent 5   # no match in .nwssconfig, uses CLI flags
+```
+
+**Supported settings:** `output`, `max_concurrent`, `dns_cache`, `cache_requests`, `dumpurls`, `remove_tempfiles`, `color`, `remove_dupes`, `compress_logs`, `debug`, `silent`, `verbose`, `headful`, `keep_open`, `dry_run`, `titles`, `sub_domains`, `no_interact`, `ghost_cursor`, `plain`, `cdp`, `dnsmasq`, `unbound`, `privoxy`, `pihole`, `eval_on_doc`, `use_puppeteer_core`, `ignore_cache`, `clear_cache`, `block_ads`, `compare`, `localhost`, `append`.
+
+**Priority:** CLI flags > `.nwssconfig` > hardcoded defaults.
+
+---
+
 ### Global Configuration Options
 
 These options go at the root level of your config.json:

package/lib/colorize.js

@@ -7,7 +7,9 @@
  * @returns {boolean} True if --color or --colour flag is present
  */
 function shouldEnableColors() {
-  return process.argv.includes('--color') || process.argv.includes('--colour');
+  // Colors enabled by default, use --no-color to disable
+  if (process.argv.includes('--no-color') || process.argv.includes('--no-colour')) return false;
+  return true;
 }
 
 // Initialize color support based on command line flags

package/lib/fingerprint.js

@@ -22,6 +22,7 @@ function seededRandom(seed) {
 
 // Cache fingerprints per domain so reloads and multi-page visits stay consistent
 const _fingerprintCache = new Map();
+const FINGERPRINT_CACHE_MAX = 500;
 
 // Type-specific property spoofing functions for monomorphic optimization
 // Built-in properties that should not be modified
@@ -32,12 +33,12 @@ const BUILT_IN_PROPERTIES = new Set([
 
 // User agent collections with latest versions
 const USER_AGENT_COLLECTIONS = Object.freeze(new Map([
-  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0"],
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0"],
   ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
 ]));
 
@@ -237,7 +238,12 @@ function generateRealisticFingerprint(userAgent, domain = '') {
   };
   
   // Cache for this domain
-  if (domain) _fingerprintCache.set(domain, fingerprint);
+  if (domain) {
+    if (_fingerprintCache.size >= FINGERPRINT_CACHE_MAX) {
+      _fingerprintCache.delete(_fingerprintCache.keys().next().value);
+    }
+    _fingerprintCache.set(domain, fingerprint);
+  }
 
   return fingerprint;
 }
@@ -495,7 +501,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
               }),
               getManifest: () => ({ 
                 name: "Chrome", 
-                version: "145.0.0.0",
+                version: "146.0.0.0",
                 manifest_version: 3,
                 description: "Chrome Browser"
               }),
@@ -1609,18 +1615,41 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         }, 'enhanced mouse/pointer spoofing');
 
         safeExecute(() => {
-          // Filter DevTools/automation traces from console.debug
+          // Neutralize CDP fingerprinting traps and filter DevTools traces
+          // CDP's Runtime.enable causes the inspector to read properties on console-logged objects.
+          // Detection scripts exploit this via console.debug with Error objects (custom .stack getters)
+          // or objects with Proxy prototypes. Only override console.debug — safest, minimal footprint.
+
           const originalConsoleDebug = console.debug;
           console.debug = function(...args) {
+            // Filter DevTools-related messages
             const message = args.join(' ');
             if (typeof message === 'string' && (
                 message.includes('DevTools') ||
                 message.includes('Runtime.evaluate') ||
                 message.includes('Page.addScriptToEvaluateOnNewDocument') ||
                 message.includes('Protocol error'))) {
-              return; // Silently drop DevTools-related debug messages
+              return;
             }
-            return originalConsoleDebug.apply(this, args);
+            // Sanitize args to neutralize CDP fingerprinting traps
+            const sanitized = args.map(arg => {
+              // Strip Error objects with custom .stack getters (CDP inspector reads .stack)
+              if (arg instanceof Error) {
+                const desc = Object.getOwnPropertyDescriptor(arg, 'stack');
+                if (desc && desc.get) return `${arg.name}: ${arg.message}`;
+              }
+              // Neutralize Proxy prototype traps (CDP inspector walks prototype chain)
+              if (arg !== null && typeof arg === 'object') {
+                try {
+                  const proto = Object.getPrototypeOf(arg);
+                  if (proto && proto !== Object.prototype && proto !== Array.prototype) {
+                    try { Object.keys(proto); } catch { return '[object Object]'; }
+                  }
+                } catch { return '[object Object]'; }
+              }
+              return arg;
+            });
+            return originalConsoleDebug.apply(this, sanitized);
           };
 
         }, 'console error suppression');
@@ -1670,6 +1699,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
           if (typeof window.Image === 'function') maskAsNative(window.Image, 'Image');
           if (typeof window.fetch === 'function') maskAsNative(window.fetch, 'fetch');
           if (typeof window.PointerEvent === 'function') maskAsNative(window.PointerEvent, 'PointerEvent');
+          if (typeof console.debug === 'function') maskAsNative(console.debug, 'debug');
 
           // Mask property getters on navigator
           const navProps = ['userAgentData', 'connection', 'pdfViewerEnabled', 'webdriver',

package/lib/smart-cache.js

@@ -867,14 +867,14 @@ class SmartCache {
       netToolsCacheSize: this.netToolsCache.size,
       similarityCacheSize: this.similarityCache.size,
       regexCacheSize: this.regexCache.size,
-      requestHitRate: this._enableRequest ? 
+      requestHitRate: (this._enableRequest && this.requestCache) ?
         (requestHitRate * 100).toFixed(2) + '%' : '0% (disabled)',
-      requestCacheSize: this._enableRequest ? this.requestCache.size : 0,
-      requestCacheMemoryMB: this._enableRequest ? 
+      requestCacheSize: (this._enableRequest && this.requestCache) ? this.requestCache.size : 0,
+      requestCacheMemoryMB: (this._enableRequest && this.requestCache) ?
         Math.round((this.requestCache.calculatedSize || 0) / 1048576) : 0,
-      totalCacheEntries: this.domainCache.size + this.patternCache.size + 
-        this.responseCache.size + this.netToolsCache.size + 
-        this.similarityCache.size + this.regexCache.size + (this._enableRequest ? this.requestCache.size : 0),
+      totalCacheEntries: this.domainCache.size + this.patternCache.size +
+        this.responseCache.size + this.netToolsCache.size +
+        this.similarityCache.size + this.regexCache.size + ((this._enableRequest && this.requestCache) ? this.requestCache.size : 0),
       memoryUsageMB: Math.round(heapUsed / 1048576),
       memoryMaxMB: Math.round(maxHeap / 1048576),
       memoryUsagePercent: ((heapUsed / maxHeap) * 100).toFixed(1) + '%',

package/nwss.js

@@ -2,7 +2,8 @@
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
-const usePuppeteerCore = process.argv.includes('--use-puppeteer-core');
+const useObscura = process.argv.includes('--use-obscura');
+const usePuppeteerCore = process.argv.includes('--use-puppeteer-core') || useObscura;
 const puppeteer = usePuppeteerCore ? require('puppeteer-core') : require('puppeteer');
 const fs = require('fs');
 const os = require('os');
@@ -104,12 +105,12 @@ const CONCURRENCY_LIMITS = Object.freeze({
 
 // V8 Optimization: Use Map for user agent lookups instead of object
 const USER_AGENTS = Object.freeze(new Map([
-  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0"],
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0"],
   ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
 ]));
 
@@ -177,6 +178,82 @@ if (args.length === 0) {
   args.push('--help');
 }
 
+// --- .nwssconfig support: inject per-config settings into args ---
+const NWSSCONFIG_PATH = path.join(__dirname, '.nwssconfig');
+if (fs.existsSync(NWSSCONFIG_PATH)) {
+  try {
+    const nwssConfig = JSON.parse(fs.readFileSync(NWSSCONFIG_PATH, 'utf-8'));
+    // Find which config file is being used (--custom-json <file> or positional .json arg)
+    const customJsonIdx = args.findIndex(arg => arg === '--custom-json');
+    const configFilename = (customJsonIdx !== -1 && args[customJsonIdx + 1])
+      ? args[customJsonIdx + 1]
+      : args.find(a => a.endsWith('.json') && !a.startsWith('--'));
+
+    if (configFilename && nwssConfig.configs && nwssConfig.configs[configFilename]) {
+      const settings = nwssConfig.configs[configFilename];
+      const originalArgs = args.join(' ');
+
+      // Map settings keys to CLI flags — only inject if not already in args
+      const settingsMap = {
+        output: ['-o', '--output'],
+        max_concurrent: ['--max-concurrent'],
+        dns_cache: ['--dns-cache'],
+        cache_requests: ['--cache-requests'],
+        dumpurls: ['--dumpurls'],
+        remove_tempfiles: ['--remove-tempfiles'],
+        color: ['--color'],
+        remove_dupes: ['--remove-dupes', '--remove-dubes'],
+        'remove-dupes': ['--remove-dupes', '--remove-dubes'],
+        'remove-dubes': ['--remove-dupes', '--remove-dubes'],
+        compress_logs: ['--compress-logs'],
+        debug: ['--debug'],
+        silent: ['--silent'],
+        verbose: ['--verbose'],
+        headful: ['--headful'],
+        keep_open: ['--keep-open'],
+        dry_run: ['--dry-run'],
+        titles: ['--titles'],
+        sub_domains: ['--sub-domains'],
+        no_interact: ['--no-interact'],
+        ghost_cursor: ['--ghost-cursor'],
+        plain: ['--plain'],
+        cdp: ['--cdp'],
+        dnsmasq: ['--dnsmasq'],
+        unbound: ['--unbound'],
+        privoxy: ['--privoxy'],
+        pihole: ['--pihole'],
+        eval_on_doc: ['--eval-on-doc'],
+        use_puppeteer_core: ['--use-puppeteer-core'],
+        ignore_cache: ['--ignore-cache'],
+        clear_cache: ['--clear-cache'],
+        block_ads: ['--block-ads'],
+        compare: ['--compare'],
+        localhost: ['--localhost'],
+        append: ['--append']
+      };
+
+      for (const [key, flags] of Object.entries(settingsMap)) {
+        // Support both underscore and hyphen variants (e.g. dns_cache or dns-cache)
+        const value = settings[key] !== undefined ? settings[key]
+          : settings[key.replace(/_/g, '-')] !== undefined ? settings[key.replace(/_/g, '-')]
+          : settings[key.replace(/-/g, '_')] !== undefined ? settings[key.replace(/-/g, '_')]
+          : undefined;
+        if (value === undefined) continue;
+        // Skip if any variant of the flag is already in CLI args
+        if (flags.some(f => originalArgs.includes(f))) continue;
+
+        if (typeof value === 'boolean') {
+          if (value) args.push(flags[flags.length - 1]);
+        } else if (typeof value === 'string' || typeof value === 'number') {
+          args.push(flags[flags.length - 1], String(value));
+        }
+      }
+    }
+  } catch (e) {
+    console.error(`Warning: Failed to parse .nwssconfig: ${e.message}`);
+  }
+}
+
 const headfulMode = args.includes('--headful');
 const SOURCES_FOLDER = 'sources';
 
@@ -565,6 +642,12 @@ Request Blocking:
   --block-ads=<file>             Block ads/trackers using EasyList format rules (||domain.com^, /ads/*, etc)
                                  Works at request-level for maximum performance
 
+Per-config settings file (.nwssconfig):
+  Place a .nwssconfig file in the project root to define per-config settings.
+  When a config filename matches a key in .nwssconfig, those settings are used.
+  CLI flags merge with and override .nwssconfig settings.
+  See README.md for format details.
+
 General Options:
   --verbose                      Force verbose mode globally
   --debug                        Force debug mode globally
@@ -580,6 +663,8 @@ General Options:
   --headful                      Launch browser with GUI (not headless)
   --keep-open                    Keep browser open after scan completes (use with --headful)
   --use-puppeteer-core           Use puppeteer-core with system Chrome instead of bundled Chromium
+  --use-obscura                  Connect to running Obscura CDP server (ws://127.0.0.1:9222 or OBSCURA_WS env)
+                                 Skips fingerprint injection — Obscura provides built-in stealth
   --load-extension <path>        Load unpacked Chrome extension from directory
   --cdp                          Enable Chrome DevTools Protocol logging (now per-page if enabled)
   --remove-dupes                 Remove duplicate domains from output (only with -o)
@@ -788,6 +873,23 @@ const globalBlockedRegexes = Array.isArray(globalBlocked)
   ? globalBlocked.map(pattern => new RegExp(pattern))
   : [];
 
+// Cache compiled regexes by pattern string — avoids recompiling same patterns across URLs
+const _compiledRegexCache = new Map();
+function getCompiledRegex(pattern) {
+  let compiled = _compiledRegexCache.get(pattern);
+  if (!compiled) {
+    compiled = new RegExp(pattern.replace(/^\/(.*)\/$/, '$1'));
+    if (_compiledRegexCache.size > 2000) _compiledRegexCache.clear();
+    _compiledRegexCache.set(pattern, compiled);
+  }
+  return compiled;
+}
+function getCompiledRegexes(patterns) {
+  if (!patterns) return [];
+  const arr = Array.isArray(patterns) ? patterns : [patterns];
+  return arr.map(p => getCompiledRegex(p));
+}
+
 // Pre-split ignoreDomains into exact Set (O(1) lookup) and wildcard array
 const _ignoreDomainsExact = new Set();
 const _ignoreDomainsWildcard = [];
@@ -1116,12 +1218,19 @@ if (forceDebug && globalComments) {
  * @param {string} url - The URL string to parse.
  * @returns {string} The root domain, or the original hostname if parsing fails (e.g., for IP addresses or invalid URLs), or an empty string on error.
  */
+const _rootDomainCache = new Map();
 function getRootDomain(url) {
+  const cached = _rootDomainCache.get(url);
+  if (cached !== undefined) return cached;
   try {
     const { hostname } = new URL(url);
     const parsed = psl.parse(hostname);
-    return parsed.domain || hostname;
+    const result = parsed.domain || hostname;
+    if (_rootDomainCache.size > 5000) _rootDomainCache.clear();
+    _rootDomainCache.set(url, result);
+    return result;
   } catch {
+    _rootDomainCache.set(url, '');
     return '';
   }
 }
@@ -1416,6 +1525,23 @@ function setupFrameHandling(page, forceDebug) {
    * @returns {Promise<import('puppeteer').Browser>} Browser instance
    */
   async function createBrowser(extraArgs = []) {
+    // Obscura mode: connect to a running Obscura CDP server instead of launching Chrome
+    if (useObscura) {
+      const obscuraEndpoint = process.env.OBSCURA_WS || 'ws://127.0.0.1:9222/devtools/browser';
+      if (forceDebug) console.log(formatLogMessage('debug', `Connecting to Obscura at ${obscuraEndpoint}`));
+      try {
+        const browser = await puppeteer.connect({ browserWSEndpoint: obscuraEndpoint });
+        if (!silentMode) console.log(messageColors.success(`Connected to Obscura CDP at ${obscuraEndpoint}`));
+        browser._nwssUserDataDir = null; // No temp dir to clean
+        browser._nwssIsObscura = true;
+        return browser;
+      } catch (err) {
+        console.error(formatLogMessage('error', `Failed to connect to Obscura: ${err.message}`));
+        console.error(formatLogMessage('error', `Start Obscura first: obscura serve --port 9222 --stealth`));
+        process.exit(1);
+      }
+    }
+
     // Create temporary user data directory that we can fully control and clean up
     const tempUserDataDir = path.join(os.tmpdir(), `puppeteer-${Date.now()}-${Math.random().toString(36).substring(7)}`);
     userDataDir = tempUserDataDir; // Store for cleanup tracking (use outer scope variable)
@@ -2231,11 +2357,16 @@ function setupFrameHandling(page, forceDebug) {
       }
 
       // --- Apply all fingerprint spoofing (user agent, Brave, fingerprint protection) ---
+      // Skip when using Obscura — it has built-in stealth that conflicts with our injection
       try {
-        await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+        if (!useObscura) {
+          await applyAllFingerprintSpoofing(page, siteConfig, forceDebug, currentUrl);
+        } else if (forceDebug) {
+          console.log(formatLogMessage('debug', `Skipping fingerprint injection — Obscura provides built-in stealth`));
+        }
         
-        // Client Hints protection for Chrome user agents
-        if (siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
+        // Client Hints protection for Chrome user agents (skipped under Obscura — it sets its own)
+        if (!useObscura && siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
           const userAgentKey = siteConfig.userAgent.toLowerCase();
           let platform = 'Windows';
           let platformVersion = '15.0.0';
@@ -2252,14 +2383,14 @@ function setupFrameHandling(page, forceDebug) {
           }
                     
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Not:A-Brand";v="99", "Google Chrome";v="145", "Chromium";v="145"',
+            'Sec-CH-UA': '"Not:A-Brand";v="99", "Google Chrome";v="146", "Chromium";v="146"',
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"145.0.7632.160"',
-            'Sec-CH-UA-Full-Version-List': '"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="145.0.7632.160", "Chromium";v="145.0.7632.160"'
+            'Sec-CH-UA-Full-Version': '"146.0.0.0"',
+            'Sec-CH-UA-Full-Version-List': '"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="146.0.0.0", "Chromium";v="146.0.0.0"'
           });
         }
       } catch (fingerprintErr) {
@@ -2272,11 +2403,7 @@ function setupFrameHandling(page, forceDebug) {
         }
       }
 
-      const regexes = Array.isArray(siteConfig.filterRegex)
-        ? siteConfig.filterRegex.map(r => new RegExp(r.replace(/^\/(.*)\/$/, '$1')))
-        : siteConfig.filterRegex
-          ? [new RegExp(siteConfig.filterRegex.replace(/^\/(.*)\/$/, '$1'))]
-          : [];
+      const regexes = getCompiledRegexes(siteConfig.filterRegex);
 
       // NEW: Get regex_and setting (defaults to false for backward compatibility)
       const useRegexAnd = siteConfig.regex_and === true;
@@ -2424,7 +2551,7 @@ function setupFrameHandling(page, forceDebug) {
   }
 
       const blockedRegexes = Array.isArray(siteConfig.blocked)
-        ? siteConfig.blocked.map(pattern => new RegExp(pattern))
+        ? siteConfig.blocked.map(pattern => getCompiledRegex(pattern))
         : [];
 		
       // Combine site-specific with pre-compiled global blocked patterns
@@ -3201,7 +3328,23 @@ function setupFrameHandling(page, forceDebug) {
           ? { ...defaultGotoOptions, ...siteConfig.goto_options } : defaultGotoOptions;
 
         // Enhanced navigation with redirect handling - passes existing gotoOptions
-        const navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOptions, forceDebug, formatLogMessage);
+        let navigationResult;
+        try {
+          navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOptions, forceDebug, formatLogMessage);
+        } catch (navErr) {
+          // Only retry on genuine timeouts, not chrome-error:// redirects
+          let pageUrl = '';
+          try { if (!page.isClosed()) pageUrl = page.url(); } catch {}
+          const isPopupFailure = navErr.message.includes('chrome-error://') || navErr.message.includes('invalid URL') ||
+            pageUrl.startsWith('chrome-error://') || pageUrl === 'about:blank';
+          if ((navErr.message.includes('timeout') || navErr.message.includes('Timeout')) && !isPopupFailure) {
+            if (forceDebug) console.log(formatLogMessage('debug', `Navigation timeout, retrying with waitUntil:networkidle2 for ${currentUrl}`));
+            const fallbackOptions = { ...gotoOptions, waitUntil: 'networkidle2', timeout: Math.min(timeout, 10000) };
+            navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, fallbackOptions, forceDebug, formatLogMessage);
+          } else {
+            throw navErr;
+          }
+        }
         
         const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains } = navigationResult;
         
@@ -3257,7 +3400,8 @@ function setupFrameHandling(page, forceDebug) {
           }
           
           if (originalDomain !== finalDomain) {
-            if (!silentMode) {
+            const isPopupRedirect = !finalUrl || finalUrl === 'about:blank' || finalUrl.startsWith('chrome-error://');
+            if (!silentMode && !isPopupRedirect) {
               console.log(`🔄 Redirect detected: ${originalDomain} → ${finalDomain}`);
             }
             
@@ -3284,13 +3428,11 @@ function setupFrameHandling(page, forceDebug) {
                 }
               }
             } else {
-              // Invalid final URL - don't update currentUrl, treat as failed redirect
-              console.warn(`⚠ Redirect to invalid URL ignored: ${originalDomain} → ${finalUrl}`);
+              // Invalid final URL (ad popup redirect) - continue with original URL
               if (forceDebug) {
-                console.log(formatLogMessage('debug', `Redirect chain ended with invalid URL, keeping original: ${originalUrl}`));
+                console.log(formatLogMessage('debug', `Popup redirect ignored: ${originalDomain} → ${finalUrl}, keeping original: ${originalUrl}`));
               }
-              // Keep processing with the original URL or throw an error
-              throw new Error(`Redirect resulted in invalid URL: ${finalUrl}`);
+              // Continue with original URL — requests captured before the redirect are still valid
             }
           }
         }
@@ -3431,7 +3573,10 @@ function setupFrameHandling(page, forceDebug) {
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
         
         if (timeoutResult.success) {
-          console.log(`⚠ Partial redirect timeout recovered: ${safeGetDomain(currentUrl)} → ${safeGetDomain(timeoutResult.finalUrl)}`);
+          const isPopupRedirect = timeoutResult.finalUrl && (timeoutResult.finalUrl === 'about:blank' || timeoutResult.finalUrl.startsWith('chrome-error://'));
+          if (!isPopupRedirect) {
+            console.log(`⚠ Partial redirect timeout recovered: ${safeGetDomain(currentUrl)} → ${safeGetDomain(timeoutResult.finalUrl)}`);
+          }
           currentUrl = timeoutResult.finalUrl; // Use the partial redirect URL
           siteCounter++;
           // Continue processing with the redirected URL instead of throwing error
@@ -4060,6 +4205,10 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
 
+ // Track domain timeout counts — skip domain after 3 failures
+ const domainTimeoutCounts = new Map();
+ const DOMAIN_TIMEOUT_THRESHOLD = 3;
+
  // Enhanced hang detection with browser restart recovery
  let currentBatchInfo = { batchStart: 0, batchSize: 0 };
  let lastProcessedCount = 0;
@@ -4285,8 +4434,17 @@ function setupFrameHandling(page, forceDebug) {
       console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
- // Create tasks with timeout protection
- const batchTasks = currentBatch.map(task => originalLimit(() => processUrl(task.url, task.config, browser)));
+ // Create tasks with timeout protection — skip domains that repeatedly timed out
+ const batchTasks = currentBatch.map(task => originalLimit(() => {
+   try {
+     const taskDomain = new URL(task.url).hostname;
+     if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
+       if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
+       return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+     }
+   } catch {}
+   return processUrl(task.url, task.config, browser);
+ }));
  
  let batchResults;
  try {
@@ -4316,6 +4474,16 @@ function setupFrameHandling(page, forceDebug) {
    }
  }
 
+    // Track domain timeout counts — skip after threshold
+    for (const result of batchResults) {
+      if (!result.success && !result.skipped && result.error && result.error.includes('timeout')) {
+        try {
+          const domain = new URL(result.url).hostname;
+          domainTimeoutCounts.set(domain, (domainTimeoutCounts.get(domain) || 0) + 1);
+        } catch {}
+      }
+    }
+
     // IMPROVED: Much more conservative emergency restart logic
     const criticalRestartCount = batchResults.filter(r => r.needsImmediateRestart).length;
     // Require either:
@@ -4651,15 +4819,23 @@ function setupFrameHandling(page, forceDebug) {
     if (forceDebug) console.log(formatLogMessage('debug', `Browser connection check failed: ${connErr.message}`));
   }
 
-  const cleanupResult = await handleBrowserExit(browser, {
-    forceDebug,
-    timeout: 10000,
-    exitOnFailure: true,
-    cleanTempFiles: true,
-    comprehensiveCleanup: removeTempFiles,  // Use --remove-tempfiles flag
-    userDataDir: browser._nwssUserDataDir,
-    verbose: !silentMode && removeTempFiles  // Show verbose output only if removing temp files and not silent
-  });
+  // Obscura: just disconnect, don't kill — we don't own the browser process
+  let cleanupResult;
+  if (browser._nwssIsObscura) {
+    try { await browser.disconnect(); } catch {}
+    cleanupResult = { success: true, browserClosed: true, tempFilesCleanedCount: 0, userDataCleaned: false, errors: [] };
+    if (forceDebug) console.log(formatLogMessage('debug', `Disconnected from Obscura (process left running)`));
+  } else {
+    cleanupResult = await handleBrowserExit(browser, {
+      forceDebug,
+      timeout: 10000,
+      exitOnFailure: true,
+      cleanTempFiles: true,
+      comprehensiveCleanup: removeTempFiles,
+      userDataDir: browser._nwssUserDataDir,
+      verbose: !silentMode && removeTempFiles
+    });
+  }
 
   if (forceDebug) {
     console.log(formatLogMessage('debug', `Final cleanup results: ${cleanupResult.success ? 'success' : 'failed'}`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.60",
+  "version": "2.0.62",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {