@fanboynz/network-scanner 2.0.65 → 3.0.0

package/nwss.js

@@ -9,6 +9,7 @@ const fs = require('fs');
 const os = require('os');
 const psl = require('psl');
 const path = require('path');
+const dnsPromises = require('node:dns/promises');
 const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
 const { compressMultipleFiles, formatFileSize } = require('./lib/compress');
 const { parseSearchStrings, createResponseHandler, createCurlHandler } = require('./lib/searchstring');
@@ -27,13 +28,13 @@ const {
   cleanup: cleanupCloudflareCache
 } = require('./lib/cloudflare');
 // FP Bypass
-const { handleFlowProxyProtection, getFlowProxyTimeouts } = require('./lib/flowproxy');
+const { handleFlowProxyProtection, getFlowProxyTimeouts, attachFlowProxyHeaderListener } = require('./lib/flowproxy');
 // ignore_similar rules
 const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore_similar');
 // Graceful exit
-const { handleBrowserExit, cleanupChromeTempFiles } = require('./lib/browserexit');
+const { handleBrowserExit, cleanupChromeTempFiles, cleanupUserDataDir } = require('./lib/browserexit');
 // Whois & Dig
-const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats } = require('./lib/nettools');
+const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats, domainKnownToResolve } = require('./lib/nettools');
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // CDP functionality
@@ -41,7 +42,29 @@ const { createCDPSession, createPageWithTimeout, setRequestInterceptionWithTimeo
 // Post-processing cleanup
 const { processResults } = require('./lib/post-processing');
 // Colorize various text when used
-const { colorize, colors, messageColors, tags, formatLogMessage } = require('./lib/colorize');
+const { messageColors, formatLogMessage } = require('./lib/colorize');
+const TIMEOUT_TAG = messageColors.processing('[TIMEOUT]');
+const INTERACTION_TAG = messageColors.processing('[interaction]');
+const GHOST_CURSOR_TAG = messageColors.processing('[ghost-cursor]');
+const PROXY_TAG = messageColors.processing('[proxy]');
+const GREP_RESPONSE_TAG = messageColors.processing('[grep-response]');
+const IGNORE_DOMAINS_BY_URL_TAG = messageColors.processing('[ignoreDomainsByUrl]');
+const BLOCK_DOMAINS_BY_URL_TAG = messageColors.processing('[blockDomainsByUrl]');
+const IGNORE_SIMILAR_IGNORED_DOMAINS_TAG = messageColors.processing('[ignore_similar_ignored_domains]');
+const IGNORE_SIMILAR_TAG = messageColors.processing('[ignore_similar]');
+const CLEAR_SITEDATA_TAG = messageColors.processing('[clear_sitedata]');
+const CSS_BLOCKED_TAG = messageColors.processing('[css_blocked]');
+const EVAL_ON_DOC_TAG = messageColors.processing('[evalOnDoc]');
+const REALTIME_CLEANUP_TAG = messageColors.processing('[realtime_cleanup]');
+const VPN_TAG = messageColors.processing('[vpn]');
+// Precomputed colored '[SmartCache]' subsystem prefix — paired with the
+// same constant in lib/smart-cache.js so debug lines from both files
+// produce consistently colored output. formatLogMessage only colors the
+// [severity] tag; this constant colors the subsystem prefix.
+const SMART_CACHE_TAG = messageColors.processing('[SmartCache]');
+// Precomputed colored '[CONCURRENCY]' subsystem prefix for batch-throughput
+// log lines (start/completed). Same cyan as the other monitoring tags.
+const CONCURRENCY_TAG = messageColors.processing('[CONCURRENCY]');
 // Enhanced mouse interaction and page simulation
 const { performPageInteraction, createInteractionConfig, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
 // Optional ghost-cursor support for advanced Bezier-based mouse movements
@@ -50,7 +73,7 @@ const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostR
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
-const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy } = require('./lib/proxy');
+const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy, prepareSocksRelays, closeAllSocksRelays } = require('./lib/proxy');
 // Dry run functionality
 const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
@@ -157,7 +180,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload, purgeStaleTrackers } = require('./lib/browserhealth');
+// purgeStaleTrackers removed from import: browserhealth's pageCreationTracker
+// and pageUsageTracker are now WeakMaps, so GC reclaims dead-page entries
+// automatically — manual purging is no longer needed.
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
 const VERSION = '2.0.33'; // Script version
@@ -266,6 +292,13 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
 }
 
 const headfulMode = args.includes('--headful');
+// Sites (esp. video/streaming) call element.requestFullscreen() on load or
+// click. In --headful that hijacks the real Chrome window into true
+// fullscreen, forcing a manual ESC. Neutralize the Fullscreen API by
+// default so it can't. Harmless in headless (no screen — the API is
+// already inert there), so default-on keeps headful consistent with the
+// primary headless path. --allow-fullscreen restores native behavior.
+const allowFullscreen = args.includes('--allow-fullscreen');
 const SOURCES_FOLDER = 'sources';
 
 let outputFile = null;
@@ -326,6 +359,36 @@ const cacheRequests = args.includes('--cache-requests');
 const dnsCacheMode = args.includes('--dns-cache');
 if (dnsCacheMode) enableDiskCache();
 
+// DNS pre-check before page.goto() — default-on, --no-dns-precheck disables.
+// Filters NXDOMAIN / unresolvable hostnames in <100ms before paying the
+// ~5-15s Puppeteer + Cloudflare detection round-trip on each.
+const dnsPrecheckEnabled = !args.includes('--no-dns-precheck');
+const dnsPrecheckTimeoutMs = 2000;
+
+// Per-scan cache of negative DNS lookups. OS resolvers don't always cache
+// NXDOMAIN responses, and a scan can hit the same dead hostname many times
+// (different URL paths on the same site). Positive results are left to the
+// OS cache; failure-cache avoids repeated lookup latency for known-dead hosts.
+// FIFO eviction at DNS_NEGATIVE_CACHE_MAX so pathological scans (thousands
+// of unique dead hosts) can't grow the cache unboundedly. Same pattern as
+// the rest of the codebase's in-memory caches.
+const dnsNegativeCache = new Map(); // hostname -> { error, timestamp }
+const DNS_NEGATIVE_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+const DNS_NEGATIVE_CACHE_MAX = 1000;
+let dnsPrecheckSkips = 0;          // URLs skipped because hostname is NXDOMAIN-cached
+let dnsPositiveSkips = 0;          // URLs skipped because dig/whois cache proves resolution
+const dnsPositiveSkippedHosts = new Set(); // unique hostnames that triggered the positive skip path
+// c-ares transient codes — read-only, hoisted out of the per-task DNS
+// pre-check so we don't allocate a fresh Set per URL.
+const DNS_TRANSIENT_ERRORS = new Set(['ETIMEOUT', 'ESERVFAIL', 'EREFUSED', 'ECONNREFUSED']);
+
+function dnsNegativeCacheSet(hostname, error) {
+  if (dnsNegativeCache.size >= DNS_NEGATIVE_CACHE_MAX) {
+    dnsNegativeCache.delete(dnsNegativeCache.keys().next().value);
+  }
+  dnsNegativeCache.set(hostname, { error, timestamp: Date.now() });
+}
+
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
 if (validateRulesIndex !== -1 && args[validateRulesIndex + 1] && !args[validateRulesIndex + 1].startsWith('--')) {
@@ -643,6 +706,8 @@ General Options:
   --custom-json <file>           Use a custom config JSON file instead of config.json
   --headful                      Launch browser with GUI (not headless)
   --keep-open                    Keep browser open after scan completes (use with --headful)
+  --allow-fullscreen             Allow sites to use the Fullscreen API. By default it is
+                                 neutralized so sites can't hijack the window in --headful
   --use-puppeteer-core           Use puppeteer-core with system Chrome instead of bundled Chromium
   --use-obscura                  Connect to running Obscura CDP server (ws://127.0.0.1:9222 or OBSCURA_WS env)
                                  Skips fingerprint injection — Obscura provides built-in stealth
@@ -658,7 +723,10 @@ General Options:
 
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
-  --dns-cache                    Persist dig/whois results to disk between runs (3hr/4hr TTL)
+  --dns-cache                    Persist dig/whois results to disk between runs (20h TTL, 2000-entry cap each)
+  --no-dns-precheck              Disable per-URL DNS resolution check before page navigation.
+                                 By default, URLs whose hostname doesn't resolve are skipped
+                                 immediately (saves ~5-15s of Puppeteer time per dead host).
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -669,6 +737,7 @@ Validation Options:
 Global config.json options:
   ignoreDomains: ["domain.com", "*.ads.com"]     Domains to completely ignore (supports wildcards)
   ignoreDomainsByUrl: ["regex1", "regex2"]       Regex patterns; if any request URL matches, the request's root domain is ignored for the rest of the scan
+  blockDomainsByUrl: ["regex1", "regex2"]        Regex patterns; if any request URL matches, ALL subsequent requests on that root domain (and subdomains) are aborted via Puppeteer for the rest of the scan
   blocked: ["regex1", "regex2"]                   Global regex patterns to block requests (combined with per-site blocked)
   whois_server_mode: "random" or "cycle"      Default server selection mode for all sites (default: random)
   ignore_similar: true/false                      Ignore domains similar to already found domains (default: true)
@@ -838,6 +907,7 @@ const {
   sites = [], 
   ignoreDomains = [],
   ignoreDomainsByUrl = [],
+  blockDomainsByUrl = [],
   blocked: globalBlocked = [],
   whois_delay = 3000, 
   whois_server_mode = 'random', 
@@ -927,10 +997,11 @@ if (validateConfig) {
   }
 }
 
-// Pre-compile global blocked regexes ONCE (used in every processUrl call)
-const globalBlockedRegexes = Array.isArray(globalBlocked)
-  ? globalBlocked.map(pattern => new RegExp(pattern))
-  : [];
+// Pre-compile global blocked regexes ONCE (used in every processUrl call).
+// Was: bare `.map(pattern => new RegExp(pattern))` which hard-threw at
+// module load on a single bad pattern, killing scan startup. Helper now
+// warns + skips so the rest of the config can still run.
+const globalBlockedRegexes = compilePatternList('blocked (global)', globalBlocked);
 
 // Cache compiled regexes by pattern string — avoids recompiling same patterns across URLs
 const _compiledRegexCache = new Map();
@@ -949,6 +1020,44 @@ function getCompiledRegexes(patterns) {
   return arr.map(p => getCompiledRegex(p));
 }
 
+/**
+ * Compile a list of regex pattern strings, WARNING loudly on any that fail
+ * compilation instead of:
+ *   (a) silently dropping them (old ignoreDomainsByUrl/blockDomainsByUrl
+ *       behavior) -- made debugging "why isn't my pattern matching?"
+ *       miserable, and
+ *   (b) hard-throwing at module load (old `blocked` behavior) -- one bad
+ *       pattern would kill the whole scan startup.
+ *
+ * Returns the array of successfully compiled regexes. Failed patterns are
+ * skipped with a single warn line per failure naming the config key + the
+ * source string + the regex error -- enough to find and fix without
+ * grepping through diff history.
+ *
+ * @param {string} configKey - name of the config key, for warn context
+ * @param {string[]} patterns - raw regex source strings
+ * @param {(p:string)=>RegExp} [compile] - compile fn (defaults to new RegExp)
+ * @returns {RegExp[]}
+ */
+function compilePatternList(configKey, patterns, compile = (p) => new RegExp(p)) {
+  if (!Array.isArray(patterns)) return [];
+  const out = [];
+  for (const p of patterns) {
+    try {
+      out.push(compile(p));
+    } catch (err) {
+      console.warn(formatLogMessage('warn', `[config] ${configKey} pattern dropped (compile error): ${JSON.stringify(p)} -- ${err.message}`));
+    }
+  }
+  return out;
+}
+
+// Per-pattern match counters for the `blocked` regex (site + global,
+// combined). Keyed by RegExp.source so the same pattern appearing in both
+// site and global lists rolls up into one row. Reported at scan end so
+// stale patterns that match zero requests are easy to spot and prune.
+const _blockedPatternHits = new Map();
+
 // Pre-split ignoreDomains into exact Set (O(1) lookup) and wildcard array
 const _ignoreDomainsExact = new Set();
 const _ignoreDomainsWildcard = [];
@@ -960,15 +1069,23 @@ for (const pattern of ignoreDomains) {
   }
 }
 
-// Compile ignoreDomainsByUrl patterns once — match request URLs to dynamically ignore domains
-const _ignoreDomainsByUrlRegexes = Array.isArray(ignoreDomainsByUrl)
-  ? ignoreDomainsByUrl.map(p => {
-      try { return getCompiledRegex(p); } catch { return null; }
-    }).filter(r => r)
-  : [];
+// Compile ignoreDomainsByUrl patterns once — match request URLs to dynamically ignore domains.
+// Bad patterns warn (via compilePatternList) instead of silently dropping.
+const _ignoreDomainsByUrlRegexes = compilePatternList('ignoreDomainsByUrl', ignoreDomainsByUrl, getCompiledRegex);
 // Runtime Set of domains marked ignored by URL pattern matches — shared across all sites in this scan
 const _dynamicallyIgnoredDomains = new Set();
 
+// blockDomainsByUrl: symmetric to ignoreDomainsByUrl but for active
+// blocking via Puppeteer's request.abort(). When a request URL matches
+// one of these regex patterns, the request's root domain is added to
+// _dynamicallyBlockedDomains; subsequent requests on that domain (and
+// its subdomains, via parent-walk in matchesDynamicBlock) get aborted
+// before reaching the network. The triggering request itself is also
+// aborted -- same "gate fires immediately after trigger" semantic the
+// ignoreDomainsByUrl path uses for the dynamic Set short-circuit.
+const _blockDomainsByUrlRegexes = compilePatternList('blockDomainsByUrl', blockDomainsByUrl, getCompiledRegex);
+const _dynamicallyBlockedDomains = new Set();
+
 // Apply global configuration overrides with validation
 // Priority: Command line args > config.json > defaults
 const MAX_CONCURRENT_SITES = (() => {
@@ -1065,7 +1182,7 @@ function safeMarkDomainProcessed(domain, context, metadata) {
       }
     } catch (cacheErr) {
       if (forceDebug) {
-        console.log(formatLogMessage('debug', `[SmartCache] Error marking domain: ${cacheErr.message}`));
+        console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Error marking domain: ${cacheErr.message}`));
       }
     }
   }
@@ -1379,16 +1496,58 @@ function shouldBypassCacheForUrl(url, siteConfig) {
 // ability to use wildcards in ignoreDomains
 // Cache compiled wildcard regexes to avoid recompilation on every request
 const _wildcardRegexCache = new Map();
+
+// Generic parent-walk helper: returns true if `domain` or any of its
+// parents (one label at a time, up to the TLD) is present in `set`.
+// Mirrors the static/dynamic parent-walk inside matchesIgnoreDomain but
+// usable against an arbitrary single Set -- consumed by
+// matchesDynamicBlock below. matchesIgnoreDomain keeps its inline
+// dual-Set probe so the hot path stays single-split, but new single-Set
+// consumers (block, future similar features) share this helper.
+function _domainOrParentInSet(set, domain) {
+  if (set.size === 0) return false;
+  if (set.has(domain)) return true;
+  const parts = domain.split('.');
+  for (let i = 1; i < parts.length; i++) {
+    if (set.has(parts.slice(i).join('.'))) return true;
+  }
+  return false;
+}
+
+/**
+ * Block-side counterpart to the ignore gate. Returns true if `domain`
+ * (or any of its parents) has been added to _dynamicallyBlockedDomains
+ * by an earlier blockDomainsByUrl pattern match. Called per-request to
+ * decide whether to request.abort() before the static blocked-regex
+ * check fires.
+ */
+function matchesDynamicBlock(domain) {
+  return _domainOrParentInSet(_dynamicallyBlockedDomains, domain);
+}
+
 function matchesIgnoreDomain(domain, ignorePatterns) {
-  // Dynamically ignored domains (from URL pattern matches via ignoreDomainsByUrl)
-  if (_dynamicallyIgnoredDomains.has(domain)) return true;
-  // Fast path: exact match or suffix match against Set (O(n) for parts, but no regex)
-  if (_ignoreDomainsExact.size > 0) {
-    if (_ignoreDomainsExact.has(domain)) return true;
-    // Check parent domains: sub.ads.example.com → ads.example.com → example.com
+  // Both dynamic and static ignore lists are walked parent-by-parent so a
+  // subdomain of an ignored root inherits the ignore. Previously the
+  // dynamic check was exact-only, creating an asymmetry: a static-config
+  // `example.com` ignored cdn.example.com transitively, but a runtime
+  // ignoreDomainsByUrl match for the same root (stored as root via
+  // checkedRootDomain at line ~2993) did NOT cascade -- subdomains slipped
+  // through to dig/whois/regex despite the root being ignored. Now
+  // unified: parts split once, shared between both Set probes.
+  const hasDynamic = _dynamicallyIgnoredDomains.size > 0;
+  const hasExact   = _ignoreDomainsExact.size > 0;
+
+  if (hasDynamic || hasExact) {
+    // Exact-domain hit on either set wins early.
+    if (hasDynamic && _dynamicallyIgnoredDomains.has(domain)) return true;
+    if (hasExact   && _ignoreDomainsExact.has(domain))         return true;
+
+    // Parent-walk: sub.ads.example.com → ads.example.com → example.com
     const parts = domain.split('.');
     for (let i = 1; i < parts.length; i++) {
-      if (_ignoreDomainsExact.has(parts.slice(i).join('.'))) return true;
+      const parent = parts.slice(i).join('.');
+      if (hasDynamic && _dynamicallyIgnoredDomains.has(parent)) return true;
+      if (hasExact   && _ignoreDomainsExact.has(parent))         return true;
     }
   }
 
@@ -1830,7 +1989,7 @@ function setupFrameHandling(page, forceDebug) {
     wgDisconnectAll(forceDebug);
     ovpnDisconnectAll(forceDebug);
     cleanupCloudflareCache();
-    purgeStaleTrackers();
+    try { await closeAllSocksRelays(forceDebug); } catch (_) {}
   }
  
   let siteCounter = 0;
@@ -1981,28 +2140,46 @@ function setupFrameHandling(page, forceDebug) {
       'Browser disconnected'
     ]);
 
+    // Popup-capture cleanup registry — declared outside the try so the
+    // finally block (which is a separate lexical scope from try) can see
+    // it. Populated by the capture_popups setup block if siteConfig
+    // .capture_popups is true; iterated in finally to deregister the
+    // browser 'targetcreated' listener and close any tracked popup pages.
+    const popupCleanups = [];
+    // Race-window guard: 'targetcreated' fires synchronously, but
+    // onTargetCreated does an `await target.page()`. If a popup target
+    // is created right as the per-URL try block winds down, the await
+    // can resolve AFTER finally has already iterated popupCleanups —
+    // leaving the popup unregistered for manual cleanup (it still gets
+    // closed by its own 3s auto-close timer, but in the meantime its
+    // request listener could capture matches into matchedDomains for a
+    // URL that already "finished"). The flag is set in finally and
+    // checked at the start of onTargetCreated to short-circuit late
+    // events cleanly.
+    let urlFinished = false;
+
     try {
 
       // --- Connect VPN if configured for this site ---
       if (siteConfig.vpn) {
         const vpnResult = await wgConnect(siteConfig, forceDebug);
         if (!vpnResult.success) {
-          console.warn(formatLogMessage('warn', `[vpn] WireGuard failed for ${currentUrl}: ${vpnResult.error}`));
+          console.warn(formatLogMessage('warn', `${VPN_TAG} WireGuard failed for ${currentUrl}: ${vpnResult.error}`));
           return { url: currentUrl, rules: [], success: false, vpnFailed: true };
         }
         if (!silentMode) {
           const ipInfo = vpnResult.externalIP ? ` (${vpnResult.externalIP})` : '';
-          console.log(formatLogMessage('info', `[vpn] WireGuard connected via ${vpnResult.interface}${ipInfo} for ${currentUrl}`));
+          console.log(formatLogMessage('info', `${VPN_TAG} WireGuard connected via ${vpnResult.interface}${ipInfo} for ${currentUrl}`));
         }
       } else if (siteConfig.openvpn) {
         const ovpnResult = await ovpnConnect(siteConfig, forceDebug);
         if (!ovpnResult.success) {
-          console.warn(formatLogMessage('warn', `[vpn] OpenVPN failed for ${currentUrl}: ${ovpnResult.error}`));
+          console.warn(formatLogMessage('warn', `${VPN_TAG} OpenVPN failed for ${currentUrl}: ${ovpnResult.error}`));
           return { url: currentUrl, rules: [], success: false, vpnFailed: true };
         }
         if (!silentMode) {
           const ipInfo = ovpnResult.externalIP ? ` (${ovpnResult.externalIP})` : '';
-          console.log(formatLogMessage('info', `[vpn] OpenVPN connected via ${ovpnResult.connection}${ipInfo} for ${currentUrl}`));
+          console.log(formatLogMessage('info', `${VPN_TAG} OpenVPN connected via ${ovpnResult.connection}${ipInfo} for ${currentUrl}`));
         }
       }
 
@@ -2036,12 +2213,12 @@ function setupFrameHandling(page, forceDebug) {
         const totalDelay = siteDelay + bufferTime;
         
         if (forceDebug && hasCloudflareConfig) {
-          console.log(formatLogMessage('debug', `[realtime_cleanup] Using extended delay for Cloudflare site: ${totalDelay}ms (${siteDelay}ms + ${bufferTime}ms CF buffer)`));
+          console.log(formatLogMessage('debug', `${REALTIME_CLEANUP_TAG} Using extended delay for Cloudflare site: ${totalDelay}ms (${siteDelay}ms + ${bufferTime}ms CF buffer)`));
         }
         
         const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, totalDelay);
         if (realtimeResult.success && realtimeResult.closedCount > 0 && forceDebug) {
-          console.log(formatLogMessage('debug', `[realtime_cleanup] Cleaned ${realtimeResult.closedCount} old pages, ${realtimeResult.remainingPages} remaining`));
+          console.log(formatLogMessage('debug', `${REALTIME_CLEANUP_TAG} Cleaned ${realtimeResult.closedCount} old pages, ${realtimeResult.remainingPages} remaining`));
         }
       } 
     
@@ -2052,7 +2229,7 @@ function setupFrameHandling(page, forceDebug) {
       // Aggressive timeouts prevent hanging in Puppeteer 23.x while maintaining speed
       
       page.on('console', (msg) => {
-        if (forceDebug && msg.type() === 'error') console.log(`[debug] Console error: ${msg.text()}`);
+        if (forceDebug && msg.type() === 'error') console.log(formatLogMessage('debug', `Console error: ${msg.text()}`));
       });
       
       // Add page crash handler
@@ -2113,6 +2290,11 @@ function setupFrameHandling(page, forceDebug) {
         const flowproxyTimeouts = getFlowProxyTimeouts(siteConfig);
         page.setDefaultTimeout(Math.min(flowproxyTimeouts.pageTimeout, TIMEOUTS.DEFAULT_NAVIGATION));
         page.setDefaultNavigationTimeout(Math.min(flowproxyTimeouts.navigationTimeout, TIMEOUTS.DEFAULT_PAGE));
+        // Attach the response/header listener BEFORE navigation so the
+        // document response's own headers (Server, Set-Cookie, X-FlowProxy-*,
+        // etc.) are observed. The listener accumulates state in a WeakMap
+        // keyed by page; analyzeFlowProxyProtection reads from it later.
+        attachFlowProxyHeaderListener(page);
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Applied flowProxy timeouts - page: ${flowproxyTimeouts.pageTimeout}ms, nav: ${flowproxyTimeouts.navigationTimeout}ms`));
         }
@@ -2131,9 +2313,9 @@ function setupFrameHandling(page, forceDebug) {
       if (shouldInjectEvalForPage) {
           if (forceDebug) {
               if (globalEvalOnDoc) {
-                  console.log(formatLogMessage('debug', `[evalOnDoc] Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
+                  console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
               } else { // siteConfig.evaluateOnNewDocument must be true
-                  console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
+                  console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
               }
           }
           
@@ -2154,7 +2336,7 @@ function setupFrameHandling(page, forceDebug) {
               browserResponsive = true;
           } catch (healthErr) {
               if (forceDebug) {
-                  console.log(formatLogMessage('debug', `[evalOnDoc] Browser health check failed: ${healthErr.message}`));
+                  console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Browser health check failed: ${healthErr.message}`));
               }
               browserResponsive = false;
           }
@@ -2253,7 +2435,7 @@ function setupFrameHandling(page, forceDebug) {
                   ]);
                   evalOnDocSuccess = true;
                   if (forceDebug) {
-                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
+                      console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Full injection successful for ${currentUrl}`));
                   }
               } catch (fullInjectionErr) {
                   // Enhanced error detection for CDP issues
@@ -2264,12 +2446,12 @@ function setupFrameHandling(page, forceDebug) {
                   
                   if (forceDebug) {
                       const errorType = isCDPError ? 'CDP/Protocol error' : 'timeout/other';
-                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed (${errorType}): ${fullInjectionErr.message}`));
+                      console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Full injection failed (${errorType}): ${fullInjectionErr.message}`));
                   }
 
                   // Skip fallback for CDP errors - they indicate browser communication issues
                   if (isCDPError) {
-                      console.warn(formatLogMessage('warn', `[evalOnDoc] CDP communication failure - skipping injection for ${currentUrl}`));
+                      console.warn(formatLogMessage('warn', `${EVAL_ON_DOC_TAG} CDP communication failure - skipping injection for ${currentUrl}`));
                       evalOnDocSuccess = false;
                   } else {
                   
@@ -2316,11 +2498,11 @@ function setupFrameHandling(page, forceDebug) {
                       ]);
                       evalOnDocSuccess = true;
                       if (forceDebug) {
-                          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection successful for ${currentUrl}`));
+                          console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Minimal injection successful for ${currentUrl}`));
                       }
                   } catch (minimalInjectionErr) {
                       if (forceDebug) {
-                          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection also failed: ${minimalInjectionErr.message}`));
+                          console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Minimal injection also failed: ${minimalInjectionErr.message}`));
                       }
                       evalOnDocSuccess = false;
                   }
@@ -2328,14 +2510,14 @@ function setupFrameHandling(page, forceDebug) {
            } 
           } else {
               if (forceDebug) {
-                  console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
+                  console.log(formatLogMessage('debug', `${EVAL_ON_DOC_TAG} Browser unresponsive, skipping injection for ${currentUrl}`));
               }
               evalOnDocSuccess = false;
           }
           
           // Final status logging
           if (!evalOnDocSuccess) {
-              console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
+              console.warn(formatLogMessage('warn', `${EVAL_ON_DOC_TAG} All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
           }
       // Allow realtime cleanup to proceed after injection completes
       if (shouldInjectEvalForPage && siteConfig.window_cleanup === "realtime") {
@@ -2364,7 +2546,7 @@ function setupFrameHandling(page, forceDebug) {
             }
           }, { selectors: cssBlockedSelectors });
         } catch (cssErr) {
-          console.warn(formatLogMessage('warn', `[css_blocked] Failed to set up CSS element blocking for ${currentUrl}: ${cssErr.message}`));
+          console.warn(formatLogMessage('warn', `${CSS_BLOCKED_TAG} Failed to set up CSS element blocking for ${currentUrl}: ${cssErr.message}`));
         }
       }
       // --- END: CSS Element Blocking Setup ---
@@ -2421,7 +2603,7 @@ function setupFrameHandling(page, forceDebug) {
           const clearResult = await clearSiteData(page, currentUrl, forceDebug);
           if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data for ${currentUrl}`));
         } catch (clearErr) {
-          if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Failed for ${currentUrl}: ${clearErr.message}`));
+          if (forceDebug) console.log(formatLogMessage('debug', `${CLEAR_SITEDATA_TAG} Failed for ${currentUrl}: ${clearErr.message}`));
         }
       }
 
@@ -2438,6 +2620,29 @@ function setupFrameHandling(page, forceDebug) {
         } else if (forceDebug) {
           console.log(formatLogMessage('debug', `Skipping fingerprint injection — Obscura provides built-in stealth`));
         }
+
+        // Neutralize the Fullscreen API before any page script runs so a
+        // site can't force the real browser window fullscreen in --headful
+        // (or trip an anti-bot check that reads document.fullscreenElement).
+        // requestFullscreen is stubbed to a resolved no-op — which is also
+        // how browsers already behave when it's called without a user
+        // gesture, so this looks normal, not automated. fullscreenElement
+        // stays null naturally since we never enter fullscreen.
+        if (!allowFullscreen) {
+          try {
+            await page.evaluateOnNewDocument(() => {
+              const noop = function () { return Promise.resolve(); };
+              const legacyNoop = function () {};
+              try { Element.prototype.requestFullscreen = noop; } catch (_) {}
+              try { Element.prototype.webkitRequestFullscreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.webkitRequestFullScreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.mozRequestFullScreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.msRequestFullscreen = legacyNoop; } catch (_) {}
+            });
+          } catch (fsErr) {
+            if (forceDebug) console.log(formatLogMessage('debug', `Fullscreen neutralization injection failed: ${fsErr.message}`));
+          }
+        }
         
         // Client Hints protection for Chrome user agents (skipped under Obscura — it sets its own)
         if (!useObscura && siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
@@ -2624,19 +2829,41 @@ function setupFrameHandling(page, forceDebug) {
     });
   }
 
-      const blockedRegexes = Array.isArray(siteConfig.blocked)
-        ? siteConfig.blocked.map(pattern => getCompiledRegex(pattern))
-        : [];
+      // Per-site blocked compile -- helper warns on bad patterns instead of
+      // throwing out of processUrl and breaking that site's scan.
+      const blockedRegexes = compilePatternList(`blocked (site: ${siteConfig.url || 'unknown'})`, siteConfig.blocked, getCompiledRegex);
+
+      // Per-site escape hatch: disable_adblock turns off the two layers of
+      // "global" ad-blocking for this URL — the adblock-rs filter-list engine
+      // and the globalBlockedRegexes pattern list. Per-site siteConfig.blocked
+      // is preserved (it's an explicit per-site choice, not "global" blocking).
+      //
+      // The use case: capture_popups + popunder/redirect chains. The global
+      // adblock often aborts the exact requests that fire the popup or chain
+      // to the tracker, defeating capture. Setting disable_adblock: true for
+      // those specific URLs lets the chain play out naturally so the popup
+      // request listener can observe the full hop sequence.
+      const disableAdblock = siteConfig.disable_adblock === true;
 
       // Pre-build Set for O(1) resourceType lookups (fired per request)
       const allowedResourceTypesSet = Array.isArray(siteConfig.resourceTypes)
         ? new Set(siteConfig.resourceTypes)
         : null;
-		
-      // Combine site-specific with pre-compiled global blocked patterns
-      const allBlockedRegexes = blockedRegexes.length > 0
-        ? [...blockedRegexes, ...globalBlockedRegexes]
-        : globalBlockedRegexes; // Avoid spread when no site-specific patterns
+
+      // Combine site-specific with pre-compiled global blocked patterns.
+      // When disable_adblock is true, globalBlockedRegexes is omitted so
+      // only the per-site list applies.
+      const allBlockedRegexes = disableAdblock
+        ? blockedRegexes
+        : (blockedRegexes.length > 0
+            ? [...blockedRegexes, ...globalBlockedRegexes]
+            : globalBlockedRegexes); // Avoid spread when no site-specific patterns
+
+      if (disableAdblock && forceDebug) {
+        const dropped = globalBlockedRegexes.length;
+        const adblockNote = adblockEnabled && adblockMatcher ? ' + adblock-rs engine' : '';
+        console.log(formatLogMessage('debug', `[adblock] disable_adblock=true for ${currentUrl} — skipping ${dropped} global blocked patterns${adblockNote} (site-level ${blockedRegexes.length} pattern(s) still apply)`));
+      }
 
       /**
        * Helper function to add domain to matched collection
@@ -2663,7 +2890,7 @@ function setupFrameHandling(page, forceDebug) {
            const cachedSimilarity = smartCache.getCachedSimilarity(domain, existingDomain);
            if (cachedSimilarity !== null && cachedSimilarity >= similarityThreshold) {
              if (forceDebug) {
-               console.log(formatLogMessage('debug', `[SmartCache] Used cached similarity: ${domain} ~= ${existingDomain} (${cachedSimilarity}%)`));
+               console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Used cached similarity: ${domain} ~= ${existingDomain} (${cachedSimilarity}%)`));
              }
              return; // Skip adding this domain
            }
@@ -2687,7 +2914,7 @@ function setupFrameHandling(page, forceDebug) {
        
        if (smartCache && smartCache.shouldSkipDomain(domain, context)) {
          if (forceDebug) {
-           console.log(formatLogMessage('debug', `[SmartCache] Skipping cached domain: ${domain}`));
+           console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Skipping cached domain: ${domain}`));
          }
          return; // Skip adding this domain
        }
@@ -2705,7 +2932,7 @@ function setupFrameHandling(page, forceDebug) {
          
          if (similarCheck.shouldIgnore) {
            if (forceDebug) {
-             console.log(formatLogMessage('debug', `[ignore_similar] Skipping ${domain}: ${similarCheck.reason}`));
+             console.log(formatLogMessage('debug', `${IGNORE_SIMILAR_TAG} Skipping ${domain}: ${similarCheck.reason}`));
            }
            return; // Skip adding this domain
          }
@@ -2721,7 +2948,7 @@ function setupFrameHandling(page, forceDebug) {
         
         if (ignoredSimilarCheck.shouldIgnore) {
           if (forceDebug) {
-            console.log(formatLogMessage('debug', `[ignore_similar_ignored_domains] Skipping ${domain}: ${ignoredSimilarCheck.reason} (similar to ignoreDomains)`));
+            console.log(formatLogMessage('debug', `${IGNORE_SIMILAR_IGNORED_DOMAINS_TAG} Skipping ${domain}: ${ignoredSimilarCheck.reason} (similar to ignoreDomains)`));
           }
           return; // Skip adding this domain
         }
@@ -2742,7 +2969,7 @@ function setupFrameHandling(page, forceDebug) {
     }
   } catch (cacheErr) {
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[SmartCache] Error marking domain: ${cacheErr.message}`));
+      console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Error marking domain: ${cacheErr.message}`));
     }
   }
       }
@@ -2760,6 +2987,247 @@ function setupFrameHandling(page, forceDebug) {
         }
       }
 
+      // === POPUP CAPTURE (opt-in via siteConfig.capture_popups: true) ===
+      // Many ad networks fire popunders / new-tab opens (window.open, target=
+      // "_blank") that navigate to trackers and disappear from view. Those
+      // pages are SEPARATE Puppeteer targets — page.on('request', ...) on the
+      // main page never sees their network traffic.
+      //
+      // IMPORTANT: modern Chromium blocks programmatic window.open() unless
+      // it's triggered by a real user gesture. In practice that means
+      // capture_popups only catches anything when the scanner is actually
+      // clicking on the page — i.e., the site config also has
+      // `interact: true` AND `interact_clicks: true`. Setting capture_popups
+      // alone will register the listener but no popups will fire.
+      //
+      // When capture_popups is true, we attach a browser-level 'targetcreated'
+      // listener for THIS URL only. New page targets whose opener-chain leads
+      // back to our main page (within maxDepth levels) get a stripped-down
+      // request listener — same regex/first-party/ignoreDomains filter as
+      // the main handler, same addMatchedDomain() sink, same domain
+      // detection cache, same nettools/similarity logic (all inherited via
+      // addMatchedDomain). Cloudflare bypass, adblock-rs matching, curl/grep
+      // content download, and request.abort() are intentionally skipped on
+      // popups — they're observation-only.
+      //
+      // Each popup's request listener stays attached across in-window
+      // navigations, so a single popup that redirects A -> B -> C captures
+      // every hop. The capture window (default 5s, configurable per-site
+      // via capture_popups_window_ms) is the wall-clock budget for that
+      // chain — bump it for long redirect chains, lower it for high-popup-
+      // rate sites where memory pressure matters more than chain coverage.
+      const capturePopups = siteConfig.capture_popups === true;
+      // Per-site overrides (with sane defaults). Parsed as numbers so config
+      // values from JSON come through correctly; falsy / non-positive values
+      // fall back to the default rather than silently disabling capture.
+      const POPUP_MAX_DEPTH = (() => {
+        const v = parseInt(siteConfig.capture_popups_max_depth, 10);
+        return Number.isFinite(v) && v > 0 ? v : 2;
+      })();
+      const POPUP_CAPTURE_WINDOW_MS = (() => {
+        const v = parseInt(siteConfig.capture_popups_window_ms, 10);
+        return Number.isFinite(v) && v > 0 ? v : 5000;
+      })();
+
+      if (capturePopups && forceDebug) {
+        // One-time setup-time warning if the click prerequisite isn't met.
+        // Without clicks, capture_popups is a no-op in practice.
+        const hasClicks = siteConfig.interact === true && siteConfig.interact_clicks === true;
+        if (!hasClicks) {
+          console.log(formatLogMessage('debug', `[popup] capture_popups is enabled but interact_clicks is not — popups need user-gesture clicks to fire; expect no captures unless the page opens popups via in-page redirects`));
+        }
+        console.log(formatLogMessage('debug', `[popup] capture_popups settings: maxDepth=${POPUP_MAX_DEPTH}, windowMs=${POPUP_CAPTURE_WINDOW_MS}`));
+      }
+
+      if (capturePopups) {
+        const mainTarget = page.target();
+
+        // Walk target.opener() chain to find depth relative to mainTarget.
+        // Returns 0 if the target isn't a descendant of mainTarget at all,
+        // 1 for a direct popup of the main page, 2 for popup-of-popup, etc.
+        const getPopupDepth = (target) => {
+          let depth = 0;
+          let cur = target.opener();
+          while (cur && depth <= POPUP_MAX_DEPTH + 1) {
+            depth++;
+            if (cur === mainTarget) return depth;
+            cur = cur.opener();
+          }
+          return 0;
+        };
+
+        // Attach observation-only request listener to a popup page. No
+        // setRequestInterception(true) — page.on('request') fires for every
+        // request regardless of interception state, and we don't need to
+        // block anything on popups.
+        const attachPopupRequestCapture = (popupPage, depth) => {
+          popupPage.on('request', (request) => {
+            try {
+              const checkedUrl = request.url();
+              let fullSubdomain = '';
+              let checkedRootDomain = '';
+              try {
+                const parsedUrl = new URL(checkedUrl);
+                fullSubdomain = parsedUrl.hostname;
+                const pslResult = psl.parse(fullSubdomain);
+                checkedRootDomain = pslResult.domain || fullSubdomain;
+              } catch (_) { return; }
+              if (!checkedRootDomain) return;
+
+              // ignoreDomainsByUrl — if any pattern matches this popup URL,
+              // mark the root domain as ignored for the rest of the scan
+              // (main page + all popups). Mirrors the main handler so a
+              // tracker URL surfaced via popup chain has the same dampening
+              // effect as one surfaced on the main page.
+              if (_ignoreDomainsByUrlRegexes.length > 0 && !_dynamicallyIgnoredDomains.has(checkedRootDomain)) {
+                for (let i = 0; i < _ignoreDomainsByUrlRegexes.length; i++) {
+                  if (_ignoreDomainsByUrlRegexes[i].test(checkedUrl)) {
+                    _dynamicallyIgnoredDomains.add(checkedRootDomain);
+                    if (forceDebug) {
+                      console.log(formatLogMessage('debug', `${IGNORE_DOMAINS_BY_URL_TAG} ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
+                    }
+                    break;
+                  }
+                }
+              }
+
+              // blockDomainsByUrl trigger — symmetric to ignoreDomainsByUrl
+              // above; populating the dynamic block Set from popup URLs lets
+              // tracker URLs surfaced via popup chains poison their root
+              // domain for the rest of the scan just like main-page hits do.
+              if (_blockDomainsByUrlRegexes.length > 0 && !_dynamicallyBlockedDomains.has(checkedRootDomain)) {
+                for (let i = 0; i < _blockDomainsByUrlRegexes.length; i++) {
+                  if (_blockDomainsByUrlRegexes[i].test(checkedUrl)) {
+                    _dynamicallyBlockedDomains.add(checkedRootDomain);
+                    if (forceDebug) {
+                      console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} ${checkedRootDomain} blocked — matched pattern: ${_blockDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
+                    }
+                    break;
+                  }
+                }
+              }
+
+              // ignoreDomains gate (global; matchesIgnoreDomain also short-
+              // circuits on _dynamicallyIgnoredDomains, so a domain we just
+              // added above will be caught here on the same request).
+              if (matchesIgnoreDomain(checkedRootDomain, ignoreDomains)) return;
+
+              // Dynamic-block gate for popup requests — early return on
+              // matched root or any parent (parent-walk in
+              // matchesDynamicBlock). Popups don't have a request object
+              // available here, so we just return rather than abort; the
+              // popup-request observer treats this as "don't process".
+              if (matchesDynamicBlock(checkedRootDomain)) return;
+
+              // First-party / third-party gate (popup belongs to the main URL's
+              // domain group — its OWN URL doesn't redefine first-party).
+              const isFirstParty = firstPartyDomains.has(checkedRootDomain);
+              if (siteConfig.firstParty === false && isFirstParty) return;
+              if (siteConfig.thirdParty === false && !isFirstParty) return;
+
+              // Regex match against the site's filterRegex list
+              const resourceType = request.resourceType();
+              let regexMatched = false;
+              for (const re of regexes) {
+                if (re.test(checkedUrl)) {
+                  regexMatched = true;
+                  if (forceDebug) {
+                    console.log(formatLogMessage('debug', `[popup depth=${depth}] Matched ${checkedRootDomain} via ${re} (${resourceType})`));
+                  }
+                  break;
+                }
+              }
+
+              if (!regexMatched) return;
+
+              // hasNetTools is the same flag the main handler uses (line ~2639).
+              // When the site config carries whois/dig terms, regex match is
+              // not sufficient by itself — the URL must ALSO pass the whois/
+              // dig validation before it counts. Mirrors the main handler's
+              // behavior so 'capture popup domains that match regex/dig/whois'
+              // means the same thing for popups as for the main page.
+              if (hasNetTools) {
+                const popupNetToolsHandler = createNetToolsHandler({
+                  whoisTerms, whoisOrTerms,
+                  processedWhoisDomains: globalProcessedWhoisDomains,
+                  processedDigDomains: globalProcessedDigDomains,
+                  whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
+                  whoisServer,
+                  whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
+                  debugLogFile,
+                  digTerms, digOrTerms, digRecordType,
+                  digSubdomain: siteConfig.dig_subdomain === true,
+                  dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
+                  matchedDomains, addMatchedDomain,
+                  isDomainAlreadyDetected: isLocallyDetected,
+                  onWhoisResult: smartCache ? (domain, result) => smartCache.cacheNetTools(domain, 'whois', result) : undefined,
+                  onDigResult: smartCache ? (domain, result, recordType) => smartCache.cacheNetTools(domain, 'dig', result, recordType) : undefined,
+                  cachedWhois: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'whois') : null,
+                  cachedDig: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'dig', digRecordType) : null,
+                  currentUrl, getRootDomain, siteConfig, dumpUrls, matchedUrlsLogFile, forceDebug, fs,
+                  ignoreDomains, matchesIgnoreDomain
+                });
+                setImmediate(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
+              } else {
+                // No nettools required — regex match alone counts.
+                addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
+              }
+            } catch (_) { /* observation-only — never let a popup error escape */ }
+          });
+        };
+
+        const onTargetCreated = async (target) => {
+          // Short-circuit guard: if finally has already started, don't attach
+          // a request listener whose closure would outlive its meaningful
+          // scope. The race is narrow (a targetcreated firing while we're
+          // mid-await on target.page() across the finally boundary), but
+          // without this guard a late popup could push matches into
+          // matchedDomains for a URL whose processing has already returned.
+          if (urlFinished) return;
+          if (target.type() !== 'page') return;
+          const depth = getPopupDepth(target);
+          if (depth < 1) return; // Not one of ours
+          if (depth > POPUP_MAX_DEPTH) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `[popup] Skipping depth-${depth} popup (max=${POPUP_MAX_DEPTH}): ${target.url() || 'about:blank'}`));
+            }
+            return;
+          }
+
+          let popupPage;
+          try { popupPage = await target.page(); } catch (_) { return; }
+          if (!popupPage) return;
+          // Re-check after the await — the per-URL finally may have flipped
+          // the flag while target.page() was resolving.
+          if (urlFinished) {
+            try { if (!popupPage.isClosed()) popupPage.close().catch(() => {}); } catch (_) {}
+            return;
+          }
+
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[popup depth=${depth}] Capturing popup: ${target.url() || 'about:blank'}`));
+          }
+
+          attachPopupRequestCapture(popupPage, depth);
+
+          // Auto-close after the capture window so popups don't pile up.
+          const closeTimer = setTimeout(() => {
+            try { if (!popupPage.isClosed()) popupPage.close().catch(() => {}); } catch (_) {}
+          }, POPUP_CAPTURE_WINDOW_MS);
+          if (typeof closeTimer.unref === 'function') closeTimer.unref();
+
+          popupCleanups.push(() => {
+            clearTimeout(closeTimer);
+            try { if (!popupPage.isClosed()) popupPage.close().catch(() => {}); } catch (_) {}
+          });
+        };
+
+        browser.on('targetcreated', onTargetCreated);
+        popupCleanups.push(() => {
+          try { browser.off('targetcreated', onTargetCreated); } catch (_) {}
+        });
+      }
+
       // --- page.on('request', ...) Handler: Core Network Request Logic ---
       // This handler is triggered for every network request made by the page.
       // It decides whether to allow, block, or process the request based on:
@@ -2820,15 +3288,17 @@ function setupFrameHandling(page, forceDebug) {
           console.log(formatLogMessage('debug', `${messageColors.highlight('[req]')}[frame: ${isMainFrame ? 'main' : 'iframe'}] ${debugFrameUrl} → ${checkedUrl}`));
         }
 
-        // Apply adblock rules BEFORE expensive regex checks for better performance
-        if (adblockEnabled && adblockMatcher) {
+        // Apply adblock-rs filter-list rules BEFORE expensive regex checks
+        // for better performance. Gated on !disableAdblock so per-URL configs
+        // (e.g. for popup/redirect chain capture) can bypass it.
+        if (!disableAdblock && adblockEnabled && adblockMatcher) {
           try {
             const result = adblockMatcher.shouldBlock(
               checkedUrl,
               currentUrl,
               request.resourceType()
             );
-            
+
             if (result.blocked) {
               adblockStats.blocked++;
               if (forceDebug) {
@@ -2862,13 +3332,42 @@ function setupFrameHandling(page, forceDebug) {
             if (_ignoreDomainsByUrlRegexes[i].test(reqUrl)) {
               _dynamicallyIgnoredDomains.add(checkedRootDomain);
               if (forceDebug) {
-                console.log(formatLogMessage('debug', `[ignoreDomainsByUrl] ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source}`));
+                console.log(formatLogMessage('debug', `${IGNORE_DOMAINS_BY_URL_TAG} ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source}`));
               }
               break;
             }
           }
         }
 
+        // blockDomainsByUrl trigger — symmetric to ignoreDomainsByUrl above.
+        // If any pattern matches this URL, mark the root domain as blocked
+        // for the rest of the scan. The gate immediately below catches the
+        // triggering request itself + any future request on this domain or
+        // its subdomains (parent-walk via matchesDynamicBlock).
+        if (_blockDomainsByUrlRegexes.length > 0 && checkedRootDomain && !_dynamicallyBlockedDomains.has(checkedRootDomain)) {
+          for (let i = 0; i < _blockDomainsByUrlRegexes.length; i++) {
+            if (_blockDomainsByUrlRegexes[i].test(reqUrl)) {
+              _dynamicallyBlockedDomains.add(checkedRootDomain);
+              if (forceDebug) {
+                console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} ${checkedRootDomain} blocked — matched pattern: ${_blockDomainsByUrlRegexes[i].source}`));
+              }
+              break;
+            }
+          }
+        }
+        // blockDomainsByUrl gate — abort if reqDomain (or a parent) is in
+        // the dynamic block Set. Fires BEFORE the static blocked-regex
+        // check so domain-based blocks short-circuit without paying the
+        // per-URL regex scan. Same abort reason as the static path so
+        // request.failure() observers see consistent metadata.
+        if (reqDomain && _dynamicallyBlockedDomains.size > 0 && matchesDynamicBlock(reqDomain)) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} aborting ${reqUrl} (domain ${reqDomain} dynamically blocked)`));
+          }
+          request.abort('blockedbyclient');
+          return;
+        }
+
         let blockedMatchIndex = -1;
         for (let i = 0; i < allBlockedRegexes.length; i++) {
           if (allBlockedRegexes[i].test(reqUrl)) {
@@ -2877,8 +3376,16 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
         if (blockedMatchIndex !== -1) {
+          // Always track the hit (zero-cost on the un-debug path) so the
+          // scan-end summary can show which patterns are doing work vs.
+          // which are stale and ready to prune. Keyed by pattern.source --
+          // identical patterns from site + global lists roll up together,
+          // which matches how users think about them.
+          const matchedPatternSrc = allBlockedRegexes[blockedMatchIndex].source;
+          _blockedPatternHits.set(matchedPatternSrc, (_blockedPatternHits.get(matchedPatternSrc) || 0) + 1);
+
           if (forceDebug) {
-            const matchedPattern = allBlockedRegexes[blockedMatchIndex].source;
+            const matchedPattern = matchedPatternSrc;
             const patternSource = blockedMatchIndex < blockedRegexes.length ? 'site' : 'global';
             console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked]')}[${simplifiedCurrentUrl}] ${reqUrl} blocked by ${patternSource} pattern: ${matchedPattern}`));
             
@@ -2950,6 +3457,19 @@ function setupFrameHandling(page, forceDebug) {
           return;
         }
 
+        // Early ignoreDomains gate — skip regex + dig/whois entirely for domains
+        // in the ignoreDomains list (or dynamically-ignored ones populated by
+        // ignoreDomainsByUrl above). Mirrors the popup handler's early gate so
+        // the main path doesn't waste a dig/whois lookup on domains that
+        // post-processing/output filters will strip anyway.
+        if (matchesIgnoreDomain(reqDomain, ignoreDomains)) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `Skipping ignoreDomains match: ${reqDomain}`));
+          }
+          request.continue();
+          return;
+        }
+
         // === ENHANCED REGEX MATCHING WITH AND/OR LOGIC ===
         let regexMatched = false;
         let matchedRegexPattern = null;
@@ -3046,9 +3566,11 @@ function setupFrameHandling(page, forceDebug) {
                 dumpUrls,
                 matchedUrlsLogFile,
                 forceDebug,
-                fs
+                fs,
+                ignoreDomains,
+                matchesIgnoreDomain
               });
-              
+
               // Execute nettools check asynchronously
               const originalDomain = fullSubdomain;
               setImmediate(() => netToolsHandler(reqDomain, originalDomain));
@@ -3122,7 +3644,7 @@ function setupFrameHandling(page, forceDebug) {
              const cachedDig = smartCache ? smartCache.getCachedNetTools(reqDomain, 'dig', digRecordType) : null;
              
              if ((cachedWhois || cachedDig) && forceDebug) {
-               console.log(formatLogMessage('debug', `[SmartCache] Using cached nettools results for ${reqDomain}`));
+               console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Using cached nettools results for ${reqDomain}`));
              }
              
              // Create nettools handler with cache callbacks (if cache is enabled)
@@ -3159,9 +3681,11 @@ function setupFrameHandling(page, forceDebug) {
                dumpUrls,
                matchedUrlsLogFile,
                forceDebug,
-               fs
+               fs,
+               ignoreDomains,
+               matchesIgnoreDomain
              });
-             
+
              // Execute nettools check asynchronously
             const originalDomain = fullSubdomain; // Use full subdomain for nettools
             setImmediate(() => netToolsHandler(reqDomain, originalDomain));
@@ -3218,7 +3742,7 @@ function setupFrameHandling(page, forceDebug) {
              }
              
              if (cachedContent && forceDebug) {
-               console.log(formatLogMessage('debug', `[SmartCache] Using cached response content for ${reqUrl.substring(0, 50)}...`));
+               console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Using cached response content for ${reqUrl.substring(0, 50)}...`));
                // Process cached content instead of fetching
              } else {
              try {
@@ -3248,7 +3772,12 @@ function setupFrameHandling(page, forceDebug) {
                    forceDebug,
                    userAgent: curlUserAgent,
                    resourceType,
-                   hasSearchString: hasSearchString || hasSearchStringAnd,
+                   // Pass both flags separately — createGrepHandler now
+                   // applies AND logic when hasSearchStringAnd is set.
+                   // Previously OR'd into hasSearchString and the AND
+                   // patterns were silently dropped.
+                   hasSearchString,
+                   hasSearchStringAnd,
                    grepOptions: {
                      ignoreCase: true,
                      wholeWord: false,
@@ -3298,7 +3827,7 @@ function setupFrameHandling(page, forceDebug) {
             } else if (useGrep && (hasSearchString || hasSearchStringAnd)) {
              // Use grep with response handler (no curl)
              if (forceDebug) {
-               console.log(formatLogMessage('debug', `[grep-response] Queuing ${reqUrl} for grep analysis via response handler`));
+               console.log(formatLogMessage('debug', `${GREP_RESPONSE_TAG} Queuing ${reqUrl} for grep analysis via response handler`));
              }
              
              // Queue for grep processing via response handler
@@ -3386,7 +3915,7 @@ function setupFrameHandling(page, forceDebug) {
               }
             }, cssBlockedSelectors);
           } catch (cssRuntimeErr) {
-            console.warn(formatLogMessage('warn', `[css_blocked] Failed to apply runtime CSS blocking for ${currentUrl}: ${cssRuntimeErr.message}`));
+            console.warn(formatLogMessage('warn', `${CSS_BLOCKED_TAG} Failed to apply runtime CSS blocking for ${currentUrl}: ${cssRuntimeErr.message}`));
           }
         }
       }
@@ -3698,8 +4227,8 @@ function setupFrameHandling(page, forceDebug) {
             const proxyErr = proxyErrors.find(e => err.message.includes(e));
             if (proxyErr) {
               const info = getProxyInfo(siteConfig);
-              console.error(formatLogMessage('error', `[proxy] ${proxyErr} — proxy: ${info} — URL: ${currentUrl}`));
-              console.error(formatLogMessage('error', `[proxy] Check: is the proxy running? Are credentials correct? Is the target reachable from the proxy?`));
+              console.error(formatLogMessage('error', `${PROXY_TAG} ${proxyErr} — proxy: ${info} — URL: ${currentUrl}`));
+              console.error(formatLogMessage('error', `${PROXY_TAG} Check: is the proxy running? Are credentials correct? Is the target reachable from the proxy?`));
             }
           }
           console.error(formatLogMessage('error', `Failed on ${currentUrl}: ${err.message}`));
@@ -3751,7 +4280,7 @@ function setupFrameHandling(page, forceDebug) {
         try {
           if (ghostConfig) {
             // Ghost-cursor mode: Bezier-based mouse movements
-            if (forceDebug) console.log(formatLogMessage('debug', `[ghost-cursor] Using ghost-cursor for ${currentUrl}`));
+            if (forceDebug) console.log(formatLogMessage('debug', `${GHOST_CURSOR_TAG} Using ghost-cursor for ${currentUrl}`));
             const cursor = createGhostCursor(page, { forceDebug });
             if (cursor) {
               await Promise.race([
@@ -3789,8 +4318,7 @@ function setupFrameHandling(page, forceDebug) {
                     await performPageInteraction(page, currentUrl, {
                       ...interactionConfig,
                       mouseMovements: 0,
-                      includeElementClicks: false,
-                      includeTyping: false
+                      includeElementClicks: false
                     }, forceDebug);
                   }
                 })(),
@@ -3811,7 +4339,7 @@ function setupFrameHandling(page, forceDebug) {
             ]);
           }
         } catch (interactTimeoutErr) {
-          if (forceDebug) console.log(formatLogMessage('debug', `[interaction] Aborted after ${INTERACTION_HARD_TIMEOUT}ms: ${interactTimeoutErr.message}`));
+          if (forceDebug) console.log(formatLogMessage('debug', `${INTERACTION_TAG} Aborted after ${INTERACTION_HARD_TIMEOUT}ms: ${interactTimeoutErr.message}`));
         }
       })();
 
@@ -3946,7 +4474,7 @@ function setupFrameHandling(page, forceDebug) {
             const clearResult = await clearSiteData(page, currentUrl, forceDebug, true); // Quick mode for reloads
             if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data before reload #${i} for ${currentUrl}`));
           } catch (reloadClearErr) {
-            if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Before reload failed for ${currentUrl}`));
+            if (forceDebug) console.log(formatLogMessage('debug', `${CLEAR_SITEDATA_TAG} Before reload failed for ${currentUrl}`));
           }
         }
         
@@ -4140,8 +4668,8 @@ function setupFrameHandling(page, forceDebug) {
         const proxyErr = proxyErrors.find(e => err.message.includes(e));
         if (proxyErr) {
           const info = getProxyInfo(siteConfig);
-          console.error(formatLogMessage('error', `[proxy] ${proxyErr} — proxy: ${info} — URL: ${currentUrl}`));
-          console.error(formatLogMessage('error', `[proxy] Check: is the proxy running? Are credentials correct? Is the target reachable from the proxy?`));
+          console.error(formatLogMessage('error', `${PROXY_TAG} ${proxyErr} — proxy: ${info} — URL: ${currentUrl}`));
+          console.error(formatLogMessage('error', `${PROXY_TAG} Check: is the proxy running? Are credentials correct? Is the target reachable from the proxy?`));
         }
       }
 
@@ -4208,17 +4736,33 @@ function setupFrameHandling(page, forceDebug) {
       };
     } finally {
       // Guaranteed resource cleanup - this runs regardless of success or failure
-    
+
+      // Flip the popup-capture race-window guard first so any in-flight
+      // 'targetcreated' handler that resolves after this point sees the
+      // flag and bails (closing its own popup if it managed to fetch one).
+      urlFinished = true;
+
+      // Popup capture teardown (opt-in via siteConfig.capture_popups). Each
+      // entry is either the browser.off('targetcreated', ...) deregistration
+      // or a per-popup (clearTimeout + popupPage.close) cleanup. Iterate even
+      // if one fails so the rest still run.
+      if (popupCleanups.length) {
+        for (const cleanup of popupCleanups) {
+          try { cleanup(); } catch (_) {}
+        }
+        popupCleanups.length = 0;
+      }
+
       // Disconnect VPN for this site
       if (siteConfig.vpn) {
         const vpnDown = wgDisconnect(siteConfig, forceDebug);
         if (vpnDown.tornDown && forceDebug) {
-          console.log(formatLogMessage('debug', `[vpn] WireGuard interface torn down for ${currentUrl}`));
+          console.log(formatLogMessage('debug', `${VPN_TAG} WireGuard interface torn down for ${currentUrl}`));
         }
       } else if (siteConfig.openvpn) {
         const ovpnDown = ovpnDisconnect(siteConfig, forceDebug);
         if (ovpnDown.tornDown && forceDebug) {
-          console.log(formatLogMessage('debug', `[vpn] OpenVPN connection torn down for ${currentUrl}`));
+          console.log(formatLogMessage('debug', `${VPN_TAG} OpenVPN connection torn down for ${currentUrl}`));
         }
       }
 
@@ -4300,6 +4844,19 @@ function setupFrameHandling(page, forceDebug) {
   // Sort tasks so proxy groups are contiguous — direct connections first, then each proxy
   allTasks.sort((a, b) => proxyKeyFor(a.config).localeCompare(proxyKeyFor(b.config)));
 
+  // Pre-start local no-auth SOCKS5 relays for any authenticated socks5://
+  // upstreams. Done once here (the only async step) so getProxyArgs stays a
+  // sync lookup in the per-batch browser-launch path. Chromium can't auth
+  // SOCKS5; the relay does the upstream auth transparently.
+  try {
+    const relayCount = await prepareSocksRelays(sites, forceDebug);
+    if (relayCount > 0 && !silentMode) {
+      console.log(messageColors.processing(`Started ${relayCount} SOCKS5 auth relay(s)`));
+    }
+  } catch (relayErr) {
+    console.warn(formatLogMessage('proxy', `SOCKS5 relay setup failed: ${relayErr.message}`));
+  }
+
   let results = [];
   let processedUrlCount = 0;
   let urlsSinceLastCleanup = 0;
@@ -4320,7 +4877,13 @@ function setupFrameHandling(page, forceDebug) {
  let lastProcessedCount = 0;
  let hangCheckCount = 0;
  let forceRestartFlag = false; // Flag to trigger restart on next iteration
- 
+
+ // Precomputed colored '[HANG CHECK]' subsystem prefix. formatLogMessage
+ // only colors the [severity] tag; the '[HANG CHECK]' substring was
+ // sitting plain inside the message string. Colored once at function
+ // entry so the interval callback doesn't re-colorize per tick.
+ const HANG_CHECK_TAG = messageColors.processing('[HANG CHECK]');
+
  const hangDetectionInterval = setInterval(() => {
    // Progress check, counter, and forceRestartFlag MUST run regardless of
    // debug mode — previously the entire body was gated on forceDebug, which
@@ -4331,10 +4894,10 @@ function setupFrameHandling(page, forceDebug) {
    if (processedUrlCount === lastProcessedCount) {
      hangCheckCount++;
      if (forceDebug) {
-       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
+       console.log(formatLogMessage('warn', `${HANG_CHECK_TAG} No progress for ${hangCheckCount * 30}s`));
      }
      if (hangCheckCount >= 5) {
-       console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
+       console.log(formatLogMessage('error', `${HANG_CHECK_TAG} Hung for 2.5 minutes. Triggering emergency browser restart.`));
        forceRestartFlag = true; // Set flag instead of exiting
        hangCheckCount = 0; // Reset counter for next cycle
      }
@@ -4347,8 +4910,8 @@ function setupFrameHandling(page, forceDebug) {
    if (forceDebug) {
      const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
      const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
-     console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
-     console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
+     console.log(formatLogMessage('debug', `${HANG_CHECK_TAG} Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
+     console.log(formatLogMessage('debug', `${HANG_CHECK_TAG} URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
    }
  }, 30000);
  // Don't keep the event loop alive solely for the hang-check interval — the
@@ -4359,29 +4922,46 @@ function setupFrameHandling(page, forceDebug) {
  // Process URLs in batches with exception handling
  let siteGroupIndex = 0;
  let currentProxyKey = '';  // Track active proxy config — '' means direct connection
+ // Map of site-config object -> index in sites[], built once. Per-batch
+ // grouping below uses this for O(1) lookup instead of sites.indexOf which
+ // walked the array per task (batch=80 * sites=20 was ~1600 cmps per batch).
+ const configToIndex = new Map();
+ for (let i = 0; i < sites.length; i++) configToIndex.set(sites[i], i);
  try {
    for (let batchStart = 0; batchStart < totalUrls; batchStart += RESOURCE_CLEANUP_INTERVAL) {
     const batchEnd = Math.min(batchStart + RESOURCE_CLEANUP_INTERVAL, totalUrls);
     const currentBatch = allTasks.slice(batchStart, batchEnd);
 
-    
-    // Group tasks by their source site configuration for window cleanup
+
+    // Group tasks by their source site configuration for window cleanup.
+    // Single get-or-set replaces has + get + set (one Map lookup not two).
+    // The `?? -1` preserves the old `sites.indexOf` semantics for a task
+    // whose config isn't in sites[] — that case shouldn't happen, but if
+    // it ever does the routing stays identical to the prior code's
+    // 'site_-1' bucket rather than silently shifting to 'site_undefined'.
     const tasksBySite = new Map();
-    currentBatch.forEach(task => {
-      const siteKey = `site_${sites.indexOf(task.config)}`;
-      if (!tasksBySite.has(siteKey)) {
-        tasksBySite.set(siteKey, []);
-      }
-      tasksBySite.get(siteKey).push(task);
-    });
+    for (let i = 0; i < currentBatch.length; i++) {
+      const task = currentBatch[i];
+      const siteKey = `site_${configToIndex.get(task.config) ?? -1}`;
+      let arr = tasksBySite.get(siteKey);
+      if (!arr) tasksBySite.set(siteKey, arr = []);
+      arr.push(task);
+    }
     
     // IMPROVED: Only check health if we have indicators of problems
     let healthCheck = { shouldRestart: false, reason: null };
     const recentResults = results.slice(-8); // Check more results for better pattern detection
-    const recentFailureRate = recentResults.length > 0 ? 
-      recentResults.filter(r => !r.success).length / recentResults.length : 0;
+    // Single-pass count for both failure rate and critical-error tally —
+    // was two .filter(...).length calls allocating two intermediate arrays.
+    let recentFailures = 0, recentCritical = 0;
+    for (let i = 0; i < recentResults.length; i++) {
+      const r = recentResults[i];
+      if (!r.success) recentFailures++;
+      if (r.needsImmediateRestart) recentCritical++;
+    }
+    const recentFailureRate = recentResults.length > 0 ? recentFailures / recentResults.length : 0;
     const hasHighFailureRate = recentFailureRate > 0.75; // 75% failure threshold (more conservative)
-    const hasCriticalErrors = recentResults.filter(r => r.needsImmediateRestart).length > 2;
+    const hasCriticalErrors = recentCritical > 2;
     
     // Only run health checks when we have STRONG indicators of problems
     if (urlsSinceLastCleanup > 15 && (
@@ -4390,15 +4970,21 @@ function setupFrameHandling(page, forceDebug) {
         urlsSinceLastCleanup > RESOURCE_CLEANUP_INTERVAL * 0.9  // Very close to cleanup limit
     )) {
      try {
+       // Race the health check against a 30s timeout. Attach .catch on the
+       // health promise itself so that if the timeout wins, the still-running
+       // monitorBrowserHealth's eventual rejection doesn't surface as an
+       // unhandledRejection warning.
+       const healthPromise = monitorBrowserHealth(browser, {}, {
+         siteIndex: Math.floor(batchStart / RESOURCE_CLEANUP_INTERVAL),
+         totalSites: Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL),
+         urlsSinceCleanup: urlsSinceLastCleanup,
+         cleanupInterval: RESOURCE_CLEANUP_INTERVAL,
+         forceDebug,
+         silentMode
+       });
+       healthPromise.catch(() => {});
        healthCheck = await Promise.race([
-         monitorBrowserHealth(browser, {}, {
-           siteIndex: Math.floor(batchStart / RESOURCE_CLEANUP_INTERVAL),
-           totalSites: Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL),
-           urlsSinceCleanup: urlsSinceLastCleanup,
-           cleanupInterval: RESOURCE_CLEANUP_INTERVAL,
-           forceDebug,
-           silentMode
-         }),
+         healthPromise,
          new Promise((_, reject) => setTimeout(() => reject(new Error('Health check timeout')), 30000))
        ]);
      } catch (healthError) {
@@ -4427,8 +5013,17 @@ function setupFrameHandling(page, forceDebug) {
     // timeout) bypasses the urlsSinceLastCleanup > 8 gate — a confirmed hang
     // needs immediate restart even if we just cleaned up. Proactive triggers
     // keep the gate to prevent thrashing.
+    //
+    // hasHighFailureRate is computed (and still used for the health-check
+    // gate above) but intentionally NOT folded into proactiveRestart:
+    // wouldExceedLimit is always true at every batch boundary with the
+    // default RESOURCE_CLEANUP_INTERVAL == batch size, so the high-failure-
+    // rate branch was dead code reached only at the same boundary that
+    // wouldExceedLimit already triggers. If failure-rate ever needs to
+    // interrupt mid-cleanup-interval, that requires interrupting the
+    // running Promise.all — a real behavior change, not an OR addition.
     const hangRecoveryRestart = forceRestartFlag;
-    const proactiveRestart = (wouldExceedLimit || shouldRestartFromHealth || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8;
+    const proactiveRestart = (wouldExceedLimit || shouldRestartFromHealth) && urlsSinceLastCleanup > 8;
     if ((hangRecoveryRestart || proactiveRestart) && isNotLastBatch) {
       let restartReason = 'Unknown';
       if (forceRestartFlag) {
@@ -4436,8 +5031,6 @@ function setupFrameHandling(page, forceDebug) {
         forceRestartFlag = false; // Reset the flag
       } else if (shouldRestartFromHealth) {
         restartReason = healthCheck.reason;
-      } else if (hasHighFailureRate) {
-        restartReason = `High failure rate: ${Math.round(recentFailureRate * 100)}% in recent batch`;
       } else if (wouldExceedLimit) {
         restartReason = `Processed ${urlsSinceLastCleanup} URLs (scheduled maintenance)`;
       }
@@ -4452,7 +5045,7 @@ function setupFrameHandling(page, forceDebug) {
         if (requestCacheStats.enabled && requestCacheStats.size > 0) {
           const clearedCount = smartCache.clearRequestCache();
           if (forceDebug) {
-            console.log(formatLogMessage('debug', `[SmartCache] Cleared ${clearedCount} request cache entries during browser restart`));
+            console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Cleared ${clearedCount} request cache entries during browser restart`));
           }
         }
       }
@@ -4467,24 +5060,21 @@ function setupFrameHandling(page, forceDebug) {
         });
 
         // Clean up the specific user data directory
-        if (userDataDir && fs.existsSync(userDataDir)) {
-          fs.rmSync(userDataDir, { recursive: true, force: true });
-          if (forceDebug) console.log(formatLogMessage('debug', `Cleaned user data dir: ${userDataDir}`));
-        }
+        if (userDataDir) await cleanupUserDataDir(userDataDir, forceDebug);
 
         // Additional cleanup for any remaining Chrome processes
         if (removeTempFiles) {
-          await cleanupChromeTempFiles({ 
-            includeSnapTemp: true, 
+          await cleanupChromeTempFiles({
+            includeSnapTemp: true,
             forceDebug,
-            comprehensive: true 
+            comprehensive: true
           });
         }
 
       } catch (browserCloseErr) {
         if (forceDebug) console.log(formatLogMessage('debug', `Browser cleanup warning: ${browserCloseErr.message}`));
       }
-      
+
       // Create new browser for next batch (preserve current proxy config)
       const restartProxyArgs = currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : [];
       browser = await createBrowser(restartProxyArgs);
@@ -4492,7 +5082,6 @@ function setupFrameHandling(page, forceDebug) {
       
       // Reset cleanup counter and add delay
       urlsSinceLastCleanup = 0;
-      purgeStaleTrackers();
       await fastTimeout(TIMEOUTS.BROWSER_STABILIZE_DELAY);
     }
 
@@ -4512,9 +5101,7 @@ function setupFrameHandling(page, forceDebug) {
           forceDebug, timeout: 10000, exitOnFailure: false,
           cleanTempFiles: true, comprehensiveCleanup: removeTempFiles
         });
-        if (userDataDir && fs.existsSync(userDataDir)) {
-          fs.rmSync(userDataDir, { recursive: true, force: true });
-        }
+        if (userDataDir) await cleanupUserDataDir(userDataDir, forceDebug);
       } catch (proxyRestartErr) {
         if (forceDebug) console.log(formatLogMessage('debug', `Proxy switch browser cleanup: ${proxyRestartErr.message}`));
       }
@@ -4526,8 +5113,8 @@ function setupFrameHandling(page, forceDebug) {
         const health = await testProxy(currentBatch[0].config, 5000);
         if (!health.reachable) {
           const info = getProxyInfo(currentBatch[0].config);
-          console.error(formatLogMessage('error', `[proxy] Unreachable: ${info} — ${health.error}`));
-          console.error(formatLogMessage('error', `[proxy] Skipping ${currentBatch.length} URL(s) in this batch`));
+          console.error(formatLogMessage('error', `${PROXY_TAG} Unreachable: ${info} — ${health.error}`));
+          console.error(formatLogMessage('error', `${PROXY_TAG} Skipping ${currentBatch.length} URL(s) in this batch`));
           const skipResults = currentBatch.map(task => ({
             success: false, url: task.url, rules: [],
             error: `Proxy unreachable: ${health.error}`
@@ -4545,7 +5132,6 @@ function setupFrameHandling(page, forceDebug) {
       browser = await createBrowser(proxyArgs);
       currentProxyKey = batchProxyKey;
       urlsSinceLastCleanup = 0;
-      purgeStaleTrackers();
       await fastTimeout(TIMEOUTS.BROWSER_STABILIZE_DELAY);
     }
     
@@ -4555,7 +5141,7 @@ function setupFrameHandling(page, forceDebug) {
     
     // Log start of concurrent processing for hang detection
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
+      console.log(formatLogMessage('debug', `${CONCURRENCY_TAG} Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
  // Create tasks with timeout protection — skip domains that repeatedly timed out.
@@ -4567,7 +5153,7 @@ function setupFrameHandling(page, forceDebug) {
    try {
      // Short-circuit queued URLs once any URL in this batch has triggered a
      // restart. Without this, the 80-URL batch in the user's hang trace
-     // would have to fail one-by-one at 120s each (~28 min total) before
+     // would have to fail one-by-one at 75s each (~25 min total) before
      // the boundary restart could fire. Now: first hang fires the flag,
      // remaining queued URLs return immediately, batch completes, restart.
      if (forceRestartFlag) {
@@ -4580,25 +5166,111 @@ function setupFrameHandling(page, forceDebug) {
          if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
          return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
        }
+
+       // DNS pre-check — fails fast on NXDOMAIN/unresolvable hosts before
+       // we pay ~5-15s for Puppeteer navigation + Cloudflare detection.
+       // Skips IP literals. Respects an in-memory negative cache so a dead
+       // host hit by many URL paths only costs one DNS round-trip per TTL.
+       //
+       // Uses dns.resolve* (c-ares, async network I/O) NOT dns.lookup
+       // (getaddrinfo, libuv threadpool). Under scan concurrency Puppeteer
+       // saturates the default 4-slot threadpool with filesystem I/O, so
+       // dns.lookup calls sit queued and blow the timeout while never
+       // actually starting — wrongly skipping live domains. c-ares isn't
+       // threadpool-bound so it's immune to that contention.
+       if (dnsPrecheckEnabled && taskDomain && !/^[\d.:]+$|^\[/.test(taskDomain)) {
+         const cached = dnsNegativeCache.get(taskDomain);
+         if (cached && Date.now() - cached.timestamp < DNS_NEGATIVE_CACHE_TTL_MS) {
+           dnsPrecheckSkips++;
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check (cached): ${taskDomain} — ${cached.error}`));
+           return { url: task.url, rules: [], success: false, error: `DNS: ${cached.error}`, skipped: true };
+         }
+         // Positive-resolution shortcut: dig or whois has already proven this
+         // hostname live within their 20h cache TTL (populated either by an
+         // earlier URL this run or by --dns-cache disk-load from a prior run).
+         // Order matters -- negative cache (5min TTL, fresher data) wins
+         // first, then this 20h-TTL positive index, then the actual resolve.
+         if (domainKnownToResolve(taskDomain)) {
+           dnsPositiveSkips++;
+           dnsPositiveSkippedHosts.add(taskDomain);
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check skipped (dig/whois cache confirms resolution): ${taskDomain}`));
+           // Fall through to navigation -- pre-check "passed" by proxy.
+         } else {
+         const dnsResolve = async () => {
+           // resolve4 first; on no-IPv4 (ENODATA / ENOTFOUND) fall back to
+           // resolve6 so IPv6-only hosts aren't wrongly skipped. ANY OTHER
+           // error code (ESERVFAIL, ETIMEOUT, EREFUSED, etc.) propagates
+           // unchanged so the outer transient-retry path sees the real
+           // resolver code and the negative cache records the right reason.
+           // Previously a bare .catch swallowed everything and tried
+           // resolve6, which masked transient v4-side errors behind
+           // whatever resolve6 ended up reporting.
+           // 2s timeout kept as a real safety net — with c-ares off the
+           // threadpool it should now rarely fire.
+           let timer;
+           try {
+             const timeoutP = new Promise((_, reject) => {
+               timer = setTimeout(() => reject(new Error('DNS timeout')), dnsPrecheckTimeoutMs);
+             });
+             const resolveChain = dnsPromises.resolve4(taskDomain)
+               .catch(err => {
+                 if (err && (err.code === 'ENODATA' || err.code === 'ENOTFOUND')) {
+                   return dnsPromises.resolve6(taskDomain);
+                 }
+                 throw err;
+               });
+             await Promise.race([resolveChain, timeoutP]);
+           } finally {
+             if (timer) clearTimeout(timer);
+           }
+         };
+         // c-ares transient codes — retry once so a momentary resolver
+         // hiccup doesn't poison the negative cache for 5 minutes.
+         // DNS_TRANSIENT_ERRORS is module-level so we don't allocate per task.
+         try {
+           try {
+             await dnsResolve();
+           } catch (firstErr) {
+             const code = firstErr && firstErr.code;
+             if (DNS_TRANSIENT_ERRORS.has(code) || (firstErr && firstErr.message === 'DNS timeout')) {
+               if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check transient (${code || 'timeout'}) for ${taskDomain}, retrying once`));
+               await dnsResolve();
+             } else {
+               throw firstErr;
+             }
+           }
+         } catch (dnsErr) {
+           const errCode = dnsErr.code || dnsErr.message || 'DNS resolve failed';
+           dnsNegativeCacheSet(taskDomain, errCode);
+           dnsPrecheckSkips++;
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
+           return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+         }
+         } // close `else` from domainKnownToResolve shortcut above
+       }
      } catch {}
 
      // Per-URL timeout so a single hung processUrl can't block the batch
-     // forever. 120s is well past any legitimate slow page: Cloudflare
-     // adaptive max ~25s, nettools overall ~65s, navigation 15s.
+     // forever. 75s sits comfortably above the realistic legit-page ceiling
+     // (nav 35s + Cloudflare adaptive ~25s + interaction ~10s + network-idle
+     // wait ~10s ≈ ~70s), well short of the old 120s safety net. Cuts
+     // hang-recovery time roughly in half when an entire batch's URLs all
+     // hang and we're waiting on this timeout to advance processedUrlCount.
+     const PER_URL_TIMEOUT_MS = 75000;
      const processUrlPromise = processUrl(task.url, task.config, browser);
      let perUrlTimer;
      try {
        return await Promise.race([
          processUrlPromise,
          new Promise((_, reject) => {
-           perUrlTimer = setTimeout(() => reject(new Error('Per-URL timeout (120s)')), 120000);
+           perUrlTimer = setTimeout(() => reject(new Error('Per-URL timeout (75s)')), PER_URL_TIMEOUT_MS);
          })
        ]);
      } catch (err) {
-       if (err && err.message === 'Per-URL timeout (120s)') {
+       if (err && err.message === 'Per-URL timeout (75s)') {
          processUrlPromise.catch(() => {});
          forceRestartFlag = true;
-         return { url: task.url, rules: [], success: false, error: 'Per-URL timeout (120s)', needsImmediateRestart: true };
+         return { url: task.url, rules: [], success: false, error: 'Per-URL timeout (75s)', needsImmediateRestart: true };
        }
        throw err;
      } finally {
@@ -4614,21 +5286,29 @@ function setupFrameHandling(page, forceDebug) {
  
  let batchResults;
  try {
+   // Same orphan-promise pattern as the health-check race above: if the
+   // 10-min batch timeout wins, the still-running Promise.all keeps going
+   // until every batchTask settles. Each individual task is already wrapped
+   // in p-limit's error handling so unhandled rejections should not surface,
+   // but the .catch is free belt-and-braces against future refactors that
+   // change task internals.
+   const batchPromise = Promise.all(batchTasks);
+   batchPromise.catch(() => {});
    batchResults = await Promise.race([
-     Promise.all(batchTasks),
-     new Promise((_, reject) => 
+     batchPromise,
+     new Promise((_, reject) =>
        setTimeout(() => reject(new Error('Batch timeout')), 600000) // 10 min timeout
      )
    ]);
  } catch (timeoutError) {
    if (timeoutError.message.includes('timeout')) {
-     console.log(formatLogMessage('error', `[TIMEOUT] Batch hung. Restarting browser.`));
+     console.log(formatLogMessage('error', `${TIMEOUT_TAG} Batch hung. Restarting browser.`));
      try {
        await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false });
+       if (userDataDir) await cleanupUserDataDir(userDataDir, forceDebug);
        const timeoutProxyArgs = currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : [];
        browser = await createBrowser(timeoutProxyArgs);
        urlsSinceLastCleanup = 0;
-       purgeStaleTrackers();
      } catch (restartErr) {
        throw restartErr;
      }
@@ -4665,7 +5345,7 @@ function setupFrameHandling(page, forceDebug) {
     
     // Log completion of concurrent processing
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[CONCURRENCY] Completed ${batchSize} concurrent tasks, ${batchResults.filter(r => r.success).length} successful`));
+      console.log(formatLogMessage('debug', `${CONCURRENCY_TAG} Completed ${batchSize} concurrent tasks, ${batchResults.filter(r => r.success).length} successful`));
     }
 
     // Enhanced error reporting for Puppeteer 23.x
@@ -4727,7 +5407,7 @@ function setupFrameHandling(page, forceDebug) {
         if (requestCacheStats.enabled && requestCacheStats.size > 0) {
           const clearedCount = smartCache.clearRequestCache();
           if (forceDebug) {
-            console.log(formatLogMessage('debug', `[SmartCache] Cleared ${clearedCount} request cache entries during emergency restart`));
+            console.log(formatLogMessage('debug', `${SMART_CACHE_TAG} Cleared ${clearedCount} request cache entries during emergency restart`));
           }
         }
       }
@@ -4748,17 +5428,23 @@ function setupFrameHandling(page, forceDebug) {
         }
 
         await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false, cleanTempFiles: true, comprehensiveCleanup: removeTempFiles });
+        if (userDataDir) await cleanupUserDataDir(userDataDir, forceDebug);
         // Additional cleanup after emergency restart
         if (removeTempFiles) {
-          await cleanupChromeTempFiles({ 
-            includeSnapTemp: true, 
+          await cleanupChromeTempFiles({
+            includeSnapTemp: true,
             forceDebug,
-            comprehensive: true 
+            comprehensive: true
           });
         }
         browser = await createBrowser(currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : []);
         urlsSinceLastCleanup = 0; // Reset counter
-        purgeStaleTrackers();
+        // Reset the hang-detection flag too: this restart path is triggered
+        // by needsImmediateRestart errors, which the per-URL 75s timeout
+        // sets in lockstep with forceRestartFlag. Without this reset, the
+        // hang-fallback restart below would fire a SECOND back-to-back
+        // browser restart on the same batch boundary.
+        forceRestartFlag = false;
         await fastTimeout(TIMEOUTS.EMERGENCY_RESTART_DELAY); // Give browser time to stabilize
       } catch (emergencyRestartErr) {
         if (forceDebug) console.log(formatLogMessage('debug', `Emergency restart failed: ${emergencyRestartErr.message}`));
@@ -4769,9 +5455,9 @@ function setupFrameHandling(page, forceDebug) {
       console.log(`\n${messageColors.fileOp('🔄 Emergency hang detection restart:')} Browser appears hung, forcing restart`);
       try {
         await handleBrowserExit(browser, { forceDebug, timeout: 5000, exitOnFailure: false, cleanTempFiles: true });
+        if (userDataDir) await cleanupUserDataDir(userDataDir, forceDebug);
         browser = await createBrowser(currentProxyKey ? getProxyArgs(currentBatch[0].config, forceDebug) : []);
         urlsSinceLastCleanup = 0;
-        purgeStaleTrackers();
         forceRestartFlag = false; // Reset flag
         await fastTimeout(TIMEOUTS.EMERGENCY_RESTART_DELAY);
         if (forceDebug) console.log(formatLogMessage('debug', `Emergency hang detection restart completed`));
@@ -4820,11 +5506,11 @@ function setupFrameHandling(page, forceDebug) {
     if (requestCacheStats.enabled && requestCacheStats.size > 0) {
       const clearedCount = smartCache.clearRequestCache();
       if (!silentMode && clearedCount > 0) {
-        console.log(`\n🗑️  Cleared request cache: ${clearedCount} entries after JSON processing`);
+        console.log(`\n${messageColors.cleanup(`🗑️  Cleared request cache: ${clearedCount} entries after JSON processing`)}`);
       }
       if (forceDebug) {
         console.log(formatLogMessage('debug', 
-          `[SmartCache] Request cache cleared after JSON scan completion (hit rate: ${requestCacheStats.hitRate})`
+          `${SMART_CACHE_TAG} Request cache cleared after JSON scan completion (hit rate: ${requestCacheStats.hitRate})`
         ));
       }
     }
@@ -4896,6 +5582,43 @@ function setupFrameHandling(page, forceDebug) {
      if (cloudflareScanStats.errorPages > 0) {
        console.log(formatLogMessage('debug', `Cloudflare 5xx origin-error pages: ${cloudflareScanStats.errorPages} (no bypass possible — origin unreachable)`));
      }
+     if (dnsPrecheckEnabled && (dnsPrecheckSkips > 0 || dnsPositiveSkips > 0)) {
+       // Two skip mechanisms, each with its own counter + unique-host count:
+       //   - dnsPrecheckSkips:  URLs short-circuited via the NXDOMAIN-cache
+       //     (dnsNegativeCache). Unique-host count = dnsNegativeCache.size.
+       //   - dnsPositiveSkips:  URLs short-circuited via dig/whois cache
+       //     proof of resolution (knownResolvedHostnames index in nettools).
+       //     Unique-host count = dnsPositiveSkippedHosts.size (this Set is
+       //     populated only on actual skip events, not on every Set add in
+       //     nettools, so it's a true per-scan visibility metric).
+       const parts = [];
+       if (dnsPrecheckSkips > 0) {
+         parts.push(`${dnsPrecheckSkips} URL(s) via ${dnsNegativeCache.size} unresolvable host(s)`);
+       }
+       if (dnsPositiveSkips > 0) {
+         parts.push(`${dnsPositiveSkips} URL(s) via ${dnsPositiveSkippedHosts.size} resolved host(s)`);
+       }
+       console.log(formatLogMessage('debug', `DNS pre-check skipped: ${parts.join(', ')}`));
+     }
+     // Blocked-pattern hit stats. Surfaces which patterns are actually
+     // doing work this scan and (by absence) which are stale enough to
+     // prune from config. Top 10 by hit count to keep the log scannable
+     // on configs with dozens of patterns; full counts available via
+     // _blockedPatternHits if needed for tooling. Fires only when at
+     // least one pattern matched -- silent on scans with no blocks.
+     if (_blockedPatternHits.size > 0) {
+       let totalBlocks = 0;
+       for (const n of _blockedPatternHits.values()) totalBlocks += n;
+       console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked-stats]')} ${_blockedPatternHits.size} pattern(s) hit ${totalBlocks} time(s) total`));
+       const sorted = [..._blockedPatternHits.entries()].sort((a, b) => b[1] - a[1]);
+       const top = sorted.slice(0, 10);
+       for (const [pattern, hits] of top) {
+         console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked-stats]')}   ${hits.toString().padStart(6)} × ${pattern}`));
+       }
+       if (sorted.length > top.length) {
+         console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked-stats]')}   ... and ${sorted.length - top.length} more pattern(s)`));
+       }
+     }
      // Log smart cache statistics (if cache is enabled)
      // Adblock statistics
      if (adblockEnabled) {
@@ -5112,7 +5835,7 @@ function setupFrameHandling(page, forceDebug) {
   try { cleanupCloudflareCache(); } catch (_) {}
   try { wgDisconnectAll(forceDebug); } catch (_) {}
   try { ovpnDisconnectAll(forceDebug); } catch (_) {}
-  try { purgeStaleTrackers(); } catch (_) {}
+  try { await closeAllSocksRelays(forceDebug); } catch (_) {}
 
   // Clean process termination
   if (forceDebug) console.log(formatLogMessage('debug', `About to exit process...`));