@fanboynz/network-scanner 3.1.2 → 3.3.0

package/nwss.js

@@ -9,9 +9,9 @@ const fs = require('fs');
 const os = require('os');
 const psl = require('psl');
 const path = require('path');
-const dnsPromises = require('node:dns/promises');
+const { createRotatingResolver, createDnsCircuitBreaker, parseDnsServers, isNonExistenceError } = require('./lib/dns');
 const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
-const { compressMultipleFiles, formatFileSize } = require('./lib/compress');
+const { compressMultipleFiles } = require('./lib/compress');
 const { parseSearchStrings, createResponseHandler } = require('./lib/searchstring');
 const { applyAllFingerprintSpoofing, USER_AGENT_COLLECTIONS, CHROME_BUILD, CHROME_GREASE_BRAND } = require('./lib/fingerprint');
 const { formatRules, handleOutput, getFormatDescription } = require('./lib/output');
@@ -34,9 +34,7 @@ const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore
 // Graceful exit
 const { handleBrowserExit, cleanupChromeTempFiles, cleanupUserDataDir } = require('./lib/browserexit');
 // Whois & Dig
-const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats, domainKnownToResolve } = require('./lib/nettools');
-// File compare
-const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
+const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats, domainKnownToResolve, loadDiskCache, saveDiskCache, setDigResolvers } = require('./lib/nettools');
 // CDP functionality
 const { createCDPSession, createPageWithTimeout, setRequestInterceptionWithTimeout } = require('./lib/cdp');
 // Post-processing cleanup
@@ -57,6 +55,7 @@ const CSS_BLOCKED_TAG = messageColors.processing('[css_blocked]');
 const EVAL_ON_DOC_TAG = messageColors.processing('[evalOnDoc]');
 const REALTIME_CLEANUP_TAG = messageColors.processing('[realtime_cleanup]');
 const VPN_TAG = messageColors.processing('[vpn]');
+const POPUP_TAG = messageColors.processing('[popup]');
 // Precomputed colored '[SmartCache]' subsystem prefix — paired with the
 // same constant in lib/smart-cache.js so debug lines from both files
 // produce consistently colored output. formatLogMessage only colors the
@@ -68,14 +67,14 @@ const CONCURRENCY_TAG = messageColors.processing('[CONCURRENCY]');
 // Enhanced mouse interaction and page simulation
 const { performPageInteraction, createInteractionConfig, computeInteractionCeilingMs, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
 // Optional ghost-cursor support for advanced Bezier-based mouse movements
-const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
+const { createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
 // Domain detection cache for performance optimization
-const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
+const { createGlobalHelpers, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
 const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy, prepareSocksRelays, closeAllSocksRelays } = require('./lib/proxy');
 // Dry run functionality
-const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
+const { initializeDryRunCollections, addDryRunMatch, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
 const { clearSiteData } = require('./lib/clear_sitedata');
 // Referrer header generation
@@ -137,6 +136,7 @@ const CONCURRENCY_LIMITS = Object.freeze({
 // Keep using the imported map directly so the two can never diverge again.
 
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
+const REALTIME_CLEANUP_BUFFER_MS = 25000; // Buffer added after site delay before realtime window cleanup
 
 /**
  * Detects the installed Puppeteer version dynamically
@@ -181,7 +181,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 // purgeStaleTrackers removed from import: browserhealth's pageCreationTracker
 // and pageUsageTracker are now WeakMaps, so GC reclaims dead-page entries
 // automatically — manual purging is no longer needed.
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
 const VERSION = '2.0.33'; // Script version
@@ -191,7 +191,12 @@ const startTime = Date.now();
 
 // Initialize domain cache helpers with debug logging if enabled
 const domainCacheOptions = { enableLogging: false }; // Set to true for cache debug logs
-const { isDomainAlreadyDetected, markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
+// Only markDomainAsDetected is used — the global cache feeds the end-of-scan
+// "unique domains cached" stat (getDetectedDomainsCount). The skip-check
+// (isDomainAlreadyDetected) is intentionally not wired in: cross-URL dedup is
+// already handled by nettools' global processed-domain sets, smart-cache, and
+// the per-URL local set, so a cache-level skip would be redundant.
+const { markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
 
 // Smart cache will be initialized after config is loaded
 let smartCache = null;
@@ -232,6 +237,9 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
       const settingsMap = {
         output: ['-o', '--output'],
         max_concurrent: ['--max-concurrent'],
+        cleanup_interval: ['--cleanup-interval'],
+        resource_cleanup_interval: ['--cleanup-interval'],
+        dns: ['--dns'],
         dns_cache: ['--dns-cache'],
         cache_requests: ['--cache-requests'],
         dumpurls: ['--dumpurls'],
@@ -243,20 +251,25 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
         compress_logs: ['--compress-logs'],
         debug: ['--debug'],
         silent: ['--silent'],
-        verbose: ['--verbose'],
         headful: ['--headful'],
         keep_open: ['--keep-open'],
         dry_run: ['--dry-run'],
         titles: ['--titles'],
         sub_domains: ['--sub-domains'],
         no_interact: ['--no-interact'],
+        show_dead_domains: ['--show-dead-domains'],
         ghost_cursor: ['--ghost-cursor'],
         plain: ['--plain'],
         cdp: ['--cdp'],
         dnsmasq: ['--dnsmasq'],
+        dnsmasq_old: ['--dnsmasq-old'],
         unbound: ['--unbound'],
         privoxy: ['--privoxy'],
         pihole: ['--pihole'],
+        adblock_rules: ['--adblock-rules'],
+        no_dns_precheck: ['--no-dns-precheck'],
+        allow_fullscreen: ['--allow-fullscreen'],
+        load_extension: ['--load-extension'],
         eval_on_doc: ['--eval-on-doc'],
         use_puppeteer_core: ['--use-puppeteer-core'],
         ignore_cache: ['--ignore-cache'],
@@ -314,7 +327,6 @@ if (compareIndex !== -1 && args[compareIndex + 1]) {
 }
 
 
-const forceVerbose = args.includes('--verbose');
 const forceDebug = args.includes('--debug');
 const silentMode = args.includes('--silent');
 const showTitles = args.includes('--titles');
@@ -337,12 +349,16 @@ const disableInteract = args.includes('--no-interact');
 const globalGhostCursor = args.includes('--ghost-cursor');
 const plainOutput = args.includes('--plain');
 const enableCDP = args.includes('--cdp');
-const dnsmasqMode = args.includes('--dnsmasq');
-const dnsmasqOldMode = args.includes('--dnsmasq-old');
-const unboundMode = args.includes('--unbound');
+// These six are reassigned to false by the incompatible-flag validation
+// blocks below (e.g. --dnsmasq + --unbound), so they must be `let` — as
+// `const` that fallback threw "Assignment to constant variable" the moment
+// two conflicting output modes were combined.
+let dnsmasqMode = args.includes('--dnsmasq');
+let dnsmasqOldMode = args.includes('--dnsmasq-old');
+let unboundMode = args.includes('--unbound');
 const removeDupes = args.includes('--remove-dupes') || args.includes('--remove-dubes');
-const privoxyMode = args.includes('--privoxy');
-const piholeMode = args.includes('--pihole');
+let privoxyMode = args.includes('--privoxy');
+let piholeMode = args.includes('--pihole');
 const globalEvalOnDoc = args.includes('--eval-on-doc'); // For Fetch/XHR interception
 const dryRunMode = args.includes('--dry-run');
 const compressLogs = args.includes('--compress-logs');
@@ -363,6 +379,25 @@ if (dnsCacheMode) enableDiskCache();
 const dnsPrecheckEnabled = !args.includes('--no-dns-precheck');
 const dnsPrecheckTimeoutMs = 2000;
 
+// --show-dead-domains: collect hostnames that are definitively DEAD (do not
+// exist / unreachable) and print them at the end of the scan so they can be
+// pruned. Only hard signals count — NXDOMAIN/ENODATA from the pre-check and
+// ERR_NAME_NOT_RESOLVED / ERR_ADDRESS_UNREACHABLE from navigation. Transient
+// failures (403/429 blocks, timeouts, Cloudflare challenges) mean the domain is
+// ALIVE and are deliberately excluded. host -> reason (first seen).
+const showDeadDomains = args.includes('--show-dead-domains');
+const _deadDomains = new Map();
+function recordDeadDomain(urlOrHost, reason) {
+  // Populate unconditionally — the pre-check skip reads _deadDomains to drop
+  // repeat URLs on a host already proven dead this run, which must work whether
+  // or not --show-dead-domains is set. The end-of-scan REPORT is separately
+  // gated on showDeadDomains, so the flag still controls output, not recording.
+  if (!urlOrHost) return;
+  let host = urlOrHost;
+  try { host = new URL(urlOrHost).hostname; } catch { /* already a bare host */ }
+  if (host && !_deadDomains.has(host)) _deadDomains.set(host, reason);
+}
+
 // Per-scan cache of negative DNS lookups. OS resolvers don't always cache
 // NXDOMAIN responses, and a scan can hit the same dead hostname many times
 // (different URL paths on the same site). Positive results are left to the
@@ -371,14 +406,67 @@ const dnsPrecheckTimeoutMs = 2000;
 // of unique dead hosts) can't grow the cache unboundedly. Same pattern as
 // the rest of the codebase's in-memory caches.
 const dnsNegativeCache = new Map(); // hostname -> { error, timestamp }
-const DNS_NEGATIVE_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
 const DNS_NEGATIVE_CACHE_MAX = 1000;
+// The negative cache holds ONLY definitive non-existence (NXDOMAIN/ENODATA) —
+// resolver errors fail open and never enter it (see the pre-check catch), so
+// persisting it can't silently drop a live host. Opt-in via --dns-cache: dead
+// hosts are remembered for DNS_NEGATIVE_PERSIST_TTL_MS and reloaded next run;
+// otherwise it's a 5-min in-memory-only cache. The persist TTL is deliberately
+// shorter than the dig/whois positive cache (dig 20h / whois 36h): a domain that doesn't exist
+// now MAY get registered, and this is a domain-hunting scanner, so the dead
+// ones are re-checked twice a day rather than trusted for ~a day.
+const DNS_NEGATIVE_PERSIST_TTL_MS = 12 * 60 * 60 * 1000; // 12 hours
+const DNS_NEGATIVE_CACHE_TTL_MS = dnsCacheMode ? DNS_NEGATIVE_PERSIST_TTL_MS : 5 * 60 * 1000;
+const DNS_NEGATIVE_CACHE_FILE = path.join(__dirname, '.dnsnegcache');
+if (dnsCacheMode) {
+  // Reuse the dig/whois caches' generic load/save (atomic write, TTL + size
+  // bounded). The 'exit' flush is synchronous (writeFileSync) so it fires on
+  // any exit path, mirroring nettools' dig/whois flush.
+  loadDiskCache(DNS_NEGATIVE_CACHE_FILE, dnsNegativeCache, DNS_NEGATIVE_CACHE_TTL_MS, DNS_NEGATIVE_CACHE_MAX);
+  process.on('exit', () => saveDiskCache(DNS_NEGATIVE_CACHE_FILE, dnsNegativeCache, DNS_NEGATIVE_CACHE_TTL_MS, DNS_NEGATIVE_CACHE_MAX));
+}
 let dnsPrecheckSkips = 0;          // URLs skipped because hostname is NXDOMAIN-cached
 let dnsPositiveSkips = 0;          // URLs skipped because dig/whois cache proves resolution
 const dnsPositiveSkippedHosts = new Set(); // unique hostnames that triggered the positive skip path
-// c-ares transient codes — read-only, hoisted out of the per-task DNS
-// pre-check so we don't allocate a fresh Set per URL.
-const DNS_TRANSIENT_ERRORS = new Set(['ETIMEOUT', 'ESERVFAIL', 'EREFUSED', 'ECONNREFUSED']);
+// DNS pre-check resolver (rotation + resolution logic lives in lib/dns.js).
+// `--dns <ip[,ip...]>` (or a `dns` setting in .nwssconfig, mapped to the same
+// flag) pins/rotates an explicit resolver list; otherwise the resolv.conf
+// nameservers are rotated. Rotation spreads the c-ares burst so one server
+// (e.g. a flaky ISP resolver) doesn't absorb every query and answer REFUSED.
+const dnsServerIndex = args.findIndex(arg => arg === '--dns');
+const dnsServersOverride = (dnsServerIndex !== -1 && args[dnsServerIndex + 1])
+  ? parseDnsServers(args[dnsServerIndex + 1])
+  : [];
+const dnsResolver = createRotatingResolver({ servers: dnsServersOverride, forceDebug });
+// Route nettools' dig through the same --dns resolvers (dig otherwise uses the
+// system /etc/resolv.conf, which on a flaky setup times out and silently drops
+// dig-gated domains). Only when --dns is explicitly set.
+if (dnsServersOverride.length > 0) setDigResolvers(dnsServersOverride);
+// Circuit breaker: if resolver errors dominate, suspend the pre-check for a
+// cooldown so a refusal storm doesn't keep hammering a broken resolver (sites
+// still load — a suspended pre-check just proceeds to navigation).
+const dnsBreaker = createDnsCircuitBreaker({ forceDebug });
+if (dnsResolver.pinned && !silentMode) {
+  const how = dnsResolver.servers.length === 1 ? 'pinned to' : 'rotating';
+  console.log(formatLogMessage('info', `DNS pre-check ${how} ${dnsResolver.servers.join(', ')}`));
+} else if (forceDebug && dnsResolver.rotates) {
+  console.log(formatLogMessage('debug', `DNS pre-check rotating ${dnsResolver.servers.length} resolv.conf nameservers: ${dnsResolver.servers.join(', ')}`));
+}
+
+// Idle-hang watchdog registry: in-flight main pages, iterable (the
+// browserhealth page trackers are WeakMaps and can't be scanned). Registered
+// when a task starts navigating, removed on completion. The hang check probes
+// these ONLY while global progress is stalled and force-closes any page that is
+// unresponsive across consecutive probes — recovering a single hung URL in ~the
+// hang-check window instead of waiting out its full per-URL ceiling (which is
+// the backstop). Acting only during a stall + requiring unresponsiveness avoids
+// killing a page that's merely slow (a page in a config delay is idle but
+// RESPONDS to a trivial evaluate; a hung one does not). Entries self-heal via
+// isClosed() so timeout/error paths that skip the normal close can't leak.
+const _inFlightPages = new Map(); // page -> { url, unresponsiveStrikes }
+const PAGE_HANG_PROBE_TIMEOUT_MS = 2000;   // liveness-probe (page.evaluate) cap; no response within this = hung
+const PAGE_HANG_PROBE_INTERVAL_MS = 15000; // how often to probe in-flight pages while the scan is stalled
+const PAGE_HANG_STRIKES_TO_KILL = 2;       // consecutive HUNG probes before force-close (~30s recovery at the 15s interval)
 
 function dnsNegativeCacheSet(hostname, error) {
   if (dnsNegativeCache.size >= DNS_NEGATIVE_CACHE_MAX) {
@@ -632,6 +720,9 @@ if (blockAdsIndex !== -1) {
 
   adblockEnabled = true;
   const engine = adblockEngineName === 'rust' ? adblockRust : adblockJs;
+  // Only ever assigned the os.tmpdir() path below — never a user file — so the
+  // unlink in finally can never touch the caller's own lists.
+  let combinedTmpFile = null;
   try {
     if (engine === adblockRust) {
       // Rust wrapper accepts an array directly — no temp file needed.
@@ -640,15 +731,22 @@ if (blockAdsIndex !== -1) {
       // JS engine takes a single path; concat to a temp file when multiple lists.
       let rulesFile = rulesFiles[0];
       if (rulesFiles.length > 1) {
-        rulesFile = path.join(os.tmpdir(), `nwss-adblock-combined-${Date.now()}.txt`);
+        combinedTmpFile = path.join(os.tmpdir(), `nwss-adblock-combined-${Date.now()}.txt`);
+        rulesFile = combinedTmpFile;
         const combined = rulesFiles.map(f => fs.readFileSync(f, 'utf-8')).join('\n');
         fs.writeFileSync(rulesFile, combined);
       }
+      // parseAdblockRules reads the file synchronously and in full before
+      // returning, so the temp copy is safe to remove immediately afterwards.
       adblockMatcher = engine.parseAdblockRules(rulesFile, { enableLogging: forceDebug });
     }
   } catch (err) {
     console.log(`Error: Failed to load adblock engine '${adblockEngineName}': ${err.message}`);
     process.exit(1);
+  } finally {
+    if (combinedTmpFile) {
+      try { fs.unlinkSync(combinedTmpFile); } catch { /* best effort — OS reaps tmpdir */ }
+    }
   }
   const stats = adblockMatcher.getStats();
   const ruleDesc = stats.total != null
@@ -691,7 +789,6 @@ Per-config settings file (.nwssconfig):
   See README.md for format details.
 
 General Options:
-  --verbose                      Force verbose mode globally
   --debug                        Force debug mode globally
   --silent                       Suppress normal console logs
   --titles                       Add ! <url> title before each site's group
@@ -721,10 +818,16 @@ General Options:
 
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
-  --dns-cache                    Persist dig/whois results to disk between runs (20h TTL, 2000-entry cap each)
+  --dns <ip[,ip,...]>            Resolver(s) for the DNS pre-check AND nettools' dig (not Chrome nav / whois).
+                                 One pins all queries to it; several rotate per query. Overrides /etc/resolv.conf.
+  --dns-cache                    Persist dig/whois results to disk between runs (dig 20h / whois 36h TTL, 2000-entry cap each),
+                                 plus the DNS pre-check negative cache (NXDOMAIN only, 12h TTL, .dnsnegcache)
   --no-dns-precheck              Disable per-URL DNS resolution check before page navigation.
                                  By default, URLs whose hostname doesn't resolve are skipped
                                  immediately (saves ~5-15s of Puppeteer time per dead host).
+  --show-dead-domains            At end of scan, list hostnames that did not resolve / were
+                                 unreachable (NXDOMAIN/ENODATA + ERR_NAME_NOT_RESOLVED/ERR_ADDRESS_UNREACHABLE).
+                                 Excludes blocks/timeouts (those mean the domain is alive). For pruning.
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -741,7 +844,7 @@ Global config.json options:
   ignore_similar: true/false                      Ignore domains similar to already found domains (default: true)
   ignore_similar_threshold: 80                    Similarity threshold percentage for ignore_similar (default: 80)
   ignore_similar_ignored_domains: true/false      Ignore domains similar to ignoreDomains list (default: true)
-  max_concurrent_sites: 8                        Maximum concurrent site processing (1-50, default: 8)
+  max_concurrent_sites: 6                        Maximum concurrent site processing (1-50, default: 6)
   resource_cleanup_interval: 80                  Browser restart interval in URLs processed (1-1000, default: 80)
   disable_ad_tagging: true/false                 Disable Chrome AdTagging to prevent ad frame throttling (default: true)
 
@@ -752,8 +855,7 @@ Per-site config.json options:
                                               When true, ALL regex patterns must match the same URL
   
 Redirect Handling Options:
-  follow_redirects: true/false               Follow redirects to new domains (default: true)
-  max_redirects: 10                          Maximum number of redirects to follow (default: 10)
+  max_redirects: 10                          Maximum number of redirects to follow (default: 10; 0 = follow none)
   js_redirect_timeout: 5000                  Milliseconds to wait for JavaScript redirects (default: 5000)
   detect_js_patterns: true/false             Analyze page source for redirect patterns (default: true)
   redirect_timeout_multiplier: 1.5          Increase timeout for redirected URLs (default: 1.5)
@@ -846,7 +948,7 @@ Advanced Options:
   whois_delay: <milliseconds>                Delay between whois requests for this site (default: global whois_delay)
   dig: ["term1", "term2"]                     Check dig output for ALL specified terms (AND logic)
   dig-or: ["term1", "term2"]                  Check dig output for ANY specified term (OR logic)
-  goto_options: {"waitUntil": "domcontentloaded"} Custom page.goto() options (default: {"waitUntil": "load"})
+  goto_options: {"waitUntil": "domcontentloaded"} Custom page.goto() options (default: {"waitUntil": "domcontentloaded"})
   dig_subdomain: true/false                    Use subdomain for dig lookup instead of root domain (default: false)
   digRecordType: "A"                          DNS record type for dig (default: A)
 
@@ -1336,6 +1438,7 @@ if (dumpUrls) {
 // Avoids blocking I/O on every intercepted request in debug/dumpurls mode
 const _logBuffers = new Map();  // filePath -> string[]
 const LOG_FLUSH_INTERVAL = 2000; // Flush every 2 seconds
+const LOG_BUFFER_MAX_RETAINED = 10000; // Cap a file's retry backlog (lines) so a permanently unwritable path can't grow memory unboundedly
 let _logFlushTimer = null;
 
 function bufferedLogWrite(filePath, entry) {
@@ -1348,18 +1451,20 @@ function bufferedLogWrite(filePath, entry) {
 
 function flushLogBuffers() {
   for (const [filePath, entries] of _logBuffers) {
-    if (entries.length > 0) {
-      try {
-        const data = entries.join('');
-        entries.length = 0; // Clear buffer immediately
-        fs.writeFile(filePath, data, { flag: 'a' }, (err) => {
-          if (err) {
-            console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
-          }
-        });
-      } catch (err) {
-        console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
-      }
+    if (entries.length === 0) continue;
+    try {
+      // Synchronous append on purpose: the batched 2s flush is small, and a
+      // blocking append cannot overlap the next timer tick (it holds the event
+      // loop for its duration) — eliminating the interleaved concurrent-append
+      // hazard of the old async fs.writeFile({flag:'a'}). Clear ONLY after the
+      // write succeeds, so a transient failure retries next tick instead of
+      // being silently dropped (the old code cleared before the async write
+      // confirmed). Bounded so a permanently unwritable path can't grow memory.
+      fs.appendFileSync(filePath, entries.join(''));
+      entries.length = 0;
+    } catch (err) {
+      console.warn(formatLogMessage('warn', `Failed to flush log buffer to ${filePath}: ${err.message}`));
+      if (entries.length > LOG_BUFFER_MAX_RETAINED) entries.length = 0;
     }
   }
 }
@@ -1403,21 +1508,29 @@ if (forceDebug && globalComments) {
  * @param {string} url - The URL string to parse.
  * @returns {string} The root domain, or the original hostname if parsing fails (e.g., for IP addresses or invalid URLs), or an empty string on error.
  */
-const _rootDomainCache = new Map();
-function getRootDomain(url) {
-  const cached = _rootDomainCache.get(url);
+// psl.parse memoized by hostname. The request handlers parse the root domain
+// of EVERY request, and a page hits the same few hosts repeatedly (CDN,
+// analytics, ad domains) — so a hostname-keyed memo turns almost all of those
+// into Map hits instead of repeated public-suffix-list lookups. Keyed by
+// hostname (not full URL) so distinct paths/queries on one host share one
+// entry: higher hit rate, fewer + shorter keys than a URL-keyed cache.
+// psl.parse is pure and never throws (malformed input → {domain: null}), so
+// the catch is defensive only.
+const _hostRootCache = new Map();
+function rootDomainForHost(hostname) {
+  if (!hostname) return '';
+  const cached = _hostRootCache.get(hostname);
   if (cached !== undefined) return cached;
-  try {
-    const { hostname } = new URL(url);
-    const parsed = psl.parse(hostname);
-    const result = parsed.domain || hostname;
-    if (_rootDomainCache.size > 5000) _rootDomainCache.clear();
-    _rootDomainCache.set(url, result);
-    return result;
-  } catch {
-    _rootDomainCache.set(url, '');
-    return '';
-  }
+  let result;
+  try { const parsed = psl.parse(hostname); result = parsed.domain || hostname; }
+  catch { result = hostname; }
+  if (_hostRootCache.size > 5000) _hostRootCache.clear();
+  _hostRootCache.set(hostname, result);
+  return result;
+}
+function getRootDomain(url) {
+  try { return rootDomainForHost(new URL(url).hostname); }
+  catch { return ''; }
 }
 
 /**
@@ -1525,7 +1638,12 @@ function matchesDynamicBlock(domain) {
   return _domainOrParentInSet(_dynamicallyBlockedDomains, domain);
 }
 
-function matchesIgnoreDomain(domain, ignorePatterns) {
+// `_ignorePatterns` is intentionally unused (underscore-marked): every caller
+// and the grep/curl/nettools/searchstring callback contract pass the ignore
+// list as a 2nd arg, but the ignore-state actually lives in the module-level
+// _dynamicallyIgnoredDomains / _ignoreDomainsExact Sets walked below. Kept in
+// the signature only to preserve that shared call shape.
+function matchesIgnoreDomain(domain, _ignorePatterns) {
   // Both dynamic and static ignore lists are walked parent-by-parent so a
   // subdomain of an ignored root inherits the ignore. Previously the
   // dynamic check was exact-only, creating an asymmetry: a static-config
@@ -1747,7 +1865,19 @@ function setupFrameHandling(page, forceDebug) {
 
   // Declare userDataDir in outer scope for cleanup access
   let userDataDir = null;
-  
+
+  // Browser-level decision (the browser launches once per batch, so this can't
+  // be per-site): only disable Chrome's pop-up blocker when at least one site
+  // actually wants popups captured. A real browser blocks non-gesture
+  // window.open(), so non-popup scans keep the blocker on for stealth.
+  // capture_popups scans turn it off so non-gesture popunders (document-level
+  // onclick / timer SDKs) fire and get captured too — gesture-triggered
+  // popups already work via the synthetic-click path regardless of this flag.
+  const wantPopups = Array.isArray(sites) && sites.some(s => s && s.capture_popups === true);
+  if (wantPopups && forceDebug) {
+    console.log(formatLogMessage('debug', `${POPUP_TAG} capture_popups set — launching with --disable-popup-blocking (non-gesture popunders allowed)`));
+  }
+
   /**
    * Creates a new browser instance with consistent configuration
    * Uses system Chrome and temporary directories to minimize disk usage
@@ -1838,6 +1968,12 @@ function setupFrameHandling(page, forceDebug) {
       // Puppeteer 22.x headless mode optimization
       // Auto-detect best headless mode based on Puppeteer version
       headless: headlessMode,
+      // Bypass TLS cert errors at the browser level (drives CDP
+      // Security.setIgnoreCertificateErrors). Robust on new-headless Chrome,
+      // where the --ignore-certificate-errors *flag* is increasingly ignored.
+      // An ad/tracker scanner must reach self-signed / mismatched-cert ad and
+      // embed domains; we observe traffic, we don't transmit secrets.
+      acceptInsecureCerts: true,
       args: [
         // CRITICAL: Remove automation detection markers
         '--disable-blink-features=AutomationControlled',
@@ -1926,6 +2062,10 @@ function setupFrameHandling(page, forceDebug) {
         '--memory-pressure-off',
         '--max_old_space_size=2048',   // V8 heap limit
         '--disable-prompt-on-repost',  // Fixes form popup on page reload
+        // Disable Chrome's pop-up blocker (chrome://settings/content/popups)
+        // ONLY when a site wants popups captured — lets non-gesture popunders
+        // fire. Gated so non-popup scans keep the blocker on for stealth.
+        ...(wantPopups ? ['--disable-popup-blocking'] : []),
         ...(keepBrowserOpen ? [] : ['--disable-background-networking']),
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -2116,22 +2256,17 @@ function setupFrameHandling(page, forceDebug) {
       bypass_cache
     } = siteConfig;
     
-    const allowFirstParty = firstParty === true || firstParty === 1;
-    const allowThirdParty = thirdParty === undefined || thirdParty === true || thirdParty === 1;
     const perSiteSubDomains = subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhostIP = localhost || null;
-    const cloudflarePhishBypass = cloudflare_phish === true;
-    const cloudflareBypass = cloudflare_bypass === true;
     // Add redirect and same-page loop protection
-    const MAX_REDIRECT_DEPTH = siteConfig.max_redirects || 10;
+    // Number check (not ||) so max_redirects: 0 isn't swallowed as falsy → 10.
+    const MAX_REDIRECT_DEPTH = (typeof siteConfig.max_redirects === 'number' && siteConfig.max_redirects >= 0)
+      ? siteConfig.max_redirects : 10;
     const redirectHistory = new Set();
     let redirectCount = 0;
     const pageLoadHistory = new Map(); // Track same-page reloads
     const MAX_SAME_PAGE_LOADS = 3;
     let currentPageUrl = currentUrl;
 
-    const sitePrivoxy = privoxy === true;
-    const sitePihole = pihole === true;
     const flowproxyDetection = flowproxy_detection === true;
     
     const evenBlocked = even_blocked === true;
@@ -2298,6 +2433,9 @@ function setupFrameHandling(page, forceDebug) {
 
       // Track page for realtime cleanup
       trackPageForRealtime(page);
+      // Register with the idle-hang watchdog (force-closed if it goes
+      // unresponsive while the whole scan has stalled).
+      _inFlightPages.set(page, { url: currentUrl, unresponsiveStrikes: 0 });
 
       // Mark page as actively processing
       updatePageUsage(page, true);
@@ -2822,12 +2960,27 @@ function setupFrameHandling(page, forceDebug) {
 
       const regexes = getCompiledRegexes(siteConfig.filterRegex);
 
+      // output_regex (optional per-site): extract the rule body from each matched
+      // URL via capture group 1 (or the whole match), so output becomes
+      // ||<capture> (e.g. ||host/script/) instead of ||host^ — lets a stable
+      // folder/file be blocked on a host that also serves legit content. Compiled
+      // silently here; config-load validation (validate_rules) warns on a bad
+      // pattern, so a throw here just disables the feature for this site.
+      // Reuse the memoized regex compiler (same cache as filterRegex) so the
+      // pattern compiles once per unique source, not once per URL. try/catch
+      // because getCompiledRegex throws on a bad pattern — config-load
+      // validation already warned; a throw here just disables the feature.
+      let outputRegex = null;
+      if (siteConfig.output_regex) {
+        try { outputRegex = getCompiledRegexes(siteConfig.output_regex)[0] || null; } catch (_) { outputRegex = null; }
+      }
+
       // NEW: Get regex_and setting (defaults to false for backward compatibility)
       const useRegexAnd = siteConfig.regex_and === true;
 
    // Parse searchstring patterns using module
    const { searchStrings, searchStringsAnd, hasSearchString, hasSearchStringAnd } = parseSearchStrings(siteConfig.searchstring, siteConfig.searchstring_and);
-   const useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring
+   let useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring (reassigned to false below if curl is unavailable)
    let useGrep = siteConfig.grep === true; // Grep can work independently
 
    // Get user agent for curl if needed
@@ -3009,9 +3162,30 @@ function setupFrameHandling(page, forceDebug) {
        * @param {string} fullSubdomain - Full subdomain for cache tracking
        * @param {string} resourceType - Resource type (for --adblock-rules mode)
        */
-      function addMatchedDomain(domain, resourceType = null, fullSubdomain = null) {
+      function addMatchedDomain(domain, resourceType = null, fullSubdomain = null, matchedUrl = null) {
        // Use fullSubdomain for cache tracking if provided, otherwise fall back to domain
        const cacheKey = fullSubdomain || domain;
+       // output_regex: derive the rule body from the matched URL. Capture group 1
+       // (or the whole match) becomes the stored key, e.g. "host/script/", which
+       // formatDomain emits as ||host/script/ for adblock and falls back to the
+       // bare host for domain-only formats. All similarity / dedup / smart-cache
+       // logic below still runs on the bare host (domain); only the final stored
+       // key changes. The capture must contain both '/' and '.' (i.e. host+path),
+       // otherwise we keep the host so a mis-written regex can't emit garbage.
+       let outputKey = domain;
+       if (outputRegex && matchedUrl) {
+         const m = matchedUrl.match(outputRegex);
+         if (m) {
+           const cap = (m[1] != null ? m[1] : m[0]);
+           // Accept only a host+path shape: a '/' with a real host before it
+           // (segment before the first '/' must contain a '.'). Rejects a
+           // capture that accidentally includes the scheme (host part would be
+           // "https:") or a path-only capture with no host — both fall back to
+           // the bare-host ||host^ rule rather than emit garbage.
+           const sl = cap ? cap.indexOf('/') : -1;
+           if (sl > 0 && cap.slice(0, sl).includes('.')) outputKey = cap;
+         }
+       }
        // Check if we should ignore similar domains
        const ignoreSimilarEnabled = siteConfig.ignore_similar !== undefined ? siteConfig.ignore_similar : ignore_similar;
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
@@ -3113,15 +3287,15 @@ function setupFrameHandling(page, forceDebug) {
       }
 
         if (matchedDomains instanceof Map) {
-          if (!matchedDomains.has(domain)) {
-            matchedDomains.set(domain, new Set());
+          if (!matchedDomains.has(outputKey)) {
+            matchedDomains.set(outputKey, new Set());
           }
           // Only add the specific resourceType that was matched, not all types for this domain
           if (resourceType) {
-            matchedDomains.get(domain).add(resourceType);
+            matchedDomains.get(outputKey).add(resourceType);
           }
         } else {
-          matchedDomains.add(domain);
+          matchedDomains.add(outputKey);
         }
       }
 
@@ -3160,12 +3334,17 @@ function setupFrameHandling(page, forceDebug) {
       // fall back to the default rather than silently disabling capture.
       const POPUP_MAX_DEPTH = (() => {
         const v = parseInt(siteConfig.capture_popups_max_depth, 10);
-        return Number.isFinite(v) && v > 0 ? v : 2;
+        return Number.isFinite(v) && v > 0 ? v : 4;
       })();
       const POPUP_CAPTURE_WINDOW_MS = (() => {
         const v = parseInt(siteConfig.capture_popups_window_ms, 10);
         return Number.isFinite(v) && v > 0 ? v : 5000;
       })();
+      // interact_popups: click inside captured popups so they cascade to their
+      // next ad/redirect (requires capture_popups — no popups exist otherwise).
+      // Light pass; the request listener catches whatever the clicks surface.
+      const interactPopups = capturePopups && siteConfig.interact_popups === true;
+      const POPUP_INTERACT_CLICKS = 3; // enough to fire popunder/redirect SDKs (incl. SDKs that suppress the 1st/2nd click as warmup) without runaway cascades
 
       if (capturePopups && forceDebug) {
         // One-time setup-time warning if the click prerequisite isn't met.
@@ -3231,8 +3410,7 @@ function setupFrameHandling(page, forceDebug) {
             try {
               const parsedUrl = new URL(checkedUrl);
               fullSubdomain = parsedUrl.hostname;
-              const pslResult = psl.parse(fullSubdomain);
-              checkedRootDomain = pslResult.domain || fullSubdomain;
+              checkedRootDomain = rootDomainForHost(fullSubdomain);
             } catch (_) { return; }
             if (!checkedRootDomain) return;
 
@@ -3331,7 +3509,7 @@ function setupFrameHandling(page, forceDebug) {
               trackNetToolsHandler(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
             } else {
               // No nettools required — regex match alone counts.
-              addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
+              addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain, checkedUrl);
             }
           } catch (_) { /* observation-only — never let a popup error escape */ }
         };
@@ -3453,6 +3631,24 @@ function setupFrameHandling(page, forceDebug) {
 
           attachPopupRequestCapture(popupPage, depth);
 
+          // interact_popups: click inside the popup so it can cascade to its next
+          // ad/redirect — popunder/redirect SDKs fire on a document-level click,
+          // and a captured-but-unclicked popup only ever shows its landing URL.
+          // Light pass (POPUP_INTERACT_CLICKS random content-zone clicks), only
+          // on popups shallower than max depth so a clicked popup's spawned child
+          // (depth+1) is still within the capture depth. Fire-and-forget: it must
+          // not block onTargetCreated, and the popup may close/navigate mid-click
+          // (performContentClicks no-ops on a closed page). The request listener
+          // above captures whatever the clicks surface; the close timer bounds it.
+          if (interactPopups && depth < POPUP_MAX_DEPTH && !popupPage.isClosed()) {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup depth=${depth}] interact_popups: ${POPUP_INTERACT_CLICKS} content click(s)`));
+            performContentClicks(popupPage, {
+              clicks: POPUP_INTERACT_CLICKS,
+              forceDebug,
+              realistic: siteConfig.realistic_click === true,
+            }).catch(() => {}); // popup is transient — non-fatal
+          }
+
           // Auto-close after the capture window so popups don't pile up.
           const closeTimer = setTimeout(() => {
             try { if (!popupPage.isClosed()) popupPage.close().catch(() => {}); } catch (_) {}
@@ -3489,30 +3685,24 @@ function setupFrameHandling(page, forceDebug) {
         try {
           const parsedUrl = new URL(checkedUrl);
           fullSubdomain = parsedUrl.hostname;
-          const pslResult = psl.parse(fullSubdomain);
-          checkedRootDomain = pslResult.domain || fullSubdomain;
+          checkedRootDomain = rootDomainForHost(fullSubdomain);
         } catch (e) {}
 
+        // Never BLOCK the top-level document (the scanned page OR a main-frame
+        // redirect target). Aborting it makes the navigation never commit (page
+        // stays at about:blank → navigation timeout), silently breaking any
+        // scanned URL that matches our own filter lists (adblock / blocked /
+        // blockDomainsByUrl) — common on adult/pirate/stream domains. This flag
+        // ONLY guards the abort paths below; the request still flows through the
+        // match logic, so a main-frame redirect destination (e.g. a
+        // filecrypt → ad-domain hop) is still captured via filterRegex/dig/whois.
+        // isNavigationRequest is true for sub-frame docs too, so the mainFrame()
+        // check keeps ad iframes blockable.
+        let isMainFrameDoc = false;
+        try { isMainFrameDoc = request.isNavigationRequest() && request.frame() === page.mainFrame(); } catch (_) {}
+
         // Check against ALL first-party domains (original + all redirects)
         const isFirstParty = checkedRootDomain && firstPartyDomains.has(checkedRootDomain);
-        
-        // Block infinite iframe loops - safely access frame URL
-        const frameUrl = (() => {
-          try {
-            const frame = request.frame();
-            return frame ? frame.url() : '';
-          } catch (err) {
-            return '';
-          }
-        })();
-        if (frameUrl && frameUrl.includes('creative.dmzjmp.com') && 
-            checkedUrl.includes('go.dmzjmp.com/api/models')) {
-          if (forceDebug) {
-            console.log(formatLogMessage('debug', `Blocking potential infinite iframe loop: ${checkedUrl}`));
-          }
-          request.abort();
-          return;
-        }
 
         // Enhanced debug logging to show which frame the request came from
         if (forceDebug) {
@@ -3542,7 +3732,7 @@ function setupFrameHandling(page, forceDebug) {
               request.resourceType()
             );
 
-            if (result.blocked) {
+            if (result.blocked && !isMainFrameDoc) {
               adblockStats.blocked++;
               if (forceDebug) {
                 console.log(formatLogMessage('debug', `${messageColors.blocked('[adblock]')} ${checkedUrl} (${result.reason})`));
@@ -3550,6 +3740,12 @@ function setupFrameHandling(page, forceDebug) {
               request.abort('blockedbyclient');
               return;
             }
+            if (result.blocked && isMainFrameDoc && forceDebug) {
+              // Matched a filter rule but it's the page we're scanning (or a
+              // main-frame redirect target) — allow it (blocking the top-level
+              // document aborts navigation). It still flows through the matcher.
+              console.log(formatLogMessage('debug', `${messageColors.highlight('[adblock]')} top-level document ${checkedUrl} matched (${result.reason}) — allowed (never block the scanned page)`));
+            }
             adblockStats.allowed++;
           } catch (err) { /* Silently continue on adblock errors */ }
         }
@@ -3603,7 +3799,7 @@ function setupFrameHandling(page, forceDebug) {
         // check so domain-based blocks short-circuit without paying the
         // per-URL regex scan. Same abort reason as the static path so
         // request.failure() observers see consistent metadata.
-        if (reqDomain && _dynamicallyBlockedDomains.size > 0 && matchesDynamicBlock(reqDomain)) {
+        if (reqDomain && _dynamicallyBlockedDomains.size > 0 && matchesDynamicBlock(reqDomain) && !isMainFrameDoc) {
           if (forceDebug) {
             console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} aborting ${reqUrl} (domain ${reqDomain} dynamically blocked)`));
           }
@@ -3618,7 +3814,7 @@ function setupFrameHandling(page, forceDebug) {
             break;
           }
         }
-        if (blockedMatchIndex !== -1) {
+        if (blockedMatchIndex !== -1 && !isMainFrameDoc) {
           // Always track the hit (zero-cost on the un-debug path) so the
           // scan-end summary can show which patterns are doing work vs.
           // which are stale and ready to prune. Keyed by pattern.source --
@@ -3658,7 +3854,7 @@ function setupFrameHandling(page, forceDebug) {
                         wasBlocked: true
                       });
                     } else {
-                      addMatchedDomain(reqDomain, resourceType, fullSubdomain);
+                      addMatchedDomain(reqDomain, resourceType, fullSubdomain, reqUrl);
                     }
                     matchedRegexPatterns.add(evenBlockedRegexPattern);
 
@@ -3836,7 +4032,10 @@ function setupFrameHandling(page, forceDebug) {
                  isFirstParty: isFirstParty
                });
              } else {
-               addMatchedDomain(reqDomain, resourceType);
+               // Pass null for fullSubdomain (not the in-scope hostname) to keep
+               // this path's dedup key as the root domain exactly as before —
+               // only matchedUrl is new here, for output_regex.
+               addMatchedDomain(reqDomain, resourceType, null, reqUrl);
              }
              if (matchedRegexPattern) matchedRegexPatterns.add(matchedRegexPattern);
              if (siteConfig.verbose === 1) {
@@ -4197,15 +4396,43 @@ function setupFrameHandling(page, forceDebug) {
         try {
           navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOptions, forceDebug, formatLogMessage);
         } catch (navErr) {
-          // Only retry on genuine timeouts, not chrome-error:// redirects
+          // Only handle genuine timeouts here, not chrome-error:// redirects.
+          // pageUrl === 'about:blank' means the navigation never committed
+          // (server never responded) — treat as a real failure, not a partial
+          // page; only a page that actually reached a URL is worth observing.
           let pageUrl = '';
           try { if (!page.isClosed()) pageUrl = page.url(); } catch {}
           const isPopupFailure = navErr.message.includes('chrome-error://') || navErr.message.includes('invalid URL') ||
             pageUrl.startsWith('chrome-error://') || pageUrl === 'about:blank';
           if ((navErr.message.includes('timeout') || navErr.message.includes('Timeout')) && !isPopupFailure) {
-            if (forceDebug) console.log(formatLogMessage('debug', `Navigation timeout, retrying with waitUntil:networkidle2 for ${currentUrl}`));
-            const fallbackOptions = { ...gotoOptions, waitUntil: 'networkidle2', timeout: Math.min(timeout, 10000) };
-            navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, fallbackOptions, forceDebug, formatLogMessage);
+            // The OLD fallback retried with networkidle2 — STRICTER than the
+            // domcontentloaded default, so it could never rescue a
+            // domcontentloaded timeout (and Puppeteer 25 has no 'commit', i.e.
+            // nothing more lenient). Two-tier recovery instead:
+            //   1. If the site used a wait STRICTER than domcontentloaded, do one
+            //      lenient retry with domcontentloaded (it fires earlier).
+            //   2. Otherwise proceed with the partially-loaded page rather than
+            //      discarding the URL — it exists and requests already fired
+            //      (captured by page.on('request')); the delay/interact phase
+            //      below keeps capturing. Streaming/embed/media pages routinely
+            //      never reach DOM-ready (a connection stays open) but their
+            //      ad/tracker calls fired early.
+            const primaryWait = gotoOptions.waitUntil || defaultWaitUntil;
+            let recovered = false;
+            if (primaryWait !== 'domcontentloaded') {
+              try {
+                if (forceDebug) console.log(formatLogMessage('debug', `Navigation timeout (${primaryWait}), retrying with waitUntil:domcontentloaded for ${currentUrl}`));
+                const fallbackOptions = { ...gotoOptions, waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) };
+                navigationResult = await navigateWithRedirectHandling(page, currentUrl, siteConfig, fallbackOptions, forceDebug, formatLogMessage);
+                recovered = true;
+              } catch (_) { /* fall through to proceed-with-partial */ }
+            }
+            if (!recovered) {
+              let partialUrl = currentUrl;
+              try { if (!page.isClosed()) partialUrl = page.url() || currentUrl; } catch {}
+              if (forceDebug) console.log(formatLogMessage('debug', `Navigation timeout — proceeding with partially-loaded page for ${currentUrl}`));
+              navigationResult = { finalUrl: partialUrl, redirected: false, redirectChain: [currentUrl], originalUrl: currentUrl, redirectDomains: [], httpStatus: null, cfRay: null };
+            }
           } else {
             throw navErr;
           }
@@ -4475,12 +4702,50 @@ function setupFrameHandling(page, forceDebug) {
             }
           }
           console.error(formatLogMessage('error', `Failed on ${currentUrl}: ${err.message}`));
+          // Capture hard "dead domain" navigation errors for --show-dead-domains
+          // (DNS doesn't resolve / host unreachable). Blocks, timeouts and CF
+          // challenges are NOT dead — they're excluded by this match.
+          // Only DEFINITIVE non-existence / unreachable signals — these now drive
+          // the in-scan dead-domain SKIP (not just --show-dead-domains reporting),
+          // so transient DNS errors must NOT match. The bare `ERR_DNS` used to
+          // catch ERR_DNS_TIMED_OUT / ERR_DNS_MALFORMED_RESPONSE / ERR_DNS_SERVER_FAILED
+          // (all transient) — dropped so a slow-DNS blip can't false-skip the
+          // rest of a live host's URLs.
+          const deadNav = /ERR_NAME_NOT_RESOLVED|ERR_ADDRESS_UNREACHABLE/.exec(err.message || '');
+          if (deadNav) {
+            recordDeadDomain(currentUrl, deadNav[0]);
+            // Corroborate-then-persist to the negative cache (.dnsnegcache with
+            // --dns-cache → cross-scan skip; else in-memory). Chrome resolves via
+            // the possibly-flaky SYSTEM resolver, so its ERR_NAME_NOT_RESOLVED may
+            // be a glitch on a LIVE host. Re-confirm via the reliable --dns
+            // resolver and cache ONLY if it ALSO returns a definitive NXDOMAIN.
+            // ERR_ADDRESS_UNREACHABLE is routing (the host resolves), so the
+            // resolve succeeds and it's correctly not cached. Fire-and-forget:
+            // off the critical path; saveDiskCache flushes on exit.
+            if (dnsPrecheckEnabled && deadNav[0] === 'ERR_NAME_NOT_RESOLVED') {
+              let navHost = '';
+              try { navHost = new URL(currentUrl).hostname; } catch {}
+              if (navHost && !/^[\d.:]+$|^\[/.test(navHost) && !dnsNegativeCache.has(navHost)) {
+                dnsResolver.resolveHost(navHost, dnsPrecheckTimeoutMs).then(
+                  () => { /* reliable resolver resolves it — system-resolver glitch, do NOT cache */ },
+                  (e) => {
+                    const code = (e && (e.code || e.message)) || '';
+                    if (isNonExistenceError(code)) {
+                      dnsNegativeCacheSet(navHost, code);
+                      recordDeadDomain(navHost, code);
+                      if (forceDebug) console.log(formatLogMessage('debug', `Dead domain confirmed by --dns resolver (${code}) — caching ${navHost} (skips next run with --dns-cache)`));
+                    }
+                  }
+                ).catch(() => {});
+              }
+            }
+          }
           throw err;
         }
       }
       }
 
-      const delayMs = siteConfig.delay || DEFAULT_DELAY;
+      const delayMs = siteConfig.delay || TIMEOUTS.DEFAULT_DELAY;
 
       // Optimized delays for Puppeteer 23.x performance
       const isFastSite = timeout <= TIMEOUTS.FAST_SITE_THRESHOLD;
@@ -4560,8 +4825,21 @@ function setupFrameHandling(page, forceDebug) {
                 const ghostStart = Date.now();
                 const ghostTimeLeft = () => ghostDuration - (Date.now() - ghostStart);
 
-                // Time-based Bezier mouse movements — runs for ghostDuration ms
-                while (ghostTimeLeft() > 200) {
+                // Honor interact_click_count in ghost mode too (built-in default
+                // is 3 — ad SDKs often swallow the 1st/2nd click as warmup). Same
+                // default + 20-cap as the built-in content-click path. 0 when
+                // element clicks are disabled.
+                const ghostClickCount = interactionConfig.includeElementClicks
+                  ? Math.min(Math.max(Number(siteConfig.interact_click_count) || 3, 1), 20)
+                  : 0;
+                // Reserve part of the duration budget for those clicks so the
+                // movement loop doesn't consume all of ghost_cursor_duration.
+                // Capped at half the budget so movement still happens; raise
+                // ghost_cursor_duration to fit more clicks.
+                const clickReserveMs = Math.min(ghostClickCount * 600, ghostDuration * 0.5);
+
+                // Time-based Bezier mouse movements — runs for the unreserved budget
+                while (ghostTimeLeft() > 200 + clickReserveMs) {
                   const toX = Math.floor(Math.random() * (viewport.width - 100)) + 50;
                   const toY = Math.floor(Math.random() * (viewport.height - 100)) + 50;
                   await ghostMove(cursor, toX, toY, {
@@ -4569,18 +4847,23 @@ function setupFrameHandling(page, forceDebug) {
                     overshootThreshold: ghostConfig.overshootThreshold,
                     forceDebug
                   });
-                  if (ghostTimeLeft() > 100) {
+                  if (ghostTimeLeft() > 100 + clickReserveMs) {
                     await new Promise(r => setTimeout(r, 25 + Math.random() * 75));
                   }
                 }
                 if (ghostTimeLeft() > 100 && Math.random() < 0.3) {
                   await ghostRandomMove(cursor, { forceDebug });
                 }
-                if (interactionConfig.includeElementClicks && ghostTimeLeft() > 100) {
+                // interact_click_count clicks, each to a fresh content-zone point.
+                // The time guard stops early if the budget runs out (raise
+                // ghost_cursor_duration for more).
+                for (let gc = 0; gc < ghostClickCount && ghostTimeLeft() > 100; gc++) {
                   const clickX = Math.floor(viewport.width * 0.2 + Math.random() * viewport.width * 0.6);
                   const clickY = Math.floor(viewport.height * 0.2 + Math.random() * viewport.height * 0.6);
                   await ghostClick(cursor, { x: clickX, y: clickY }, {
                     hesitate: ghostConfig.hesitate,
+                    page,
+                    realistic: siteConfig.realistic_click === true,
                     forceDebug
                   });
                 }
@@ -4895,7 +5178,7 @@ function setupFrameHandling(page, forceDebug) {
       // Only add delay if we're continuing with more reloads
       if (i < totalReloads) {
     // Reduce delay for problematic sites
-    const adjustedDelay = i > 1 ? Math.min(DEFAULT_DELAY, 2000) : DEFAULT_DELAY;
+    const adjustedDelay = i > 1 ? Math.min(TIMEOUTS.DEFAULT_DELAY, 2000) : TIMEOUTS.DEFAULT_DELAY;
     await fastTimeout(adjustedDelay);
       }
     }
@@ -5088,7 +5371,7 @@ function setupFrameHandling(page, forceDebug) {
           const safeUrl = currentUrl.replace(/https?:\/\//, '').replace(/[^a-zA-Z0-9]/g, '_').substring(0, 80);
           const filename = `screenshots/${safeUrl}-${timestamp}.png`;
           try {
-            if (!fs.existsSync('screenshots')) fs.mkdirSync('screenshots', { recursive: true });
+            fs.mkdirSync('screenshots', { recursive: true }); // recursive:true is a no-op if it already exists
             await page.screenshot({ path: filename, type: 'png', fullPage: true });
             console.log(formatLogMessage('info', `Screenshot saved: ${filename}`));
           } catch (screenshotErr) {
@@ -5099,6 +5382,7 @@ function setupFrameHandling(page, forceDebug) {
         if (!keepBrowserOpen) {
           try {
             untrackPage(page);
+            _inFlightPages.delete(page);
             await page.close();
             if (forceDebug) console.log(formatLogMessage('debug', `Page closed for ${currentUrl}`));
           } catch (pageCloseErr) {
@@ -5199,6 +5483,12 @@ function setupFrameHandling(page, forceDebug) {
  let lastProcessedCount = 0;
  let hangCheckCount = 0;
  let forceRestartFlag = false; // Flag to trigger restart on next iteration
+ // Largest per-URL timeout budget seen across tasks. The hang-check restart
+ // scales to this so it can't false-fire on a legitimately-slow config (high
+ // delay × reload × interact) whose per-URL budget exceeds a flat threshold —
+ // the emergency restart should only fire once the per-URL timeout ITSELF has
+ // had its chance and failed (a true browser hang).
+ let maxPerUrlTimeoutMs = 0;
 
  // Precomputed colored '[HANG CHECK]' subsystem prefix. formatLogMessage
  // only colors the [severity] tag; the '[HANG CHECK]' substring was
@@ -5206,6 +5496,48 @@ function setupFrameHandling(page, forceDebug) {
  // entry so the interval callback doesn't re-colorize per tick.
  const HANG_CHECK_TAG = messageColors.processing('[HANG CHECK]');
 
+ // Idle-hang watchdog. Runs only while the scan is stalled (no URL completing).
+ // The probe distinguishes a HUNG renderer from one that's merely NAVIGATING,
+ // which is the key to probing aggressively without false-kills:
+ //   - evaluate resolves            -> 'alive'      -> reset strikes
+ //   - evaluate rejects fast (e.g. "Execution context destroyed" mid goto/
+ //     reload)                       -> 'navigating' -> inconclusive: neither
+ //                                                      strike nor reset, so a
+ //                                                      navigation can NEVER trip
+ //                                                      the kill regardless of cadence
+ //   - no response within the cap    -> 'hung'       -> strike
+ // PAGE_HANG_STRIKES_TO_KILL consecutive HUNG probes force-close the page, so the
+ // stuck task's awaits reject and its batch completes instead of waiting out the
+ // full per-URL ceiling. Parallel, guarded against overlap; zero overhead off a stall.
+ let _hangProbeInProgress = false;
+ const probeInFlightPagesForHang = async () => {
+   if (_hangProbeInProgress || _inFlightPages.size === 0) return;
+   _hangProbeInProgress = true;
+   try {
+     await Promise.all([..._inFlightPages.entries()].map(async ([page, info]) => {
+       if (page.isClosed()) { _inFlightPages.delete(page); return; }
+       let verdict;
+       try {
+         verdict = await Promise.race([
+           page.evaluate(() => true).then(() => 'alive', () => 'navigating'),
+           new Promise(r => setTimeout(() => r('hung'), PAGE_HANG_PROBE_TIMEOUT_MS)),
+         ]);
+       } catch { verdict = 'hung'; }
+       if (verdict === 'alive') { info.unresponsiveStrikes = 0; return; }
+       if (verdict === 'navigating') return; // context destroyed mid-nav — not a hang; don't strike or reset
+       // verdict === 'hung' — renderer gave no response within the cap
+       info.unresponsiveStrikes++;
+       if (info.unresponsiveStrikes >= PAGE_HANG_STRIKES_TO_KILL) {
+         console.log(formatLogMessage('warn', `${HANG_CHECK_TAG} Force-closing hung page after ${info.unresponsiveStrikes} unresponsive probes: ${info.url}`));
+         _inFlightPages.delete(page);
+         page.close().catch(() => {}); // stuck task's awaits reject -> task errors -> batch completes
+       }
+     }));
+   } finally {
+     _hangProbeInProgress = false;
+   }
+ };
+
  const hangDetectionInterval = setInterval(() => {
    // Progress check, counter, and forceRestartFlag MUST run regardless of
    // debug mode — previously the entire body was gated on forceDebug, which
@@ -5218,8 +5550,18 @@ function setupFrameHandling(page, forceDebug) {
      if (forceDebug) {
        console.log(formatLogMessage('warn', `${HANG_CHECK_TAG} No progress for ${hangCheckCount * 30}s`));
      }
-     if (hangCheckCount >= 5) {
-       console.log(formatLogMessage('error', `${HANG_CHECK_TAG} Hung for 2.5 minutes. Triggering emergency browser restart.`));
+     // The faster 15s probe interval below does surgical per-page recovery; this
+     // 30s interval owns only the slower nuclear-restart escalation. Deadline-
+     // aware: the restart only fires once the stall has OUTLASTED the heaviest
+     // in-flight per-URL budget (+ grace) — i.e. the per-URL timeout itself had
+     // its chance and failed, a true hang. A flat threshold (the old 2.5min)
+     // false-fires on legitimately-slow configs (high delay × reload × interact)
+     // whose per-URL budget exceeds it, restarting the browser mid-work. Floor
+     // at 150s so light configs behave exactly as before.
+     // +45s buffer covers the per-URL 8s orphan grace + the 30s tick granularity + slack.
+     const restartAfterMs = Math.max(150000, maxPerUrlTimeoutMs + 45000);
+     if (hangCheckCount * 30000 >= restartAfterMs) {
+       console.log(formatLogMessage('error', `${HANG_CHECK_TAG} No progress for ${Math.round(hangCheckCount * 30)}s (past the ${Math.round(restartAfterMs / 1000)}s per-URL budget). Triggering emergency browser restart.`));
        forceRestartFlag = true; // Set flag instead of exiting
        hangCheckCount = 0; // Reset counter for next cycle
      }
@@ -5241,6 +5583,22 @@ function setupFrameHandling(page, forceDebug) {
  // cleanup, this is belt-and-suspenders in case a future refactor moves them.
  hangDetectionInterval.unref();
 
+ // Fast surgical recovery on its own 15s cadence (the 30s interval above owns
+ // the slower nuclear-restart escalation). Probes in-flight pages only while
+ // progress is stalled and force-closes confirmed-hung ones; clears strikes when
+ // progress resumes so a fresh stall starts from zero. Starts at -1 so the very
+ // first window is grace (processedUrlCount begins at 0).
+ let lastProbeCount = -1;
+ const pageHangProbeInterval = setInterval(() => {
+   if (processedUrlCount === lastProbeCount) {
+     probeInFlightPagesForHang(); // fire-and-forget; self-guarded against overlap
+   } else {
+     for (const info of _inFlightPages.values()) info.unresponsiveStrikes = 0;
+   }
+   lastProbeCount = processedUrlCount;
+ }, PAGE_HANG_PROBE_INTERVAL_MS);
+ pageHangProbeInterval.unref();
+
  // Process URLs in batches with exception handling
  let siteGroupIndex = 0;
  let currentProxyKey = '';  // Track active proxy config — '' means direct connection
@@ -5509,6 +5867,19 @@ function setupFrameHandling(page, forceDebug) {
        // actually starting — wrongly skipping live domains. c-ares isn't
        // threadpool-bound so it's immune to that contention.
        if (dnsPrecheckEnabled && taskDomain && !/^[\d.:]+$|^\[/.test(taskDomain)) {
+         // Already proven dead earlier THIS run — either a pre-check NXDOMAIN or
+         // a prior URL's navigation hit ERR_NAME_NOT_RESOLVED / ERR_ADDRESS_UNREACHABLE
+         // (recordDeadDomain populates _deadDomains for both). Skip the repeat
+         // instead of paying another fail-open navigation on a multi-URL dead
+         // host (e.g. dlstreams.top?id=39/54/347). In-scan only (NOT persisted):
+         // Chrome resolves via the system resolver, so a nav-level failure could
+         // be a system-resolver glitch on a live host — a false "dead" must not
+         // carry across runs. Cheap: a Map lookup, no DNS resolve.
+         if (_deadDomains.has(taskDomain)) {
+           dnsPrecheckSkips++;
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check: ${taskDomain} already dead this run (${_deadDomains.get(taskDomain)}) — skipping`));
+           return { url: task.url, rules: [], success: false, error: `DNS: ${_deadDomains.get(taskDomain)}`, skipped: true };
+         }
          const cached = dnsNegativeCache.get(taskDomain);
          if (cached && Date.now() - cached.timestamp < DNS_NEGATIVE_CACHE_TTL_MS) {
            dnsPrecheckSkips++;
@@ -5525,58 +5896,38 @@ function setupFrameHandling(page, forceDebug) {
            dnsPositiveSkippedHosts.add(taskDomain);
            if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check skipped (dig/whois cache confirms resolution): ${taskDomain}`));
            // Fall through to navigation -- pre-check "passed" by proxy.
+         } else if (dnsBreaker.isTripped()) {
+           // Resolver is in a refusal storm — pre-checking is futile and only
+           // adds load. Skip the resolve and proceed to navigation (same effect
+           // as a fail-open); no breaker record since no resolve happened.
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check suspended (resolver circuit open) — proceeding: ${taskDomain}`));
          } else {
-         const dnsResolve = async () => {
-           // resolve4 first; on no-IPv4 (ENODATA / ENOTFOUND) fall back to
-           // resolve6 so IPv6-only hosts aren't wrongly skipped. ANY OTHER
-           // error code (ESERVFAIL, ETIMEOUT, EREFUSED, etc.) propagates
-           // unchanged so the outer transient-retry path sees the real
-           // resolver code and the negative cache records the right reason.
-           // Previously a bare .catch swallowed everything and tried
-           // resolve6, which masked transient v4-side errors behind
-           // whatever resolve6 ended up reporting.
-           // 2s timeout kept as a real safety net — with c-ares off the
-           // threadpool it should now rarely fire.
-           let timer;
-           try {
-             const timeoutP = new Promise((_, reject) => {
-               timer = setTimeout(() => reject(new Error('DNS timeout')), dnsPrecheckTimeoutMs);
-             });
-             const resolveChain = dnsPromises.resolve4(taskDomain)
-               .catch(err => {
-                 if (err && (err.code === 'ENODATA' || err.code === 'ENOTFOUND')) {
-                   return dnsPromises.resolve6(taskDomain);
-                 }
-                 throw err;
-               });
-             await Promise.race([resolveChain, timeoutP]);
-           } finally {
-             if (timer) clearTimeout(timer);
-           }
-         };
-         // c-ares transient codes — retry once so a momentary resolver
-         // hiccup doesn't poison the negative cache for 5 minutes.
-         // DNS_TRANSIENT_ERRORS is module-level so we don't allocate per task.
          try {
-           try {
-             await dnsResolve();
-           } catch (firstErr) {
-             const code = firstErr && firstErr.code;
-             if (DNS_TRANSIENT_ERRORS.has(code) || (firstErr && firstErr.message === 'DNS timeout')) {
-               if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check transient (${code || 'timeout'}) for ${taskDomain}, retrying once`));
-               await dnsResolve();
-             } else {
-               throw firstErr;
-             }
-           }
+           // Rotates the lead nameserver per attempt and retries once on a
+           // transient error; rejects with the final error (code intact) on
+           // failure. See lib/dns.js.
+           await dnsResolver.resolveHost(taskDomain, dnsPrecheckTimeoutMs);
+           dnsBreaker.record(false); // resolved OK — resolver healthy
          } catch (dnsErr) {
            const errCode = dnsErr.code || dnsErr.message || 'DNS resolve failed';
-           dnsNegativeCacheSet(taskDomain, errCode);
-           dnsPrecheckSkips++;
-           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
-           return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+           // Only a definitive "host does not exist / has no address" answer
+           // (ENOTFOUND/ENODATA) justifies dropping the URL. A resolver-level
+           // failure (EREFUSED/ESERVFAIL/ETIMEOUT/ECONNREFUSED/timeout) says
+           // nothing about whether the domain is live — fail open: don't cache,
+           // don't skip, let it proceed to real browser navigation (a genuinely
+           // dead host still fails fast there).
+           if (isNonExistenceError(errCode)) {
+             dnsBreaker.record(false); // resolver answered NXDOMAIN — healthy
+             dnsNegativeCacheSet(taskDomain, errCode);
+             recordDeadDomain(taskDomain, errCode);
+             dnsPrecheckSkips++;
+             if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
+             return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+           }
+           dnsBreaker.record(true); // resolver error — counts toward tripping the circuit
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check inconclusive (${errCode}) for ${taskDomain} — proceeding (resolver issue, not a dead host)`));
          }
-         } // close `else` from domainKnownToResolve shortcut above
+         } // close the resolve `else` (domainKnownToResolve / circuit-open shortcuts above)
        }
      } catch {}
 
@@ -5609,6 +5960,9 @@ function setupFrameHandling(page, forceDebug) {
          + ((task.config.delay || 0) + INTERACTION_OVERHEAD_MS) * (1 + reloadCount)
          + 30000
      );
+     // Feed the hang-check restart so it never escalates before this URL's own
+     // timeout could have fired (see maxPerUrlTimeoutMs).
+     if (PER_URL_TIMEOUT_MS > maxPerUrlTimeoutMs) maxPerUrlTimeoutMs = PER_URL_TIMEOUT_MS;
      // Grace period after primary timeout — gives the orphan a chance to
      // finish drainPendingNetTools() and emit "Saving N rules despite page
      // load failure" before we abandon its result. Drain typically completes
@@ -5868,11 +6222,13 @@ function setupFrameHandling(page, forceDebug) {
  } catch (processingError) {
    console.log(formatLogMessage('error', `Critical error: ${processingError.message}`));
    clearInterval(hangDetectionInterval);
+   clearInterval(pageHangProbeInterval);
    throw processingError;
   }
 
-  // Clear hang detection interval
+  // Clear hang detection intervals
   clearInterval(hangDetectionInterval);
+  clearInterval(pageHangProbeInterval);
 
   // === POST-SCAN PROCESSING ===
   // Clean up first-party domains and validate results
@@ -5954,7 +6310,6 @@ function setupFrameHandling(page, forceDebug) {
   const totalMatches = results.reduce((sum, r) => sum + (r.rules ? r.rules.length : 0), 0);
 
   // Debug: Show output format being used
-  const totalDomainsSkipped = getTotalDomainsSkipped();
   const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
@@ -5969,7 +6324,7 @@ function setupFrameHandling(page, forceDebug) {
     };
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
-     console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
+     console.log(formatLogMessage('debug', `Performance: ${detectedDomainsCount} unique domains cached`));
      // Cloudflare cache statistics
      const cloudflareStats = getCacheStats();
      if (cloudflareStats.size > 0) {
@@ -5998,6 +6353,13 @@ function setupFrameHandling(page, forceDebug) {
        }
        console.log(formatLogMessage('debug', `DNS pre-check skipped: ${parts.join(', ')}`));
      }
+     // Surface circuit-breaker activity in the end-of-scan summary (each trip
+     // also warns in real time). Shown outside forceDebug because a resolver
+     // refusal storm is something the operator should know happened.
+     const dnsBreakerTrips = dnsBreaker.stats().trips;
+     if (dnsBreakerTrips > 0 && !silentMode) {
+       console.log(formatLogMessage('info', `DNS pre-check circuit tripped ${dnsBreakerTrips}× this scan (resolver refusal back-off)`));
+     }
      // Blocked-pattern hit stats. Surfaces which patterns are actually
      // doing work this scan and (by absence) which are stale enough to
      // prune from config. Top 10 by hit count to keep the log scannable
@@ -6200,8 +6562,18 @@ function setupFrameHandling(page, forceDebug) {
     } else if (outputResult.totalRules > 0 && dryRunMode) {
       console.log(messageColors.success('Found') + ` ${outputResult.totalRules} total matches across all URLs`);
     }
-    if (totalDomainsSkipped > 0) {
-      console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
+    // --show-dead-domains: list hostnames that didn't resolve / were unreachable
+    // this scan (NXDOMAIN/ENODATA + ERR_NAME_NOT_RESOLVED/ERR_ADDRESS_UNREACHABLE).
+    // One host per line so it's greppable for pruning; reason in the trailing column.
+    if (showDeadDomains) {
+      if (_deadDomains.size > 0) {
+        console.log(`\n${messageColors.warn(`Dead domains (${_deadDomains.size}) — did not resolve / unreachable:`)}`);
+        for (const [host, reason] of [..._deadDomains].sort((a, b) => a[0].localeCompare(b[0]))) {
+          console.log(`  ${host}\t${reason}`);
+        }
+      } else {
+        console.log(`\n${messageColors.success('Dead domains: none detected')}`);
+      }
     }
     if (ignoreCache && forceDebug) {
       console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);