@fanboynz/network-scanner 3.1.0 → 3.2.0

package/nwss.js

@@ -9,11 +9,11 @@ const fs = require('fs');
 const os = require('os');
 const psl = require('psl');
 const path = require('path');
-const dnsPromises = require('node:dns/promises');
+const { createRotatingResolver, createDnsCircuitBreaker, parseDnsServers, isNonExistenceError } = require('./lib/dns');
 const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
-const { compressMultipleFiles, formatFileSize } = require('./lib/compress');
+const { compressMultipleFiles } = require('./lib/compress');
 const { parseSearchStrings, createResponseHandler } = require('./lib/searchstring');
-const { applyAllFingerprintSpoofing, USER_AGENT_COLLECTIONS } = require('./lib/fingerprint');
+const { applyAllFingerprintSpoofing, USER_AGENT_COLLECTIONS, CHROME_BUILD, CHROME_GREASE_BRAND } = require('./lib/fingerprint');
 const { formatRules, handleOutput, getFormatDescription } = require('./lib/output');
 // Curl functionality (replace searchstring curl handler)
 const { validateCurlAvailability, createCurlHandler: createCurlModuleHandler } = require('./lib/curl');
@@ -34,9 +34,7 @@ const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore
 // Graceful exit
 const { handleBrowserExit, cleanupChromeTempFiles, cleanupUserDataDir } = require('./lib/browserexit');
 // Whois & Dig
-const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats, domainKnownToResolve } = require('./lib/nettools');
-// File compare
-const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
+const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats, domainKnownToResolve, loadDiskCache, saveDiskCache, setDigResolvers } = require('./lib/nettools');
 // CDP functionality
 const { createCDPSession, createPageWithTimeout, setRequestInterceptionWithTimeout } = require('./lib/cdp');
 // Post-processing cleanup
@@ -68,14 +66,14 @@ const CONCURRENCY_TAG = messageColors.processing('[CONCURRENCY]');
 // Enhanced mouse interaction and page simulation
 const { performPageInteraction, createInteractionConfig, computeInteractionCeilingMs, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
 // Optional ghost-cursor support for advanced Bezier-based mouse movements
-const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
+const { createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
 // Domain detection cache for performance optimization
-const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
+const { createGlobalHelpers, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
 const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy, prepareSocksRelays, closeAllSocksRelays } = require('./lib/proxy');
 // Dry run functionality
-const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
+const { initializeDryRunCollections, addDryRunMatch, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
 const { clearSiteData } = require('./lib/clear_sitedata');
 // Referrer header generation
@@ -137,6 +135,7 @@ const CONCURRENCY_LIMITS = Object.freeze({
 // Keep using the imported map directly so the two can never diverge again.
 
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
+const REALTIME_CLEANUP_BUFFER_MS = 25000; // Buffer added after site delay before realtime window cleanup
 
 /**
  * Detects the installed Puppeteer version dynamically
@@ -181,7 +180,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 // purgeStaleTrackers removed from import: browserhealth's pageCreationTracker
 // and pageUsageTracker are now WeakMaps, so GC reclaims dead-page entries
 // automatically — manual purging is no longer needed.
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
 const VERSION = '2.0.33'; // Script version
@@ -191,7 +190,12 @@ const startTime = Date.now();
 
 // Initialize domain cache helpers with debug logging if enabled
 const domainCacheOptions = { enableLogging: false }; // Set to true for cache debug logs
-const { isDomainAlreadyDetected, markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
+// Only markDomainAsDetected is used — the global cache feeds the end-of-scan
+// "unique domains cached" stat (getDetectedDomainsCount). The skip-check
+// (isDomainAlreadyDetected) is intentionally not wired in: cross-URL dedup is
+// already handled by nettools' global processed-domain sets, smart-cache, and
+// the per-URL local set, so a cache-level skip would be redundant.
+const { markDomainAsDetected } = createGlobalHelpers(domainCacheOptions);
 
 // Smart cache will be initialized after config is loaded
 let smartCache = null;
@@ -232,6 +236,9 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
       const settingsMap = {
         output: ['-o', '--output'],
         max_concurrent: ['--max-concurrent'],
+        cleanup_interval: ['--cleanup-interval'],
+        resource_cleanup_interval: ['--cleanup-interval'],
+        dns: ['--dns'],
         dns_cache: ['--dns-cache'],
         cache_requests: ['--cache-requests'],
         dumpurls: ['--dumpurls'],
@@ -243,20 +250,25 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
         compress_logs: ['--compress-logs'],
         debug: ['--debug'],
         silent: ['--silent'],
-        verbose: ['--verbose'],
         headful: ['--headful'],
         keep_open: ['--keep-open'],
         dry_run: ['--dry-run'],
         titles: ['--titles'],
         sub_domains: ['--sub-domains'],
         no_interact: ['--no-interact'],
+        show_dead_domains: ['--show-dead-domains'],
         ghost_cursor: ['--ghost-cursor'],
         plain: ['--plain'],
         cdp: ['--cdp'],
         dnsmasq: ['--dnsmasq'],
+        dnsmasq_old: ['--dnsmasq-old'],
         unbound: ['--unbound'],
         privoxy: ['--privoxy'],
         pihole: ['--pihole'],
+        adblock_rules: ['--adblock-rules'],
+        no_dns_precheck: ['--no-dns-precheck'],
+        allow_fullscreen: ['--allow-fullscreen'],
+        load_extension: ['--load-extension'],
         eval_on_doc: ['--eval-on-doc'],
         use_puppeteer_core: ['--use-puppeteer-core'],
         ignore_cache: ['--ignore-cache'],
@@ -314,7 +326,6 @@ if (compareIndex !== -1 && args[compareIndex + 1]) {
 }
 
 
-const forceVerbose = args.includes('--verbose');
 const forceDebug = args.includes('--debug');
 const silentMode = args.includes('--silent');
 const showTitles = args.includes('--titles');
@@ -337,12 +348,16 @@ const disableInteract = args.includes('--no-interact');
 const globalGhostCursor = args.includes('--ghost-cursor');
 const plainOutput = args.includes('--plain');
 const enableCDP = args.includes('--cdp');
-const dnsmasqMode = args.includes('--dnsmasq');
-const dnsmasqOldMode = args.includes('--dnsmasq-old');
-const unboundMode = args.includes('--unbound');
+// These six are reassigned to false by the incompatible-flag validation
+// blocks below (e.g. --dnsmasq + --unbound), so they must be `let` — as
+// `const` that fallback threw "Assignment to constant variable" the moment
+// two conflicting output modes were combined.
+let dnsmasqMode = args.includes('--dnsmasq');
+let dnsmasqOldMode = args.includes('--dnsmasq-old');
+let unboundMode = args.includes('--unbound');
 const removeDupes = args.includes('--remove-dupes') || args.includes('--remove-dubes');
-const privoxyMode = args.includes('--privoxy');
-const piholeMode = args.includes('--pihole');
+let privoxyMode = args.includes('--privoxy');
+let piholeMode = args.includes('--pihole');
 const globalEvalOnDoc = args.includes('--eval-on-doc'); // For Fetch/XHR interception
 const dryRunMode = args.includes('--dry-run');
 const compressLogs = args.includes('--compress-logs');
@@ -363,6 +378,21 @@ if (dnsCacheMode) enableDiskCache();
 const dnsPrecheckEnabled = !args.includes('--no-dns-precheck');
 const dnsPrecheckTimeoutMs = 2000;
 
+// --show-dead-domains: collect hostnames that are definitively DEAD (do not
+// exist / unreachable) and print them at the end of the scan so they can be
+// pruned. Only hard signals count — NXDOMAIN/ENODATA from the pre-check and
+// ERR_NAME_NOT_RESOLVED / ERR_ADDRESS_UNREACHABLE from navigation. Transient
+// failures (403/429 blocks, timeouts, Cloudflare challenges) mean the domain is
+// ALIVE and are deliberately excluded. host -> reason (first seen).
+const showDeadDomains = args.includes('--show-dead-domains');
+const _deadDomains = new Map();
+function recordDeadDomain(urlOrHost, reason) {
+  if (!showDeadDomains || !urlOrHost) return;
+  let host = urlOrHost;
+  try { host = new URL(urlOrHost).hostname; } catch { /* already a bare host */ }
+  if (host && !_deadDomains.has(host)) _deadDomains.set(host, reason);
+}
+
 // Per-scan cache of negative DNS lookups. OS resolvers don't always cache
 // NXDOMAIN responses, and a scan can hit the same dead hostname many times
 // (different URL paths on the same site). Positive results are left to the
@@ -371,14 +401,67 @@ const dnsPrecheckTimeoutMs = 2000;
 // of unique dead hosts) can't grow the cache unboundedly. Same pattern as
 // the rest of the codebase's in-memory caches.
 const dnsNegativeCache = new Map(); // hostname -> { error, timestamp }
-const DNS_NEGATIVE_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
 const DNS_NEGATIVE_CACHE_MAX = 1000;
+// The negative cache holds ONLY definitive non-existence (NXDOMAIN/ENODATA) —
+// resolver errors fail open and never enter it (see the pre-check catch), so
+// persisting it can't silently drop a live host. Opt-in via --dns-cache: dead
+// hosts are remembered for DNS_NEGATIVE_PERSIST_TTL_MS and reloaded next run;
+// otherwise it's a 5-min in-memory-only cache. The persist TTL is deliberately
+// shorter than the dig/whois positive cache (20h): a domain that doesn't exist
+// now MAY get registered, and this is a domain-hunting scanner, so the dead
+// ones are re-checked twice a day rather than trusted for ~a day.
+const DNS_NEGATIVE_PERSIST_TTL_MS = 12 * 60 * 60 * 1000; // 12 hours
+const DNS_NEGATIVE_CACHE_TTL_MS = dnsCacheMode ? DNS_NEGATIVE_PERSIST_TTL_MS : 5 * 60 * 1000;
+const DNS_NEGATIVE_CACHE_FILE = path.join(__dirname, '.dnsnegcache');
+if (dnsCacheMode) {
+  // Reuse the dig/whois caches' generic load/save (atomic write, TTL + size
+  // bounded). The 'exit' flush is synchronous (writeFileSync) so it fires on
+  // any exit path, mirroring nettools' dig/whois flush.
+  loadDiskCache(DNS_NEGATIVE_CACHE_FILE, dnsNegativeCache, DNS_NEGATIVE_CACHE_TTL_MS, DNS_NEGATIVE_CACHE_MAX);
+  process.on('exit', () => saveDiskCache(DNS_NEGATIVE_CACHE_FILE, dnsNegativeCache, DNS_NEGATIVE_CACHE_TTL_MS, DNS_NEGATIVE_CACHE_MAX));
+}
 let dnsPrecheckSkips = 0;          // URLs skipped because hostname is NXDOMAIN-cached
 let dnsPositiveSkips = 0;          // URLs skipped because dig/whois cache proves resolution
 const dnsPositiveSkippedHosts = new Set(); // unique hostnames that triggered the positive skip path
-// c-ares transient codes — read-only, hoisted out of the per-task DNS
-// pre-check so we don't allocate a fresh Set per URL.
-const DNS_TRANSIENT_ERRORS = new Set(['ETIMEOUT', 'ESERVFAIL', 'EREFUSED', 'ECONNREFUSED']);
+// DNS pre-check resolver (rotation + resolution logic lives in lib/dns.js).
+// `--dns <ip[,ip...]>` (or a `dns` setting in .nwssconfig, mapped to the same
+// flag) pins/rotates an explicit resolver list; otherwise the resolv.conf
+// nameservers are rotated. Rotation spreads the c-ares burst so one server
+// (e.g. a flaky ISP resolver) doesn't absorb every query and answer REFUSED.
+const dnsServerIndex = args.findIndex(arg => arg === '--dns');
+const dnsServersOverride = (dnsServerIndex !== -1 && args[dnsServerIndex + 1])
+  ? parseDnsServers(args[dnsServerIndex + 1])
+  : [];
+const dnsResolver = createRotatingResolver({ servers: dnsServersOverride, forceDebug });
+// Route nettools' dig through the same --dns resolvers (dig otherwise uses the
+// system /etc/resolv.conf, which on a flaky setup times out and silently drops
+// dig-gated domains). Only when --dns is explicitly set.
+if (dnsServersOverride.length > 0) setDigResolvers(dnsServersOverride);
+// Circuit breaker: if resolver errors dominate, suspend the pre-check for a
+// cooldown so a refusal storm doesn't keep hammering a broken resolver (sites
+// still load — a suspended pre-check just proceeds to navigation).
+const dnsBreaker = createDnsCircuitBreaker({ forceDebug });
+if (dnsResolver.pinned && !silentMode) {
+  const how = dnsResolver.servers.length === 1 ? 'pinned to' : 'rotating';
+  console.log(formatLogMessage('info', `DNS pre-check ${how} ${dnsResolver.servers.join(', ')}`));
+} else if (forceDebug && dnsResolver.rotates) {
+  console.log(formatLogMessage('debug', `DNS pre-check rotating ${dnsResolver.servers.length} resolv.conf nameservers: ${dnsResolver.servers.join(', ')}`));
+}
+
+// Idle-hang watchdog registry: in-flight main pages, iterable (the
+// browserhealth page trackers are WeakMaps and can't be scanned). Registered
+// when a task starts navigating, removed on completion. The hang check probes
+// these ONLY while global progress is stalled and force-closes any page that is
+// unresponsive across consecutive probes — recovering a single hung URL in ~the
+// hang-check window instead of waiting out its full per-URL ceiling (which is
+// the backstop). Acting only during a stall + requiring unresponsiveness avoids
+// killing a page that's merely slow (a page in a config delay is idle but
+// RESPONDS to a trivial evaluate; a hung one does not). Entries self-heal via
+// isClosed() so timeout/error paths that skip the normal close can't leak.
+const _inFlightPages = new Map(); // page -> { url, unresponsiveStrikes }
+const PAGE_HANG_PROBE_TIMEOUT_MS = 2000;   // liveness-probe (page.evaluate) cap; no response within this = hung
+const PAGE_HANG_PROBE_INTERVAL_MS = 15000; // how often to probe in-flight pages while the scan is stalled
+const PAGE_HANG_STRIKES_TO_KILL = 2;       // consecutive HUNG probes before force-close (~30s recovery at the 15s interval)
 
 function dnsNegativeCacheSet(hostname, error) {
   if (dnsNegativeCache.size >= DNS_NEGATIVE_CACHE_MAX) {
@@ -691,7 +774,6 @@ Per-config settings file (.nwssconfig):
   See README.md for format details.
 
 General Options:
-  --verbose                      Force verbose mode globally
   --debug                        Force debug mode globally
   --silent                       Suppress normal console logs
   --titles                       Add ! <url> title before each site's group
@@ -721,10 +803,16 @@ General Options:
 
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
-  --dns-cache                    Persist dig/whois results to disk between runs (20h TTL, 2000-entry cap each)
+  --dns <ip[,ip,...]>            Resolver(s) for the DNS pre-check AND nettools' dig (not Chrome nav / whois).
+                                 One pins all queries to it; several rotate per query. Overrides /etc/resolv.conf.
+  --dns-cache                    Persist dig/whois results to disk between runs (20h TTL, 2000-entry cap each),
+                                 plus the DNS pre-check negative cache (NXDOMAIN only, 12h TTL, .dnsnegcache)
   --no-dns-precheck              Disable per-URL DNS resolution check before page navigation.
                                  By default, URLs whose hostname doesn't resolve are skipped
                                  immediately (saves ~5-15s of Puppeteer time per dead host).
+  --show-dead-domains            At end of scan, list hostnames that did not resolve / were
+                                 unreachable (NXDOMAIN/ENODATA + ERR_NAME_NOT_RESOLVED/ERR_ADDRESS_UNREACHABLE).
+                                 Excludes blocks/timeouts (those mean the domain is alive). For pruning.
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -741,7 +829,7 @@ Global config.json options:
   ignore_similar: true/false                      Ignore domains similar to already found domains (default: true)
   ignore_similar_threshold: 80                    Similarity threshold percentage for ignore_similar (default: 80)
   ignore_similar_ignored_domains: true/false      Ignore domains similar to ignoreDomains list (default: true)
-  max_concurrent_sites: 8                        Maximum concurrent site processing (1-50, default: 8)
+  max_concurrent_sites: 6                        Maximum concurrent site processing (1-50, default: 6)
   resource_cleanup_interval: 80                  Browser restart interval in URLs processed (1-1000, default: 80)
   disable_ad_tagging: true/false                 Disable Chrome AdTagging to prevent ad frame throttling (default: true)
 
@@ -752,8 +840,7 @@ Per-site config.json options:
                                               When true, ALL regex patterns must match the same URL
   
 Redirect Handling Options:
-  follow_redirects: true/false               Follow redirects to new domains (default: true)
-  max_redirects: 10                          Maximum number of redirects to follow (default: 10)
+  max_redirects: 10                          Maximum number of redirects to follow (default: 10; 0 = follow none)
   js_redirect_timeout: 5000                  Milliseconds to wait for JavaScript redirects (default: 5000)
   detect_js_patterns: true/false             Analyze page source for redirect patterns (default: true)
   redirect_timeout_multiplier: 1.5          Increase timeout for redirected URLs (default: 1.5)
@@ -1525,7 +1612,12 @@ function matchesDynamicBlock(domain) {
   return _domainOrParentInSet(_dynamicallyBlockedDomains, domain);
 }
 
-function matchesIgnoreDomain(domain, ignorePatterns) {
+// `_ignorePatterns` is intentionally unused (underscore-marked): every caller
+// and the grep/curl/nettools/searchstring callback contract pass the ignore
+// list as a 2nd arg, but the ignore-state actually lives in the module-level
+// _dynamicallyIgnoredDomains / _ignoreDomainsExact Sets walked below. Kept in
+// the signature only to preserve that shared call shape.
+function matchesIgnoreDomain(domain, _ignorePatterns) {
   // Both dynamic and static ignore lists are walked parent-by-parent so a
   // subdomain of an ignored root inherits the ignore. Previously the
   // dynamic check was exact-only, creating an asymmetry: a static-config
@@ -2116,22 +2208,17 @@ function setupFrameHandling(page, forceDebug) {
       bypass_cache
     } = siteConfig;
     
-    const allowFirstParty = firstParty === true || firstParty === 1;
-    const allowThirdParty = thirdParty === undefined || thirdParty === true || thirdParty === 1;
     const perSiteSubDomains = subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhostIP = localhost || null;
-    const cloudflarePhishBypass = cloudflare_phish === true;
-    const cloudflareBypass = cloudflare_bypass === true;
     // Add redirect and same-page loop protection
-    const MAX_REDIRECT_DEPTH = siteConfig.max_redirects || 10;
+    // Number check (not ||) so max_redirects: 0 isn't swallowed as falsy → 10.
+    const MAX_REDIRECT_DEPTH = (typeof siteConfig.max_redirects === 'number' && siteConfig.max_redirects >= 0)
+      ? siteConfig.max_redirects : 10;
     const redirectHistory = new Set();
     let redirectCount = 0;
     const pageLoadHistory = new Map(); // Track same-page reloads
     const MAX_SAME_PAGE_LOADS = 3;
     let currentPageUrl = currentUrl;
 
-    const sitePrivoxy = privoxy === true;
-    const sitePihole = pihole === true;
     const flowproxyDetection = flowproxy_detection === true;
     
     const evenBlocked = even_blocked === true;
@@ -2298,6 +2385,9 @@ function setupFrameHandling(page, forceDebug) {
 
       // Track page for realtime cleanup
       trackPageForRealtime(page);
+      // Register with the idle-hang watchdog (force-closed if it goes
+      // unresponsive while the whole scan has stalled).
+      _inFlightPages.set(page, { url: currentUrl, unresponsiveStrikes: 0 });
 
       // Mark page as actively processing
       updatePageUsage(page, true);
@@ -2750,7 +2840,7 @@ function setupFrameHandling(page, forceDebug) {
         if (!useObscura && siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
           const userAgentKey = siteConfig.userAgent.toLowerCase();
           let platform = 'Windows';
-          let platformVersion = '15.0.0';
+          let platformVersion = '19.0.0'; // Win11 — MUST match fingerprint.js's userAgentData platformVersion
           let arch = 'x86';
           
           if (userAgentKey === 'chrome_mac') {
@@ -2769,21 +2859,46 @@ function setupFrameHandling(page, forceDebug) {
           // never drift out of sync with navigator.userAgent. The version
           // used to be hardcoded ('146') while the UA list moved to 148 —
           // a detector cross-checking UA vs Sec-CH-UA saw the mismatch.
-          // Chrome's UA-reduction means the full version is "<major>.0.0.0".
+          // The full-version hints carry the REAL build (major.0.BUILD) — the
+          // reduced UA hides it, these reveal it. Build comes from
+          // lib/fingerprint's CHROME_BUILD, the same source the JS
+          // getHighEntropyValues spoof uses, so HTTP and JS can't disagree.
           const browserUa = USER_AGENT_COLLECTIONS.get(userAgentKey) || '';
           const chromeMajor = (browserUa.match(/Chrome\/(\d+)/) || [])[1] || '148';
-          const fullVer = `${chromeMajor}.0.0.0`;
+          const fullVer = `${chromeMajor}.0.${CHROME_BUILD}`;
 
-          await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': `"Not:A-Brand";v="99", "Google Chrome";v="${chromeMajor}", "Chromium";v="${chromeMajor}"`,
+          const chHeaders = {
+            // Brand list order + grease string match real Chrome of this major
+            // exactly (deterministic GREASE): Chromium, Google Chrome, <grease>.
+            // Same order/grease the JS brands spoof uses, so HTTP and JS agree.
+            'Sec-CH-UA': `"Chromium";v="${chromeMajor}", "Google Chrome";v="${chromeMajor}", "${CHROME_GREASE_BRAND}";v="99"`,
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
+            'Sec-CH-UA-WoW64': '?0',
+            'Sec-CH-UA-Model': '""',
             'Sec-CH-UA-Full-Version': `"${fullVer}"`,
-            'Sec-CH-UA-Full-Version-List': `"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="${fullVer}", "Chromium";v="${fullVer}"`
-          });
+            'Sec-CH-UA-Full-Version-List': `"Chromium";v="${fullVer}", "Google Chrome";v="${fullVer}", "${CHROME_GREASE_BRAND}";v="99.0.0.0"`,
+            // Real Chrome (128+) sends this for desktop; pairs with the
+            // formFactors value in fingerprint.js's getHighEntropyValues spoof.
+            'Sec-CH-UA-Form-Factors': '"Desktop"'
+          };
+          // Sec-CH-Device-Memory must mirror the JS navigator.deviceMemory
+          // override (8) so a server reading BOTH can't cross-check a mismatch.
+          // That JS override lives in applyFingerprintProtection, so it only
+          // runs when fingerprint_protection is set — gate the header the same
+          // way. Without this gate, a userAgent-only site (no fp_protection)
+          // would get JS deviceMemory = the real host RAM (e.g. 32) but HTTP
+          // = 8, a fresh mismatch. With fp off we send neither and both sides
+          // report the native value, which is also consistent. (RAM isn't
+          // server-observable, so spoofing it down hides datacenter specs with
+          // nothing external to contradict — unlike rtt, which we leave native.)
+          if (siteConfig.fingerprint_protection) {
+            chHeaders['Sec-CH-Device-Memory'] = '8';
+          }
+          await page.setExtraHTTPHeaders(chHeaders);
         }
       } catch (fingerprintErr) {
         if (fingerprintErr.message.includes('Session closed') || 
@@ -2797,12 +2912,27 @@ function setupFrameHandling(page, forceDebug) {
 
       const regexes = getCompiledRegexes(siteConfig.filterRegex);
 
+      // output_regex (optional per-site): extract the rule body from each matched
+      // URL via capture group 1 (or the whole match), so output becomes
+      // ||<capture> (e.g. ||host/script/) instead of ||host^ — lets a stable
+      // folder/file be blocked on a host that also serves legit content. Compiled
+      // silently here; config-load validation (validate_rules) warns on a bad
+      // pattern, so a throw here just disables the feature for this site.
+      // Reuse the memoized regex compiler (same cache as filterRegex) so the
+      // pattern compiles once per unique source, not once per URL. try/catch
+      // because getCompiledRegex throws on a bad pattern — config-load
+      // validation already warned; a throw here just disables the feature.
+      let outputRegex = null;
+      if (siteConfig.output_regex) {
+        try { outputRegex = getCompiledRegexes(siteConfig.output_regex)[0] || null; } catch (_) { outputRegex = null; }
+      }
+
       // NEW: Get regex_and setting (defaults to false for backward compatibility)
       const useRegexAnd = siteConfig.regex_and === true;
 
    // Parse searchstring patterns using module
    const { searchStrings, searchStringsAnd, hasSearchString, hasSearchStringAnd } = parseSearchStrings(siteConfig.searchstring, siteConfig.searchstring_and);
-   const useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring
+   let useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring (reassigned to false below if curl is unavailable)
    let useGrep = siteConfig.grep === true; // Grep can work independently
 
    // Get user agent for curl if needed
@@ -2984,9 +3114,30 @@ function setupFrameHandling(page, forceDebug) {
        * @param {string} fullSubdomain - Full subdomain for cache tracking
        * @param {string} resourceType - Resource type (for --adblock-rules mode)
        */
-      function addMatchedDomain(domain, resourceType = null, fullSubdomain = null) {
+      function addMatchedDomain(domain, resourceType = null, fullSubdomain = null, matchedUrl = null) {
        // Use fullSubdomain for cache tracking if provided, otherwise fall back to domain
        const cacheKey = fullSubdomain || domain;
+       // output_regex: derive the rule body from the matched URL. Capture group 1
+       // (or the whole match) becomes the stored key, e.g. "host/script/", which
+       // formatDomain emits as ||host/script/ for adblock and falls back to the
+       // bare host for domain-only formats. All similarity / dedup / smart-cache
+       // logic below still runs on the bare host (domain); only the final stored
+       // key changes. The capture must contain both '/' and '.' (i.e. host+path),
+       // otherwise we keep the host so a mis-written regex can't emit garbage.
+       let outputKey = domain;
+       if (outputRegex && matchedUrl) {
+         const m = matchedUrl.match(outputRegex);
+         if (m) {
+           const cap = (m[1] != null ? m[1] : m[0]);
+           // Accept only a host+path shape: a '/' with a real host before it
+           // (segment before the first '/' must contain a '.'). Rejects a
+           // capture that accidentally includes the scheme (host part would be
+           // "https:") or a path-only capture with no host — both fall back to
+           // the bare-host ||host^ rule rather than emit garbage.
+           const sl = cap ? cap.indexOf('/') : -1;
+           if (sl > 0 && cap.slice(0, sl).includes('.')) outputKey = cap;
+         }
+       }
        // Check if we should ignore similar domains
        const ignoreSimilarEnabled = siteConfig.ignore_similar !== undefined ? siteConfig.ignore_similar : ignore_similar;
        const similarityThreshold = siteConfig.ignore_similar_threshold || ignore_similar_threshold;
@@ -3088,15 +3239,15 @@ function setupFrameHandling(page, forceDebug) {
       }
 
         if (matchedDomains instanceof Map) {
-          if (!matchedDomains.has(domain)) {
-            matchedDomains.set(domain, new Set());
+          if (!matchedDomains.has(outputKey)) {
+            matchedDomains.set(outputKey, new Set());
           }
           // Only add the specific resourceType that was matched, not all types for this domain
           if (resourceType) {
-            matchedDomains.get(domain).add(resourceType);
+            matchedDomains.get(outputKey).add(resourceType);
           }
         } else {
-          matchedDomains.add(domain);
+          matchedDomains.add(outputKey);
         }
       }
 
@@ -3135,12 +3286,17 @@ function setupFrameHandling(page, forceDebug) {
       // fall back to the default rather than silently disabling capture.
       const POPUP_MAX_DEPTH = (() => {
         const v = parseInt(siteConfig.capture_popups_max_depth, 10);
-        return Number.isFinite(v) && v > 0 ? v : 2;
+        return Number.isFinite(v) && v > 0 ? v : 4;
       })();
       const POPUP_CAPTURE_WINDOW_MS = (() => {
         const v = parseInt(siteConfig.capture_popups_window_ms, 10);
         return Number.isFinite(v) && v > 0 ? v : 5000;
       })();
+      // interact_popups: click inside captured popups so they cascade to their
+      // next ad/redirect (requires capture_popups — no popups exist otherwise).
+      // Light pass; the request listener catches whatever the clicks surface.
+      const interactPopups = capturePopups && siteConfig.interact_popups === true;
+      const POPUP_INTERACT_CLICKS = 3; // enough to fire popunder/redirect SDKs (incl. SDKs that suppress the 1st/2nd click as warmup) without runaway cascades
 
       if (capturePopups && forceDebug) {
         // One-time setup-time warning if the click prerequisite isn't met.
@@ -3306,7 +3462,7 @@ function setupFrameHandling(page, forceDebug) {
               trackNetToolsHandler(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
             } else {
               // No nettools required — regex match alone counts.
-              addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
+              addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain, checkedUrl);
             }
           } catch (_) { /* observation-only — never let a popup error escape */ }
         };
@@ -3428,6 +3584,24 @@ function setupFrameHandling(page, forceDebug) {
 
           attachPopupRequestCapture(popupPage, depth);
 
+          // interact_popups: click inside the popup so it can cascade to its next
+          // ad/redirect — popunder/redirect SDKs fire on a document-level click,
+          // and a captured-but-unclicked popup only ever shows its landing URL.
+          // Light pass (POPUP_INTERACT_CLICKS random content-zone clicks), only
+          // on popups shallower than max depth so a clicked popup's spawned child
+          // (depth+1) is still within the capture depth. Fire-and-forget: it must
+          // not block onTargetCreated, and the popup may close/navigate mid-click
+          // (performContentClicks no-ops on a closed page). The request listener
+          // above captures whatever the clicks surface; the close timer bounds it.
+          if (interactPopups && depth < POPUP_MAX_DEPTH && !popupPage.isClosed()) {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup depth=${depth}] interact_popups: ${POPUP_INTERACT_CLICKS} content click(s)`));
+            performContentClicks(popupPage, {
+              clicks: POPUP_INTERACT_CLICKS,
+              forceDebug,
+              realistic: siteConfig.realistic_click === true,
+            }).catch(() => {}); // popup is transient — non-fatal
+          }
+
           // Auto-close after the capture window so popups don't pile up.
           const closeTimer = setTimeout(() => {
             try { if (!popupPage.isClosed()) popupPage.close().catch(() => {}); } catch (_) {}
@@ -3633,7 +3807,7 @@ function setupFrameHandling(page, forceDebug) {
                         wasBlocked: true
                       });
                     } else {
-                      addMatchedDomain(reqDomain, resourceType, fullSubdomain);
+                      addMatchedDomain(reqDomain, resourceType, fullSubdomain, reqUrl);
                     }
                     matchedRegexPatterns.add(evenBlockedRegexPattern);
 
@@ -3811,7 +3985,10 @@ function setupFrameHandling(page, forceDebug) {
                  isFirstParty: isFirstParty
                });
              } else {
-               addMatchedDomain(reqDomain, resourceType);
+               // Pass null for fullSubdomain (not the in-scope hostname) to keep
+               // this path's dedup key as the root domain exactly as before —
+               // only matchedUrl is new here, for output_regex.
+               addMatchedDomain(reqDomain, resourceType, null, reqUrl);
              }
              if (matchedRegexPattern) matchedRegexPatterns.add(matchedRegexPattern);
              if (siteConfig.verbose === 1) {
@@ -4450,12 +4627,17 @@ function setupFrameHandling(page, forceDebug) {
             }
           }
           console.error(formatLogMessage('error', `Failed on ${currentUrl}: ${err.message}`));
+          // Capture hard "dead domain" navigation errors for --show-dead-domains
+          // (DNS doesn't resolve / host unreachable). Blocks, timeouts and CF
+          // challenges are NOT dead — they're excluded by this match.
+          const deadNav = /ERR_NAME_NOT_RESOLVED|ERR_ADDRESS_UNREACHABLE|ERR_DNS/.exec(err.message || '');
+          if (deadNav) recordDeadDomain(currentUrl, deadNav[0]);
           throw err;
         }
       }
       }
 
-      const delayMs = siteConfig.delay || DEFAULT_DELAY;
+      const delayMs = siteConfig.delay || TIMEOUTS.DEFAULT_DELAY;
 
       // Optimized delays for Puppeteer 23.x performance
       const isFastSite = timeout <= TIMEOUTS.FAST_SITE_THRESHOLD;
@@ -4535,8 +4717,21 @@ function setupFrameHandling(page, forceDebug) {
                 const ghostStart = Date.now();
                 const ghostTimeLeft = () => ghostDuration - (Date.now() - ghostStart);
 
-                // Time-based Bezier mouse movements — runs for ghostDuration ms
-                while (ghostTimeLeft() > 200) {
+                // Honor interact_click_count in ghost mode too (built-in default
+                // is 3 — ad SDKs often swallow the 1st/2nd click as warmup). Same
+                // default + 20-cap as the built-in content-click path. 0 when
+                // element clicks are disabled.
+                const ghostClickCount = interactionConfig.includeElementClicks
+                  ? Math.min(Math.max(Number(siteConfig.interact_click_count) || 3, 1), 20)
+                  : 0;
+                // Reserve part of the duration budget for those clicks so the
+                // movement loop doesn't consume all of ghost_cursor_duration.
+                // Capped at half the budget so movement still happens; raise
+                // ghost_cursor_duration to fit more clicks.
+                const clickReserveMs = Math.min(ghostClickCount * 600, ghostDuration * 0.5);
+
+                // Time-based Bezier mouse movements — runs for the unreserved budget
+                while (ghostTimeLeft() > 200 + clickReserveMs) {
                   const toX = Math.floor(Math.random() * (viewport.width - 100)) + 50;
                   const toY = Math.floor(Math.random() * (viewport.height - 100)) + 50;
                   await ghostMove(cursor, toX, toY, {
@@ -4544,18 +4739,23 @@ function setupFrameHandling(page, forceDebug) {
                     overshootThreshold: ghostConfig.overshootThreshold,
                     forceDebug
                   });
-                  if (ghostTimeLeft() > 100) {
+                  if (ghostTimeLeft() > 100 + clickReserveMs) {
                     await new Promise(r => setTimeout(r, 25 + Math.random() * 75));
                   }
                 }
                 if (ghostTimeLeft() > 100 && Math.random() < 0.3) {
                   await ghostRandomMove(cursor, { forceDebug });
                 }
-                if (interactionConfig.includeElementClicks && ghostTimeLeft() > 100) {
+                // interact_click_count clicks, each to a fresh content-zone point.
+                // The time guard stops early if the budget runs out (raise
+                // ghost_cursor_duration for more).
+                for (let gc = 0; gc < ghostClickCount && ghostTimeLeft() > 100; gc++) {
                   const clickX = Math.floor(viewport.width * 0.2 + Math.random() * viewport.width * 0.6);
                   const clickY = Math.floor(viewport.height * 0.2 + Math.random() * viewport.height * 0.6);
                   await ghostClick(cursor, { x: clickX, y: clickY }, {
                     hesitate: ghostConfig.hesitate,
+                    page,
+                    realistic: siteConfig.realistic_click === true,
                     forceDebug
                   });
                 }
@@ -4870,7 +5070,7 @@ function setupFrameHandling(page, forceDebug) {
       // Only add delay if we're continuing with more reloads
       if (i < totalReloads) {
     // Reduce delay for problematic sites
-    const adjustedDelay = i > 1 ? Math.min(DEFAULT_DELAY, 2000) : DEFAULT_DELAY;
+    const adjustedDelay = i > 1 ? Math.min(TIMEOUTS.DEFAULT_DELAY, 2000) : TIMEOUTS.DEFAULT_DELAY;
     await fastTimeout(adjustedDelay);
       }
     }
@@ -5074,6 +5274,7 @@ function setupFrameHandling(page, forceDebug) {
         if (!keepBrowserOpen) {
           try {
             untrackPage(page);
+            _inFlightPages.delete(page);
             await page.close();
             if (forceDebug) console.log(formatLogMessage('debug', `Page closed for ${currentUrl}`));
           } catch (pageCloseErr) {
@@ -5174,6 +5375,12 @@ function setupFrameHandling(page, forceDebug) {
  let lastProcessedCount = 0;
  let hangCheckCount = 0;
  let forceRestartFlag = false; // Flag to trigger restart on next iteration
+ // Largest per-URL timeout budget seen across tasks. The hang-check restart
+ // scales to this so it can't false-fire on a legitimately-slow config (high
+ // delay × reload × interact) whose per-URL budget exceeds a flat threshold —
+ // the emergency restart should only fire once the per-URL timeout ITSELF has
+ // had its chance and failed (a true browser hang).
+ let maxPerUrlTimeoutMs = 0;
 
  // Precomputed colored '[HANG CHECK]' subsystem prefix. formatLogMessage
  // only colors the [severity] tag; the '[HANG CHECK]' substring was
@@ -5181,6 +5388,48 @@ function setupFrameHandling(page, forceDebug) {
  // entry so the interval callback doesn't re-colorize per tick.
  const HANG_CHECK_TAG = messageColors.processing('[HANG CHECK]');
 
+ // Idle-hang watchdog. Runs only while the scan is stalled (no URL completing).
+ // The probe distinguishes a HUNG renderer from one that's merely NAVIGATING,
+ // which is the key to probing aggressively without false-kills:
+ //   - evaluate resolves            -> 'alive'      -> reset strikes
+ //   - evaluate rejects fast (e.g. "Execution context destroyed" mid goto/
+ //     reload)                       -> 'navigating' -> inconclusive: neither
+ //                                                      strike nor reset, so a
+ //                                                      navigation can NEVER trip
+ //                                                      the kill regardless of cadence
+ //   - no response within the cap    -> 'hung'       -> strike
+ // PAGE_HANG_STRIKES_TO_KILL consecutive HUNG probes force-close the page, so the
+ // stuck task's awaits reject and its batch completes instead of waiting out the
+ // full per-URL ceiling. Parallel, guarded against overlap; zero overhead off a stall.
+ let _hangProbeInProgress = false;
+ const probeInFlightPagesForHang = async () => {
+   if (_hangProbeInProgress || _inFlightPages.size === 0) return;
+   _hangProbeInProgress = true;
+   try {
+     await Promise.all([..._inFlightPages.entries()].map(async ([page, info]) => {
+       if (page.isClosed()) { _inFlightPages.delete(page); return; }
+       let verdict;
+       try {
+         verdict = await Promise.race([
+           page.evaluate(() => true).then(() => 'alive', () => 'navigating'),
+           new Promise(r => setTimeout(() => r('hung'), PAGE_HANG_PROBE_TIMEOUT_MS)),
+         ]);
+       } catch { verdict = 'hung'; }
+       if (verdict === 'alive') { info.unresponsiveStrikes = 0; return; }
+       if (verdict === 'navigating') return; // context destroyed mid-nav — not a hang; don't strike or reset
+       // verdict === 'hung' — renderer gave no response within the cap
+       info.unresponsiveStrikes++;
+       if (info.unresponsiveStrikes >= PAGE_HANG_STRIKES_TO_KILL) {
+         console.log(formatLogMessage('warn', `${HANG_CHECK_TAG} Force-closing hung page after ${info.unresponsiveStrikes} unresponsive probes: ${info.url}`));
+         _inFlightPages.delete(page);
+         page.close().catch(() => {}); // stuck task's awaits reject -> task errors -> batch completes
+       }
+     }));
+   } finally {
+     _hangProbeInProgress = false;
+   }
+ };
+
  const hangDetectionInterval = setInterval(() => {
    // Progress check, counter, and forceRestartFlag MUST run regardless of
    // debug mode — previously the entire body was gated on forceDebug, which
@@ -5193,8 +5442,18 @@ function setupFrameHandling(page, forceDebug) {
      if (forceDebug) {
        console.log(formatLogMessage('warn', `${HANG_CHECK_TAG} No progress for ${hangCheckCount * 30}s`));
      }
-     if (hangCheckCount >= 5) {
-       console.log(formatLogMessage('error', `${HANG_CHECK_TAG} Hung for 2.5 minutes. Triggering emergency browser restart.`));
+     // The faster 15s probe interval below does surgical per-page recovery; this
+     // 30s interval owns only the slower nuclear-restart escalation. Deadline-
+     // aware: the restart only fires once the stall has OUTLASTED the heaviest
+     // in-flight per-URL budget (+ grace) — i.e. the per-URL timeout itself had
+     // its chance and failed, a true hang. A flat threshold (the old 2.5min)
+     // false-fires on legitimately-slow configs (high delay × reload × interact)
+     // whose per-URL budget exceeds it, restarting the browser mid-work. Floor
+     // at 150s so light configs behave exactly as before.
+     // +45s buffer covers the per-URL 8s orphan grace + the 30s tick granularity + slack.
+     const restartAfterMs = Math.max(150000, maxPerUrlTimeoutMs + 45000);
+     if (hangCheckCount * 30000 >= restartAfterMs) {
+       console.log(formatLogMessage('error', `${HANG_CHECK_TAG} No progress for ${Math.round(hangCheckCount * 30)}s (past the ${Math.round(restartAfterMs / 1000)}s per-URL budget). Triggering emergency browser restart.`));
        forceRestartFlag = true; // Set flag instead of exiting
        hangCheckCount = 0; // Reset counter for next cycle
      }
@@ -5216,6 +5475,22 @@ function setupFrameHandling(page, forceDebug) {
  // cleanup, this is belt-and-suspenders in case a future refactor moves them.
  hangDetectionInterval.unref();
 
+ // Fast surgical recovery on its own 15s cadence (the 30s interval above owns
+ // the slower nuclear-restart escalation). Probes in-flight pages only while
+ // progress is stalled and force-closes confirmed-hung ones; clears strikes when
+ // progress resumes so a fresh stall starts from zero. Starts at -1 so the very
+ // first window is grace (processedUrlCount begins at 0).
+ let lastProbeCount = -1;
+ const pageHangProbeInterval = setInterval(() => {
+   if (processedUrlCount === lastProbeCount) {
+     probeInFlightPagesForHang(); // fire-and-forget; self-guarded against overlap
+   } else {
+     for (const info of _inFlightPages.values()) info.unresponsiveStrikes = 0;
+   }
+   lastProbeCount = processedUrlCount;
+ }, PAGE_HANG_PROBE_INTERVAL_MS);
+ pageHangProbeInterval.unref();
+
  // Process URLs in batches with exception handling
  let siteGroupIndex = 0;
  let currentProxyKey = '';  // Track active proxy config — '' means direct connection
@@ -5500,58 +5775,38 @@ function setupFrameHandling(page, forceDebug) {
            dnsPositiveSkippedHosts.add(taskDomain);
            if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check skipped (dig/whois cache confirms resolution): ${taskDomain}`));
            // Fall through to navigation -- pre-check "passed" by proxy.
+         } else if (dnsBreaker.isTripped()) {
+           // Resolver is in a refusal storm — pre-checking is futile and only
+           // adds load. Skip the resolve and proceed to navigation (same effect
+           // as a fail-open); no breaker record since no resolve happened.
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check suspended (resolver circuit open) — proceeding: ${taskDomain}`));
          } else {
-         const dnsResolve = async () => {
-           // resolve4 first; on no-IPv4 (ENODATA / ENOTFOUND) fall back to
-           // resolve6 so IPv6-only hosts aren't wrongly skipped. ANY OTHER
-           // error code (ESERVFAIL, ETIMEOUT, EREFUSED, etc.) propagates
-           // unchanged so the outer transient-retry path sees the real
-           // resolver code and the negative cache records the right reason.
-           // Previously a bare .catch swallowed everything and tried
-           // resolve6, which masked transient v4-side errors behind
-           // whatever resolve6 ended up reporting.
-           // 2s timeout kept as a real safety net — with c-ares off the
-           // threadpool it should now rarely fire.
-           let timer;
-           try {
-             const timeoutP = new Promise((_, reject) => {
-               timer = setTimeout(() => reject(new Error('DNS timeout')), dnsPrecheckTimeoutMs);
-             });
-             const resolveChain = dnsPromises.resolve4(taskDomain)
-               .catch(err => {
-                 if (err && (err.code === 'ENODATA' || err.code === 'ENOTFOUND')) {
-                   return dnsPromises.resolve6(taskDomain);
-                 }
-                 throw err;
-               });
-             await Promise.race([resolveChain, timeoutP]);
-           } finally {
-             if (timer) clearTimeout(timer);
-           }
-         };
-         // c-ares transient codes — retry once so a momentary resolver
-         // hiccup doesn't poison the negative cache for 5 minutes.
-         // DNS_TRANSIENT_ERRORS is module-level so we don't allocate per task.
          try {
-           try {
-             await dnsResolve();
-           } catch (firstErr) {
-             const code = firstErr && firstErr.code;
-             if (DNS_TRANSIENT_ERRORS.has(code) || (firstErr && firstErr.message === 'DNS timeout')) {
-               if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check transient (${code || 'timeout'}) for ${taskDomain}, retrying once`));
-               await dnsResolve();
-             } else {
-               throw firstErr;
-             }
-           }
+           // Rotates the lead nameserver per attempt and retries once on a
+           // transient error; rejects with the final error (code intact) on
+           // failure. See lib/dns.js.
+           await dnsResolver.resolveHost(taskDomain, dnsPrecheckTimeoutMs);
+           dnsBreaker.record(false); // resolved OK — resolver healthy
          } catch (dnsErr) {
            const errCode = dnsErr.code || dnsErr.message || 'DNS resolve failed';
-           dnsNegativeCacheSet(taskDomain, errCode);
-           dnsPrecheckSkips++;
-           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
-           return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+           // Only a definitive "host does not exist / has no address" answer
+           // (ENOTFOUND/ENODATA) justifies dropping the URL. A resolver-level
+           // failure (EREFUSED/ESERVFAIL/ETIMEOUT/ECONNREFUSED/timeout) says
+           // nothing about whether the domain is live — fail open: don't cache,
+           // don't skip, let it proceed to real browser navigation (a genuinely
+           // dead host still fails fast there).
+           if (isNonExistenceError(errCode)) {
+             dnsBreaker.record(false); // resolver answered NXDOMAIN — healthy
+             dnsNegativeCacheSet(taskDomain, errCode);
+             recordDeadDomain(taskDomain, errCode);
+             dnsPrecheckSkips++;
+             if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
+             return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+           }
+           dnsBreaker.record(true); // resolver error — counts toward tripping the circuit
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check inconclusive (${errCode}) for ${taskDomain} — proceeding (resolver issue, not a dead host)`));
          }
-         } // close `else` from domainKnownToResolve shortcut above
+         } // close the resolve `else` (domainKnownToResolve / circuit-open shortcuts above)
        }
      } catch {}
 
@@ -5584,6 +5839,9 @@ function setupFrameHandling(page, forceDebug) {
          + ((task.config.delay || 0) + INTERACTION_OVERHEAD_MS) * (1 + reloadCount)
          + 30000
      );
+     // Feed the hang-check restart so it never escalates before this URL's own
+     // timeout could have fired (see maxPerUrlTimeoutMs).
+     if (PER_URL_TIMEOUT_MS > maxPerUrlTimeoutMs) maxPerUrlTimeoutMs = PER_URL_TIMEOUT_MS;
      // Grace period after primary timeout — gives the orphan a chance to
      // finish drainPendingNetTools() and emit "Saving N rules despite page
      // load failure" before we abandon its result. Drain typically completes
@@ -5843,11 +6101,13 @@ function setupFrameHandling(page, forceDebug) {
  } catch (processingError) {
    console.log(formatLogMessage('error', `Critical error: ${processingError.message}`));
    clearInterval(hangDetectionInterval);
+   clearInterval(pageHangProbeInterval);
    throw processingError;
   }
 
-  // Clear hang detection interval
+  // Clear hang detection intervals
   clearInterval(hangDetectionInterval);
+  clearInterval(pageHangProbeInterval);
 
   // === POST-SCAN PROCESSING ===
   // Clean up first-party domains and validate results
@@ -5929,7 +6189,6 @@ function setupFrameHandling(page, forceDebug) {
   const totalMatches = results.reduce((sum, r) => sum + (r.rules ? r.rules.length : 0), 0);
 
   // Debug: Show output format being used
-  const totalDomainsSkipped = getTotalDomainsSkipped();
   const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
@@ -5944,7 +6203,7 @@ function setupFrameHandling(page, forceDebug) {
     };
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
-     console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
+     console.log(formatLogMessage('debug', `Performance: ${detectedDomainsCount} unique domains cached`));
      // Cloudflare cache statistics
      const cloudflareStats = getCacheStats();
      if (cloudflareStats.size > 0) {
@@ -5973,6 +6232,13 @@ function setupFrameHandling(page, forceDebug) {
        }
        console.log(formatLogMessage('debug', `DNS pre-check skipped: ${parts.join(', ')}`));
      }
+     // Surface circuit-breaker activity in the end-of-scan summary (each trip
+     // also warns in real time). Shown outside forceDebug because a resolver
+     // refusal storm is something the operator should know happened.
+     const dnsBreakerTrips = dnsBreaker.stats().trips;
+     if (dnsBreakerTrips > 0 && !silentMode) {
+       console.log(formatLogMessage('info', `DNS pre-check circuit tripped ${dnsBreakerTrips}× this scan (resolver refusal back-off)`));
+     }
      // Blocked-pattern hit stats. Surfaces which patterns are actually
      // doing work this scan and (by absence) which are stale enough to
      // prune from config. Top 10 by hit count to keep the log scannable
@@ -6175,8 +6441,18 @@ function setupFrameHandling(page, forceDebug) {
     } else if (outputResult.totalRules > 0 && dryRunMode) {
       console.log(messageColors.success('Found') + ` ${outputResult.totalRules} total matches across all URLs`);
     }
-    if (totalDomainsSkipped > 0) {
-      console.log(messageColors.info('Performance:') + ` ${totalDomainsSkipped} domains skipped (already detected)`);
+    // --show-dead-domains: list hostnames that didn't resolve / were unreachable
+    // this scan (NXDOMAIN/ENODATA + ERR_NAME_NOT_RESOLVED/ERR_ADDRESS_UNREACHABLE).
+    // One host per line so it's greppable for pruning; reason in the trailing column.
+    if (showDeadDomains) {
+      if (_deadDomains.size > 0) {
+        console.log(`\n${messageColors.warn(`Dead domains (${_deadDomains.size}) — did not resolve / unreachable:`)}`);
+        for (const [host, reason] of [..._deadDomains].sort((a, b) => a[0].localeCompare(b[0]))) {
+          console.log(`  ${host}\t${reason}`);
+        }
+      } else {
+        console.log(`\n${messageColors.success('Dead domains: none detected')}`);
+      }
     }
     if (ignoreCache && forceDebug) {
       console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);