@fanboynz/network-scanner 3.0.2 → 3.1.0

package/nwss.js

@@ -12,13 +12,13 @@ const path = require('path');
 const dnsPromises = require('node:dns/promises');
 const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
 const { compressMultipleFiles, formatFileSize } = require('./lib/compress');
-const { parseSearchStrings, createResponseHandler, createCurlHandler } = require('./lib/searchstring');
-const { applyAllFingerprintSpoofing } = require('./lib/fingerprint');
+const { parseSearchStrings, createResponseHandler } = require('./lib/searchstring');
+const { applyAllFingerprintSpoofing, USER_AGENT_COLLECTIONS } = require('./lib/fingerprint');
 const { formatRules, handleOutput, getFormatDescription } = require('./lib/output');
 // Curl functionality (replace searchstring curl handler)
 const { validateCurlAvailability, createCurlHandler: createCurlModuleHandler } = require('./lib/curl');
 // Rule validation
-const { validateRulesetFile, validateFullConfig, testDomainValidation, cleanRulesetFile } = require('./lib/validate_rules');
+const { validateRulesetFile, validateFullConfig, testDomainValidation, cleanRulesetFile, normalizeSiteConfig } = require('./lib/validate_rules');
 // CF Bypass
 const { 
   handleCloudflareProtection,
@@ -66,7 +66,7 @@ const SMART_CACHE_TAG = messageColors.processing('[SmartCache]');
 // log lines (start/completed). Same cyan as the other monitoring tags.
 const CONCURRENCY_TAG = messageColors.processing('[CONCURRENCY]');
 // Enhanced mouse interaction and page simulation
-const { performPageInteraction, createInteractionConfig, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
+const { performPageInteraction, createInteractionConfig, computeInteractionCeilingMs, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
 // Optional ghost-cursor support for advanced Bezier-based mouse movements
 const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
 // Domain detection cache for performance optimization
@@ -129,15 +129,12 @@ const CONCURRENCY_LIMITS = Object.freeze({
 });
 
 // V8 Optimization: Use Map for user agent lookups instead of object
-const USER_AGENTS = Object.freeze(new Map([
-  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
-  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
-  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
-  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"],
-  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0"],
-  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0"],
-  ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
-]));
+// User-Agent strings come from the single source of truth in lib/fingerprint
+// (USER_AGENT_COLLECTIONS, imported above) — the same map page.setUserAgent
+// applies to the browser. The previous local duplicate had silently drifted
+// (Chrome 146 vs the browser's 148, Firefox 148 vs 151, Safari 18.6 vs 19.5),
+// so curl content-fetches advertised a different browser than the page did.
+// Keep using the imported map directly so the two can never diverge again.
 
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
 
@@ -776,13 +773,14 @@ Redirect Handling Options:
   resourceTypes: ["script", "stylesheet"]     Only process requests of these resource types (default: all types)
   interact: true/false                         Simulate mouse movements/clicks
   isBrave: true/false                          Spoof Brave browser detection
-  userAgent: "chrome"|"chrome_mac"|"chrome_linux"|"firefox"|"firefox_mac"|"firefox_linux"|"safari"  Custom desktop User-Agent
+  userAgent: "chrome"|"chrome_mac"|"chrome_linux"|"firefox"|"firefox_mac"|"firefox_linux"|"safari"  Desktop User-Agent (defaults to "chrome" if unset; set false to scan with the raw headless UA)
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
   delay: <milliseconds>                        Delay after load (default: 6000, capped at 2000ms unless delay_uncapped: true)
   delay_uncapped: true/false                   Honor 'delay' up to half the per-URL timeout instead of the 2s default cap. Use for sites with setTimeout-deferred lazy ad/tracker loaders that fire well past the standard post-networkidle window
   reload: <number>                             Reload page n times after load (default: 1)
   forcereload: true/false or ["domain1.com", "domain2.com"]  Force cache-clearing reload for all URLs or specific domains
   clear_sitedata: true/false                   Clear all cookies, cache, storage before each load (default: false)
+  clear_sitedata_full_on_reload: true/false    With clear_sitedata: true, also clear heavy storage (IndexedDB, WebSQL, service workers) between reloads — quick mode (cookies+cache+local/session storage) is the default for reloads; this flag promotes them to full clears at ~100-500ms latency cost per reload. Use for sites with IndexedDB/service-worker-backed session caps. Off by default.
   subDomains: 1/0                              Output full subdomains (default: 0)
   localhost: true/false                        Force localhost output (127.0.0.1)
   localhost_0_0_0_0: true/false                Force localhost output (0.0.0.0)
@@ -1864,15 +1862,65 @@ function setupFrameHandling(page, forceDebug) {
         '--log-level=3',               // Fatal errors only (suppresses verbose disk logging)
         '--no-service-autorun',        // No background service disk activity
         '--disable-domain-reliability', // No reliability monitor disk writes
+        // Suppress Chrome's auto-update subsystem entirely in headful runs.
+        // --disable-component-update + --disable-background-networking above
+        // stop the network-level check, but Chrome's UI can still show the
+        // "update available" toolbar dot / banner / "relaunch to update"
+        // modal if Chrome has cached state from a prior check by the same
+        // installed chrome binary. These two flags neutralize that:
+        //   simulate-outdated-no-au=DATE — the no-auto-update simulation
+        //     date is treated as DATE. Far-future date = never shows the
+        //     'outdated' UI. Quotes around the date required by Chrome.
+        //   check-for-update-interval=N — seconds between update checks.
+        //     31536000 = 1 year. Even if the above somehow gets bypassed,
+        //     the check itself won't fire within any reasonable scan.
+        // Both are no-ops in pure headless modes but matter in --headful
+        // and headless='new' (which can render UI in some cases).
+        '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"',
+        '--check-for-update-interval=31536000',
         // PERFORMANCE: Disable non-essential Chrome features in a single flag
         // IMPORTANT: Chrome only reads the LAST --disable-features flag, so combine all into one
-        // AccountConsistencyMirror + AccountConsistencyDice prevent the
-        // Chrome sign-in subsystem from initialising at startup. Combined
-        // with --disable-sync + --allow-browser-signin=false below, this
-        // suppresses the "Something went wrong when opening your profile"
-        // popup that fires in headful + --keep-open mode (temp userDataDir
-        // has no real profile, so the sync init errors out and pops up).
-        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding,AccountConsistencyMirror,AccountConsistencyDice${disable_ad_tagging ? ',AdTagging' : ''}`,
+        //
+        // Sign-in / profile suppression family (prevents the "Something went
+        // wrong when opening your profile. Please sign out then sign in
+        // again" popup that fires in headful when Chrome's sign-in/sync
+        // subsystem can't make sense of our fresh-each-launch temp
+        // userDataDir):
+        //   AccountConsistencyMirror, AccountConsistencyDice
+        //     Older Chrome's identity consistency layer. Disabling stops
+        //     the sync subsystem from initialising at startup.
+        //   ProfilePicker, EnableProfilePicker
+        //     Two names for the same Chrome feature (renamed in Chrome
+        //     ~120s). Disabling stops the profile-picker dialog that some
+        //     Chrome versions display when launching with no recognised
+        //     profile. Was the new offender in Chrome 148 for this case.
+        //   IdentityConsistency
+        //     Chrome's identity-consistency-with-google.com checks. Tries
+        //     to read profile credentials at startup; trips the popup if
+        //     profile is fresh/empty.
+        //   SyncDisabledWithProfilePicker
+        //     Sync subsystem variant that activates when profile picker
+        //     would otherwise show. Disabling is harmless when picker is
+        //     also disabled but covers the gap if a Chrome version honors
+        //     only one of the two.
+        //   SigninInterceptBubble
+        //     Sign-in interception bubble that pops when Chrome detects
+        //     'enterprise' sign-in patterns. Defensive.
+        // Combined with --disable-sync + --allow-browser-signin=false
+        // below + --profile-directory=Default flag (explicit profile name
+        // instead of letting Chrome auto-detect/pick), this should fully
+        // suppress sign-in popups in headful from Chrome 118 through 148+.
+        //
+        // ChromeWhatsNewUI: suppresses the post-update "What's New" page
+        // that auto-opens in a new tab after Chrome installs an update —
+        // not popunder-relevant but visually noisy in headful sessions.
+        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding,AccountConsistencyMirror,AccountConsistencyDice,ProfilePicker,EnableProfilePicker,IdentityConsistency,SyncDisabledWithProfilePicker,SigninInterceptBubble,ChromeWhatsNewUI${disable_ad_tagging ? ',AdTagging' : ''}`,
+        // Explicit profile directory — without this, Chrome may probe for
+        // available profiles at launch and trigger the picker dialog (or
+        // the "something went wrong" popup if no profile is found). With
+        // a fresh temp userDataDir each launch, Chrome will create
+        // 'Default' on its own; explicitly naming it skips the probe.
+        '--profile-directory=Default',
         '--disable-ipc-flooding-protection',
         '--aggressive-cache-discard',
         '--memory-pressure-off',
@@ -1931,7 +1979,20 @@ function setupFrameHandling(page, forceDebug) {
 
 
   const pLimit = (await import('p-limit')).default;
-  const limit = pLimit(MAX_CONCURRENT_SITES);
+  // VPN connect/disconnect is per-URL (wgConnect/ovpnConnect at scan start,
+  // wgDisconnect/ovpnDisconnect in the finally) and manipulates the SHARED
+  // system routing table. Interface names are derived from a hash of the VPN
+  // config and connect/disconnect is not refcounted, so two concurrent URLs
+  // that share a VPN config resolve to the same interface and one task's
+  // teardown rips the interface out from under the other mid-scan. Force
+  // serial execution whenever any site uses vpn/openvpn — correctness over
+  // throughput, and VPN scans are network-bound rather than CPU-bound anyway.
+  const vpnInUse = sites.some(site => site.vpn || site.openvpn);
+  const effectiveConcurrency = vpnInUse ? 1 : MAX_CONCURRENT_SITES;
+  if (vpnInUse && MAX_CONCURRENT_SITES > 1 && (forceDebug || !silentMode)) {
+    console.log(formatLogMessage('info', `${VPN_TAG} VPN configured — forcing concurrency 1 (was ${MAX_CONCURRENT_SITES}) to avoid routing-table races`));
+  }
+  const limit = pLimit(effectiveConcurrency);
 
   const perSiteHeadful = sites.some(site => site.headful === true);
   const launchHeadless = !(headfulMode || perSiteHeadful);
@@ -2694,23 +2755,34 @@ function setupFrameHandling(page, forceDebug) {
           
           if (userAgentKey === 'chrome_mac') {
             platform = 'macOS';
-            platformVersion = '13.5.0'; 
+            platformVersion = '13.5.0';
             arch = 'arm';
           } else if (userAgentKey === 'chrome_linux') {
             platform = 'Linux';
             platformVersion = '6.5.0';
             arch = 'x86';
           }
-                    
+
+          // Derive the Chrome major version from the SAME UA string the
+          // browser actually sends (USER_AGENT_COLLECTIONS, via
+          // page.setUserAgent in applyUserAgentSpoofing) so Sec-CH-UA can
+          // never drift out of sync with navigator.userAgent. The version
+          // used to be hardcoded ('146') while the UA list moved to 148 —
+          // a detector cross-checking UA vs Sec-CH-UA saw the mismatch.
+          // Chrome's UA-reduction means the full version is "<major>.0.0.0".
+          const browserUa = USER_AGENT_COLLECTIONS.get(userAgentKey) || '';
+          const chromeMajor = (browserUa.match(/Chrome\/(\d+)/) || [])[1] || '148';
+          const fullVer = `${chromeMajor}.0.0.0`;
+
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Not:A-Brand";v="99", "Google Chrome";v="146", "Chromium";v="146"',
+            'Sec-CH-UA': `"Not:A-Brand";v="99", "Google Chrome";v="${chromeMajor}", "Chromium";v="${chromeMajor}"`,
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"146.0.0.0"',
-            'Sec-CH-UA-Full-Version-List': '"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="146.0.0.0", "Chromium";v="146.0.0.0"'
+            'Sec-CH-UA-Full-Version': `"${fullVer}"`,
+            'Sec-CH-UA-Full-Version-List': `"Not:A-Brand";v="99.0.0.0", "Google Chrome";v="${fullVer}", "Chromium";v="${fullVer}"`
           });
         }
       } catch (fingerprintErr) {
@@ -2736,7 +2808,7 @@ function setupFrameHandling(page, forceDebug) {
    // Get user agent for curl if needed
    let curlUserAgent = '';
    if (useCurl && siteConfig.userAgent) {
-     curlUserAgent = USER_AGENTS.get(siteConfig.userAgent.toLowerCase()) || '';
+     curlUserAgent = USER_AGENT_COLLECTIONS.get(siteConfig.userAgent.toLowerCase()) || '';
    }
 
    if (useCurl && forceDebug) {
@@ -3072,10 +3144,22 @@ function setupFrameHandling(page, forceDebug) {
 
       if (capturePopups && forceDebug) {
         // One-time setup-time warning if the click prerequisite isn't met.
-        // Without clicks, capture_popups is a no-op in practice.
-        const hasClicks = siteConfig.interact === true && siteConfig.interact_clicks === true;
-        if (!hasClicks) {
-          console.log(formatLogMessage('debug', `[popup] capture_popups is enabled but interact_clicks is not — popups need user-gesture clicks to fire; expect no captures unless the page opens popups via in-page redirects`));
+        // Without clicks, capture_popups is a no-op in practice. Previous
+        // version blamed `interact_clicks` for both missing-piece cases — but
+        // when the actual culprit is `interact: 1` (number, silently disabled
+        // by strict `=== true`), the message misled users into debugging
+        // interact_clicks while the real problem was interact itself.
+        // (normalizeSiteConfig now coerces interact: 1 → true with a warning,
+        // so by the time we get here both should be booleans — but keep the
+        // diagnostic accurate for the truly-missing case.)
+        const interactOn = siteConfig.interact === true;
+        const clicksOn = siteConfig.interact_clicks === true;
+        if (!interactOn && !clicksOn) {
+          console.log(formatLogMessage('debug', `[popup] capture_popups is enabled but neither 'interact' nor 'interact_clicks' is — set BOTH to true to fire user-gesture clicks; without them, only popups opened via in-page redirects will capture`));
+        } else if (!interactOn) {
+          console.log(formatLogMessage('debug', `[popup] capture_popups is enabled but 'interact' is not — set interact: true to enable the interaction loop (interact_clicks is already set); without it, no fake clicks fire`));
+        } else if (!clicksOn) {
+          console.log(formatLogMessage('debug', `[popup] capture_popups is enabled but 'interact_clicks' is not — set interact_clicks: true to enable element-targeted clicks; without it, only random content-zone clicks fire and may miss overlay-based popunders`));
         }
         console.log(formatLogMessage('debug', `[popup] capture_popups settings: maxDepth=${POPUP_MAX_DEPTH}, windowMs=${POPUP_CAPTURE_WINDOW_MS}`));
       }
@@ -3101,133 +3185,200 @@ function setupFrameHandling(page, forceDebug) {
         // setRequestInterception(true) — page.on('request') fires for every
         // request regardless of interception state, and we don't need to
         // block anything on popups.
-        const attachPopupRequestCapture = (popupPage, depth) => {
-          popupPage.on('request', (request) => {
+        // Evaluate ANY URL surfaced from a popup (the popup's own navigation URL
+        // OR an in-popup request) against the same filter pipeline the main-page
+        // request handler uses. Factored out so:
+        //   1. attachPopupRequestCapture's `popupPage.on('request', ...)` calls
+        //      this once per in-popup request (with the request's resourceType).
+        //   2. onTargetCreated calls this once with `target.url()` and resourceType
+        //      'document' BEFORE attaching the request listener — catches the
+        //      popup's navigation URL itself, which fires before our listener can
+        //      attach (targetcreated → page resolve → attach is async, and the
+        //      browser dispatches the navigation immediately on window.open).
+        //      Without #2, popunder destinations whose own URL contains the
+        //      filterRegex pattern (e.g. AdsCore campaign URLs with &campaign=)
+        //      were seen-but-not-evaluated.
+        const evaluatePopupUrl = (checkedUrl, depth, resourceType) => {
+          try {
+            if (!checkedUrl || checkedUrl === 'about:blank') return;
+            let fullSubdomain = '';
+            let checkedRootDomain = '';
             try {
-              const checkedUrl = request.url();
-              let fullSubdomain = '';
-              let checkedRootDomain = '';
-              try {
-                const parsedUrl = new URL(checkedUrl);
-                fullSubdomain = parsedUrl.hostname;
-                const pslResult = psl.parse(fullSubdomain);
-                checkedRootDomain = pslResult.domain || fullSubdomain;
-              } catch (_) { return; }
-              if (!checkedRootDomain) return;
-
-              // ignoreDomainsByUrl — if any pattern matches this popup URL,
-              // mark the root domain as ignored for the rest of the scan
-              // (main page + all popups). Mirrors the main handler so a
-              // tracker URL surfaced via popup chain has the same dampening
-              // effect as one surfaced on the main page.
-              if (_ignoreDomainsByUrlRegexes.length > 0 && !_dynamicallyIgnoredDomains.has(checkedRootDomain)) {
-                for (let i = 0; i < _ignoreDomainsByUrlRegexes.length; i++) {
-                  if (_ignoreDomainsByUrlRegexes[i].test(checkedUrl)) {
-                    _dynamicallyIgnoredDomains.add(checkedRootDomain);
-                    if (forceDebug) {
-                      console.log(formatLogMessage('debug', `${IGNORE_DOMAINS_BY_URL_TAG} ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
-                    }
-                    break;
-                  }
-                }
-              }
-
-              // blockDomainsByUrl trigger — symmetric to ignoreDomainsByUrl
-              // above; populating the dynamic block Set from popup URLs lets
-              // tracker URLs surfaced via popup chains poison their root
-              // domain for the rest of the scan just like main-page hits do.
-              if (_blockDomainsByUrlRegexes.length > 0 && !_dynamicallyBlockedDomains.has(checkedRootDomain)) {
-                for (let i = 0; i < _blockDomainsByUrlRegexes.length; i++) {
-                  if (_blockDomainsByUrlRegexes[i].test(checkedUrl)) {
-                    _dynamicallyBlockedDomains.add(checkedRootDomain);
-                    if (forceDebug) {
-                      console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} ${checkedRootDomain} blocked — matched pattern: ${_blockDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
-                    }
-                    break;
+              const parsedUrl = new URL(checkedUrl);
+              fullSubdomain = parsedUrl.hostname;
+              const pslResult = psl.parse(fullSubdomain);
+              checkedRootDomain = pslResult.domain || fullSubdomain;
+            } catch (_) { return; }
+            if (!checkedRootDomain) return;
+
+            // ignoreDomainsByUrl — if any pattern matches this popup URL,
+            // mark the root domain as ignored for the rest of the scan
+            // (main page + all popups). Mirrors the main handler so a
+            // tracker URL surfaced via popup chain has the same dampening
+            // effect as one surfaced on the main page.
+            if (_ignoreDomainsByUrlRegexes.length > 0 && !_dynamicallyIgnoredDomains.has(checkedRootDomain)) {
+              for (let i = 0; i < _ignoreDomainsByUrlRegexes.length; i++) {
+                if (_ignoreDomainsByUrlRegexes[i].test(checkedUrl)) {
+                  _dynamicallyIgnoredDomains.add(checkedRootDomain);
+                  if (forceDebug) {
+                    console.log(formatLogMessage('debug', `${IGNORE_DOMAINS_BY_URL_TAG} ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
                   }
+                  break;
                 }
               }
+            }
 
-              // ignoreDomains gate (global; matchesIgnoreDomain also short-
-              // circuits on _dynamicallyIgnoredDomains, so a domain we just
-              // added above will be caught here on the same request).
-              if (matchesIgnoreDomain(checkedRootDomain, ignoreDomains)) return;
-
-              // Dynamic-block gate for popup requests — early return on
-              // matched root or any parent (parent-walk in
-              // matchesDynamicBlock). Popups don't have a request object
-              // available here, so we just return rather than abort; the
-              // popup-request observer treats this as "don't process".
-              if (matchesDynamicBlock(checkedRootDomain)) return;
-
-              // First-party / third-party gate (popup belongs to the main URL's
-              // domain group — its OWN URL doesn't redefine first-party).
-              const isFirstParty = firstPartyDomains.has(checkedRootDomain);
-              if (siteConfig.firstParty === false && isFirstParty) return;
-              if (siteConfig.thirdParty === false && !isFirstParty) return;
-
-              // Regex match against the site's filterRegex list
-              const resourceType = request.resourceType();
-              let regexMatched = false;
-              for (const re of regexes) {
-                if (re.test(checkedUrl)) {
-                  regexMatched = true;
+            // blockDomainsByUrl trigger — symmetric to ignoreDomainsByUrl
+            // above; populating the dynamic block Set from popup URLs lets
+            // tracker URLs surfaced via popup chains poison their root
+            // domain for the rest of the scan just like main-page hits do.
+            if (_blockDomainsByUrlRegexes.length > 0 && !_dynamicallyBlockedDomains.has(checkedRootDomain)) {
+              for (let i = 0; i < _blockDomainsByUrlRegexes.length; i++) {
+                if (_blockDomainsByUrlRegexes[i].test(checkedUrl)) {
+                  _dynamicallyBlockedDomains.add(checkedRootDomain);
                   if (forceDebug) {
-                    console.log(formatLogMessage('debug', `[popup depth=${depth}] Matched ${checkedRootDomain} via ${re} (${resourceType})`));
+                    console.log(formatLogMessage('debug', `${BLOCK_DOMAINS_BY_URL_TAG} ${checkedRootDomain} blocked — matched pattern: ${_blockDomainsByUrlRegexes[i].source} (from popup depth=${depth})`));
                   }
                   break;
                 }
               }
+            }
 
-              if (!regexMatched) return;
-
-              // hasNetTools is the same flag the main handler uses (line ~2639).
-              // When the site config carries whois/dig terms, regex match is
-              // not sufficient by itself — the URL must ALSO pass the whois/
-              // dig validation before it counts. Mirrors the main handler's
-              // behavior so 'capture popup domains that match regex/dig/whois'
-              // means the same thing for popups as for the main page.
-              if (hasNetTools) {
-                const popupNetToolsHandler = createNetToolsHandler({
-                  whoisTerms, whoisOrTerms,
-                  processedWhoisDomains: globalProcessedWhoisDomains,
-                  processedDigDomains: globalProcessedDigDomains,
-                  whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
-                  whoisServer,
-                  whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
-                  debugLogFile,
-                  digTerms, digOrTerms, digRecordType,
-                  digSubdomain: siteConfig.dig_subdomain === true,
-                  dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
-                  matchedDomains, addMatchedDomain,
-                  isDomainAlreadyDetected: isLocallyDetected,
-                  onWhoisResult: smartCache ? (domain, result) => smartCache.cacheNetTools(domain, 'whois', result) : undefined,
-                  onDigResult: smartCache ? (domain, result, recordType) => smartCache.cacheNetTools(domain, 'dig', result, recordType) : undefined,
-                  cachedWhois: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'whois') : null,
-                  cachedDig: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'dig', digRecordType) : null,
-                  currentUrl, getRootDomain, siteConfig, dumpUrls, matchedUrlsLogFile, forceDebug, fs,
-                  ignoreDomains, matchesIgnoreDomain
-                });
-                trackNetToolsHandler(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
-              } else {
-                // No nettools required — regex match alone counts.
-                addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
+            // ignoreDomains gate (global; matchesIgnoreDomain also short-
+            // circuits on _dynamicallyIgnoredDomains, so a domain we just
+            // added above will be caught here on the same request).
+            if (matchesIgnoreDomain(checkedRootDomain, ignoreDomains)) return;
+
+            // Dynamic-block gate for popup requests — early return on
+            // matched root or any parent (parent-walk in
+            // matchesDynamicBlock). Popups don't have a request object
+            // available here, so we just return rather than abort; the
+            // popup-request observer treats this as "don't process".
+            if (matchesDynamicBlock(checkedRootDomain)) return;
+
+            // First-party / third-party gate (popup belongs to the main URL's
+            // domain group — its OWN URL doesn't redefine first-party).
+            const isFirstParty = firstPartyDomains.has(checkedRootDomain);
+            if (siteConfig.firstParty === false && isFirstParty) return;
+            if (siteConfig.thirdParty === false && !isFirstParty) return;
+
+            // Regex match against the site's filterRegex list
+            let regexMatched = false;
+            for (const re of regexes) {
+              if (re.test(checkedUrl)) {
+                regexMatched = true;
+                if (forceDebug) {
+                  console.log(formatLogMessage('debug', `[popup depth=${depth}] Matched ${checkedRootDomain} via ${re} (${resourceType})`));
+                }
+                break;
               }
-            } catch (_) { /* observation-only — never let a popup error escape */ }
+            }
+
+            if (!regexMatched) return;
+
+            // hasNetTools is the same flag the main handler uses (line ~2639).
+            // When the site config carries whois/dig terms, regex match is
+            // not sufficient by itself — the URL must ALSO pass the whois/
+            // dig validation before it counts. Mirrors the main handler's
+            // behavior so 'capture popup domains that match regex/dig/whois'
+            // means the same thing for popups as for the main page.
+            if (hasNetTools) {
+              const popupNetToolsHandler = createNetToolsHandler({
+                whoisTerms, whoisOrTerms,
+                processedWhoisDomains: globalProcessedWhoisDomains,
+                processedDigDomains: globalProcessedDigDomains,
+                whoisDelay: siteConfig.whois_delay !== undefined ? siteConfig.whois_delay : whois_delay,
+                whoisServer,
+                whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
+                debugLogFile,
+                digTerms, digOrTerms, digRecordType,
+                digSubdomain: siteConfig.dig_subdomain === true,
+                dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
+                matchedDomains, addMatchedDomain,
+                isDomainAlreadyDetected: isLocallyDetected,
+                onWhoisResult: smartCache ? (domain, result) => smartCache.cacheNetTools(domain, 'whois', result) : undefined,
+                onDigResult: smartCache ? (domain, result, recordType) => smartCache.cacheNetTools(domain, 'dig', result, recordType) : undefined,
+                cachedWhois: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'whois') : null,
+                cachedDig: smartCache ? smartCache.getCachedNetTools(checkedRootDomain, 'dig', digRecordType) : null,
+                currentUrl, getRootDomain, siteConfig, dumpUrls, matchedUrlsLogFile, forceDebug, fs,
+                ignoreDomains, matchesIgnoreDomain
+              });
+              trackNetToolsHandler(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
+            } else {
+              // No nettools required — regex match alone counts.
+              addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
+            }
+          } catch (_) { /* observation-only — never let a popup error escape */ }
+        };
+
+        // Thin wrapper around evaluatePopupUrl for the per-request listener.
+        // Under forceDebug also attach framenavigated + close listeners so
+        // the popup's full lifecycle (initial nav URL, mid-popup navigations,
+        // close) is visible in logs. Useful when investigating "I saw a
+        // Chrome window flash on screen" — the framenavigated transitions
+        // tell you what URL the window was showing and for how long.
+        const attachPopupRequestCapture = (popupPage, depth) => {
+          popupPage.on('request', (request) => {
+            evaluatePopupUrl(request.url(), depth, request.resourceType());
           });
+          if (forceDebug) {
+            try {
+              popupPage.on('framenavigated', (frame) => {
+                try {
+                  if (frame !== popupPage.mainFrame()) return; // main frame only
+                  console.log(formatLogMessage('debug', `[popup depth=${depth}] framenavigated → ${frame.url() || 'about:blank'}`));
+                } catch (_) {}
+              });
+              popupPage.on('close', () => {
+                try {
+                  const lastUrl = popupPage.url ? popupPage.url() : '(unknown)';
+                  console.log(formatLogMessage('debug', `[popup depth=${depth}] close (last URL: ${lastUrl})`));
+                } catch (_) {}
+              });
+              popupPage.on('pageerror', (err) => {
+                try { console.log(formatLogMessage('debug', `[popup depth=${depth}] pageerror: ${err.message}`)); } catch (_) {}
+              });
+            } catch (_) { /* listener attach errors aren't fatal */ }
+          }
         };
 
         const onTargetCreated = async (target) => {
+          // Log EVERY targetcreated event under forceDebug so callers can see
+          // the full set of targets Chromium creates during the scan — not
+          // just the ones we capture. Useful when investigating "is that
+          // Chrome window I saw from a popup or from somewhere else?" — if
+          // a window opens but no targetcreated fires, it's not ours. If a
+          // targetcreated fires for type=page but we skip-and-explain below,
+          // the user knows why we ignored it. Captures the FULL diagnostic
+          // surface, no behavior change.
+          let _tType, _tUrl;
+          if (forceDebug) {
+            try {
+              _tType = target.type();
+              _tUrl = target.url() || 'about:blank';
+              console.log(formatLogMessage('debug', `[popup] targetcreated: type=${_tType} url=${_tUrl}`));
+            } catch (_) {}
+          }
+
           // Short-circuit guard: if finally has already started, don't attach
           // a request listener whose closure would outlive its meaningful
           // scope. The race is narrow (a targetcreated firing while we're
           // mid-await on target.page() across the finally boundary), but
           // without this guard a late popup could push matches into
           // matchedDomains for a URL whose processing has already returned.
-          if (urlFinished) return;
-          if (target.type() !== 'page') return;
+          if (urlFinished) {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup] skipping: urlFinished=true (scan teardown in progress)`));
+            return;
+          }
+          if (target.type() !== 'page') {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup] skipping: non-page target type=${target.type()} (workers/service-workers/etc are not popunder candidates)`));
+            return;
+          }
           const depth = getPopupDepth(target);
-          if (depth < 1) return; // Not one of ours
+          if (depth < 1) {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup] skipping: depth=0 — target not in opener chain of main page (likely a new browser tab opened independently, not a popunder from our scan)`));
+            return; // Not one of ours
+          }
           if (depth > POPUP_MAX_DEPTH) {
             if (forceDebug) {
               console.log(formatLogMessage('debug', `[popup] Skipping depth-${depth} popup (max=${POPUP_MAX_DEPTH}): ${target.url() || 'about:blank'}`));
@@ -3237,7 +3388,10 @@ function setupFrameHandling(page, forceDebug) {
 
           let popupPage;
           try { popupPage = await target.page(); } catch (_) { return; }
-          if (!popupPage) return;
+          if (!popupPage) {
+            if (forceDebug) console.log(formatLogMessage('debug', `[popup depth=${depth}] target.page() returned null — popup not accessible as a Page object`));
+            return;
+          }
           // Re-check after the await — the per-URL finally may have flipped
           // the flag while target.page() was resolving.
           if (urlFinished) {
@@ -3247,8 +3401,31 @@ function setupFrameHandling(page, forceDebug) {
 
           if (forceDebug) {
             console.log(formatLogMessage('debug', `[popup depth=${depth}] Capturing popup: ${target.url() || 'about:blank'}`));
+            // Window dimensions are useful for the "is the popup visible on
+            // my screen?" question — a popup with non-zero viewport in a
+            // headless=new launch shouldn't be visible but on some display
+            // servers (WSLg, X11) it can briefly flash on screen. Log the
+            // viewport so callers can correlate with what they saw.
+            try {
+              const vp = popupPage.viewport();
+              if (vp) console.log(formatLogMessage('debug', `[popup depth=${depth}] viewport: ${vp.width}x${vp.height}`));
+            } catch (_) {}
           }
 
+          // Evaluate the popup's own navigation URL against the same filter
+          // pipeline used for in-popup requests. Required because targetcreated
+          // → target.page() → on('request', ...) is async, and the browser
+          // dispatches the popup's navigation request immediately on window.open
+          // — by the time the listener attaches below, the navigation request
+          // has already fired and won't be re-emitted. resourceType 'document'
+          // mirrors what Chrome would emit for a top-level navigation request.
+          // Without this call, AdsCore-style popunder destinations (URL contains
+          // &campaign=, &v=, etc) were seen-but-not-evaluated: the popup was
+          // logged but its domain never matched the filter regex, so it never
+          // became a rule. Only secondary in-popup requests (tracking pixels,
+          // sub-resources) ever got tested against the regex.
+          evaluatePopupUrl(target.url(), depth, 'document');
+
           attachPopupRequestCapture(popupPage, depth);
 
           // Auto-close after the capture window so popups don't pile up.
@@ -4322,7 +4499,26 @@ function setupFrameHandling(page, forceDebug) {
 
         // Mark page as processing during interactions
         updatePageUsage(page, true);
-        const INTERACTION_HARD_TIMEOUT = 15000;
+        // Work-aware ceiling (scales with click count / realistic_click /
+        // intensity) instead of a flat 15s, which truncated high-click
+        // popunder configs mid-pass. Single source of truth shared with
+        // interaction.js's own internal hard cap so the two can't disagree.
+        const INTERACTION_HARD_TIMEOUT = computeInteractionCeilingMs(interactionConfig);
+
+        // Capture-and-clear timer wrapper — same fix as cdp.js (0772ccd) and
+        // the per-URL grace (577ad66). The 3 inline Promise.race patterns
+        // below previously used `new Promise((_, reject) => setTimeout(...))`
+        // without capturing the timer ID, leaking the 15s timer + closure on
+        // reject every time interaction completed inside the cap (the common
+        // case). Centralizing avoids the same mistake recurring across the
+        // ghost-cursor / fallback / standard branches.
+        const raceWithTimer = (promise, msg) => {
+          let t;
+          return Promise.race([
+            promise,
+            new Promise((_, reject) => { t = setTimeout(() => reject(new Error(msg)), INTERACTION_HARD_TIMEOUT); })
+          ]).finally(() => clearTimeout(t));
+        };
 
         // Check if ghost-cursor mode is enabled for this site
         const ghostConfig = resolveGhostCursorConfig(siteConfig, globalGhostCursor, forceDebug);
@@ -4333,60 +4529,51 @@ function setupFrameHandling(page, forceDebug) {
             if (forceDebug) console.log(formatLogMessage('debug', `${GHOST_CURSOR_TAG} Using ghost-cursor for ${currentUrl}`));
             const cursor = createGhostCursor(page, { forceDebug });
             if (cursor) {
-              await Promise.race([
-                (async () => {
-                  const viewport = page.viewport() || { width: 1200, height: 800 };
-                  const ghostDuration = ghostConfig.duration || 2000;
-                  const ghostStart = Date.now();
-                  const ghostTimeLeft = () => ghostDuration - (Date.now() - ghostStart);
-
-                  // Time-based Bezier mouse movements — runs for ghostDuration ms
-                  while (ghostTimeLeft() > 200) {
-                    const toX = Math.floor(Math.random() * (viewport.width - 100)) + 50;
-                    const toY = Math.floor(Math.random() * (viewport.height - 100)) + 50;
-                    await ghostMove(cursor, toX, toY, {
-                      moveSpeed: ghostConfig.moveSpeed,
-                      overshootThreshold: ghostConfig.overshootThreshold,
-                      forceDebug
-                    });
-                    if (ghostTimeLeft() > 100) {
-                      await new Promise(r => setTimeout(r, 25 + Math.random() * 75));
-                    }
-                  }
-                  if (ghostTimeLeft() > 100 && Math.random() < 0.3) {
-                    await ghostRandomMove(cursor, { forceDebug });
-                  }
-                  if (interactionConfig.includeElementClicks && ghostTimeLeft() > 100) {
-                    const clickX = Math.floor(viewport.width * 0.2 + Math.random() * viewport.width * 0.6);
-                    const clickY = Math.floor(viewport.height * 0.2 + Math.random() * viewport.height * 0.6);
-                    await ghostClick(cursor, { x: clickX, y: clickY }, {
-                      hesitate: ghostConfig.hesitate,
-                      forceDebug
-                    });
+              await raceWithTimer((async () => {
+                const viewport = page.viewport() || { width: 1200, height: 800 };
+                const ghostDuration = ghostConfig.duration || 2000;
+                const ghostStart = Date.now();
+                const ghostTimeLeft = () => ghostDuration - (Date.now() - ghostStart);
+
+                // Time-based Bezier mouse movements — runs for ghostDuration ms
+                while (ghostTimeLeft() > 200) {
+                  const toX = Math.floor(Math.random() * (viewport.width - 100)) + 50;
+                  const toY = Math.floor(Math.random() * (viewport.height - 100)) + 50;
+                  await ghostMove(cursor, toX, toY, {
+                    moveSpeed: ghostConfig.moveSpeed,
+                    overshootThreshold: ghostConfig.overshootThreshold,
+                    forceDebug
+                  });
+                  if (ghostTimeLeft() > 100) {
+                    await new Promise(r => setTimeout(r, 25 + Math.random() * 75));
                   }
-                  if (interactionConfig.includeScrolling) {
-                    await performPageInteraction(page, currentUrl, {
-                      ...interactionConfig,
-                      mouseMovements: 0,
-                      includeElementClicks: false
-                    }, forceDebug);
-                  }
-                })(),
-                new Promise((_, reject) => setTimeout(() => reject(new Error('ghost-cursor interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
-              ]);
+                }
+                if (ghostTimeLeft() > 100 && Math.random() < 0.3) {
+                  await ghostRandomMove(cursor, { forceDebug });
+                }
+                if (interactionConfig.includeElementClicks && ghostTimeLeft() > 100) {
+                  const clickX = Math.floor(viewport.width * 0.2 + Math.random() * viewport.width * 0.6);
+                  const clickY = Math.floor(viewport.height * 0.2 + Math.random() * viewport.height * 0.6);
+                  await ghostClick(cursor, { x: clickX, y: clickY }, {
+                    hesitate: ghostConfig.hesitate,
+                    forceDebug
+                  });
+                }
+                if (interactionConfig.includeScrolling) {
+                  await performPageInteraction(page, currentUrl, {
+                    ...interactionConfig,
+                    mouseMovements: 0,
+                    includeElementClicks: false
+                  }, forceDebug);
+                }
+              })(), 'ghost-cursor interaction hard timeout');
             } else {
               if (forceDebug) console.log(formatLogMessage('debug', '[ghost-cursor] Falling back to built-in mouse'));
-              await Promise.race([
-                performPageInteraction(page, currentUrl, interactionConfig, forceDebug),
-                new Promise((_, reject) => setTimeout(() => reject(new Error('interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
-              ]);
+              await raceWithTimer(performPageInteraction(page, currentUrl, interactionConfig, forceDebug), 'interaction hard timeout');
             }
           } else {
             // Standard built-in mouse interaction
-            await Promise.race([
-              performPageInteraction(page, currentUrl, interactionConfig, forceDebug),
-              new Promise((_, reject) => setTimeout(() => reject(new Error('interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
-            ]);
+            await raceWithTimer(performPageInteraction(page, currentUrl, interactionConfig, forceDebug), 'interaction hard timeout');
           }
         } catch (interactTimeoutErr) {
           if (forceDebug) console.log(formatLogMessage('debug', `${INTERACTION_TAG} Aborted after ${INTERACTION_HARD_TIMEOUT}ms: ${interactTimeoutErr.message}`));
@@ -4521,8 +4708,16 @@ function setupFrameHandling(page, forceDebug) {
 
         if (siteConfig.clear_sitedata === true) {
           try {
-            const clearResult = await clearSiteData(page, currentUrl, forceDebug, true); // Quick mode for reloads
-            if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data before reload #${i} for ${currentUrl}`));
+            // Default reload clear is quick mode (cookies + cache +
+            // localStorage + sessionStorage — the storage layers where
+            // session-cap tracking typically lives). Sites that put their
+            // session cap in IndexedDB / WebSQL / service workers can opt
+            // into a full clear-per-reload via clear_sitedata_full_on_reload.
+            // Costs ~100-500ms extra per reload and may unregister a
+            // service worker the page depends on; off by default.
+            const fullOnReload = siteConfig.clear_sitedata_full_on_reload === true;
+            const clearResult = await clearSiteData(page, currentUrl, forceDebug, !fullOnReload);
+            if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data (${fullOnReload ? 'full' : 'quick'}) before reload #${i} for ${currentUrl}`));
           } catch (reloadClearErr) {
             if (forceDebug) console.log(formatLogMessage('debug', `${CLEAR_SITEDATA_TAG} Before reload failed for ${currentUrl}`));
           }
@@ -4536,20 +4731,26 @@ function setupFrameHandling(page, forceDebug) {
       if (useForceReload && !reloadSuccess && !skipForceReload) {
         // Attempt force reload: disable cache, reload, re-enable cache
           try {
+          // Local race-with-timer helper — capture-and-clear pattern from
+          // cdp.js / interact (6ad36e7). Without this, every successful
+          // setCacheEnabled() left an 8s setTimeout running with closure
+          // on `reject` (2 leaks per reload cycle × N reload cycles).
+          const raceWithTimer = (promise, msg, ms) => {
+            let t;
+            return Promise.race([
+              promise,
+              new Promise((_, reject) => { t = setTimeout(() => reject(new Error(msg)), ms); })
+            ]).finally(() => clearTimeout(t));
+          };
+
           // Timeout-protected cache disable
-          await Promise.race([
-            page.setCacheEnabled(false),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache disable timeout')), 8000))
-          ]);
-          
+          await raceWithTimer(page.setCacheEnabled(false), 'Cache disable timeout', 8000);
+
             // Use networkidle2 for force reload to better detect when page is actually loaded
             await page.reload({ waitUntil: 'networkidle2', timeout: Math.min(timeout, 15000) });
-          
+
           // Timeout-protected cache enable
-          await Promise.race([
-            page.setCacheEnabled(true),
-            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache enable timeout')), 8000))
-          ]);
+          await raceWithTimer(page.setCacheEnabled(true), 'Cache enable timeout', 8000);
          
           reloadSuccess = true;
             if (forceDebug) console.log(formatLogMessage('debug', `Force reload #${i} completed for ${currentUrl}`));
@@ -4644,8 +4845,21 @@ function setupFrameHandling(page, forceDebug) {
             const endY = 200 + Math.floor(Math.random() * (vp.height - 400));
             await humanLikeMouseMove(page, startX, startY, endX, endY, { steps: 3, curve: 0.04, jitter: 1 });
           }
-          // Content clicks to trigger document-level onclick handlers
-          await performContentClicks(page, { clicks: 2, preDelay: 200, forceDebug });
+          // Content clicks to trigger document-level onclick handlers.
+          // Honor siteConfig.interact_click_count so popunder-discovery configs
+          // get the same click volume on every reload, not just the initial load.
+          // Omit `clicks` when no override is set so performContentClicks uses
+          // its CONTENT_CLICK.CLICK_COUNT default (single source of truth).
+          // realistic forwards siteConfig.realistic_click; always passed
+          // (defaults to false) so realistic mode applies to every reload's
+          // clicks, not just the initial pass.
+          const postReloadClickOpts = {
+            preDelay: 200,
+            forceDebug,
+            realistic: !!interactionConfig.realistic
+          };
+          if (interactionConfig.clickCount) postReloadClickOpts.clicks = interactionConfig.clickCount;
+          await performContentClicks(page, postReloadClickOpts);
           if (forceDebug) console.log(formatLogMessage('debug', `Post-reload interaction completed for reload #${i}`));
         } catch (postReloadInteractErr) {
           // Non-critical — continue with remaining reloads
@@ -4870,9 +5084,21 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
 
-// Temporarily store the pLimit function  
+// Temporarily store the pLimit function
   const originalLimit = limit;
 
+  // Per-site config normalization (always runs, not gated on --validate-config).
+  // Catches typo'd keys (whois_terms vs whois) with "did you mean" suggestions
+  // and coerces boolean-like values (interact: 1 → interact: true) before any
+  // downstream strict-equality check silently treats them as disabled. Mutates
+  // each site in place so the rest of the scan sees normalized values.
+  // Reports via console.warn so messages surface even when --silent is set.
+  for (let i = 0; i < sites.length; i++) {
+    const { warnings, errors } = normalizeSiteConfig(sites[i], i);
+    for (const e of errors) console.warn(messageColors.error('⚠ ' + e));
+    for (const w of warnings) console.warn(messageColors.warn('⚠ [config] ' + w));
+  }
+
   // V8 Optimization: Calculate total URLs first to pre-allocate array
   let totalUrls = 0;
   for (const site of sites) {
@@ -4890,7 +5116,17 @@ function setupFrameHandling(page, forceDebug) {
     for (const url of urlsToProcess) {
       allTasks[taskIndex++] = {
         url,
-        config: { ...site, _originalUrl: url }, // Preserve original URL for CDP domain checking
+        // Default userAgent to 'chrome' when a site doesn't set one. Without
+        // it the browser sends its bundled default UA, which literally
+        // contains "HeadlessChrome" (verified, both headless modes) — an
+        // instant automation tell. Defaulting here (rather than at launch)
+        // activates the whole coherent path, since UA-string spoofing, the
+        // navigator/webdriver/plugins/userAgentData JS masking, the Sec-CH-UA
+        // request headers, and the curl content-fetch UA all gate on
+        // config.userAgent. Placing 'chrome' BEFORE the spread means an
+        // explicit site value wins — including userAgent:false / null to opt
+        // out and scan with the raw headless UA.
+        config: { userAgent: 'chrome', ...site, _originalUrl: url },
         taskId: taskIndex - 1 // For tracking
       };
     }
@@ -4923,7 +5159,7 @@ function setupFrameHandling(page, forceDebug) {
   let urlsSinceLastCleanup = 0;
   
   if (!silentMode && totalUrls > 0) {
-    console.log(`\n${messageColors.processing('Processing')} ${totalUrls} URLs with TRUE concurrency ${MAX_CONCURRENT_SITES}...`);
+    console.log(`\n${messageColors.processing('Processing')} ${totalUrls} URLs with TRUE concurrency ${effectiveConcurrency}...`);
     if (totalUrls > RESOURCE_CLEANUP_INTERVAL) {
       console.log(messageColors.processing('Browser will restart every') + ` ~${RESOURCE_CLEANUP_INTERVAL} URLs to free resources`);
     }
@@ -5044,10 +5280,18 @@ function setupFrameHandling(page, forceDebug) {
          silentMode
        });
        healthPromise.catch(() => {});
-       healthCheck = await Promise.race([
-         healthPromise,
-         new Promise((_, reject) => setTimeout(() => reject(new Error('Health check timeout')), 30000))
-       ]);
+       // Capture-and-clear timer pattern (cdp.js 0772ccd, interact 6ad36e7) —
+       // when healthPromise wins the race, the inline setTimeout would
+       // otherwise hold reject's closure for the full 30s grace window.
+       let healthTimer;
+       try {
+         healthCheck = await Promise.race([
+           healthPromise,
+           new Promise((_, reject) => { healthTimer = setTimeout(() => reject(new Error('Health check timeout')), 30000); })
+         ]);
+       } finally {
+         if (healthTimer) clearTimeout(healthTimer);
+       }
      } catch (healthError) {
        console.log(formatLogMessage('warn', `[HEALTH CHECK] Timeout, assuming restart needed`));
        healthCheck = { shouldRestart: true, reason: 'Health check timeout' };
@@ -5312,26 +5556,94 @@ function setupFrameHandling(page, forceDebug) {
      } catch {}
 
      // Per-URL timeout so a single hung processUrl can't block the batch
-     // forever. 75s sits comfortably above the realistic legit-page ceiling
-     // (nav 35s + Cloudflare adaptive ~25s + interaction ~10s + network-idle
-     // wait ~10s ≈ ~70s), well short of the old 120s safety net. Cuts
-     // hang-recovery time roughly in half when an entire batch's URLs all
-     // hang and we're waiting on this timeout to advance processedUrlCount.
-     const PER_URL_TIMEOUT_MS = 75000;
+     // forever. Scaled from siteConfig.timeout + (delay + interaction) ×
+     // (1 + reload) + 30s headroom, with a 75s floor.
+     //
+     // The (1 + reload) multiplier was missing from the previous formula
+     // (13dd4fa) — `reload: 4` configs perform 5 total cycles (initial +
+     // 4 reloads), each with its own delay + interaction overhead, so the
+     // 80s ceiling for the user's lean config (timeout:35000, delay:15000,
+     // reload:4) fired DURING the 3rd reload while the orphan still had
+     // 2 more cycles + drain to go — far longer than the 8s grace could
+     // bridge. Multiplying by cycle count brings the ceiling above the
+     // legitimate work envelope.
+     const reloadCount = task.config.reload || 0;
+     // Interaction overhead per cycle must match interaction.js's actual
+     // ceiling, which is now work-aware (high interact_click_count /
+     // realistic_click configs legitimately run far longer than the old flat
+     // 15s). Compute the same value here so the per-URL ceiling stays above
+     // the real interaction envelope and can't fire mid-pass. Zero when
+     // interaction is disabled for this task (no interaction cost to budget).
+     const interactionOnForTask = task.config.interact === true && !disableInteract;
+     const INTERACTION_OVERHEAD_MS = interactionOnForTask
+       ? computeInteractionCeilingMs(createInteractionConfig(task.url, task.config))
+       : 0;
+     const PER_URL_TIMEOUT_MS = Math.max(
+       75000,
+       (task.config.timeout || 35000)
+         + ((task.config.delay || 0) + INTERACTION_OVERHEAD_MS) * (1 + reloadCount)
+         + 30000
+     );
+     // Grace period after primary timeout — gives the orphan a chance to
+     // finish drainPendingNetTools() and emit "Saving N rules despite page
+     // load failure" before we abandon its result. Drain typically completes
+     // in <1s with cached nettools; 8s is the safety ceiling.
+     const PER_URL_GRACE_MS = 8000;
+     const PER_URL_TIMEOUT_MARKER = 'PER_URL_TIMEOUT_FIRED';
+
      const processUrlPromise = processUrl(task.url, task.config, browser);
      let perUrlTimer;
      try {
        return await Promise.race([
          processUrlPromise,
          new Promise((_, reject) => {
-           perUrlTimer = setTimeout(() => reject(new Error('Per-URL timeout (75s)')), PER_URL_TIMEOUT_MS);
+           perUrlTimer = setTimeout(() => {
+             const e = new Error(`Per-URL timeout (${Math.round(PER_URL_TIMEOUT_MS / 1000)}s)`);
+             e.code = PER_URL_TIMEOUT_MARKER;
+             reject(e);
+           }, PER_URL_TIMEOUT_MS);
          })
        ]);
      } catch (err) {
-       if (err && err.message === 'Per-URL timeout (75s)') {
-         processUrlPromise.catch(() => {});
+       if (err && err.code === PER_URL_TIMEOUT_MARKER) {
          forceRestartFlag = true;
-         return { url: task.url, rules: [], success: false, error: 'Per-URL timeout (75s)', needsImmediateRestart: true };
+         // Log the timeout fire — was invisible before; only ended up in the
+         // returned result.error field which is never printed. Makes
+         // ceiling-tuning regressions visible without source-reading.
+         if (forceDebug) {
+           console.log(formatLogMessage('warn', `${err.message} for ${task.url} — orphan in ${PER_URL_GRACE_MS / 1000}s grace`));
+         }
+         // Grace period — wait briefly for the orphan to drain + recover
+         // partial matches. Browser is still in a bad state (we hit the
+         // primary ceiling) so the restart still fires either way; only the
+         // rules payload differs.
+         let graceTimer;
+         try {
+           const graceResult = await Promise.race([
+             processUrlPromise,
+             new Promise((_, reject) => {
+               // Capture the timer ID so the finally can clear it when the
+               // orphan wins the race — otherwise the setTimeout keeps the
+               // event loop ref + closure on `reject` alive for the full
+               // grace window, even though the race already settled.
+               // Same leak pattern fixed in cdp.js (0772ccd) and
+               // clear_sitedata (780b443).
+               graceTimer = setTimeout(() => reject(new Error('Grace timeout')), PER_URL_GRACE_MS);
+             })
+           ]);
+           if (forceDebug) {
+             console.log(formatLogMessage('debug', `Grace recovered ${(graceResult && graceResult.rules ? graceResult.rules.length : 0)} rules for ${task.url}`));
+           }
+           return { ...graceResult, needsImmediateRestart: true };
+         } catch (_) {
+           if (forceDebug) {
+             console.log(formatLogMessage('warn', `Grace timed out for ${task.url} — discarding orphan`));
+           }
+           processUrlPromise.catch(() => {});
+           return { url: task.url, rules: [], success: false, error: err.message, needsImmediateRestart: true };
+         } finally {
+           if (graceTimer) clearTimeout(graceTimer);
+         }
        }
        throw err;
      } finally {