@fanboynz/network-scanner 2.0.64 → 2.0.66

package/nwss.js

@@ -9,6 +9,7 @@ const fs = require('fs');
 const os = require('os');
 const psl = require('psl');
 const path = require('path');
+const dnsPromises = require('node:dns/promises');
 const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
 const { compressMultipleFiles, formatFileSize } = require('./lib/compress');
 const { parseSearchStrings, createResponseHandler, createCurlHandler } = require('./lib/searchstring');
@@ -50,7 +51,7 @@ const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostR
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
-const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy } = require('./lib/proxy');
+const { needsProxy, getProxyArgs, applyProxyAuth, getProxyInfo, testProxy, prepareSocksRelays, closeAllSocksRelays } = require('./lib/proxy');
 // Dry run functionality
 const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
@@ -266,6 +267,13 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
 }
 
 const headfulMode = args.includes('--headful');
+// Sites (esp. video/streaming) call element.requestFullscreen() on load or
+// click. In --headful that hijacks the real Chrome window into true
+// fullscreen, forcing a manual ESC. Neutralize the Fullscreen API by
+// default so it can't. Harmless in headless (no screen — the API is
+// already inert there), so default-on keeps headful consistent with the
+// primary headless path. --allow-fullscreen restores native behavior.
+const allowFullscreen = args.includes('--allow-fullscreen');
 const SOURCES_FOLDER = 'sources';
 
 let outputFile = null;
@@ -326,6 +334,31 @@ const cacheRequests = args.includes('--cache-requests');
 const dnsCacheMode = args.includes('--dns-cache');
 if (dnsCacheMode) enableDiskCache();
 
+// DNS pre-check before page.goto() — default-on, --no-dns-precheck disables.
+// Filters NXDOMAIN / unresolvable hostnames in <100ms before paying the
+// ~5-15s Puppeteer + Cloudflare detection round-trip on each.
+const dnsPrecheckEnabled = !args.includes('--no-dns-precheck');
+const dnsPrecheckTimeoutMs = 2000;
+
+// Per-scan cache of negative DNS lookups. OS resolvers don't always cache
+// NXDOMAIN responses, and a scan can hit the same dead hostname many times
+// (different URL paths on the same site). Positive results are left to the
+// OS cache; failure-cache avoids repeated lookup latency for known-dead hosts.
+// FIFO eviction at DNS_NEGATIVE_CACHE_MAX so pathological scans (thousands
+// of unique dead hosts) can't grow the cache unboundedly. Same pattern as
+// the rest of the codebase's in-memory caches.
+const dnsNegativeCache = new Map(); // hostname -> { error, timestamp }
+const DNS_NEGATIVE_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+const DNS_NEGATIVE_CACHE_MAX = 1000;
+let dnsPrecheckSkips = 0;
+
+function dnsNegativeCacheSet(hostname, error) {
+  if (dnsNegativeCache.size >= DNS_NEGATIVE_CACHE_MAX) {
+    dnsNegativeCache.delete(dnsNegativeCache.keys().next().value);
+  }
+  dnsNegativeCache.set(hostname, { error, timestamp: Date.now() });
+}
+
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
 if (validateRulesIndex !== -1 && args[validateRulesIndex + 1] && !args[validateRulesIndex + 1].startsWith('--')) {
@@ -360,6 +393,12 @@ let adblockEnabled = false;
 let adblockMatcher = null;
 let adblockStats = { blocked: 0, allowed: 0 };
 
+// Cloudflare scan-wide stats. errorPages counts URLs where the returned page
+// was a Cloudflare-served 5xx origin error (522/523/etc.) — no bypass
+// possible, useful signal for diagnosing dead-origin scans. Named distinct
+// from the local cloudflareStats = getCacheStats() in the debug stats block.
+let cloudflareScanStats = { errorPages: 0 };
+
 // Validate --adblock-rules usage - ignore if used incorrectly instead of erroring
 if (adblockRulesMode) {
   if (!outputFile) {
@@ -478,78 +517,10 @@ if (testValidation) {
   }
 }
 
-if (validateConfig) {
-  console.log(`\n${messageColors.processing('Validating configuration file...')}`);
-  try {
-    const validation = validateFullConfig(config, { forceDebug, silentMode });
-    
-    // Validate referrer_headers format
-    for (const site of sites) {
-       if (site.referrer_headers && typeof site.referrer_headers === 'object' && !Array.isArray(site.referrer_headers)) {
-         const validation = validateReferrerConfig(site.referrer_headers);
-         if (!validation.isValid) {
-           console.warn(`⚠ Invalid referrer_headers configuration: ${validation.errors.join(', ')}`);
-         }
-         if (validation.warnings.length > 0) {
-           console.warn(`⚠ Referrer warnings: ${validation.warnings.join(', ')}`);
-         }
-       }
-       // Validate referrer_disable format
-       if (site.referrer_disable) {
-         const disableValidation = validateReferrerDisable(site.referrer_disable);
-         if (!disableValidation.isValid) {
-           console.warn(`⚠ Invalid referrer_disable configuration: ${disableValidation.errors.join(', ')}`);
-         }
-         if (disableValidation.warnings.length > 0) {
-           console.warn(`⚠ Referrer disable warnings: ${disableValidation.warnings.join(', ')}`);
-         }
-       }
-    }
-
-    // Validate VPN configurations
-    for (const site of sites) {
-      if (site.vpn) {
-        const vpnNorm = normalizeVpnConfig(site.vpn);
-        const vpnValidation = validateVpnConfig(vpnNorm);
-        if (!vpnValidation.isValid) {
-          console.warn(`⚠ Invalid vpn configuration for ${site.url}: ${vpnValidation.errors.join(', ')}`);
-        }
-        if (vpnValidation.warnings.length > 0) {
-          vpnValidation.warnings.forEach(w => console.warn(`⚠ VPN warning for ${site.url}: ${w}`));
-        }
-      }
-      if (site.openvpn) {
-        const ovpnNorm = normalizeOvpnConfig(site.openvpn);
-        const ovpnValidation = validateOvpnConfig(ovpnNorm);
-        if (!ovpnValidation.isValid) {
-          console.warn(`⚠ Invalid openvpn configuration for ${site.url}: ${ovpnValidation.errors.join(', ')}`);
-        }
-        if (ovpnValidation.warnings.length > 0) {
-          ovpnValidation.warnings.forEach(w => console.warn(`⚠ OpenVPN warning for ${site.url}: ${w}`));
-        }
-      }
-      if (site.vpn && site.openvpn) {
-        console.warn(`⚠ ${site.url} has both vpn and openvpn configured — only one will be used (vpn takes precedence)`);
-      }
-    }
-
-    if (validation.isValid) {
-      console.log(`${messageColors.success('✅ Configuration is valid!')}`);
-      console.log(`${messageColors.info('Summary:')} ${validation.summary.validSites}/${validation.summary.totalSites} sites valid`);
-      if (validation.summary.sitesWithWarnings > 0) {
-        console.log(`${messageColors.warn('⚠ Warnings:')} ${validation.summary.sitesWithWarnings} sites have warnings`);
-      }
-      process.exit(0);
-    } else {
-      console.log(`${messageColors.error('❌ Configuration validation failed!')}`);
-      console.log(`${messageColors.error('Errors:')} ${validation.globalErrors.length} global, ${validation.summary.sitesWithErrors} site-specific`);
-      process.exit(1);
-    }
-  } catch (validationErr) {
-    console.error(`❌ Validation failed: ${validationErr.message}`);
-    process.exit(1);
-  }
-}
+// Note: --validate-config is handled further down, AFTER the config file is
+// loaded and `config`/`sites` are populated. Running it here would fail with
+// "Cannot access 'config' before initialization" since those are declared
+// later in the module.
 
 if (validateRules || validateRulesFile) {
   const filesToValidate = validateRulesFile ? [validateRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -705,6 +676,8 @@ General Options:
   --custom-json <file>           Use a custom config JSON file instead of config.json
   --headful                      Launch browser with GUI (not headless)
   --keep-open                    Keep browser open after scan completes (use with --headful)
+  --allow-fullscreen             Allow sites to use the Fullscreen API. By default it is
+                                 neutralized so sites can't hijack the window in --headful
   --use-puppeteer-core           Use puppeteer-core with system Chrome instead of bundled Chromium
   --use-obscura                  Connect to running Obscura CDP server (ws://127.0.0.1:9222 or OBSCURA_WS env)
                                  Skips fingerprint injection — Obscura provides built-in stealth
@@ -721,6 +694,9 @@ General Options:
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
   --dns-cache                    Persist dig/whois results to disk between runs (3hr/4hr TTL)
+  --no-dns-precheck              Disable per-URL DNS resolution check before page navigation.
+                                 By default, URLs whose hostname doesn't resolve are skipped
+                                 immediately (saves ~5-15s of Puppeteer time per dead host).
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -909,10 +885,86 @@ const {
   disable_ad_tagging = true,
   max_concurrent_sites = 6,
   resource_cleanup_interval = 80,
-  comments: globalComments, 
-  ...otherGlobalConfig 
+  comments: globalComments,
+  ...otherGlobalConfig
 } = config;
 
+// --validate-config runs here, after `config` and `sites` are populated.
+// Previously this block lived above the config load and triggered a TDZ
+// "Cannot access 'config' before initialization" error.
+if (validateConfig) {
+  console.log(`\n${messageColors.processing('Validating configuration file...')}`);
+  try {
+    const validation = validateFullConfig(config, { forceDebug, silentMode });
+
+    // Validate referrer_headers format
+    for (const site of sites) {
+       if (site.referrer_headers && typeof site.referrer_headers === 'object' && !Array.isArray(site.referrer_headers)) {
+         const refValidation = validateReferrerConfig(site.referrer_headers);
+         if (!refValidation.isValid) {
+           console.warn(`⚠ Invalid referrer_headers configuration: ${refValidation.errors.join(', ')}`);
+         }
+         if (refValidation.warnings.length > 0) {
+           console.warn(`⚠ Referrer warnings: ${refValidation.warnings.join(', ')}`);
+         }
+       }
+       // Validate referrer_disable format
+       if (site.referrer_disable) {
+         const disableValidation = validateReferrerDisable(site.referrer_disable);
+         if (!disableValidation.isValid) {
+           console.warn(`⚠ Invalid referrer_disable configuration: ${disableValidation.errors.join(', ')}`);
+         }
+         if (disableValidation.warnings.length > 0) {
+           console.warn(`⚠ Referrer disable warnings: ${disableValidation.warnings.join(', ')}`);
+         }
+       }
+    }
+
+    // Validate VPN configurations
+    for (const site of sites) {
+      if (site.vpn) {
+        const vpnNorm = normalizeVpnConfig(site.vpn);
+        const vpnValidation = validateVpnConfig(vpnNorm);
+        if (!vpnValidation.isValid) {
+          console.warn(`⚠ Invalid vpn configuration for ${site.url}: ${vpnValidation.errors.join(', ')}`);
+        }
+        if (vpnValidation.warnings.length > 0) {
+          vpnValidation.warnings.forEach(w => console.warn(`⚠ VPN warning for ${site.url}: ${w}`));
+        }
+      }
+      if (site.openvpn) {
+        const ovpnNorm = normalizeOvpnConfig(site.openvpn);
+        const ovpnValidation = validateOvpnConfig(ovpnNorm);
+        if (!ovpnValidation.isValid) {
+          console.warn(`⚠ Invalid openvpn configuration for ${site.url}: ${ovpnValidation.errors.join(', ')}`);
+        }
+        if (ovpnValidation.warnings.length > 0) {
+          ovpnValidation.warnings.forEach(w => console.warn(`⚠ OpenVPN warning for ${site.url}: ${w}`));
+        }
+      }
+      if (site.vpn && site.openvpn) {
+        console.warn(`⚠ ${site.url} has both vpn and openvpn configured — only one will be used (vpn takes precedence)`);
+      }
+    }
+
+    if (validation.isValid) {
+      console.log(`${messageColors.success('✅ Configuration is valid!')}`);
+      console.log(`${messageColors.info('Summary:')} ${validation.summary.validSites}/${validation.summary.totalSites} sites valid`);
+      if (validation.summary.sitesWithWarnings > 0) {
+        console.log(`${messageColors.warn('⚠ Warnings:')} ${validation.summary.sitesWithWarnings} sites have warnings`);
+      }
+      process.exit(0);
+    } else {
+      console.log(`${messageColors.error('❌ Configuration validation failed!')}`);
+      console.log(`${messageColors.error('Errors:')} ${validation.globalErrors.length} global, ${validation.summary.sitesWithErrors} site-specific`);
+      process.exit(1);
+    }
+  } catch (validationErr) {
+    console.error(`❌ Validation failed: ${validationErr.message}`);
+    process.exit(1);
+  }
+}
+
 // Pre-compile global blocked regexes ONCE (used in every processUrl call)
 const globalBlockedRegexes = Array.isArray(globalBlocked)
   ? globalBlocked.map(pattern => new RegExp(pattern))
@@ -1817,6 +1869,7 @@ function setupFrameHandling(page, forceDebug) {
     ovpnDisconnectAll(forceDebug);
     cleanupCloudflareCache();
     purgeStaleTrackers();
+    try { await closeAllSocksRelays(forceDebug); } catch (_) {}
   }
  
   let siteCounter = 0;
@@ -2424,6 +2477,29 @@ function setupFrameHandling(page, forceDebug) {
         } else if (forceDebug) {
           console.log(formatLogMessage('debug', `Skipping fingerprint injection — Obscura provides built-in stealth`));
         }
+
+        // Neutralize the Fullscreen API before any page script runs so a
+        // site can't force the real browser window fullscreen in --headful
+        // (or trip an anti-bot check that reads document.fullscreenElement).
+        // requestFullscreen is stubbed to a resolved no-op — which is also
+        // how browsers already behave when it's called without a user
+        // gesture, so this looks normal, not automated. fullscreenElement
+        // stays null naturally since we never enter fullscreen.
+        if (!allowFullscreen) {
+          try {
+            await page.evaluateOnNewDocument(() => {
+              const noop = function () { return Promise.resolve(); };
+              const legacyNoop = function () {};
+              try { Element.prototype.requestFullscreen = noop; } catch (_) {}
+              try { Element.prototype.webkitRequestFullscreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.webkitRequestFullScreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.mozRequestFullScreen = legacyNoop; } catch (_) {}
+              try { Element.prototype.msRequestFullscreen = legacyNoop; } catch (_) {}
+            });
+          } catch (fsErr) {
+            if (forceDebug) console.log(formatLogMessage('debug', `Fullscreen neutralization injection failed: ${fsErr.message}`));
+          }
+        }
         
         // Client Hints protection for Chrome user agents (skipped under Obscura — it sets its own)
         if (!useObscura && siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
@@ -3425,7 +3501,7 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
         
-        const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains } = navigationResult;
+        const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains, httpStatus, cfRay } = navigationResult;
         
         // Check for same-page reload loops BEFORE redirect processing
         const loadCount = pageLoadHistory.get(currentUrl) || 0;
@@ -3534,8 +3610,14 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
 
-        // Handle all Cloudflare protections using the enhanced module
-        const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug);
+        // Handle all Cloudflare protections using the enhanced module. Pass
+        // httpStatus and cfRay captured at goto time so the outcome log can
+        // surface them — Puppeteer's response object is only available
+        // immediately after page.goto, so handleCloudflareProtection can't
+        // recover them from `page` alone.
+        const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug, { httpStatus, cfRay });
+
+        if (cloudflareResult.cloudflareErrorPage) cloudflareScanStats.errorPages++;
 
         // Check if Cloudflare handling exceeded max retries and should terminate processing
         if (!cloudflareResult.overallSuccess && 
@@ -3568,7 +3650,10 @@ function setupFrameHandling(page, forceDebug) {
             console.warn(`   - ${error}`);
           });
           // Continue with scan despite Cloudflare issues
-        } else if (cloudflareResult.verificationChallenge?.success && forceDebug) {
+        } else if (cloudflareResult.verificationChallenge?.attempted && cloudflareResult.verificationChallenge?.success && forceDebug) {
+          // Require attempted === true so we don't log "Challenge solved using:
+          // undefined" for pages that had no challenge to solve (success: true
+          // is the natural state for that case).
           console.log(formatLogMessage('debug', `[cloudflare] Challenge solved using: ${cloudflareResult.verificationChallenge.method}`));
         }
 
@@ -4277,6 +4362,19 @@ function setupFrameHandling(page, forceDebug) {
   // Sort tasks so proxy groups are contiguous — direct connections first, then each proxy
   allTasks.sort((a, b) => proxyKeyFor(a.config).localeCompare(proxyKeyFor(b.config)));
 
+  // Pre-start local no-auth SOCKS5 relays for any authenticated socks5://
+  // upstreams. Done once here (the only async step) so getProxyArgs stays a
+  // sync lookup in the per-batch browser-launch path. Chromium can't auth
+  // SOCKS5; the relay does the upstream auth transparently.
+  try {
+    const relayCount = await prepareSocksRelays(sites, forceDebug);
+    if (relayCount > 0 && !silentMode) {
+      console.log(messageColors.processing(`Started ${relayCount} SOCKS5 auth relay(s)`));
+    }
+  } catch (relayErr) {
+    console.warn(formatLogMessage('proxy', `SOCKS5 relay setup failed: ${relayErr.message}`));
+  }
+
   let results = [];
   let processedUrlCount = 0;
   let urlsSinceLastCleanup = 0;
@@ -4299,27 +4397,39 @@ function setupFrameHandling(page, forceDebug) {
  let forceRestartFlag = false; // Flag to trigger restart on next iteration
  
  const hangDetectionInterval = setInterval(() => {
+   // Progress check, counter, and forceRestartFlag MUST run regardless of
+   // debug mode — previously the entire body was gated on forceDebug, which
+   // made hang recovery a debug-only feature even though the restart
+   // machinery exists for production scans. Only the verbose diagnostic
+   // logs stay debug-gated; the "no progress" warning and the
+   // "triggering restart" error are user-visible recovery events.
+   if (processedUrlCount === lastProcessedCount) {
+     hangCheckCount++;
+     if (forceDebug) {
+       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
+     }
+     if (hangCheckCount >= 5) {
+       console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
+       forceRestartFlag = true; // Set flag instead of exiting
+       hangCheckCount = 0; // Reset counter for next cycle
+     }
+   } else {
+     hangCheckCount = 0;
+   }
+   lastProcessedCount = processedUrlCount;
+
+   // Debug-only diagnostic snapshot
    if (forceDebug) {
      const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
      const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
      console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
      console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
-     
-     // Check progress and trigger browser restart if hung
-     if (processedUrlCount === lastProcessedCount) {
-       hangCheckCount++;
-       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
-       if (hangCheckCount >= 5) {
-         console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
-         forceRestartFlag = true; // Set flag instead of exiting
-         hangCheckCount = 0; // Reset counter for next cycle
-       }
-     } else {
-       hangCheckCount = 0;
-     }
-     lastProcessedCount = processedUrlCount;
    }
  }, 30000);
+ // Don't keep the event loop alive solely for the hang-check interval — the
+ // clearInterval calls at the normal-exit and error paths already cover the
+ // cleanup, this is belt-and-suspenders in case a future refactor moves them.
+ hangDetectionInterval.unref();
 
  // Process URLs in batches with exception handling
  let siteGroupIndex = 0;
@@ -4387,8 +4497,14 @@ function setupFrameHandling(page, forceDebug) {
       !healthCheck.reason?.includes('Scheduled cleanup') && 
       (healthCheck.reason?.includes('Critical') || healthCheck.reason?.includes('disconnected'));
     
-    // Restart browser if we've processed enough URLs, health check suggests it, hang detected, and this isn't the last site
-    if ((wouldExceedLimit || shouldRestartFromHealth || forceRestartFlag || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8 && isNotLastBatch) {      
+    // Restart conditions split into hang recovery vs proactive triggers.
+    // Hang recovery (forceRestartFlag set by 2.5-min HANG CHECK or a per-URL
+    // timeout) bypasses the urlsSinceLastCleanup > 8 gate — a confirmed hang
+    // needs immediate restart even if we just cleaned up. Proactive triggers
+    // keep the gate to prevent thrashing.
+    const hangRecoveryRestart = forceRestartFlag;
+    const proactiveRestart = (wouldExceedLimit || shouldRestartFromHealth || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8;
+    if ((hangRecoveryRestart || proactiveRestart) && isNotLastBatch) {
       let restartReason = 'Unknown';
       if (forceRestartFlag) {
         restartReason = 'Emergency restart due to 2.5-minute hang detection';
@@ -4517,16 +4633,118 @@ function setupFrameHandling(page, forceDebug) {
       console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
- // Create tasks with timeout protection — skip domains that repeatedly timed out
- const batchTasks = currentBatch.map(task => originalLimit(() => {
+ // Create tasks with timeout protection — skip domains that repeatedly timed out.
+ // Wrapped in an outer try/finally so processedUrlCount is incremented exactly
+ // once per URL no matter which return/throw path is taken — that turns HANG
+ // CHECK's signal from "did the batch finish?" into "did any URL finish?",
+ // which is what 30-second tick granularity actually needs.
+ const batchTasks = currentBatch.map(task => originalLimit(async () => {
    try {
-     const taskDomain = new URL(task.url).hostname;
-     if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
-       if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
-       return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+     // Short-circuit queued URLs once any URL in this batch has triggered a
+     // restart. Without this, the 80-URL batch in the user's hang trace
+     // would have to fail one-by-one at 120s each (~28 min total) before
+     // the boundary restart could fire. Now: first hang fires the flag,
+     // remaining queued URLs return immediately, batch completes, restart.
+     if (forceRestartFlag) {
+       return { url: task.url, rules: [], success: false, error: 'Browser restart pending', skipped: true };
      }
-   } catch {}
-   return processUrl(task.url, task.config, browser);
+
+     try {
+       const taskDomain = new URL(task.url).hostname;
+       if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
+         if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
+         return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+       }
+
+       // DNS pre-check — fails fast on NXDOMAIN/unresolvable hosts before
+       // we pay ~5-15s for Puppeteer navigation + Cloudflare detection.
+       // Skips IP literals. Respects an in-memory negative cache so a dead
+       // host hit by many URL paths only costs one DNS round-trip per TTL.
+       //
+       // Uses dns.resolve* (c-ares, async network I/O) NOT dns.lookup
+       // (getaddrinfo, libuv threadpool). Under scan concurrency Puppeteer
+       // saturates the default 4-slot threadpool with filesystem I/O, so
+       // dns.lookup calls sit queued and blow the timeout while never
+       // actually starting — wrongly skipping live domains. c-ares isn't
+       // threadpool-bound so it's immune to that contention.
+       if (dnsPrecheckEnabled && taskDomain && !/^[\d.:]+$|^\[/.test(taskDomain)) {
+         const cached = dnsNegativeCache.get(taskDomain);
+         if (cached && Date.now() - cached.timestamp < DNS_NEGATIVE_CACHE_TTL_MS) {
+           dnsPrecheckSkips++;
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check (cached): ${taskDomain} — ${cached.error}`));
+           return { url: task.url, rules: [], success: false, error: `DNS: ${cached.error}`, skipped: true };
+         }
+         const dnsResolve = async () => {
+           // resolve4 first; on no-IPv4 (ENODATA / ENOTFOUND) fall back to
+           // resolve6 so IPv6-only hosts aren't wrongly skipped. Only a
+           // failure of BOTH means the host is genuinely unresolvable.
+           // 2s timeout kept as a real safety net — with c-ares off the
+           // threadpool it should now rarely fire.
+           let timer;
+           try {
+             const timeoutP = new Promise((_, reject) => {
+               timer = setTimeout(() => reject(new Error('DNS timeout')), dnsPrecheckTimeoutMs);
+             });
+             const resolveChain = dnsPromises.resolve4(taskDomain)
+               .catch(() => dnsPromises.resolve6(taskDomain));
+             await Promise.race([resolveChain, timeoutP]);
+           } finally {
+             if (timer) clearTimeout(timer);
+           }
+         };
+         // c-ares transient codes — retry once so a momentary resolver
+         // hiccup doesn't poison the negative cache for 5 minutes.
+         const TRANSIENT = new Set(['ETIMEOUT', 'ESERVFAIL', 'EREFUSED', 'ECONNREFUSED']);
+         try {
+           try {
+             await dnsResolve();
+           } catch (firstErr) {
+             const code = firstErr && firstErr.code;
+             if (TRANSIENT.has(code) || (firstErr && firstErr.message === 'DNS timeout')) {
+               if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check transient (${code || 'timeout'}) for ${taskDomain}, retrying once`));
+               await dnsResolve();
+             } else {
+               throw firstErr;
+             }
+           }
+         } catch (dnsErr) {
+           const errCode = dnsErr.code || dnsErr.message || 'DNS resolve failed';
+           dnsNegativeCacheSet(taskDomain, errCode);
+           dnsPrecheckSkips++;
+           if (forceDebug) console.log(formatLogMessage('debug', `DNS pre-check failed: ${taskDomain} — ${errCode}`));
+           return { url: task.url, rules: [], success: false, error: `DNS: ${errCode}`, skipped: true };
+         }
+       }
+     } catch {}
+
+     // Per-URL timeout so a single hung processUrl can't block the batch
+     // forever. 120s is well past any legitimate slow page: Cloudflare
+     // adaptive max ~25s, nettools overall ~65s, navigation 15s.
+     const processUrlPromise = processUrl(task.url, task.config, browser);
+     let perUrlTimer;
+     try {
+       return await Promise.race([
+         processUrlPromise,
+         new Promise((_, reject) => {
+           perUrlTimer = setTimeout(() => reject(new Error('Per-URL timeout (120s)')), 120000);
+         })
+       ]);
+     } catch (err) {
+       if (err && err.message === 'Per-URL timeout (120s)') {
+         processUrlPromise.catch(() => {});
+         forceRestartFlag = true;
+         return { url: task.url, rules: [], success: false, error: 'Per-URL timeout (120s)', needsImmediateRestart: true };
+       }
+       throw err;
+     } finally {
+       if (perUrlTimer) clearTimeout(perUrlTimer);
+     }
+   } finally {
+     // Always count completion — even on unexpected throw — so HANG CHECK's
+     // per-tick progress signal stays accurate. Replaces the old
+     // `processedUrlCount += batchSize` that ran after the whole batch.
+     processedUrlCount++;
+   }
  }));
  
  let batchResults;
@@ -4628,7 +4846,8 @@ function setupFrameHandling(page, forceDebug) {
       }
     }
     
-    processedUrlCount += batchSize;
+    // processedUrlCount is now incremented per-URL inside the batchTasks
+    // wrapper above; no batch-level += batchSize here.
     urlsSinceLastCleanup += batchSize;
 
     // Force browser restart if any URL had critical errors
@@ -4809,12 +5028,43 @@ function setupFrameHandling(page, forceDebug) {
        console.log(formatLogMessage('debug', `Cache hit rate: ${cloudflareStats.hitRate}, Total hits: ${cloudflareStats.hits}, Misses: ${cloudflareStats.misses}`));
        console.log(formatLogMessage('debug', `Cached detections: ${cloudflareStats.size}`));
      }
+     if (cloudflareScanStats.errorPages > 0) {
+       console.log(formatLogMessage('debug', `Cloudflare 5xx origin-error pages: ${cloudflareScanStats.errorPages} (no bypass possible — origin unreachable)`));
+     }
+     if (dnsPrecheckEnabled && dnsPrecheckSkips > 0) {
+       console.log(formatLogMessage('debug', `DNS pre-check skipped: ${dnsPrecheckSkips} URL(s) via ${dnsNegativeCache.size} unresolvable host(s)`));
+     }
      // Log smart cache statistics (if cache is enabled)
      // Adblock statistics
      if (adblockEnabled) {
        console.log(formatLogMessage('debug', '=== Adblock Statistics ==='));
        const blockRate = ((adblockStats.blocked / (adblockStats.blocked + adblockStats.allowed)) * 100).toFixed(1);
        console.log(formatLogMessage('debug', `Blocked: ${adblockStats.blocked} requests (${blockRate}% block rate), Allowed: ${adblockStats.allowed}`));
+
+       // Engine-specific stats from the matcher itself. Both engines expose
+       // getStats() but with slightly different cache shapes — JS engine
+       // tracks urlCacheSize + resultCacheSize separately, rust wrapper
+       // tracks a single size. Handle both.
+       if (adblockMatcher && typeof adblockMatcher.getStats === 'function') {
+         try {
+           const es = adblockMatcher.getStats();
+           const engine = es.engine || 'js';
+           console.log(formatLogMessage('debug', `Engine: ${engine}${es.fromDiskCache ? ' (loaded from disk cache)' : ''}`));
+           if (es.cache && (es.cache.hits != null || es.cache.misses != null)) {
+             // rust wrapper: single `size`; JS engine: split into urlCacheSize + resultCacheSize
+             const sizeDesc = es.cache.size != null
+               ? `${es.cache.size}/${es.cache.maxSize}`
+               : `url ${es.cache.urlCacheSize}, result ${es.cache.resultCacheSize}, cap ${es.cache.maxSize}`;
+             console.log(formatLogMessage('debug', `Matcher cache: ${es.cache.hits} hits / ${es.cache.misses} misses (${es.cache.hitRate}), ${sizeDesc}`));
+           }
+           if (es.exceptions != null && es.exceptions > 0) {
+             console.log(formatLogMessage('debug', `Whitelist exceptions: ${es.exceptions}`));
+           }
+           if (es.errors != null && es.errors > 0) {
+             console.log(formatLogMessage('debug', `Engine errors: ${es.errors}`));
+           }
+         } catch (_) { /* getStats shape mismatch — don't crash the exit path */ }
+       }
      }
     if (smartCache) {
     const cacheStats = smartCache.getStats();  
@@ -4993,8 +5243,18 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
   
+  // Run the same cleanup the SIGINT/SIGTERM emergency handler does, so normal
+  // scan completion isn't left depending on process.exit(0) to override
+  // lingering setInterval handles (the cloudflare detection cache schedules
+  // one that's otherwise only stopped on signal-driven shutdown).
+  try { cleanupCloudflareCache(); } catch (_) {}
+  try { wgDisconnectAll(forceDebug); } catch (_) {}
+  try { ovpnDisconnectAll(forceDebug); } catch (_) {}
+  try { purgeStaleTrackers(); } catch (_) {}
+  try { await closeAllSocksRelays(forceDebug); } catch (_) {}
+
   // Clean process termination
   if (forceDebug) console.log(formatLogMessage('debug', `About to exit process...`));
   process.exit(0);
-  
+
 })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.64",
+  "version": "2.0.66",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {
@@ -14,11 +14,12 @@
     "lru-cache": "^11.3.5",
     "p-limit": "^7.3.0",
     "psl": "^1.15.0",
-    "puppeteer": ">=20.0.0"
+    "puppeteer": ">=20.0.0",
+    "socks": "^2.8.9"
   },
   "overrides": {
     "tar-fs": "3.1.1",
-    "ws": "8.18.3",
+    "ws": ">=8.20.1",
     "yauzl": ">=3.2.1"
   },
   "keywords": [