@fanboynz/network-scanner 2.0.64 → 2.0.65

package/nwss.js

@@ -360,6 +360,12 @@ let adblockEnabled = false;
 let adblockMatcher = null;
 let adblockStats = { blocked: 0, allowed: 0 };
 
+// Cloudflare scan-wide stats. errorPages counts URLs where the returned page
+// was a Cloudflare-served 5xx origin error (522/523/etc.) — no bypass
+// possible, useful signal for diagnosing dead-origin scans. Named distinct
+// from the local cloudflareStats = getCacheStats() in the debug stats block.
+let cloudflareScanStats = { errorPages: 0 };
+
 // Validate --adblock-rules usage - ignore if used incorrectly instead of erroring
 if (adblockRulesMode) {
   if (!outputFile) {
@@ -478,78 +484,10 @@ if (testValidation) {
   }
 }
 
-if (validateConfig) {
-  console.log(`\n${messageColors.processing('Validating configuration file...')}`);
-  try {
-    const validation = validateFullConfig(config, { forceDebug, silentMode });
-    
-    // Validate referrer_headers format
-    for (const site of sites) {
-       if (site.referrer_headers && typeof site.referrer_headers === 'object' && !Array.isArray(site.referrer_headers)) {
-         const validation = validateReferrerConfig(site.referrer_headers);
-         if (!validation.isValid) {
-           console.warn(`⚠ Invalid referrer_headers configuration: ${validation.errors.join(', ')}`);
-         }
-         if (validation.warnings.length > 0) {
-           console.warn(`⚠ Referrer warnings: ${validation.warnings.join(', ')}`);
-         }
-       }
-       // Validate referrer_disable format
-       if (site.referrer_disable) {
-         const disableValidation = validateReferrerDisable(site.referrer_disable);
-         if (!disableValidation.isValid) {
-           console.warn(`⚠ Invalid referrer_disable configuration: ${disableValidation.errors.join(', ')}`);
-         }
-         if (disableValidation.warnings.length > 0) {
-           console.warn(`⚠ Referrer disable warnings: ${disableValidation.warnings.join(', ')}`);
-         }
-       }
-    }
-
-    // Validate VPN configurations
-    for (const site of sites) {
-      if (site.vpn) {
-        const vpnNorm = normalizeVpnConfig(site.vpn);
-        const vpnValidation = validateVpnConfig(vpnNorm);
-        if (!vpnValidation.isValid) {
-          console.warn(`⚠ Invalid vpn configuration for ${site.url}: ${vpnValidation.errors.join(', ')}`);
-        }
-        if (vpnValidation.warnings.length > 0) {
-          vpnValidation.warnings.forEach(w => console.warn(`⚠ VPN warning for ${site.url}: ${w}`));
-        }
-      }
-      if (site.openvpn) {
-        const ovpnNorm = normalizeOvpnConfig(site.openvpn);
-        const ovpnValidation = validateOvpnConfig(ovpnNorm);
-        if (!ovpnValidation.isValid) {
-          console.warn(`⚠ Invalid openvpn configuration for ${site.url}: ${ovpnValidation.errors.join(', ')}`);
-        }
-        if (ovpnValidation.warnings.length > 0) {
-          ovpnValidation.warnings.forEach(w => console.warn(`⚠ OpenVPN warning for ${site.url}: ${w}`));
-        }
-      }
-      if (site.vpn && site.openvpn) {
-        console.warn(`⚠ ${site.url} has both vpn and openvpn configured — only one will be used (vpn takes precedence)`);
-      }
-    }
-
-    if (validation.isValid) {
-      console.log(`${messageColors.success('✅ Configuration is valid!')}`);
-      console.log(`${messageColors.info('Summary:')} ${validation.summary.validSites}/${validation.summary.totalSites} sites valid`);
-      if (validation.summary.sitesWithWarnings > 0) {
-        console.log(`${messageColors.warn('⚠ Warnings:')} ${validation.summary.sitesWithWarnings} sites have warnings`);
-      }
-      process.exit(0);
-    } else {
-      console.log(`${messageColors.error('❌ Configuration validation failed!')}`);
-      console.log(`${messageColors.error('Errors:')} ${validation.globalErrors.length} global, ${validation.summary.sitesWithErrors} site-specific`);
-      process.exit(1);
-    }
-  } catch (validationErr) {
-    console.error(`❌ Validation failed: ${validationErr.message}`);
-    process.exit(1);
-  }
-}
+// Note: --validate-config is handled further down, AFTER the config file is
+// loaded and `config`/`sites` are populated. Running it here would fail with
+// "Cannot access 'config' before initialization" since those are declared
+// later in the module.
 
 if (validateRules || validateRulesFile) {
   const filesToValidate = validateRulesFile ? [validateRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -909,10 +847,86 @@ const {
   disable_ad_tagging = true,
   max_concurrent_sites = 6,
   resource_cleanup_interval = 80,
-  comments: globalComments, 
-  ...otherGlobalConfig 
+  comments: globalComments,
+  ...otherGlobalConfig
 } = config;
 
+// --validate-config runs here, after `config` and `sites` are populated.
+// Previously this block lived above the config load and triggered a TDZ
+// "Cannot access 'config' before initialization" error.
+if (validateConfig) {
+  console.log(`\n${messageColors.processing('Validating configuration file...')}`);
+  try {
+    const validation = validateFullConfig(config, { forceDebug, silentMode });
+
+    // Validate referrer_headers format
+    for (const site of sites) {
+       if (site.referrer_headers && typeof site.referrer_headers === 'object' && !Array.isArray(site.referrer_headers)) {
+         const refValidation = validateReferrerConfig(site.referrer_headers);
+         if (!refValidation.isValid) {
+           console.warn(`⚠ Invalid referrer_headers configuration: ${refValidation.errors.join(', ')}`);
+         }
+         if (refValidation.warnings.length > 0) {
+           console.warn(`⚠ Referrer warnings: ${refValidation.warnings.join(', ')}`);
+         }
+       }
+       // Validate referrer_disable format
+       if (site.referrer_disable) {
+         const disableValidation = validateReferrerDisable(site.referrer_disable);
+         if (!disableValidation.isValid) {
+           console.warn(`⚠ Invalid referrer_disable configuration: ${disableValidation.errors.join(', ')}`);
+         }
+         if (disableValidation.warnings.length > 0) {
+           console.warn(`⚠ Referrer disable warnings: ${disableValidation.warnings.join(', ')}`);
+         }
+       }
+    }
+
+    // Validate VPN configurations
+    for (const site of sites) {
+      if (site.vpn) {
+        const vpnNorm = normalizeVpnConfig(site.vpn);
+        const vpnValidation = validateVpnConfig(vpnNorm);
+        if (!vpnValidation.isValid) {
+          console.warn(`⚠ Invalid vpn configuration for ${site.url}: ${vpnValidation.errors.join(', ')}`);
+        }
+        if (vpnValidation.warnings.length > 0) {
+          vpnValidation.warnings.forEach(w => console.warn(`⚠ VPN warning for ${site.url}: ${w}`));
+        }
+      }
+      if (site.openvpn) {
+        const ovpnNorm = normalizeOvpnConfig(site.openvpn);
+        const ovpnValidation = validateOvpnConfig(ovpnNorm);
+        if (!ovpnValidation.isValid) {
+          console.warn(`⚠ Invalid openvpn configuration for ${site.url}: ${ovpnValidation.errors.join(', ')}`);
+        }
+        if (ovpnValidation.warnings.length > 0) {
+          ovpnValidation.warnings.forEach(w => console.warn(`⚠ OpenVPN warning for ${site.url}: ${w}`));
+        }
+      }
+      if (site.vpn && site.openvpn) {
+        console.warn(`⚠ ${site.url} has both vpn and openvpn configured — only one will be used (vpn takes precedence)`);
+      }
+    }
+
+    if (validation.isValid) {
+      console.log(`${messageColors.success('✅ Configuration is valid!')}`);
+      console.log(`${messageColors.info('Summary:')} ${validation.summary.validSites}/${validation.summary.totalSites} sites valid`);
+      if (validation.summary.sitesWithWarnings > 0) {
+        console.log(`${messageColors.warn('⚠ Warnings:')} ${validation.summary.sitesWithWarnings} sites have warnings`);
+      }
+      process.exit(0);
+    } else {
+      console.log(`${messageColors.error('❌ Configuration validation failed!')}`);
+      console.log(`${messageColors.error('Errors:')} ${validation.globalErrors.length} global, ${validation.summary.sitesWithErrors} site-specific`);
+      process.exit(1);
+    }
+  } catch (validationErr) {
+    console.error(`❌ Validation failed: ${validationErr.message}`);
+    process.exit(1);
+  }
+}
+
 // Pre-compile global blocked regexes ONCE (used in every processUrl call)
 const globalBlockedRegexes = Array.isArray(globalBlocked)
   ? globalBlocked.map(pattern => new RegExp(pattern))
@@ -3425,7 +3439,7 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
         
-        const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains } = navigationResult;
+        const { finalUrl, redirected, redirectChain, originalUrl, redirectDomains, httpStatus, cfRay } = navigationResult;
         
         // Check for same-page reload loops BEFORE redirect processing
         const loadCount = pageLoadHistory.get(currentUrl) || 0;
@@ -3534,8 +3548,14 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
 
-        // Handle all Cloudflare protections using the enhanced module
-        const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug);
+        // Handle all Cloudflare protections using the enhanced module. Pass
+        // httpStatus and cfRay captured at goto time so the outcome log can
+        // surface them — Puppeteer's response object is only available
+        // immediately after page.goto, so handleCloudflareProtection can't
+        // recover them from `page` alone.
+        const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug, { httpStatus, cfRay });
+
+        if (cloudflareResult.cloudflareErrorPage) cloudflareScanStats.errorPages++;
 
         // Check if Cloudflare handling exceeded max retries and should terminate processing
         if (!cloudflareResult.overallSuccess && 
@@ -3568,7 +3588,10 @@ function setupFrameHandling(page, forceDebug) {
             console.warn(`   - ${error}`);
           });
           // Continue with scan despite Cloudflare issues
-        } else if (cloudflareResult.verificationChallenge?.success && forceDebug) {
+        } else if (cloudflareResult.verificationChallenge?.attempted && cloudflareResult.verificationChallenge?.success && forceDebug) {
+          // Require attempted === true so we don't log "Challenge solved using:
+          // undefined" for pages that had no challenge to solve (success: true
+          // is the natural state for that case).
           console.log(formatLogMessage('debug', `[cloudflare] Challenge solved using: ${cloudflareResult.verificationChallenge.method}`));
         }
 
@@ -4299,27 +4322,39 @@ function setupFrameHandling(page, forceDebug) {
  let forceRestartFlag = false; // Flag to trigger restart on next iteration
  
  const hangDetectionInterval = setInterval(() => {
+   // Progress check, counter, and forceRestartFlag MUST run regardless of
+   // debug mode — previously the entire body was gated on forceDebug, which
+   // made hang recovery a debug-only feature even though the restart
+   // machinery exists for production scans. Only the verbose diagnostic
+   // logs stay debug-gated; the "no progress" warning and the
+   // "triggering restart" error are user-visible recovery events.
+   if (processedUrlCount === lastProcessedCount) {
+     hangCheckCount++;
+     if (forceDebug) {
+       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
+     }
+     if (hangCheckCount >= 5) {
+       console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
+       forceRestartFlag = true; // Set flag instead of exiting
+       hangCheckCount = 0; // Reset counter for next cycle
+     }
+   } else {
+     hangCheckCount = 0;
+   }
+   lastProcessedCount = processedUrlCount;
+
+   // Debug-only diagnostic snapshot
    if (forceDebug) {
      const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
      const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
      console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
      console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
-     
-     // Check progress and trigger browser restart if hung
-     if (processedUrlCount === lastProcessedCount) {
-       hangCheckCount++;
-       console.log(formatLogMessage('warn', `[HANG CHECK] No progress for ${hangCheckCount * 30}s`));
-       if (hangCheckCount >= 5) {
-         console.log(formatLogMessage('error', `[HANG CHECK] Hung for 2.5 minutes. Triggering emergency browser restart.`));
-         forceRestartFlag = true; // Set flag instead of exiting
-         hangCheckCount = 0; // Reset counter for next cycle
-       }
-     } else {
-       hangCheckCount = 0;
-     }
-     lastProcessedCount = processedUrlCount;
    }
  }, 30000);
+ // Don't keep the event loop alive solely for the hang-check interval — the
+ // clearInterval calls at the normal-exit and error paths already cover the
+ // cleanup, this is belt-and-suspenders in case a future refactor moves them.
+ hangDetectionInterval.unref();
 
  // Process URLs in batches with exception handling
  let siteGroupIndex = 0;
@@ -4387,8 +4422,14 @@ function setupFrameHandling(page, forceDebug) {
       !healthCheck.reason?.includes('Scheduled cleanup') && 
       (healthCheck.reason?.includes('Critical') || healthCheck.reason?.includes('disconnected'));
     
-    // Restart browser if we've processed enough URLs, health check suggests it, hang detected, and this isn't the last site
-    if ((wouldExceedLimit || shouldRestartFromHealth || forceRestartFlag || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8 && isNotLastBatch) {      
+    // Restart conditions split into hang recovery vs proactive triggers.
+    // Hang recovery (forceRestartFlag set by 2.5-min HANG CHECK or a per-URL
+    // timeout) bypasses the urlsSinceLastCleanup > 8 gate — a confirmed hang
+    // needs immediate restart even if we just cleaned up. Proactive triggers
+    // keep the gate to prevent thrashing.
+    const hangRecoveryRestart = forceRestartFlag;
+    const proactiveRestart = (wouldExceedLimit || shouldRestartFromHealth || (hasHighFailureRate && recentResults.length >= 6)) && urlsSinceLastCleanup > 8;
+    if ((hangRecoveryRestart || proactiveRestart) && isNotLastBatch) {
       let restartReason = 'Unknown';
       if (forceRestartFlag) {
         restartReason = 'Emergency restart due to 2.5-minute hang detection';
@@ -4517,16 +4558,58 @@ function setupFrameHandling(page, forceDebug) {
       console.log(formatLogMessage('debug', `[CONCURRENCY] Starting ${batchSize} concurrent tasks with limit ${MAX_CONCURRENT_SITES}`));
     }
     
- // Create tasks with timeout protection — skip domains that repeatedly timed out
- const batchTasks = currentBatch.map(task => originalLimit(() => {
+ // Create tasks with timeout protection — skip domains that repeatedly timed out.
+ // Wrapped in an outer try/finally so processedUrlCount is incremented exactly
+ // once per URL no matter which return/throw path is taken — that turns HANG
+ // CHECK's signal from "did the batch finish?" into "did any URL finish?",
+ // which is what 30-second tick granularity actually needs.
+ const batchTasks = currentBatch.map(task => originalLimit(async () => {
    try {
-     const taskDomain = new URL(task.url).hostname;
-     if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
-       if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
-       return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+     // Short-circuit queued URLs once any URL in this batch has triggered a
+     // restart. Without this, the 80-URL batch in the user's hang trace
+     // would have to fail one-by-one at 120s each (~28 min total) before
+     // the boundary restart could fire. Now: first hang fires the flag,
+     // remaining queued URLs return immediately, batch completes, restart.
+     if (forceRestartFlag) {
+       return { url: task.url, rules: [], success: false, error: 'Browser restart pending', skipped: true };
+     }
+
+     try {
+       const taskDomain = new URL(task.url).hostname;
+       if ((domainTimeoutCounts.get(taskDomain) || 0) >= DOMAIN_TIMEOUT_THRESHOLD) {
+         if (!silentMode) console.log(formatLogMessage('info', `Skipping ${task.url} — ${taskDomain} timed out ${DOMAIN_TIMEOUT_THRESHOLD} times`));
+         return { url: task.url, rules: [], success: false, error: 'Domain repeatedly timed out', skipped: true };
+       }
+     } catch {}
+
+     // Per-URL timeout so a single hung processUrl can't block the batch
+     // forever. 120s is well past any legitimate slow page: Cloudflare
+     // adaptive max ~25s, nettools overall ~65s, navigation 15s.
+     const processUrlPromise = processUrl(task.url, task.config, browser);
+     let perUrlTimer;
+     try {
+       return await Promise.race([
+         processUrlPromise,
+         new Promise((_, reject) => {
+           perUrlTimer = setTimeout(() => reject(new Error('Per-URL timeout (120s)')), 120000);
+         })
+       ]);
+     } catch (err) {
+       if (err && err.message === 'Per-URL timeout (120s)') {
+         processUrlPromise.catch(() => {});
+         forceRestartFlag = true;
+         return { url: task.url, rules: [], success: false, error: 'Per-URL timeout (120s)', needsImmediateRestart: true };
+       }
+       throw err;
+     } finally {
+       if (perUrlTimer) clearTimeout(perUrlTimer);
      }
-   } catch {}
-   return processUrl(task.url, task.config, browser);
+   } finally {
+     // Always count completion — even on unexpected throw — so HANG CHECK's
+     // per-tick progress signal stays accurate. Replaces the old
+     // `processedUrlCount += batchSize` that ran after the whole batch.
+     processedUrlCount++;
+   }
  }));
  
  let batchResults;
@@ -4628,7 +4711,8 @@ function setupFrameHandling(page, forceDebug) {
       }
     }
     
-    processedUrlCount += batchSize;
+    // processedUrlCount is now incremented per-URL inside the batchTasks
+    // wrapper above; no batch-level += batchSize here.
     urlsSinceLastCleanup += batchSize;
 
     // Force browser restart if any URL had critical errors
@@ -4809,12 +4893,40 @@ function setupFrameHandling(page, forceDebug) {
        console.log(formatLogMessage('debug', `Cache hit rate: ${cloudflareStats.hitRate}, Total hits: ${cloudflareStats.hits}, Misses: ${cloudflareStats.misses}`));
        console.log(formatLogMessage('debug', `Cached detections: ${cloudflareStats.size}`));
      }
+     if (cloudflareScanStats.errorPages > 0) {
+       console.log(formatLogMessage('debug', `Cloudflare 5xx origin-error pages: ${cloudflareScanStats.errorPages} (no bypass possible — origin unreachable)`));
+     }
      // Log smart cache statistics (if cache is enabled)
      // Adblock statistics
      if (adblockEnabled) {
        console.log(formatLogMessage('debug', '=== Adblock Statistics ==='));
        const blockRate = ((adblockStats.blocked / (adblockStats.blocked + adblockStats.allowed)) * 100).toFixed(1);
        console.log(formatLogMessage('debug', `Blocked: ${adblockStats.blocked} requests (${blockRate}% block rate), Allowed: ${adblockStats.allowed}`));
+
+       // Engine-specific stats from the matcher itself. Both engines expose
+       // getStats() but with slightly different cache shapes — JS engine
+       // tracks urlCacheSize + resultCacheSize separately, rust wrapper
+       // tracks a single size. Handle both.
+       if (adblockMatcher && typeof adblockMatcher.getStats === 'function') {
+         try {
+           const es = adblockMatcher.getStats();
+           const engine = es.engine || 'js';
+           console.log(formatLogMessage('debug', `Engine: ${engine}${es.fromDiskCache ? ' (loaded from disk cache)' : ''}`));
+           if (es.cache && (es.cache.hits != null || es.cache.misses != null)) {
+             // rust wrapper: single `size`; JS engine: split into urlCacheSize + resultCacheSize
+             const sizeDesc = es.cache.size != null
+               ? `${es.cache.size}/${es.cache.maxSize}`
+               : `url ${es.cache.urlCacheSize}, result ${es.cache.resultCacheSize}, cap ${es.cache.maxSize}`;
+             console.log(formatLogMessage('debug', `Matcher cache: ${es.cache.hits} hits / ${es.cache.misses} misses (${es.cache.hitRate}), ${sizeDesc}`));
+           }
+           if (es.exceptions != null && es.exceptions > 0) {
+             console.log(formatLogMessage('debug', `Whitelist exceptions: ${es.exceptions}`));
+           }
+           if (es.errors != null && es.errors > 0) {
+             console.log(formatLogMessage('debug', `Engine errors: ${es.errors}`));
+           }
+         } catch (_) { /* getStats shape mismatch — don't crash the exit path */ }
+       }
      }
     if (smartCache) {
     const cacheStats = smartCache.getStats();  
@@ -4993,8 +5105,17 @@ function setupFrameHandling(page, forceDebug) {
     }
   }
   
+  // Run the same cleanup the SIGINT/SIGTERM emergency handler does, so normal
+  // scan completion isn't left depending on process.exit(0) to override
+  // lingering setInterval handles (the cloudflare detection cache schedules
+  // one that's otherwise only stopped on signal-driven shutdown).
+  try { cleanupCloudflareCache(); } catch (_) {}
+  try { wgDisconnectAll(forceDebug); } catch (_) {}
+  try { ovpnDisconnectAll(forceDebug); } catch (_) {}
+  try { purgeStaleTrackers(); } catch (_) {}
+
   // Clean process termination
   if (forceDebug) console.log(formatLogMessage('debug', `About to exit process...`));
   process.exit(0);
-  
+
 })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.64",
+  "version": "2.0.65",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {