@fanboynz/network-scanner 2.0.64 → 2.0.65

package/lib/nettools.js

@@ -4,11 +4,9 @@
  */
 
 const { exec, execSync } = require('child_process');
-const util = require('util');
 const fs = require('fs');
 const path = require('path');
 const { formatLogMessage, messageColors } = require('./colorize');
-const execPromise = util.promisify(exec);
 const ANSI_REGEX = /\x1b\[[0-9;]*m/g;
 
 // Cycling index for whois server rotation
@@ -80,7 +78,11 @@ function saveDiskCache(filePath, cache, ttl, maxSize) {
       }
     }
 
-    // If over max, keep only the newest entries
+    // If over max, keep only the newest entries. Drop the pretty-print —
+    // saveDiskCache runs on the synchronous 'exit' handler when --dns-cache
+    // is set, so any work here directly delays scan exit. Compact JSON is
+    // several times faster on multi-megabyte caches and the file is not
+    // intended for human reading.
     if (count > maxSize) {
       const sorted = Object.entries(entries)
         .sort((a, b) => b[1].timestamp - a[1].timestamp)
@@ -89,9 +91,9 @@ function saveDiskCache(filePath, cache, ttl, maxSize) {
       for (const [key, entry] of sorted) {
         trimmed[key] = entry;
       }
-      fs.writeFileSync(filePath, JSON.stringify(trimmed, null, 2));
+      fs.writeFileSync(filePath, JSON.stringify(trimmed));
     } else {
-      fs.writeFileSync(filePath, JSON.stringify(entries, null, 2));
+      fs.writeFileSync(filePath, JSON.stringify(entries));
     }
   } catch {
     // Disk write failed — non-fatal, in-memory cache still works
@@ -125,14 +127,18 @@ function enableDiskCache() {
   loadDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
   loadDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
 
-  // Save caches to disk once on process exit instead of per-lookup
+  // Save caches to disk once on process exit instead of per-lookup. The
+  // 'exit' handler fires synchronously regardless of how the process exits
+  // (normal completion, signal, uncaught exception), so a separate signal
+  // handler is redundant. We deliberately do NOT install SIGINT/SIGTERM
+  // handlers here — nwss.js installs its own async ones that perform
+  // browser/VPN cleanup, and a sync handler here would call process.exit(0)
+  // first and skip that cleanup entirely.
   const flushCaches = () => {
     saveDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
     saveDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
   };
   process.on('exit', flushCaches);
-  process.on('SIGINT', () => { flushCaches(); process.exit(0); });
-  process.on('SIGTERM', () => { flushCaches(); process.exit(0); });
 }
 
 /**
@@ -217,14 +223,18 @@ function execWithTimeout(command, timeout = 10000) {
     // Set up timeout
     const timer = setTimeout(() => {
       child.kill('SIGTERM');
-      
-      // Force kill after 2 seconds if SIGTERM doesn't work
-      setTimeout(() => {
+
+      // Force kill after 2 seconds if SIGTERM doesn't work. unref() so this
+      // tail timer doesn't keep the event loop alive past scan completion —
+      // a dig that times out near the end of a scan would otherwise delay
+      // exit by ~2 seconds.
+      const killTimer = setTimeout(() => {
         if (!child.killed) {
           child.kill('SIGKILL');
         }
       }, 2000);
-      
+      killTimer.unref();
+
       reject(new Error(`Command timeout after ${timeout}ms: ${command}`));
     }, timeout);
     
@@ -925,6 +935,31 @@ function createNetToolsHandler(config) {
   const hasWhoisOr = whoisOrTerms && Array.isArray(whoisOrTerms) && whoisOrTerms.length > 0;
   const hasDig = digTerms && Array.isArray(digTerms) && digTerms.length > 0;
   const hasDigOr = digOrTerms && Array.isArray(digOrTerms) && digOrTerms.length > 0;
+
+  // Pre-lowercase search terms once per handler so the per-domain check loop
+  // doesn't re-lowercase the same constants for every output it scans.
+  const whoisTermsLower   = hasWhois   ? whoisTerms.map(t => t.toLowerCase())   : null;
+  const whoisOrTermsLower = hasWhoisOr ? whoisOrTerms.map(t => t.toLowerCase()) : null;
+  const digTermsLower     = hasDig     ? digTerms.map(t => t.toLowerCase())     : null;
+  const digOrTermsLower   = hasDigOr   ? digOrTerms.map(t => t.toLowerCase())   : null;
+
+  // Hoisted out of handleNetToolsCheck so the closure is constructed once per
+  // handler rather than once per invocation. References forceDebug, debugLogFile,
+  // and fs from the destructured config above.
+  function logToConsoleAndFile(message) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', message));
+    }
+    if (debugLogFile && fs) {
+      try {
+        const timestamp = new Date().toISOString();
+        const cleanMessage = stripAnsiColors(message);
+        fs.appendFileSync(debugLogFile, `${timestamp} [debug nettools] ${cleanMessage}\n`);
+      } catch (_) {
+        // Silently fail file logging to avoid disrupting whois operations
+      }
+    }
+  }
   
   // Create config-aware cache keys for deduplication
   // Whois: Only include search terms + server (domain registry data is consistent across subdomains)
@@ -948,10 +983,7 @@ function createNetToolsHandler(config) {
   // DNS results are the same regardless of search terms
   
   return async function handleNetToolsCheck(domain, fullSubdomain) {
-    // Use fullSubdomain parameter instead of originalDomain to maintain consistency
-    // with the domain cache fix approach
     const originalDomain = fullSubdomain;
-    // Helper function to log to BOTH console and debug file
 
     // Check if domain was already detected (skip expensive operations)
     if (typeof isDomainAlreadyDetected === 'function' && isDomainAlreadyDetected(fullSubdomain)) {
@@ -960,36 +992,7 @@ function createNetToolsHandler(config) {
       }
       return;
     }
-    
-    // NOTE: The logToConsoleAndFile function needs to be declared INSIDE this function
-    // so it has access to the closure variables (forceDebug, debugLogFile, fs) from the 
-    // createNetToolsHandler config. This function was being called but not declared
-    // within the scope where whoisLookup and whoisLookupWithRetry try to use it.
-    // This is why we were getting "logToConsoleAndFile is not defined" errors.
 
-    // Move the logToConsoleAndFile function declaration from later in the file to here:
-    function logToConsoleAndFile(message) {
-      // Note: This function needs access to forceDebug, debugLogFile, and fs from the parent scope
-      // These are passed in via the config object to createNetToolsHandler
-      // forceDebug, debugLogFile, and fs are available in this closure
-
-      // Always log to console when in debug mode
-      if (forceDebug) {
-        console.log(formatLogMessage('debug', message));
-      }
-      
-      // Also log to file if debug file logging is enabled
-      if (debugLogFile && fs) {
-        try {
-          const timestamp = new Date().toISOString();
-          const cleanMessage = stripAnsiColors(message);
-          fs.appendFileSync(debugLogFile, `${timestamp} [debug nettools] ${cleanMessage}\n`);
-        } catch (logErr) {
-          // Silently fail file logging to avoid disrupting whois operations
-        }
-      }
-    }
-    
     // Determine which domain will be used for dig lookup
     const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
 
@@ -1152,8 +1155,13 @@ function createNetToolsHandler(config) {
             try {
               const lookupPromise = whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
               pendingWhoisLookups.set(whoisCacheKey, lookupPromise);
-              whoisResult = await lookupPromise;
-              pendingWhoisLookups.delete(whoisCacheKey);
+              // try/finally so a rejected lookup still clears the pending
+              // entry — see matching comment on pendingDigLookups below.
+              try {
+                whoisResult = await lookupPromise;
+              } finally {
+                pendingWhoisLookups.delete(whoisCacheKey);
+              }
 
               // Cache successful results (and certain types of failures)
               if (whoisResult.success ||
@@ -1196,11 +1204,18 @@ function createNetToolsHandler(config) {
         
         // Process whois result (whether from cache or fresh lookup)
         if (whoisResult) {
-          
+
           if (whoisResult.success) {
+            // Lowercase the output ONCE — checkWhoisTerms / checkWhoisTermsOr
+            // each call .toLowerCase() on their input independently, which
+            // re-allocates a multi-KB lowercased string per call. Pre-lowering
+            // here lets the AND check, OR check, and matched-term find share
+            // a single allocation.
+            const whoisOutputLower = whoisResult.output.toLowerCase();
+
             // Check AND terms if configured
             if (hasWhois) {
-              whoisMatched = checkWhoisTerms(whoisResult.output, whoisTerms);
+              whoisMatched = whoisTermsLower.every(t => whoisOutputLower.includes(t));
               if (whoisMatched && dryRunCallback) {
                 dryRunCallback(domain, 'whois', 'AND logic', whoisTerms.join(', '), 'All terms found in whois data', {
                   server: whoisResult.whoisServer || 'default',
@@ -1214,12 +1229,13 @@ function createNetToolsHandler(config) {
               }
 
             }
-          
+
             // Check OR terms if configured
             if (hasWhoisOr) {
-              whoisOrMatched = checkWhoisTermsOr(whoisResult.output, whoisOrTerms);
+              whoisOrMatched = whoisOrTermsLower.some(t => whoisOutputLower.includes(t));
               if (whoisOrMatched && dryRunCallback) {
-                const matchedTerm = whoisOrTerms.find(term => whoisResult.output.toLowerCase().includes(term.toLowerCase()));
+                const matchedIdx = whoisOrTermsLower.findIndex(t => whoisOutputLower.includes(t));
+                const matchedTerm = whoisOrTerms[matchedIdx];
                 dryRunCallback(domain, 'whois', 'OR logic', matchedTerm, 'Term found in whois data', {
                   server: whoisResult.whoisServer || 'default',
                   duration: whoisResult.duration,
@@ -1371,8 +1387,15 @@ function createNetToolsHandler(config) {
             } else {
               const lookupPromise = digLookup(digDomain, digRecordType, 5000);
               pendingDigLookups.set(digCacheKey, lookupPromise);
-              digResult = await lookupPromise;
-              pendingDigLookups.delete(digCacheKey);
+              // try/finally so a rejected lookup still clears the pending
+              // entry — otherwise the Map would retain a rejected-Promise
+              // entry forever and any subsequent caller for the same key
+              // would await that rejection.
+              try {
+                digResult = await lookupPromise;
+              } finally {
+                pendingDigLookups.delete(digCacheKey);
+              }
 
               // Cache the result for future use
               globalDigResultCache.set(digCacheKey, {
@@ -1389,9 +1412,13 @@ function createNetToolsHandler(config) {
           }
           
           if (digResult.success) {
+            // Lowercase the output ONCE — see matching comment in the whois
+            // branch above for rationale.
+            const digOutputLower = digResult.output.toLowerCase();
+
             // Check AND terms if configured
             if (hasDig) {
-              digMatched = checkDigTerms(digResult.output, digTerms);
+              digMatched = digTermsLower.every(t => digOutputLower.includes(t));
               if (digMatched && dryRunCallback) {
                 dryRunCallback(domain, 'dig', 'AND logic', digTerms.join(', '), `All terms found in ${digRecordType} records`, {
                   queriedDomain: digDomain,
@@ -1400,12 +1427,13 @@ function createNetToolsHandler(config) {
                 });
               }
             }
-            
+
             // Check OR terms if configured
             if (hasDigOr) {
-              digOrMatched = checkDigTermsOr(digResult.output, digOrTerms);
+              digOrMatched = digOrTermsLower.some(t => digOutputLower.includes(t));
               if (digOrMatched && dryRunCallback) {
-                const matchedTerm = digOrTerms.find(term => digResult.output.toLowerCase().includes(term.toLowerCase()));
+                const matchedIdx = digOrTermsLower.findIndex(t => digOutputLower.includes(t));
+                const matchedTerm = digOrTerms[matchedIdx];
                 dryRunCallback(domain, 'dig', 'OR logic', matchedTerm, `Term found in ${digRecordType} records`, {
                   queriedDomain: digDomain,
                   recordType: digRecordType,

package/lib/redirect.js

@@ -15,6 +15,9 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
   const redirectChain = [currentUrl];
   let finalUrl = currentUrl;
   let redirected = false;
+  // Hoisted so they're in scope at the return outside the try block below.
+  let httpStatus = null;
+  let cfRay = null;
   const jsRedirectTimeout = siteConfig.js_redirect_timeout || 5000; // Wait 5s for JS redirects
   const maxRedirects = siteConfig.max_redirects || 10;
   const detectJSPatterns = siteConfig.detect_js_patterns !== false; // Default to true
@@ -23,7 +26,12 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
   const navigationHandler = (frame) => {
     if (frame === page.mainFrame()) {
       const frameUrl = frame.url();
-      if (frameUrl && frameUrl !== 'about:blank' && !redirectChain.includes(frameUrl)) {
+      // Skip about:blank and chrome-error:// — the latter is what Puppeteer
+      // navigates to on DNS/connection failures, and pushing it into the
+      // redirect chain produces bogus entries like
+      // "chrome-error://chromewebdata/" that downstream consumers
+      // (redirectDomains, logs) treat as a real intermediate hop.
+      if (frameUrl && frameUrl !== 'about:blank' && !frameUrl.startsWith('chrome-error://') && !redirectChain.includes(frameUrl)) {
         // Check redirect limit before adding
         if (redirectChain.length >= maxRedirects) {
           if (forceDebug) {
@@ -161,9 +169,21 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
       console.log(formatLogMessage('debug', `Using goto options: ${JSON.stringify(gotoOptions)}`));
     }
 
-    // Initial navigation
+    // Initial navigation. Puppeteer's page.goto returns the response for the
+    // last HTTP request in the chain (it follows HTTP redirects internally),
+    // so response.status() reflects the page that actually rendered, not the
+    // 301/302 hop. JS redirects via window.location detected later in this
+    // function will land on a different page, in which case httpStatus/cfRay
+    // captured here are pre-JS-redirect — a known limitation.
     const response = await page.goto(currentUrl, gotoOptions);
-    
+    if (response) {
+      try {
+        httpStatus = response.status();
+        const headers = response.headers();
+        if (headers && headers['cf-ray']) cfRay = headers['cf-ray'];
+      } catch (_) { /* response disposed or detached — fine, stays null */ }
+    }
+
     if (response && response.url() !== currentUrl) {
       // Check redirect limit before adding
       if (redirectChain.length >= maxRedirects) {
@@ -295,7 +315,7 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
     redirectDomains = intermediateDomains;
   }
 
-  return { finalUrl, redirected, redirectChain, originalUrl: currentUrl, redirectDomains };
+  return { finalUrl, redirected, redirectChain, originalUrl: currentUrl, redirectDomains, httpStatus, cfRay };
 }
 
 /**
@@ -306,13 +326,23 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
  * @returns {Promise<Array>} Array of detected patterns
  */
 async function detectCommonJSRedirects(page, forceDebug = false, formatLogMessage) {
+  // This function's only externally-visible behavior is the per-pattern
+  // debug log below. The return value isn't read by any caller. Bail
+  // before the expensive page.evaluate + outerHTML serialization when
+  // there's no debug consumer for the result.
+  if (!forceDebug) return [];
+
   try {
     const redirectPatterns = await page.evaluate(() => {
       const patterns = [];
-      
-      // Check for common redirect patterns in page source
-      const pageSource = document.documentElement.outerHTML;
-      
+
+      // Cap the source read to 100KB. document.documentElement.outerHTML
+      // materializes the full page (potentially many MB on content-heavy
+      // sites) AND serializes it over CDP back to Node. JS redirects all
+      // appear early — in head meta tags or top-of-body inline scripts —
+      // so a head-anchored cap is enough for real-world coverage.
+      const pageSource = document.documentElement.outerHTML.substring(0, 100000);
+
       // Pattern 1: window.location = "url"
       const locationAssign = pageSource.match(/window\.location\s*=\s*["']([^"']+)["']/g);
       if (locationAssign) {