@fanboynz/network-scanner 2.0.25 → 2.0.26

package/README.md

@@ -223,7 +223,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `whois_timeout_multiplier` | Number | `1.5` | Timeout increase multiplier per retry |
 | `whois_use_fallback` | Boolean | `true` | Add TLD-specific fallback servers |
 | `whois_retry_on_timeout` | Boolean | `true` | Retry on timeout errors |
-| `whois_retry_on_error` | Boolean | `false` | Retry on connection/other errors |
+| `whois_retry_on_error` | Boolean | `true` | Retry on connection/other errors |
 | `dig`                | Array | - | Check dig output for ALL specified terms (AND logic) |
 | `dig-or`             | Array | - | Check dig output for ANY specified term (OR logic) |
 | `dig_subdomain`      | Boolean | `false` | Use subdomain for dig lookup instead of root domain |

package/lib/fingerprint.js

@@ -59,7 +59,7 @@ const PRECOMPILED_MOCKS = Object.freeze({
       postMessage: () => {},
       disconnect: () => {}
     }),
-    getManifest: () => Object.freeze({ name: "Chrome", version: "140.0.0.0" }),
+    getManifest: () => Object.freeze({ name: "Chrome", version: "141.0.0.0" }),
     getURL: (path) => `chrome-extension://invalid/${path}`,
     id: undefined
   }),
@@ -91,9 +91,9 @@ const BUILT_IN_PROPERTIES = new Set([
 // User agent collections with latest versions
 const USER_AGENT_COLLECTIONS = {
   chrome: [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
   ],
   firefox: [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.0) Gecko/20100101 Firefox/143.0",
@@ -535,7 +535,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
                 onMessage: { addListener: () => {}, removeListener: () => {} },
                 sendMessage: () => {},
                 connect: () => ({ onMessage: { addListener: () => {}, removeListener: () => {} }, postMessage: () => {}, disconnect: () => {} }),
-                getManifest: () => ({ name: "Chrome", version: "140.0.0.0" }),
+                getManifest: () => ({ name: "Chrome", version: "141.0.0.0" }),
                 getURL: (path) => `chrome-extension://invalid/${path}`,
                 id: undefined
               },

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.25 ===
+// === Network scanner script (nwss.js) v2.0.26 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -56,7 +56,7 @@ function fastTimeout(ms) {
 }
 
 // --- Configuration Constants ---
-const TIMEOUTS = {
+const TIMEOUTS = Object.freeze({
   DEFAULT_PAGE: 35000,                // Standard page load timeout (35s)
   DEFAULT_NAVIGATION: 25000,          // Navigation operation timeout
   DEFAULT_NAVIGATION_REDUCED: 20000,  // Reduced timeout for faster failures
@@ -71,21 +71,32 @@ const TIMEOUTS = {
   CURL_HANDLER_DELAY: 3000,           // Wait for async curl operations
   PROTOCOL_TIMEOUT: 180000,           // Chrome DevTools Protocol timeout
   REDIRECT_JS_TIMEOUT: 5000           // JavaScript redirect detection timeout
-};
+});
 
-const CACHE_LIMITS = {
+const CACHE_LIMITS = Object.freeze({
   DISK_CACHE_SIZE: 52428800, // 50MB
   MEDIA_CACHE_SIZE: 52428800, // 50MB
   DEFAULT_CACHE_PATH: '.cache',
   DEFAULT_MAX_SIZE: 5000
-};
+});
 
-const CONCURRENCY_LIMITS = {
+const CONCURRENCY_LIMITS = Object.freeze({
   MIN: 1,
   MAX: 50,
   DEFAULT: 6,
   HIGH_CONCURRENCY_THRESHOLD: 12  // Auto-enable aggressive caching above this
-};
+});
+
+// V8 Optimization: Use Map for user agent lookups instead of object
+const USER_AGENTS = Object.freeze(new Map([
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
+]));
 
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
 
@@ -132,7 +143,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.25'; // Script version
+const VERSION = '2.0.26'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1211,6 +1222,9 @@ function setupFrameHandling(page, forceDebug) {
       '/usr/bin/chromium',
       '/snap/bin/chromium'
     ];
+    // V8 Optimization: Freeze the Chrome paths array since it's constant
+    Object.freeze(systemChromePaths);
+
 
     let executablePath = null;
     for (const chromePath of systemChromePaths) {
@@ -1387,12 +1401,28 @@ function setupFrameHandling(page, forceDebug) {
    * @returns {Promise<object>} A promise that resolves to an object containing scan results.
    */
   async function processUrl(currentUrl, siteConfig, browserInstance) {
-    const allowFirstParty = siteConfig.firstParty === true || siteConfig.firstParty === 1;
-    const allowThirdParty = siteConfig.thirdParty === undefined || siteConfig.thirdParty === true || siteConfig.thirdParty === 1;
-    const perSiteSubDomains = siteConfig.subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhostIP = siteConfig.localhost || null;
-    const cloudflarePhishBypass = siteConfig.cloudflare_phish === true;
-    const cloudflareBypass = siteConfig.cloudflare_bypass === true;
+    // V8 Optimization: Single destructuring to avoid multiple property lookups
+    const {
+      firstParty,
+      thirdParty,
+      subDomains,
+      localhost,
+      cloudflare_phish,
+      cloudflare_bypass,
+      flowproxy_detection,
+      privoxy,
+      pihole,
+      even_blocked,
+      comments,
+      bypass_cache
+    } = siteConfig;
+    
+    const allowFirstParty = firstParty === true || firstParty === 1;
+    const allowThirdParty = thirdParty === undefined || thirdParty === true || thirdParty === 1;
+    const perSiteSubDomains = subDomains === 1 ? true : subDomainsMode;
+    const siteLocalhostIP = localhost || null;
+    const cloudflarePhishBypass = cloudflare_phish === true;
+    const cloudflareBypass = cloudflare_bypass === true;
     // Add redirect and same-page loop protection
     const MAX_REDIRECT_DEPTH = siteConfig.max_redirects || 10;
     const redirectHistory = new Set();
@@ -1401,14 +1431,14 @@ function setupFrameHandling(page, forceDebug) {
     const MAX_SAME_PAGE_LOADS = 3;
     let currentPageUrl = currentUrl;
 
-    const sitePrivoxy = siteConfig.privoxy === true;
-    const sitePihole = siteConfig.pihole === true;
-    const flowproxyDetection = siteConfig.flowproxy_detection === true;
+    const sitePrivoxy = privoxy === true;
+    const sitePihole = pihole === true;
+    const flowproxyDetection = flowproxy_detection === true;
     
-    const evenBlocked = siteConfig.even_blocked === true;
+    const evenBlocked = even_blocked === true;
     // Log site-level comments if debug mode is enabled
-    if (forceDebug && siteConfig.comments) {
-      const siteComments = Array.isArray(siteConfig.comments) ? siteConfig.comments : [siteConfig.comments];
+    if (forceDebug && comments) {
+      const siteComments = Array.isArray(comments) ? comments : [comments];
       console.log(formatLogMessage('debug', `Site comments for ${currentUrl}: ${siteComments.length} item(s)`));
       siteComments.forEach((comment, idx) => 
         console.log(formatLogMessage('debug', `  Site comment ${idx + 1}: ${comment}`))
@@ -1416,11 +1446,11 @@ function setupFrameHandling(page, forceDebug) {
     }
 
    // Log bypass_cache setting if enabled
-   if (forceDebug && siteConfig.bypass_cache === true) {
+   if (forceDebug && bypass_cache === true) {
      console.log(formatLogMessage('debug', `Cache bypass enabled for all URLs in site: ${currentUrl}`));
    }
 
-    if (siteConfig.firstParty === 0 && siteConfig.thirdParty === 0) {
+    if (firstParty === 0 && thirdParty === 0) {
       console.warn(`⚠ Skipping ${currentUrl} because both firstParty and thirdParty are disabled.`);
       return { url: currentUrl, rules: [], success: false, skipped: true };
     }
@@ -1475,14 +1505,14 @@ function setupFrameHandling(page, forceDebug) {
     let finalUrlAfterRedirect = null;
 
     // Enhanced error types for Puppeteer 23.x compatibility
-    const CRITICAL_BROWSER_ERRORS = [
+    const CRITICAL_BROWSER_ERRORS = Object.freeze([
       'Protocol error',
       'Target closed',
       'Browser has been closed',
       'Browser protocol broken',
       'Browser process exited',
       'Browser disconnected'
-    ];
+    ]);
 
     try {
 
@@ -1907,29 +1937,30 @@ function setupFrameHandling(page, forceDebug) {
         
         // Client Hints protection for Chrome user agents
         if (siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
+          const userAgentKey = siteConfig.userAgent.toLowerCase();
           let platform = 'Windows';
           let platformVersion = '15.0.0';
           let arch = 'x86';
           
-          if (siteConfig.userAgent.toLowerCase() === 'chrome_mac') {
+          if (userAgentKey === 'chrome_mac') {
             platform = 'macOS';
-            platformVersion = '13.5.0';
+            platformVersion = '13.5.0'; 
             arch = 'arm';
-          } else if (siteConfig.userAgent.toLowerCase() === 'chrome_linux') {
+          } else if (userAgentKey === 'chrome_linux') {
             platform = 'Linux';
             platformVersion = '6.5.0';
             arch = 'x86';
           }
-          
+                    
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Chromium";v="140", "Not=A?Brand";v="24", "Google Chrome";v="140"',
+            'Sec-CH-UA': '"Chromium";v="141", "Not=A?Brand";v="24", "Google Chrome";v="141"',
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"140.0.7339.208"',
-            'Sec-CH-UA-Full-Version-List': '"Chromium";v="140.0.7339.208", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="140.0.7339.208"'
+            'Sec-CH-UA-Full-Version': '"141.0.7390.55"',
+            'Sec-CH-UA-Full-Version-List': '"Chromium";v="141.0.7390.55", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="141.0.7390.55"'
           });
         }
       } catch (fingerprintErr) {
@@ -1959,16 +1990,7 @@ function setupFrameHandling(page, forceDebug) {
    // Get user agent for curl if needed
    let curlUserAgent = '';
    if (useCurl && siteConfig.userAgent) {
-     const userAgents = {
-       chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       chrome_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       chrome_linux: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0",
-       firefox_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0",
-       firefox_linux: "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0",
-       safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"
-     };
-     curlUserAgent = userAgents[siteConfig.userAgent.toLowerCase()] || '';
+     curlUserAgent = USER_AGENTS.get(siteConfig.userAgent.toLowerCase()) || '';
    }
 
    if (useCurl && forceDebug) {
@@ -3487,20 +3509,29 @@ function setupFrameHandling(page, forceDebug) {
 // Temporarily store the pLimit function  
   const originalLimit = limit;
 
-  // Create a flat list of all URL tasks with their site configs for true concurrency
-  const allTasks = [];
+  // V8 Optimization: Calculate total URLs first to pre-allocate array
+  let totalUrls = 0;
   for (const site of sites) {
     const urlsToProcess = Array.isArray(site.url) ? site.url : [site.url];
-    urlsToProcess.forEach(url => {
-      allTasks.push({
+    totalUrls += urlsToProcess.length;
+  }
+  
+  // Pre-allocate array with exact size to prevent multiple reallocations
+  const allTasks = new Array(totalUrls);
+  let taskIndex = 0;
+  
+  // Populate the pre-allocated array
+  for (const site of sites) {
+    const urlsToProcess = Array.isArray(site.url) ? site.url : [site.url];
+    for (const url of urlsToProcess) {
+      allTasks[taskIndex++] = {
         url,
         config: { ...site, _originalUrl: url }, // Preserve original URL for CDP domain checking
-        taskId: allTasks.length // For tracking
-      });
-    });
+        taskId: taskIndex - 1 // For tracking
+      };
+    }
   }
-  
-  const totalUrls = allTasks.length;
+
 
   let results = [];
   let processedUrlCount = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.25",
+  "version": "2.0.26",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {