@fanboynz/network-scanner 2.0.24 → 2.0.26

package/README.md

@@ -223,7 +223,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `whois_timeout_multiplier` | Number | `1.5` | Timeout increase multiplier per retry |
 | `whois_use_fallback` | Boolean | `true` | Add TLD-specific fallback servers |
 | `whois_retry_on_timeout` | Boolean | `true` | Retry on timeout errors |
-| `whois_retry_on_error` | Boolean | `false` | Retry on connection/other errors |
+| `whois_retry_on_error` | Boolean | `true` | Retry on connection/other errors |
 | `dig`                | Array | - | Check dig output for ALL specified terms (AND logic) |
 | `dig-or`             | Array | - | Check dig output for ANY specified term (OR logic) |
 | `dig_subdomain`      | Boolean | `false` | Use subdomain for dig lookup instead of root domain |

package/lib/fingerprint.js

@@ -59,7 +59,7 @@ const PRECOMPILED_MOCKS = Object.freeze({
       postMessage: () => {},
       disconnect: () => {}
     }),
-    getManifest: () => Object.freeze({ name: "Chrome", version: "140.0.0.0" }),
+    getManifest: () => Object.freeze({ name: "Chrome", version: "141.0.0.0" }),
     getURL: (path) => `chrome-extension://invalid/${path}`,
     id: undefined
   }),
@@ -91,9 +91,9 @@ const BUILT_IN_PROPERTIES = new Set([
 // User agent collections with latest versions
 const USER_AGENT_COLLECTIONS = {
   chrome: [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
   ],
   firefox: [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.0) Gecko/20100101 Firefox/143.0",
@@ -535,7 +535,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
                 onMessage: { addListener: () => {}, removeListener: () => {} },
                 sendMessage: () => {},
                 connect: () => ({ onMessage: { addListener: () => {}, removeListener: () => {} }, postMessage: () => {}, disconnect: () => {} }),
-                getManifest: () => ({ name: "Chrome", version: "140.0.0.0" }),
+                getManifest: () => ({ name: "Chrome", version: "141.0.0.0" }),
                 getURL: (path) => `chrome-extension://invalid/${path}`,
                 id: undefined
               },
@@ -1347,6 +1347,19 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
 
   try {
     await page.evaluateOnNewDocument(({ spoof, debugEnabled }) => {
+    
+      // Define helper functions FIRST in this context
+      function spoofNavigatorProperties(navigator, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(navigator, prop, descriptor);
+        }
+      }
+      
+      function spoofScreenProperties(screen, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(screen, prop, descriptor);
+        }
+      }
       
       function safeDefinePropertyLocal(target, property, descriptor) {
         try {

package/lib/searchstring.js

@@ -3,6 +3,7 @@
 
 const fs = require('fs');
 const { spawnSync } = require('child_process');
+const { grepContent } = require('./grep');
 
 // Configuration constants for search logic
 const SEARCH_CONFIG = {
@@ -51,11 +52,12 @@ function parseSearchStrings(searchstring, searchstringAnd) {
  * @param {Function} addMatchedDomain - Optional helper function for adding domains
  * @param {string} domain - Domain to add
  * @param {string} resourceType - Resource type (for --adblock-rules mode)
+ * @param {string} fullSubdomain - Full subdomain for cache tracking (optional)
  */
-function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null) {
+function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null, fullSubdomain = null) {
   // Use helper function if provided (preferred method)
   if (typeof addMatchedDomain === 'function') {
-    addMatchedDomain(domain, resourceType);
+    addMatchedDomain(domain, resourceType, fullSubdomain);
     return;
   }
   
@@ -575,6 +577,7 @@ function createResponseHandler(config) {
     siteConfig,
     dumpUrls,
     matchedUrlsLogFile,
+    useGrep = false,
     forceDebug,
     resourceType // Will be null for response handler
   } = config;
@@ -584,22 +587,16 @@ function createResponseHandler(config) {
     const respDomain = perSiteSubDomains ? (new URL(respUrl)).hostname : getRootDomain(respUrl);
     
     // Only process responses that match our regex patterns
-    const matchesRegex = regexes.some(re => re.test(respUrl));
-    if (!matchesRegex) return;
+    const fullSubdomain = (new URL(respUrl)).hostname; // Always get full subdomain for cache tracking
     
-    // Extract domain and check if already detected (skip expensive operations)
-    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(respDomain)) {
-      if (forceDebug) {
-        console.log(`[debug] Skipping response analysis for already detected domain: ${respDomain}`);
-      }
+    // Skip if already detected to avoid duplicates
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(fullSubdomain)) {
       return;
     }
-
-    // Check if this is a first-party response (same domain as the URL being scanned)
-    const currentUrlHostname = new URL(currentUrl).hostname;
-    const responseHostname = new URL(respUrl).hostname;
-    const isFirstParty = currentUrlHostname === responseHostname;
+    const matchesRegex = regexes.some(re => re.test(respUrl));
+    if (!matchesRegex) return;
     
+    // Extract domain and check if already detected (skip expensive operations)   
     // The main request handler already filtered first-party/third-party requests
     // This response handler only runs for requests that passed that filter
     // However, we need to apply the same first-party/third-party logic here for searchstring analysis
@@ -607,6 +604,10 @@ function createResponseHandler(config) {
     
     // Apply first-party/third-party filtering for searchstring analysis
     // Use the exact same logic as the main request handler
+
+    const currentUrlHostname = new URL(currentUrl).hostname;
+    const responseHostname = new URL(respUrl).hostname;
+    const isFirstParty = currentUrlHostname === responseHostname;
     if (isFirstParty && siteConfig.firstParty === false) {
       if (forceDebug) {
         console.log(`[debug] Skipping first-party response for searchstring analysis (firstParty=false): ${respUrl}`);
@@ -632,9 +633,61 @@ function createResponseHandler(config) {
       }
       
       const content = await response.text();
+
+      // Cache the fetched content if callback provided
+      if (config.onContentFetched) {
+        try {
+          config.onContentFetched(respUrl, content);
+        } catch (cacheErr) {
+          if (forceDebug) {
+            console.log(`[debug] Content caching failed: ${cacheErr.message}`);
+          }
+        }
+      }
       
       // Check if content contains search strings (OR or AND logic)
-      const { found, matchedString, logicType, error } = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      let searchResult;
+      
+      if (useGrep && (searchStrings.length > 0 || searchStringsAnd.length > 0)) {
+        // Use grep for pattern matching
+        try {
+          const allPatterns = [...(searchStrings || []), ...(searchStringsAnd || [])];
+          const grepResult = await grepContent(content, allPatterns, {
+            ignoreCase: true,
+            wholeWord: false,
+            regex: false
+          });
+          
+          if (hasSearchStringAnd && searchStringsAnd.length > 0) {
+            // For AND logic, check that all patterns were found
+            const foundPatterns = grepResult.allMatches.map(match => match.pattern);
+            const allFound = searchStringsAnd.every(pattern => foundPatterns.includes(pattern));
+            searchResult = {
+              found: allFound,
+              matchedString: allFound ? foundPatterns.join(' AND ') : null,
+              logicType: 'AND'
+            };
+          } else {
+            // For OR logic, any match is sufficient
+            searchResult = {
+              found: grepResult.found,
+              matchedString: grepResult.matchedPattern,
+              logicType: 'OR'
+            };
+          }
+        } catch (grepErr) {
+          if (forceDebug) {
+            console.log(`[debug] Grep failed for ${respUrl}, falling back to JavaScript: ${grepErr.message}`);
+          }
+          // Fallback to JavaScript search
+          searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+        }
+      } else {
+        // Use JavaScript search
+        searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      }
+      
+      const { found, matchedString, logicType, error } = searchResult;
       
       if (found) {
         if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
@@ -642,27 +695,31 @@ function createResponseHandler(config) {
         }
         
         // Response handler doesn't have access to specific resource type
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, null);
+        // Use the addMatchedDomain helper which handles fullSubdomain properly
+        addMatchedDomain(respDomain, null, fullSubdomain);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
           const partyType = isFirstParty ? 'first-party' : 'third-party';
-          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}) contains searchstring (${logicType}): "${matchedString}"`);
+          const searchMethod = useGrep ? 'grep' : 'js';
+          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}) contains searchstring (${logicType}): "${matchedString}"`);
         }
         
         if (dumpUrls) {
           const timestamp = new Date().toISOString();
           const partyType = isFirstParty ? 'first-party' : 'third-party';
+          const searchMethod = useGrep ? 'grep' : 'js';
           try {
             fs.appendFileSync(matchedUrlsLogFile, 
-              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, searchstring (${logicType}): "${matchedString}")\n`);
+              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}, searchstring (${logicType}): "${matchedString}")\n`);
           } catch (logErr) {
             console.warn(`[warn] Failed to write to matched URLs log: ${logErr.message}`);
           }
         }
       } else if (forceDebug) {
         const partyType = isFirstParty ? 'first-party' : 'third-party';
-        console.log(`[debug] ${respUrl} (${partyType}) matched regex but no searchstring found`);
+        const searchMethod = useGrep ? 'grep' : 'js';
+        console.log(`[debug] ${respUrl} (${partyType}, ${searchMethod}) matched regex but no searchstring found`);
         if (error) {
           console.log(`[debug] Search error: ${error}`);
         }

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.24 ===
+// === Network scanner script (nwss.js) v2.0.26 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -56,7 +56,7 @@ function fastTimeout(ms) {
 }
 
 // --- Configuration Constants ---
-const TIMEOUTS = {
+const TIMEOUTS = Object.freeze({
   DEFAULT_PAGE: 35000,                // Standard page load timeout (35s)
   DEFAULT_NAVIGATION: 25000,          // Navigation operation timeout
   DEFAULT_NAVIGATION_REDUCED: 20000,  // Reduced timeout for faster failures
@@ -71,21 +71,32 @@ const TIMEOUTS = {
   CURL_HANDLER_DELAY: 3000,           // Wait for async curl operations
   PROTOCOL_TIMEOUT: 180000,           // Chrome DevTools Protocol timeout
   REDIRECT_JS_TIMEOUT: 5000           // JavaScript redirect detection timeout
-};
+});
 
-const CACHE_LIMITS = {
+const CACHE_LIMITS = Object.freeze({
   DISK_CACHE_SIZE: 52428800, // 50MB
   MEDIA_CACHE_SIZE: 52428800, // 50MB
   DEFAULT_CACHE_PATH: '.cache',
   DEFAULT_MAX_SIZE: 5000
-};
+});
 
-const CONCURRENCY_LIMITS = {
+const CONCURRENCY_LIMITS = Object.freeze({
   MIN: 1,
   MAX: 50,
   DEFAULT: 6,
   HIGH_CONCURRENCY_THRESHOLD: 12  // Auto-enable aggressive caching above this
-};
+});
+
+// V8 Optimization: Use Map for user agent lookups instead of object
+const USER_AGENTS = Object.freeze(new Map([
+  ['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"],
+  ['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0"],
+  ['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
+]));
 
 const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
 
@@ -132,7 +143,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.24'; // Script version
+const VERSION = '2.0.26'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -997,7 +1008,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Map(); // Use Map to track frame state
+  let activeFrames = new Set(); // Use Set to track frame references
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1031,7 +1042,6 @@ function setupFrameHandling(page, forceDebug) {
       // Enhanced frame validation with multiple safety checks
       let frameUrl;
       try {
-        // Test frame accessibility first
         frameUrl = frame.url();
         
         // Check if frame is detached (if method exists)
@@ -1041,12 +1051,17 @@ function setupFrameHandling(page, forceDebug) {
           }
           return;
         }
+        
+        activeFrames.add(frame);
+        
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
+        }
       } catch (frameAccessError) {
         // Frame is not accessible (likely detached)
         return;
       }
-      
-      activeFrames.add(frame);
+
     } catch (detachError) {
       // Frame state checking can throw in 23.x, handle gracefully
       if (forceDebug) {
@@ -1055,14 +1070,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    // Store frame with timestamp for tracking
-    activeFrames.set(frame, Date.now());
     
     if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
-      try {       
-        if (forceDebug) {
-          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
-        }
+      let frameUrl;
+        frameUrl = frame.url();
         
         // Don't try to navigate to frames with invalid/empty URLs
         if (!frameUrl ||
@@ -1100,6 +1111,7 @@ function setupFrameHandling(page, forceDebug) {
         // Let frames load naturally - manual navigation often causes Protocol errors
         // await frame.goto(frame.url(), { waitUntil: 'domcontentloaded', timeout: 5000 });
         
+        try {        
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Frame will load naturally: ${frameUrl}`));
         }
@@ -1117,11 +1129,11 @@ function setupFrameHandling(page, forceDebug) {
   });
   // Handle frame navigations (keep this for monitoring)
   page.on('framenavigated', (frame) => {
-    let frameUrl;
 
     // Skip if frame is not in our active set
     if (!activeFrames.has(frame)) return;
 
+    let frameUrl;
     try {
       frameUrl = frame.url();
     } catch (urlErr) {
@@ -1143,17 +1155,14 @@ function setupFrameHandling(page, forceDebug) {
   // Optional: Handle frame detachment for cleanup
   page.on('framedetached', (frame) => {
     // Remove from active tracking
-    activeFrames.delete(frame);
+    activeFrames.delete(frame); // This works for both Map and Set
     
-    // Skip logging if we can't access frame URL
-    let frameUrl;
+
     if (forceDebug) {
+      let frameUrl;
       try {
         frameUrl = frame.url();
-      } catch (urlErr) {
-        // Frame already detached, can't get URL
-        return;
-      }
+
       if (frameUrl &&
           frameUrl !== 'about:blank' &&
           frameUrl !== 'about:srcdoc' &&
@@ -1162,6 +1171,11 @@ function setupFrameHandling(page, forceDebug) {
           !frameUrl.startsWith('chrome-extension://')) {
         console.log(formatLogMessage('debug', `Frame detached: ${frameUrl}`));
       }
+      } catch (urlErr) {
+        // Frame already detached, can't get URL - this is expected
+        return;
+      }
+
     }
   });
 }
@@ -1208,6 +1222,9 @@ function setupFrameHandling(page, forceDebug) {
       '/usr/bin/chromium',
       '/snap/bin/chromium'
     ];
+    // V8 Optimization: Freeze the Chrome paths array since it's constant
+    Object.freeze(systemChromePaths);
+
 
     let executablePath = null;
     for (const chromePath of systemChromePaths) {
@@ -1384,12 +1401,28 @@ function setupFrameHandling(page, forceDebug) {
    * @returns {Promise<object>} A promise that resolves to an object containing scan results.
    */
   async function processUrl(currentUrl, siteConfig, browserInstance) {
-    const allowFirstParty = siteConfig.firstParty === true || siteConfig.firstParty === 1;
-    const allowThirdParty = siteConfig.thirdParty === undefined || siteConfig.thirdParty === true || siteConfig.thirdParty === 1;
-    const perSiteSubDomains = siteConfig.subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhostIP = siteConfig.localhost || null;
-    const cloudflarePhishBypass = siteConfig.cloudflare_phish === true;
-    const cloudflareBypass = siteConfig.cloudflare_bypass === true;
+    // V8 Optimization: Single destructuring to avoid multiple property lookups
+    const {
+      firstParty,
+      thirdParty,
+      subDomains,
+      localhost,
+      cloudflare_phish,
+      cloudflare_bypass,
+      flowproxy_detection,
+      privoxy,
+      pihole,
+      even_blocked,
+      comments,
+      bypass_cache
+    } = siteConfig;
+    
+    const allowFirstParty = firstParty === true || firstParty === 1;
+    const allowThirdParty = thirdParty === undefined || thirdParty === true || thirdParty === 1;
+    const perSiteSubDomains = subDomains === 1 ? true : subDomainsMode;
+    const siteLocalhostIP = localhost || null;
+    const cloudflarePhishBypass = cloudflare_phish === true;
+    const cloudflareBypass = cloudflare_bypass === true;
     // Add redirect and same-page loop protection
     const MAX_REDIRECT_DEPTH = siteConfig.max_redirects || 10;
     const redirectHistory = new Set();
@@ -1398,14 +1431,14 @@ function setupFrameHandling(page, forceDebug) {
     const MAX_SAME_PAGE_LOADS = 3;
     let currentPageUrl = currentUrl;
 
-    const sitePrivoxy = siteConfig.privoxy === true;
-    const sitePihole = siteConfig.pihole === true;
-    const flowproxyDetection = siteConfig.flowproxy_detection === true;
+    const sitePrivoxy = privoxy === true;
+    const sitePihole = pihole === true;
+    const flowproxyDetection = flowproxy_detection === true;
     
-    const evenBlocked = siteConfig.even_blocked === true;
+    const evenBlocked = even_blocked === true;
     // Log site-level comments if debug mode is enabled
-    if (forceDebug && siteConfig.comments) {
-      const siteComments = Array.isArray(siteConfig.comments) ? siteConfig.comments : [siteConfig.comments];
+    if (forceDebug && comments) {
+      const siteComments = Array.isArray(comments) ? comments : [comments];
       console.log(formatLogMessage('debug', `Site comments for ${currentUrl}: ${siteComments.length} item(s)`));
       siteComments.forEach((comment, idx) => 
         console.log(formatLogMessage('debug', `  Site comment ${idx + 1}: ${comment}`))
@@ -1413,11 +1446,11 @@ function setupFrameHandling(page, forceDebug) {
     }
 
    // Log bypass_cache setting if enabled
-   if (forceDebug && siteConfig.bypass_cache === true) {
+   if (forceDebug && bypass_cache === true) {
      console.log(formatLogMessage('debug', `Cache bypass enabled for all URLs in site: ${currentUrl}`));
    }
 
-    if (siteConfig.firstParty === 0 && siteConfig.thirdParty === 0) {
+    if (firstParty === 0 && thirdParty === 0) {
       console.warn(`⚠ Skipping ${currentUrl} because both firstParty and thirdParty are disabled.`);
       return { url: currentUrl, rules: [], success: false, skipped: true };
     }
@@ -1472,14 +1505,14 @@ function setupFrameHandling(page, forceDebug) {
     let finalUrlAfterRedirect = null;
 
     // Enhanced error types for Puppeteer 23.x compatibility
-    const CRITICAL_BROWSER_ERRORS = [
+    const CRITICAL_BROWSER_ERRORS = Object.freeze([
       'Protocol error',
       'Target closed',
       'Browser has been closed',
       'Browser protocol broken',
       'Browser process exited',
       'Browser disconnected'
-    ];
+    ]);
 
     try {
 
@@ -1904,29 +1937,30 @@ function setupFrameHandling(page, forceDebug) {
         
         // Client Hints protection for Chrome user agents
         if (siteConfig.userAgent && siteConfig.userAgent.toLowerCase().includes('chrome')) {
+          const userAgentKey = siteConfig.userAgent.toLowerCase();
           let platform = 'Windows';
           let platformVersion = '15.0.0';
           let arch = 'x86';
           
-          if (siteConfig.userAgent.toLowerCase() === 'chrome_mac') {
+          if (userAgentKey === 'chrome_mac') {
             platform = 'macOS';
-            platformVersion = '13.5.0';
+            platformVersion = '13.5.0'; 
             arch = 'arm';
-          } else if (siteConfig.userAgent.toLowerCase() === 'chrome_linux') {
+          } else if (userAgentKey === 'chrome_linux') {
             platform = 'Linux';
             platformVersion = '6.5.0';
             arch = 'x86';
           }
-          
+                    
           await page.setExtraHTTPHeaders({
-            'Sec-CH-UA': '"Chromium";v="140", "Not=A?Brand";v="24", "Google Chrome";v="140"',
+            'Sec-CH-UA': '"Chromium";v="141", "Not=A?Brand";v="24", "Google Chrome";v="141"',
             'Sec-CH-UA-Platform': `"${platform}"`,
             'Sec-CH-UA-Platform-Version': `"${platformVersion}"`,
             'Sec-CH-UA-Mobile': '?0',
             'Sec-CH-UA-Arch': `"${arch}"`,
             'Sec-CH-UA-Bitness': '"64"',
-            'Sec-CH-UA-Full-Version': '"140.0.7339.208"',
-            'Sec-CH-UA-Full-Version-List': '"Chromium";v="140.0.7339.208", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="140.0.7339.208"'
+            'Sec-CH-UA-Full-Version': '"141.0.7390.55"',
+            'Sec-CH-UA-Full-Version-List': '"Chromium";v="141.0.7390.55", "Not=A?Brand";v="24.0.0.0", "Google Chrome";v="141.0.7390.55"'
           });
         }
       } catch (fingerprintErr) {
@@ -1951,21 +1985,12 @@ function setupFrameHandling(page, forceDebug) {
    // Parse searchstring patterns using module
    const { searchStrings, searchStringsAnd, hasSearchString, hasSearchStringAnd } = parseSearchStrings(siteConfig.searchstring, siteConfig.searchstring_and);
    const useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring
-   let useGrep = siteConfig.grep === true && useCurl; // Grep requires curl to be enabled
+   let useGrep = siteConfig.grep === true; // Grep can work independently
 
    // Get user agent for curl if needed
    let curlUserAgent = '';
    if (useCurl && siteConfig.userAgent) {
-     const userAgents = {
-       chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       chrome_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       chrome_linux: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-       firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/143.0",
-       firefox_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/143.0",
-       firefox_linux: "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/143.0",
-       safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"
-     };
-     curlUserAgent = userAgents[siteConfig.userAgent.toLowerCase()] || '';
+     curlUserAgent = USER_AGENTS.get(siteConfig.userAgent.toLowerCase()) || '';
    }
 
    if (useCurl && forceDebug) {
@@ -1973,7 +1998,7 @@ function setupFrameHandling(page, forceDebug) {
    }
 
    if (useGrep && forceDebug) {
-     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}`));
+     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}${useCurl ? ' (with curl)' : ' (with response handler)'}`));
    }
    
    // Validate grep availability if needed
@@ -1993,7 +2018,6 @@ function setupFrameHandling(page, forceDebug) {
      if (!curlCheck.isAvailable) {
        console.warn(formatLogMessage('warn', `Curl not available for ${currentUrl}: ${curlCheck.error}. Skipping curl-based analysis.`));
        useCurl = false;
-       useGrep = false; // Grep requires curl
      } else if (forceDebug) {
        console.log(formatLogMessage('debug', `Using curl: ${curlCheck.version}`));
      }
@@ -2643,7 +2667,7 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check bypass_cache before attempting cache lookup
+             // Check bypass_cache before attempting cache lookup (curl mode)
              let cachedContent = null;
              if (!shouldBypassCacheForUrl(reqUrl, siteConfig)) {
                // Check request cache first if smart cache is available and caching is enabled
@@ -2732,8 +2756,30 @@ function setupFrameHandling(page, forceDebug) {
                }
              }
              }
+            } else if (useGrep && (hasSearchString || hasSearchStringAnd)) {
+             // Use grep with response handler (no curl)
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `[grep-response] Queuing ${reqUrl} for grep analysis via response handler`));
+             }
+             
+             // Queue for grep processing via response handler
+             // The response handler will download content and call grep
+             if (dryRunMode) {
+               matchedDomains.get('dryRunMatches').push({
+                 regex: matchedRegexPattern,
+                 domain: reqDomain,
+                 resourceType: resourceType,
+                 fullUrl: reqUrl,
+                 isFirstParty: isFirstParty,
+                 needsGrepCheck: true
+               });
+             }
+             
+             // Don't process immediately - let response handler do the work
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `URL ${reqUrl} queued for grep analysis via response handler`));
+             }
            }
-
           // No break needed since we've already determined if regex matched
         }
         request.continue();
@@ -2742,8 +2788,8 @@ function setupFrameHandling(page, forceDebug) {
       // Mark page as actively processing network requests
       updatePageUsage(page, true);
 
-     // Add response handler ONLY if searchstring/searchstring_and is defined AND neither curl nor grep is enabled
-     if ((hasSearchString || hasSearchStringAnd) && !useCurl && !useGrep) {
+     // Add response handler if searchstring is defined and either no curl, or grep without curl
+     if ((hasSearchString || hasSearchStringAnd) && (!useCurl || (useGrep && !useCurl))) {
        const responseHandler = createResponseHandler({
          searchStrings,
          searchStringsAnd,
@@ -2761,6 +2807,7 @@ function setupFrameHandling(page, forceDebug) {
          } : undefined,
          currentUrl,
          perSiteSubDomains,
+         useGrep, // Pass grep flag to response handler
          ignoreDomains,
          matchesIgnoreDomain,
          getRootDomain,
@@ -3462,20 +3509,29 @@ function setupFrameHandling(page, forceDebug) {
 // Temporarily store the pLimit function  
   const originalLimit = limit;
 
-  // Create a flat list of all URL tasks with their site configs for true concurrency
-  const allTasks = [];
+  // V8 Optimization: Calculate total URLs first to pre-allocate array
+  let totalUrls = 0;
   for (const site of sites) {
     const urlsToProcess = Array.isArray(site.url) ? site.url : [site.url];
-    urlsToProcess.forEach(url => {
-      allTasks.push({
+    totalUrls += urlsToProcess.length;
+  }
+  
+  // Pre-allocate array with exact size to prevent multiple reallocations
+  const allTasks = new Array(totalUrls);
+  let taskIndex = 0;
+  
+  // Populate the pre-allocated array
+  for (const site of sites) {
+    const urlsToProcess = Array.isArray(site.url) ? site.url : [site.url];
+    for (const url of urlsToProcess) {
+      allTasks[taskIndex++] = {
         url,
         config: { ...site, _originalUrl: url }, // Preserve original URL for CDP domain checking
-        taskId: allTasks.length // For tracking
-      });
-    });
+        taskId: taskIndex - 1 // For tracking
+      };
+    }
   }
-  
-  const totalUrls = allTasks.length;
+
 
   let results = [];
   let processedUrlCount = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.24",
+  "version": "2.0.26",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {