@fanboynz/network-scanner 2.0.23 → 2.0.25

package/lib/fingerprint.js

@@ -7,6 +7,81 @@
 const DEFAULT_PLATFORM = 'Win32';
 const DEFAULT_TIMEZONE = 'America/New_York';
 
+// Cached property descriptors for V8 optimization
+const CACHED_DESCRIPTORS = {
+  readOnlyValue: (value) => ({ value, writable: false, enumerable: true, configurable: true }),
+  getter: (fn) => ({ get: fn, enumerable: true, configurable: true }),
+  hiddenValue: (value) => ({ value, writable: false, enumerable: false, configurable: true })
+};
+
+// Type-specific property spoofing functions for monomorphic optimization
+function spoofNavigatorProperties(navigator, properties, options = {}) {
+  if (!navigator || typeof navigator !== 'object') return false;
+  
+  for (const [prop, descriptor] of Object.entries(properties)) {
+    if (!safeDefineProperty(navigator, prop, descriptor, options)) {
+      if (options.debug) console.log(`[fingerprint] Failed to spoof navigator.${prop}`);
+    }
+  }
+  return true;
+}
+
+function spoofScreenProperties(screen, properties, options = {}) {
+  if (!screen || typeof screen !== 'object') return false;
+  
+  for (const [prop, descriptor] of Object.entries(properties)) {
+    if (!safeDefineProperty(screen, prop, descriptor, options)) {
+      if (options.debug) console.log(`[fingerprint] Failed to spoof screen.${prop}`);
+    }
+  }
+  return true;
+}
+
+function spoofWindowProperties(window, properties, options = {}) {
+  if (!window || typeof window !== 'object') return false;
+  
+  for (const [prop, descriptor] of Object.entries(properties)) {
+    if (!safeDefineProperty(window, prop, descriptor, options)) {
+      if (options.debug) console.log(`[fingerprint] Failed to spoof window.${prop}`);
+    }
+  }
+  return true;
+}
+
+// Pre-compiled mock objects for V8 optimization
+const PRECOMPILED_MOCKS = Object.freeze({
+  chromeRuntime: Object.freeze({
+    onConnect: Object.freeze({ addListener: () => {}, removeListener: () => {} }),
+    onMessage: Object.freeze({ addListener: () => {}, removeListener: () => {} }),
+    sendMessage: () => {},
+    connect: () => Object.freeze({
+      onMessage: Object.freeze({ addListener: () => {}, removeListener: () => {} }),
+      postMessage: () => {},
+      disconnect: () => {}
+    }),
+    getManifest: () => Object.freeze({ name: "Chrome", version: "140.0.0.0" }),
+    getURL: (path) => `chrome-extension://invalid/${path}`,
+    id: undefined
+  }),
+  
+  fingerprintResult: Object.freeze({
+    visitorId: 'mock_visitor_' + Math.random().toString(36).substr(2, 9),
+    confidence: Object.freeze({ score: 0.99 }),
+    components: Object.freeze({
+      screen: Object.freeze({ value: Object.freeze({ width: 1920, height: 1080 }) }),
+      timezone: Object.freeze({ value: 'America/New_York' }),
+      language: Object.freeze({ value: 'en-US' })
+    })
+  }),
+  
+  loadTimes: Object.freeze({
+    commitLoadTime: performance.now() - Math.random() * 1000,
+    connectionInfo: 'http/1.1',
+    finishDocumentLoadTime: performance.now() - Math.random() * 500,
+    navigationType: 'Navigation'
+  })
+});
+
 // Built-in properties that should not be modified
 const BUILT_IN_PROPERTIES = new Set([
   'href', 'origin', 'protocol', 'host', 'hostname', 'port', 'pathname', 'search', 'hash',
@@ -55,10 +130,7 @@ function safeDefineProperty(target, property, descriptor, options = {}) {
       return false;
     }
 
-    Object.defineProperty(target, property, {
-      ...descriptor,
-      configurable: true
-    });
+    Object.defineProperty(target, property, descriptor);
     return true;
   } catch (err) {
     if (options.debug) console.log(`[fingerprint] Failed to define ${property}: ${err.message}`);
@@ -99,11 +171,61 @@ function getRealisticScreenResolution() {
 /**
  * Generates randomized but realistic browser fingerprint values
  */
-function getRandomFingerprint() {
-  const resolution = getRealisticScreenResolution();
+function generateRealisticFingerprint(userAgent) {
+  // Determine OS from user agent
+  let osType = 'windows';
+  if (userAgent.includes('Macintosh') || userAgent.includes('Mac OS X')) {
+    osType = 'mac';
+  } else if (userAgent.includes('X11; Linux') || userAgent.includes('Ubuntu')) {
+    osType = 'linux';
+  }
+  
+  // Generate OS-appropriate hardware specs
+  const profiles = {
+    windows: {
+      deviceMemory: [8, 16], // Common Windows configurations
+      hardwareConcurrency: [4, 6, 8], // Typical consumer CPUs
+      platform: 'Win32',
+      timezone: 'America/New_York',
+      language: 'en-US',
+      resolutions: [
+        { width: 1920, height: 1080 },
+        { width: 2560, height: 1440 },
+        { width: 1366, height: 768 }
+      ]
+    },
+    mac: {
+      deviceMemory: [8, 16], // MacBook/iMac typical configs
+      hardwareConcurrency: [8, 10], // Apple Silicon M1/M2 cores
+      platform: 'MacIntel',
+      timezone: 'America/Los_Angeles',
+      language: 'en-US',
+      resolutions: [
+        { width: 2560, height: 1600 }, // MacBook Pro
+        { width: 3840, height: 2160 }, // iMac 4K
+        { width: 1440, height: 900 }   // MacBook Air
+      ]
+    },
+    linux: {
+      deviceMemory: [8, 16],
+      hardwareConcurrency: [4, 8, 12], // Wide variety on Linux
+      platform: 'Linux x86_64',
+      timezone: 'America/New_York',
+      language: 'en-US',
+      resolutions: [
+        { width: 1920, height: 1080 },
+        { width: 2560, height: 1440 },
+        { width: 1600, height: 900 }
+      ]
+    }
+  };
+  
+  const profile = profiles[osType];
+  const resolution = profile.resolutions[Math.floor(Math.random() * profile.resolutions.length)];
+  
   return {
-    deviceMemory: [4, 8, 16, 32][Math.floor(Math.random() * 4)],
-    hardwareConcurrency: [2, 4, 6, 8, 12, 16][Math.floor(Math.random() * 6)],
+    deviceMemory: profile.deviceMemory[Math.floor(Math.random() * profile.deviceMemory.length)],
+    hardwareConcurrency: profile.hardwareConcurrency[Math.floor(Math.random() * profile.hardwareConcurrency.length)],
     screen: {
       width: resolution.width,
       height: resolution.height,
@@ -112,11 +234,11 @@ function getRandomFingerprint() {
       colorDepth: 24,
       pixelDepth: 24
     },
-    platform: Math.random() > 0.3 ? 'Win32' : 'MacIntel',
-    timezone: ['America/New_York', 'America/Los_Angeles', 'Europe/London', 'America/Chicago'][Math.floor(Math.random() * 4)],
-    language: ['en-US', 'en-GB', 'en-CA'][Math.floor(Math.random() * 3)],
+    platform: profile.platform,
+    timezone: profile.timezone,
+    language: profile.language,
     cookieEnabled: true,
-    doNotTrack: Math.random() > 0.7 ? '1' : null
+    doNotTrack: null // Most users don't enable DNT
   };
 }
 
@@ -124,19 +246,7 @@ function getRandomFingerprint() {
  * Creates mock Chrome runtime objects
  */
 function createMockChromeRuntime() {
-  return {
-    onConnect: { addListener: () => {}, removeListener: () => {} },
-    onMessage: { addListener: () => {}, removeListener: () => {} },
-    sendMessage: () => {},
-    connect: () => ({
-      onMessage: { addListener: () => {}, removeListener: () => {} },
-      postMessage: () => {},
-      disconnect: () => {}
-    }),
-    getManifest: () => ({ name: "Chrome", version: "131.0.0.0" }),
-    getURL: (path) => `chrome-extension://invalid/${path}`,
-    id: undefined
-  };
+  return PRECOMPILED_MOCKS.chromeRuntime;
 }
 
 /**
@@ -144,21 +254,8 @@ function createMockChromeRuntime() {
  */
 function generateRealisticLoadTimes() {
   const now = performance.now();
-  return {
-    commitLoadTime: now - Math.random() * 1000,
-    connectionInfo: 'http/1.1',
-    finishDocumentLoadTime: now - Math.random() * 500,
-    finishLoadTime: now - Math.random() * 100,
-    firstPaintAfterLoadTime: now - Math.random() * 50,
-    firstPaintTime: now - Math.random() * 200,
-    navigationType: 'Navigation',
-    npnNegotiatedProtocol: 'unknown',
-    requestTime: now - Math.random() * 2000,
-    startLoadTime: now - Math.random() * 1500,
-    wasAlternateProtocolAvailable: false,
-    wasFetchedViaSpdy: false,
-    wasNpnNegotiated: false
-  };
+  // Return a copy with updated timing values
+  return { ...PRECOMPILED_MOCKS.loadTimes, commitLoadTime: now - Math.random() * 1000 };
 }
 
 /**
@@ -194,22 +291,14 @@ async function validatePageForInjection(page, currentUrl, forceDebug) {
  * Creates mock fingerprinting objects
  */
 function createFingerprintMocks() {
-  const mockResult = {
-    visitorId: 'mock_visitor_' + Math.random().toString(36).substr(2, 9),
-    confidence: { score: 0.99 },
-    components: {
-      screen: { value: { width: 1920, height: 1080 } },
-      timezone: { value: DEFAULT_TIMEZONE },
-      language: { value: 'en-US' }
-    }
-  };
+  const mockResult = PRECOMPILED_MOCKS.fingerprintResult;
   
   return {
     fp: {
       getResult: (callback) => callback ? setTimeout(() => callback(mockResult), 0) : mockResult,
       get: (callback) => Promise.resolve(mockResult),
       load: () => Promise.resolve(window.fp),
-      components: { screen: { value: { width: 1920, height: 1080 } } },
+      components: mockResult.components,
       x64hash128: () => 'mock_hash',
       tz: DEFAULT_TIMEZONE,
       timezone: DEFAULT_TIMEZONE
@@ -351,7 +440,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         // Create safe property definition helper
         function safeDefinePropertyLocal(target, property, descriptor) {
           const builtInProps = new Set(['href', 'origin', 'protocol', 'host', 'hostname', 'port', 'pathname', 'search', 'hash', 'constructor', 'prototype', '__proto__', 'toString', 'valueOf', 'assign', 'reload', 'replace']);
-          
+       
           if (builtInProps.has(property)) {
             if (debugEnabled) console.log(`[fingerprint] Skipping built-in property: ${property}`);
             return false;
@@ -374,6 +463,24 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
             return false;
           }
         }
+
+        // Add cached descriptors helper for page context
+        const CACHED_DESCRIPTORS = {
+          getter: (fn) => ({ get: fn, enumerable: true, configurable: true })
+        };
+        
+        // Add monomorphic spoofing functions for page context
+        function spoofNavigatorProperties(navigator, properties) {
+          for (const [prop, descriptor] of Object.entries(properties)) {
+            safeDefinePropertyLocal(navigator, prop, descriptor);
+          }
+        }
+        
+        function spoofScreenProperties(screen, properties) {
+          for (const [prop, descriptor] of Object.entries(properties)) {
+            safeDefinePropertyLocal(screen, prop, descriptor);
+          }
+        }
         
         // Safe execution wrapper
         function safeExecute(fn, description) {
@@ -428,7 +535,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
                 onMessage: { addListener: () => {}, removeListener: () => {} },
                 sendMessage: () => {},
                 connect: () => ({ onMessage: { addListener: () => {}, removeListener: () => {} }, postMessage: () => {}, disconnect: () => {} }),
-                getManifest: () => ({ name: "Chrome", version: "131.0.0.0" }),
+                getManifest: () => ({ name: "Chrome", version: "140.0.0.0" }),
                 getURL: (path) => `chrome-extension://invalid/${path}`,
                 id: undefined
               },
@@ -507,8 +614,11 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         //
         safeExecute(() => {
           const languages = ['en-US', 'en'];
-          safeDefinePropertyLocal(navigator, 'languages', { get: () => languages });
-          safeDefinePropertyLocal(navigator, 'language', { get: () => languages[0] });
+          const languageProps = {
+            languages: { get: () => languages },
+            language: { get: () => languages[0] }
+          };
+          spoofNavigatorProperties(navigator, languageProps);
         }, 'language spoofing');
 
         // Spoof vendor information
@@ -523,8 +633,11 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
             vendor = 'Apple Computer, Inc.';
           }
           
-          safeDefinePropertyLocal(navigator, 'vendor', { get: () => vendor });
-          safeDefinePropertyLocal(navigator, 'product', { get: () => product });
+          const vendorProps = {
+            vendor: { get: () => vendor },
+            product: { get: () => product }
+          };
+          spoofNavigatorProperties(navigator, vendorProps);
         }, 'vendor/product spoofing');
 
         // Enhanced OS fingerprinting protection based on actual user agent content
@@ -574,7 +687,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
         // Hardware concurrency spoofing (universal coverage)
         //
         safeExecute(() => {
-          safeDefinePropertyLocal(navigator, 'hardwareConcurrency', { get: () => [4, 6, 8, 12][Math.floor(Math.random() * 4)] });
+          const hardwareProps = {
+            hardwareConcurrency: { get: () => [4, 6, 8, 12][Math.floor(Math.random() * 4)] }
+          };
+          spoofNavigatorProperties(navigator, hardwareProps);
         }, 'hardware concurrency spoofing');
 
 
@@ -593,10 +709,13 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
           ];
           const resolution = commonResolutions[Math.floor(Math.random() * commonResolutions.length)];
           
-          safeDefinePropertyLocal(window.screen, 'width', { get: () => resolution.width });
-          safeDefinePropertyLocal(window.screen, 'height', { get: () => resolution.height });
-          safeDefinePropertyLocal(window.screen, 'availWidth', { get: () => resolution.width });
-          safeDefinePropertyLocal(window.screen, 'availHeight', { get: () => resolution.height - 40 });
+          const screenProps = {
+            width: { get: () => resolution.width },
+            height: { get: () => resolution.height },
+            availWidth: { get: () => resolution.width },
+            availHeight: { get: () => resolution.height - 40 }
+          };
+          spoofScreenProperties(window.screen, screenProps);
         }, 'screen resolution protection');
 
         // Spoof MIME types
@@ -1213,7 +1332,9 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
   // Validate page state before injection
   if (!(await validatePageForInjection(page, currentUrl, forceDebug))) return;
   
-  const spoof = fingerprintSetting === 'random' ? getRandomFingerprint() : {
+ const currentUserAgent = await page.evaluate(() => navigator.userAgent);
+  
+ const spoof = fingerprintSetting === 'random' ? generateRealisticFingerprint(currentUserAgent) : {
     deviceMemory: 8, 
     hardwareConcurrency: 4,
     screen: { width: 1920, height: 1080, availWidth: 1920, availHeight: 1040, colorDepth: 24, pixelDepth: 24 },
@@ -1226,6 +1347,19 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
 
   try {
     await page.evaluateOnNewDocument(({ spoof, debugEnabled }) => {
+    
+      // Define helper functions FIRST in this context
+      function spoofNavigatorProperties(navigator, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(navigator, prop, descriptor);
+        }
+      }
+      
+      function spoofScreenProperties(screen, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(screen, prop, descriptor);
+        }
+      }
       
       function safeDefinePropertyLocal(target, property, descriptor) {
         try {
@@ -1244,11 +1378,15 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
       }
 
       // Platform spoofing
-      safeDefinePropertyLocal(navigator, 'platform', { get: () => spoof.platform });
+      const navigatorProps = {
+        platform: { get: () => spoof.platform },
+        deviceMemory: { get: () => spoof.deviceMemory },
+        hardwareConcurrency: { get: () => spoof.hardwareConcurrency }
+      };
+      spoofNavigatorProperties(navigator, navigatorProps);
       
-      // Memory spoofing
-      safeDefinePropertyLocal(navigator, 'deviceMemory', { get: () => spoof.deviceMemory });
-      safeDefinePropertyLocal(navigator, 'hardwareConcurrency', { get: () => spoof.hardwareConcurrency });
+      // Platform, memory, and hardware spoofing combined for better V8 optimization
+      // (moved into navigatorProps above);
 
       // Connection type spoofing
       safeDefinePropertyLocal(navigator, 'connection', {
@@ -1262,16 +1400,21 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
       });
       
       // Screen properties spoofing
+      const screenSpoofProps = {};
       ['width', 'height', 'availWidth', 'availHeight', 'colorDepth', 'pixelDepth'].forEach(prop => {
         if (spoof.screen[prop] !== undefined) {
-          safeDefinePropertyLocal(window.screen, prop, { get: () => spoof.screen[prop] });
+          screenSpoofProps[prop] = { get: () => spoof.screen[prop] };
         }
       });
+      spoofScreenProperties(window.screen, screenSpoofProps);
       
       // Language spoofing
       const languages = Array.isArray(spoof.language) ? spoof.language : [spoof.language, spoof.language.split('-')[0]];
-      safeDefinePropertyLocal(navigator, 'languages', { get: () => languages });
-      safeDefinePropertyLocal(navigator, 'language', { get: () => languages[0] });
+      const languageSpoofProps = {
+        languages: { get: () => languages },
+        language: { get: () => languages[0] }
+      };
+      spoofNavigatorProperties(navigator, languageSpoofProps);
 
       // Timezone spoofing
       if (spoof.timezone && window.Intl?.DateTimeFormat) {
@@ -1480,7 +1623,7 @@ function safeExecuteSpoofing(spoofFunction, description, forceDebug = false) {
 
 
 module.exports = {
-  getRandomFingerprint,
+  generateRealisticFingerprint,
   getRealisticScreenResolution,
   applyUserAgentSpoofing,
   applyBraveSpoofing,

package/lib/searchstring.js

@@ -3,6 +3,7 @@
 
 const fs = require('fs');
 const { spawnSync } = require('child_process');
+const { grepContent } = require('./grep');
 
 // Configuration constants for search logic
 const SEARCH_CONFIG = {
@@ -51,11 +52,12 @@ function parseSearchStrings(searchstring, searchstringAnd) {
  * @param {Function} addMatchedDomain - Optional helper function for adding domains
  * @param {string} domain - Domain to add
  * @param {string} resourceType - Resource type (for --adblock-rules mode)
+ * @param {string} fullSubdomain - Full subdomain for cache tracking (optional)
  */
-function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null) {
+function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null, fullSubdomain = null) {
   // Use helper function if provided (preferred method)
   if (typeof addMatchedDomain === 'function') {
-    addMatchedDomain(domain, resourceType);
+    addMatchedDomain(domain, resourceType, fullSubdomain);
     return;
   }
   
@@ -575,6 +577,7 @@ function createResponseHandler(config) {
     siteConfig,
     dumpUrls,
     matchedUrlsLogFile,
+    useGrep = false,
     forceDebug,
     resourceType // Will be null for response handler
   } = config;
@@ -584,22 +587,16 @@ function createResponseHandler(config) {
     const respDomain = perSiteSubDomains ? (new URL(respUrl)).hostname : getRootDomain(respUrl);
     
     // Only process responses that match our regex patterns
-    const matchesRegex = regexes.some(re => re.test(respUrl));
-    if (!matchesRegex) return;
+    const fullSubdomain = (new URL(respUrl)).hostname; // Always get full subdomain for cache tracking
     
-    // Extract domain and check if already detected (skip expensive operations)
-    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(respDomain)) {
-      if (forceDebug) {
-        console.log(`[debug] Skipping response analysis for already detected domain: ${respDomain}`);
-      }
+    // Skip if already detected to avoid duplicates
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(fullSubdomain)) {
       return;
     }
-
-    // Check if this is a first-party response (same domain as the URL being scanned)
-    const currentUrlHostname = new URL(currentUrl).hostname;
-    const responseHostname = new URL(respUrl).hostname;
-    const isFirstParty = currentUrlHostname === responseHostname;
+    const matchesRegex = regexes.some(re => re.test(respUrl));
+    if (!matchesRegex) return;
     
+    // Extract domain and check if already detected (skip expensive operations)   
     // The main request handler already filtered first-party/third-party requests
     // This response handler only runs for requests that passed that filter
     // However, we need to apply the same first-party/third-party logic here for searchstring analysis
@@ -607,6 +604,10 @@ function createResponseHandler(config) {
     
     // Apply first-party/third-party filtering for searchstring analysis
     // Use the exact same logic as the main request handler
+
+    const currentUrlHostname = new URL(currentUrl).hostname;
+    const responseHostname = new URL(respUrl).hostname;
+    const isFirstParty = currentUrlHostname === responseHostname;
     if (isFirstParty && siteConfig.firstParty === false) {
       if (forceDebug) {
         console.log(`[debug] Skipping first-party response for searchstring analysis (firstParty=false): ${respUrl}`);
@@ -632,9 +633,61 @@ function createResponseHandler(config) {
       }
       
       const content = await response.text();
+
+      // Cache the fetched content if callback provided
+      if (config.onContentFetched) {
+        try {
+          config.onContentFetched(respUrl, content);
+        } catch (cacheErr) {
+          if (forceDebug) {
+            console.log(`[debug] Content caching failed: ${cacheErr.message}`);
+          }
+        }
+      }
       
       // Check if content contains search strings (OR or AND logic)
-      const { found, matchedString, logicType, error } = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      let searchResult;
+      
+      if (useGrep && (searchStrings.length > 0 || searchStringsAnd.length > 0)) {
+        // Use grep for pattern matching
+        try {
+          const allPatterns = [...(searchStrings || []), ...(searchStringsAnd || [])];
+          const grepResult = await grepContent(content, allPatterns, {
+            ignoreCase: true,
+            wholeWord: false,
+            regex: false
+          });
+          
+          if (hasSearchStringAnd && searchStringsAnd.length > 0) {
+            // For AND logic, check that all patterns were found
+            const foundPatterns = grepResult.allMatches.map(match => match.pattern);
+            const allFound = searchStringsAnd.every(pattern => foundPatterns.includes(pattern));
+            searchResult = {
+              found: allFound,
+              matchedString: allFound ? foundPatterns.join(' AND ') : null,
+              logicType: 'AND'
+            };
+          } else {
+            // For OR logic, any match is sufficient
+            searchResult = {
+              found: grepResult.found,
+              matchedString: grepResult.matchedPattern,
+              logicType: 'OR'
+            };
+          }
+        } catch (grepErr) {
+          if (forceDebug) {
+            console.log(`[debug] Grep failed for ${respUrl}, falling back to JavaScript: ${grepErr.message}`);
+          }
+          // Fallback to JavaScript search
+          searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+        }
+      } else {
+        // Use JavaScript search
+        searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      }
+      
+      const { found, matchedString, logicType, error } = searchResult;
       
       if (found) {
         if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
@@ -642,27 +695,31 @@ function createResponseHandler(config) {
         }
         
         // Response handler doesn't have access to specific resource type
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, null);
+        // Use the addMatchedDomain helper which handles fullSubdomain properly
+        addMatchedDomain(respDomain, null, fullSubdomain);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
           const partyType = isFirstParty ? 'first-party' : 'third-party';
-          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}) contains searchstring (${logicType}): "${matchedString}"`);
+          const searchMethod = useGrep ? 'grep' : 'js';
+          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}) contains searchstring (${logicType}): "${matchedString}"`);
         }
         
         if (dumpUrls) {
           const timestamp = new Date().toISOString();
           const partyType = isFirstParty ? 'first-party' : 'third-party';
+          const searchMethod = useGrep ? 'grep' : 'js';
           try {
             fs.appendFileSync(matchedUrlsLogFile, 
-              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, searchstring (${logicType}): "${matchedString}")\n`);
+              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}, searchstring (${logicType}): "${matchedString}")\n`);
           } catch (logErr) {
             console.warn(`[warn] Failed to write to matched URLs log: ${logErr.message}`);
           }
         }
       } else if (forceDebug) {
         const partyType = isFirstParty ? 'first-party' : 'third-party';
-        console.log(`[debug] ${respUrl} (${partyType}) matched regex but no searchstring found`);
+        const searchMethod = useGrep ? 'grep' : 'js';
+        console.log(`[debug] ${respUrl} (${partyType}, ${searchMethod}) matched regex but no searchstring found`);
         if (error) {
           console.log(`[debug] Search error: ${error}`);
         }

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.23 ===
+// === Network scanner script (nwss.js) v2.0.25 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -132,7 +132,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.23'; // Script version
+const VERSION = '2.0.25'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -997,7 +997,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Map(); // Use Map to track frame state
+  let activeFrames = new Set(); // Use Set to track frame references
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1031,7 +1031,6 @@ function setupFrameHandling(page, forceDebug) {
       // Enhanced frame validation with multiple safety checks
       let frameUrl;
       try {
-        // Test frame accessibility first
         frameUrl = frame.url();
         
         // Check if frame is detached (if method exists)
@@ -1041,12 +1040,17 @@ function setupFrameHandling(page, forceDebug) {
           }
           return;
         }
+        
+        activeFrames.add(frame);
+        
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
+        }
       } catch (frameAccessError) {
         // Frame is not accessible (likely detached)
         return;
       }
-      
-      activeFrames.add(frame);
+
     } catch (detachError) {
       // Frame state checking can throw in 23.x, handle gracefully
       if (forceDebug) {
@@ -1055,14 +1059,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    // Store frame with timestamp for tracking
-    activeFrames.set(frame, Date.now());
     
     if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
-      try {       
-        if (forceDebug) {
-          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
-        }
+      let frameUrl;
+        frameUrl = frame.url();
         
         // Don't try to navigate to frames with invalid/empty URLs
         if (!frameUrl ||
@@ -1100,6 +1100,7 @@ function setupFrameHandling(page, forceDebug) {
         // Let frames load naturally - manual navigation often causes Protocol errors
         // await frame.goto(frame.url(), { waitUntil: 'domcontentloaded', timeout: 5000 });
         
+        try {        
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Frame will load naturally: ${frameUrl}`));
         }
@@ -1117,11 +1118,11 @@ function setupFrameHandling(page, forceDebug) {
   });
   // Handle frame navigations (keep this for monitoring)
   page.on('framenavigated', (frame) => {
-    let frameUrl;
 
     // Skip if frame is not in our active set
     if (!activeFrames.has(frame)) return;
 
+    let frameUrl;
     try {
       frameUrl = frame.url();
     } catch (urlErr) {
@@ -1143,17 +1144,14 @@ function setupFrameHandling(page, forceDebug) {
   // Optional: Handle frame detachment for cleanup
   page.on('framedetached', (frame) => {
     // Remove from active tracking
-    activeFrames.delete(frame);
+    activeFrames.delete(frame); // This works for both Map and Set
     
-    // Skip logging if we can't access frame URL
-    let frameUrl;
+
     if (forceDebug) {
+      let frameUrl;
       try {
         frameUrl = frame.url();
-      } catch (urlErr) {
-        // Frame already detached, can't get URL
-        return;
-      }
+
       if (frameUrl &&
           frameUrl !== 'about:blank' &&
           frameUrl !== 'about:srcdoc' &&
@@ -1162,6 +1160,11 @@ function setupFrameHandling(page, forceDebug) {
           !frameUrl.startsWith('chrome-extension://')) {
         console.log(formatLogMessage('debug', `Frame detached: ${frameUrl}`));
       }
+      } catch (urlErr) {
+        // Frame already detached, can't get URL - this is expected
+        return;
+      }
+
     }
   });
 }
@@ -1951,7 +1954,7 @@ function setupFrameHandling(page, forceDebug) {
    // Parse searchstring patterns using module
    const { searchStrings, searchStringsAnd, hasSearchString, hasSearchStringAnd } = parseSearchStrings(siteConfig.searchstring, siteConfig.searchstring_and);
    const useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring
-   let useGrep = siteConfig.grep === true && useCurl; // Grep requires curl to be enabled
+   let useGrep = siteConfig.grep === true; // Grep can work independently
 
    // Get user agent for curl if needed
    let curlUserAgent = '';
@@ -1973,7 +1976,7 @@ function setupFrameHandling(page, forceDebug) {
    }
 
    if (useGrep && forceDebug) {
-     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}`));
+     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}${useCurl ? ' (with curl)' : ' (with response handler)'}`));
    }
    
    // Validate grep availability if needed
@@ -1993,7 +1996,6 @@ function setupFrameHandling(page, forceDebug) {
      if (!curlCheck.isAvailable) {
        console.warn(formatLogMessage('warn', `Curl not available for ${currentUrl}: ${curlCheck.error}. Skipping curl-based analysis.`));
        useCurl = false;
-       useGrep = false; // Grep requires curl
      } else if (forceDebug) {
        console.log(formatLogMessage('debug', `Using curl: ${curlCheck.version}`));
      }
@@ -2643,7 +2645,7 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check bypass_cache before attempting cache lookup
+             // Check bypass_cache before attempting cache lookup (curl mode)
              let cachedContent = null;
              if (!shouldBypassCacheForUrl(reqUrl, siteConfig)) {
                // Check request cache first if smart cache is available and caching is enabled
@@ -2732,8 +2734,30 @@ function setupFrameHandling(page, forceDebug) {
                }
              }
              }
+            } else if (useGrep && (hasSearchString || hasSearchStringAnd)) {
+             // Use grep with response handler (no curl)
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `[grep-response] Queuing ${reqUrl} for grep analysis via response handler`));
+             }
+             
+             // Queue for grep processing via response handler
+             // The response handler will download content and call grep
+             if (dryRunMode) {
+               matchedDomains.get('dryRunMatches').push({
+                 regex: matchedRegexPattern,
+                 domain: reqDomain,
+                 resourceType: resourceType,
+                 fullUrl: reqUrl,
+                 isFirstParty: isFirstParty,
+                 needsGrepCheck: true
+               });
+             }
+             
+             // Don't process immediately - let response handler do the work
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `URL ${reqUrl} queued for grep analysis via response handler`));
+             }
            }
-
           // No break needed since we've already determined if regex matched
         }
         request.continue();
@@ -2742,8 +2766,8 @@ function setupFrameHandling(page, forceDebug) {
       // Mark page as actively processing network requests
       updatePageUsage(page, true);
 
-     // Add response handler ONLY if searchstring/searchstring_and is defined AND neither curl nor grep is enabled
-     if ((hasSearchString || hasSearchStringAnd) && !useCurl && !useGrep) {
+     // Add response handler if searchstring is defined and either no curl, or grep without curl
+     if ((hasSearchString || hasSearchStringAnd) && (!useCurl || (useGrep && !useCurl))) {
        const responseHandler = createResponseHandler({
          searchStrings,
          searchStringsAnd,
@@ -2761,6 +2785,7 @@ function setupFrameHandling(page, forceDebug) {
          } : undefined,
          currentUrl,
          perSiteSubDomains,
+         useGrep, // Pass grep flag to response handler
          ignoreDomains,
          matchesIgnoreDomain,
          getRootDomain,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.23",
+  "version": "2.0.25",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {