@fanboynz/network-scanner 2.0.24 → 2.0.25

package/lib/fingerprint.js

@@ -1347,6 +1347,19 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
 
   try {
     await page.evaluateOnNewDocument(({ spoof, debugEnabled }) => {
+    
+      // Define helper functions FIRST in this context
+      function spoofNavigatorProperties(navigator, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(navigator, prop, descriptor);
+        }
+      }
+      
+      function spoofScreenProperties(screen, properties) {
+        for (const [prop, descriptor] of Object.entries(properties)) {
+          safeDefinePropertyLocal(screen, prop, descriptor);
+        }
+      }
       
       function safeDefinePropertyLocal(target, property, descriptor) {
         try {

package/lib/searchstring.js

@@ -3,6 +3,7 @@
 
 const fs = require('fs');
 const { spawnSync } = require('child_process');
+const { grepContent } = require('./grep');
 
 // Configuration constants for search logic
 const SEARCH_CONFIG = {
@@ -51,11 +52,12 @@ function parseSearchStrings(searchstring, searchstringAnd) {
  * @param {Function} addMatchedDomain - Optional helper function for adding domains
  * @param {string} domain - Domain to add
  * @param {string} resourceType - Resource type (for --adblock-rules mode)
+ * @param {string} fullSubdomain - Full subdomain for cache tracking (optional)
  */
-function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null) {
+function addDomainToCollection(matchedDomains, addMatchedDomain, domain, resourceType = null, fullSubdomain = null) {
   // Use helper function if provided (preferred method)
   if (typeof addMatchedDomain === 'function') {
-    addMatchedDomain(domain, resourceType);
+    addMatchedDomain(domain, resourceType, fullSubdomain);
     return;
   }
   
@@ -575,6 +577,7 @@ function createResponseHandler(config) {
     siteConfig,
     dumpUrls,
     matchedUrlsLogFile,
+    useGrep = false,
     forceDebug,
     resourceType // Will be null for response handler
   } = config;
@@ -584,22 +587,16 @@ function createResponseHandler(config) {
     const respDomain = perSiteSubDomains ? (new URL(respUrl)).hostname : getRootDomain(respUrl);
     
     // Only process responses that match our regex patterns
-    const matchesRegex = regexes.some(re => re.test(respUrl));
-    if (!matchesRegex) return;
+    const fullSubdomain = (new URL(respUrl)).hostname; // Always get full subdomain for cache tracking
     
-    // Extract domain and check if already detected (skip expensive operations)
-    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(respDomain)) {
-      if (forceDebug) {
-        console.log(`[debug] Skipping response analysis for already detected domain: ${respDomain}`);
-      }
+    // Skip if already detected to avoid duplicates
+    if (typeof config.isDomainAlreadyDetected === 'function' && config.isDomainAlreadyDetected(fullSubdomain)) {
       return;
     }
-
-    // Check if this is a first-party response (same domain as the URL being scanned)
-    const currentUrlHostname = new URL(currentUrl).hostname;
-    const responseHostname = new URL(respUrl).hostname;
-    const isFirstParty = currentUrlHostname === responseHostname;
+    const matchesRegex = regexes.some(re => re.test(respUrl));
+    if (!matchesRegex) return;
     
+    // Extract domain and check if already detected (skip expensive operations)   
     // The main request handler already filtered first-party/third-party requests
     // This response handler only runs for requests that passed that filter
     // However, we need to apply the same first-party/third-party logic here for searchstring analysis
@@ -607,6 +604,10 @@ function createResponseHandler(config) {
     
     // Apply first-party/third-party filtering for searchstring analysis
     // Use the exact same logic as the main request handler
+
+    const currentUrlHostname = new URL(currentUrl).hostname;
+    const responseHostname = new URL(respUrl).hostname;
+    const isFirstParty = currentUrlHostname === responseHostname;
     if (isFirstParty && siteConfig.firstParty === false) {
       if (forceDebug) {
         console.log(`[debug] Skipping first-party response for searchstring analysis (firstParty=false): ${respUrl}`);
@@ -632,9 +633,61 @@ function createResponseHandler(config) {
       }
       
       const content = await response.text();
+
+      // Cache the fetched content if callback provided
+      if (config.onContentFetched) {
+        try {
+          config.onContentFetched(respUrl, content);
+        } catch (cacheErr) {
+          if (forceDebug) {
+            console.log(`[debug] Content caching failed: ${cacheErr.message}`);
+          }
+        }
+      }
       
       // Check if content contains search strings (OR or AND logic)
-      const { found, matchedString, logicType, error } = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      let searchResult;
+      
+      if (useGrep && (searchStrings.length > 0 || searchStringsAnd.length > 0)) {
+        // Use grep for pattern matching
+        try {
+          const allPatterns = [...(searchStrings || []), ...(searchStringsAnd || [])];
+          const grepResult = await grepContent(content, allPatterns, {
+            ignoreCase: true,
+            wholeWord: false,
+            regex: false
+          });
+          
+          if (hasSearchStringAnd && searchStringsAnd.length > 0) {
+            // For AND logic, check that all patterns were found
+            const foundPatterns = grepResult.allMatches.map(match => match.pattern);
+            const allFound = searchStringsAnd.every(pattern => foundPatterns.includes(pattern));
+            searchResult = {
+              found: allFound,
+              matchedString: allFound ? foundPatterns.join(' AND ') : null,
+              logicType: 'AND'
+            };
+          } else {
+            // For OR logic, any match is sufficient
+            searchResult = {
+              found: grepResult.found,
+              matchedString: grepResult.matchedPattern,
+              logicType: 'OR'
+            };
+          }
+        } catch (grepErr) {
+          if (forceDebug) {
+            console.log(`[debug] Grep failed for ${respUrl}, falling back to JavaScript: ${grepErr.message}`);
+          }
+          // Fallback to JavaScript search
+          searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+        }
+      } else {
+        // Use JavaScript search
+        searchResult = searchContent(content, searchStrings, searchStringsAnd, contentType, respUrl);
+      }
+      
+      const { found, matchedString, logicType, error } = searchResult;
       
       if (found) {
         if (!respDomain || matchesIgnoreDomain(respDomain, ignoreDomains)) {
@@ -642,27 +695,31 @@ function createResponseHandler(config) {
         }
         
         // Response handler doesn't have access to specific resource type
-        addDomainToCollection(matchedDomains, addMatchedDomain, respDomain, null);
+        // Use the addMatchedDomain helper which handles fullSubdomain properly
+        addMatchedDomain(respDomain, null, fullSubdomain);
         const simplifiedUrl = getRootDomain(currentUrl);
         
         if (siteConfig.verbose === 1) {
           const partyType = isFirstParty ? 'first-party' : 'third-party';
-          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}) contains searchstring (${logicType}): "${matchedString}"`);
+          const searchMethod = useGrep ? 'grep' : 'js';
+          console.log(`[match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}) contains searchstring (${logicType}): "${matchedString}"`);
         }
         
         if (dumpUrls) {
           const timestamp = new Date().toISOString();
           const partyType = isFirstParty ? 'first-party' : 'third-party';
+          const searchMethod = useGrep ? 'grep' : 'js';
           try {
             fs.appendFileSync(matchedUrlsLogFile, 
-              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, searchstring (${logicType}): "${matchedString}")\n`);
+              `${timestamp} [match][${simplifiedUrl}] ${respUrl} (${partyType}, ${searchMethod}, searchstring (${logicType}): "${matchedString}")\n`);
           } catch (logErr) {
             console.warn(`[warn] Failed to write to matched URLs log: ${logErr.message}`);
           }
         }
       } else if (forceDebug) {
         const partyType = isFirstParty ? 'first-party' : 'third-party';
-        console.log(`[debug] ${respUrl} (${partyType}) matched regex but no searchstring found`);
+        const searchMethod = useGrep ? 'grep' : 'js';
+        console.log(`[debug] ${respUrl} (${partyType}, ${searchMethod}) matched regex but no searchstring found`);
         if (error) {
           console.log(`[debug] Search error: ${error}`);
         }

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.24 ===
+// === Network scanner script (nwss.js) v2.0.25 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -132,7 +132,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.24'; // Script version
+const VERSION = '2.0.25'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -997,7 +997,7 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
 
 function setupFrameHandling(page, forceDebug) {
   // Track active frames and clear on navigation to prevent detached frame access
-  let activeFrames = new Map(); // Use Map to track frame state
+  let activeFrames = new Set(); // Use Set to track frame references
   
   // Clear frame tracking on navigation to prevent stale references
   page.on('framenavigated', (frame) => {
@@ -1031,7 +1031,6 @@ function setupFrameHandling(page, forceDebug) {
       // Enhanced frame validation with multiple safety checks
       let frameUrl;
       try {
-        // Test frame accessibility first
         frameUrl = frame.url();
         
         // Check if frame is detached (if method exists)
@@ -1041,12 +1040,17 @@ function setupFrameHandling(page, forceDebug) {
           }
           return;
         }
+        
+        activeFrames.add(frame);
+        
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
+        }
       } catch (frameAccessError) {
         // Frame is not accessible (likely detached)
         return;
       }
-      
-      activeFrames.add(frame);
+
     } catch (detachError) {
       // Frame state checking can throw in 23.x, handle gracefully
       if (forceDebug) {
@@ -1055,14 +1059,10 @@ function setupFrameHandling(page, forceDebug) {
       return;
     }
 
-    // Store frame with timestamp for tracking
-    activeFrames.set(frame, Date.now());
     
     if (frame !== page.mainFrame() && frame.parentFrame()) { // Only handle child frames
-      try {       
-        if (forceDebug) {
-          console.log(formatLogMessage('debug', `New frame attached: ${frameUrl || 'about:blank'}`));
-        }
+      let frameUrl;
+        frameUrl = frame.url();
         
         // Don't try to navigate to frames with invalid/empty URLs
         if (!frameUrl ||
@@ -1100,6 +1100,7 @@ function setupFrameHandling(page, forceDebug) {
         // Let frames load naturally - manual navigation often causes Protocol errors
         // await frame.goto(frame.url(), { waitUntil: 'domcontentloaded', timeout: 5000 });
         
+        try {        
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Frame will load naturally: ${frameUrl}`));
         }
@@ -1117,11 +1118,11 @@ function setupFrameHandling(page, forceDebug) {
   });
   // Handle frame navigations (keep this for monitoring)
   page.on('framenavigated', (frame) => {
-    let frameUrl;
 
     // Skip if frame is not in our active set
     if (!activeFrames.has(frame)) return;
 
+    let frameUrl;
     try {
       frameUrl = frame.url();
     } catch (urlErr) {
@@ -1143,17 +1144,14 @@ function setupFrameHandling(page, forceDebug) {
   // Optional: Handle frame detachment for cleanup
   page.on('framedetached', (frame) => {
     // Remove from active tracking
-    activeFrames.delete(frame);
+    activeFrames.delete(frame); // This works for both Map and Set
     
-    // Skip logging if we can't access frame URL
-    let frameUrl;
+
     if (forceDebug) {
+      let frameUrl;
       try {
         frameUrl = frame.url();
-      } catch (urlErr) {
-        // Frame already detached, can't get URL
-        return;
-      }
+
       if (frameUrl &&
           frameUrl !== 'about:blank' &&
           frameUrl !== 'about:srcdoc' &&
@@ -1162,6 +1160,11 @@ function setupFrameHandling(page, forceDebug) {
           !frameUrl.startsWith('chrome-extension://')) {
         console.log(formatLogMessage('debug', `Frame detached: ${frameUrl}`));
       }
+      } catch (urlErr) {
+        // Frame already detached, can't get URL - this is expected
+        return;
+      }
+
     }
   });
 }
@@ -1951,7 +1954,7 @@ function setupFrameHandling(page, forceDebug) {
    // Parse searchstring patterns using module
    const { searchStrings, searchStringsAnd, hasSearchString, hasSearchStringAnd } = parseSearchStrings(siteConfig.searchstring, siteConfig.searchstring_and);
    const useCurl = siteConfig.curl === true; // Use curl if enabled, regardless of searchstring
-   let useGrep = siteConfig.grep === true && useCurl; // Grep requires curl to be enabled
+   let useGrep = siteConfig.grep === true; // Grep can work independently
 
    // Get user agent for curl if needed
    let curlUserAgent = '';
@@ -1973,7 +1976,7 @@ function setupFrameHandling(page, forceDebug) {
    }
 
    if (useGrep && forceDebug) {
-     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}`));
+     console.log(formatLogMessage('debug', `Grep-based pattern matching enabled for ${currentUrl}${useCurl ? ' (with curl)' : ' (with response handler)'}`));
    }
    
    // Validate grep availability if needed
@@ -1993,7 +1996,6 @@ function setupFrameHandling(page, forceDebug) {
      if (!curlCheck.isAvailable) {
        console.warn(formatLogMessage('warn', `Curl not available for ${currentUrl}: ${curlCheck.error}. Skipping curl-based analysis.`));
        useCurl = false;
-       useGrep = false; // Grep requires curl
      } else if (forceDebug) {
        console.log(formatLogMessage('debug', `Using curl: ${curlCheck.version}`));
      }
@@ -2643,7 +2645,7 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check bypass_cache before attempting cache lookup
+             // Check bypass_cache before attempting cache lookup (curl mode)
              let cachedContent = null;
              if (!shouldBypassCacheForUrl(reqUrl, siteConfig)) {
                // Check request cache first if smart cache is available and caching is enabled
@@ -2732,8 +2734,30 @@ function setupFrameHandling(page, forceDebug) {
                }
              }
              }
+            } else if (useGrep && (hasSearchString || hasSearchStringAnd)) {
+             // Use grep with response handler (no curl)
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `[grep-response] Queuing ${reqUrl} for grep analysis via response handler`));
+             }
+             
+             // Queue for grep processing via response handler
+             // The response handler will download content and call grep
+             if (dryRunMode) {
+               matchedDomains.get('dryRunMatches').push({
+                 regex: matchedRegexPattern,
+                 domain: reqDomain,
+                 resourceType: resourceType,
+                 fullUrl: reqUrl,
+                 isFirstParty: isFirstParty,
+                 needsGrepCheck: true
+               });
+             }
+             
+             // Don't process immediately - let response handler do the work
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `URL ${reqUrl} queued for grep analysis via response handler`));
+             }
            }
-
           // No break needed since we've already determined if regex matched
         }
         request.continue();
@@ -2742,8 +2766,8 @@ function setupFrameHandling(page, forceDebug) {
       // Mark page as actively processing network requests
       updatePageUsage(page, true);
 
-     // Add response handler ONLY if searchstring/searchstring_and is defined AND neither curl nor grep is enabled
-     if ((hasSearchString || hasSearchStringAnd) && !useCurl && !useGrep) {
+     // Add response handler if searchstring is defined and either no curl, or grep without curl
+     if ((hasSearchString || hasSearchStringAnd) && (!useCurl || (useGrep && !useCurl))) {
        const responseHandler = createResponseHandler({
          searchStrings,
          searchStringsAnd,
@@ -2761,6 +2785,7 @@ function setupFrameHandling(page, forceDebug) {
          } : undefined,
          currentUrl,
          perSiteSubDomains,
+         useGrep, // Pass grep flag to response handler
          ignoreDomains,
          matchesIgnoreDomain,
          getRootDomain,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.24",
+  "version": "2.0.25",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {