@fanboynz/network-scanner 1.0.84 → 1.0.86

package/lib/browserhealth.js

@@ -5,6 +5,89 @@
 
 const { formatLogMessage, messageColors } = require('./colorize');
 
+/**
+ * Quick browser responsiveness test for use during page setup
+ * Designed to catch browser degradation between operations
+ * @param {import('puppeteer').Browser} browserInstance - Puppeteer browser instance
+ * @param {number} timeout - Timeout in milliseconds (default: 3000)
+ * @returns {Promise<boolean>} True if browser responds quickly, false otherwise
+ */
+async function isQuicklyResponsive(browserInstance, timeout = 3000) {
+  try {
+    await Promise.race([
+      browserInstance.version(), // Quick responsiveness test
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Quick responsiveness timeout')), timeout)
+      )
+    ]);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+/**
+ * Tests if browser can handle network operations (like Network.enable)
+ * Creates a test page and attempts basic network setup
+ * @param {import('puppeteer').Browser} browserInstance - Puppeteer browser instance
+ * @param {number} timeout - Timeout in milliseconds (default: 10000)
+ * @returns {Promise<object>} Network capability test result
+ */
+async function testNetworkCapability(browserInstance, timeout = 10000) {
+  const result = {
+    capable: false,
+    error: null,
+    responseTime: 0
+  };
+
+  const startTime = Date.now();
+  let testPage = null;
+
+  try {
+    // Create test page
+    testPage = await Promise.race([
+      browserInstance.newPage(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Test page creation timeout')), timeout)
+      )
+    ]);
+
+    // Test network operations (the critical operation that's failing)
+    await Promise.race([
+      testPage.setRequestInterception(true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Network.enable test timeout')), timeout)
+      )
+    ]);
+
+    // Turn off interception and close
+    await testPage.setRequestInterception(false);
+    result.capable = true;
+    result.responseTime = Date.now() - startTime;
+
+  } catch (error) {
+    result.error = error.message;
+    result.responseTime = Date.now() - startTime;
+
+    // Classify the error type
+    if (error.message.includes('Network.enable') || 
+        error.message.includes('timed out') ||
+        error.message.includes('Protocol error')) {
+      result.error = `Network capability test failed: ${error.message}`;
+    }
+  } finally {
+    if (testPage && !testPage.isClosed()) {
+      try { 
+        await testPage.close(); 
+      } catch (closeErr) { 
+        /* ignore cleanup errors */ 
+      }
+    }
+  }
+
+  return result;
+}
+
 /**
  * Checks if browser instance is still responsive
  * @param {import('puppeteer').Browser} browserInstance - Puppeteer browser instance
@@ -18,7 +101,8 @@ async function checkBrowserHealth(browserInstance, timeout = 8000) {
     error: null,
     responseTime: 0,
     recommendations: [],
-    criticalError: false
+    criticalError: false,
+    networkCapable: false
   };
 
   const startTime = Date.now();
@@ -82,13 +166,25 @@ async function checkBrowserHealth(browserInstance, timeout = 8000) {
       return healthResult;
     }
 
-    // Test 5: Check response time performance
+    // Test 5: Network capability test (critical for Network.enable issues)
+    const networkTest = await testNetworkCapability(browserInstance, Math.min(timeout, 5000));
+    healthResult.networkCapable = networkTest.capable;
+    
+    if (!networkTest.capable) {
+      healthResult.recommendations.push(`Network operations failing: ${networkTest.error}`);
+      if (networkTest.error && networkTest.error.includes('Network.enable')) {
+        healthResult.criticalError = true;
+      }
+    }
+
+    // Test 6: Check response time performance
     if (healthResult.responseTime > 5000) {
       healthResult.recommendations.push('Slow browser response - consider restart');
     }
 
-    // If all tests pass
-    healthResult.healthy = true;
+    // If all tests pass (including network capability)
+    healthResult.healthy = networkTest.capable; // Network capability is now critical for health
+
 
   } catch (error) {
     healthResult.error = error.message;
@@ -195,7 +291,11 @@ function isCriticalProtocolError(error) {
     'Browser process exited',
     'Navigation timeout of',
     'Page crashed',
-    'Renderer process crashed'
+    'Renderer process crashed',
+    // Network-specific critical errors
+    'Network.enable timed out',
+    'Network.disable timed out',
+    'Network service not available'
   ];
   
   return criticalErrors.some(criticalError => 
@@ -413,14 +513,25 @@ async function monitorBrowserHealth(browserInstance, context = {}, options = {})
 
 /**
  * Simple health check function for quick integration
+ * Enhanced version that includes network capability testing
  * @param {import('puppeteer').Browser} browserInstance - Puppeteer browser instance
+ * @param {boolean} includeNetworkTest - Whether to test network capabilities (default: true)
  * @returns {Promise<boolean>} True if browser is healthy, false otherwise
  */
-async function isBrowserHealthy(browserInstance) {
+async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
   try {
-    const health = await checkBrowserHealth(browserInstance, 5000); // Faster timeout
+    // Quick responsiveness test first (fastest check)
+    const quickCheck = await isQuicklyResponsive(browserInstance, 2500);
+    if (!quickCheck) return false;
+    
+    // More comprehensive health check if quick test passes
+    const health = await checkBrowserHealth(browserInstance, includeNetworkTest ? 8000 : 5000);
     const connectivity = await testBrowserConnectivity(browserInstance, 3000);
-    return health.healthy && connectivity.connected && connectivity.cdpResponsive;
+    
+    const baseHealth = health.healthy && connectivity.connected && connectivity.cdpResponsive;
+    
+    // Include network capability in health assessment if requested
+    return includeNetworkTest ? (baseHealth && health.networkCapable) : baseHealth;
   } catch (error) {
     return false;
   }
@@ -430,6 +541,8 @@ module.exports = {
   checkBrowserHealth,
   checkBrowserMemory,
   testBrowserConnectivity,
+  testNetworkCapability,
+  isQuicklyResponsive,
   performHealthAssessment,
   monitorBrowserHealth,
   isBrowserHealthy,

package/lib/output.js

@@ -18,13 +18,35 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
   
   return ignorePatterns.some(pattern => {
     if (pattern.includes('*')) {
-      // Convert wildcard pattern to regex
-      const regexPattern = pattern
-        .replace(/\./g, '\\.')  // Escape dots
-        .replace(/\*/g, '.*');  // Convert * to .*
-      return new RegExp(`^${regexPattern}$`).test(domain);
+      // Enhanced wildcard pattern handling
+      if (pattern.startsWith('*.')) {
+        // Pattern: *.example.com
+        const wildcardDomain = pattern.substring(2); // Remove "*."
+        const wildcardRoot = extractDomainFromRule(`||${wildcardDomain}^`) || wildcardDomain;
+        const domainRoot = extractDomainFromRule(`||${domain}^`) || domain;
+        
+        // Use basic root domain comparison for output filtering
+        const getSimpleRoot = (d) => {
+          const parts = d.split('.');
+          return parts.length >= 2 ? parts.slice(-2).join('.') : d;
+        };
+        
+        return getSimpleRoot(domainRoot) === getSimpleRoot(wildcardRoot);
+      } else if (pattern.endsWith('.*')) {
+        // Pattern: example.*
+        const baseDomain = pattern.slice(0, -2); // Remove ".*"
+        return domain.startsWith(baseDomain + '.');
+      } else {
+        // Complex wildcard pattern
+        const regexPattern = pattern
+          .replace(/\./g, '\\.')  // Escape dots
+          .replace(/\*/g, '.*');  // Convert * to .*
+        return new RegExp(`^${regexPattern}$`).test(domain);
+      }
+    } else {
+      // Exact pattern matching
+      return domain === pattern || domain.endsWith('.' + pattern);
     }
-    return domain.endsWith(pattern);
   });
 }
 

package/lib/post-processing.js

@@ -316,6 +316,77 @@ function cleanupIgnoreDomains(results, ignoreDomains, options = {}) {
   return cleanedResults;
 }
 
+/**
+ * Enhanced domain extraction helper that reuses existing parsing logic
+ * @param {string} rule - Rule string in various formats
+ * @returns {string|null} Extracted domain or null if not found
+ */
+function extractDomainFromRule(rule) {
+  if (!rule || typeof rule !== 'string') {
+    return null;
+  }
+
+  try {
+    // Reuse the existing parsing logic from cleanupFirstPartyDomains
+    let extractedDomain = null;
+    
+    if (rule.startsWith('||') && rule.includes('^')) {
+      // ||domain.com^ format (adblock)
+      const match = rule.match(/^\|\|([^/\^]+)/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.startsWith('127.0.0.1 ') || rule.startsWith('0.0.0.0 ')) {
+      // hosts file format
+      const parts = rule.split(/\s+/);
+      if (parts.length >= 2) {
+        extractedDomain = parts[1];
+      }
+    } else if (rule.includes('local=/') && rule.includes('/')) {
+      // dnsmasq format: local=/domain.com/
+      const match = rule.match(/local=\/([^/]+)\//);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('server=/') && rule.includes('/')) {
+      // dnsmasq old format: server=/domain.com/
+      const match = rule.match(/server=\/([^/]+)\//);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('local-zone:') && rule.includes('always_null')) {
+      // unbound format: local-zone: "domain.com." always_null
+      const match = rule.match(/local-zone:\s*"([^"]+)\.?"/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('+block') && rule.includes('.')) {
+      // privoxy format: { +block } .domain.com
+      const match = rule.match(/\{\s*\+block\s*\}\s*\.?([^\s]+)/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.match(/^\(\^\|\\\.\).*\\\.\w+\$$/)) {
+      // pi-hole regex format: (^|\.)domain\.com$
+      const match = rule.match(/^\(\^\|\\\.\)(.+)\\\.\w+\$$/);
+      if (match) {
+        // Unescape the domain
+        extractedDomain = match[1].replace(/\\\./g, '.');
+      }
+    } else {
+      // Try to extract any domain-like pattern as fallback
+      const domainMatch = rule.match(/([a-zA-Z0-9][a-zA-Z0-9.-]*\.[a-zA-Z]{2,})/);
+      if (domainMatch) {
+        extractedDomain = domainMatch[1];
+      }
+    }
+    
+    return extractedDomain;
+  } catch (parseErr) {
+    return null;
+  }
+}
+
 /**
  * Post-scan cleanup function to remove first-party domains from results
  * Only processes sites that have firstParty: false in their configuration
@@ -564,6 +635,111 @@ function validateScanResults(results, options = {}) {
   return validatedResults;
 }
 
+
+/**
+ * Final validation check for firstParty: false violations
+ * Reuses existing domain extraction and matching logic
+ * 
+ * @param {Array} results - Array of scan results
+ * @param {Array} sites - Array of site configurations  
+ * @param {Object} options - Options object
+ * @returns {Array} Results with any remaining first-party domains removed
+ */
+function finalFirstPartyValidation(results, sites, options = {}) {
+  const { forceDebug = false, silentMode = false } = options;
+  
+  if (!results || results.length === 0) {
+    return results;
+  }
+
+  // Reuse the URL-to-config mapping pattern from cleanupFirstPartyDomains
+  const urlToSiteConfig = new Map();
+  sites.forEach(site => {
+    const urls = Array.isArray(site.url) ? site.url : [site.url];
+    urls.forEach(url => {
+      urlToSiteConfig.set(url, site);
+    });
+  });
+
+  const finalResults = [];
+  let totalViolationsFound = 0;
+  let sitesWithViolations = 0;
+
+  results.forEach(result => {
+    const siteConfig = urlToSiteConfig.get(result.url);
+    
+    // Only validate sites with firstParty: false
+    const shouldValidate = siteConfig && siteConfig.firstParty === false;
+    
+    if (!shouldValidate || !result.rules || result.rules.length === 0) {
+      finalResults.push(result);
+      return;
+    }
+
+    const scannedDomain = safeGetDomain(result.url, false);
+    if (!scannedDomain) {
+      finalResults.push(result);
+      return;
+    }
+
+    // Reuse the same filtering logic pattern from cleanupFirstPartyDomains
+    const cleanedRules = [];
+    const violatingRules = [];
+
+    result.rules.forEach(rule => {
+      const extractedDomain = extractDomainFromRule(rule);
+      if (extractedDomain) {
+        // Reuse the shouldRemoveAsFirstParty logic
+        const matchResult = shouldRemoveAsFirstParty(extractedDomain, scannedDomain, forceDebug);
+        
+        if (matchResult.shouldRemove) {
+          violatingRules.push({
+            rule: rule,
+            domain: extractedDomain,
+            reason: `VALIDATION FAILURE: ${matchResult.reason}`
+          });
+          totalViolationsFound++;
+          return;
+        }
+      }
+      cleanedRules.push(rule);
+    });
+
+    if (violatingRules.length > 0) {
+      sitesWithViolations++;
+      
+      if (!silentMode) {
+        const errorMessage = `? CONFIG VIOLATION: Found ${violatingRules.length} first-party rule(s) in ${scannedDomain} (firstParty: false)`;
+        if (messageColors && messageColors.error) {
+          console.log(messageColors.error(errorMessage));
+        } else {
+          console.log(errorMessage);
+        }
+      }
+
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[final-validation] Violations found for ${result.url}:`));
+        violatingRules.forEach((violation, idx) => {
+          console.log(formatLogMessage('debug', `  [${idx + 1}] ${violation.rule} -> ${violation.domain}`));
+        });
+      }
+    }
+
+    finalResults.push({ ...result, rules: cleanedRules });
+  });
+
+  // Summary using existing message patterns
+  if (totalViolationsFound > 0 && !silentMode) {
+    const summaryMessage = `\n? SCAN FILTERING FAILURE: Removed ${totalViolationsFound} first-party rules from ${sitesWithViolations} site(s) in post-processing`;
+    console.log(summaryMessage);
+    console.log('??  This indicates firstParty: false filtering failed during scan - consider investigating root cause.');
+  } else if (forceDebug) {
+    console.log(formatLogMessage('debug', '[final-validation] No first-party violations found - filtering working correctly'));
+  }
+
+  return finalResults;
+}
+
 /**
  * Main post-processing function that runs all cleanup and validation steps
  * 
@@ -587,8 +763,11 @@ function processResults(results, sites, options = {}) {
   
   // Step 2: Clean up ignoreDomains (final safety net)
   processedResults = cleanupIgnoreDomains(processedResults, options.ignoreDomains || [], options);
+  
+  // Step 3: Final validation for firstParty: false configurations
+  processedResults = finalFirstPartyValidation(processedResults, sites, options);
 
-  // Step 3: Validate results
+  // Step 4: Validate results
   processedResults = validateScanResults(processedResults, options);
   
   if (forceDebug) {
@@ -602,6 +781,8 @@ function processResults(results, sites, options = {}) {
 module.exports = {
   cleanupFirstPartyDomains,
   cleanupIgnoreDomains,
+  finalFirstPartyValidation,
+  extractDomainFromRule,
   validateScanResults,
   processResults
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.84 ===
+// === Network scanner script (nwss.js) v1.0.86 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -120,10 +120,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.84'; // Script version
+const VERSION = '1.0.86'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1435,6 +1435,9 @@ function setupFrameHandling(page, forceDebug) {
     const originalRootDomain = safeGetDomain(currentUrl, false);
     if (originalRootDomain) {
       firstPartyDomains.add(originalRootDomain);
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Initial first-party domain: ${originalRootDomain} for ${currentUrl}`));
+      }
     } 
 
     // Track redirect domains to exclude from matching
@@ -1480,6 +1483,22 @@ function setupFrameHandling(page, forceDebug) {
       if (!page || page.isClosed()) {
         throw new Error('Failed to create valid page instance');
       }
+
+      // Additional health check after page creation but before critical setup
+      const stillHealthy = await isQuicklyResponsive(browserInstance, 3000);
+      
+      if (!stillHealthy) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Browser unresponsive during page setup for ${currentUrl} - triggering restart`));
+        }
+        return { 
+          url: currentUrl, 
+          rules: [], 
+          success: false, 
+          needsImmediateRestart: true,
+          error: 'Browser became unresponsive during page setup - restart required'
+        };
+      }
       
       // Set aggressive timeouts for problematic operations
       // Optimized timeouts for Puppeteer 23.x responsiveness
@@ -1569,6 +1588,22 @@ function setupFrameHandling(page, forceDebug) {
                   console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
               }
           }
+          // Quick browser health check before script injection
+          let browserResponsive = false;
+          try {
+              await Promise.race([
+                  browserInstance.version(), // Quick responsiveness test
+                  new Promise((_, reject) => 
+                      setTimeout(() => reject(new Error('Browser health check timeout')), 5000)
+                  )
+              ]);
+              browserResponsive = true;
+          } catch (healthErr) {
+              console.warn(formatLogMessage('warn', `[evalOnDoc] Browser unresponsive for ${currentUrl}: ${healthErr.message} - skipping script injection`));
+              browserResponsive = false;
+          }
+          
+          if (browserResponsive) {
           try {
               await page.evaluateOnNewDocument(() => {
                   // Prevent infinite reload loops
@@ -1633,7 +1668,16 @@ function setupFrameHandling(page, forceDebug) {
                   };
               });
           } catch (evalErr) {
-              console.warn(formatLogMessage('warn', `[evalOnDoc] Failed to set up Fetch/XHR interception for ${currentUrl}: ${evalErr.message}`));
+                  if (evalErr.message.includes('timed out') || evalErr.message.includes('ProtocolError')) {
+                      console.warn(formatLogMessage('warn', `[evalOnDoc] Script injection protocol timeout for ${currentUrl} - continuing without XHR/Fetch interception`));
+                  } else {
+                      console.warn(formatLogMessage('warn', `[evalOnDoc] Failed to set up Fetch/XHR interception for ${currentUrl}: ${evalErr.message}`));
+                  }
+          }
+          } else {
+              if (forceDebug) {
+                  console.log(formatLogMessage('debug', `[evalOnDoc] Continuing ${currentUrl} without XHR/Fetch interception due to browser health`));
+              }
           }
       }
       // --- END: evaluateOnNewDocument for Fetch/XHR Interception ---
@@ -1680,7 +1724,34 @@ function setupFrameHandling(page, forceDebug) {
       }
       // --- End of Per-Page CDP Setup ---
 
-      await page.setRequestInterception(true);
+      // Protected request interception setup with timeout
+      try {
+        // Test if network operations are responsive before enabling request interception
+        await Promise.race([
+          page.setRequestInterception(true),
+          new Promise((_, reject) => 
+            setTimeout(() => reject(new Error('Network.enable timeout')), 10000)
+          )
+        ]);
+        
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Request interception enabled successfully for ${currentUrl}`));
+        }
+      } catch (networkErr) {
+        if (networkErr.message.includes('timed out') || 
+            networkErr.message.includes('Network.enable') || 
+            networkErr.message.includes('timeout')) {
+          console.warn(formatLogMessage('warn', `Network setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
+          return { 
+            url: currentUrl, 
+            rules: [], 
+            success: false, 
+            needsImmediateRestart: true,
+            error: 'Network.enable timeout - browser restart required'
+          };
+        }
+        throw networkErr; // Re-throw other errors
+      }
 	  
 	  // Set up frame handling to suppress invalid URL errors
       setupFrameHandling(page, forceDebug);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.84",
+  "version": "1.0.86",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {