@fanboynz/network-scanner 1.0.94 → 1.0.96

package/README.md

@@ -73,10 +73,13 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 
 | Argument                  | Description |
 |:---------------------------|:------------|
+| `--cache-requests`          | Cache HTTP requests to avoid re-requesting same URLs within scan |
 | `--validate-config`         | Validate config.json file and exit |
 | `--validate-rules [file]`   | Validate rule file format (uses --output/--compare files if no file specified) |
 | `--clean-rules [file]`      | Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified) |
 | `--test-validation`         | Run domain validation tests and exit |
+| `--clear-cache`             | Clear persistent cache before scanning (improves fresh start performance) |
+| `--ignore-cache`            | Bypass all smart caching functionality during scanning |
 
 ---
 
@@ -133,6 +136,7 @@ Example:
 | `url`                | String or Array |   -     | Website URL(s) to scan |
 | `userAgent`          | `chrome`, `firefox`, `safari` | - | User agent for page (latest versions: Chrome 131, Firefox 133, Safari 18.2) |
 | `filterRegex`        | String or Array | `.*` | Regex or list of regexes to match requests |
+| `regex_and`          | Boolean | `false` | Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL |
 | `comments`           | String or Array | - | String of comments or references |
 | `resourceTypes`      | Array | `["script", "xhr", "image", "stylesheet"]` | What resource types to monitor |
 | `reload`             | Integer | `1` | Number of times to reload page |
@@ -146,6 +150,8 @@ Example:
 | `subDomains`         | `0` or `1` | `0` | 1 = preserve subdomains in output |
 | `blocked`            | Array | - | Domains or regexes to block during scanning |
 | `even_blocked`       | Boolean | `false` | Add matching rules even if requests are blocked |
+| `bypass_cache`       | Boolean | `false` | Skip all caching for this site's URLs |
+
 
 ### Redirect Handling Options
 
@@ -190,6 +196,10 @@ When a page redirects to a new domain, first-party/third-party detection is base
 |:---------------------|:-------|:-------:|:------------|
 | `cloudflare_phish`   | Boolean | `false` | Auto-click through Cloudflare phishing warnings |
 | `cloudflare_bypass`  | Boolean | `false` | Auto-solve Cloudflare "Verify you are human" challenges |
+| `cloudflare_parallel_detection` | Boolean | `true` | Use parallel detection for faster Cloudflare checks |
+| `cloudflare_max_retries` | Integer | `3` | Maximum retry attempts for Cloudflare operations |
+| `cloudflare_cache_ttl` | Milliseconds | `300000` | TTL for Cloudflare detection cache (5 minutes) |
+| `cloudflare_retry_on_error` | Boolean | `true` | Enable retry logic for Cloudflare operations |
 | `flowproxy_detection` | Boolean | `false` | Enable flowProxy protection detection and handling |
 | `flowproxy_page_timeout` | Milliseconds | `45000` | Page timeout for flowProxy sites |
 | `flowproxy_nav_timeout` | Milliseconds | `45000` | Navigation timeout for flowProxy sites |
@@ -240,6 +250,11 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |
 | `headful`            | Boolean | `false` | Launch browser with GUI for this site |
 | `adblock_rules`      | Boolean | `false` | Generate adblock filter rules with resource types for this site |
+| `interact_duration`  | Milliseconds | `2000` | Duration of interaction simulation |
+| `interact_scrolling` | Boolean | `true` | Enable scrolling simulation |
+| `interact_clicks`    | Boolean | `false` | Enable element clicking simulation |
+| `interact_typing`    | Boolean | `false` | Enable typing simulation |
+| `interact_intensity` | String | `"medium"` | Interaction simulation intensity: "low", "medium", "high" |
 
 ### Global Configuration Options
 
@@ -254,7 +269,11 @@ These options go at the root level of your config.json:
 | `ignore_similar_threshold` | Integer | `80` | Similarity threshold percentage for ignore_similar |
 | `ignore_similar_ignored_domains` | Boolean | `true` | Ignore domains similar to ignoreDomains list |
 | `max_concurrent_sites` | Integer | `6` | Maximum concurrent site processing (1-50) |
-| `resource_cleanup_interval` | Integer | `180` | Browser restart interval in URLs processed (1-1000) |
+| `resource_cleanup_interval` | Integer | `80` | Browser restart interval in URLs processed (1-1000) |
+| `cache_path`         | String | `".cache"` | Directory path for persistent cache storage |
+| `cache_max_size`     | Integer | `5000` | Maximum number of entries in cache |
+| `cache_autosave_minutes` | Integer | `1` | Interval for automatic cache saves (minutes) |
+| `cache_requests`     | Boolean | `false` | Enable HTTP request response caching |
 
 ---
 

package/lib/cdp.js

@@ -27,6 +27,36 @@
 
 const { formatLogMessage } = require('./colorize');
 
+/**
+ * Creates a new page with timeout protection to prevent CDP hangs
+ * @param {import('puppeteer').Browser} browser - Browser instance
+ * @param {number} timeout - Timeout in milliseconds (default: 30000)
+ * @returns {Promise<import('puppeteer').Page>} Page instance
+ */
+async function createPageWithTimeout(browser, timeout = 30000) {
+  return Promise.race([
+    browser.newPage(),
+    new Promise((_, reject) => 
+      setTimeout(() => reject(new Error('Page creation timeout - browser may be unresponsive')), timeout)
+    )
+  ]);
+}
+
+/**
+ * Sets request interception with timeout protection
+ * @param {import('puppeteer').Page} page - Page instance
+ * @param {number} timeout - Timeout in milliseconds (default: 15000)
+ * @returns {Promise<void>}
+ */
+async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
+  return Promise.race([
+    page.setRequestInterception(true),
+    new Promise((_, reject) => 
+      setTimeout(() => reject(new Error('Request interception setup timeout')), timeout)
+    )
+  ]);
+}
+
 /**
  * Creates and manages a CDP session for network monitoring
  * 
@@ -88,8 +118,13 @@ async function createCDPSession(page, currentUrl, options = {}) {
 
   try {
     // Create CDP session using modern Puppeteer 20+ API
-    // page.target().createCDPSession() was deprecated in 19+ and removed in 20+
-    cdpSession = await page.createCDPSession();
+    // Add timeout protection for CDP session creation
+    cdpSession = await Promise.race([
+      page.createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('CDP session creation timeout')), 20000)
+      )
+    ]);
     
     // Enable network domain - required for network event monitoring  
     await cdpSession.send('Network.enable');
@@ -302,6 +337,8 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
 // - Some features may not work in --no-sandbox mode
 module.exports = {
   createCDPSession,
+  createPageWithTimeout,
+  setRequestInterceptionWithTimeout,
   validateCDPConfig,
   createEnhancedCDPSession
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.94 ===
+// === Network scanner script (nwss.js) v1.0.96 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -33,7 +33,7 @@ const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvaila
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // CDP functionality
-const { createCDPSession } = require('./lib/cdp');
+const { createCDPSession, createPageWithTimeout, setRequestInterceptionWithTimeout } = require('./lib/cdp');
 // Post-processing cleanup
 const { processResults } = require('./lib/post-processing');
 // Colorize various text when used
@@ -125,7 +125,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.94'; // Script version
+const VERSION = '1.0.96'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1465,7 +1465,7 @@ function setupFrameHandling(page, forceDebug) {
       if (browserInstance.process() && browserInstance.process().killed) {
         throw new Error('Browser process was killed - restart required');
       }
-      page = await browserInstance.newPage();
+      page = await createPageWithTimeout(browserInstance, 30000);
 
       // Enhanced page validation for Puppeteer 23.x
       if (!page || page.isClosed()) {
@@ -1584,8 +1584,16 @@ function setupFrameHandling(page, forceDebug) {
           // Strategy 2: Try injection with reduced complexity if browser is responsive
           if (browserResponsive) {
               try {
+                  // Add comprehensive timeout protection for evaluateOnNewDocument
                   await Promise.race([
+                      // Main injection with all safety checks
                       page.evaluateOnNewDocument(() => {
+                          // Wrap everything in try-catch to prevent page crashes
+                          try {
+                              // Add timeout check within the injection
+                              const injectionTimeout = setTimeout(() => {
+                                  console.log('[evalOnDoc] Injection taking too long, aborting');
+                              }, 3000);
                   // Prevent infinite reload loops
                   let reloadCount = 0;
                   const MAX_RELOADS = 2;
@@ -1646,19 +1654,39 @@ function setupFrameHandling(page, forceDebug) {
                           return originalXHROpen.apply(this, arguments);
                       }
                   };
+                              clearTimeout(injectionTimeout);
+                          } catch (injectionError) {
+                              console.log('[evalOnDoc][error]', 'Injection failed:', injectionError.message);
+                          }
               }),
-                      new Promise((_, reject) => 
-                          setTimeout(() => reject(new Error('Injection timeout')), 8000)
-                      )
+                      // Reduced timeout for faster failure
+                      new Promise((_, reject) => {
+                          setTimeout(() => {
+                              reject(new Error('evaluateOnNewDocument timeout - browser may be unresponsive'));
+                          }, 5000); // Reduced from 8000ms
+                      })
                   ]);
                   evalOnDocSuccess = true;
                   if (forceDebug) {
                       console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
                   }
               } catch (fullInjectionErr) {
+                  // Enhanced error detection for CDP issues
+                  const isCDPError = fullInjectionErr.constructor.name === 'ProtocolError' ||
+                                    fullInjectionErr.name === 'ProtocolError' ||
+                                    fullInjectionErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+                                    fullInjectionErr.message.includes('Protocol error');
+                  
                   if (forceDebug) {
-                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
+                      const errorType = isCDPError ? 'CDP/Protocol error' : 'timeout/other';
+                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed (${errorType}): ${fullInjectionErr.message}`));
                   }
+
+                  // Skip fallback for CDP errors - they indicate browser communication issues
+                  if (isCDPError) {
+                      console.warn(formatLogMessage('warn', `[evalOnDoc] CDP communication failure - skipping injection for ${currentUrl}`));
+                      evalOnDocSuccess = false;
+                  } else {
                   
                   // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
                   try {
@@ -1692,6 +1720,7 @@ function setupFrameHandling(page, forceDebug) {
                       evalOnDocSuccess = false;
                   }
               }
+           } 
           } else {
               if (forceDebug) {
                   console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
@@ -1750,13 +1779,8 @@ function setupFrameHandling(page, forceDebug) {
 
       // Protected request interception setup with timeout
       try {
-        // Test if network operations are responsive before enabling request interception
-        await Promise.race([
-          page.setRequestInterception(true),
-          new Promise((_, reject) => 
-            setTimeout(() => reject(new Error('Network.enable timeout')), 10000)
-          )
-        ]);
+        // Use timeout-protected request interception setup
+        await setRequestInterceptionWithTimeout(page, 15000);
         
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Request interception enabled successfully for ${currentUrl}`));
@@ -1765,13 +1789,13 @@ function setupFrameHandling(page, forceDebug) {
         if (networkErr.message.includes('timed out') || 
             networkErr.message.includes('Network.enable') || 
             networkErr.message.includes('timeout')) {
-          console.warn(formatLogMessage('warn', `Network setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
+          console.warn(formatLogMessage('warn', `Request interception setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
           return { 
             url: currentUrl, 
             rules: [], 
             success: false, 
             needsImmediateRestart: true,
-            error: 'Network.enable timeout - browser restart required'
+            error: 'Request interception timeout - browser restart required'
           };
         }
         throw networkErr; // Re-throw other errors
@@ -2984,18 +3008,45 @@ function setupFrameHandling(page, forceDebug) {
       // Fallback to standard reload if force reload failed or wasn't attempted
       if (!reloadSuccess) {
         try {
-          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
+          // Reduced timeout for faster failure detection
+          const standardReloadTimeout = Math.min(timeout, 8000); // Reduced from 15000ms
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: standardReloadTimeout });
           if (forceDebug) console.log(formatLogMessage('debug', `Standard reload #${i} completed for ${currentUrl}`));
         } catch (standardReloadErr) {
           console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+          
+          // Check if this is a persistent failure that should skip remaining reloads
+          const isPersistentFailure = standardReloadErr.message.includes('Navigation timeout') || 
+                                     standardReloadErr.message.includes('net::ERR_') ||
+                                     standardReloadErr.message.includes('Protocol error') ||
+                                     standardReloadErr.message.includes('Page crashed') ||
+                                     // CDP and injection failures
+                                     standardReloadErr.constructor.name === 'ProtocolError' ||
+                                     standardReloadErr.name === 'ProtocolError' ||
+                                     standardReloadErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+                                     standardReloadErr.message.includes('Runtime.callFunctionOn timed out') ||
+                                     standardReloadErr.message.includes('CDP injection timeout');
+          
+          if (isPersistentFailure) {
+            const remainingReloads = totalReloads - i;
+            if (remainingReloads > 0 && forceDebug) {
+              console.log(formatLogMessage('debug', `Persistent failure detected - skipping ${remainingReloads} remaining reload(s) for ${currentUrl}`));
+            }
+            // Break out of reload loop to move to next URL faster
+            break;
+          }
           }
         } else {
           // Regular reload
           await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
         }
-        
+
+      // Only add delay if we're continuing with more reloads
+      if (i < totalReloads) {
+
         await fastTimeout(delayMs);
       }
+    }
 
       if (dryRunMode) {
         // Get page title for dry run output

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.94",
+  "version": "1.0.96",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {