@fanboynz/network-scanner 1.0.93 → 1.0.95

package/README.md

@@ -73,10 +73,13 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 
 | Argument                  | Description |
 |:---------------------------|:------------|
+| `--cache-requests`          | Cache HTTP requests to avoid re-requesting same URLs within scan |
 | `--validate-config`         | Validate config.json file and exit |
 | `--validate-rules [file]`   | Validate rule file format (uses --output/--compare files if no file specified) |
 | `--clean-rules [file]`      | Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified) |
 | `--test-validation`         | Run domain validation tests and exit |
+| `--clear-cache`             | Clear persistent cache before scanning (improves fresh start performance) |
+| `--ignore-cache`            | Bypass all smart caching functionality during scanning |
 
 ---
 
@@ -133,6 +136,7 @@ Example:
 | `url`                | String or Array |   -     | Website URL(s) to scan |
 | `userAgent`          | `chrome`, `firefox`, `safari` | - | User agent for page (latest versions: Chrome 131, Firefox 133, Safari 18.2) |
 | `filterRegex`        | String or Array | `.*` | Regex or list of regexes to match requests |
+| `regex_and`          | Boolean | `false` | Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL |
 | `comments`           | String or Array | - | String of comments or references |
 | `resourceTypes`      | Array | `["script", "xhr", "image", "stylesheet"]` | What resource types to monitor |
 | `reload`             | Integer | `1` | Number of times to reload page |
@@ -146,6 +150,8 @@ Example:
 | `subDomains`         | `0` or `1` | `0` | 1 = preserve subdomains in output |
 | `blocked`            | Array | - | Domains or regexes to block during scanning |
 | `even_blocked`       | Boolean | `false` | Add matching rules even if requests are blocked |
+| `bypass_cache`       | Boolean | `false` | Skip all caching for this site's URLs |
+
 
 ### Redirect Handling Options
 
@@ -190,6 +196,10 @@ When a page redirects to a new domain, first-party/third-party detection is base
 |:---------------------|:-------|:-------:|:------------|
 | `cloudflare_phish`   | Boolean | `false` | Auto-click through Cloudflare phishing warnings |
 | `cloudflare_bypass`  | Boolean | `false` | Auto-solve Cloudflare "Verify you are human" challenges |
+| `cloudflare_parallel_detection` | Boolean | `true` | Use parallel detection for faster Cloudflare checks |
+| `cloudflare_max_retries` | Integer | `3` | Maximum retry attempts for Cloudflare operations |
+| `cloudflare_cache_ttl` | Milliseconds | `300000` | TTL for Cloudflare detection cache (5 minutes) |
+| `cloudflare_retry_on_error` | Boolean | `true` | Enable retry logic for Cloudflare operations |
 | `flowproxy_detection` | Boolean | `false` | Enable flowProxy protection detection and handling |
 | `flowproxy_page_timeout` | Milliseconds | `45000` | Page timeout for flowProxy sites |
 | `flowproxy_nav_timeout` | Milliseconds | `45000` | Navigation timeout for flowProxy sites |
@@ -240,6 +250,11 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |
 | `headful`            | Boolean | `false` | Launch browser with GUI for this site |
 | `adblock_rules`      | Boolean | `false` | Generate adblock filter rules with resource types for this site |
+| `interact_duration`  | Milliseconds | `2000` | Duration of interaction simulation |
+| `interact_scrolling` | Boolean | `true` | Enable scrolling simulation |
+| `interact_clicks`    | Boolean | `false` | Enable element clicking simulation |
+| `interact_typing`    | Boolean | `false` | Enable typing simulation |
+| `interact_intensity` | String | `"medium"` | Interaction simulation intensity: "low", "medium", "high" |
 
 ### Global Configuration Options
 
@@ -254,7 +269,11 @@ These options go at the root level of your config.json:
 | `ignore_similar_threshold` | Integer | `80` | Similarity threshold percentage for ignore_similar |
 | `ignore_similar_ignored_domains` | Boolean | `true` | Ignore domains similar to ignoreDomains list |
 | `max_concurrent_sites` | Integer | `6` | Maximum concurrent site processing (1-50) |
-| `resource_cleanup_interval` | Integer | `180` | Browser restart interval in URLs processed (1-1000) |
+| `resource_cleanup_interval` | Integer | `80` | Browser restart interval in URLs processed (1-1000) |
+| `cache_path`         | String | `".cache"` | Directory path for persistent cache storage |
+| `cache_max_size`     | Integer | `5000` | Maximum number of entries in cache |
+| `cache_autosave_minutes` | Integer | `1` | Interval for automatic cache saves (minutes) |
+| `cache_requests`     | Boolean | `false` | Enable HTTP request response caching |
 
 ---
 

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.91 ===
+// === Network scanner script (nwss.js) v1.0.95 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -125,7 +125,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.91'; // Script version
+const VERSION = '1.0.95'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1584,8 +1584,16 @@ function setupFrameHandling(page, forceDebug) {
           // Strategy 2: Try injection with reduced complexity if browser is responsive
           if (browserResponsive) {
               try {
+                  // Add comprehensive timeout protection for evaluateOnNewDocument
                   await Promise.race([
+                      // Main injection with all safety checks
                       page.evaluateOnNewDocument(() => {
+                          // Wrap everything in try-catch to prevent page crashes
+                          try {
+                              // Add timeout check within the injection
+                              const injectionTimeout = setTimeout(() => {
+                                  console.log('[evalOnDoc] Injection taking too long, aborting');
+                              }, 3000);
                   // Prevent infinite reload loops
                   let reloadCount = 0;
                   const MAX_RELOADS = 2;
@@ -1646,19 +1654,39 @@ function setupFrameHandling(page, forceDebug) {
                           return originalXHROpen.apply(this, arguments);
                       }
                   };
+                              clearTimeout(injectionTimeout);
+                          } catch (injectionError) {
+                              console.log('[evalOnDoc][error]', 'Injection failed:', injectionError.message);
+                          }
               }),
-                      new Promise((_, reject) => 
-                          setTimeout(() => reject(new Error('Injection timeout')), 8000)
-                      )
+                      // Reduced timeout for faster failure
+                      new Promise((_, reject) => {
+                          setTimeout(() => {
+                              reject(new Error('evaluateOnNewDocument timeout - browser may be unresponsive'));
+                          }, 5000); // Reduced from 8000ms
+                      })
                   ]);
                   evalOnDocSuccess = true;
                   if (forceDebug) {
                       console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
                   }
               } catch (fullInjectionErr) {
+                  // Enhanced error detection for CDP issues
+                  const isCDPError = fullInjectionErr.constructor.name === 'ProtocolError' ||
+                                    fullInjectionErr.name === 'ProtocolError' ||
+                                    fullInjectionErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+                                    fullInjectionErr.message.includes('Protocol error');
+                  
                   if (forceDebug) {
-                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
+                      const errorType = isCDPError ? 'CDP/Protocol error' : 'timeout/other';
+                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed (${errorType}): ${fullInjectionErr.message}`));
                   }
+
+                  // Skip fallback for CDP errors - they indicate browser communication issues
+                  if (isCDPError) {
+                      console.warn(formatLogMessage('warn', `[evalOnDoc] CDP communication failure - skipping injection for ${currentUrl}`));
+                      evalOnDocSuccess = false;
+                  } else {
                   
                   // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
                   try {
@@ -1692,6 +1720,7 @@ function setupFrameHandling(page, forceDebug) {
                       evalOnDocSuccess = false;
                   }
               }
+           } 
           } else {
               if (forceDebug) {
                   console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
@@ -2950,23 +2979,79 @@ function setupFrameHandling(page, forceDebug) {
           }
         }
         
-        if (useForceReload) {
-          // Force reload: disable cache, reload, re-enable cache
+      let reloadSuccess = false;
+      
+      if (useForceReload && !reloadSuccess) {
+        // Attempt force reload: disable cache, reload, re-enable cache
           try {
-            await page.setCacheEnabled(false);
-            await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
-            await page.setCacheEnabled(true);
+          // Add timeout protection for setCacheEnabled operations
+          await Promise.race([
+            page.setCacheEnabled(false),
+            new Promise((_, reject) => 
+              setTimeout(() => reject(new Error('setCacheEnabled(false) timeout')), 5000)
+            )
+          ]);
+          
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
+          
+          await Promise.race([
+            page.setCacheEnabled(true),
+            new Promise((_, reject) => 
+              setTimeout(() => reject(new Error('setCacheEnabled(true) timeout')), 5000)
+            )
+          ]);
+          
+          reloadSuccess = true;
             if (forceDebug) console.log(formatLogMessage('debug', `Force reload #${i} completed for ${currentUrl}`));
+
           } catch (forceReloadErr) {
-            console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message}`));
+          console.warn(messageColors.warn(`[force reload #${i} failed] ${currentUrl}: ${forceReloadErr.message} - falling back to standard reload`));
+          reloadSuccess = false; // Ensure we try standard reload
+        }
+      }
+      
+      // Fallback to standard reload if force reload failed or wasn't attempted
+      if (!reloadSuccess) {
+        try {
+          // Reduced timeout for faster failure detection
+          const standardReloadTimeout = Math.min(timeout, 8000); // Reduced from 15000ms
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: standardReloadTimeout });
+          if (forceDebug) console.log(formatLogMessage('debug', `Standard reload #${i} completed for ${currentUrl}`));
+        } catch (standardReloadErr) {
+          console.warn(messageColors.warn(`[standard reload #${i} failed] ${currentUrl}: ${standardReloadErr.message}`));
+          
+          // Check if this is a persistent failure that should skip remaining reloads
+          const isPersistentFailure = standardReloadErr.message.includes('Navigation timeout') || 
+                                     standardReloadErr.message.includes('net::ERR_') ||
+                                     standardReloadErr.message.includes('Protocol error') ||
+                                     standardReloadErr.message.includes('Page crashed') ||
+                                     // CDP and injection failures
+                                     standardReloadErr.constructor.name === 'ProtocolError' ||
+                                     standardReloadErr.name === 'ProtocolError' ||
+                                     standardReloadErr.message.includes('addScriptToEvaluateOnNewDocument timed out') ||
+                                     standardReloadErr.message.includes('Runtime.callFunctionOn timed out') ||
+                                     standardReloadErr.message.includes('CDP injection timeout');
+          
+          if (isPersistentFailure) {
+            const remainingReloads = totalReloads - i;
+            if (remainingReloads > 0 && forceDebug) {
+              console.log(formatLogMessage('debug', `Persistent failure detected - skipping ${remainingReloads} remaining reload(s) for ${currentUrl}`));
+            }
+            // Break out of reload loop to move to next URL faster
+            break;
+          }
           }
         } else {
           // Regular reload
           await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 15000) });
         }
-        
+
+      // Only add delay if we're continuing with more reloads
+      if (i < totalReloads) {
+
         await fastTimeout(delayMs);
       }
+    }
 
       if (dryRunMode) {
         // Get page title for dry run output

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.93",
+  "version": "1.0.95",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {