@fanboynz/network-scanner 1.0.90 → 1.0.92

package/lib/interaction.js

@@ -84,7 +84,7 @@ const MOUSE_MOVEMENT = {
   MAX_DELAY: 25,             // Maximum milliseconds between movement steps
   DEFAULT_CURVE: 0.2,        // Default curve intensity (reduced for performance)
   DEFAULT_JITTER: 2,         // Default random jitter in pixels
-  DISTANCE_STEP_RATIO: 50,   // Pixels per step (FIXED: increased from 10 to prevent excessive steps)
+  DISTANCE_STEP_RATIO: 200,  // CRITICAL: 4x increase to drastically reduce steps
   CURVE_INTENSITY_RATIO: 0.01 // Multiplier for curve calculation
 };
 
@@ -339,19 +339,19 @@ async function humanLikeMouseMove(page, fromX, fromY, toX, toY, options = {}) {
   // FIXED: More aggressive step capping to prevent excessive delays
   let actualSteps;
   if (options.steps) {
-    // Use explicit steps parameter but cap at MAX_STEPS
-    actualSteps = Math.min(options.steps, MOUSE_MOVEMENT.MAX_STEPS);
+    // CRITICAL: Much lower step cap to prevent timeouts
+    actualSteps = Math.min(options.steps, 8); // Was MAX_STEPS (30), now max 8
   } else {
     // Calculate steps based on distance with strict limits
     const calculatedSteps = Math.floor(distance / MOUSE_MOVEMENT.DISTANCE_STEP_RATIO);
     actualSteps = Math.max(
-      MOUSE_MOVEMENT.MIN_STEPS,
-      Math.min(calculatedSteps, MOUSE_MOVEMENT.MAX_STEPS)
+      2,  // Min 2 steps instead of 5
+      Math.min(calculatedSteps, 6)  // Max 6 steps instead of 30
     );
   }
 
-  // FIXED: Add performance safeguard - never exceed 2 seconds total
-  const maxTotalTime = 2000; // 2 seconds maximum
+  // CRITICAL: Emergency timeout - never exceed 300ms for mouse movement
+  const maxTotalTime = 300; // 300ms maximum (was 2000ms)
   const estimatedTime = actualSteps * maxDelay;
   if (estimatedTime > maxTotalTime) {
     actualSteps = Math.floor(maxTotalTime / maxDelay);
@@ -787,6 +787,14 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
   } = options;
 
   try {
+    // CRITICAL: Emergency timeout wrapper for entire interaction
+    const MAX_INTERACTION_TIME = 15000; // 15 seconds absolute maximum
+    const interactionStartTime = Date.now();
+    
+    const checkTimeout = () => {
+      return Date.now() - interactionStartTime > MAX_INTERACTION_TIME;
+    };
+
     // Validate page state before starting interaction
     try {
       // Optimized timeout calculation - shorter for better performance
@@ -845,20 +853,26 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
       return; // Exit gracefully if mouse operations fail
     }
 
-    const startTime = Date.now();
+
     const totalDuration = duration * settings.pauseMultiplier;
-    const actionInterval = totalDuration / (actualMovements + (includeScrolling ? settings.scrolls : 0));
+    // CRITICAL: Cap action intervals to prevent long waits
+    const baseInterval = totalDuration / (actualMovements + (includeScrolling ? settings.scrolls : 0));
+    const actionInterval = Math.min(baseInterval, 100); // Never wait more than 100ms
+    
+    // Start timing ONLY the actual interaction operations  
+    const actualInteractionStartTime = Date.now();
 
     // Perform mouse movements
     for (let i = 0; i < actualMovements; i++) {
+      if (checkTimeout()) break; // Emergency timeout check
       const targetPos = generateRandomCoordinates(maxX, maxY, { 
         avoidCenter: i % 2 === 0,
         preferEdges: i % 3 === 0 
       });
 
       await humanLikeMouseMove(page, currentPos.x, currentPos.y, targetPos.x, targetPos.y, {
-          steps: 8 + Math.floor(Math.random() * 10), // FIXED: Reduced step range (was 10-25, now 8-18)
-          curve: 0.1 + Math.random() * 0.2, // FIXED: Reduced curve intensity
+          steps: 3 + Math.floor(Math.random() * 4), // CRITICAL: 3-6 steps (was 8-18)
+          curve: 0.05 + Math.random() * 0.05, // CRITICAL: Minimal curve
         jitter: 1 + Math.random() * 2
       });
 
@@ -866,24 +880,25 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
 
       // Occasional pause
       if (Math.random() < PROBABILITIES.PAUSE_CHANCE) {
-        await fastTimeout(TIMING.CLICK_PAUSE_MIN + Math.random() * TIMING.POST_CLICK_MIN);
+        await fastTimeout(25 + Math.random() * 50); // CRITICAL: Much shorter pauses
       }
 
       // Time-based spacing
-      await fastTimeout(actionInterval);
+      await fastTimeout(Math.min(actionInterval, 50)); // CRITICAL: Cap at 50ms
     }
 
     // Scrolling simulation
     if (includeScrolling) {
       for (let i = 0; i < settings.scrolls; i++) {
+        if (checkTimeout()) break; // Emergency timeout check
         const direction = Math.random() < PROBABILITIES.SCROLL_DOWN_BIAS ? 'down' : 'up';
         await simulateScrolling(page, {
           direction,
-          amount: 2 + Math.floor(Math.random() * 3),
-          smoothness: 3 + Math.floor(Math.random() * 4)
+          amount: 1 + Math.floor(Math.random() * 2), // CRITICAL: Less scrolling
+          smoothness: 1 + Math.floor(Math.random() * 2) // CRITICAL: Much less smooth
         });
         
-        await fastTimeout(actionInterval);
+        await fastTimeout(Math.min(actionInterval, 100)); // CRITICAL: Cap intervals
       }
     }
 
@@ -912,9 +927,16 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
       // Silently handle hover failures - not critical
     }
 
-    const elapsedTime = Date.now() - startTime;
+    // End timing ONLY after actual interaction operations complete
+    const interactionElapsedTime = Date.now() - actualInteractionStartTime
+
+    // CRITICAL: Warn about slow interactions
+    if (interactionElapsedTime > 8000) {
+      console.warn(`[interaction] WARNING: Interaction took ${interactionElapsedTime}ms for ${currentUrl}`);
+    }
+
     if (forceDebug) {
-      console.log(`[interaction] Completed interaction simulation in ${elapsedTime}ms (${actualMovements} movements, ${includeScrolling ? settings.scrolls : 0} scrolls)`);
+      console.log(`[interaction] Completed interaction simulation in ${interactionElapsedTime}ms (${actualMovements} movements, ${includeScrolling ? settings.scrolls : 0} scrolls)`);
     }
 
   } catch (interactionErr) {

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.90 ===
+// === Network scanner script (nwss.js) v1.0.91 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -45,9 +45,6 @@ const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } =
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
 
-// Evaluate on new document functionality
-const { applyEvaluateOnNewDocument } = require('./lib/evaldocument');
-
 // Fast setTimeout helper for Puppeteer 22.x compatibility
 // Uses standard Promise constructor for better performance than node:timers/promises
 function fastTimeout(ms) {
@@ -128,7 +125,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.90'; // Script version
+const VERSION = '1.0.91'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1555,9 +1552,158 @@ function setupFrameHandling(page, forceDebug) {
 
       // --- START: evaluateOnNewDocument for Fetch/XHR Interception (Moved and Fixed) ---
       // This script is injected if --eval-on-doc is used or siteConfig.evaluateOnNewDocument is true.
-      const evalOnDocSuccess = await applyEvaluateOnNewDocument(
-        page, currentUrl, siteConfig, globalEvalOnDoc, forceDebug, formatLogMessage
-      );
+      const shouldInjectEvalForPage = siteConfig.evaluateOnNewDocument === true || globalEvalOnDoc;
+      let evalOnDocSuccess = false; // Track injection success for fallback logic
+      
+      if (shouldInjectEvalForPage) {
+          if (forceDebug) {
+              if (globalEvalOnDoc) {
+                  console.log(formatLogMessage('debug', `[evalOnDoc] Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
+              } else { // siteConfig.evaluateOnNewDocument must be true
+                  console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
+              }
+          }
+          
+          // Strategy 1: Try full injection with health check
+          let browserResponsive = false;
+          try {
+              await Promise.race([
+                  browserInstance.version(), // Quick responsiveness test
+                  new Promise((_, reject) => 
+                      setTimeout(() => reject(new Error('Browser health check timeout')), 3000)
+                  )
+              ]);
+              browserResponsive = true;
+          } catch (healthErr) {
+              if (forceDebug) {
+                  console.log(formatLogMessage('debug', `[evalOnDoc] Browser health check failed: ${healthErr.message}`));
+              }
+              browserResponsive = false;
+          }
+          
+          // Strategy 2: Try injection with reduced complexity if browser is responsive
+          if (browserResponsive) {
+              try {
+                  await Promise.race([
+                      page.evaluateOnNewDocument(() => {
+                  // Prevent infinite reload loops
+                  let reloadCount = 0;
+                  const MAX_RELOADS = 2;
+                  const originalReload = window.location.reload;
+                  const originalReplace = window.location.replace;
+                  const originalAssign = window.location.assign;
+                  
+                  window.location.reload = function() {
+                      if (++reloadCount > MAX_RELOADS) {
+                          console.log('[loop-protection] Blocked excessive reload attempt');
+                          return;
+                      }
+                      return originalReload.apply(this, arguments);
+                  };
+                  
+                  // Also protect against location.replace/assign to same URL
+                  const currentHref = window.location.href;
+                  window.location.replace = function(url) {
+                      if (url === currentHref && ++reloadCount > MAX_RELOADS) {
+                          console.log('[loop-protection] Blocked same-page replace attempt');
+                          return;
+                      }
+                      return originalReplace.apply(this, arguments);
+                  };
+
+                  // This script intercepts and logs Fetch and XHR requests
+                  // from within the page context at the earliest possible moment.
+                  const originalFetch = window.fetch;
+                  window.fetch = (...args) => {
+                      try {
+                          console.log('[evalOnDoc][fetch]', args[0]); // Log fetch requests
+                          const fetchPromise = originalFetch.apply(this, args);
+                          
+                          // Add network error handling to prevent page errors
+                          return fetchPromise.catch(fetchErr => {
+                              console.log('[evalOnDoc][fetch-error]', args[0], fetchErr.message);
+                              throw fetchErr; // Re-throw to maintain normal error flow
+                          });
+                      } catch (fetchWrapperErr) {
+                          console.log('[evalOnDoc][fetch-wrapper-error]', fetchWrapperErr.message);
+                          return originalFetch.apply(this, args);
+                      }
+                  };
+
+                  const originalXHROpen = XMLHttpRequest.prototype.open;
+                  XMLHttpRequest.prototype.open = function (method, xhrUrl) {
+                      try {
+                          console.log('[evalOnDoc][xhr]', xhrUrl); // Log XHR requests
+                          
+                          // Add error handling for XHR
+                          this.addEventListener('error', function(event) {
+                              console.log('[evalOnDoc][xhr-error]', xhrUrl, 'Network error occurred');
+                          });
+                          
+                          return originalXHROpen.apply(this, arguments);
+                      } catch (xhrOpenErr) {
+                          console.log('[evalOnDoc][xhr-open-error]', xhrOpenErr.message);
+                          return originalXHROpen.apply(this, arguments);
+                      }
+                  };
+              }),
+                      new Promise((_, reject) => 
+                          setTimeout(() => reject(new Error('Injection timeout')), 8000)
+                      )
+                  ]);
+                  evalOnDocSuccess = true;
+                  if (forceDebug) {
+                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
+                  }
+              } catch (fullInjectionErr) {
+                  if (forceDebug) {
+                      console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
+                  }
+                  
+                  // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
+                  try {
+                      await Promise.race([
+                          page.evaluateOnNewDocument(() => {
+                              // Minimal injection - just fetch monitoring
+                              if (window.fetch) {
+                                  const originalFetch = window.fetch;
+                                  window.fetch = (...args) => {
+                                      try {
+                                          console.log('[evalOnDoc][fetch-minimal]', args[0]);
+                                          return originalFetch.apply(this, args);
+                                      } catch (err) {
+                                          return originalFetch.apply(this, args);
+                                      }
+                                  };
+                              }
+                          }),
+                          new Promise((_, reject) => 
+                              setTimeout(() => reject(new Error('Minimal injection timeout')), 3000)
+                          )
+                      ]);
+                      evalOnDocSuccess = true;
+                      if (forceDebug) {
+                          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection successful for ${currentUrl}`));
+                      }
+                  } catch (minimalInjectionErr) {
+                      if (forceDebug) {
+                          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection also failed: ${minimalInjectionErr.message}`));
+                      }
+                      evalOnDocSuccess = false;
+                  }
+              }
+          } else {
+              if (forceDebug) {
+                  console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
+              }
+              evalOnDocSuccess = false;
+          }
+          
+          // Final status logging
+          if (!evalOnDocSuccess) {
+              console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
+          }
+      }
       // --- END: evaluateOnNewDocument for Fetch/XHR Interception ---
 
       // --- CSS Element Blocking Setup ---
@@ -1608,7 +1754,7 @@ function setupFrameHandling(page, forceDebug) {
         await Promise.race([
           page.setRequestInterception(true),
           new Promise((_, reject) => 
-            setTimeout(() => reject(new Error('Network.enable timeout')), 8000)
+            setTimeout(() => reject(new Error('Network.enable timeout')), 10000)
           )
         ]);
         
@@ -1618,9 +1764,7 @@ function setupFrameHandling(page, forceDebug) {
       } catch (networkErr) {
         if (networkErr.message.includes('timed out') || 
             networkErr.message.includes('Network.enable') || 
-            networkErr.message.includes('timeout') ||
-            networkErr.constructor.name === 'ProtocolError' ||
-            networkErr.name === 'ProtocolError') {
+            networkErr.message.includes('timeout')) {
           console.warn(formatLogMessage('warn', `Network setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
           return { 
             url: currentUrl, 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.90",
+  "version": "1.0.92",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {

package/lib/evaldocument.js

@@ -1,193 +0,0 @@
-/**
- * Module for handling evaluateOnNewDocument functionality
- * Provides Fetch/XHR interception and page protection mechanisms
- */
-
-/**
- * Applies evaluateOnNewDocument script injection to a page
- * @param {import('puppeteer').Page} page - Puppeteer page instance
- * @param {string} currentUrl - Current URL being processed
- * @param {Object} siteConfig - Site configuration
- * @param {boolean} globalEvalOnDoc - Global eval-on-doc flag
- * @param {boolean} forceDebug - Debug logging flag
- * @param {Function} formatLogMessage - Log formatting function
- * @returns {Promise<boolean>} Success status of injection
- */
-async function applyEvaluateOnNewDocument(page, currentUrl, siteConfig, globalEvalOnDoc, forceDebug, formatLogMessage) {
-  const shouldInjectEvalForPage = siteConfig.evaluateOnNewDocument === true || globalEvalOnDoc;
-  let evalOnDocSuccess = false;
-  
-  if (!shouldInjectEvalForPage) {
-    return false;
-  }
-
-  if (forceDebug) {
-    if (globalEvalOnDoc) {
-      console.log(formatLogMessage('debug', `[evalOnDoc] Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
-    } else {
-      console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
-    }
-  }
-  
-  // Strategy 1: Try full injection with health check
-  let browserResponsive = false;
-  try {
-    await Promise.race([
-      page.browser().version(), // Quick responsiveness test
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Browser health check timeout')), 3000)
-      )
-    ]);
-    browserResponsive = true;
-  } catch (healthErr) {
-    if (forceDebug) {
-      console.log(formatLogMessage('debug', `[evalOnDoc] Browser health check failed: ${healthErr.message}`));
-    }
-    browserResponsive = false;
-  }
-  
-  // Strategy 2: Try injection with reduced complexity if browser is responsive
-  if (browserResponsive) {
-    try {
-      await Promise.race([
-        page.evaluateOnNewDocument(createFullInterceptionScript()),
-        new Promise((_, reject) => 
-          setTimeout(() => reject(new Error('Injection timeout')), 8000)
-        )
-      ]);
-      evalOnDocSuccess = true;
-      if (forceDebug) {
-        console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
-      }
-    } catch (fullInjectionErr) {
-      if (forceDebug) {
-        console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
-      }
-      
-      // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
-      try {
-        await Promise.race([
-          page.evaluateOnNewDocument(createMinimalInterceptionScript()),
-          new Promise((_, reject) => 
-            setTimeout(() => reject(new Error('Minimal injection timeout')), 3000)
-          )
-        ]);
-        evalOnDocSuccess = true;
-        if (forceDebug) {
-          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection successful for ${currentUrl}`));
-        }
-      } catch (minimalInjectionErr) {
-        if (forceDebug) {
-          console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection also failed: ${minimalInjectionErr.message}`));
-        }
-        evalOnDocSuccess = false;
-      }
-    }
-  } else {
-    if (forceDebug) {
-      console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
-    }
-    evalOnDocSuccess = false;
-  }
-  
-  // Final status logging
-  if (!evalOnDocSuccess) {
-    console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
-  }
-  
-  return evalOnDocSuccess;
-}
-
-/**
- * Creates the full interception script with all protections
- * @returns {Function} Script function for evaluateOnNewDocument
- */
-function createFullInterceptionScript() {
-  return () => {
-    // Prevent infinite reload loops
-    let reloadCount = 0;
-    const MAX_RELOADS = 2;
-    const originalReload = window.location.reload;
-    const originalReplace = window.location.replace;
-    const originalAssign = window.location.assign;
-    
-    window.location.reload = function() {
-      if (++reloadCount > MAX_RELOADS) {
-        console.log('[loop-protection] Blocked excessive reload attempt');
-        return;
-      }
-      return originalReload.apply(this, arguments);
-    };
-    
-    // Also protect against location.replace/assign to same URL
-    const currentHref = window.location.href;
-    window.location.replace = function(url) {
-      if (url === currentHref && ++reloadCount > MAX_RELOADS) {
-        console.log('[loop-protection] Blocked same-page replace attempt');
-        return;
-      }
-      return originalReplace.apply(this, arguments);
-    };
-
-    // This script intercepts and logs Fetch and XHR requests
-    // from within the page context at the earliest possible moment.
-    const originalFetch = window.fetch;
-    window.fetch = (...args) => {
-      try {
-        console.log('[evalOnDoc][fetch]', args[0]); // Log fetch requests
-        const fetchPromise = originalFetch.apply(this, args);
-        
-        // Add network error handling to prevent page errors
-        return fetchPromise.catch(fetchErr => {
-          console.log('[evalOnDoc][fetch-error]', args[0], fetchErr.message);
-          throw fetchErr; // Re-throw to maintain normal error flow
-        });
-      } catch (fetchWrapperErr) {
-        console.log('[evalOnDoc][fetch-wrapper-error]', fetchWrapperErr.message);
-        return originalFetch.apply(this, args);
-      }
-    };
-
-    const originalXHROpen = XMLHttpRequest.prototype.open;
-    XMLHttpRequest.prototype.open = function (method, xhrUrl) {
-      try {
-        console.log('[evalOnDoc][xhr]', xhrUrl); // Log XHR requests
-        
-        // Add error handling for XHR
-        this.addEventListener('error', function(event) {
-          console.log('[evalOnDoc][xhr-error]', xhrUrl, 'Network error occurred');
-        });
-        
-        return originalXHROpen.apply(this, arguments);
-      } catch (xhrOpenErr) {
-        console.log('[evalOnDoc][xhr-open-error]', xhrOpenErr.message);
-        return originalXHROpen.apply(this, arguments);
-      }
-    };
-  };
-}
-
-/**
- * Creates the minimal interception script (fetch only)
- * @returns {Function} Script function for evaluateOnNewDocument
- */
-function createMinimalInterceptionScript() {
-  return () => {
-    // Minimal injection - just fetch monitoring
-    if (window.fetch) {
-      const originalFetch = window.fetch;
-      window.fetch = (...args) => {
-        try {
-          console.log('[evalOnDoc][fetch-minimal]', args[0]);
-          return originalFetch.apply(this, args);
-        } catch (err) {
-          return originalFetch.apply(this, args);
-        }
-      };
-    }
-  };
-}
-
-module.exports = {
-  applyEvaluateOnNewDocument
-};