@fanboynz/network-scanner 1.0.71 → 1.0.72

package/lib/cloudflare.js

@@ -1,13 +1,13 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
- * Version: 2.1.0 - Enhanced with quick detection, adaptive timeouts, and comprehensive debug logging
+ * Version: 2.2.0 - Enhanced with retry logic, caching, and improved error handling
  * Handles phishing warnings, Turnstile challenges, and modern Cloudflare protections
  */
 
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.1.0';
+const CLOUDFLARE_MODULE_VERSION = '2.2.0';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -29,7 +29,12 @@ const TIMEOUTS = {
   ADAPTIVE_TIMEOUT_WITH_INDICATORS: 25000,    // Adaptive timeout when indicators found + explicit config
   ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS: 20000, // Adaptive timeout with explicit config only
   ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS: 15000,   // Adaptive timeout for auto-detected with indicators
-  ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000 // Adaptive timeout for auto-detected without indicators
+  ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000, // Adaptive timeout for auto-detected without indicators
+  // New timeouts for enhanced functionality
+  RETRY_DELAY: 1000,              // Delay between retry attempts
+  MAX_RETRIES: 3,                 // Maximum retry attempts for operations
+  CHALLENGE_POLL_INTERVAL: 500,   // Interval for polling challenge completion
+  CHALLENGE_MAX_POLLS: 20         // Maximum polling attempts
 };
 
 // Fast timeout constants - optimized for speed
@@ -45,6 +50,101 @@ const FAST_TIMEOUTS = {
   CHALLENGE_COMPLETION: 3000      // Fast completion check
 };
 
+/**
+ * Error categories for better handling
+ */
+const ERROR_TYPES = {
+  NETWORK: 'network',
+  TIMEOUT: 'timeout',
+  ELEMENT_NOT_FOUND: 'element_not_found',
+  EVALUATION_FAILED: 'evaluation_failed',
+  NAVIGATION_FAILED: 'navigation_failed',
+  UNKNOWN: 'unknown'
+};
+
+/**
+ * Retry configuration with exponential backoff
+ */
+const RETRY_CONFIG = {
+  maxAttempts: 3,
+  baseDelay: 1000,
+  maxDelay: 8000,
+  backoffMultiplier: 2,
+  retryableErrors: [ERROR_TYPES.NETWORK, ERROR_TYPES.TIMEOUT, ERROR_TYPES.ELEMENT_NOT_FOUND]
+};
+
+/**
+ * Performance cache for detection results
+ * Stores detection results per domain to avoid redundant checks
+ */
+class CloudflareDetectionCache {
+  constructor(ttl = 300000) { // 5 minutes TTL by default
+    this.cache = new Map();
+    this.ttl = ttl;
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  getCacheKey(url) {
+    try {
+      const urlObj = new URL(url);
+      return `${urlObj.hostname}${urlObj.pathname}`;
+    } catch {
+      return url;
+    }
+  }
+
+  get(url) {
+    const key = this.getCacheKey(url);
+    const cached = this.cache.get(key);
+    
+    if (cached && Date.now() - cached.timestamp < this.ttl) {
+      this.hits++;
+      return cached.data;
+    }
+    
+    if (cached) {
+      this.cache.delete(key); // Remove expired entry
+    }
+    
+    this.misses++;
+    return null;
+  }
+
+  set(url, data) {
+    const key = this.getCacheKey(url);
+    this.cache.set(key, {
+      data,
+      timestamp: Date.now()
+    });
+    
+    // Prevent cache from growing too large
+    if (this.cache.size > 1000) {
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+  }
+
+  clear() {
+    this.cache.clear();
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  getStats() {
+    const total = this.hits + this.misses;
+    return {
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? (this.hits / total * 100).toFixed(2) + '%' : '0%',
+      size: this.cache.size
+    };
+  }
+}
+
+// Initialize cache singleton
+const detectionCache = new CloudflareDetectionCache();
+
 /**
  * Gets module version information
  * @returns {object} Version information object
@@ -108,27 +208,96 @@ async function waitForTimeout(page, timeout) {
 }
 
 /**
- * Safe page evaluation with timeout protection
+ * Categorizes errors for better handling
  */
-async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE) {
-  try {
-    return await Promise.race([
-      page.evaluate(func),
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('Page evaluation timeout')), timeout)
-      )
-    ]);
-  } catch (error) {
-    console.warn(`[cloudflare] Page evaluation failed: ${error.message}`);
-    return {
-      isChallengePresent: false,
-      isPhishingWarning: false,
-      isTurnstile: false,
-      isJSChallenge: false,
-      isChallengeCompleted: false,
-      error: error.message
-    };
+function categorizeError(error) {
+  const errorMessage = error.message || '';
+  
+  if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
+    return ERROR_TYPES.TIMEOUT;
   }
+  if (errorMessage.includes('Protocol error') || errorMessage.includes('Target closed')) {
+    return ERROR_TYPES.NETWORK;
+  }
+  if (errorMessage.includes('evaluation') || errorMessage.includes('Evaluation')) {
+    return ERROR_TYPES.EVALUATION_FAILED;
+  }
+  if (errorMessage.includes('navigation') || errorMessage.includes('Navigation')) {
+    return ERROR_TYPES.NAVIGATION_FAILED;
+  }
+  
+  return ERROR_TYPES.UNKNOWN;
+}
+
+/**
+ * Implements exponential backoff delay
+ */
+async function getRetryDelay(attempt) {
+  const delay = Math.min(
+    RETRY_CONFIG.baseDelay * Math.pow(RETRY_CONFIG.backoffMultiplier, attempt - 1),
+    RETRY_CONFIG.maxDelay
+  );
+  return new Promise(resolve => setTimeout(resolve, delay));
+}
+
+/**
+ * Enhanced safe page evaluation with retry logic and better error handling
+ */
+async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE, options = {}) {
+  const { maxRetries = RETRY_CONFIG.maxAttempts, forceDebug = false } = options;
+  let lastError = null;
+  
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      const result = await Promise.race([
+        page.evaluate(func),
+        new Promise((_, reject) => 
+          setTimeout(() => reject(new Error('Page evaluation timeout')), timeout)
+        )
+      ]);
+      
+      if (forceDebug && attempt > 1) {
+        console.log(`[cloudflare] Page evaluation succeeded on attempt ${attempt}`);
+      }
+      
+      return result;
+    } catch (error) {
+      lastError = error;
+      const errorType = categorizeError(error);
+      
+      if (forceDebug) {
+        console.warn(`[cloudflare] Page evaluation failed (attempt ${attempt}/${maxRetries}): ${error.message} [${errorType}]`);
+      }
+      
+      // Don't retry if error type is not retryable or if it's the last attempt
+      if (!RETRY_CONFIG.retryableErrors.includes(errorType) || attempt === maxRetries) {
+        return {
+          isChallengePresent: false,
+          isPhishingWarning: false,
+          isTurnstile: false,
+          isJSChallenge: false,
+          isChallengeCompleted: false,
+          error: error.message,
+          errorType: errorType,
+          attempts: attempt
+        };
+      }
+      
+      // Wait before retrying with exponential backoff
+      await getRetryDelay(attempt);
+    }
+  }
+  
+  return {
+    isChallengePresent: false,
+    isPhishingWarning: false,
+    isTurnstile: false,
+    isJSChallenge: false,
+    isChallengeCompleted: false,
+    error: lastError?.message || 'Unknown error',
+    errorType: categorizeError(lastError),
+    attempts: maxRetries
+  };
 }
 
 /**
@@ -165,7 +334,7 @@ async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT
 }
 
 /**
- * Quick Cloudflare detection - faster initial check to avoid unnecessary waiting
+ * Quick Cloudflare detection with caching for performance
  */
 async function quickCloudflareDetection(page, forceDebug = false) {
   try {
@@ -179,23 +348,34 @@ async function quickCloudflareDetection(page, forceDebug = false) {
       return { hasIndicators: false, skippedInvalidUrl: true };
     }
 
-    // Continue with existing detection logic only for valid HTTP(S) URLs
-    
+    // Check cache first
+    const cachedResult = detectionCache.get(currentPageUrl);
+    if (cachedResult !== null) {
+      if (forceDebug) {
+        const stats = detectionCache.getStats();
+        console.log(`[debug][cloudflare] Using cached detection result (cache hit rate: ${stats.hitRate})`);
+      }
+      return cachedResult;
+    }
+
+    // Perform actual detection with enhanced error handling
     const quickCheck = await safePageEvaluate(page, () => {
       const title = document.title || '';
       const bodyText = document.body ? document.body.textContent.substring(0, 500) : '';
       const url = window.location.href;
       
-      // Quick indicators of Cloudflare presence
+      // Enhanced indicators with 2025 patterns
       const hasCloudflareIndicators = 
         title.includes('Just a moment') ||
         title.includes('Checking your browser') ||
         title.includes('Attention Required') ||
+        title.includes('Security check') || // New pattern
         bodyText.includes('Cloudflare') ||
         bodyText.includes('cf-ray') ||
         bodyText.includes('Verify you are human') ||
         bodyText.includes('This website has been reported for potential phishing') ||
         bodyText.includes('Please wait while we verify') ||
+        bodyText.includes('Checking if the site connection is secure') || // New pattern
         url.includes('/cdn-cgi/challenge-platform/') ||
         url.includes('cloudflare.com') ||
         document.querySelector('[data-ray]') ||
@@ -207,7 +387,10 @@ async function quickCloudflareDetection(page, forceDebug = false) {
         document.querySelector('iframe[src*="challenges.cloudflare.com"]') ||
         document.querySelector('iframe[title*="Cloudflare security challenge"]') ||
         document.querySelector('script[src*="/cdn-cgi/challenge-platform/"]') ||
-        document.querySelector('a[href*="continue"]');
+        document.querySelector('a[href*="continue"]') ||
+        // New selectors for 2025
+        document.querySelector('.cf-managed-challenge') ||
+        document.querySelector('[data-cf-managed]');
       
       return {
         hasIndicators: hasCloudflareIndicators,
@@ -215,12 +398,21 @@ async function quickCloudflareDetection(page, forceDebug = false) {
         url,
         bodySnippet: bodyText.substring(0, 200)
       };
-    }, FAST_TIMEOUTS.QUICK_DETECTION);
+    }, FAST_TIMEOUTS.QUICK_DETECTION, { maxRetries: 2, forceDebug });
+    
+    // Cache the result
+    detectionCache.set(currentPageUrl, quickCheck);
     
-    if (forceDebug && quickCheck.hasIndicators) {
-      console.log(`[debug][cloudflare] Quick detection found Cloudflare indicators on ${quickCheck.url}`);
-    } else if (forceDebug && !quickCheck.hasIndicators) {
-      console.log(`[debug][cloudflare] Quick detection found no Cloudflare indicators on ${quickCheck.url}`);
+    if (forceDebug) {
+      if (quickCheck.hasIndicators) {
+        console.log(`[debug][cloudflare] Quick detection found Cloudflare indicators on ${quickCheck.url}`);
+      } else {
+        console.log(`[debug][cloudflare] Quick detection found no Cloudflare indicators on ${quickCheck.url}`);
+      }
+      
+      if (quickCheck.attempts && quickCheck.attempts > 1) {
+        console.log(`[debug][cloudflare] Detection required ${quickCheck.attempts} attempts`);
+      }
     }
     
     return quickCheck;
@@ -1017,6 +1209,87 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
   return result;
 }
 
+/**
+ * Performs parallel detection of multiple challenge types for better performance
+ */
+async function parallelChallengeDetection(page, forceDebug = false) {
+  const detectionPromises = [];
+  
+  // Check for JS challenge
+  detectionPromises.push(
+    page.evaluate(() => {
+      return {
+        type: 'js',
+        detected: document.querySelector('script[src*="/cdn-cgi/challenge-platform/"]') !== null ||
+                  document.body?.textContent?.includes('Checking your browser') ||
+                  document.body?.textContent?.includes('Please wait while we verify')
+      };
+    }).catch(err => ({ type: 'js', detected: false, error: err.message }))
+  );
+  
+  // Check for Turnstile
+  detectionPromises.push(
+    page.evaluate(() => {
+      return {
+        type: 'turnstile',
+        detected: document.querySelector('.cf-turnstile') !== null ||
+                  document.querySelector('iframe[src*="challenges.cloudflare.com"]') !== null ||
+                  document.querySelector('.ctp-checkbox-container') !== null
+      };
+    }).catch(err => ({ type: 'turnstile', detected: false, error: err.message }))
+  );
+  
+  // Check for phishing warning
+  detectionPromises.push(
+    page.evaluate(() => {
+      return {
+        type: 'phishing',
+        detected: document.body?.textContent?.includes('This website has been reported for potential phishing') ||
+                  document.querySelector('a[href*="continue"]') !== null
+      };
+    }).catch(err => ({ type: 'phishing', detected: false, error: err.message }))
+  );
+  
+  // Check for managed challenge
+  detectionPromises.push(
+    page.evaluate(() => {
+      return {
+        type: 'managed',
+        detected: document.querySelector('.cf-managed-challenge') !== null ||
+                  document.querySelector('[data-cf-managed]') !== null
+      };
+    }).catch(err => ({ type: 'managed', detected: false, error: err.message }))
+  );
+  
+  const results = await Promise.all(detectionPromises);
+  
+  const detectedChallenges = results.filter(r => r.detected).map(r => r.type);
+  
+  if (forceDebug && detectedChallenges.length > 0) {
+    console.log(`[debug][cloudflare] Parallel detection found challenges: ${detectedChallenges.join(', ')}`);
+  }
+  
+  return {
+    challenges: detectedChallenges,
+    hasAnyChallenge: detectedChallenges.length > 0,
+    details: results
+  };
+}
+
+/**
+ * Gets cache statistics for performance monitoring
+ */
+function getCacheStats() {
+  return detectionCache.getStats();
+}
+
+/**
+ * Clears the detection cache
+ */
+function clearDetectionCache() {
+  detectionCache.clear();
+}
+
 module.exports = {
   analyzeCloudflareChallenge,
   handlePhishingWarning,
@@ -1029,5 +1302,12 @@ module.exports = {
   checkChallengeCompletion,
   quickCloudflareDetection,
   getModuleInfo,
-  CLOUDFLARE_MODULE_VERSION
-};
+  CLOUDFLARE_MODULE_VERSION,
+  // New exports
+  parallelChallengeDetection,
+  getCacheStats,
+  clearDetectionCache,
+  categorizeError,
+  ERROR_TYPES,
+  RETRY_CONFIG
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.69 ===
+// === Network scanner script (nwss.js) v1.0.72 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -14,7 +14,12 @@ const { formatRules, handleOutput, getFormatDescription } = require('./lib/outpu
 // Rule validation
 const { validateRulesetFile, validateFullConfig, testDomainValidation, cleanRulesetFile } = require('./lib/validate_rules');
 // CF Bypass
-const { handleCloudflareProtection } = require('./lib/cloudflare');
+const { 
+  handleCloudflareProtection,
+  getCacheStats,
+  clearDetectionCache,
+  parallelChallengeDetection
+} = require('./lib/cloudflare');
 // FP Bypass
 const { handleFlowProxyProtection, getFlowProxyTimeouts } = require('./lib/flowproxy');
 // ignore_similar rules
@@ -83,7 +88,7 @@ function detectPuppeteerVersion() {
   try {
     const puppeteer = require('puppeteer');
     let versionString = null;
-    
+ 
     // Try multiple methods to get version
     if (puppeteer.version) {
       versionString = puppeteer.version;
@@ -118,7 +123,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.69'; // Script version
+const VERSION = '1.0.72'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -311,6 +316,10 @@ if (clearCache && !dryRunMode) {
     forceDebug,
     cachePath: CACHE_LIMITS.DEFAULT_CACHE_PATH // Default path, will be updated after config loads if needed
   });
+  
+  // Also clear Cloudflare detection cache
+  clearDetectionCache();
+  if (forceDebug) console.log(formatLogMessage('debug', 'Cleared Cloudflare detection cache'));
 }
 
 // Handle validation-only operations before main help
@@ -517,6 +526,10 @@ Redirect Handling Options:
 Cloudflare Protection Options:
   cloudflare_phish: true/false                 Auto-click through Cloudflare phishing warnings (default: false)
   cloudflare_bypass: true/false               Auto-solve Cloudflare "Verify you are human" challenges (default: false)
+  cloudflare_parallel_detection: true/false    Use parallel detection for faster Cloudflare checks (default: true)
+  cloudflare_max_retries: <number>            Maximum retry attempts for Cloudflare operations (default: 3)
+  cloudflare_cache_ttl: <milliseconds>        TTL for Cloudflare detection cache (default: 300000 - 5 minutes)
+  cloudflare_retry_on_error: true/false       Enable retry logic for Cloudflare operations (default: true)
 
 FlowProxy Protection Options:
   flowproxy_detection: true/false              Enable flowProxy protection detection and handling (default: false)
@@ -2409,15 +2422,41 @@ function setupFrameHandling(page, forceDebug) {
         
         siteCounter++;
 
-        // Handle all Cloudflare protections using the dedicated module
+        // Enhanced Cloudflare handling with parallel detection
+        if (siteConfig.cloudflare_parallel_detection !== false) { // Enable by default
+          try {
+            const parallelResult = await parallelChallengeDetection(page, forceDebug);
+            if (parallelResult.hasAnyChallenge && forceDebug) {
+              console.log(formatLogMessage('debug', `[cloudflare] Parallel detection found: ${parallelResult.challenges.join(', ')}`));
+            }
+          } catch (parallelErr) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `[cloudflare] Parallel detection failed: ${parallelErr.message}`));
+            }
+          }
+        }
+
+        // Handle all Cloudflare protections using the enhanced module
         const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug);
-        
+        // Check for retry recommendations
+        if (cloudflareResult.errors && cloudflareResult.errors.length > 0) {
+          const hasRetryableErrors = cloudflareResult.errors.some(err => 
+            err.includes('timeout') || err.includes('network')
+          );
+          
+          if (hasRetryableErrors && forceDebug) {
+            console.log(formatLogMessage('debug', '[cloudflare] Errors may be retryable - consider enabling retry logic'));
+          }
+        }        
+
         if (!cloudflareResult.overallSuccess) {
           console.warn(`⚠ [cloudflare] Protection handling failed for ${currentUrl}:`);
           cloudflareResult.errors.forEach(error => {
             console.warn(`   - ${error}`);
           });
           // Continue with scan despite Cloudflare issues
+        } else if (cloudflareResult.verificationChallenge?.success && forceDebug) {
+          console.log(formatLogMessage('debug', `[cloudflare] Challenge solved using: ${cloudflareResult.verificationChallenge.method}`));
         }
 
         // Handle flowProxy protection if enabled
@@ -2959,6 +2998,13 @@ function setupFrameHandling(page, forceDebug) {
      console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
      console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
      console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
+     // Cloudflare cache statistics
+     const cloudflareStats = getCacheStats();
+     if (cloudflareStats.size > 0) {
+       console.log(formatLogMessage('debug', '=== Cloudflare Cache Statistics ==='));
+       console.log(formatLogMessage('debug', `Cache hit rate: ${cloudflareStats.hitRate}, Total hits: ${cloudflareStats.hits}, Misses: ${cloudflareStats.misses}`));
+       console.log(formatLogMessage('debug', `Cached detections: ${cloudflareStats.size}`));
+     }
      // Log smart cache statistics (if cache is enabled)
     if (smartCache) {
     const cacheStats = smartCache.getStats();  

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.71",
+  "version": "1.0.72",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {