@fanboynz/network-scanner 1.0.75 → 1.0.77

package/lib/cloudflare.js

@@ -1,13 +1,13 @@
 /**
  * Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
- * Version: 2.2.0 - Enhanced with retry logic, caching, and improved error handling
+ * Version: 2.3.0 - Support iframe CF, and better error handling
  * Handles phishing warnings, Turnstile challenges, and modern Cloudflare protections
  */
 
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.2.0';
+const CLOUDFLARE_MODULE_VERSION = '2.3.0';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -749,6 +749,144 @@ async function attemptChallengeSolve(page, currentUrl, challengeInfo, forceDebug
   return result;
 }
 
+ /**
+ * Enhanced embedded iframe challenge detection and interaction
+ */
+async function handleEmbeddedIframeChallenge(page, forceDebug = false) {
+  const result = {
+    success: false,
+    error: null
+  };
+
+  try {
+    if (forceDebug) console.log(`[debug][cloudflare] Checking for embedded iframe challenges`);
+
+    // Enhanced iframe selectors including challenges.cloudflare.com
+    const iframeSelectors = [
+      'iframe[src*="challenges.cloudflare.com"]',
+      'iframe[title*="Verify you are human"]',
+      'iframe[title*="Cloudflare security challenge"]',
+      'iframe[title*="Widget containing a Cloudflare"]'
+    ];
+
+    // Wait for iframe to appear
+    let iframeFound = false;
+    for (const selector of iframeSelectors) {
+      try {
+        await Promise.race([
+          page.waitForSelector(selector, { timeout: FAST_TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), FAST_TIMEOUTS.SELECTOR_WAIT + 1000))
+        ]);
+        iframeFound = true;
+        if (forceDebug) console.log(`[debug][cloudflare] Found iframe: ${selector}`);
+        break;
+      } catch (e) {
+        continue;
+      }
+    }
+
+    if (!iframeFound) {
+      result.error = 'No embedded iframe found';
+      return result;
+    }
+
+    // Find challenge frame using existing frame detection logic
+    const frames = await page.frames();
+    const challengeFrame = frames.find(frame => {
+      const frameUrl = frame.url();
+      return frameUrl.includes('challenges.cloudflare.com') ||
+             frameUrl.includes('/turnstile/if/') ||
+             frameUrl.includes('captcha-delivery.com') ||
+             frameUrl.includes('/challenge-platform/') ||
+             frameUrl.includes('turnstile');
+    });
+
+    if (!challengeFrame) {
+      result.error = 'Challenge iframe not accessible';
+      return result;
+    }
+
+    if (forceDebug) console.log(`[debug][cloudflare] Interacting with iframe: ${challengeFrame.url()}`);
+
+    // Reuse existing checkbox interaction logic
+    const checkboxSelectors = [
+      'input[type="checkbox"]',
+      '.ctp-checkbox',
+      'input.ctp-checkbox',
+      '.cf-turnstile input',
+      '.ctp-checkbox-label'
+    ];
+    
+    let checkboxInteractionSuccess = false;
+    for (const selector of checkboxSelectors) {
+      try {
+        await Promise.race([
+          challengeFrame.waitForSelector(selector, { timeout: FAST_TIMEOUTS.SELECTOR_WAIT }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), FAST_TIMEOUTS.SELECTOR_WAIT + 1000))
+        ]);
+        
+        await waitForTimeout(page, FAST_TIMEOUTS.ELEMENT_INTERACTION_DELAY);
+        await challengeFrame.click(selector);
+        
+        if (forceDebug) console.log(`[debug][cloudflare] Clicked iframe element: ${selector}`);
+        checkboxInteractionSuccess = true;
+        break;
+      } catch (e) {
+        continue;
+      }
+    }
+
+    // Try alternative interaction only if standard selectors failed
+    if (!checkboxInteractionSuccess) {
+      if (forceDebug) console.log(`[debug][cloudflare] Checkbox interactions failed, trying container fallback`);
+      await waitForTimeout(page, 1000);
+
+      try {
+        // Try clicking on the iframe container itself as fallback
+        const iframeElement = await page.$('iframe[src*="challenges.cloudflare.com"]');
+        if (iframeElement) {
+          await iframeElement.click();
+          if (forceDebug) console.log(`[debug][cloudflare] Clicked iframe container as fallback`);
+        }
+      } catch (containerClickError) {
+        if (forceDebug) console.log(`[debug][cloudflare] Container click failed: ${containerClickError.message}`);
+      }
+    } else {
+      if (forceDebug) console.log(`[debug][cloudflare] Checkbox interaction successful, skipping container fallback`);
+    }
+
+    // Reuse existing completion check pattern with error handling
+    try {
+      await Promise.race([
+        page.waitForFunction(
+          () => {
+            const responseInput = document.querySelector('input[name="cf-turnstile-response"]');
+            const hasResponse = responseInput && responseInput.value && responseInput.value.length > 0;
+            const hasClearance = document.cookie.includes('cf_clearance');
+            const noChallenge = !document.body.textContent.includes('Verify you are human');
+            
+            return hasResponse || hasClearance || noChallenge;
+          },
+          { timeout: TIMEOUTS.TURNSTILE_COMPLETION }
+        ),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('Completion check timeout')), TIMEOUTS.TURNSTILE_COMPLETION_BUFFER))
+      ]);
+
+      result.success = true;
+      if (forceDebug) console.log(`[debug][cloudflare] Embedded iframe challenge completed`);
+    } catch (completionError) {
+      result.error = `Challenge completion check failed: ${completionError.message}`;
+      if (forceDebug) console.log(`[debug][cloudflare] Completion check failed: ${completionError.message}`);
+    }
+
+  } catch (error) {
+    result.error = `Embedded iframe handling failed: ${error.message}`;
+    if (forceDebug) console.log(`[debug][cloudflare] ${result.error}`);
+  }
+
+  return result;
+}
+
 /**
  * Waits for JS challenge completion with timeout protection and enhanced debug logging
  */
@@ -796,6 +934,14 @@ async function handleTurnstileChallenge(page, forceDebug = false) {
     error: null
   };
 
+  // Try embedded iframe approach first
+  const iframeResult = await handleEmbeddedIframeChallenge(page, forceDebug);
+  if (iframeResult.success) {
+    return { ...result, success: true };
+  }
+  
+  if (forceDebug) console.log(`[debug][cloudflare] Embedded iframe failed: ${iframeResult.error}, trying legacy method`);
+
   try {
     // Use fast timeout for Turnstile operations
     const turnstileTimeout = FAST_TIMEOUTS.TURNSTILE_OPERATION;
@@ -1276,6 +1422,29 @@ async function parallelChallengeDetection(page, forceDebug = false) {
   };
 }
 
+/**
+ * Enhanced parallel detection including embedded iframe challenges
+ */
+async function enhancedParallelChallengeDetection(page, forceDebug = false) {
+  const existingDetection = await parallelChallengeDetection(page, forceDebug);
+  
+  try {
+    const hasEmbeddedIframe = await page.evaluate(() => {
+      return document.querySelector('iframe[src*="challenges.cloudflare.com"]') !== null ||
+             document.querySelector('iframe[title*="Verify you are human"]') !== null;
+    });
+    
+    if (hasEmbeddedIframe && !existingDetection.challenges.includes('embedded_iframe')) {
+      existingDetection.challenges.push('embedded_iframe');
+      existingDetection.hasAnyChallenge = true;
+    }
+  } catch (e) {
+    // Ignore detection errors
+  }
+  
+  return existingDetection;
+}
+
 /**
  * Gets cache statistics for performance monitoring
  */
@@ -1300,6 +1469,8 @@ module.exports = {
   waitForJSChallengeCompletion,
   handleLegacyCheckbox,
   checkChallengeCompletion,
+  handleEmbeddedIframeChallenge,
+  enhancedParallelChallengeDetection,
   quickCloudflareDetection,
   getModuleInfo,
   CLOUDFLARE_MODULE_VERSION,

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.75 ===
+// === Network scanner script (nwss.js) v1.0.77 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -123,7 +123,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.75'; // Script version
+const VERSION = '1.0.77'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -187,6 +187,7 @@ const testValidation = args.includes('--test-validation');
 let cleanRules = args.includes('--clean-rules');
 const clearCache = args.includes('--clear-cache');
 const ignoreCache = args.includes('--ignore-cache');
+const cacheRequests = args.includes('--cache-requests');
 
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
@@ -454,6 +455,7 @@ General Options:
   --remove-tempfiles             Remove Chrome/Puppeteer temporary files before exit
 
 Validation Options:
+  --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -522,6 +524,7 @@ Redirect Handling Options:
   adblock_rules: true/false                    Generate adblock filter rules with resource types for this site
   even_blocked: true/false                     Add matching rules even if requests are blocked (default: false)
   
+  bypass_cache: true/false                     Skip all caching for this site's URLs (default: false)
   referrer_headers: "url" or ["url1", "url2"] Set referrer header for realistic traffic sources
   custom_headers: {"Header": "value"}         Add custom HTTP headers to requests
 
@@ -682,6 +685,7 @@ if (ignoreCache) {
 } else {
 smartCache = createSmartCache({
   ...config,
+  cache_requests: cacheRequests, // NEW: Pass request caching flag
   forceDebug,
   max_concurrent_sites: MAX_CONCURRENT_SITES,  // Pass concurrency info
   cache_aggressive_mode: MAX_CONCURRENT_SITES > CONCURRENCY_LIMITS.HIGH_CONCURRENCY_THRESHOLD,  // Auto-enable for high concurrency
@@ -692,6 +696,24 @@ smartCache = createSmartCache({
 });
 }
 
+// Add safe domain processing helper after smartCache initialization
+function safeMarkDomainProcessed(domain, context, metadata) {
+  if (smartCache) {
+    try {
+      if (typeof smartCache.markDomainProcessed === 'function') {
+        smartCache.markDomainProcessed(domain, context, metadata);
+      } else {
+        // Fallback: trigger cache via shouldSkipDomain
+        smartCache.shouldSkipDomain(domain, context);
+      }
+    } catch (cacheErr) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[SmartCache] Error marking domain: ${cacheErr.message}`));
+      }
+    }
+  }
+}
+
 // Handle --clean-rules after config is loaded (so we have access to sites)
 if (cleanRules || cleanRulesFile) {
   const filesToClean = cleanRulesFile ? [cleanRulesFile] : [outputFile, compareFile].filter(Boolean);
@@ -1000,6 +1022,16 @@ function shouldProcessUrl(url, forceDebug) {
   }
 }
 
+/**
+ * Check if URL should bypass all caching for this site
+ * @param {string} url - URL to check
+ * @param {Object} siteConfig - Site configuration
+ * @returns {boolean} True if should bypass cache
+ */
+function shouldBypassCacheForUrl(url, siteConfig) {
+  return siteConfig.bypass_cache === true;
+}
+
 // ability to use widcards in ignoreDomains
 function matchesIgnoreDomain(domain, ignorePatterns) {
   return ignorePatterns.some(pattern => {
@@ -1365,6 +1397,11 @@ function setupFrameHandling(page, forceDebug) {
       );
     }
 
+   // Log bypass_cache setting if enabled
+   if (forceDebug && siteConfig.bypass_cache === true) {
+     console.log(formatLogMessage('debug', `Cache bypass enabled for all URLs in site: ${currentUrl}`));
+   }
+
     if (siteConfig.firstParty === 0 && siteConfig.thirdParty === 0) {
       console.warn(`⚠ Skipping ${currentUrl} because both firstParty and thirdParty are disabled.`);
       return { url: currentUrl, rules: [], success: false, skipped: true };
@@ -1891,7 +1928,18 @@ function setupFrameHandling(page, forceDebug) {
       
       // Also mark in smart cache with context (if cache is enabled)
       if (smartCache) {
-        smartCache.markDomainProcessed(domain, context, { resourceType, fullSubdomain });
+  try {
+    if (smartCache.markDomainProcessed) {
+      safeMarkDomainProcessed(domain, context, { resourceType, fullSubdomain });
+    } else {
+      // Fallback: use shouldSkipDomain to indirectly cache
+      smartCache.shouldSkipDomain(domain, context);
+    }
+  } catch (cacheErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[SmartCache] Error marking domain: ${cacheErr.message}`));
+    }
+  }
       }
 
         if (matchedDomains instanceof Map) {
@@ -2303,8 +2351,16 @@ function setupFrameHandling(page, forceDebug) {
            
            // If curl is enabled, download and analyze content immediately
            if (useCurl) {
-             // Check response cache first if smart cache is available and caching is enabled
-             const cachedContent = smartCache ? smartCache.getCachedResponse(reqUrl) : null;
+             // Check bypass_cache before attempting cache lookup
+             let cachedContent = null;
+             if (!shouldBypassCacheForUrl(reqUrl, siteConfig)) {
+               // Check request cache first if smart cache is available and caching is enabled
+               cachedContent = smartCache ? smartCache.getCachedRequest(reqUrl, {
+                 method: 'GET',
+                 headers: { 'user-agent': curlUserAgent },
+                 siteConfig: siteConfig
+               }) : null;
+             }
              
              if (cachedContent && forceDebug) {
                console.log(formatLogMessage('debug', `[SmartCache] Using cached response content for ${reqUrl.substring(0, 50)}...`));
@@ -2321,7 +2377,10 @@ function setupFrameHandling(page, forceDebug) {
                    addMatchedDomain, // Pass the helper function
                    isDomainAlreadyDetected,
                    onContentFetched: smartCache && !ignoreCache ? (url, content) => {
-                     smartCache.cacheResponse(url, content);
+                     // Only cache if not bypassing cache
+                     if (!shouldBypassCacheForUrl(url, siteConfig)) {
+                       smartCache.cacheRequest(url, { method: 'GET', siteConfig }, { body: content, status: 200 });
+                     }
                    } : undefined,
                    currentUrl,
                    perSiteSubDomains,
@@ -2354,6 +2413,12 @@ function setupFrameHandling(page, forceDebug) {
                    matchedDomains,
                    addMatchedDomain, // Pass the helper function
                    isDomainAlreadyDetected,
+                 onContentFetched: smartCache && !ignoreCache ? (url, content) => {
+                   // Only cache if not bypassing cache
+                   if (!shouldBypassCacheForUrl(url, siteConfig)) {
+                     smartCache.cacheRequest(url, { method: 'GET', siteConfig }, { body: content, status: 200 });
+                   }
+                 } : undefined,
                    currentUrl,
                    perSiteSubDomains,
                    ignoreDomains,
@@ -2392,7 +2457,14 @@ function setupFrameHandling(page, forceDebug) {
          regexes,
          matchedDomains,
          addMatchedDomain, // Pass the helper function
+         bypassCache: (url) => shouldBypassCacheForUrl(url, siteConfig),
          isDomainAlreadyDetected,
+         onContentFetched: smartCache && !ignoreCache ? (url, content) => {
+           // Only cache if not bypassing cache
+           if (!shouldBypassCacheForUrl(url, siteConfig)) {
+             smartCache.cacheRequest(url, { method: 'GET', siteConfig }, { body: content, status: 200 });
+           }
+         } : undefined,
          currentUrl,
          perSiteSubDomains,
          ignoreDomains,
@@ -2910,6 +2982,17 @@ function setupFrameHandling(page, forceDebug) {
         console.log(`\n${messageColors.fileOp('🔄 Browser restart triggered:')} ${restartReason}`);
       }
       
+      // NEW: Clear request cache during browser restart to ensure fresh session
+      if (smartCache && cacheRequests) {
+        const requestCacheStats = smartCache.getRequestCacheStats();
+        if (requestCacheStats.enabled && requestCacheStats.size > 0) {
+          const clearedCount = smartCache.clearRequestCache();
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[SmartCache] Cleared ${clearedCount} request cache entries during browser restart`));
+          }
+        }
+      }
+
       try {
         await handleBrowserExit(browser, {
           forceDebug,
@@ -2986,6 +3069,17 @@ function setupFrameHandling(page, forceDebug) {
         console.log(`\n${messageColors.fileOp('🔄 Emergency browser restart:')} Critical browser errors detected`);
       }
       
+      // NEW: Clear request cache during emergency restart
+      if (smartCache && cacheRequests) {
+        const requestCacheStats = smartCache.getRequestCacheStats();
+        if (requestCacheStats.enabled && requestCacheStats.size > 0) {
+          const clearedCount = smartCache.clearRequestCache();
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[SmartCache] Cleared ${clearedCount} request cache entries during emergency restart`));
+          }
+        }
+      }
+
       // Force browser restart immediately
       try {
         // Enhanced emergency restart for Puppeteer 23.x
@@ -3048,6 +3142,22 @@ function setupFrameHandling(page, forceDebug) {
 
   let outputResult;
   
+  // NEW: Clear request cache after processing all sites in the JSON config
+  if (smartCache && cacheRequests) {
+    const requestCacheStats = smartCache.getRequestCacheStats();
+    if (requestCacheStats.enabled && requestCacheStats.size > 0) {
+      const clearedCount = smartCache.clearRequestCache();
+      if (!silentMode && clearedCount > 0) {
+        console.log(`\n🗑️  Cleared request cache: ${clearedCount} entries after JSON processing`);
+      }
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', 
+          `[SmartCache] Request cache cleared after JSON scan completion (hit rate: ${requestCacheStats.hitRate})`
+        ));
+      }
+    }
+  }
+  
   if (!dryRunMode) {
     // Handle all output using the output module
     const outputConfig = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.75",
+  "version": "1.0.77",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {