@fanboynz/network-scanner 2.0.5 → 2.0.7

package/lib/cdp.js

@@ -49,12 +49,37 @@ async function createPageWithTimeout(browser, timeout = 30000) {
  * @returns {Promise<void>}
  */
 async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
-  return Promise.race([
-    page.setRequestInterception(true),
-    new Promise((_, reject) => 
-      setTimeout(() => reject(new Error('Request interception setup timeout')), timeout)
-    )
-  ]);
+  try {
+    await Promise.race([
+      page.setRequestInterception(true),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Request interception timeout - first attempt')), timeout)
+      )
+    ]);
+  } catch (firstError) {
+    // Check for immediate critical failures
+    if (firstError.message.includes('Target closed') || 
+        firstError.message.includes('Session closed') ||
+        firstError.message.includes('Browser has been closed')) {
+      throw new Error('CRITICAL_BROWSER_ERROR: ' + firstError.message);
+    }
+    
+    // Retry with extended timeout
+    try {
+      await Promise.race([
+        page.setRequestInterception(true),
+        new Promise((_, reject) => 
+          setTimeout(() => reject(new Error('Request interception timeout - retry failed')), timeout * 2)
+        )
+      ]);
+    } catch (retryError) {
+      if (retryError.message.includes('Network.enable timed out') || 
+          retryError.message.includes('ProtocolError')) {
+        throw new Error('CRITICAL_NETWORK_ERROR: ' + retryError.message);
+      }
+      throw retryError;
+    }
+  }
 }
 
 /**

package/lib/clear_sitedata.js

@@ -0,0 +1,350 @@
+// === Clear Site Data Module ===
+// Handles comprehensive site data clearing via CDP and page-level fallbacks
+// Resolves SecurityError issues with localStorage/sessionStorage access
+
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * Clears site data using CDP (bypasses same-origin restrictions)
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag
+ * @param {boolean} quickMode - If true, only clear cache/cookies (for reloads)
+ * @returns {Promise<{success: boolean, operations: string[]}>}
+ */
+async function clearSiteDataViaCDP(page, currentUrl, forceDebug, quickMode = false) {
+  let clearDataSession = null;
+  const completedOperations = [];
+  
+  try {
+    clearDataSession = await Promise.race([
+      page.target().createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('CDP session timeout')), 10000)
+      )
+    ]);
+    
+    const origin = new URL(currentUrl).origin;
+    
+    // Always clear cache and cookies (even in quick mode)
+    const basicOperations = [
+      { cmd: 'Network.clearBrowserCookies', name: 'cookies' },
+      { cmd: 'Network.clearBrowserCache', name: 'cache' }
+    ];
+    
+    for (const op of basicOperations) {
+      try {
+        await clearDataSession.send(op.cmd);
+        completedOperations.push(op.name);
+      } catch (opErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[clear_sitedata] ${op.name} clear failed: ${opErr.message}`));
+        }
+      }
+    }
+    
+    // Full storage clearing (skip in quick mode for reloads)
+    if (!quickMode) {
+      // Try comprehensive storage clearing first
+      try {
+        await clearDataSession.send('Storage.clearDataForOrigin', {
+          origin: origin,
+          storageTypes: 'all'
+        });
+        completedOperations.push('all_storage');
+      } catch (allStorageErr) {
+        // Fallback: try individual storage types
+        const storageTypes = [
+          { type: 'local_storage', name: 'localStorage' },
+          { type: 'session_storage', name: 'sessionStorage' },
+          { type: 'indexeddb', name: 'indexedDB' },
+          { type: 'websql', name: 'webSQL' },
+          { type: 'service_workers', name: 'serviceWorkers' }
+        ];
+        
+        for (const storage of storageTypes) {
+          try {
+            await clearDataSession.send('Storage.clearDataForOrigin', {
+              origin: origin,
+              storageTypes: storage.type
+            });
+            completedOperations.push(storage.name);
+          } catch (individualErr) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `[clear_sitedata] ${storage.name} clear failed: ${individualErr.message}`));
+            }
+          }
+        }
+      }
+    }
+    
+    if (forceDebug && completedOperations.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] CDP cleared: ${completedOperations.join(', ')}`));
+    }
+    
+    return { success: completedOperations.length > 0, operations: completedOperations };
+    
+  } catch (cdpErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] CDP session failed: ${cdpErr.message}`));
+    }
+    return { success: false, operations: completedOperations };
+  } finally {
+    if (clearDataSession) {
+      try { 
+        await clearDataSession.detach(); 
+      } catch (detachErr) { 
+        // Ignore detach errors
+      }
+    }
+  }
+}
+
+/**
+ * Fallback page-level clearing with security error handling
+ * @param {Page} page - Puppeteer page instance  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<{success: boolean, operations: string[]}>}
+ */
+async function clearSiteDataViaPage(page, forceDebug) {
+  try {
+    const result = await page.evaluate(() => {
+      const cleared = [];
+      
+      // Test and clear localStorage
+      try {
+        if (window.localStorage && typeof window.localStorage.setItem === 'function') {
+          const testKey = '__nwss_access_test__';
+          localStorage.setItem(testKey, 'test');
+          localStorage.removeItem(testKey);
+          localStorage.clear();
+          cleared.push('localStorage');
+        }
+      } catch (e) {
+        // Security error expected on some sites
+      }
+      
+      // Test and clear sessionStorage  
+      try {
+        if (window.sessionStorage && typeof window.sessionStorage.setItem === 'function') {
+          const testKey = '__nwss_access_test__';
+          sessionStorage.setItem(testKey, 'test');
+          sessionStorage.removeItem(testKey);
+          sessionStorage.clear();
+          cleared.push('sessionStorage');
+        }
+      } catch (e) {
+        // Security error expected on some sites
+      }
+      
+      // Clear IndexedDB
+      try {
+        if (window.indexedDB && typeof window.indexedDB.databases === 'function') {
+          window.indexedDB.databases().then(dbs => {
+            dbs.forEach(db => {
+              try {
+                window.indexedDB.deleteDatabase(db.name);
+              } catch (dbErr) {
+                // Individual DB deletion may fail
+              }
+            });
+          }).catch(() => {
+            // Database listing may fail
+          });
+          cleared.push('indexedDB');
+        }
+      } catch (e) {
+        // IndexedDB may not be available
+      }
+      
+      return cleared;
+    });
+    
+    if (forceDebug && result.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] Page-level cleared: ${result.join(', ')}`));
+    }
+    
+    return { success: result.length > 0, operations: result };
+  } catch (pageErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] Page evaluation failed: ${pageErr.message}`));
+    }
+    return { success: false, operations: [] };
+  }
+}
+
+/**
+ * Main entry point for site data clearing
+ * Attempts CDP clearing first, falls back to page-level if needed
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag
+ * @param {boolean} quickMode - If true, only clear cache/cookies (for reloads)
+ * @returns {Promise<{success: boolean, operations: string[], method: string}>}
+ */
+async function clearSiteData(page, currentUrl, forceDebug, quickMode = false) {
+  // Try CDP clearing first (preferred method)
+  const cdpResult = await clearSiteDataViaCDP(page, currentUrl, forceDebug, quickMode);
+  
+  if (cdpResult.success) {
+    return {
+      success: true,
+      operations: cdpResult.operations,
+      method: 'CDP'
+    };
+  }
+  
+  // Fallback to page-level clearing if CDP failed and not in quick mode
+  if (!quickMode) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `CDP clearing failed, attempting page-level fallback for ${currentUrl}`));
+    }
+    
+    const pageResult = await clearSiteDataViaPage(page, forceDebug);
+    
+    return {
+      success: pageResult.success,
+      operations: pageResult.operations,
+      method: pageResult.success ? 'page-level' : 'failed'
+    };
+  }
+  
+  return {
+    success: false,
+    operations: [],
+    method: 'failed'
+  };
+}
+
+/**
+ * Enhanced site data clearing with additional browser-level operations
+ * Includes cache warming prevention and comprehensive storage cleanup
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<{success: boolean, operations: string[], method: string}>}
+ */
+async function clearSiteDataEnhanced(page, currentUrl, forceDebug) {
+  let clearDataSession = null;
+  const completedOperations = [];
+  
+  try {
+    clearDataSession = await Promise.race([
+      page.target().createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Enhanced CDP session timeout')), 15000)
+      )
+    ]);
+    
+    const origin = new URL(currentUrl).origin;
+    
+    // Enhanced clearing operations
+    const enhancedOperations = [
+      // Network layer
+      { cmd: 'Network.clearBrowserCookies', name: 'cookies' },
+      { cmd: 'Network.clearBrowserCache', name: 'cache' },
+      
+      // Storage layer - comprehensive
+      {
+        cmd: 'Storage.clearDataForOrigin',
+        params: { origin, storageTypes: 'all' },
+        name: 'all_storage'
+      },
+      
+      // Runtime layer
+      { cmd: 'Runtime.discardConsoleEntries', name: 'console' },
+      
+      // Security layer
+      { cmd: 'Security.disable', name: 'security_reset' }
+    ];
+    
+    for (const op of enhancedOperations) {
+      try {
+        if (op.params) {
+          await clearDataSession.send(op.cmd, op.params);
+        } else {
+          await clearDataSession.send(op.cmd);
+        }
+        completedOperations.push(op.name);
+      } catch (opErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] ${op.name} failed: ${opErr.message}`));
+        }
+        
+        // For storage operations, try individual fallbacks
+        if (op.name === 'all_storage') {
+          const individualTypes = ['local_storage', 'session_storage', 'indexeddb', 'websql', 'service_workers'];
+          for (const type of individualTypes) {
+            try {
+              await clearDataSession.send('Storage.clearDataForOrigin', {
+                origin,
+                storageTypes: type
+              });
+              completedOperations.push(type);
+            } catch (individualErr) {
+              // Continue trying other types
+            }
+          }
+        }
+      }
+    }
+    
+    // Additional DOM cleanup via page evaluation
+    try {
+      await page.evaluate(() => {
+        // Clear any cached DOM queries
+        if (window.document && document.querySelectorAll) {
+          // Force garbage collection of cached selectors
+          const div = document.createElement('div');
+          document.body.appendChild(div);
+          document.body.removeChild(div);
+        }
+        
+        // Clear performance entries
+        if (window.performance && performance.clearMarks) {
+          performance.clearMarks();
+          performance.clearMeasures();
+        }
+      });
+      completedOperations.push('dom_cleanup');
+    } catch (domErr) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] DOM cleanup failed: ${domErr.message}`));
+      }
+    }
+    
+    if (forceDebug && completedOperations.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] Cleared: ${completedOperations.join(', ')}`));
+    }
+    
+    return {
+      success: completedOperations.length > 0,
+      operations: completedOperations,
+      method: 'enhanced_CDP'
+    };
+    
+  } catch (enhancedErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] Failed: ${enhancedErr.message}`));
+    }
+    
+    // Fallback to regular clearing
+    return await clearSiteData(page, currentUrl, forceDebug, false);
+    
+  } finally {
+    if (clearDataSession) {
+      try { 
+        await clearDataSession.detach(); 
+      } catch (detachErr) { 
+        // Ignore detach errors
+      }
+    }
+  }
+}
+
+module.exports = {
+  clearSiteData,
+  clearSiteDataViaCDP,
+  clearSiteDataViaPage,
+  clearSiteDataEnhanced
+};

package/lib/cloudflare.js

@@ -19,7 +19,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.1';
+const CLOUDFLARE_MODULE_VERSION = '2.6.2';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -1880,4 +1880,4 @@ module.exports = {
   detectChallengeLoop,
   // Memory management
   cleanup
-};
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.5 ===
+// === Network scanner script (nwss.js) v2.0.7 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -44,6 +44,8 @@ const { performPageInteraction, createInteractionConfig } = require('./lib/inter
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
+// Enhanced site data clearing functionality
+const { clearSiteData } = require('./lib/clear_sitedata');
 
 // Fast setTimeout helper for Puppeteer 22.x compatibility
 // Uses standard Promise constructor for better performance than node:timers/promises
@@ -127,7 +129,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.5'; // Script version
+const VERSION = '2.0.7'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1903,7 +1905,10 @@ function setupFrameHandling(page, forceDebug) {
           console.log(formatLogMessage('debug', `Request interception enabled successfully for ${currentUrl}`));
         }
       } catch (networkErr) {
-        if (networkErr.message.includes('timed out') || 
+        if (networkErr.message.includes('CRITICAL_NETWORK_ERROR') || 
+            networkErr.message.includes('CRITICAL_BROWSER_ERROR') ||
+            networkErr.message.includes('ProtocolError') ||
+            networkErr.message.includes('timed out') || 
             networkErr.message.includes('Network.enable') || 
             networkErr.message.includes('timeout')) {
           console.warn(formatLogMessage('warn', `Request interception setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
@@ -1923,24 +1928,10 @@ function setupFrameHandling(page, forceDebug) {
 	  
       if (siteConfig.clear_sitedata === true) {
         try {
-          let clearDataSession = null;
-          try {
-            clearDataSession = await page.target().createCDPSession();
-            await clearDataSession.send('Network.clearBrowserCookies');
-            await clearDataSession.send('Network.clearBrowserCache');
-          } finally {
-            if (clearDataSession) {
-              try { await clearDataSession.detach(); } catch (detachErr) { /* ignore */ }
-            }
-          }
-          await page.evaluate(() => {
-            localStorage.clear();
-            sessionStorage.clear();
-            indexedDB.databases().then(dbs => dbs.forEach(db => indexedDB.deleteDatabase(db.name)));
-          });
+          const clearResult = await clearSiteData(page, currentUrl, forceDebug);
           if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data for ${currentUrl}`));
         } catch (clearErr) {
-          console.warn(messageColors.warn(`[clear_sitedata failed] ${currentUrl}: ${clearErr.message}`));
+          if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Failed for ${currentUrl}: ${clearErr.message}`));
         }
       }
 
@@ -3113,24 +3104,10 @@ function setupFrameHandling(page, forceDebug) {
 
         if (siteConfig.clear_sitedata === true) {
           try {
-            let reloadClearSession = null;
-            try {
-              reloadClearSession = await page.target().createCDPSession();
-              await reloadClearSession.send('Network.clearBrowserCookies');
-              await reloadClearSession.send('Network.clearBrowserCache');
-            } finally {
-              if (reloadClearSession) {
-                try { await reloadClearSession.detach(); } catch (detachErr) { /* ignore */ }
-              }
-            }
-            await page.evaluate(() => {
-              localStorage.clear();
-              sessionStorage.clear();
-              indexedDB.databases().then(dbs => dbs.forEach(db => indexedDB.deleteDatabase(db.name)));
-            });
+            const clearResult = await clearSiteData(page, currentUrl, forceDebug, true); // Quick mode for reloads
             if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data before reload #${i} for ${currentUrl}`));
           } catch (reloadClearErr) {
-            console.warn(messageColors.warn(`[clear_sitedata before reload failed] ${currentUrl}: ${reloadClearErr.message}`));
+            if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Before reload failed for ${currentUrl}`));
           }
         }
         
@@ -3390,10 +3367,13 @@ function setupFrameHandling(page, forceDebug) {
   // Hang detection for debugging concurrency issues
   let currentBatchInfo = { batchStart: 0, batchSize: 0 };
   const hangDetectionInterval = setInterval(() => {
+  // Only show hang detection messages in debug mode
+  if (forceDebug) {
     const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
     const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
     console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
     console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
+    }
   }, 30000); // Check every 30 seconds
 
   // Process URLs in batches to maintain concurrency while allowing browser restarts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.5",
+  "version": "2.0.7",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {