@fanboynz/network-scanner 2.0.6 → 2.0.8

package/README.md

@@ -134,7 +134,7 @@ Example:
 | Field                | Values | Default | Description |
 |:---------------------|:-------|:-------:|:------------|
 | `url`                | String or Array |   -     | Website URL(s) to scan |
-| `userAgent`          | `chrome`, `firefox`, `safari` | - | User agent for page (latest versions: Chrome 131, Firefox 133, Safari 18.2) |
+| `userAgent`          | `chrome`, `chrome_mac`, `chrome_linux`, `firefox`, `firefox_mac`, `firefox_linux`, `safari` | - | User agent for page |
 | `filterRegex`        | String or Array | `.*` | Regex or list of regexes to match requests |
 | `regex_and`          | Boolean | `false` | Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL |
 | `comments`           | String or Array | - | String of comments or references |

package/lib/clear_sitedata.js

@@ -0,0 +1,350 @@
+// === Clear Site Data Module ===
+// Handles comprehensive site data clearing via CDP and page-level fallbacks
+// Resolves SecurityError issues with localStorage/sessionStorage access
+
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * Clears site data using CDP (bypasses same-origin restrictions)
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag
+ * @param {boolean} quickMode - If true, only clear cache/cookies (for reloads)
+ * @returns {Promise<{success: boolean, operations: string[]}>}
+ */
+async function clearSiteDataViaCDP(page, currentUrl, forceDebug, quickMode = false) {
+  let clearDataSession = null;
+  const completedOperations = [];
+  
+  try {
+    clearDataSession = await Promise.race([
+      page.target().createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('CDP session timeout')), 10000)
+      )
+    ]);
+    
+    const origin = new URL(currentUrl).origin;
+    
+    // Always clear cache and cookies (even in quick mode)
+    const basicOperations = [
+      { cmd: 'Network.clearBrowserCookies', name: 'cookies' },
+      { cmd: 'Network.clearBrowserCache', name: 'cache' }
+    ];
+    
+    for (const op of basicOperations) {
+      try {
+        await clearDataSession.send(op.cmd);
+        completedOperations.push(op.name);
+      } catch (opErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[clear_sitedata] ${op.name} clear failed: ${opErr.message}`));
+        }
+      }
+    }
+    
+    // Full storage clearing (skip in quick mode for reloads)
+    if (!quickMode) {
+      // Try comprehensive storage clearing first
+      try {
+        await clearDataSession.send('Storage.clearDataForOrigin', {
+          origin: origin,
+          storageTypes: 'all'
+        });
+        completedOperations.push('all_storage');
+      } catch (allStorageErr) {
+        // Fallback: try individual storage types
+        const storageTypes = [
+          { type: 'local_storage', name: 'localStorage' },
+          { type: 'session_storage', name: 'sessionStorage' },
+          { type: 'indexeddb', name: 'indexedDB' },
+          { type: 'websql', name: 'webSQL' },
+          { type: 'service_workers', name: 'serviceWorkers' }
+        ];
+        
+        for (const storage of storageTypes) {
+          try {
+            await clearDataSession.send('Storage.clearDataForOrigin', {
+              origin: origin,
+              storageTypes: storage.type
+            });
+            completedOperations.push(storage.name);
+          } catch (individualErr) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `[clear_sitedata] ${storage.name} clear failed: ${individualErr.message}`));
+            }
+          }
+        }
+      }
+    }
+    
+    if (forceDebug && completedOperations.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] CDP cleared: ${completedOperations.join(', ')}`));
+    }
+    
+    return { success: completedOperations.length > 0, operations: completedOperations };
+    
+  } catch (cdpErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] CDP session failed: ${cdpErr.message}`));
+    }
+    return { success: false, operations: completedOperations };
+  } finally {
+    if (clearDataSession) {
+      try { 
+        await clearDataSession.detach(); 
+      } catch (detachErr) { 
+        // Ignore detach errors
+      }
+    }
+  }
+}
+
+/**
+ * Fallback page-level clearing with security error handling
+ * @param {Page} page - Puppeteer page instance  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<{success: boolean, operations: string[]}>}
+ */
+async function clearSiteDataViaPage(page, forceDebug) {
+  try {
+    const result = await page.evaluate(() => {
+      const cleared = [];
+      
+      // Test and clear localStorage
+      try {
+        if (window.localStorage && typeof window.localStorage.setItem === 'function') {
+          const testKey = '__nwss_access_test__';
+          localStorage.setItem(testKey, 'test');
+          localStorage.removeItem(testKey);
+          localStorage.clear();
+          cleared.push('localStorage');
+        }
+      } catch (e) {
+        // Security error expected on some sites
+      }
+      
+      // Test and clear sessionStorage  
+      try {
+        if (window.sessionStorage && typeof window.sessionStorage.setItem === 'function') {
+          const testKey = '__nwss_access_test__';
+          sessionStorage.setItem(testKey, 'test');
+          sessionStorage.removeItem(testKey);
+          sessionStorage.clear();
+          cleared.push('sessionStorage');
+        }
+      } catch (e) {
+        // Security error expected on some sites
+      }
+      
+      // Clear IndexedDB
+      try {
+        if (window.indexedDB && typeof window.indexedDB.databases === 'function') {
+          window.indexedDB.databases().then(dbs => {
+            dbs.forEach(db => {
+              try {
+                window.indexedDB.deleteDatabase(db.name);
+              } catch (dbErr) {
+                // Individual DB deletion may fail
+              }
+            });
+          }).catch(() => {
+            // Database listing may fail
+          });
+          cleared.push('indexedDB');
+        }
+      } catch (e) {
+        // IndexedDB may not be available
+      }
+      
+      return cleared;
+    });
+    
+    if (forceDebug && result.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] Page-level cleared: ${result.join(', ')}`));
+    }
+    
+    return { success: result.length > 0, operations: result };
+  } catch (pageErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata] Page evaluation failed: ${pageErr.message}`));
+    }
+    return { success: false, operations: [] };
+  }
+}
+
+/**
+ * Main entry point for site data clearing
+ * Attempts CDP clearing first, falls back to page-level if needed
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed
+ * @param {boolean} forceDebug - Debug logging flag
+ * @param {boolean} quickMode - If true, only clear cache/cookies (for reloads)
+ * @returns {Promise<{success: boolean, operations: string[], method: string}>}
+ */
+async function clearSiteData(page, currentUrl, forceDebug, quickMode = false) {
+  // Try CDP clearing first (preferred method)
+  const cdpResult = await clearSiteDataViaCDP(page, currentUrl, forceDebug, quickMode);
+  
+  if (cdpResult.success) {
+    return {
+      success: true,
+      operations: cdpResult.operations,
+      method: 'CDP'
+    };
+  }
+  
+  // Fallback to page-level clearing if CDP failed and not in quick mode
+  if (!quickMode) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `CDP clearing failed, attempting page-level fallback for ${currentUrl}`));
+    }
+    
+    const pageResult = await clearSiteDataViaPage(page, forceDebug);
+    
+    return {
+      success: pageResult.success,
+      operations: pageResult.operations,
+      method: pageResult.success ? 'page-level' : 'failed'
+    };
+  }
+  
+  return {
+    success: false,
+    operations: [],
+    method: 'failed'
+  };
+}
+
+/**
+ * Enhanced site data clearing with additional browser-level operations
+ * Includes cache warming prevention and comprehensive storage cleanup
+ * @param {Page} page - Puppeteer page instance
+ * @param {string} currentUrl - URL being processed  
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<{success: boolean, operations: string[], method: string}>}
+ */
+async function clearSiteDataEnhanced(page, currentUrl, forceDebug) {
+  let clearDataSession = null;
+  const completedOperations = [];
+  
+  try {
+    clearDataSession = await Promise.race([
+      page.target().createCDPSession(),
+      new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Enhanced CDP session timeout')), 15000)
+      )
+    ]);
+    
+    const origin = new URL(currentUrl).origin;
+    
+    // Enhanced clearing operations
+    const enhancedOperations = [
+      // Network layer
+      { cmd: 'Network.clearBrowserCookies', name: 'cookies' },
+      { cmd: 'Network.clearBrowserCache', name: 'cache' },
+      
+      // Storage layer - comprehensive
+      {
+        cmd: 'Storage.clearDataForOrigin',
+        params: { origin, storageTypes: 'all' },
+        name: 'all_storage'
+      },
+      
+      // Runtime layer
+      { cmd: 'Runtime.discardConsoleEntries', name: 'console' },
+      
+      // Security layer
+      { cmd: 'Security.disable', name: 'security_reset' }
+    ];
+    
+    for (const op of enhancedOperations) {
+      try {
+        if (op.params) {
+          await clearDataSession.send(op.cmd, op.params);
+        } else {
+          await clearDataSession.send(op.cmd);
+        }
+        completedOperations.push(op.name);
+      } catch (opErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] ${op.name} failed: ${opErr.message}`));
+        }
+        
+        // For storage operations, try individual fallbacks
+        if (op.name === 'all_storage') {
+          const individualTypes = ['local_storage', 'session_storage', 'indexeddb', 'websql', 'service_workers'];
+          for (const type of individualTypes) {
+            try {
+              await clearDataSession.send('Storage.clearDataForOrigin', {
+                origin,
+                storageTypes: type
+              });
+              completedOperations.push(type);
+            } catch (individualErr) {
+              // Continue trying other types
+            }
+          }
+        }
+      }
+    }
+    
+    // Additional DOM cleanup via page evaluation
+    try {
+      await page.evaluate(() => {
+        // Clear any cached DOM queries
+        if (window.document && document.querySelectorAll) {
+          // Force garbage collection of cached selectors
+          const div = document.createElement('div');
+          document.body.appendChild(div);
+          document.body.removeChild(div);
+        }
+        
+        // Clear performance entries
+        if (window.performance && performance.clearMarks) {
+          performance.clearMarks();
+          performance.clearMeasures();
+        }
+      });
+      completedOperations.push('dom_cleanup');
+    } catch (domErr) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] DOM cleanup failed: ${domErr.message}`));
+      }
+    }
+    
+    if (forceDebug && completedOperations.length > 0) {
+      console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] Cleared: ${completedOperations.join(', ')}`));
+    }
+    
+    return {
+      success: completedOperations.length > 0,
+      operations: completedOperations,
+      method: 'enhanced_CDP'
+    };
+    
+  } catch (enhancedErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[clear_sitedata_enhanced] Failed: ${enhancedErr.message}`));
+    }
+    
+    // Fallback to regular clearing
+    return await clearSiteData(page, currentUrl, forceDebug, false);
+    
+  } finally {
+    if (clearDataSession) {
+      try { 
+        await clearDataSession.detach(); 
+      } catch (detachErr) { 
+        // Ignore detach errors
+      }
+    }
+  }
+}
+
+module.exports = {
+  clearSiteData,
+  clearSiteDataViaCDP,
+  clearSiteDataViaPage,
+  clearSiteDataEnhanced
+};

package/lib/cloudflare.js

@@ -19,7 +19,7 @@ const { formatLogMessage } = require('./colorize');
 /**
  * Module version information
  */
-const CLOUDFLARE_MODULE_VERSION = '2.6.1';
+const CLOUDFLARE_MODULE_VERSION = '2.6.2';
 
 /**
  * Timeout constants for various operations (in milliseconds)
@@ -1880,4 +1880,4 @@ module.exports = {
   detectChallengeLoop,
   // Memory management
   cleanup
-};
+};

package/nwss.1

@@ -280,7 +280,8 @@ Array of CSS selectors to hide elements on the page.
 
 .TP
 .B userAgent
-Spoof User-Agent: \fB"chrome"\fR, \fB"firefox"\fR, or \fB"safari"\fR.
+Spoof User-Agent: \fB"chrome"\fR, \fB"chrome_mac"\fR, \fB"chrome_linux"\fR, \fB"firefox"\fR, \fB"firefox_mac"\fR, \fB"firefox_linux"\fR, or \fB"safari"\fR.
+
 
 .TP
 .B interact

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.6 ===
+// === Network scanner script (nwss.js) v2.0.8 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -44,6 +44,8 @@ const { performPageInteraction, createInteractionConfig } = require('./lib/inter
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
+// Enhanced site data clearing functionality
+const { clearSiteData } = require('./lib/clear_sitedata');
 
 // Fast setTimeout helper for Puppeteer 22.x compatibility
 // Uses standard Promise constructor for better performance than node:timers/promises
@@ -127,7 +129,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.6'; // Script version
+const VERSION = '2.0.8'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -505,7 +507,7 @@ Redirect Handling Options:
   resourceTypes: ["script", "stylesheet"]     Only process requests of these resource types (default: all types)
   interact: true/false                         Simulate mouse movements/clicks
   isBrave: true/false                          Spoof Brave browser detection
-  userAgent: "chrome"|"firefox"|"safari"        Custom desktop User-Agent
+  userAgent: "chrome"|"chrome_mac"|"chrome_linux"|"firefox"|"firefox_mac"|"firefox_linux"|"safari"  Custom desktop User-Agent
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
   delay: <milliseconds>                        Delay after load (default: 4000)
   reload: <number>                             Reload page n times after load (default: 1)
@@ -1926,24 +1928,10 @@ function setupFrameHandling(page, forceDebug) {
 	  
       if (siteConfig.clear_sitedata === true) {
         try {
-          let clearDataSession = null;
-          try {
-            clearDataSession = await page.target().createCDPSession();
-            await clearDataSession.send('Network.clearBrowserCookies');
-            await clearDataSession.send('Network.clearBrowserCache');
-          } finally {
-            if (clearDataSession) {
-              try { await clearDataSession.detach(); } catch (detachErr) { /* ignore */ }
-            }
-          }
-          await page.evaluate(() => {
-            localStorage.clear();
-            sessionStorage.clear();
-            indexedDB.databases().then(dbs => dbs.forEach(db => indexedDB.deleteDatabase(db.name)));
-          });
+          const clearResult = await clearSiteData(page, currentUrl, forceDebug);
           if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data for ${currentUrl}`));
         } catch (clearErr) {
-          console.warn(messageColors.warn(`[clear_sitedata failed] ${currentUrl}: ${clearErr.message}`));
+          if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Failed for ${currentUrl}: ${clearErr.message}`));
         }
       }
 
@@ -1969,7 +1957,11 @@ function setupFrameHandling(page, forceDebug) {
    if (useCurl && siteConfig.userAgent) {
      const userAgents = {
        chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+       chrome_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+       chrome_linux: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
        firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/142.0",
+       firefox_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0",
+       firefox_linux: "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0",
        safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"
      };
      curlUserAgent = userAgents[siteConfig.userAgent.toLowerCase()] || '';
@@ -3116,24 +3108,10 @@ function setupFrameHandling(page, forceDebug) {
 
         if (siteConfig.clear_sitedata === true) {
           try {
-            let reloadClearSession = null;
-            try {
-              reloadClearSession = await page.target().createCDPSession();
-              await reloadClearSession.send('Network.clearBrowserCookies');
-              await reloadClearSession.send('Network.clearBrowserCache');
-            } finally {
-              if (reloadClearSession) {
-                try { await reloadClearSession.detach(); } catch (detachErr) { /* ignore */ }
-              }
-            }
-            await page.evaluate(() => {
-              localStorage.clear();
-              sessionStorage.clear();
-              indexedDB.databases().then(dbs => dbs.forEach(db => indexedDB.deleteDatabase(db.name)));
-            });
+            const clearResult = await clearSiteData(page, currentUrl, forceDebug, true); // Quick mode for reloads
             if (forceDebug) console.log(formatLogMessage('debug', `Cleared site data before reload #${i} for ${currentUrl}`));
           } catch (reloadClearErr) {
-            console.warn(messageColors.warn(`[clear_sitedata before reload failed] ${currentUrl}: ${reloadClearErr.message}`));
+            if (forceDebug) console.log(formatLogMessage('debug', `[clear_sitedata] Before reload failed for ${currentUrl}`));
           }
         }
         
@@ -3393,10 +3371,13 @@ function setupFrameHandling(page, forceDebug) {
   // Hang detection for debugging concurrency issues
   let currentBatchInfo = { batchStart: 0, batchSize: 0 };
   const hangDetectionInterval = setInterval(() => {
+  // Only show hang detection messages in debug mode
+  if (forceDebug) {
     const currentBatch = Math.floor(currentBatchInfo.batchStart / RESOURCE_CLEANUP_INTERVAL) + 1;
     const totalBatches = Math.ceil(totalUrls / RESOURCE_CLEANUP_INTERVAL);
     console.log(formatLogMessage('debug', `[HANG CHECK] Processed: ${processedUrlCount}/${totalUrls} URLs, Batch: ${currentBatch}/${totalBatches}, Current batch size: ${currentBatchInfo.batchSize}`));
     console.log(formatLogMessage('debug', `[HANG CHECK] URLs since cleanup: ${urlsSinceLastCleanup}, Recent failures: ${results.slice(-3).filter(r => !r.success).length}/3`));
+    }
   }, 30000); // Check every 30 seconds
 
   // Process URLs in batches to maintain concurrency while allowing browser restarts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.6",
+  "version": "2.0.8",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {