@fanboynz/network-scanner 2.0.7 → 2.0.9

package/README.md

@@ -134,7 +134,7 @@ Example:
 | Field                | Values | Default | Description |
 |:---------------------|:-------|:-------:|:------------|
 | `url`                | String or Array |   -     | Website URL(s) to scan |
-| `userAgent`          | `chrome`, `firefox`, `safari` | - | User agent for page (latest versions: Chrome 131, Firefox 133, Safari 18.2) |
+| `userAgent`          | `chrome`, `chrome_mac`, `chrome_linux`, `firefox`, `firefox_mac`, `firefox_linux`, `safari` | - | User agent for page |
 | `filterRegex`        | String or Array | `.*` | Regex or list of regexes to match requests |
 | `regex_and`          | Boolean | `false` | Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL |
 | `comments`           | String or Array | - | String of comments or references |
@@ -248,6 +248,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `isBrave`            | Boolean | `false` | Spoof Brave browser detection |
 | `evaluateOnNewDocument` | Boolean | `false` | Inject fetch/XHR interceptor in page |
 | `cdp`                | Boolean | `false` | Enable CDP logging for this site |
+| `cdp_specific`       | Array | - | Enable CDP logging only for specific domains in the URL list |
 | `css_blocked`        | Array | - | CSS selectors to hide elements |
 | `source`             | Boolean | `false` | Save page source HTML after load |
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |

package/lib/cdp.js

@@ -116,6 +116,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
  * @param {boolean} options.enableCDP - Global CDP flag (from --cdp command line)
  * @param {boolean} options.siteSpecificCDP - Site-specific CDP flag (from config)
  * @param {boolean} options.forceDebug - Debug logging flag
+ * @param {string} options.currentUrl - Current URL for domain-specific CDP decisions
  * @returns {Promise<object>} CDP session object with cleanup method
  */
 async function createCDPSession(page, currentUrl, options = {}) {
@@ -132,10 +133,14 @@ async function createCDPSession(page, currentUrl, options = {}) {
 
   // Log which CDP mode is being used
   if (forceDebug) {
+    const urlHostname = (() => {
+      try { return new URL(currentUrl).hostname; } catch { return 'unknown'; }
+    })();
+
     if (enableCDP) {
-      console.log(formatLogMessage('debug', `CDP logging globally enabled by --cdp, applying to page: ${currentUrl}`));
+      console.log(formatLogMessage('debug', `[cdp] Global CDP enabled by --cdp flag for ${urlHostname}`));
     } else if (siteSpecificCDP === true) {
-      console.log(formatLogMessage('debug', `CDP logging enabled for page ${currentUrl} via site-specific 'cdp: true' config.`));
+      console.log(formatLogMessage('debug', `[cdp] Site-specific CDP enabled for ${urlHostname} (via cdp: true or cdp_specific domain match)`));
     }
   }
 
@@ -163,14 +168,22 @@ async function createCDPSession(page, currentUrl, options = {}) {
       // Extract hostname for logging context (handles URL parsing errors gracefully)
       let hostnameForLog = 'unknown-host';
       try {
-        hostnameForLog = new URL(currentUrl).hostname;
+        const currentHostname = new URL(currentUrl).hostname;
+        const requestHostname = new URL(requestUrl).hostname;
+        // Show both hostnames if different (cross-domain requests)
+        if (currentHostname !== requestHostname) {
+          hostnameForLog = `${currentHostname}?${requestHostname}`;
+        } else {
+          hostnameForLog = currentHostname;
+        }
       } catch (_) { 
         // Ignore URL parsing errors for logging context
       }
       
-      // Log the request with context - customize this for your needs
-      // Format: [cdp][hostname] METHOD url (initiator: type)
-      console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
+ // Log the request with context only if debug mode is enabled
+ if (forceDebug) {
+   console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
+ }
     });
 
     if (forceDebug) {
@@ -200,6 +213,15 @@ async function createCDPSession(page, currentUrl, options = {}) {
   } catch (cdpErr) {
     cdpSession = null; // Reset on failure
     
+    // Enhanced error context for CDP domain-specific debugging
+    const urlContext = (() => {
+      try {
+        return new URL(currentUrl).hostname;
+      } catch {
+        return currentUrl.substring(0, 50) + '...';
+      }
+    })();
+
     // Categorize CDP errors for proper handling
     // Enhanced error handling for Puppeteer 20+ error patterns
     if (cdpErr.message.includes('Network.enable timed out') || 
@@ -236,9 +258,10 @@ async function createCDPSession(page, currentUrl, options = {}) {
  *
  * @param {object} siteConfig - Site configuration object
  * @param {boolean} globalCDP - Global CDP flag
+ * @param {Array} cdpSpecificDomains - Array of domains for cdp_specific feature
  * @returns {object} Validation result with recommendations
  */
-function validateCDPConfig(siteConfig, globalCDP) {
+function validateCDPConfig(siteConfig, globalCDP, cdpSpecificDomains = []) {
   const warnings = [];
   const recommendations = [];
   
@@ -247,8 +270,25 @@ function validateCDPConfig(siteConfig, globalCDP) {
     warnings.push('Site-specific CDP disabled but global CDP is enabled - global setting will override');
   }
   
+  // Validate cdp_specific configuration
+  if (siteConfig.cdp_specific) {
+    if (!Array.isArray(siteConfig.cdp_specific)) {
+      warnings.push('cdp_specific must be an array of domain strings');
+    } else if (siteConfig.cdp_specific.length === 0) {
+      warnings.push('cdp_specific is empty - no domains will have CDP enabled');
+    } else {
+      // Validate domain format
+      const invalidDomains = siteConfig.cdp_specific.filter(domain => {
+        return typeof domain !== 'string' || domain.trim() === '';
+      });
+      if (invalidDomains.length > 0) {
+        warnings.push(`cdp_specific contains invalid domains: ${invalidDomains.join(', ')}`);
+      }
+    }
+  }
+  
   // Performance recommendations
-  if (globalCDP || siteConfig.cdp === true) {
+  if (globalCDP || siteConfig.cdp === true || (siteConfig.cdp_specific && siteConfig.cdp_specific.length > 0)) {
     recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
     
     if (siteConfig.timeout && siteConfig.timeout < 30000) {

package/nwss.1

@@ -280,7 +280,7 @@ Array of CSS selectors to hide elements on the page.
 
 .TP
 .B userAgent
-Spoof User-Agent: \fB"chrome"\fR, \fB"firefox"\fR, or \fB"safari"\fR.
+Spoof User-Agent: \fB"chrome"\fR, \fB"chrome_mac"\fR, \fB"chrome_linux"\fR, \fB"firefox"\fR, \fB"firefox_mac"\fR, \fB"firefox_linux"\fR, or \fB"safari"\fR.
 
 .TP
 .B interact
@@ -441,6 +441,10 @@ Boolean. Inject Fetch/XHR interceptor scripts into page context.
 .B cdp
 Boolean. Enable Chrome DevTools Protocol logging for this specific site.
 
+.TP
+.B cdp_specific
+Array of domain names. Enable Chrome DevTools Protocol logging only for URLs matching these specific domains within a multi-URL site configuration. Takes precedence over \fBcdp: false\fR but is ignored if \fBcdp: true\fR is set. Supports exact hostname matching and subdomain matching (e.g., "example.com" matches both "example.com" and "subdomain.example.com"). Useful for selective debugging of network requests on specific domains while avoiding CDP overhead on others.
+
 .TP
 .B source
 Boolean. Save page source HTML after loading.
@@ -879,6 +883,26 @@ node nwss.js --ignore-cache --debug -o rules.txt
 }
 .EE
 
+.SS Selective CDP logging for specific domains:
+.EX
+{
+  "url": [
+    "https://site1.com/page1",
+    "https://debug-target.com/page2", 
+    "https://site2.com/page3"
+  ],
+  "filterRegex": "\\\\.(space|website)\\\\b",
+  "cdp_specific": ["debug-target.com"],
+  "resourceTypes": ["script", "fetch"],
+  "comments": [
+    "CDP enabled only for debug-target.com",
+    "Other URLs run without CDP overhead"
+  ]
+}
+.EE
+
+Note: If \fBcdp: true\fR is also set, \fBcdp_specific\fR is ignored and CDP is enabled for all URLs.
+
 .SS FlowProxy protection handling:
 .EX
 {

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.7 ===
+// === Network scanner script (nwss.js) v2.0.9 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -129,7 +129,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.7'; // Script version
+const VERSION = '2.0.9'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -507,7 +507,7 @@ Redirect Handling Options:
   resourceTypes: ["script", "stylesheet"]     Only process requests of these resource types (default: all types)
   interact: true/false                         Simulate mouse movements/clicks
   isBrave: true/false                          Spoof Brave browser detection
-  userAgent: "chrome"|"firefox"|"safari"        Custom desktop User-Agent
+  userAgent: "chrome"|"chrome_mac"|"chrome_linux"|"firefox"|"firefox_mac"|"firefox_linux"|"safari"  Custom desktop User-Agent
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
   delay: <milliseconds>                        Delay after load (default: 4000)
   reload: <number>                             Reload page n times after load (default: 1)
@@ -555,6 +555,7 @@ FlowProxy Protection Options:
 Advanced Options:
   evaluateOnNewDocument: true/false           Inject fetch/XHR interceptor in page (for this site)
   cdp: true/false                            Enable CDP logging for this site Inject fetch/XHR interceptor in page
+  cdp_specific: ["domain1.com", "domain2.com"] Enable CDP logging only for specific domains in the URL list
   interact_duration: <milliseconds>           Duration of interaction simulation (default: 2000)
   interact_scrolling: true/false              Enable scrolling simulation (default: true)
   interact_clicks: true/false                 Enable element clicking simulation (default: false)
@@ -923,6 +924,30 @@ function safeGetDomain(url, getFullHostname = false) {
   }
 }
 
+/**
+ * Checks if a URL matches any domain in the cdp_specific list
+ * @param {string} url - The URL to check
+ * @param {Array} cdpSpecificList - Array of domains that should have CDP enabled
+ * @returns {boolean} True if URL matches a domain in the list
+ */
+function shouldEnableCDPForUrl(url, cdpSpecificList) {
+  if (!cdpSpecificList || !Array.isArray(cdpSpecificList) || cdpSpecificList.length === 0) {
+    return false;
+  }
+  
+  try {
+    const urlHostname = new URL(url).hostname;
+    return cdpSpecificList.some(domain => {
+      // Remove protocol if present and clean domain
+      const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
+      // Match exact hostname or subdomain
+      return urlHostname === cleanDomain || urlHostname.endsWith('.' + cleanDomain);
+    });
+  } catch (urlErr) {
+    return false;
+  }
+}
+
 /**
  * Outputs dry run results to console with formatted display
  * If outputFile is specified, also captures output for file writing
@@ -1475,6 +1500,24 @@ function setupFrameHandling(page, forceDebug) {
       return { url: currentUrl, rules: [], success: false, skipped: true };
     }
 
+    // Determine CDP enablement based on cdp_specific or traditional cdp setting
+    let shouldEnableCDPForThisUrl = false;
+    if (siteConfig.cdp === true) {
+      // If cdp: true is set, enable CDP for all URLs and ignore cdp_specific
+      shouldEnableCDPForThisUrl = true;
+      if (forceDebug && siteConfig.cdp_specific) {
+        console.log(formatLogMessage('debug', `CDP enabled for all URLs via cdp: true - ignoring cdp_specific for ${currentUrl}`));
+      }
+    } else if (siteConfig.cdp_specific && Array.isArray(siteConfig.cdp_specific)) {
+      // Only use cdp_specific if cdp is not explicitly set to true
+      shouldEnableCDPForThisUrl = shouldEnableCDPForUrl(currentUrl, siteConfig.cdp_specific);
+      if (forceDebug && shouldEnableCDPForThisUrl) {
+        console.log(formatLogMessage('debug', `CDP enabled for ${currentUrl} via cdp_specific domain match`));
+      }
+    } else {
+      shouldEnableCDPForThisUrl = false;
+    }
+
     let page = null;
     let cdpSession = null;
     let cdpSessionManager = null;
@@ -1884,7 +1927,7 @@ function setupFrameHandling(page, forceDebug) {
       try {
         cdpSessionManager = await createCDPSession(page, currentUrl, {
           enableCDP,
-          siteSpecificCDP: siteConfig.cdp,
+          siteSpecificCDP: shouldEnableCDPForThisUrl,
           forceDebug
         });
       } catch (cdpErr) {
@@ -1957,7 +2000,11 @@ function setupFrameHandling(page, forceDebug) {
    if (useCurl && siteConfig.userAgent) {
      const userAgents = {
        chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+       chrome_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+       chrome_linux: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
        firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/142.0",
+       firefox_mac: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0",
+       firefox_linux: "Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0",
        safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"
      };
      curlUserAgent = userAgents[siteConfig.userAgent.toLowerCase()] || '';
@@ -3116,12 +3163,18 @@ function setupFrameHandling(page, forceDebug) {
       if (useForceReload && !reloadSuccess) {
         // Attempt force reload: disable cache, reload, re-enable cache
           try {
-          // Add timeout protection for setCacheEnabled operations
+          // Timeout-protected cache disable
           await Promise.race([
             page.setCacheEnabled(false),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('setCacheEnabled(false) timeout')), 5000)
-            )
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache disable timeout')), 8000))
+          ]);
+          
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
+          
+          // Timeout-protected cache enable
+          await Promise.race([
+            page.setCacheEnabled(true),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Cache enable timeout')), 8000))
           ]);
           
           await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeout, 12000) });
@@ -3345,7 +3398,7 @@ function setupFrameHandling(page, forceDebug) {
     urlsToProcess.forEach(url => {
       allTasks.push({
         url,
-        config: site,
+        config: { ...site, _originalUrl: url }, // Preserve original URL for CDP domain checking
         taskId: allTasks.length // For tracking
       });
     });
@@ -3863,4 +3916,4 @@ function setupFrameHandling(page, forceDebug) {
   if (forceDebug) console.log(formatLogMessage('debug', `About to exit process...`));
   process.exit(0);
   
-})();
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.7",
+  "version": "2.0.9",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {