@fanboynz/network-scanner 2.0.61 → 2.0.63

package/README.md

@@ -5,7 +5,7 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 - Scan websites and detect matching third-party or first-party resources
 - Output Adblock-formatted blocking rules
 - Support for multiple filters per site
-- Grouped titles (! <url>) before site matches
+- Grouped titles (! <url>) before site matches, including redirect source and matching regex
 - Ignore unwanted domains (global and per-site)
 - Block unwanted domains during scan (simulate adblock)
 - Support Chrome, Firefox, Safari user agents (desktop or mobile)
@@ -64,6 +64,7 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 | `--headful`                 | Launch browser with GUI (not headless) |
 | `--keep-open`               | Keep browser and tabs open after scan completes (use with `--headful` for debugging) |
 | `--use-puppeteer-core`      | Use `puppeteer-core` with system Chrome instead of bundled Chromium |
+| `--use-obscura`             | Connect to running Obscura CDP server (`ws://127.0.0.1:9222` or `OBSCURA_WS` env). Skips fingerprint injection — Obscura provides built-in stealth |
 | `--load-extension <path>`   | Load unpacked Chrome extension from directory (can be used multiple times) |
 | `--dns-cache`               | Persist dig/whois results to disk between runs (14hr TTL, `.digcache`/`.whoiscache`) |
 | `--block-ads=<files>`       | Block ads using EasyList format rules (comma-separated: `easylist.txt,easyprivacy.txt`) |
@@ -448,7 +449,7 @@ node nwss.js config-clean2.json --debug             # .nwssconfig + debug overri
 node nwss.js config-other.json --max-concurrent 5   # no match in .nwssconfig, uses CLI flags
 ```
 
-**Supported settings:** `output`, `max_concurrent`, `dns_cache`, `cache_requests`, `dumpurls`, `remove_tempfiles`, `color`, `remove_dupes`, `compress_logs`, `debug`, `silent`, `verbose`, `headful`, `keep_open`, `dry_run`, `titles`, `sub_domains`, `no_interact`, `ghost_cursor`, `plain`, `cdp`, `dnsmasq`, `unbound`, `privoxy`, `pihole`, `eval_on_doc`, `use_puppeteer_core`, `ignore_cache`, `clear_cache`, `block_ads`, `compare`, `localhost`, `append`.
+**Supported settings:** `output`, `max_concurrent`, `dns_cache`, `cache_requests`, `dumpurls`, `remove_tempfiles`, `color`, `remove_dupes`, `compress_logs`, `debug`, `silent`, `verbose`, `headful`, `keep_open`, `dry_run`, `titles`, `sub_domains`, `no_interact`, `ghost_cursor`, `plain`, `cdp`, `dnsmasq`, `unbound`, `privoxy`, `pihole`, `eval_on_doc`, `use_puppeteer_core`, `use_obscura`, `ignore_cache`, `clear_cache`, `block_ads`, `compare`, `localhost`, `append`.
 
 **Priority:** CLI flags > `.nwssconfig` > hardcoded defaults.
 
@@ -461,6 +462,7 @@ These options go at the root level of your config.json:
 | Field                | Values | Default | Description |
 |:---------------------|:-------|:-------:|:------------|
 | `ignoreDomains`      | Array | - | Domains to completely ignore (supports wildcards like `*.ads.com`) |
+| `ignoreDomainsByUrl` | Array | - | Regex patterns; if a request URL matches, the request's root domain is dynamically ignored for the rest of the scan (e.g. `["\\/jwplayer\\/", "\\/build\\/assets\\/"]`) |
 | `blocked`            | Array | - | Global regex patterns to block requests (combined with per-site blocked) |
 | `whois_server_mode`  | String | `"random"` | Default server selection mode for all sites |
 | `ignore_similar`     | Boolean | `true` | Ignore domains similar to already found domains |

package/config.json

@@ -38,7 +38,7 @@
  "sites": [
     {
     "url": "https://www.anandtech.com/",
-    "filterRegex": ".",
+    "filterRegex": "teststring",
     "resourceTypes": ["script", "xhr", "document"],
     "reload": 1,
     "timeout": 25000,
@@ -50,7 +50,7 @@
   },
   {
     "url": "https://www.tomshardware.com/",
-    "filterRegex": ".",
+    "filterRegex": "anotherstrng",
     "resourceTypes": ["all"],
     "reload": 2,
     "timeout": 25000,
@@ -61,7 +61,7 @@
   },
   {
     "url": ["https://www.tomshardware.com/", "https://www.anandtech.com/"],
-    "filterRegex": ".",
+    "filterRegex": "morestrings",
     "resourceTypes": ["all"],
     "reload": 2,
     "timeout": 25000,

package/lib/output.js

@@ -324,35 +324,40 @@ function buildOutputLines(results, options = {}) {
   const { showTitles = false, removeDupes = false, ignoreDomains = [], forLogFile = false } = options;
   
   // Consolidate rules from all results, handling multiple results for same URL
-  const consolidatedRules = new Map(); // URL -> Set of rules
+  const consolidatedRules = new Map(); // URL -> { rules: Set, originalUrl, regexes: Set }
   let successfulPageLoads = 0;
-  
+
   results.forEach(result => {
     if (result) {
       if (result.success) {
         successfulPageLoads++;
       }
       if (result.rules && result.rules.length > 0) {
-        // Consolidate rules by URL to handle multiple site entries for same URL
         if (!consolidatedRules.has(result.url)) {
-          consolidatedRules.set(result.url, new Set());
+          consolidatedRules.set(result.url, { rules: new Set(), originalUrl: result.originalUrl || result.url, regexes: new Set() });
+        }
+        const entry = consolidatedRules.get(result.url);
+        result.rules.forEach(rule => entry.rules.add(rule));
+        if (Array.isArray(result.matchedRegexes)) {
+          result.matchedRegexes.forEach(rx => entry.regexes.add(rx));
+        }
+        // Prefer the original URL from any result entry that has one different from final
+        if (result.originalUrl && result.originalUrl !== result.url) {
+          entry.originalUrl = result.originalUrl;
         }
-        
-        // Add all rules from this result to the consolidated set
-        result.rules.forEach(rule => {
-          consolidatedRules.get(result.url).add(rule);
-        });
       }
     }
   });
 
   // Convert consolidated rules back to array format
   const finalSiteRules = [];
-  consolidatedRules.forEach((rulesSet, url) => {
-    if (rulesSet.size > 0) {
-      finalSiteRules.push({ 
-        url: url, 
-        rules: Array.from(rulesSet) 
+  consolidatedRules.forEach((entry, url) => {
+    if (entry.rules.size > 0) {
+      finalSiteRules.push({
+        url: url,
+        originalUrl: entry.originalUrl,
+        regexes: Array.from(entry.regexes),
+        rules: Array.from(entry.rules)
       });
     }
   });
@@ -362,35 +367,41 @@ function buildOutputLines(results, options = {}) {
   const outputLinesWithTitles = [];
   let filteredOutCount = 0;
   
-  for (const { url, rules } of finalSiteRules) {
+  for (const { url, originalUrl, regexes, rules } of finalSiteRules) {
     if (rules.length > 0) {
+      // Build title comments — include redirect source if URL changed and matched regex(es)
+      const titleLines = [`! ${url}`];
+      if (originalUrl && originalUrl !== url) {
+        titleLines.push(`! Redirected from: ${originalUrl}`);
+      }
+      if (regexes && regexes.length > 0) {
+        titleLines.push(`! Regex: ${regexes.join(', ')}`);
+      }
+
       // Regular output (for -o files and console) - only add titles if --titles flag used
       if (showTitles) {
-        outputLines.push(`! ${url}`);
+        outputLines.push(...titleLines);
       }
-      
+
       // Filter out ignored domains from rules
       const filteredRules = rules.filter(rule => {
         const domain = extractDomainFromRule(rule);
         if (domain && matchesIgnoreDomain(domain, ignoreDomains)) {
           filteredOutCount++;
-    
-          // Log each filtered domain
           if (options.forceDebug) {
             console.log(formatLogMessage('debug', `[output-filter] Removed rule matching ignoreDomains: ${rule} (domain: ${domain})`));
           } else if (!options.silentMode) {
             console.log(formatLogMessage('info', `Filtered out: ${domain}`));
-         }
-
+          }
           return false;
         }
         return true;
       });
-      
+
       outputLines.push(...filteredRules);
-      
+
       // Output with titles (for auto-saved log files) - always add titles
-      outputLinesWithTitles.push(`! ${url}`);
+      outputLinesWithTitles.push(...titleLines);
       outputLinesWithTitles.push(...filteredRules);
     }
   }

package/lib/smart-cache.js

@@ -867,14 +867,14 @@ class SmartCache {
       netToolsCacheSize: this.netToolsCache.size,
       similarityCacheSize: this.similarityCache.size,
       regexCacheSize: this.regexCache.size,
-      requestHitRate: this._enableRequest ? 
+      requestHitRate: (this._enableRequest && this.requestCache) ?
         (requestHitRate * 100).toFixed(2) + '%' : '0% (disabled)',
-      requestCacheSize: this._enableRequest ? this.requestCache.size : 0,
-      requestCacheMemoryMB: this._enableRequest ? 
+      requestCacheSize: (this._enableRequest && this.requestCache) ? this.requestCache.size : 0,
+      requestCacheMemoryMB: (this._enableRequest && this.requestCache) ?
         Math.round((this.requestCache.calculatedSize || 0) / 1048576) : 0,
-      totalCacheEntries: this.domainCache.size + this.patternCache.size + 
-        this.responseCache.size + this.netToolsCache.size + 
-        this.similarityCache.size + this.regexCache.size + (this._enableRequest ? this.requestCache.size : 0),
+      totalCacheEntries: this.domainCache.size + this.patternCache.size +
+        this.responseCache.size + this.netToolsCache.size +
+        this.similarityCache.size + this.regexCache.size + ((this._enableRequest && this.requestCache) ? this.requestCache.size : 0),
       memoryUsageMB: Math.round(heapUsed / 1048576),
       memoryMaxMB: Math.round(maxHeap / 1048576),
       memoryUsagePercent: ((heapUsed / maxHeap) * 100).toFixed(1) + '%',

package/nwss.js

@@ -185,9 +185,19 @@ if (fs.existsSync(NWSSCONFIG_PATH)) {
     const nwssConfig = JSON.parse(fs.readFileSync(NWSSCONFIG_PATH, 'utf-8'));
     // Find which config file is being used (--custom-json <file> or positional .json arg)
     const customJsonIdx = args.findIndex(arg => arg === '--custom-json');
+    const positionalJson = (customJsonIdx === -1)
+      ? args.find(a => a.endsWith('.json') && !a.startsWith('--'))
+      : null;
     const configFilename = (customJsonIdx !== -1 && args[customJsonIdx + 1])
       ? args[customJsonIdx + 1]
-      : args.find(a => a.endsWith('.json') && !a.startsWith('--'));
+      : positionalJson;
+
+    // If a positional .json was used (not --custom-json), wire it to --custom-json
+    // so the real config loader picks it up instead of defaulting to config.json
+    if (positionalJson && customJsonIdx === -1) {
+      args.push('--custom-json', positionalJson);
+      process.argv.push('--custom-json', positionalJson);
+    }
 
     if (configFilename && nwssConfig.configs && nwssConfig.configs[configFilename]) {
       const settings = nwssConfig.configs[configFilename];
@@ -687,6 +697,7 @@ Validation Options:
   
 Global config.json options:
   ignoreDomains: ["domain.com", "*.ads.com"]     Domains to completely ignore (supports wildcards)
+  ignoreDomainsByUrl: ["regex1", "regex2"]       Regex patterns; if any request URL matches, the request's root domain is ignored for the rest of the scan
   blocked: ["regex1", "regex2"]                   Global regex patterns to block requests (combined with per-site blocked)
   whois_server_mode: "random" or "cycle"      Default server selection mode for all sites (default: random)
   ignore_similar: true/false                      Ignore domains similar to already found domains (default: true)
@@ -854,8 +865,9 @@ try {
 // Extract config values while ignoring 'comments' field at global and site levels
 const { 
   sites = [], 
-  ignoreDomains = [], 
-  blocked: globalBlocked = [], 
+  ignoreDomains = [],
+  ignoreDomainsByUrl = [],
+  blocked: globalBlocked = [],
   whois_delay = 3000, 
   whois_server_mode = 'random', 
   ignore_similar = true, 
@@ -901,6 +913,15 @@ for (const pattern of ignoreDomains) {
   }
 }
 
+// Compile ignoreDomainsByUrl patterns once — match request URLs to dynamically ignore domains
+const _ignoreDomainsByUrlRegexes = Array.isArray(ignoreDomainsByUrl)
+  ? ignoreDomainsByUrl.map(p => {
+      try { return getCompiledRegex(p); } catch { return null; }
+    }).filter(r => r)
+  : [];
+// Runtime Set of domains marked ignored by URL pattern matches — shared across all sites in this scan
+const _dynamicallyIgnoredDomains = new Set();
+
 // Apply global configuration overrides with validation
 // Priority: Command line args > config.json > defaults
 const MAX_CONCURRENT_SITES = (() => {
@@ -1312,6 +1333,8 @@ function shouldBypassCacheForUrl(url, siteConfig) {
 // Cache compiled wildcard regexes to avoid recompilation on every request
 const _wildcardRegexCache = new Map();
 function matchesIgnoreDomain(domain, ignorePatterns) {
+  // Dynamically ignored domains (from URL pattern matches via ignoreDomainsByUrl)
+  if (_dynamicallyIgnoredDomains.has(domain)) return true;
   // Fast path: exact match or suffix match against Set (O(n) for parts, but no regex)
   if (_ignoreDomainsExact.size > 0) {
     if (_ignoreDomainsExact.has(domain)) return true;
@@ -1789,6 +1812,10 @@ function setupFrameHandling(page, forceDebug) {
    * @returns {Promise<object>} A promise that resolves to an object containing scan results.
    */
   async function processUrl(currentUrl, siteConfig, browserInstance) {
+    // Preserve the original URL (before any redirect) for output display
+    const originalRequestedUrl = currentUrl;
+    // Track regex patterns that produced matches (for title comments in output)
+    const matchedRegexPatterns = new Set();
     // V8 Optimization: Single destructuring to avoid multiple property lookups
     const {
       firstParty,
@@ -2553,6 +2580,11 @@ function setupFrameHandling(page, forceDebug) {
       const blockedRegexes = Array.isArray(siteConfig.blocked)
         ? siteConfig.blocked.map(pattern => getCompiledRegex(pattern))
         : [];
+
+      // Pre-build Set for O(1) resourceType lookups (fired per request)
+      const allowedResourceTypesSet = Array.isArray(siteConfig.resourceTypes)
+        ? new Set(siteConfig.resourceTypes)
+        : null;
 		
       // Combine site-specific with pre-compiled global blocked patterns
       const allBlockedRegexes = blockedRegexes.length > 0
@@ -2774,9 +2806,22 @@ function setupFrameHandling(page, forceDebug) {
           bufferedLogWrite(debugLogFile, logEntry);
         }
         const reqUrl = checkedUrl;
-        
+
         const reqDomain = perSiteSubDomains ? fullSubdomain : checkedRootDomain;
 
+        // ignoreDomainsByUrl — if any pattern matches this URL, mark the root domain as ignored for the rest of the scan
+        if (_ignoreDomainsByUrlRegexes.length > 0 && checkedRootDomain && !_dynamicallyIgnoredDomains.has(checkedRootDomain)) {
+          for (let i = 0; i < _ignoreDomainsByUrlRegexes.length; i++) {
+            if (_ignoreDomainsByUrlRegexes[i].test(reqUrl)) {
+              _dynamicallyIgnoredDomains.add(checkedRootDomain);
+              if (forceDebug) {
+                console.log(formatLogMessage('debug', `[ignoreDomainsByUrl] ${checkedRootDomain} ignored — matched pattern: ${_ignoreDomainsByUrlRegexes[i].source}`));
+              }
+              break;
+            }
+          }
+        }
+
         let blockedMatchIndex = -1;
         for (let i = 0; i < allBlockedRegexes.length; i++) {
           if (allBlockedRegexes[i].test(reqUrl)) {
@@ -2801,14 +2846,14 @@ function setupFrameHandling(page, forceDebug) {
             if (reqDomain && !matchesIgnoreDomain(reqDomain, ignoreDomains)) {
               for (const re of regexes) {
                 if (re.test(reqUrl)) {
+                  const evenBlockedRegexPattern = re.source;
                   const resourceType = request.resourceType();
-                  
+
                   // Apply same filtering logic as unblocked requests
-                  const allowedResourceTypes = siteConfig.resourceTypes;
-                  if (!allowedResourceTypes || !Array.isArray(allowedResourceTypes) || allowedResourceTypes.includes(resourceType)) {
+                  if (!allowedResourceTypesSet || allowedResourceTypesSet.has(resourceType)) {
                     if (dryRunMode) {
                       addDryRunMatch(matchedDomains, {
-                        regex: matchedRegexPattern,
+                        regex: evenBlockedRegexPattern,
                         domain: reqDomain,
                         resourceType: resourceType,
                         fullUrl: reqUrl,
@@ -2818,10 +2863,11 @@ function setupFrameHandling(page, forceDebug) {
                     } else {
                       addMatchedDomain(reqDomain, resourceType, fullSubdomain);
                     }
-                    
+                    matchedRegexPatterns.add(evenBlockedRegexPattern);
+
                     if (siteConfig.verbose === 1) {
                       const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
-                      console.log(formatLogMessage('match', `[${simplifiedCurrentUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
+                      console.log(formatLogMessage('match', `[${simplifiedCurrentUrl}] ${reqUrl} matched regex: ${evenBlockedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
                     }
                     if (dumpUrls) {
                       const timestamp = new Date().toISOString();
@@ -2889,11 +2935,10 @@ function setupFrameHandling(page, forceDebug) {
             
            // *** UNIVERSAL RESOURCE TYPE FILTER ***
            // Check resourceTypes filter FIRST, before ANY processing (nettools, searchstring, immediate matching)
-           const allowedResourceTypes = siteConfig.resourceTypes;
-           if (allowedResourceTypes && Array.isArray(allowedResourceTypes) && allowedResourceTypes.length > 0) {
-             if (!allowedResourceTypes.includes(resourceType)) {
+           if (allowedResourceTypesSet && allowedResourceTypesSet.size > 0) {
+             if (!allowedResourceTypesSet.has(resourceType)) {
                if (forceDebug) {
-                 console.log(formatLogMessage('debug', `URL ${reqUrl} matches regex but resourceType '${resourceType}' not in allowed types [${allowedResourceTypes.join(', ')}]. Skipping ALL processing.`));
+                 console.log(formatLogMessage('debug', `URL ${reqUrl} matches regex but resourceType '${resourceType}' not in allowed types [${Array.from(allowedResourceTypesSet).join(', ')}]. Skipping ALL processing.`));
                }
                // Skip this URL entirely - doesn't match required resource types
                request.continue();
@@ -2981,6 +3026,7 @@ function setupFrameHandling(page, forceDebug) {
              } else {
                addMatchedDomain(reqDomain, resourceType);
              }
+             if (matchedRegexPattern) matchedRegexPatterns.add(matchedRegexPattern);
              if (siteConfig.verbose === 1) {
                const resourceInfo = (adblockRulesMode || siteConfig.adblock_rules) ? ` (${resourceType})` : '';
               console.log(formatLogMessage('match', `[${simplifiedCurrentUrl}] ${reqUrl} matched regex: ${matchedRegexPattern} and resourceType: ${resourceType}${resourceInfo}`));
@@ -4011,12 +4057,14 @@ function setupFrameHandling(page, forceDebug) {
       };
         const formattedRules = formatRules(matchedDomains, siteConfig, globalOptions);
         
-        return { 
-          url: currentUrl, 
-          rules: formattedRules, 
+        return {
+          url: currentUrl,
+          originalUrl: originalRequestedUrl,
+          rules: formattedRules,
           success: true,
           finalUrl: finalUrlAfterRedirect || currentUrl,
-          redirectDomains: redirectDomainsToExclude
+          redirectDomains: redirectDomainsToExclude,
+          matchedRegexes: Array.from(matchedRegexPatterns)
         };
       }
       
@@ -4072,13 +4120,15 @@ function setupFrameHandling(page, forceDebug) {
         };
         const formattedRules = formatRules(matchedDomains, siteConfig, globalOptions);
         if (forceDebug) console.log(formatLogMessage('debug', `Saving ${formattedRules.length} rules despite page load failure`));
-        return { 
-          url: currentUrl, 
-          rules: formattedRules, 
-          success: false, 
+        return {
+          url: currentUrl,
+          originalUrl: originalRequestedUrl,
+          rules: formattedRules,
+          success: false,
           hasMatches: true,
           finalUrl: finalUrlAfterRedirect || currentUrl,
-          redirectDomains: redirectDomainsToExclude
+          redirectDomains: redirectDomainsToExclude,
+          matchedRegexes: Array.from(matchedRegexPatterns)
         };
       }
       

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.61",
+  "version": "2.0.63",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {