@fanboynz/network-scanner 2.0.18 → 2.0.20

package/README.md

@@ -35,8 +35,8 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 
 | Argument                  | Description |
 |:---------------------------|:------------|
-| `--localhost`               | Output as `127.0.0.1 domain.com` |
-| `--localhost-0.0.0.0`       | Output as `0.0.0.0 domain.com` |
+| `--localhost[=IP]`          | Output as `IP domain.com` (default: 127.0.0.1) |
+|                             | Examples: `--localhost`, `--localhost=0.0.0.0`, `--localhost=192.168.1.1` |
 | `--plain`                   | Output just domains (no adblock formatting) |
 | `--dnsmasq`                 | Output as `local=/domain.com/` (dnsmasq format) |
 | `--dnsmasq-old`             | Output as `server=/domain.com/` (dnsmasq old format) |
@@ -253,6 +253,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `source`             | Boolean | `false` | Save page source HTML after load |
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |
 | `headful`            | Boolean | `false` | Launch browser with GUI for this site |
+| `localhost`          | String | - | Force custom IP format for this site (e.g., "127.0.0.1", "0.0.0.0", "192.168.1.1") |
 | `adblock_rules`      | Boolean | `false` | Generate adblock filter rules with resource types for this site |
 | `interact_duration`  | Milliseconds | `2000` | Duration of interaction simulation |
 | `interact_scrolling` | Boolean | `true` | Enable scrolling simulation |

package/lib/output.js

@@ -93,8 +93,7 @@ function extractDomainFromRule(rule) {
  * Formats a domain according to the specified output mode
  * @param {string} domain - The domain to format
  * @param {object} options - Formatting options
- * @param {boolean} options.localhost - Use 127.0.0.1 format
- * @param {boolean} options.localhostAlt - Use 0.0.0.0 format  
+ * @param {string|null} options.localhostIP - Use custom IP format (e.g., '127.0.0.1', '0.0.0.0') 
  * @param {boolean} options.plain - Use plain domain format (no adblock syntax)
  * @param {boolean} options.adblockRules - Generate adblock filter rules with resource types
  * @param {boolean} options.dnsmasq - Use dnsmasq local format
@@ -106,7 +105,7 @@ function extractDomainFromRule(rule) {
  * @returns {string} The formatted domain
  */
 function formatDomain(domain, options = {}) {
-  const { localhost = false, localhostAlt = false, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false, resourceType = null } = options;
+  const { localhostIP = null, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false, resourceType = null } = options;
 
   
   // Validate domain length and format
@@ -132,10 +131,8 @@ function formatDomain(domain, options = {}) {
     return `server=/${domain}/`;
   } else if (unbound) {
     return `local-zone: "${domain}." always_null`;
-  } else if (localhost) {
-    return `127.0.0.1 ${domain}`;
-  } else if (localhostAlt) {
-    return `0.0.0.0 ${domain}`;
+  } else if (localhostIP) {
+    return `${localhostIP} ${domain}`;
   } else if (adblockRules && resourceType) {
     // Generate adblock filter rules with resource type modifiers
     return `||${domain}^${resourceType}`;
@@ -178,8 +175,7 @@ function mapResourceTypeToAdblockModifier(resourceType) {
  */
 function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   const {
-    localhostMode = false,
-    localhostModeAlt = false,
+    localhostIP = null,
     plainOutput = false,
     adblockRulesMode = false,
     dnsmasqMode = false,
@@ -190,8 +186,7 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   } = globalOptions;
   
   // Site-level overrides
-  const siteLocalhost = siteConfig.localhost === true;
-  const siteLocalhostAlt = siteConfig.localhost_0_0_0_0 === true;
+  const siteLocalhostIP = siteConfig.localhost || null;
   const sitePlainSetting = siteConfig.plain === true;
   const siteAdblockRules = siteConfig.adblock_rules === true;
   const siteDnsmasq = siteConfig.dnsmasq === true;
@@ -208,16 +203,14 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
     privoxyMode || sitePrivoxy,
     piholeMode || sitePihole,
     adblockRulesMode || siteAdblockRules,
-    localhostMode || siteLocalhost,
-    localhostModeAlt || siteLocalhostAlt,
+    (localhostIP || siteLocalhostIP) ? true : false,
     plainOutput || sitePlainSetting
   ].filter(Boolean).length;
   
   if (activeFormats > 1) {
     // Multiple formats specified - fall back to standard adblock format
     const formatOptions = {
-      localhost: false,
-      localhostAlt: false,
+      localhostIP: null,
       plain: false,
       adblockRules: false,
       dnsmasq: false,
@@ -240,8 +233,7 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   
   // Determine final formatting options
   const formatOptions = {
-    localhost: localhostMode || siteLocalhost,
-    localhostAlt: localhostModeAlt || siteLocalhostAlt,
+    localhostIP: siteLocalhostIP || localhostIP,
     plain: plainOutput || sitePlainSetting,
     adblockRules: adblockRulesMode || siteAdblockRules,
     dnsmasq: dnsmasqMode || siteDnsmasq,
@@ -338,8 +330,8 @@ function removeDuplicates(lines) {
 function buildOutputLines(results, options = {}) {
   const { showTitles = false, removeDupes = false, ignoreDomains = [], forLogFile = false } = options;
   
-  // Filter and collect successful results with rules
-  const finalSiteRules = [];
+  // Consolidate rules from all results, handling multiple results for same URL
+  const consolidatedRules = new Map(); // URL -> Set of rules
   let successfulPageLoads = 0;
   
   results.forEach(result => {
@@ -348,10 +340,29 @@ function buildOutputLines(results, options = {}) {
         successfulPageLoads++;
       }
       if (result.rules && result.rules.length > 0) {
-        finalSiteRules.push({ url: result.url, rules: result.rules });
+        // Consolidate rules by URL to handle multiple site entries for same URL
+        if (!consolidatedRules.has(result.url)) {
+          consolidatedRules.set(result.url, new Set());
+        }
+        
+        // Add all rules from this result to the consolidated set
+        result.rules.forEach(rule => {
+          consolidatedRules.get(result.url).add(rule);
+        });
       }
     }
   });
+
+  // Convert consolidated rules back to array format
+  const finalSiteRules = [];
+  consolidatedRules.forEach((rulesSet, url) => {
+    if (rulesSet.size > 0) {
+      finalSiteRules.push({ 
+        url: url, 
+        rules: Array.from(rulesSet) 
+      });
+    }
+  });
   
   // Build output lines
   const outputLines = [];
@@ -621,7 +632,7 @@ function handleOutput(results, config = {}) {
  * @returns {string} Human-readable format description
  */
 function getFormatDescription(options = {}) {
-  const { localhost = false, localhostAlt = false, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false } = options;
+  const { localhostIP = null, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false } = options;
   
   // Plain always takes precedence
   if (plain) {
@@ -640,10 +651,8 @@ function getFormatDescription(options = {}) {
     return 'Unbound format (local-zone: "domain.com." always_null)';
   } else if (adblockRules) {
     return 'Adblock filter rules with resource type modifiers (||domain.com^$script)';
-  } else if (localhost) {
-    return 'Localhost format (127.0.0.1 domain.com)';
-  } else if (localhostAlt) {
-    return 'Localhost format (0.0.0.0 domain.com)';
+  } else if (localhostIP) {
+    return `Localhost format (${localhostIP} domain.com)`;
   } else {
     return 'Adblock format (||domain.com^)';
   }

package/nwss.1

@@ -33,12 +33,15 @@ Enable colored console output for status messages.
 
 .SS Output Format Options
 .TP
-.B \--localhost
-Output rules as \fB127.0.0.1 domain.com\fR format for hosts file.
+.BR \--localhost [ =\fIIP\fR ]
+Output rules as \fBIP domain.com\fR format for hosts file. If no IP is specified, defaults to \fB127.0.0.1\fR.
 
-.TP
-.B \--localhost-0.0.0.0
-Output rules as \fB0.0.0.0 domain.com\fR format for hosts file.
+Examples:
+.RS
+\fB\--localhost\fR (uses 127.0.0.1)
+\fB\--localhost=0.0.0.0\fR
+\fB\--localhost=192.168.1.1\fR
+.RE
 
 .TP
 .B \--plain
@@ -578,11 +581,9 @@ Number. Output full subdomains instead of root domains (1/0).
 
 .TP
 .B localhost
-Boolean. Force localhost output format (127.0.0.1) for this site.
+String. Force custom IP format for this site. Examples: \fB"127.0.0.1"\fR, \fB"0.0.0.0"\fR, \fB"192.168.1.1"\fR, \fB"10.0.0.1"\fR.
 
-.TP
-.B localhost_0_0_0_0
-Boolean. Force localhost output format (0.0.0.0) for this site.
+When set, overrides global \fB\--localhost\fR flag for this specific site.
 
 .TP
 .B dnsmasq
@@ -970,9 +971,11 @@ Format: \fB(^|\\.)domain\\.com$\fR
 For Pi-hole regex filters. Blocks domain and subdomains at DNS level.
 
 .SS Hosts File Formats
-Flags: \fB\--localhost\fR, \fB\--localhost-0.0.0.0\fR
+Flag: \fB\--localhost[=IP]\fR
 .br
-Formats: \fB127.0.0.1 domain.com\fR, \fB0.0.0.0 domain.com\fR
+Format: \fBIP domain.com\fR (default IP: 127.0.0.1)
+.br
+Examples: \fB\--localhost\fR, \fB\--localhost=0.0.0.0\fR, \fB\--localhost=10.0.0.1\fR
 .br
 For system hosts files.
 
@@ -988,6 +991,19 @@ Format: \fBdomain.com\fR
 .br
 Simple domain list without formatting.
 
+.SS Custom IP Examples
+.EX
+# Standard localhost blocking
+node nwss.js -o hosts.txt --localhost
+
+# Block to null route
+node nwss.js -o hosts.txt --localhost=0.0.0.0
+
+# Route to local server
+node nwss.js -o hosts.txt --localhost=192.168.1.100
+node nwss.js -o hosts.txt --localhost=10.0.0.1
+.EE
+
 .SH FILES
 
 .TP
@@ -1090,6 +1106,16 @@ Report bugs to the project repository or maintainer.
 .BR unbound (8),
 .BR privoxy (8)
 
+.SH EXAMPLES OF CONFIG WITH CUSTOM LOCALHOST IP
+.EX
+{
+  "url": "https://example.com",
+  "filterRegex": "ads|tracking",
+  "localhost": "10.0.0.1"
+}
+.EE
+This configuration will output rules in the format \fB10.0.0.1 domain.com\fR for this specific site, overriding any global \fB\--localhost\fR flag.
+
 .SH AUTHORS
 Written for malware research and network security analysis.
 

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.18 ===
+// === Network scanner script (nwss.js) v2.0.20 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -130,7 +130,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.18'; // Script version
+const VERSION = '2.0.20'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -173,8 +173,12 @@ const silentMode = args.includes('--silent');
 const showTitles = args.includes('--titles');
 const dumpUrls = args.includes('--dumpurls');
 const subDomainsMode = args.includes('--sub-domains');
-const localhostMode = args.includes('--localhost');
-const localhostModeAlt = args.includes('--localhost-0.0.0.0');
+// Parse --localhost with optional IP address
+let localhostIP = null;
+const localhostIndex = args.findIndex(arg => arg.startsWith('--localhost'));
+if (localhostIndex !== -1) {
+  localhostIP = args[localhostIndex].includes('=') ? args[localhostIndex].split('=')[1] : '127.0.0.1';
+}
 const disableInteract = args.includes('--no-interact');
 const plainOutput = args.includes('--plain');
 const enableCDP = args.includes('--cdp');
@@ -230,7 +234,7 @@ if (adblockRulesMode) {
   if (!outputFile) {
     if (forceDebug) console.log(formatLogMessage('debug', `--adblock-rules ignored: requires --output (-o) to specify an output file`));
     adblockRulesMode = false;
-  } else if (localhostMode || localhostModeAlt || plainOutput || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
+  } else if (localhostIP || plainOutput || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--adblock-rules ignored: incompatible with localhost/plain output modes`));
     adblockRulesMode = false;
   }
@@ -238,7 +242,7 @@ if (adblockRulesMode) {
 
 // Validate --dnsmasq usage
 if (dnsmasqMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--dnsmasq-old ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     dnsmasqMode = false;
   }
@@ -246,7 +250,7 @@ if (dnsmasqMode) {
 
 // Validate --dnsmasq-old usage
 if (dnsmasqOldMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || unboundMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--dnsmasq-old ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     dnsmasqOldMode = false;
   }
@@ -254,7 +258,7 @@ if (dnsmasqOldMode) {
 
 // Validate --unbound usage
 if (unboundMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--unbound ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     unboundMode = false;
   }
@@ -262,7 +266,7 @@ if (unboundMode) {
 
 // Validate --privoxy usage
 if (privoxyMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--privoxy ignored: incompatible with localhost/plain/adblock-rules/dnsmasq/unbound output modes`));
     privoxyMode = false;
   }
@@ -270,7 +274,7 @@ if (privoxyMode) {
 
 // Validate --pihole usage
 if (piholeMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--pihole ignored: incompatible with localhost/plain/adblock-rules/dnsmasq/unbound/privoxy output modes`));
     piholeMode = false;
   }
@@ -430,8 +434,8 @@ Options:
   --append                       Append new rules to output file instead of overwriting (requires -o)
     
 Output Format Options:
-  --localhost                    Output as 127.0.0.1 domain.com
-  --localhost-0.0.0.0            Output as 0.0.0.0 domain.com
+  --localhost[=IP]               Output as IP domain.com (default: 127.0.0.1)
+                                 Examples: --localhost, --localhost=0.0.0.0, --localhost=192.168.1.1
   --plain                        Output just domains (no adblock formatting)
   --dnsmasq                      Output as local=/domain.com/ (dnsmasq format)
   --dnsmasq-old                  Output as server=/domain.com/ (dnsmasq old format)
@@ -1478,8 +1482,7 @@ function setupFrameHandling(page, forceDebug) {
     const allowFirstParty = siteConfig.firstParty === true || siteConfig.firstParty === 1;
     const allowThirdParty = siteConfig.thirdParty === undefined || siteConfig.thirdParty === true || siteConfig.thirdParty === 1;
     const perSiteSubDomains = siteConfig.subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhost = siteConfig.localhost === true;
-    const siteLocalhostAlt = siteConfig.localhost_0_0_0_0 === true;
+    const siteLocalhostIP = siteConfig.localhost || null;
     const cloudflarePhishBypass = siteConfig.cloudflare_phish === true;
     const cloudflareBypass = siteConfig.cloudflare_bypass === true;
     // Add redirect and same-page loop protection
@@ -3392,8 +3395,7 @@ function setupFrameHandling(page, forceDebug) {
       } else {
         // Format rules using the output module
         const globalOptions = {
-        localhostMode,
-        localhostModeAlt,
+        localhostIP,
         plainOutput,
         adblockRulesMode,
         dnsmasqMode,
@@ -3434,8 +3436,7 @@ function setupFrameHandling(page, forceDebug) {
       // For other errors, preserve any matches we found before the error
       if (matchedDomains && (matchedDomains.size > 0 || (matchedDomains instanceof Map && matchedDomains.size > 0))) {
         const globalOptions = {
-          localhostMode,
-          localhostModeAlt,
+          localhostIP,
           plainOutput,
           adblockRulesMode,
           dnsmasqMode,
@@ -3883,8 +3884,7 @@ function setupFrameHandling(page, forceDebug) {
   const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
-      localhostMode,
-      localhostModeAlt,
+      localhostIP,
       plainOutput,
       adblockRules: adblockRulesMode,
       dnsmasq: dnsmasqMode,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.18",
+  "version": "2.0.20",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {