@fanboynz/network-scanner 2.0.19 → 2.0.21

package/README.md

@@ -35,8 +35,8 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 
 | Argument                  | Description |
 |:---------------------------|:------------|
-| `--localhost`               | Output as `127.0.0.1 domain.com` |
-| `--localhost-0.0.0.0`       | Output as `0.0.0.0 domain.com` |
+| `--localhost[=IP]`          | Output as `IP domain.com` (default: 127.0.0.1) |
+|                             | Examples: `--localhost`, `--localhost=0.0.0.0`, `--localhost=192.168.1.1` |
 | `--plain`                   | Output just domains (no adblock formatting) |
 | `--dnsmasq`                 | Output as `local=/domain.com/` (dnsmasq format) |
 | `--dnsmasq-old`             | Output as `server=/domain.com/` (dnsmasq old format) |
@@ -253,6 +253,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `source`             | Boolean | `false` | Save page source HTML after load |
 | `screenshot`         | Boolean | `false` | Capture screenshot on load failure |
 | `headful`            | Boolean | `false` | Launch browser with GUI for this site |
+| `localhost`          | String | - | Force custom IP format for this site (e.g., "127.0.0.1", "0.0.0.0", "192.168.1.1") |
 | `adblock_rules`      | Boolean | `false` | Generate adblock filter rules with resource types for this site |
 | `interact_duration`  | Milliseconds | `2000` | Duration of interaction simulation |
 | `interact_scrolling` | Boolean | `true` | Enable scrolling simulation |

package/lib/output.js

@@ -93,8 +93,7 @@ function extractDomainFromRule(rule) {
  * Formats a domain according to the specified output mode
  * @param {string} domain - The domain to format
  * @param {object} options - Formatting options
- * @param {boolean} options.localhost - Use 127.0.0.1 format
- * @param {boolean} options.localhostAlt - Use 0.0.0.0 format  
+ * @param {string|null} options.localhostIP - Use custom IP format (e.g., '127.0.0.1', '0.0.0.0') 
  * @param {boolean} options.plain - Use plain domain format (no adblock syntax)
  * @param {boolean} options.adblockRules - Generate adblock filter rules with resource types
  * @param {boolean} options.dnsmasq - Use dnsmasq local format
@@ -106,7 +105,7 @@ function extractDomainFromRule(rule) {
  * @returns {string} The formatted domain
  */
 function formatDomain(domain, options = {}) {
-  const { localhost = false, localhostAlt = false, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false, resourceType = null } = options;
+  const { localhostIP = null, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false, resourceType = null } = options;
 
   
   // Validate domain length and format
@@ -132,10 +131,8 @@ function formatDomain(domain, options = {}) {
     return `server=/${domain}/`;
   } else if (unbound) {
     return `local-zone: "${domain}." always_null`;
-  } else if (localhost) {
-    return `127.0.0.1 ${domain}`;
-  } else if (localhostAlt) {
-    return `0.0.0.0 ${domain}`;
+  } else if (localhostIP) {
+    return `${localhostIP} ${domain}`;
   } else if (adblockRules && resourceType) {
     // Generate adblock filter rules with resource type modifiers
     return `||${domain}^${resourceType}`;
@@ -178,8 +175,7 @@ function mapResourceTypeToAdblockModifier(resourceType) {
  */
 function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   const {
-    localhostMode = false,
-    localhostModeAlt = false,
+    localhostIP = null,
     plainOutput = false,
     adblockRulesMode = false,
     dnsmasqMode = false,
@@ -190,8 +186,7 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   } = globalOptions;
   
   // Site-level overrides
-  const siteLocalhost = siteConfig.localhost === true;
-  const siteLocalhostAlt = siteConfig.localhost_0_0_0_0 === true;
+  const siteLocalhostIP = siteConfig.localhost || null;
   const sitePlainSetting = siteConfig.plain === true;
   const siteAdblockRules = siteConfig.adblock_rules === true;
   const siteDnsmasq = siteConfig.dnsmasq === true;
@@ -208,16 +203,14 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
     privoxyMode || sitePrivoxy,
     piholeMode || sitePihole,
     adblockRulesMode || siteAdblockRules,
-    localhostMode || siteLocalhost,
-    localhostModeAlt || siteLocalhostAlt,
+    (localhostIP || siteLocalhostIP) ? true : false,
     plainOutput || sitePlainSetting
   ].filter(Boolean).length;
   
   if (activeFormats > 1) {
     // Multiple formats specified - fall back to standard adblock format
     const formatOptions = {
-      localhost: false,
-      localhostAlt: false,
+      localhostIP: null,
       plain: false,
       adblockRules: false,
       dnsmasq: false,
@@ -240,8 +233,7 @@ function formatRules(matchedDomains, siteConfig = {}, globalOptions = {}) {
   
   // Determine final formatting options
   const formatOptions = {
-    localhost: localhostMode || siteLocalhost,
-    localhostAlt: localhostModeAlt || siteLocalhostAlt,
+    localhostIP: siteLocalhostIP || localhostIP,
     plain: plainOutput || sitePlainSetting,
     adblockRules: adblockRulesMode || siteAdblockRules,
     dnsmasq: dnsmasqMode || siteDnsmasq,
@@ -640,7 +632,7 @@ function handleOutput(results, config = {}) {
  * @returns {string} Human-readable format description
  */
 function getFormatDescription(options = {}) {
-  const { localhost = false, localhostAlt = false, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false } = options;
+  const { localhostIP = null, plain = false, adblockRules = false, dnsmasq = false, dnsmasqOld = false, unbound = false, privoxy = false, pihole = false } = options;
   
   // Plain always takes precedence
   if (plain) {
@@ -659,10 +651,8 @@ function getFormatDescription(options = {}) {
     return 'Unbound format (local-zone: "domain.com." always_null)';
   } else if (adblockRules) {
     return 'Adblock filter rules with resource type modifiers (||domain.com^$script)';
-  } else if (localhost) {
-    return 'Localhost format (127.0.0.1 domain.com)';
-  } else if (localhostAlt) {
-    return 'Localhost format (0.0.0.0 domain.com)';
+  } else if (localhostIP) {
+    return `Localhost format (${localhostIP} domain.com)`;
   } else {
     return 'Adblock format (||domain.com^)';
   }

package/nwss.1

@@ -33,12 +33,15 @@ Enable colored console output for status messages.
 
 .SS Output Format Options
 .TP
-.B \--localhost
-Output rules as \fB127.0.0.1 domain.com\fR format for hosts file.
+.BR \--localhost [ =\fIIP\fR ]
+Output rules as \fBIP domain.com\fR format for hosts file. If no IP is specified, defaults to \fB127.0.0.1\fR.
 
-.TP
-.B \--localhost-0.0.0.0
-Output rules as \fB0.0.0.0 domain.com\fR format for hosts file.
+Examples:
+.RS
+\fB\--localhost\fR (uses 127.0.0.1)
+\fB\--localhost=0.0.0.0\fR
+\fB\--localhost=192.168.1.1\fR
+.RE
 
 .TP
 .B \--plain
@@ -300,7 +303,7 @@ Number of times to reload the page (default: 1).
 
 .TP
 .B forcereload
-Boolean. Force an additional reload after reloads.
+Boolean or Array. Force cache-clearing reload for all URLs or specific domains. Can be \fBtrue\fR/\fBfalse\fR or array of domain names like \fB["domain1.com", "domain2.com"]\fR. When set to \fBtrue\fR, applies to all URLs in the site configuration. When set to an array, applies only to URLs whose domains match the specified domains. Supports exact hostname matching and subdomain matching (e.g., "example.com" matches both "example.com" and "subdomain.example.com"). Domain matching is case-insensitive and automatically handles protocols, ports, and paths.
 
 .TP
 .B timeout
@@ -578,11 +581,9 @@ Number. Output full subdomains instead of root domains (1/0).
 
 .TP
 .B localhost
-Boolean. Force localhost output format (127.0.0.1) for this site.
+String. Force custom IP format for this site. Examples: \fB"127.0.0.1"\fR, \fB"0.0.0.0"\fR, \fB"192.168.1.1"\fR, \fB"10.0.0.1"\fR.
 
-.TP
-.B localhost_0_0_0_0
-Boolean. Force localhost output format (0.0.0.0) for this site.
+When set, overrides global \fB\--localhost\fR flag for this specific site.
 
 .TP
 .B dnsmasq
@@ -783,6 +784,37 @@ With default settings (\fBignore_similar_threshold: 80\fR):
 }
 .EE
 
+.SS Force reload examples:
+.EX
+# Force reload for all URLs in a site configuration
+{
+  "url": ["https://site1.com", "https://site2.com"],
+  "filterRegex": "ads|tracking",
+  "forcereload": true
+}
+
+# Force reload only for specific domains
+{
+  "url": [
+    "https://example.com/page1",
+    "https://test.org/page2", 
+    "https://demo.net/page3"
+  ],
+  "filterRegex": "\\\\.(space|website)\\\\b",
+  "forcereload": ["example.com", "demo.net"],
+  "comments": [
+    "Only example.com and demo.net URLs get force reload",
+    "test.org URLs use standard reload"
+  ]
+}
+
+# Mixed configuration with domain-specific force reload
+{
+  "url": "https://cdn.example.com/content/page.html",
+  "forcereload": ["example.com"]
+}
+.EE
+
 .SS Configuration with documentation comments:
 .EX
 {
@@ -970,9 +1002,11 @@ Format: \fB(^|\\.)domain\\.com$\fR
 For Pi-hole regex filters. Blocks domain and subdomains at DNS level.
 
 .SS Hosts File Formats
-Flags: \fB\--localhost\fR, \fB\--localhost-0.0.0.0\fR
+Flag: \fB\--localhost[=IP]\fR
 .br
-Formats: \fB127.0.0.1 domain.com\fR, \fB0.0.0.0 domain.com\fR
+Format: \fBIP domain.com\fR (default IP: 127.0.0.1)
+.br
+Examples: \fB\--localhost\fR, \fB\--localhost=0.0.0.0\fR, \fB\--localhost=10.0.0.1\fR
 .br
 For system hosts files.
 
@@ -988,6 +1022,19 @@ Format: \fBdomain.com\fR
 .br
 Simple domain list without formatting.
 
+.SS Custom IP Examples
+.EX
+# Standard localhost blocking
+node nwss.js -o hosts.txt --localhost
+
+# Block to null route
+node nwss.js -o hosts.txt --localhost=0.0.0.0
+
+# Route to local server
+node nwss.js -o hosts.txt --localhost=192.168.1.100
+node nwss.js -o hosts.txt --localhost=10.0.0.1
+.EE
+
 .SH FILES
 
 .TP
@@ -1090,6 +1137,16 @@ Report bugs to the project repository or maintainer.
 .BR unbound (8),
 .BR privoxy (8)
 
+.SH EXAMPLES OF CONFIG WITH CUSTOM LOCALHOST IP
+.EX
+{
+  "url": "https://example.com",
+  "filterRegex": "ads|tracking",
+  "localhost": "10.0.0.1"
+}
+.EE
+This configuration will output rules in the format \fB10.0.0.1 domain.com\fR for this specific site, overriding any global \fB\--localhost\fR flag.
+
 .SH AUTHORS
 Written for malware research and network security analysis.
 

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.19 ===
+// === Network scanner script (nwss.js) v2.0.21 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -130,7 +130,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.19'; // Script version
+const VERSION = '2.0.21'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -173,8 +173,12 @@ const silentMode = args.includes('--silent');
 const showTitles = args.includes('--titles');
 const dumpUrls = args.includes('--dumpurls');
 const subDomainsMode = args.includes('--sub-domains');
-const localhostMode = args.includes('--localhost');
-const localhostModeAlt = args.includes('--localhost-0.0.0.0');
+// Parse --localhost with optional IP address
+let localhostIP = null;
+const localhostIndex = args.findIndex(arg => arg.startsWith('--localhost'));
+if (localhostIndex !== -1) {
+  localhostIP = args[localhostIndex].includes('=') ? args[localhostIndex].split('=')[1] : '127.0.0.1';
+}
 const disableInteract = args.includes('--no-interact');
 const plainOutput = args.includes('--plain');
 const enableCDP = args.includes('--cdp');
@@ -230,7 +234,7 @@ if (adblockRulesMode) {
   if (!outputFile) {
     if (forceDebug) console.log(formatLogMessage('debug', `--adblock-rules ignored: requires --output (-o) to specify an output file`));
     adblockRulesMode = false;
-  } else if (localhostMode || localhostModeAlt || plainOutput || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
+  } else if (localhostIP || plainOutput || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--adblock-rules ignored: incompatible with localhost/plain output modes`));
     adblockRulesMode = false;
   }
@@ -238,7 +242,7 @@ if (adblockRulesMode) {
 
 // Validate --dnsmasq usage
 if (dnsmasqMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqOldMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--dnsmasq-old ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     dnsmasqMode = false;
   }
@@ -246,7 +250,7 @@ if (dnsmasqMode) {
 
 // Validate --dnsmasq-old usage
 if (dnsmasqOldMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || unboundMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || unboundMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--dnsmasq-old ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     dnsmasqOldMode = false;
   }
@@ -254,7 +258,7 @@ if (dnsmasqOldMode) {
 
 // Validate --unbound usage
 if (unboundMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || privoxyMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || privoxyMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--unbound ignored: incompatible with localhost/plain/adblock-rules/dnsmasq output modes`));
     unboundMode = false;
   }
@@ -262,7 +266,7 @@ if (unboundMode) {
 
 // Validate --privoxy usage
 if (privoxyMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || piholeMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || piholeMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--privoxy ignored: incompatible with localhost/plain/adblock-rules/dnsmasq/unbound output modes`));
     privoxyMode = false;
   }
@@ -270,7 +274,7 @@ if (privoxyMode) {
 
 // Validate --pihole usage
 if (piholeMode) {
-  if (localhostMode || localhostModeAlt || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode) {
+  if (localhostIP || plainOutput || adblockRulesMode || dnsmasqMode || dnsmasqOldMode || unboundMode || privoxyMode) {
     if (forceDebug) console.log(formatLogMessage('debug', `--pihole ignored: incompatible with localhost/plain/adblock-rules/dnsmasq/unbound/privoxy output modes`));
     piholeMode = false;
   }
@@ -430,8 +434,8 @@ Options:
   --append                       Append new rules to output file instead of overwriting (requires -o)
     
 Output Format Options:
-  --localhost                    Output as 127.0.0.1 domain.com
-  --localhost-0.0.0.0            Output as 0.0.0.0 domain.com
+  --localhost[=IP]               Output as IP domain.com (default: 127.0.0.1)
+                                 Examples: --localhost, --localhost=0.0.0.0, --localhost=192.168.1.1
   --plain                        Output just domains (no adblock formatting)
   --dnsmasq                      Output as local=/domain.com/ (dnsmasq format)
   --dnsmasq-old                  Output as server=/domain.com/ (dnsmasq old format)
@@ -512,7 +516,7 @@ Redirect Handling Options:
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
   delay: <milliseconds>                        Delay after load (default: 4000)
   reload: <number>                             Reload page n times after load (default: 1)
-  forcereload: true/false                      Force an additional reload after reloads
+  forcereload: true/false or ["domain1.com", "domain2.com"]  Force cache-clearing reload for all URLs or specific domains
   clear_sitedata: true/false                   Clear all cookies, cache, storage before each load (default: false)
   subDomains: 1/0                              Output full subdomains (default: 0)
   localhost: true/false                        Force localhost output (127.0.0.1)
@@ -1478,8 +1482,7 @@ function setupFrameHandling(page, forceDebug) {
     const allowFirstParty = siteConfig.firstParty === true || siteConfig.firstParty === 1;
     const allowThirdParty = siteConfig.thirdParty === undefined || siteConfig.thirdParty === true || siteConfig.thirdParty === 1;
     const perSiteSubDomains = siteConfig.subDomains === 1 ? true : subDomainsMode;
-    const siteLocalhost = siteConfig.localhost === true;
-    const siteLocalhostAlt = siteConfig.localhost_0_0_0_0 === true;
+    const siteLocalhostIP = siteConfig.localhost || null;
     const cloudflarePhishBypass = siteConfig.cloudflare_phish === true;
     const cloudflareBypass = siteConfig.cloudflare_bypass === true;
     // Add redirect and same-page loop protection
@@ -3199,7 +3202,80 @@ function setupFrameHandling(page, forceDebug) {
       updatePageUsage(page, true);
 
       const totalReloads = (siteConfig.reload || 1) - 1; // Subtract 1 because initial load counts as first
-      const useForceReload = siteConfig.forcereload === true;
+      
+      // Enhanced forcereload logic: support boolean or domain array
+      let useForceReload = false;
+      if (siteConfig.forcereload === true) {
+        // Original behavior: force reload for all URLs
+        useForceReload = true;
+      } else if (Array.isArray(siteConfig.forcereload)) {
+        // Input validation: filter out invalid entries
+        const validDomains = siteConfig.forcereload.filter(domain => {
+          if (typeof domain !== 'string') {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Invalid forcereload entry (not string): ${typeof domain} - ${JSON.stringify(domain)}`));
+            }
+            return false;
+          }
+          
+          if (domain.trim() === '') {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Invalid forcereload entry (empty string)`));
+            }
+            return false;
+          }
+          
+          return true;
+        });
+        
+        if (validDomains.length === 0) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `No valid domains in forcereload array for ${currentUrl}`));
+          }
+          useForceReload = false;
+        } else {
+        // New behavior: force reload only for matching domains
+        const currentDomain = safeGetDomain(currentUrl, true); // Get full hostname
+        const currentRootDomain = safeGetDomain(currentUrl, false); // Get root domain
+        
+        useForceReload = validDomains.some(domain => {
+          // Enhanced domain cleaning: handle protocols, ports, paths, and normalize case
+          let cleanDomain = domain.trim();
+          cleanDomain = cleanDomain.replace(/^https?:\/\//, '');  // Remove protocol
+          cleanDomain = cleanDomain.replace(/:\d+$/, '');         // Remove port (e.g., :8080)
+          cleanDomain = cleanDomain.replace(/\/.*$/, '');         // Remove path
+          cleanDomain = cleanDomain.toLowerCase();               // Normalize case
+          
+          // Additional validation: basic domain format check
+          if (!/^[a-z0-9.-]+$/.test(cleanDomain)) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Skipping invalid domain format in forcereload: ${domain} -> ${cleanDomain}`));
+            }
+            return false;
+          }
+          
+          // Check if current URL matches this domain
+          // Support both exact hostname match and subdomain match
+          if (currentDomain.toLowerCase() === cleanDomain || currentRootDomain.toLowerCase() === cleanDomain) {
+            return true;
+          }
+          
+          // Check if current hostname ends with the domain (subdomain match)
+          if (currentDomain.toLowerCase().endsWith('.' + cleanDomain)) {
+            return true;
+          }
+          
+          return false;
+        });
+        }
+        
+        if (forceDebug && useForceReload) {
+          console.log(formatLogMessage('debug', `Force reload enabled for ${currentUrl} - matches domain in forcereload list`));
+        } else if (forceDebug && validDomains.length > 0) {
+          console.log(formatLogMessage('debug', `Force reload not applied for ${currentUrl} - no domain match in [${validDomains.join(', ')}]`));
+        }
+      }
+      // If forcereload is not specified, false, or any other value, useForceReload remains false
       
       if (useForceReload && forceDebug) {
         console.log(formatLogMessage('debug', `Using force reload mechanism for all ${totalReloads + 1} reload(s) on ${currentUrl}`));
@@ -3392,8 +3468,7 @@ function setupFrameHandling(page, forceDebug) {
       } else {
         // Format rules using the output module
         const globalOptions = {
-        localhostMode,
-        localhostModeAlt,
+        localhostIP,
         plainOutput,
         adblockRulesMode,
         dnsmasqMode,
@@ -3434,8 +3509,7 @@ function setupFrameHandling(page, forceDebug) {
       // For other errors, preserve any matches we found before the error
       if (matchedDomains && (matchedDomains.size > 0 || (matchedDomains instanceof Map && matchedDomains.size > 0))) {
         const globalOptions = {
-          localhostMode,
-          localhostModeAlt,
+          localhostIP,
           plainOutput,
           adblockRulesMode,
           dnsmasqMode,
@@ -3883,8 +3957,7 @@ function setupFrameHandling(page, forceDebug) {
   const detectedDomainsCount = getDetectedDomainsCount();
   if (forceDebug) {
     const globalOptions = {
-      localhostMode,
-      localhostModeAlt,
+      localhostIP,
       plainOutput,
       adblockRules: adblockRulesMode,
       dnsmasq: dnsmasqMode,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.19",
+  "version": "2.0.21",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {