@fanboynz/network-scanner 3.1.0 → 3.2.0

package/lib/validate_rules.js

@@ -806,7 +806,6 @@ function cleanRulesetFile(filePath, outputPath = null, options = {}) {
   } = options;
   
   const fs = require('fs');
-  const path = require('path');
 
   let content;
   try {
@@ -1118,6 +1117,7 @@ const KNOWN_SITE_CONFIG_KEYS = new Set([
   'ignore_similar_threshold', 'interact', 'interact_click_count', 'interact_clicks',
   'interact_duration', 'interact_intensity', 'interact_scrolling', 'isBrave',
   'js_redirect_timeout', 'localhost', 'max_redirects', 'openvpn', 'pihole',
+  'output_regex',
   'plain', 'privoxy', 'proxy', 'proxy_bypass', 'proxy_debug', 'proxy_remote_dns',
   'realistic_click', 'referrer_disable', 'referrer_headers', 'regex_and',
   'reload', 'resourceTypes', 'screenshot', 'searchstring', 'searchstring_and',
@@ -1307,6 +1307,21 @@ function normalizeSiteConfig(siteConfig, siteIndex = 0) {
     }
   }
 
+  // 2b. output_regex must be a compilable regex. An invalid one is silently
+  // disabled at runtime (the use-site try/catch falls back to ||host^), so
+  // surface it here at load time where the user can fix it.
+  if ('output_regex' in siteConfig && siteConfig.output_regex != null && siteConfig.output_regex !== '') {
+    if (typeof siteConfig.output_regex !== 'string') {
+      warnings.push(`${tag}: 'output_regex' should be a string regex, got ${JSON.stringify(siteConfig.output_regex)} — will be ignored`);
+    } else {
+      try {
+        new RegExp(siteConfig.output_regex);
+      } catch (e) {
+        warnings.push(`${tag}: 'output_regex' is not a valid regex (${e.message}) — will be ignored, output falls back to ||host^`);
+      }
+    }
+  }
+
   // 3. String → single-element array coercion for fields that accept both
   // forms (dig, dig-or, whois, whois-or). Downstream consumers all gate on
   // Array.isArray(), so a bare string value previously silently disabled

package/nwss.1

@@ -72,10 +72,6 @@ Output as \fB(^|\\.)domain\\.com$\fR format for Pi-hole regex filters.
 Generate adblock filter rules with resource type modifiers (requires \fB\-o\fR).
 
 .SS General Options
-.TP
-.B \--verbose
-Enable verbose output globally for all sites.
-
 .TP
 .B \--debug
 Enable debug mode with detailed logging of all network requests.
@@ -104,6 +100,10 @@ Output full subdomains instead of collapsing to root domains.
 .B \--no-interact
 Disable mouse simulation and page interaction globally.
 
+.TP
+.B \--ghost-cursor
+Use ghost-cursor Bezier mouse movements globally (requires \fBnpm i ghost-cursor\fR). See \fBGhost Cursor Options\fR. Equivalent to per-site \fBcursor_mode: "ghost"\fR.
+
 .TP
 .BR \--custom-json " \fIFILE\fR"
 Use \fIFILE\fR instead of \fBconfig.json\fR for configuration.
@@ -136,10 +136,23 @@ Remove Chrome/Puppeteer temporary files before exit.
 .BR \--max-concurrent " \fINUMBER\fR"
 Maximum concurrent site processing (1-50, overrides config/default).
 
+.TP
+.BR \--dns " \fIIP\fR[,\fIIP\fR...]"
+Nameserver(s) for the DNS pre-check AND nettools' dig \(em does not affect Chrome
+navigation or whois. A single address pins all queries to it; several are
+rotated per query (each leading once, the rest as failover) to spread the
+load. Routing dig through these avoids dig timeouts on a flaky system resolver
+silently dropping dig-gated domains. Overrides /etc/resolv.conf. Invalid
+entries are warned and dropped.
+
 .TP
 .BR \--cleanup-interval " \fINUMBER\fR"
 Browser restart interval in URLs processed (1-1000, overrides config/default).
 
+.TP
+.B \--show-dead-domains
+At end of scan, list hostnames that did not resolve or were unreachable (\fBNXDOMAIN\fR/\fBENODATA\fR plus \fBERR_NAME_NOT_RESOLVED\fR/\fBERR_ADDRESS_UNREACHABLE\fR). Excludes blocks and timeouts, since those mean the domain is alive. Useful for pruning dead URLs.
+
 .TP
 .BR \-h ", " \--help
 Show help message and exit.
@@ -249,6 +262,10 @@ Regex pattern(s) to match suspicious requests.
 .B regex_and
 Boolean. Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL (default: false).
 
+.TP
+.B output_regex
+String. Regex applied to each matched URL to build the rule body: capture group 1 (or the whole match) becomes \fB||<capture>\fR instead of \fB||host^\fR. For example \fB^https?://([^/]+/[^/]+/)\fR turns \fBhttps://host.com/script/abc.js\fR into \fB||host.com/script/\fR, collapsing randomized filenames under a path into one rule. The capture must include the host. If the regex does not match a URL, output falls back to \fB||host^\fR. Adblock-only; domain-based formats (dnsmasq, pihole, hosts, plain) emit the bare host.
+
 .TP
 .B comments
 Documentation strings or notes - completely ignored by the scanner. Can be a single string or array of strings. Used for adding context, URLs, timestamps, or any documentation notes to configuration files.
@@ -293,6 +310,14 @@ Boolean. Simulate mouse movements and clicks.
 .B interact_intensity
 String. Interaction simulation intensity: \fB"low"\fR, \fB"medium"\fR, \fB"high"\fR (default: "medium").
 
+.TP
+.B interact_click_count
+Integer. Number of random content-zone clicks per load, capped at 20 (default: 3). The default of 3 is a primary click plus two backups, since some ad SDKs suppress the first or second click as warmup.
+
+.TP
+.B realistic_click
+Boolean. Higher click fidelity: denser mouse approach (15 steps), sub-pixel hand-tremor micro-moves during the press, and a small mouseup drift so the mousedown and mouseup coordinates differ. For sites that score click realism. Costs roughly 80-120ms per click (default: false).
+
 .TP
 .B delay
 Milliseconds to wait after page load (default: 4000).
@@ -419,7 +444,7 @@ Object. Custom page.goto() options for Puppeteer navigation. Available options:
 .IP \(bu 4
 \fB"networkidle0"\fR - Wait until 0 network requests for 500ms
 .IP \(bu 4
-\fB"networkidle2"\fR - Wait until ≤2 network requests for 500ms
+\fB"networkidle2"\fR - Wait until \(<=2 network requests for 500ms
 .RE
 .IP \(bu 4
 \fBtimeout\fR: Maximum navigation time in milliseconds (overrides site timeout)
@@ -479,15 +504,28 @@ Both modes wait 16 seconds before cleanup to allow final operations to complete,
 
 .TP
 
-.SS Redirect Handling Options
+.SS Popup Capture Options
+.TP
+.B capture_popups
+Boolean. Capture popup windows opened during the scan and evaluate their landing URL and in-popup requests against \fBfilterRegex\fR/\fBdig\fR/\fBwhois\fR. Requires \fBinteract\fR plus interaction clicks to fire the user-gesture click that opens popups; \fBcapture_popups\fR alone registers the listener but no popups will fire (default: false).
+
+.TP
+.B interact_popups
+Boolean. Mouse-click inside captured popups (content-zone clicks) so the chain cascades to its next redirect or ad. Requires \fBcapture_popups\fR. Clicks popups up to \fBcapture_popups_max_depth\fR minus 1 \(em the deepest captured popup is observed, not clicked (default: false).
 
 .TP
-.B follow_redirects
-Boolean. Follow redirects to new domains (default: true).
+.B capture_popups_max_depth
+Integer. Maximum popup-chain depth to capture, e.g. \fBsite -> p1 -> p2 -> p3 -> destination\fR. Each extra level multiplies popups and time (default: 4).
+
+.TP
+.B capture_popups_window_ms
+Integer. Per-popup capture window in milliseconds before the popup is auto-closed (default: 5000).
+
+.SS Redirect Handling Options
 
 .TP
 .B max_redirects
-Number. Maximum number of redirects to follow (default: 10).
+Number. Maximum number of redirects to follow (default: 10; 0 = follow none).
 
 .TP
 .B js_redirect_timeout
@@ -501,6 +539,29 @@ Boolean. Analyze page source for redirect patterns (default: true).
 .B redirect_timeout_multiplier
 Number. Increase timeout for redirected URLs (default: 1.5).
 
+.SS Ghost Cursor Options
+Optional Bezier-curve mouse engine (the \fBghost-cursor\fR npm package, install
+with \fBnpm i ghost-cursor\fR). Falls back to the built-in mouse if not
+installed. Enable per-site with \fBcursor_mode\fR or globally with the
+\fB\-\-ghost-cursor\fR flag.
+.TP
+.B cursor_mode
+String. Set to \fB"ghost"\fR to use ghost-cursor Bezier mouse movements for this site.
+.TP
+.B ghost_cursor_speed
+Number. Movement speed multiplier (default: auto).
+.TP
+.B ghost_cursor_hesitate
+Number. Delay in milliseconds before a click (default: 50).
+.TP
+.B ghost_cursor_overshoot
+Number. Maximum overshoot distance in pixels before correcting back to the target (default: auto).
+.TP
+.B ghost_cursor_duration
+Number. How long the Bezier movement loop runs, in milliseconds (default: \fBinteract_duration\fR or 2000). Part of this budget (up to half) is reserved for clicks.
+.PP
+Ghost-cursor only \fIclicks\fR when both \fBinteract\fR and \fBinteract_clicks\fR are true. With \fBrealistic_click\fR set, each press adds hand-tremor during the hold plus a mouseup drift so mousedown and mouseup coordinates differ. Ghost mode honors \fBinteract_click_count\fR (default 3, cap 20); since realistic clicks take roughly 600-700ms each, raise \fBghost_cursor_duration\fR (about \fBinteract_click_count\fR x 700 plus movement, e.g. 5000-8000) to fit all of them \(em the default 2000 fits about one click.
+
 .SS Cloudflare Protection Options
 
 .TP
@@ -678,15 +739,15 @@ Global and per-site boolean to enable similarity filtering against ignoreDomains
 With default settings (\fBignore_similar_threshold: 80\fR):
 .RS
 .IP \(bu 4
-\fBanimerco.com\fR vs \fBanimerco.org\fR → 100% similar → Ignored
+\fBanimerco.com\fR vs \fBanimerco.org\fR \(-> 100% similar \(-> Ignored
 .IP \(bu 4
-\fBgoogle.com\fR vs \fBgoogle.co.uk\fR → 100% similar → Ignored
+\fBgoogle.com\fR vs \fBgoogle.co.uk\fR \(-> 100% similar \(-> Ignored
 .IP \(bu 4
-\fBamazon.com\fR vs \fBamazon2.org\fR → 89% similar → Ignored
+\fBamazon.com\fR vs \fBamazon2.org\fR \(-> 89% similar \(-> Ignored
 .IP \(bu 4
-\fBfacebook.com\fR vs \fBfaceboook.com\fR → 91% similar → Ignored
+\fBfacebook.com\fR vs \fBfaceboook.com\fR \(-> 91% similar \(-> Ignored
 .IP \(bu 4
-\fBapple.com\fR vs \fBmicrosoft.com\fR → 0% similar → Kept
+\fBapple.com\fR vs \fBmicrosoft.com\fR \(-> 0% similar \(-> Kept
 .RE
 
 .SH EXAMPLES
@@ -844,7 +905,7 @@ With default settings (\fBignore_similar_threshold: 80\fR):
 
 .SS Run with debug mode and similarity filtering:
 .EX
-node nwss.js --debug --dry-run --verbose
+node nwss.js --debug --dry-run
 .EE
 
 .SS Run with adblock output format: