@fanboynz/network-scanner 2.0.44 → 2.0.46

package/README.md

@@ -152,8 +152,9 @@ Example:
 | `even_blocked`       | Boolean | `false` | Add matching rules even if requests are blocked |
 | `bypass_cache`       | Boolean | `false` | Skip all caching for this site's URLs |
 | `window_cleanup`     | Boolean or String | `false` | Close old/unused browser windows/tabs after entire URL group completes |
+| `window_cleanup_threshold` | Integer | `8` | For `"realtime"` mode: max pages to keep open before cleanup |
 
-**Window cleanup modes:** `false` (disabled), `true` (conservative - closes obvious leftovers), `"all"` (aggressive - closes all content pages). Both active modes preserve the main Puppeteer window and wait 16 seconds before cleanup to avoid interfering with active operations.
+**Window cleanup modes:** `false` (disabled), `true` (conservative - closes obvious leftovers), `"realtime"` (continuously cleanup oldest pages when threshold exceeded), `"all"` (aggressive - closes all content pages after group). Both active modes preserve the main Puppeteer window and wait 16 seconds before cleanup to avoid interfering with active operations.
 
 
 ### Redirect Handling Options
@@ -193,6 +194,11 @@ When a page redirects to a new domain, first-party/third-party detection is base
 "referrer_headers": {"mode": "custom", "custom": ["https://news.ycombinator.com/"]}
 ```
 
+**Disable referrer for specific URLs:**
+```json
+"referrer_disable": ["https://example.com/no-ref", "sensitive-site.com"]
+```
+
 ### Protection Bypassing
 
 | Field                | Values | Default | Description |
@@ -244,7 +250,7 @@ When a page redirects to a new domain, first-party/third-party detection is base
 |:---------------------|:-------|:-------:|:------------|
 | `goto_options`       | Object | `{"waitUntil": "load"}` | Custom page.goto() options |
 | `clear_sitedata`     | Boolean | `false` | Clear all cookies, cache, storage before each load |
-| `forcereload`        | Boolean | `false` | Force an additional reload after reloads |
+| `forcereload`        | Boolean or Array | `false` | Force cache-clearing reload for all URLs (`true`) or specific domains (`["domain1.com"]`) |
 | `isBrave`            | Boolean | `false` | Spoof Brave browser detection |
 | `evaluateOnNewDocument` | Boolean | `false` | Inject fetch/XHR interceptor in page |
 | `cdp`                | Boolean | `false` | Enable CDP logging for this site |
@@ -260,6 +266,54 @@ When a page redirects to a new domain, first-party/third-party detection is base
 | `interact_clicks`    | Boolean | `false` | Enable element clicking simulation |
 | `interact_typing`    | Boolean | `false` | Enable typing simulation |
 | `interact_intensity` | String | `"medium"` | Interaction simulation intensity: "low", "medium", "high" |
+| `dnsmasq`            | Boolean | `false` | Force dnsmasq output for this site |
+| `dnsmasq_old`        | Boolean | `false` | Force dnsmasq old format output for this site |
+| `unbound`            | Boolean | `false` | Force unbound output for this site |
+| `privoxy`            | Boolean | `false` | Force Privoxy output for this site |
+| `pihole`             | Boolean | `false` | Force Pi-hole regex output for this site |
+| `ignore_similar`     | Boolean | - | Override global `ignore_similar` setting for this site |
+| `ignore_similar_threshold` | Integer | - | Override global similarity threshold for this site |
+| `ignore_similar_ignored_domains` | Boolean | - | Override global `ignore_similar_ignored_domains` for this site |
+
+### VPN Options
+
+Route traffic through a VPN for specific sites. Requires `sudo` privileges. The VPN connection is established before scanning and torn down after the site completes.
+
+> **Note:** VPN modifies system-level routing. During concurrent scanning, all traffic routes through the active tunnel — not just the site that requested it. For isolated per-site VPN, run sites sequentially or use the same VPN config for all concurrent sites.
+
+#### WireGuard
+
+| Field                | Values | Default | Description |
+|:---------------------|:-------|:-------:|:------------|
+| `vpn`                | String or Object | - | WireGuard VPN configuration |
+| `vpn.config`         | String | - | Path to `.conf` file or interface name in `/etc/wireguard/` |
+| `vpn.config_inline`  | String | - | Inline WireGuard config content |
+| `vpn.interface`      | String | auto | Interface name (auto-derived from config filename) |
+| `vpn.health_check`   | Boolean | `true` | Ping through tunnel to verify connectivity |
+| `vpn.test_host`      | String | `"1.1.1.1"` | Host to ping for health check |
+| `vpn.retry`          | Boolean | `true` | Retry on connection failure |
+| `vpn.max_retries`    | Integer | `2` | Maximum retry attempts |
+
+#### OpenVPN
+
+| Field                | Values | Default | Description |
+|:---------------------|:-------|:-------:|:------------|
+| `openvpn`            | String or Object | - | OpenVPN configuration |
+| `openvpn.config`     | String | - | Path to `.ovpn` file |
+| `openvpn.config_inline` | String | - | Inline OpenVPN config content |
+| `openvpn.name`       | String | auto | Connection name (auto-derived from config filename) |
+| `openvpn.username`   | String | - | VPN username (written to secure temp file) |
+| `openvpn.password`   | String | - | VPN password (written to secure temp file) |
+| `openvpn.auth_file`  | String | - | Path to existing auth credentials file |
+| `openvpn.health_check` | Boolean | `true` | Ping through tunnel to verify connectivity |
+| `openvpn.test_host`  | String | `"1.1.1.1"` | Host to ping for health check |
+| `openvpn.retry`      | Boolean | `true` | Retry on connection failure |
+| `openvpn.max_retries` | Integer | `2` | Maximum retry attempts |
+| `openvpn.connect_timeout` | Milliseconds | `30000` | Timeout for connection establishment |
+| `openvpn.extra_args` | Array | - | Additional OpenVPN command line arguments |
+| `openvpn.verbosity`  | String | `"3"` | OpenVPN log verbosity level |
+
+> **Authentication:** If the `.ovpn` file already contains credentials (via `auth-user-pass /path/to/file` or an inline `<auth-user-pass>` block), no additional config is needed — just provide the config path. The `username`/`password` fields are only needed when the `.ovpn` file has a bare `auth-user-pass` directive that expects interactive input.
 
 ### Global Configuration Options
 
@@ -434,6 +488,41 @@ node nwss.js --max-concurrent 12 --cleanup-interval 300 -o rules.txt
 }
 ```
 
+#### Scanning through OpenVPN
+```json
+{
+  "url": "https://geo-restricted-site.com",
+  "openvpn": "/etc/openvpn/us-server.ovpn",
+  "filterRegex": "tracking|analytics",
+  "userAgent": "chrome",
+  "fingerprint_protection": "random"
+}
+```
+
+#### OpenVPN with Credentials
+```json
+{
+  "url": "https://region-locked-site.com",
+  "openvpn": {
+    "config": "/etc/openvpn/eu-server.ovpn",
+    "username": "vpn_user",
+    "password": "vpn_pass",
+    "connect_timeout": 45000
+  },
+  "filterRegex": "ads|tracking"
+}
+```
+
+#### Scanning through WireGuard
+```json
+{
+  "url": "https://another-site.com",
+  "vpn": "/etc/wireguard/wg-us.conf",
+  "filterRegex": "analytics",
+  "userAgent": "firefox"
+}
+```
+
 ---
 
 ## Memory Management
@@ -477,6 +566,39 @@ sudo apt install google-chrome-stable
 sudo apt install bind9-dnsutils whois
 ```
 
+#### OpenVPN (optional, for per-site VPN routing)
+```
+sudo apt install openvpn
+```
+
+Grant passwordless sudo for OpenVPN operations:
+```
+sudo visudo -f /etc/sudoers.d/openvpn-nwss
+```
+Add:
+```
+your_username ALL=(root) NOPASSWD: /usr/sbin/openvpn, /usr/bin/kill, /usr/bin/pgrep, /usr/bin/pkill
+```
+
+On WSL2, load the TUN module (required each reboot):
+```
+sudo modprobe tun
+```
+
+#### WireGuard (optional, for per-site VPN routing)
+```
+sudo apt install wireguard
+```
+
+Grant passwordless sudo for WireGuard operations:
+```
+sudo visudo -f /etc/sudoers.d/wg-nwss
+```
+Add:
+```
+your_username ALL=(root) NOPASSWD: /usr/bin/wg-quick, /usr/bin/wg
+```
+
 ## Notes
 
 - If both `firstParty: 0` and `thirdParty: 0` are set for a site, it will be skipped.
@@ -491,5 +613,9 @@ sudo apt install bind9-dnsutils whois
 - For maximum stealth, combine `fingerprint_protection: "random"` with appropriate `referrer_headers` modes
 - User agents are automatically updated to latest versions (Chrome 131, Firefox 133, Safari 18.2)
 - Referrer headers work independently from fingerprint protection - use both for best results
+- VPN connections (`vpn`/`openvpn`) are established before scanning and torn down after the site completes
+- If an `.ovpn` file contains embedded credentials, no additional auth config is needed in the JSON
+- VPN affects system-level routing — all concurrent scans will route through the active tunnel
+- Both `vpn` (WireGuard) and `openvpn` can be set, but `vpn` takes precedence
 
 ---

package/lib/cdp.js

@@ -425,7 +425,5 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
 module.exports = {
   createCDPSession,
   createPageWithTimeout,
-  setRequestInterceptionWithTimeout,
-  validateCDPConfig,
-  createEnhancedCDPSession
+  setRequestInterceptionWithTimeout
 };

package/lib/clear_sitedata.js

@@ -345,6 +345,5 @@ async function clearSiteDataEnhanced(page, currentUrl, forceDebug) {
 module.exports = {
   clearSiteData,
   clearSiteDataViaCDP,
-  clearSiteDataViaPage,
-  clearSiteDataEnhanced
+  clearSiteDataViaPage
 };

package/lib/interaction.js

@@ -373,7 +373,6 @@ async function humanLikeMouseMove(page, fromX, fromY, toX, toY, options = {}) {
 
     // Add slight curve to movement (more human-like)
     if (curve > 0 && i > 0 && i < actualSteps) {
-      const midpoint = actualSteps / 2;
       const curveIntensity = Math.sin((i / actualSteps) * Math.PI) * curve * distance * MOUSE_MOVEMENT.CURVE_INTENSITY_RATIO;
       const perpX = -(toY - fromY) / distance;
       const perpY = (toX - fromX) / distance;
@@ -449,7 +448,7 @@ async function simulateScrolling(page, options = {}) {
       // Smooth scrolling by breaking into smaller increments
       for (let j = 0; j < smoothness; j++) {
         await page.mouse.wheel({ deltaY: scrollDelta / smoothness });
-        await new Promise(resolve => setTimeout(resolve, SCROLLING.SMOOTH_INCREMENT_DELAY));
+        await fastTimeout(SCROLLING.SMOOTH_INCREMENT_DELAY);
       }
       
       if (i < amount - 1) {
@@ -549,7 +548,7 @@ async function interactWithElements(page, options = {}) {
     for (let attempt = 0; attempt < maxAttempts; attempt++) {
       try {
         // Find visible, clickable elements
-        const elements = await page.evaluate((selectors, avoidWords) => {
+        const elements = await page.evaluate((selectors, avoidWords, textPreviewLen) => {
           const clickableElements = [];
           
           selectors.forEach(selector => {
@@ -571,7 +570,7 @@ async function interactWithElements(page, options = {}) {
                     y: rect.top + rect.height / 2,
                     width: rect.width,
                     height: rect.height,
-                    text: text.substring(0, ELEMENT_INTERACTION.TEXT_PREVIEW_LENGTH)
+                    text: text.substring(0, textPreviewLen)
                   });
                 }
               }
@@ -579,7 +578,7 @@ async function interactWithElements(page, options = {}) {
           });
           
           return clickableElements;
-        }, elementTypes, avoidDestructive ? ['delete', 'remove', 'submit', 'buy', 'purchase', 'order'] : []);
+        }, elementTypes, avoidDestructive ? ['delete', 'remove', 'submit', 'buy', 'purchase', 'order'] : [], ELEMENT_INTERACTION.TEXT_PREVIEW_LENGTH);
 
         if (elements.length > 0) {
           // Choose a random element to interact with
@@ -590,12 +589,12 @@ async function interactWithElements(page, options = {}) {
           await humanLikeMouseMove(page, currentPos.x, currentPos.y, element.x, element.y);
           
           // Brief pause before clicking
-          await fastTimeout(TIMING.CLICK_PAUSE_MIN + Math.random() * TIMING.CLICK_PAUSE_MAX);
+          await fastTimeout(TIMING.CLICK_PAUSE_MIN + Math.random() * (TIMING.CLICK_PAUSE_MAX - TIMING.CLICK_PAUSE_MIN));
           
           await page.mouse.click(element.x, element.y);
           
           // Brief pause after clicking
-          await fastTimeout(TIMING.POST_CLICK_MIN + Math.random() * TIMING.POST_CLICK_MAX);
+          await fastTimeout(TIMING.POST_CLICK_MIN + Math.random() * (TIMING.POST_CLICK_MAX - TIMING.POST_CLICK_MIN));
         }
       } catch (elementErr) {
         // Continue to next attempt if this one fails
@@ -673,9 +672,9 @@ async function simulateTyping(page, text, options = {}) {
       if (mistakes && Math.random() < mistakeRate) {
         const wrongChar = String.fromCharCode(97 + Math.floor(Math.random() * 26));
         await page.keyboard.type(wrongChar);
-        await fastTimeout(TIMING.MISTAKE_PAUSE_MIN + Math.random() * TIMING.MISTAKE_PAUSE_MAX);
+        await fastTimeout(TIMING.MISTAKE_PAUSE_MIN + Math.random() * (TIMING.MISTAKE_PAUSE_MAX - TIMING.MISTAKE_PAUSE_MIN));
         await page.keyboard.press('Backspace');
-        await fastTimeout(TIMING.BACKSPACE_DELAY_MIN + Math.random() * TIMING.BACKSPACE_DELAY_MAX);
+        await fastTimeout(TIMING.BACKSPACE_DELAY_MIN + Math.random() * (TIMING.BACKSPACE_DELAY_MAX - TIMING.BACKSPACE_DELAY_MIN));
       }
       
       await page.keyboard.type(char);
@@ -818,6 +817,7 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
         }
         return;
       }
+      await bodyExists.dispose();
     } catch (bodyCheckErr) {
       if (forceDebug) {
         console.log(`[interaction] Page not ready for interaction on ${currentUrl} (waited ${Math.min(Math.max((options.siteTimeout || 20000) / 8, 2000), 5000)}ms): ${bodyCheckErr.message}`);
@@ -922,6 +922,7 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
       const bodyElement = await page.$('body');
       if (bodyElement) {
         await page.hover('body');
+        await bodyElement.dispose();
       }
     } catch (hoverErr) {
       // Silently handle hover failures - not critical

package/lib/openvpn_vpn.js

@@ -13,6 +13,22 @@ const fs = require('fs');
 const path = require('path');
 const { formatLogMessage } = require('./colorize');
 
+/**
+ * Fetch external IP address through the active tunnel
+ * @param {string} tunDevice - TUN device name (optional)
+ * @returns {string|null} External IP or null
+ */
+function getExternalIP(tunDevice) {
+  const services = ['https://api.ipify.org', 'https://ifconfig.me/ip', 'https://icanhazip.com'];
+  for (const service of services) {
+    try {
+      const iface = tunDevice ? `--interface ${tunDevice}` : '';
+      return execSync(`curl -s -m 5 ${iface} ${service}`, { encoding: 'utf8', timeout: 8000 }).trim();
+    } catch {}
+  }
+  return null;
+}
+
 // Track active connections: name ? { process, configPath, pid, tunDevice, startedAt, sites }
 const activeConnections = new Map();
 
@@ -107,7 +123,7 @@ function checkTunDevice() {
  */
 function ensureTempDir() {
   if (!fs.existsSync(TEMP_DIR)) {
-    fs.mkdirSync(TEMP_DIR, { recursive: true, mode: 0o700 });
+    fs.mkdirSync(TEMP_DIR, { recursive: true, mode: 0o755 });
   }
 }
 
@@ -340,12 +356,24 @@ async function startConnection(configPath, vpnConfig, forceDebug = false) {
     return { success: true, connection: connectionName, tunDevice: existing.tunDevice, alreadyActive: true };
   }
 
+  // Kill any stale processes from a previous run using this config
+  try {
+    execSync(`sudo pkill -TERM -f "openvpn.*${connectionName}" 2>/dev/null`, {
+      encoding: 'utf8', timeout: 3000
+    });
+    // Brief wait for cleanup
+    execSync('sleep 1', { timeout: 3000 });
+  } catch {}
+
   ensureTempDir();
   const logPath = path.join(TEMP_DIR, `${connectionName}.log`);
 
   // Clean stale log
   try { if (fs.existsSync(logPath)) fs.unlinkSync(logPath); } catch {}
 
+  // Pre-create log file writable by all so sudo openvpn can write and user can read
+  try { fs.writeFileSync(logPath, '', { mode: 0o666 }); } catch {}
+
   const args = buildArgs(configPath, vpnConfig, connectionName, logPath);
 
   if (forceDebug) {
@@ -365,7 +393,7 @@ async function startConnection(configPath, vpnConfig, forceDebug = false) {
     fgArgs.push(args[i]);
   }
 
-  const child = spawn('openvpn', fgArgs, {
+  const child = spawn('sudo', ['openvpn', ...fgArgs], {
     stdio: ['ignore', 'ignore', 'ignore'],
     detached: false
   });
@@ -411,17 +439,21 @@ function stopConnection(connectionName, forceDebug = false) {
   }
 
   try {
-    // Send SIGTERM to gracefully disconnect
-    if (info.process && info.process.exitCode === null) {
-      info.process.kill('SIGTERM');
-
-      // Wait briefly for graceful shutdown, then force kill
-      const killed = waitForProcessExit(info.pid, 5000);
-      if (!killed) {
-        try { info.process.kill('SIGKILL'); } catch {}
-        try { process.kill(info.pid, 'SIGKILL'); } catch {}
+    // Find the actual openvpn PID (child of sudo) and kill it
+    try {
+      execSync(`sudo pkill -TERM -f "openvpn.*${connectionName}" 2>/dev/null`, {
+        encoding: 'utf8', timeout: 3000
+      });
+    } catch {}
+
+    const killed = waitForProcessExit(info.pid, 5000);
+    if (!killed) {
+      try {
+        execSync(`sudo pkill -9 -f "openvpn.*${connectionName}" 2>/dev/null`, {
+          encoding: 'utf8', timeout: 3000
+        });
+      } catch {}
       }
-    }
   } catch (killErr) {
     // Process may already be dead
     if (forceDebug) {
@@ -766,7 +798,8 @@ async function connectForSite(siteConfig, forceDebug = false) {
       }
     }
 
-    return { success: true, connection: connectionName, tunDevice: startResult.tunDevice };
+    const externalIP = getExternalIP(startResult.tunDevice);
+    return { success: true, connection: connectionName, tunDevice: startResult.tunDevice, externalIP };
   }
 
   return { success: false, connection: connectionName, error: 'All attempts failed' };

package/lib/wireguard_vpn.js

@@ -7,6 +7,22 @@ const fs = require('fs');
 const path = require('path');
 const { formatLogMessage } = require('./colorize');
 
+/**
+ * Fetch external IP address through the active tunnel
+ * @param {string} interfaceName - WireGuard interface name (optional)
+ * @returns {string|null} External IP or null
+ */
+function getExternalIP(interfaceName) {
+  const services = ['https://api.ipify.org', 'https://ifconfig.me/ip', 'https://icanhazip.com'];
+  for (const service of services) {
+    try {
+      const iface = interfaceName ? `--interface ${interfaceName}` : '';
+      return execSync(`curl -s -m 5 ${iface} ${service}`, { encoding: 'utf8', timeout: 8000 }).trim();
+    } catch {}
+  }
+  return null;
+}
+
 // Track active interfaces for cleanup
 const activeInterfaces = new Map();
 
@@ -94,7 +110,7 @@ function interfaceUp(configPath, interfaceName, forceDebug = false) {
 
   try {
     // wg-quick accepts a config path or interface name in /etc/wireguard/
-    execSync(`wg-quick up "${configPath}"`, {
+    execSync(`sudo wg-quick up "${configPath}"`, {
       encoding: 'utf8',
       timeout: 15000
     });
@@ -109,7 +125,8 @@ function interfaceUp(configPath, interfaceName, forceDebug = false) {
       console.log(formatLogMessage('debug', `[vpn] Interface ${interfaceName} is up`));
     }
 
-    return { success: true, interface: interfaceName };
+    const externalIP = getExternalIP(interfaceName);
+    return { success: true, interface: interfaceName, externalIP };
   } catch (error) {
     return {
       success: false,
@@ -132,7 +149,7 @@ function interfaceDown(interfaceName, forceDebug = false) {
   }
 
   try {
-    execSync(`wg-quick down "${info.configPath}"`, {
+    execSync(`sudo wg-quick down "${info.configPath}"`, {
       encoding: 'utf8',
       timeout: 10000
     });

package/nwss.js

@@ -1673,7 +1673,8 @@ function setupFrameHandling(page, forceDebug) {
           return { url: currentUrl, rules: [], success: false, vpnFailed: true };
         }
         if (!silentMode) {
-          console.log(formatLogMessage('info', `[vpn] WireGuard connected via ${vpnResult.interface} for ${currentUrl}`));
+          const ipInfo = vpnResult.externalIP ? ` (${vpnResult.externalIP})` : '';
+          console.log(formatLogMessage('info', `[vpn] WireGuard connected via ${vpnResult.interface}${ipInfo} for ${currentUrl}`));
         }
       } else if (siteConfig.openvpn) {
         const ovpnResult = await ovpnConnect(siteConfig, forceDebug);
@@ -1682,7 +1683,8 @@ function setupFrameHandling(page, forceDebug) {
           return { url: currentUrl, rules: [], success: false, vpnFailed: true };
         }
         if (!silentMode) {
-          console.log(formatLogMessage('info', `[vpn] OpenVPN connected via ${ovpnResult.connection} for ${currentUrl}`));
+          const ipInfo = ovpnResult.externalIP ? ` (${ovpnResult.externalIP})` : '';
+          console.log(formatLogMessage('info', `[vpn] OpenVPN connected via ${ovpnResult.connection}${ipInfo} for ${currentUrl}`));
         }
       }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.44",
+  "version": "2.0.46",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {