@fanboynz/network-scanner 2.0.56 → 2.0.58

package/.github/workflows/npm-publish.yml

@@ -1,22 +1,21 @@
 name: Publish to NPM
 on:
-  push:
-    branches: [ main, master ]
+  workflow_dispatch:
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     permissions:
-      contents: write  # This is key!
+      contents: write
       
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
           
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: '20'
           registry-url: 'https://registry.npmjs.org'

package/CLAUDE.md

@@ -0,0 +1,65 @@
+# Network Scanner (NWSS)
+
+Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, VPN/proxy routing, and multiple output formats.
+
+## Project Structure
+
+- `nwss.js` — Main entry point (~4,600 lines). CLI args, URL processing, orchestration.
+- `config.json` — Default scan configuration (sites, filters, options).
+- `lib/` — 28 focused, single-purpose modules:
+  - `fingerprint.js` — Bot detection evasion (device/GPU/timezone spoofing)
+  - `cloudflare.js` — Cloudflare challenge detection and solving
+  - `browserhealth.js` — Memory management and browser lifecycle
+  - `interaction.js` — Human-like mouse/scroll/typing simulation
+  - `smart-cache.js` — Multi-layer caching with persistence
+  - `nettools.js` — WHOIS/dig integration
+  - `output.js` — Multi-format rule output (adblock, dnsmasq, unbound, pihole, etc.)
+  - `proxy.js` — SOCKS5/HTTP proxy support
+  - `wireguard_vpn.js` / `openvpn_vpn.js` — VPN routing
+  - `adblock.js` — Adblock filter parsing and validation
+  - `validate_rules.js` — Domain and rule format validation
+  - `colorize.js` — Console output formatting and colors
+  - `domain-cache.js` — Domain detection cache for performance
+  - `post-processing.js` — Result cleanup and deduplication
+  - `redirect.js`, `referrer.js`, `cdp.js`, `curl.js`, `grep.js`, `compare.js`, `compress.js`, `dry-run.js`, `browserexit.js`, `clear_sitedata.js`, `flowproxy.js`, `ignore_similar.js`, `searchstring.js`
+- `.github/workflows/npm-publish.yml` — Automated npm publishing
+- `nwss.1` — Man page
+
+## Tech Stack
+
+- **Node.js** >=20.0.0
+- **puppeteer** >=20.0.0 — Headless browser automation
+- **psl** — Public Suffix List for domain parsing
+- **lru-cache** — LRU cache implementation
+- **p-limit** — Concurrency limiting (dynamically imported)
+- **eslint** — Linting (`npm run lint`)
+
+## Conventions
+
+- Store modular functionality in `./lib/` with focused, single-purpose modules
+- Use `messageColors` and `formatLogMessage` from `./lib/colorize` for consistent console output
+- Implement timeout protection for all Puppeteer operations using `Promise.race` patterns
+- Handle browser lifecycle with comprehensive cleanup in try-finally blocks
+- Validate all external tool availability before use (grep, curl, whois, dig)
+- Use `forceDebug` flag for detailed logging, `silentMode` for minimal output
+- Use `Object.freeze` for constant configuration objects (TIMEOUTS, CACHE_LIMITS, CONCURRENCY_LIMITS)
+- Use `fastTimeout(ms)` helper instead of `node:timers/promises` for Puppeteer 22.x compatibility
+
+## Running
+
+```bash
+node nwss.js                          # Run with default config.json
+node nwss.js config-custom.json       # Run with custom config
+node nwss.js --validate-config        # Validate configuration
+node nwss.js --dry-run                # Preview without network calls
+node nwss.js --headful                # Launch with browser GUI
+```
+
+## Files to Ignore
+
+- `node_modules/**`
+- `logs/**`
+- `sources/**`
+- `.cache/**`
+- `*.log`
+- `*.gz`

package/lib/adblock.js

@@ -51,11 +51,12 @@ function parseAdblockRules(filePath, options = {}) {
     caseSensitive = false
   } = options;
 
-  if (!fs.existsSync(filePath)) {
+  let fileContent;
+  try {
+    fileContent = fs.readFileSync(filePath, 'utf-8');
+  } catch (err) {
     throw new Error(`Adblock rules file not found: ${filePath}`);
   }
-
-  const fileContent = fs.readFileSync(filePath, 'utf-8');
   const lines = fileContent.split('\n');
   
   const rules = {

package/lib/browserexit.js

@@ -5,6 +5,7 @@
 
 
 const fs = require('fs');
+const path = require('path');
 const { execSync } = require('child_process');
 
 // Constants for temp file cleanup
@@ -15,20 +16,55 @@ const CHROME_TEMP_PATHS = [
 ];
 
 const CHROME_TEMP_PATTERNS = [
-  'com.google.Chrome.*',         // Google Chrome temp files (no leading dot)
-  '.org.chromium.Chromium.*',
-  'puppeteer-*'
+  /^\.?com\.google\.Chrome\./,
+  /^\.?org\.chromium\.Chromium\./,
+  /^puppeteer-/
 ];
 
+/**
+ * Count and remove matching Chrome/Puppeteer temp entries from a directory using fs
+ * @param {string} basePath - Directory to scan
+ * @param {boolean} forceDebug - Whether to output debug logs
+ * @returns {number} Number of items cleaned
+ */
+function cleanTempDir(basePath, forceDebug) {
+  let entries;
+  try {
+    entries = fs.readdirSync(basePath);
+  } catch {
+    if (forceDebug) console.log(`[debug] [temp-cleanup] Cannot read ${basePath}`);
+    return 0;
+  }
+
+  let cleaned = 0;
+  for (const entry of entries) {
+    let matched = false;
+    for (const re of CHROME_TEMP_PATTERNS) {
+      if (re.test(entry)) { matched = true; break; }
+    }
+    if (!matched) continue;
+
+    try {
+      fs.rmSync(path.join(basePath, entry), { recursive: true, force: true });
+      cleaned++;
+      if (forceDebug) console.log(`[debug] [temp-cleanup] Removed ${basePath}/${entry}`);
+    } catch (rmErr) {
+      if (forceDebug) console.log(`[debug] [temp-cleanup] Failed to remove ${basePath}/${entry}: ${rmErr.message}`);
+    }
+  }
+
+  return cleaned;
+}
+
 /**
  * Clean Chrome temporary files and directories
  * @param {Object} options - Cleanup options
  * @param {boolean} options.includeSnapTemp - Whether to clean snap temp directories
  * @param {boolean} options.forceDebug - Whether to output debug logs
  * @param {boolean} options.comprehensive - Whether to perform comprehensive cleanup of all temp locations
- * @returns {Promise<Object>} Cleanup results
+ * @returns {Object} Cleanup results
  */
-async function cleanupChromeTempFiles(options = {}) {
+function cleanupChromeTempFiles(options = {}) {
   const {
     includeSnapTemp = false,
     forceDebug = false,
@@ -36,57 +72,20 @@ async function cleanupChromeTempFiles(options = {}) {
   } = options;
 
   try {
-
-    // Base cleanup commands for standard temp directories
-    const cleanupCommands = [
-      'rm -rf /tmp/com.google.Chrome.* 2>/dev/null || true',
-      'rm -rf /tmp/.com.google.Chrome.* 2>/dev/null || true',
-      'rm -rf /tmp/.org.chromium.Chromium.* 2>/dev/null || true',
-      'rm -rf /tmp/puppeteer-* 2>/dev/null || true',
-      'rm -rf /dev/shm/.com.google.Chrome.* 2>/dev/null || true',
-      'rm -rf /dev/shm/.org.chromium.Chromium.* 2>/dev/null || true'
-    ];
-
-    // Add snap-specific cleanup if requested
-    if (includeSnapTemp || comprehensive) {
-      cleanupCommands.push('rm -rf /dev/shm/com.google.Chrome.* 2>/dev/null || true');
-      cleanupCommands.push(
-        'rm -rf /tmp/snap-private-tmp/snap.chromium/tmp/.org.chromium.Chromium.* 2>/dev/null || true',
-        'rm -rf /tmp/snap-private-tmp/snap.chromium/tmp/puppeteer-* 2>/dev/null || true'
-      );
-    }
+    const paths = comprehensive || includeSnapTemp
+      ? CHROME_TEMP_PATHS
+      : CHROME_TEMP_PATHS.slice(0, 2); // /tmp and /dev/shm only
 
     let totalCleaned = 0;
-    
-    for (const command of cleanupCommands) {
-      try {
-        // Extract glob pattern and count matches before deletion
-        const globPattern = command.match(/rm -rf ([^ ]+)/)?.[1];
-        if (!globPattern) continue;
-        const fileCount = parseInt(execSync(`ls -1d ${globPattern} 2>/dev/null | wc -l || echo 0`, { stdio: 'pipe' }).toString().trim()) || 0;
-        
-        if (fileCount > 0) {
-          execSync(command, { stdio: 'ignore' });
-          totalCleaned += fileCount;
-          
-          if (forceDebug) {
-            console.log(`[debug] [temp-cleanup] Cleaned ${fileCount} items from ${globPattern}`);
-          }
-        }
-      } catch (cmdErr) {
-        // Ignore individual command errors but log in debug mode
-        if (forceDebug) {
-          console.log(`[debug] [temp-cleanup] Cleanup command failed: ${command} (${cmdErr.message})`);
-        }
-      }
+    for (const basePath of paths) {
+      totalCleaned += cleanTempDir(basePath, forceDebug);
     }
 
     if (forceDebug) {
-      console.log(`[debug] [temp-cleanup] Standard cleanup completed (${totalCleaned} items)`);
+      console.log(`[debug] [temp-cleanup] Cleanup completed (${totalCleaned} items)`);
     }
-    
+
     return { success: true, itemsCleaned: totalCleaned };
-    
   } catch (cleanupErr) {
     if (forceDebug) {
       console.log(`[debug] [temp-cleanup] Chrome cleanup error: ${cleanupErr.message}`);
@@ -96,72 +95,38 @@ async function cleanupChromeTempFiles(options = {}) {
 }
 
 /**
- * Comprehensive temp file cleanup that systematically checks all known Chrome temp locations
+ * Comprehensive temp file cleanup that checks all known Chrome temp locations
  * @param {Object} options - Cleanup options
  * @param {boolean} options.forceDebug - Whether to output debug logs
  * @param {boolean} options.verbose - Whether to show verbose output
- * @returns {Promise<Object>} Cleanup results
+ * @returns {Object} Cleanup results
  */
-async function comprehensiveChromeTempCleanup(options = {}) {
+function comprehensiveChromeTempCleanup(options = {}) {
   const { forceDebug = false, verbose = false } = options;
-  
+
   try {
-    let totalCleaned = 0;
-    
     if (verbose && !forceDebug) {
       console.log(`[temp-cleanup] Scanning Chrome/Puppeteer temporary files...`);
     }
-    
+
+    let totalCleaned = 0;
     for (const basePath of CHROME_TEMP_PATHS) {
-      // Check if the base path exists before trying to clean it
-      try {
-        const pathExists = fs.existsSync(basePath);
-        
-        if (!pathExists) {
-          if (forceDebug) {
-            console.log(`[debug] [temp-cleanup] Skipping non-existent path: ${basePath}`);
-          }
-          continue;
-        }
-        
-        for (const pattern of CHROME_TEMP_PATTERNS) {
-          const fullPattern = `${basePath}/${pattern}`;
-          
-          // Count items before deletion
-          const countCommand = `ls -1d ${fullPattern} 2>/dev/null | wc -l || echo 0`;
-          const itemCount = parseInt(execSync(countCommand, { stdio: 'pipe' }).toString().trim()) || 0;
-          
-          if (itemCount > 0) {
-            const deleteCommand = `rm -rf ${fullPattern} 2>/dev/null || true`;
-            execSync(deleteCommand, { stdio: 'ignore' });
-            totalCleaned += itemCount;
-            
-            if (forceDebug) {
-              console.log(`[debug] [temp-cleanup] Removed ${itemCount} items matching ${fullPattern}`);
-            }
-          }
-        }
-      } catch (pathErr) {
-        if (forceDebug) {
-          console.log(`[debug] [temp-cleanup] Error checking path ${basePath}: ${pathErr.message}`);
-        }
-      }
+      totalCleaned += cleanTempDir(basePath, forceDebug);
     }
-    
+
     if (verbose && totalCleaned > 0) {
-      console.log(`[temp-cleanup] ? Removed ${totalCleaned} temporary file(s)/folder(s)`);
+      console.log(`[temp-cleanup] Removed ${totalCleaned} temporary file(s)/folder(s)`);
     } else if (verbose && totalCleaned === 0) {
-      console.log(`[temp-cleanup] ? Clean - no remaining temporary files`);
+      console.log(`[temp-cleanup] Clean - no remaining temporary files`);
     } else if (forceDebug) {
       console.log(`[debug] [temp-cleanup] Comprehensive cleanup completed (${totalCleaned} items)`);
     }
-    
+
     return { success: true, itemsCleaned: totalCleaned };
-    
   } catch (err) {
     const errorMsg = `Comprehensive temp file cleanup failed: ${err.message}`;
     if (verbose) {
-      console.warn(`[temp-cleanup] ? ${errorMsg}`);
+      console.warn(`[temp-cleanup] ${errorMsg}`);
     } else if (forceDebug) {
       console.log(`[debug] [temp-cleanup] ${errorMsg}`);
     }
@@ -317,7 +282,7 @@ async function forceBrowserKill(browser, forceDebug = false) {
       }
       
       // Wait for graceful termination
-      await new Promise(resolve => setTimeout(resolve, 3000));
+      await new Promise(resolve => setTimeout(resolve, 1000));
       
       // Force kill any remaining processes with SIGKILL
       for (const pid of pidsToKill) {