@fanboynz/network-scanner 2.0.59 → 2.0.60

package/nwss.js

@@ -32,7 +32,7 @@ const { shouldIgnoreSimilarDomain, calculateSimilarity } = require('./lib/ignore
 // Graceful exit
 const { handleBrowserExit, cleanupChromeTempFiles } = require('./lib/browserexit');
 // Whois & Dig
-const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability } = require('./lib/nettools');
+const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability, enableDiskCache, getDnsCacheStats } = require('./lib/nettools');
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
 // CDP functionality
@@ -208,6 +208,12 @@ if (localhostIndex !== -1) {
   localhostIP = args[localhostIndex].includes('=') ? args[localhostIndex].split('=')[1] : '127.0.0.1';
 }
 const keepBrowserOpen = args.includes('--keep-open');
+const loadExtensionPaths = [];
+args.forEach((arg, idx) => {
+  if (arg === '--load-extension' && args[idx + 1] && !args[idx + 1].startsWith('--')) {
+    loadExtensionPaths.push(path.resolve(args[idx + 1]));
+  }
+});
 const disableInteract = args.includes('--no-interact');
 const globalGhostCursor = args.includes('--ghost-cursor');
 const plainOutput = args.includes('--plain');
@@ -229,6 +235,8 @@ let cleanRules = args.includes('--clean-rules');
 const clearCache = args.includes('--clear-cache');
 const ignoreCache = args.includes('--ignore-cache');
 const cacheRequests = args.includes('--cache-requests');
+const dnsCacheMode = args.includes('--dns-cache');
+if (dnsCacheMode) enableDiskCache();
 
 let validateRulesFile = null;
 const validateRulesIndex = args.findIndex(arg => arg === '--validate-rules');
@@ -499,22 +507,38 @@ if (validateRules || validateRulesFile) {
   }
 }
 
-// Parse --block-ads argument for request-level ad blocking
+// Parse --block-ads argument for request-level ad blocking (supports comma-separated lists)
 const blockAdsIndex = args.findIndex(arg => arg.startsWith('--block-ads'));
 if (blockAdsIndex !== -1) {
-  const rulesFile = args[blockAdsIndex].includes('=') 
-    ? args[blockAdsIndex].split('=')[1] 
+  const rulesArg = args[blockAdsIndex].includes('=')
+    ? args[blockAdsIndex].split('=')[1]
     : args[blockAdsIndex + 1];
-  
-  if (!rulesFile || !fs.existsSync(rulesFile)) {
-    console.log(`Error: Adblock rules file not found: ${rulesFile || '(not specified)'}`);
+
+  if (!rulesArg) {
+    console.log('Error: No adblock rules file specified');
     process.exit(1);
   }
-  
+
+  const rulesFiles = rulesArg.split(',').map(f => f.trim()).filter(f => f);
+  for (const file of rulesFiles) {
+    if (!fs.existsSync(file)) {
+      console.log(`Error: Adblock rules file not found: ${file}`);
+      process.exit(1);
+    }
+  }
+
+  // Concatenate multiple lists into a single temp file for the parser
+  let rulesFile = rulesFiles[0];
+  if (rulesFiles.length > 1) {
+    rulesFile = path.join(os.tmpdir(), `nwss-adblock-combined-${Date.now()}.txt`);
+    const combined = rulesFiles.map(f => fs.readFileSync(f, 'utf-8')).join('\n');
+    fs.writeFileSync(rulesFile, combined);
+  }
+
   adblockEnabled = true;
   adblockMatcher = parseAdblockRules(rulesFile, { enableLogging: forceDebug });
   const stats = adblockMatcher.getStats();
-  if (!silentMode) console.log(messageColors.success(`Adblock enabled: Loaded ${stats.total} blocking rules from ${rulesFile}`));
+  if (!silentMode) console.log(messageColors.success(`Adblock enabled: Loaded ${stats.total} blocking rules from ${rulesFiles.length} list${rulesFiles.length > 1 ? 's' : ''}`));
 }
 
 if (args.includes('--help') || args.includes('-h')) {
@@ -556,6 +580,7 @@ General Options:
   --headful                      Launch browser with GUI (not headless)
   --keep-open                    Keep browser open after scan completes (use with --headful)
   --use-puppeteer-core           Use puppeteer-core with system Chrome instead of bundled Chromium
+  --load-extension <path>        Load unpacked Chrome extension from directory
   --cdp                          Enable Chrome DevTools Protocol logging (now per-page if enabled)
   --remove-dupes                 Remove duplicate domains from output (only with -o)
   --eval-on-doc                 Globally enable evaluateOnNewDocument() for Fetch/XHR interception
@@ -567,6 +592,7 @@ General Options:
 
 Validation Options:
   --cache-requests               Cache HTTP requests to avoid re-requesting same URLs within scan
+  --dns-cache                    Persist dig/whois results to disk between runs (3hr/4hr TTL)
   --validate-config              Validate config.json file and exit
   --validate-rules [file]        Validate rule file format (uses --output/--compare files if no file specified)
   --clean-rules [file]           Clean rule files by removing invalid lines and optionally duplicates (uses --output/--compare files if no file specified)
@@ -583,6 +609,7 @@ Global config.json options:
   ignore_similar_ignored_domains: true/false      Ignore domains similar to ignoreDomains list (default: true)
   max_concurrent_sites: 8                        Maximum concurrent site processing (1-50, default: 8)
   resource_cleanup_interval: 80                  Browser restart interval in URLs processed (1-1000, default: 80)
+  disable_ad_tagging: true/false                 Disable Chrome AdTagging to prevent ad frame throttling (default: true)
 
 Per-site config.json options:
   url: "site" or ["site1", "site2"]          Single URL or list of URLs
@@ -748,7 +775,8 @@ const {
   whois_server_mode = 'random', 
   ignore_similar = true, 
   ignore_similar_threshold = 80, 
-  ignore_similar_ignored_domains = true, 
+  ignore_similar_ignored_domains = true,
+  disable_ad_tagging = true,
   max_concurrent_sites = 6,
   resource_cleanup_interval = 80,
   comments: globalComments, 
@@ -1460,7 +1488,7 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-blink-features=AutomationControlled',
         '--no-first-run',
         '--disable-default-apps',
-        '--disable-component-extensions-with-background-pages',
+        ...(keepBrowserOpen ? [] : ['--disable-component-extensions-with-background-pages']),
         // HIGH IMPACT: Normal Chrome behavior simulation
         '--password-store=basic',
         '--use-mock-keychain',
@@ -1474,30 +1502,29 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-background-downloads',
         // DISK I/O REDUCTION: Eliminate unnecessary Chrome disk writes
         '--disable-breakpad',          // No crash dump files
-        '--disable-component-update',  // No component update downloads
+        ...(keepBrowserOpen ? [] : ['--disable-component-update']),  // No component update downloads
         '--disable-logging',           // No Chrome internal log files
         '--log-level=3',               // Fatal errors only (suppresses verbose disk logging)
         '--no-service-autorun',        // No background service disk activity
         '--disable-domain-reliability', // No reliability monitor disk writes
-        // PERFORMANCE: Enhanced Puppeteer 23.x optimizations
-        '--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor',
-        '--disable-features=TranslateUI,BlinkGenPropertyTrees,Translate',
-        '--disable-features=BackForwardCache,AcceptCHFrame',
+        // PERFORMANCE: Disable non-essential Chrome features in a single flag
+        // IMPORTANT: Chrome only reads the LAST --disable-features flag, so combine all into one
+        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding${disable_ad_tagging ? ',AdTagging' : ''}`,
         '--disable-ipc-flooding-protection',
         '--aggressive-cache-discard',
         '--memory-pressure-off',
         '--max_old_space_size=2048',   // V8 heap limit
         '--disable-prompt-on-repost',  // Fixes form popup on page reload
-        '--disable-background-networking',
+        ...(keepBrowserOpen ? [] : ['--disable-background-networking']),
         '--no-sandbox',
         '--disable-setuid-sandbox',
-        '--disable-features=SafeBrowsing',
         '--disable-dev-shm-usage',
-        '--disable-sync',
+        ...(keepBrowserOpen ? [] : ['--disable-sync']),
         '--mute-audio',
         '--disable-translate',
         '--window-size=1920,1080',
-        '--disable-extensions',
+        ...(keepBrowserOpen ? [] : ['--disable-extensions', '--disable-component-update']),
+        ...(loadExtensionPaths.length ? [`--load-extension=${loadExtensionPaths.join(',')}`, '--enable-extensions'] : []),
         '--no-default-browser-check',
         '--safebrowsing-disable-auto-update',
         '--ignore-ssl-errors',
@@ -1506,18 +1533,15 @@ function setupFrameHandling(page, forceDebug) {
         '--ignore-certificate-errors-ca-list',
         '--disable-web-security',
         '--allow-running-insecure-content',
-        '--disable-features=HttpsFirstBalancedModeAutoEnable',
         // Puppeteer 23.x: Enhanced performance and stability args
         '--disable-renderer-backgrounding',
         '--disable-backgrounding-occluded-windows',
         '--disable-background-timer-throttling',
-        '--disable-features=site-per-process', // Better for single-site scanning
         '--no-zygote', // Better process isolation
         // PERFORMANCE: Process and memory reduction for high concurrency
         '--renderer-process-limit=10',  // Cap renderer processes (default: unlimited)
         '--disable-accelerated-2d-canvas', // Software canvas only (we spoof it anyway)
         '--disable-hang-monitor',      // Remove per-renderer hang check overhead
-        '--disable-features=PaintHolding', // Don't hold frames in renderer memory
         '--js-flags=--max-old-space-size=512', // Cap V8 heap per renderer to 512MB
         ...extraArgs,
         ],
@@ -4601,9 +4625,19 @@ function setupFrameHandling(page, forceDebug) {
   // Keep browser open if --keep-open flag is set (useful with --headful for inspection)
   if (keepBrowserOpen && !launchHeadless) {
     console.log(messageColors.info('Browser kept open.') + ' Close the browser window or press Ctrl+C to exit.');
+    const cleanup = async () => {
+      try {
+        if (browser.isConnected()) await browser.close();
+      } catch {}
+      process.exit(0);
+    };
+    process.on('SIGINT', cleanup);
+    process.on('SIGTERM', cleanup);
     await new Promise((resolve) => {
       browser.on('disconnected', resolve);
     });
+    process.removeListener('SIGINT', cleanup);
+    process.removeListener('SIGTERM', cleanup);
   }
 
   // Perform comprehensive final cleanup using enhanced browserexit module
@@ -4680,6 +4714,24 @@ function setupFrameHandling(page, forceDebug) {
     if (ignoreCache && forceDebug) {
       console.log(messageColors.info('Cache:') + ` Smart caching was disabled`);
     }
+    // DNS cache statistics
+    const dnsStats = getDnsCacheStats();
+    if (dnsStats.digHits + dnsStats.digMisses > 0 || dnsStats.whoisHits + dnsStats.whoisMisses > 0) {
+      const parts = [];
+      if (dnsStats.digHits + dnsStats.digMisses > 0) {
+        parts.push(`${messageColors.success(dnsStats.digHits)} dig cached, ${messageColors.timing(dnsStats.digMisses)} fresh`);
+      }
+      if (dnsStats.whoisHits + dnsStats.whoisMisses > 0) {
+        parts.push(`${messageColors.success(dnsStats.whoisHits)} whois cached, ${messageColors.timing(dnsStats.whoisMisses)} fresh`);
+      }
+      console.log(messageColors.info('DNS cache:') + ` ${parts.join(' | ')}`);
+      if (dnsStats.freshDig.length > 0) {
+        console.log(messageColors.info('  Fresh dig:') + ` ${dnsStats.freshDig.join(', ')}`);
+      }
+      if (dnsStats.freshWhois.length > 0) {
+        console.log(messageColors.info('  Fresh whois:') + ` ${dnsStats.freshWhois.join(', ')}`);
+      }
+    }
   }
   
   // Clean process termination

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.59",
+  "version": "2.0.60",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {