@fanboynz/network-scanner 3.0.1 → 3.0.3

package/nwss.js

@@ -109,6 +109,7 @@ const TIMEOUTS = Object.freeze({
   EMERGENCY_RESTART_DELAY: 2000,      // Delay after emergency browser restart
   BROWSER_STABILIZE_DELAY: 1000,      // Browser stabilization after restart
   CURL_HANDLER_DELAY: 3000,           // Wait for async curl operations
+  NETTOOLS_DRAIN_TIMEOUT: 3000,       // Hard cap for awaiting in-flight nettools (dig/whois) handlers before snapshot. Drains immediately if all complete; bounded so a hung dig can't block exit. Mirrors CURL_HANDLER_DELAY's role for curl/searchstring.
   PROTOCOL_TIMEOUT: 180000,           // Chrome DevTools Protocol timeout
   REDIRECT_JS_TIMEOUT: 5000           // JavaScript redirect detection timeout
 });
@@ -777,7 +778,8 @@ Redirect Handling Options:
   isBrave: true/false                          Spoof Brave browser detection
   userAgent: "chrome"|"chrome_mac"|"chrome_linux"|"firefox"|"firefox_mac"|"firefox_linux"|"safari"  Custom desktop User-Agent
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
-  delay: <milliseconds>                        Delay after load (default: 4000)
+  delay: <milliseconds>                        Delay after load (default: 6000, capped at 2000ms unless delay_uncapped: true)
+  delay_uncapped: true/false                   Honor 'delay' up to half the per-URL timeout instead of the 2s default cap. Use for sites with setTimeout-deferred lazy ad/tracker loaders that fire well past the standard post-networkidle window
   reload: <number>                             Reload page n times after load (default: 1)
   forcereload: true/false or ["domain1.com", "domain2.com"]  Force cache-clearing reload for all URLs or specific domains
   clear_sitedata: true/false                   Clear all cookies, cache, storage before each load (default: false)
@@ -1864,7 +1866,13 @@ function setupFrameHandling(page, forceDebug) {
         '--disable-domain-reliability', // No reliability monitor disk writes
         // PERFORMANCE: Disable non-essential Chrome features in a single flag
         // IMPORTANT: Chrome only reads the LAST --disable-features flag, so combine all into one
-        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding${disable_ad_tagging ? ',AdTagging' : ''}`,
+        // AccountConsistencyMirror + AccountConsistencyDice prevent the
+        // Chrome sign-in subsystem from initialising at startup. Combined
+        // with --disable-sync + --allow-browser-signin=false below, this
+        // suppresses the "Something went wrong when opening your profile"
+        // popup that fires in headful + --keep-open mode (temp userDataDir
+        // has no real profile, so the sync init errors out and pops up).
+        `--disable-features=AudioServiceOutOfProcess,VizDisplayCompositor,TranslateUI,BlinkGenPropertyTrees,Translate,BackForwardCache,AcceptCHFrame,SafeBrowsing,HttpsFirstBalancedModeAutoEnable,site-per-process,PaintHolding,AccountConsistencyMirror,AccountConsistencyDice${disable_ad_tagging ? ',AdTagging' : ''}`,
         '--disable-ipc-flooding-protection',
         '--aggressive-cache-discard',
         '--memory-pressure-off',
@@ -1874,7 +1882,16 @@ function setupFrameHandling(page, forceDebug) {
         '--no-sandbox',
         '--disable-setuid-sandbox',
         '--disable-dev-shm-usage',
-        ...(keepBrowserOpen ? [] : ['--disable-sync']),
+        // --disable-sync is always-on (was previously dropped in --keep-open
+        // mode, which let the sync subsystem init against our temp
+        // userDataDir and pop the "Something went wrong when opening your
+        // profile" dialog). Inspection during --keep-open doesn't need
+        // sync; nothing in the scanner flow does.
+        '--disable-sync',
+        // Prevent the sign-in promo / account banner from appearing in
+        // headful sessions. Same family of fixes as --disable-sync and the
+        // AccountConsistency* features disabled above.
+        '--allow-browser-signin=false',
         '--mute-audio',
         '--disable-translate',
         '--window-size=1920,1080',
@@ -2100,6 +2117,30 @@ function setupFrameHandling(page, forceDebug) {
     // Use Map to track domains and their resource types for --adblock-rules or --dry-run
     const matchedDomains = (adblockRulesMode || siteConfig.adblock_rules || dryRunMode) ? new Map() : new Set();
 
+    // Per-URL tracking of in-flight async nettools (dig/whois) handlers so we
+    // can drain them BEFORE snapshotting matchedDomains into the result. The
+    // previous fire-and-forget setImmediate pattern dropped late-completing
+    // matches (handler resolved after formatRules had already run). Each
+    // setImmediate-scheduled handler now registers a promise via
+    // trackNetToolsHandler; drainPendingNetTools() awaits all of them with a
+    // hard cap (TIMEOUTS.NETTOOLS_DRAIN_TIMEOUT) so a hung dig can't block.
+    const pendingNetTools = [];
+    const trackNetToolsHandler = (handlerFn) => {
+      pendingNetTools.push(new Promise((resolve) => {
+        setImmediate(async () => {
+          try { await handlerFn(); } catch (_) { /* handler logs its own errors */ }
+          finally { resolve(); }
+        });
+      }));
+    };
+    const drainPendingNetTools = async () => {
+      if (pendingNetTools.length === 0) return;
+      await Promise.race([
+        Promise.all(pendingNetTools),
+        fastTimeout(TIMEOUTS.NETTOOLS_DRAIN_TIMEOUT)
+      ]);
+    };
+
     // Local domain dedup scoped to THIS processUrl call only
     // Prevents cross-config contamination from the global domain cache
     const localDetectedDomains = new Set();
@@ -3167,7 +3208,7 @@ function setupFrameHandling(page, forceDebug) {
                   currentUrl, getRootDomain, siteConfig, dumpUrls, matchedUrlsLogFile, forceDebug, fs,
                   ignoreDomains, matchesIgnoreDomain
                 });
-                setImmediate(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
+                trackNetToolsHandler(() => popupNetToolsHandler(checkedRootDomain, fullSubdomain));
               } else {
                 // No nettools required — regex match alone counts.
                 addMatchedDomain(checkedRootDomain, resourceType, fullSubdomain);
@@ -3573,7 +3614,7 @@ function setupFrameHandling(page, forceDebug) {
 
               // Execute nettools check asynchronously
               const originalDomain = fullSubdomain;
-              setImmediate(() => netToolsHandler(reqDomain, originalDomain));
+              trackNetToolsHandler(() => netToolsHandler(reqDomain, originalDomain));
             }
             if (forceDebug) {
               console.log(formatLogMessage('debug', `${reqUrl} has nettools validation required - skipping immediate add`));
@@ -3688,7 +3729,7 @@ function setupFrameHandling(page, forceDebug) {
 
              // Execute nettools check asynchronously
             const originalDomain = fullSubdomain; // Use full subdomain for nettools
-            setImmediate(() => netToolsHandler(reqDomain, originalDomain));
+            trackNetToolsHandler(() => netToolsHandler(reqDomain, originalDomain));
 
              // Do NOT continue processing this request for immediate domain addition
              // The nettools handler is responsible for adding the domain if validation passes
@@ -4237,13 +4278,22 @@ function setupFrameHandling(page, forceDebug) {
       }
       }
 
-      const delayMs = DEFAULT_DELAY;
+      const delayMs = siteConfig.delay || DEFAULT_DELAY;
 
       // Optimized delays for Puppeteer 23.x performance
       const isFastSite = timeout <= TIMEOUTS.FAST_SITE_THRESHOLD;
       const networkIdleTime = TIMEOUTS.NETWORK_IDLE;  // Balanced: 2s for reliable network detection
       const networkIdleTimeout = Math.min(timeout / 2, TIMEOUTS.NETWORK_IDLE_MAX);  // Balanced: 10s timeout
-      const actualDelay = Math.min(delayMs, TIMEOUTS.NETWORK_IDLE);  // Balanced: 2s delay for stability
+      // Post-networkidle delay cap. Default (2s) keeps fast sites fast. Opt
+      // in with `delay_uncapped: true` to honor the configured `delay` up to
+      // half the per-URL timeout — useful for sites with setTimeout-deferred
+      // lazy ad/tracker loaders (weather.com, cbssports.com class) where
+      // late requests fire well past the 2s window. See also the per-URL
+      // drainPendingNetTools() which awaits in-flight dig/whois handlers
+      // before the matchedDomains snapshot regardless of this flag.
+      const actualDelay = siteConfig.delay_uncapped === true
+        ? Math.min(delayMs, Math.floor(timeout / 2))
+        : Math.min(delayMs, TIMEOUTS.NETWORK_IDLE);
 
       // Build delay promise (networkIdle + delay + optional flowProxy delay)
       const delayPromise = (async () => {
@@ -4625,7 +4675,8 @@ function setupFrameHandling(page, forceDebug) {
         // Wait a moment for async nettools/searchstring operations to complete
         // Use fast timeout helper for Puppeteer 22.x compatibility
         await fastTimeout(TIMEOUTS.CURL_HANDLER_DELAY); // Wait for async operations
-        
+        await drainPendingNetTools(); // Bounded wait for in-flight dig/whois (race fix)
+
         return { url: currentUrl, rules: [], success: true, dryRun: true, matchCount: dryRunResult.matchCount };
       } else {
         // Format rules using the output module
@@ -4639,6 +4690,12 @@ function setupFrameHandling(page, forceDebug) {
         privoxyMode,
         piholeMode
       };
+        // Drain pending dig/whois handlers BEFORE snapshotting matchedDomains.
+        // Without this, late-completing async validations (request fired near
+        // end of the delay window, dig still in flight) get orphaned — their
+        // addMatchedDomain calls happen but the result has already been
+        // returned. Bounded by TIMEOUTS.NETTOOLS_DRAIN_TIMEOUT.
+        await drainPendingNetTools();
         const formattedRules = formatRules(matchedDomains, siteConfig, globalOptions);
         
         return {
@@ -4690,7 +4747,11 @@ function setupFrameHandling(page, forceDebug) {
       };
     }
          
-      // For other errors, preserve any matches we found before the error
+      // For other errors, preserve any matches we found before the error.
+      // Drain pending nettools first so dig/whois handlers scheduled DURING
+      // the failed navigation get a chance to add to matchedDomains before
+      // the partial-success snapshot — same race as the success path.
+      await drainPendingNetTools();
       if (matchedDomains && (matchedDomains.size > 0 || (matchedDomains instanceof Map && matchedDomains.size > 0))) {
         const globalOptions = {
           localhostIP,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "3.0.1",
+  "version": "3.0.3",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {

package/scripts/test-stealth.js

@@ -70,7 +70,7 @@ const TARGETS = [
     extract: async (page) => {
       return await page.evaluate(() => {
         const cells = Array.from(document.querySelectorAll('td'));
-        const out = { passed: 0, failed: 0, warn: 0, total: 0, failures: [] };
+        const out = { passed: 0, failed: 0, warn: 0, total: 0, failures: [], warnings: [] };
         for (const c of cells) {
           const cls = c.className || '';
           if (cls.includes('passed')) { out.passed++; out.total++; }
@@ -81,7 +81,14 @@ const TARGETS = [
             const label = row?.querySelector('td')?.textContent?.trim() || '?';
             out.failures.push(label);
           }
-          else if (cls.includes('warn')) { out.warn++; out.total++; }
+          else if (cls.includes('warn')) {
+            out.warn++; out.total++;
+            // Capture warn-row labels too so a soft regression (cell moving
+            // passed -> warn) is debuggable without --headful.
+            const row = c.closest('tr');
+            const label = row?.querySelector('td')?.textContent?.trim() || '?';
+            out.warnings.push(label);
+          }
         }
         return out;
       });
@@ -97,15 +104,29 @@ const TARGETS = [
       await new Promise(r => setTimeout(r, 8000)); // give async tests time
       return await page.evaluate(() => {
         const text = document.body.innerText || '';
-        // CreepJS reports a "Trust Score" percentage and individual signal entries.
-        const trustMatch = text.match(/Trust Score[:\s]+(\d+(?:\.\d+)?)\s*%/i);
-        const lieMatch = text.match(/lies[:\s]+(\d+)/i);
-        const botMatch = text.match(/bot[:\s]+(true|false)/i);
+        // CreepJS's actual stealth-relevant outputs are in a "Headless"
+        // section as percentages (e.g. "67% headless", "40% stealth",
+        // "44% like headless"), not the "Trust Score" label the old
+        // regex expected. Engine identification comes from "chromium:
+        // true/false" in the same block. Lower headless % and higher
+        // stealth % are better for evasion.
+        const headlessMatch = text.match(/(\d+(?:\.\d+)?)\s*%\s+headless\b/i);
+        const likeHeadlessMatch = text.match(/(\d+(?:\.\d+)?)\s*%\s+like\s+headless/i);
+        const stealthMatch = text.match(/(\d+(?:\.\d+)?)\s*%\s+stealth\b/i);
+        const chromiumMatch = text.match(/chromium\s*:\s*(true|false)/i);
+        // FP ID is CreepJS's stable fingerprint hash — same value across
+        // reloads if the fingerprint is unchanged; lets you A/B before
+        // and after a spoof change.
+        const fpIdMatch = text.match(/FP\s*ID\s*:?\s*([0-9a-f]{16,})/i);
         return {
-          trustScore: trustMatch ? parseFloat(trustMatch[1]) : null,
-          lies: lieMatch ? parseInt(lieMatch[1], 10) : null,
-          botDetected: botMatch ? botMatch[1] === 'true' : null,
-          excerpt: text.split('\n').slice(0, 15).join('\n').slice(0, 400)
+          headlessPct: headlessMatch ? parseFloat(headlessMatch[1]) : null,
+          likeHeadlessPct: likeHeadlessMatch ? parseFloat(likeHeadlessMatch[1]) : null,
+          stealthPct: stealthMatch ? parseFloat(stealthMatch[1]) : null,
+          isChromium: chromiumMatch ? chromiumMatch[1] === 'true' : null,
+          fpId: fpIdMatch ? fpIdMatch[1].slice(0, 16) : null,
+          // Larger excerpt (40 lines, up to 2KB) so a future UI rotation
+          // is debuggable from the output without --headful.
+          excerpt: text.split('\n').slice(0, 40).join('\n').slice(0, 2000)
         };
       });
     }
@@ -165,10 +186,15 @@ function formatResult(target, result) {
     if (result.failures.length) {
       lines.push(`  failure rows: ${result.failures.slice(0, 10).join(', ')}${result.failures.length > 10 ? ` ... +${result.failures.length - 10} more` : ''}`);
     }
+    if (result.warnings && result.warnings.length) {
+      lines.push(`  warn rows: ${result.warnings.slice(0, 10).join(', ')}${result.warnings.length > 10 ? ` ... +${result.warnings.length - 10} more` : ''}`);
+    }
   } else if (target.name === 'creepjs') {
-    lines.push(`  trust score: ${result.trustScore ?? 'n/a'}%`);
-    lines.push(`  lies detected: ${result.lies ?? 'n/a'}`);
-    lines.push(`  bot flagged: ${result.botDetected ?? 'n/a'}`);
+    lines.push(`  FP ID: ${result.fpId ?? 'n/a'}  (stable across reloads if fingerprint unchanged)`);
+    lines.push(`  engine chromium: ${result.isChromium ?? 'n/a'}`);
+    lines.push(`  headless score: ${result.headlessPct ?? 'n/a'}%  (lower = better; 0% = real browser)`);
+    lines.push(`  like-headless: ${result.likeHeadlessPct ?? 'n/a'}%  (lower = better; soft headless signals)`);
+    lines.push(`  stealth score: ${result.stealthPct ?? 'n/a'}%  (lower = better; % likely to be using anti-detection tooling)`);
     if (result.excerpt) lines.push(`  excerpt:\n    ${result.excerpt.split('\n').join('\n    ')}`);
   } else if (target.name === 'browserleaks') {
     for (const [k, v] of Object.entries(result)) {