@fanboynz/network-scanner 1.0.73 → 1.0.75

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.73 ===
+// === Network scanner script (nwss.js) v1.0.75 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -123,7 +123,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.73'; // Script version
+const VERSION = '1.0.75'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -2111,7 +2111,51 @@ function setupFrameHandling(page, forceDebug) {
  
             // REMOVED: Check if this URL matches any blocked patterns - if so, skip detection but still continue browser blocking
             // This check is no longer needed here since even_blocked handles it above
-
+            
+          // Check if nettools validation is required - if so, NEVER add domains immediately
+          if (hasNetTools) {
+            // Call nettools handler BEFORE exiting
+            if (hasNetTools && !hasSearchString && !hasSearchStringAnd) {
+              // Create and execute nettools handler
+              const netToolsHandler = createNetToolsHandler({
+                whoisTerms,
+                whoisOrTerms,
+                whoisDelay: siteConfig.whois_delay || whois_delay,
+                whoisServer,
+                whoisServerMode: siteConfig.whois_server_mode || whois_server_mode,
+                debugLogFile,
+                fs,
+                digTerms,
+                digOrTerms,
+                digRecordType,
+                digSubdomain: siteConfig.dig_subdomain === true,
+                dryRunCallback: dryRunMode ? createEnhancedDryRunCallback(matchedDomains, forceDebug) : null,
+                matchedDomains,
+                addMatchedDomain,
+                isDomainAlreadyDetected,
+                onWhoisResult: smartCache ? (domain, result) => smartCache.cacheNetTools(domain, 'whois', result) : undefined,
+                onDigResult: smartCache ? (domain, result, recordType) => smartCache.cacheNetTools(domain, 'dig', result, recordType) : undefined,
+                cachedWhois: smartCache ? smartCache.getCachedNetTools(reqDomain, 'whois') : null,
+                cachedDig: smartCache ? smartCache.getCachedNetTools(reqDomain, 'dig', digRecordType) : null,
+                currentUrl,
+                getRootDomain,
+                siteConfig,
+                dumpUrls,
+                matchedUrlsLogFile,
+                forceDebug,
+                fs
+              });
+              
+              // Execute nettools check asynchronously
+              const originalDomain = fullSubdomain;
+              setImmediate(() => netToolsHandler(reqDomain, originalDomain));
+            }
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `${reqUrl} has nettools validation required - skipping immediate add`));
+            }
+            request.continue();
+            return;
+          }
            
            // If NO searchstring AND NO nettools are defined, match immediately (existing behavior)
            if (!hasSearchString && !hasSearchStringAnd && !hasNetTools) {
@@ -2152,6 +2196,12 @@ function setupFrameHandling(page, forceDebug) {
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${matchedRegexPattern} and resourceType ${resourceType}, queued for nettools check`));
              }
 
+             // IMPORTANT: Do NOT add domain immediately when nettools validation is required
+             // The nettools handler will add the domain only if validation passes
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `Domain ${reqDomain} queued for mandatory nettools validation (dig: ${JSON.stringify(siteConfig.dig)})`));
+             }
+
              if (dryRunMode) {
                // For dry run, we'll collect the domain for nettools checking
                matchedDomains.get('dryRunMatches').push({
@@ -2212,6 +2262,12 @@ function setupFrameHandling(page, forceDebug) {
              // Execute nettools check asynchronously
             const originalDomain = fullSubdomain; // Use full subdomain for nettools
             setImmediate(() => netToolsHandler(reqDomain, originalDomain));
+
+             // Do NOT continue processing this request for immediate domain addition
+             // The nettools handler is responsible for adding the domain if validation passes
+             if (forceDebug) {
+               console.log(formatLogMessage('debug', `Request processing halted for ${reqUrl} - awaiting nettools validation`));
+             }
            } else {
              // If searchstring or searchstring_and IS defined (with or without nettools), queue for content checking
              // Skip searchstring check if full subdomain was already detected
@@ -2237,6 +2293,12 @@ function setupFrameHandling(page, forceDebug) {
                  needsSearchStringCheck: true
                });
              }
+             // If we have BOTH searchstring AND nettools, ensure nettools validation still happens
+             if (hasNetTools) {
+               if (forceDebug) {
+                 console.log(formatLogMessage('debug', `${reqUrl} requires both content and nettools validation`));
+               }
+             }
            }
            
            // If curl is enabled, download and analyze content immediately

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.73",
+  "version": "1.0.75",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {