@fanboynz/network-scanner 1.0.83 → 1.0.85

package/lib/output.js

@@ -18,13 +18,35 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
   
   return ignorePatterns.some(pattern => {
     if (pattern.includes('*')) {
-      // Convert wildcard pattern to regex
-      const regexPattern = pattern
-        .replace(/\./g, '\\.')  // Escape dots
-        .replace(/\*/g, '.*');  // Convert * to .*
-      return new RegExp(`^${regexPattern}$`).test(domain);
+      // Enhanced wildcard pattern handling
+      if (pattern.startsWith('*.')) {
+        // Pattern: *.example.com
+        const wildcardDomain = pattern.substring(2); // Remove "*."
+        const wildcardRoot = extractDomainFromRule(`||${wildcardDomain}^`) || wildcardDomain;
+        const domainRoot = extractDomainFromRule(`||${domain}^`) || domain;
+        
+        // Use basic root domain comparison for output filtering
+        const getSimpleRoot = (d) => {
+          const parts = d.split('.');
+          return parts.length >= 2 ? parts.slice(-2).join('.') : d;
+        };
+        
+        return getSimpleRoot(domainRoot) === getSimpleRoot(wildcardRoot);
+      } else if (pattern.endsWith('.*')) {
+        // Pattern: example.*
+        const baseDomain = pattern.slice(0, -2); // Remove ".*"
+        return domain.startsWith(baseDomain + '.');
+      } else {
+        // Complex wildcard pattern
+        const regexPattern = pattern
+          .replace(/\./g, '\\.')  // Escape dots
+          .replace(/\*/g, '.*');  // Convert * to .*
+        return new RegExp(`^${regexPattern}$`).test(domain);
+      }
+    } else {
+      // Exact pattern matching
+      return domain === pattern || domain.endsWith('.' + pattern);
     }
-    return domain.endsWith(pattern);
   });
 }
 

package/lib/post-processing.js

@@ -316,6 +316,77 @@ function cleanupIgnoreDomains(results, ignoreDomains, options = {}) {
   return cleanedResults;
 }
 
+/**
+ * Enhanced domain extraction helper that reuses existing parsing logic
+ * @param {string} rule - Rule string in various formats
+ * @returns {string|null} Extracted domain or null if not found
+ */
+function extractDomainFromRule(rule) {
+  if (!rule || typeof rule !== 'string') {
+    return null;
+  }
+
+  try {
+    // Reuse the existing parsing logic from cleanupFirstPartyDomains
+    let extractedDomain = null;
+    
+    if (rule.startsWith('||') && rule.includes('^')) {
+      // ||domain.com^ format (adblock)
+      const match = rule.match(/^\|\|([^/\^]+)/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.startsWith('127.0.0.1 ') || rule.startsWith('0.0.0.0 ')) {
+      // hosts file format
+      const parts = rule.split(/\s+/);
+      if (parts.length >= 2) {
+        extractedDomain = parts[1];
+      }
+    } else if (rule.includes('local=/') && rule.includes('/')) {
+      // dnsmasq format: local=/domain.com/
+      const match = rule.match(/local=\/([^/]+)\//);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('server=/') && rule.includes('/')) {
+      // dnsmasq old format: server=/domain.com/
+      const match = rule.match(/server=\/([^/]+)\//);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('local-zone:') && rule.includes('always_null')) {
+      // unbound format: local-zone: "domain.com." always_null
+      const match = rule.match(/local-zone:\s*"([^"]+)\.?"/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.includes('+block') && rule.includes('.')) {
+      // privoxy format: { +block } .domain.com
+      const match = rule.match(/\{\s*\+block\s*\}\s*\.?([^\s]+)/);
+      if (match) {
+        extractedDomain = match[1];
+      }
+    } else if (rule.match(/^\(\^\|\\\.\).*\\\.\w+\$$/)) {
+      // pi-hole regex format: (^|\.)domain\.com$
+      const match = rule.match(/^\(\^\|\\\.\)(.+)\\\.\w+\$$/);
+      if (match) {
+        // Unescape the domain
+        extractedDomain = match[1].replace(/\\\./g, '.');
+      }
+    } else {
+      // Try to extract any domain-like pattern as fallback
+      const domainMatch = rule.match(/([a-zA-Z0-9][a-zA-Z0-9.-]*\.[a-zA-Z]{2,})/);
+      if (domainMatch) {
+        extractedDomain = domainMatch[1];
+      }
+    }
+    
+    return extractedDomain;
+  } catch (parseErr) {
+    return null;
+  }
+}
+
 /**
  * Post-scan cleanup function to remove first-party domains from results
  * Only processes sites that have firstParty: false in their configuration
@@ -564,6 +635,111 @@ function validateScanResults(results, options = {}) {
   return validatedResults;
 }
 
+
+/**
+ * Final validation check for firstParty: false violations
+ * Reuses existing domain extraction and matching logic
+ * 
+ * @param {Array} results - Array of scan results
+ * @param {Array} sites - Array of site configurations  
+ * @param {Object} options - Options object
+ * @returns {Array} Results with any remaining first-party domains removed
+ */
+function finalFirstPartyValidation(results, sites, options = {}) {
+  const { forceDebug = false, silentMode = false } = options;
+  
+  if (!results || results.length === 0) {
+    return results;
+  }
+
+  // Reuse the URL-to-config mapping pattern from cleanupFirstPartyDomains
+  const urlToSiteConfig = new Map();
+  sites.forEach(site => {
+    const urls = Array.isArray(site.url) ? site.url : [site.url];
+    urls.forEach(url => {
+      urlToSiteConfig.set(url, site);
+    });
+  });
+
+  const finalResults = [];
+  let totalViolationsFound = 0;
+  let sitesWithViolations = 0;
+
+  results.forEach(result => {
+    const siteConfig = urlToSiteConfig.get(result.url);
+    
+    // Only validate sites with firstParty: false
+    const shouldValidate = siteConfig && siteConfig.firstParty === false;
+    
+    if (!shouldValidate || !result.rules || result.rules.length === 0) {
+      finalResults.push(result);
+      return;
+    }
+
+    const scannedDomain = safeGetDomain(result.url, false);
+    if (!scannedDomain) {
+      finalResults.push(result);
+      return;
+    }
+
+    // Reuse the same filtering logic pattern from cleanupFirstPartyDomains
+    const cleanedRules = [];
+    const violatingRules = [];
+
+    result.rules.forEach(rule => {
+      const extractedDomain = extractDomainFromRule(rule);
+      if (extractedDomain) {
+        // Reuse the shouldRemoveAsFirstParty logic
+        const matchResult = shouldRemoveAsFirstParty(extractedDomain, scannedDomain, forceDebug);
+        
+        if (matchResult.shouldRemove) {
+          violatingRules.push({
+            rule: rule,
+            domain: extractedDomain,
+            reason: `VALIDATION FAILURE: ${matchResult.reason}`
+          });
+          totalViolationsFound++;
+          return;
+        }
+      }
+      cleanedRules.push(rule);
+    });
+
+    if (violatingRules.length > 0) {
+      sitesWithViolations++;
+      
+      if (!silentMode) {
+        const errorMessage = `? CONFIG VIOLATION: Found ${violatingRules.length} first-party rule(s) in ${scannedDomain} (firstParty: false)`;
+        if (messageColors && messageColors.error) {
+          console.log(messageColors.error(errorMessage));
+        } else {
+          console.log(errorMessage);
+        }
+      }
+
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[final-validation] Violations found for ${result.url}:`));
+        violatingRules.forEach((violation, idx) => {
+          console.log(formatLogMessage('debug', `  [${idx + 1}] ${violation.rule} -> ${violation.domain}`));
+        });
+      }
+    }
+
+    finalResults.push({ ...result, rules: cleanedRules });
+  });
+
+  // Summary using existing message patterns
+  if (totalViolationsFound > 0 && !silentMode) {
+    const summaryMessage = `\n? SCAN FILTERING FAILURE: Removed ${totalViolationsFound} first-party rules from ${sitesWithViolations} site(s) in post-processing`;
+    console.log(summaryMessage);
+    console.log('??  This indicates firstParty: false filtering failed during scan - consider investigating root cause.');
+  } else if (forceDebug) {
+    console.log(formatLogMessage('debug', '[final-validation] No first-party violations found - filtering working correctly'));
+  }
+
+  return finalResults;
+}
+
 /**
  * Main post-processing function that runs all cleanup and validation steps
  * 
@@ -587,8 +763,11 @@ function processResults(results, sites, options = {}) {
   
   // Step 2: Clean up ignoreDomains (final safety net)
   processedResults = cleanupIgnoreDomains(processedResults, options.ignoreDomains || [], options);
+  
+  // Step 3: Final validation for firstParty: false configurations
+  processedResults = finalFirstPartyValidation(processedResults, sites, options);
 
-  // Step 3: Validate results
+  // Step 4: Validate results
   processedResults = validateScanResults(processedResults, options);
   
   if (forceDebug) {
@@ -602,6 +781,8 @@ function processResults(results, sites, options = {}) {
 module.exports = {
   cleanupFirstPartyDomains,
   cleanupIgnoreDomains,
+  finalFirstPartyValidation,
+  extractDomainFromRule,
   validateScanResults,
   processResults
 };

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.83 ===
+// === Network scanner script (nwss.js) v1.0.85 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -123,7 +123,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.83'; // Script version
+const VERSION = '1.0.85'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1435,6 +1435,9 @@ function setupFrameHandling(page, forceDebug) {
     const originalRootDomain = safeGetDomain(currentUrl, false);
     if (originalRootDomain) {
       firstPartyDomains.add(originalRootDomain);
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Initial first-party domain: ${originalRootDomain} for ${currentUrl}`));
+      }
     } 
 
     // Track redirect domains to exclude from matching

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.83",
+  "version": "1.0.85",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {

package/regex-samples.md

@@ -1,10 +1,130 @@
-Just examples of usage, always review the output before using publicly
+# EasyPrivacy Generic Strings - JSON Regex Conversion
 
-Validate using;
+Tracking and analytics patterns converted to JSON-safe regex. Always validate before using.
+
+Test patterns at:
 * https://regex101.com/
 * https://regexr.com/
 
-| Domain                   | JSON Regex |
+## Event & Impression Tracking
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `&EventType=DataDealImpression&` | `&EventType=DataDealImpression&` |
+| `?logType=impression&` | `\\?logType=impression&` |
+| `?groupType=engagement&eventType=CLICK&` | `\\?groupType=engagement&eventType=CLICK&` |
+| `?type=page&event=` | `\\?type=page&event=` |
+| `?event=performancelogger:` | `\\?event=performancelogger:` |
+| `*view*pixel&` | `.*view.*pixel&` |
+
+## Script Files & Resources
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `-adobeDatalayer_bridge.js` | `-adobeDatalayer_bridge\\.js` |
+| `_chartbeat.js` | `_chartbeat\\.js` |
+| `/owa.tracker-combined-min.js` | `\\/owa\\.tracker-combined-min\\.js` |
+| `.v4.analytics.` | `\\.v4\\.analytics\\.` |
+| `/vli-platform/adb-analytics@` | `\\/vli-platform\\/adb-analytics@` |
+
+## Tracking Endpoints
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `.com/track?v=` | `\\.com\\/track\\?v=` |
+| `/track?cb=` | `\\/track\\?cb=` |
+| `/track.gif?data=` | `\\/track\\.gif\\?data=` |
+| `/track_framework_metrics?` | `\\/track_framework_metrics\\?` |
+| `/track/pageview?` | `\\/track\\/pageview\\?` |
+| `-click-tracker.` | `-click-tracker\\.` |
+
+## Service & Tracking IDs
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `.svc/?tracking_id=` | `\\.svc\\/\\?tracking_id=` |
+| `/get_site_data?requestUUID=` | `\\/get_site_data\\?requestUUID=` |
+| `/ns.html?id=GTM-` | `\\/ns\\.html\\?id=GTM-` |
+| `/tag/proxy?id=G-` | `\\/tag\\/proxy\\?id=G-` |
+| `/sk-park.php?pid=` | `\\/sk-park\\.php\\?pid=` |
+
+## PHP Action Parameters
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `.php?action=browse&` | `\\.php\\?action=browse&` |
+| `.php?action_name=` | `\\.php\\?action_name=` |
+| `_stat.php?referer=` | `_stat\\.php\\?referer=` |
+| `/pagelogger/connector.php?` | `\\/pagelogger\\/connector\\.php\\?` |
+
+## Analytics & Performance
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `/?essb_counter_` | `\\/\\?essb_counter_` |
+| `/_i?referral_url=` | `\\/_i\\?referral_url=` |
+| `/?livehit=` | `\\/\\?livehit=` |
+| `/__ssobj/rum?` | `\\/__ssobj\\/rum\\?` |
+| `/analytics/?event=` | `\\/analytics\\/\\?event=` |
+| `/analytics/visit.php` | `\\/analytics\\/visit\\.php` |
+| `/wpstatistics/v1/hit?` | `\\/wpstatistics\\/v1\\/hit\\?` |
+
+## Logging & Experiments
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `/?log=experiment&` | `\\/\\?log=experiment&` |
+| `/?log=performance-` | `\\/\\?log=performance-` |
+| `?log=stats-` | `\\?log=stats-` |
+| `?log=xhl-widgets-events&` | `\\?log=xhl-widgets-events&` |
+| `/_/_/logClientError/` | `\\/_\\/_\\/logClientError\\/` |
+| `/hits/logger?` | `\\/hits\\/logger\\?` |
+
+## Pixel & Image Tracking
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `/anonymous_user_guid.gif?` | `\\/anonymous_user_guid\\.gif\\?` |
+| `/0.gif?` | `\\/0\\.gif\\?` |
+| `_c.gif?c=` | `_c\\.gif\\?c=` |
+| `/urchin.html?` | `\\/urchin\\.html\\?` |
+
+## Counters & Views
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `/ViewCounter/` | `\\/ViewCounter\\/` |
+| `/prod/ping?` | `\\/prod\\/ping\\?` |
+
+## DataLayer & Google Analytics
+
+| Pattern | JSON Regex |
+|:--------|:-----------|
+| `?[AQB]&ndh=1&t=` | `\\?[AQB]&ndh=1&t=` |
+| `&l=dataLayer&cx=c` | `&l=dataLayer&cx=c` |
+
+## Common Variations & Wildcards
+
+| Pattern Type | Example | JSON Regex |
+|:-------------|:--------|:-----------|
+| **Any tracking script** | `*track*.js` | `.*track.*\\.js` |
+| **Any analytics path** | `/*/analytics/*` | `\\/.*\\/analytics\\/.*` |
+| **Any pixel tracking** | `*.gif?*` | `.*\\.gif\\?.*` |
+| **Any log parameter** | `?*log*=*` | `\\?.*log.*=.*` |
+| **Any event parameter** | `?*event*=*` | `\\?.*event.*=.*` |
+
+## Advanced Patterns
+
+| Use Case | Pattern | JSON Regex |
+|:---------|:--------|:-----------|
+| **Multiple parameters** | `?param1=value&param2=` | `\\?param1=value&param2=` |
+| **Optional parameters** | `/path?optional_param` | `\\/path\\?.*optional_param` |
+| **Subdomain tracking** | `tracking.*.com/` | `tracking\\..+\\.com\\/` |
+| **Version in path** | `/v1/track` or `/v2/track` | `\\/v[0-9]+\\/track` |
+| **Hash-based IDs** | `/track?id=abc123` | `\\/track\\?id=[a-zA-Z0-9]+` |
+
+
+| Pattern                  | JSON Regex |
 |:---------------------------|:------------|
 | `/api/test/`             | `\\/api\\/test\\/` |
 | `/rto.js`                | `\\/rto\\.js` |
@@ -18,10 +138,16 @@ Validate using;
 | `abcdefghjk.top/`        | `^https?:\\/\\/[a-z]{8,19}\\.top\\/$` |
 | `abcdefghjk.top/*`       | `^https?:\\/\\/[a-z]{8,19}\\.top\\/.*$` |
 | `abcdefghjk.top/com`     | `^https?:\\/\\/[a-z]{8,19}\\.(top\|com)\\/$` |
-| `abcdefghjk.top com/*`   | `^https?:\\/\\/[a-z]{8,19}\\.(top\|com)\\/.*$` |
 | `.net/bar/`              | `\\.net\\/bar\\/` |
 | `&test_me=`              | `&test_me=` |
 | `/new/` `/test/`         | `\\/(new\|test)\\/` |
-| `.com` or `.net`         | `\\.(com\|net)\\/` |       
-
+| `.com` or `.net`         | `\\.(com\|net)\\/` |    
 
+## Notes
+- Use `.*` for wildcard matching (matches any characters)
+- Escape dots: `.` ? `\\.`
+- Escape slashes: `/` ? `\\/` 
+- Escape question marks: `?` ? `\\?`
+- Use `[a-zA-Z0-9]` for alphanumeric characters
+- Use `+` for one or more characters
+- Use `*` after `.` for zero or more characters