@fanboynz/network-scanner 2.0.46 → 2.0.48

package/lib/adblock.js

@@ -62,6 +62,7 @@ function parseAdblockRules(filePath, options = {}) {
     domainMap: new Map(),          // ||domain.com^ - Exact domains for O(1) lookup
     domainRules: [],               // ||*.domain.com^ - Wildcard domains (fallback)
     thirdPartyRules: [],           // ||domain.com^$third-party
+    firstPartyRules: [],
     pathRules: [],                 // /ads/*
     scriptRules: [],               // .js$script
     regexRules: [],                // /regex/
@@ -73,6 +74,7 @@ function parseAdblockRules(filePath, options = {}) {
       domain: 0,
       domainMapEntries: 0,         // Exact domain matches in Map
       thirdParty: 0,
+      firstParty: 0,
       path: 0,
       script: 0,
       regex: 0,
@@ -137,6 +139,9 @@ function parseAdblockRules(filePath, options = {}) {
       if (parsedRule.isThirdParty) {
         rules.thirdPartyRules.push(parsedRule);
         rules.stats.thirdParty++;
+      } else if (parsedRule.isFirstParty) {
+        rules.firstPartyRules.push(parsedRule);
+        rules.stats.firstParty++;
       } else if (parsedRule.isDomain) {
         // Store exact domains in Map for O(1) lookup, wildcards in array
         if (parsedRule.domain && !parsedRule.domain.includes('*')) {
@@ -170,6 +175,7 @@ function parseAdblockRules(filePath, options = {}) {
     console.log(`    • Exact matches (Map): ${rules.stats.domainMapEntries}`);
     console.log(`    • Wildcard patterns (Array): ${rules.domainRules.length}`);
     console.log(`  - Third-party rules: ${rules.stats.thirdParty}`);
+    console.log(`  - First-party rules: ${rules.stats.firstParty}`);
     console.log(`  - Path rules: ${rules.stats.path}`);
     console.log(`  - Script rules: ${rules.stats.script}`);
     console.log(`  - Regex rules: ${rules.stats.regex}`);
@@ -193,7 +199,10 @@ function parseRule(rule, isWhitelist) {
     isWhitelist,
     isDomain: false,
     isThirdParty: false,
+    isFirstParty: false,
     isScript: false,
+    resourceTypes: null,       // Array of allowed resource types, null = all types
+    excludedResourceTypes: null, // Array of excluded resource types ($~script, $~image)
     isRegex: false,
     domainRestrictions: null,  // { include: ['site.com'], exclude: ['~site.com'] }
     pattern: '',
@@ -233,10 +242,47 @@ function parseRule(rule, isWhitelist) {
       parsed.isThirdParty = true;
     }
     
-    // Check for script option
-    if (parsed.options['script']) {
-      parsed.isScript = true;
+    // Check for first-party option ($first-party, $1p, $~third-party)
+    if (parsed.options['first-party'] || parsed.options['1p'] || parsed.options['~third-party']) {
+      parsed.isFirstParty = true;
     }
+
+    // Parse resource type options
+    const TYPE_MAP = {
+      'script': 'script',
+      'stylesheet': 'stylesheet',
+      'css': 'stylesheet',
+      'image': 'image',
+      'xmlhttprequest': 'xhr',
+      'xhr': 'xhr',
+      'font': 'font',
+      'media': 'media',
+      'websocket': 'websocket',
+      'subdocument': 'subdocument',
+      'document': 'document',
+      'ping': 'ping',
+      'other': 'other'
+    };
+    
+    const matchedTypes = Object.keys(parsed.options)
+      .filter(key => TYPE_MAP[key])
+      .map(key => TYPE_MAP[key]);
+  
+    const excludedTypes = Object.keys(parsed.options)
+      .filter(key => key.startsWith('~') && TYPE_MAP[key.substring(1)])
+      .map(key => TYPE_MAP[key.substring(1)]);
+
+    if (matchedTypes.length > 0) {
+      parsed.resourceTypes = matchedTypes;
+      if (parsed.options['script']) {
+        parsed.isScript = true;
+      }
+    }
+
+    if (excludedTypes.length > 0) {
+      parsed.excludedResourceTypes = excludedTypes;
+    }
+
     // Parse domain option: $domain=site1.com|site2.com|~excluded.com
     if (parsed.options['domain']) {
       const domainList = parsed.options['domain'];
@@ -261,15 +307,6 @@ function parseRule(rule, isWhitelist) {
         exclude: exclude.length > 0 ? exclude : null
       };
       
-      // For debugging
-      if (enableLogging && parsed.domainRestrictions) {
-        if (parsed.domainRestrictions.include) {
-          // console.log(`[Adblock] Rule includes domains: ${parsed.domainRestrictions.include.join(', ')}`);
-        }
-        if (parsed.domainRestrictions.exclude) {
-          // console.log(`[Adblock] Rule excludes domains: ${parsed.domainRestrictions.exclude.join(', ')}`);
-        }
-      }
   }
  }
 
@@ -284,7 +321,8 @@ function parseRule(rule, isWhitelist) {
   else if (pattern.startsWith('/') && pattern.endsWith('/')) {
     parsed.isRegex = true;
     const regexPattern = pattern.substring(1, pattern.length - 1);
-    parsed.matcher = new RegExp(regexPattern, 'i');
+    const regex = new RegExp(regexPattern, 'i');
+    parsed.matcher = (url) => regex.test(url);
   }
   // Path/wildcard rules: /ads/* or ad.js
   else {
@@ -344,6 +382,7 @@ function createMatcher(rules, options = {}) {
   const urlCache = new URLCache(1000);
   let cacheHits = 0;
   let cacheMisses = 0;
+  const hasPartyRules = rules.thirdPartyRules.length > 0 || rules.firstPartyRules.length > 0;
   
   return {
     rules,
@@ -378,22 +417,21 @@ function createMatcher(rules, options = {}) {
           cacheMisses++;
         }
         
-        // OPTIMIZATION #1: Only calculate third-party status if we have third-party rules to check
-        // Avoids expensive URL parsing (2x new URL() calls) when no third-party rules exist
-        const isThirdParty = (sourceUrl && rules.thirdPartyRules.length > 0) 
-          ? isThirdPartyRequest(url, sourceUrl) 
-          : false;
-        
-        // OPTIMIZATION #2: Calculate hostname parts once and reuse (avoid duplicate split operations)
+        // Calculate hostname parts once and reuse
         const hostnameParts = lowerHostname.split('.');
+
+        // Precompute parent domains once, reused for whitelist and block checks
+        const parentDomains = [];
+        const partsLen = hostnameParts.length;
+        for (let i = 1; i < partsLen; i++) {
+          parentDomains.push(hostnameParts.slice(i).join('.'));
+        }
     
-        // V8 OPT: Extract and cache source page domain for $domain option checking
+        // Extract and cache source page domain for $domain and third-party checks
         let sourceDomain = null;
-        let cachedSourceData = null;
 
         if (sourceUrl) {
-          // Check if sourceUrl is in cache (avoid duplicate URL parsing)
-          cachedSourceData = urlCache.get(sourceUrl);
+          const cachedSourceData = urlCache.get(sourceUrl);
           
           if (cachedSourceData) {
             sourceDomain = cachedSourceData.lowerHostname;
@@ -416,6 +454,11 @@ function createMatcher(rules, options = {}) {
           }
         }
 
+        // Calculate third-party status using already-parsed hostnames
+        const isThirdParty = (sourceDomain && hasPartyRules)
+          ? getBaseDomain(lowerHostname) !== getBaseDomain(sourceDomain)
+          : false;
+
         // === WHITELIST CHECK (exception rules take precedence) ===
         
         // Fast path: Check exact domain in Map (O(1))
@@ -430,10 +473,8 @@ function createMatcher(rules, options = {}) {
         }
         
         // Check parent domains for subdomain matches (e.g., sub.example.com -> example.com)
-        const partsLen = hostnameParts.length;  // V8: Cache array length
-        for (let i = 1; i < partsLen; i++) {
-          const parentDomain = hostnameParts.slice(i).join('.');
-          rule = rules.whitelistMap.get(parentDomain);  // V8: Single Map lookup
+        for (let i = 0; i < parentDomains.length; i++) {
+          rule = rules.whitelistMap.get(parentDomains[i]);
           if (rule) {
             if (enableLogging) {
               console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
@@ -470,9 +511,8 @@ function createMatcher(rules, options = {}) {
         }
         
         // Check parent domains for subdomain matches (e.g., ads.example.com -> example.com)
-        for (let i = 1; i < partsLen; i++) {  // V8: Reuse cached length
-          const parentDomain = hostnameParts.slice(i).join('.');
-          rule = rules.domainMap.get(parentDomain);  // V8: Single Map lookup
+        for (let i = 0; i < parentDomains.length; i++) {
+          rule = rules.domainMap.get(parentDomains[i]);
           if (rule) {
             if (enableLogging) {
               console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
@@ -513,6 +553,24 @@ function createMatcher(rules, options = {}) {
           }
         }
 
+        // Check first-party rules
+        if (!isThirdParty) {
+          const firstPartyLen = rules.firstPartyRules.length;
+          for (let i = 0; i < firstPartyLen; i++) {
+            const rule = rules.firstPartyRules[i];
+            if (matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain)) {
+              if (enableLogging) {
+                console.log(`[Adblock] Blocked first-party: ${url} (${rule.raw})`);
+              }
+              return {
+                blocked: true,
+                rule: rule.raw,
+                reason: 'first_party_rule'
+              };
+            }
+          }
+        }
+
         // Check script rules
         if (resourceType === 'script' || url.endsWith('.js')) {
           const scriptRulesLen = rules.scriptRules.length;  // V8: Cache length
@@ -675,6 +733,14 @@ function matchesDomainRestrictions(rule, sourceDomain) {
   return true;
 }
 
+// Module-level constant for resource type normalization (hot path)
+const RESOURCE_TYPE_ALIASES = {
+  'script': 'script', 'stylesheet': 'stylesheet', 'image': 'image',
+  'xhr': 'xhr', 'fetch': 'xhr', 'font': 'font', 'media': 'media',
+  'websocket': 'websocket', 'subdocument': 'subdocument',
+  'document': 'document', 'ping': 'ping', 'other': 'other'
+};
+
 /**
  * Check if rule matches the given URL
  * @param {Object} rule - Parsed rule object
@@ -685,6 +751,7 @@ function matchesDomainRestrictions(rule, sourceDomain) {
  * @param {string|null} sourceDomain - Source page domain (for $domain option)
  * @returns {boolean} True if rule matches
  */
+ 
 function matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDomain) {
   // Check domain restrictions first
   if (!matchesDomainRestrictions(rule, sourceDomain)) {
@@ -695,11 +762,34 @@ function matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDoma
     return false;
   }
 
-  // Check script option
-  if (rule.isScript && resourceType !== 'script' && !url.endsWith('.js')) {
+  // Check first-party option
+  if (rule.isFirstParty && isThirdParty) {
     return false;
   }
 
+  // Check resource type restrictions
+  if (rule.resourceTypes) {
+    if (!resourceType) {
+      // No resource type info available — allow match for safety
+    } else {
+      // Normalize Puppeteer resource types to match our type names
+      const normalizedType = RESOURCE_TYPE_ALIASES[resourceType] || resourceType;
+      if (!rule.resourceTypes.includes(normalizedType)) {
+        return false;
+      }
+    }
+  }
+
+  // Check negated resource type restrictions ($~script, $~image, etc.)
+  if (rule.excludedResourceTypes) {
+    if (resourceType) {
+      const normalizedType = RESOURCE_TYPE_ALIASES[resourceType] || resourceType;
+      if (rule.excludedResourceTypes.includes(normalizedType)) {
+        return false;
+      }
+    }
+  }
+
   // Apply matcher function
   if (rule.isDomain) {
     return rule.matcher(url, hostname);
@@ -708,27 +798,6 @@ function matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDoma
   }
 }
 
-/**
- * Determine if request is third-party
- * @param {string} requestUrl - URL being requested
- * @param {string} sourceUrl - URL of the page making the request
- * @returns {boolean} True if third-party request
- */
-function isThirdPartyRequest(requestUrl, sourceUrl) {
-  try {
-    const requestHostname = new URL(requestUrl).hostname;
-    const sourceHostname = new URL(sourceUrl).hostname;
-    
-    // Extract base domain (handle subdomains)
-    const requestDomain = getBaseDomain(requestHostname);
-    const sourceDomain = getBaseDomain(sourceHostname);
-    
-    return requestDomain !== sourceDomain;
-  } catch (err) {
-    return false;
-  }
-}
-
 /**
  * Extract base domain from hostname
  * @param {string} hostname - Full hostname
@@ -745,6 +814,5 @@ function getBaseDomain(hostname) {
 
 module.exports = {
   parseAdblockRules,
-  isThirdPartyRequest,
   getBaseDomain
 };

package/lib/grep.js

@@ -3,9 +3,6 @@
 
 const fs = require('fs');
 const { spawnSync } = require('child_process');
-const crypto = require('crypto');
-const path = require('path');
-const os = require('os');
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
 
 // === Constants ===
@@ -21,53 +18,9 @@ const GREP_DEFAULTS = {
   GREP_SUCCESS_STATUS: 0,
   GREP_NOT_FOUND_STATUS: 1,
   CURL_SUCCESS_STATUS: 0,
-  VERSION_LINE_INDEX: 0,
-  RANDOM_STRING_LENGTH: 9,
-  TEMP_DIR_PREFIX: 'grep_search_'
+  VERSION_LINE_INDEX: 0
 };
 
-/**
- * Creates a temporary directory and file with content for grep processing
- * Uses mkdtempSync to avoid race conditions from filename collisions
- * @param {string} content - The content to write to temp file
- * @param {string} prefix - Prefix for temp filename
- * @returns {object} Object containing tempDir and tempFile paths
- */
-function createTempFile(content, prefix = 'scanner_grep') {
-  const tempDir = os.tmpdir();
-  
-  // Create a unique temporary directory to avoid race conditions
-  const uniqueTempDir = fs.mkdtempSync(path.join(tempDir, GREP_DEFAULTS.TEMP_DIR_PREFIX));
-  
-  // Use cryptographically secure random ID for additional uniqueness
-  const uniqueId = crypto.randomBytes(8).toString('hex');
-  const tempFile = path.join(uniqueTempDir, `${prefix}_${uniqueId}.tmp`);
-  
-  try {
-    // Write atomically with error handling
-    fs.writeFileSync(tempFile, content, { 
-      encoding: 'utf8',
-      mode: 0o600 // Restrict permissions for security
-    });
-    
-    return { tempDir: uniqueTempDir, tempFile };
-  } catch (error) {
-    // Clean up temp directory on write failure
-    try {
-      if (fs.existsSync(tempFile)) {
-        fs.unlinkSync(tempFile);
-      }
-      if (fs.existsSync(uniqueTempDir)) {
-        fs.rmdirSync(uniqueTempDir);
-      }
-    } catch (cleanupErr) {
-      // Ignore cleanup errors, report original error
-    }
-
-    throw new Error(`Failed to create temp file: ${error.message}`);
-  }
-}
-
 /**
  * Searches content using grep with the provided patterns
  * @param {string} content - The content to search
@@ -86,14 +39,8 @@ async function grepContent(content, searchPatterns, options = {}) {
   if (!content || searchPatterns.length === 0) {
     return { found: false, matchedPattern: null, allMatches: [] };
   }
-
-  let tempFile = null;
-  
+ 
   try {
-    // Create temporary directory and file with content
-    const tempResult = createTempFile(content, 'grep_search');
-    tempDir = tempResult.tempDir;
-    tempFile = tempResult.tempFile;
     
     const allMatches = [];
     let firstMatch = null;
@@ -110,12 +57,12 @@ async function grepContent(content, searchPatterns, options = {}) {
       if (wholeWord) grepArgs.push('-w');
       if (!regex) grepArgs.push('-F'); // Fixed strings (literal)
       
-      // Add pattern and file
-      grepArgs.push(pattern, tempFile);
+      grepArgs.push(pattern);
       
       try {
         const result = spawnSync('grep', grepArgs, {
           encoding: 'utf8',
+          input: content,
           timeout: GREP_DEFAULTS.GREP_TIMEOUT,
           maxBuffer: GREP_DEFAULTS.MAX_BUFFER_SIZE
         });
@@ -146,22 +93,6 @@ async function grepContent(content, searchPatterns, options = {}) {
     
   } catch (error) {
     throw new Error(`Grep search failed: ${error.message}`);
-  } finally {
-    // Clean up temporary file and directory
-    if (tempFile) {
-      try {
-        fs.unlinkSync(tempFile);
-      } catch (cleanupErr) {
-        console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp file ${tempFile}: ${cleanupErr.message}`));
-      }
-    }
-    if (tempDir) {
-      try {
-        fs.rmdirSync(tempDir);
-      } catch (cleanupErr) {
-        console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp directory ${tempDir}: ${cleanupErr.message}`));
-      }
-    }
   }
 }
 
@@ -417,6 +348,5 @@ module.exports = {
   grepContent,
   downloadAndGrep,
   createGrepHandler,
-  validateGrepAvailability,
-  createTempFile
+  validateGrepAvailability
 };

package/nwss.js

@@ -1790,10 +1790,6 @@ function setupFrameHandling(page, forceDebug) {
         }
       });
       
-      page.on('response', (response) => {
-        // Response handler - removed incorrect error logging
-      });
-
       // Apply flowProxy timeouts if detection is enabled
       if (flowproxyDetection) {
         const flowproxyTimeouts = getFlowProxyTimeouts(siteConfig);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.46",
+  "version": "2.0.48",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {