@fanboynz/network-scanner 2.0.20 → 2.0.22

package/lib/dry-run.js

@@ -0,0 +1,515 @@
+// === Dry Run Module (dry-run.js) ===
+// Handles dry run mode functionality for network scanner
+
+const fs = require('fs');
+const { messageColors, formatLogMessage } = require('./colorize');
+
+// Constants for dry run collection keys
+const DRY_RUN_KEYS = {
+  MATCHES: 'dryRunMatches',
+  NET_TOOLS: 'dryRunNetTools',
+  SEARCH_STRING: 'dryRunSearchString'
+};
+
+/**
+ * Initialize dry run collections for a matched domains map
+ * @param {Map} matchedDomains - The matched domains map to initialize
+ * @throws {Error} If matchedDomains is not a Map instance
+ */
+function initializeDryRunCollections(matchedDomains) {
+  if (!(matchedDomains instanceof Map)) {
+    throw new Error('matchedDomains must be a Map instance for dry-run mode');
+  }
+  
+  matchedDomains.set(DRY_RUN_KEYS.MATCHES, []);
+  matchedDomains.set(DRY_RUN_KEYS.NET_TOOLS, []);
+  matchedDomains.set(DRY_RUN_KEYS.SEARCH_STRING, new Map());
+}
+
+/**
+ * Validates match data object structure
+ * @param {Object} matchData - Match data to validate
+ * @throws {Error} If matchData is invalid
+ */
+function validateMatchData(matchData) {
+  if (!matchData || typeof matchData !== 'object') {
+    throw new Error('Match data must be an object');
+  }
+  
+  const requiredFields = ['regex', 'domain', 'resourceType', 'fullUrl'];
+  for (const field of requiredFields) {
+    if (!(field in matchData)) {
+      throw new Error(`Match data missing required field: ${field}`);
+    }
+  }
+}
+
+/**
+ * Validates nettools data object structure
+ * @param {Object} netToolsData - NetTools data to validate
+ * @throws {Error} If netToolsData is invalid
+ */
+function validateNetToolsData(netToolsData) {
+  if (!netToolsData || typeof netToolsData !== 'object') {
+    throw new Error('NetTools data must be an object');
+  }
+  
+  const requiredFields = ['domain', 'tool', 'matchType', 'matchedTerm'];
+  for (const field of requiredFields) {
+    if (!(field in netToolsData)) {
+      throw new Error(`NetTools data missing required field: ${field}`);
+    }
+  }
+}
+
+/**
+ * Add a match to dry run collections
+ * @param {Map} matchedDomains - The matched domains map
+ * @param {Object} matchData - Match data object
+ * @throws {Error} If parameters are invalid
+ */
+function addDryRunMatch(matchedDomains, matchData) {
+  if (!(matchedDomains instanceof Map)) {
+    throw new Error('matchedDomains must be a Map instance');
+  }
+  
+  validateMatchData(matchData);
+  
+  if (!matchedDomains.has(DRY_RUN_KEYS.MATCHES)) {
+    throw new Error('Dry run collections not initialized. Call initializeDryRunCollections first.');
+  }
+  
+  matchedDomains.get(DRY_RUN_KEYS.MATCHES).push({
+    ...matchData,
+    timestamp: new Date().toISOString()
+  });
+}
+
+/**
+ * Add a nettools result to dry run collections
+ * @param {Map} matchedDomains - The matched domains map
+ * @param {Object} netToolsData - NetTools result data
+ * @throws {Error} If parameters are invalid
+ */
+function addDryRunNetTools(matchedDomains, netToolsData) {
+  if (!(matchedDomains instanceof Map)) {
+    throw new Error('matchedDomains must be a Map instance');
+  }
+  
+  validateNetToolsData(netToolsData);
+  
+  if (!matchedDomains.has(DRY_RUN_KEYS.NET_TOOLS)) {
+    throw new Error('Dry run collections not initialized. Call initializeDryRunCollections first.');
+  }
+  
+  matchedDomains.get(DRY_RUN_KEYS.NET_TOOLS).push({
+    ...netToolsData,
+    timestamp: new Date().toISOString()
+  });
+}
+
+/**
+ * Add a search string result to dry run collections
+ * @param {Map} matchedDomains - The matched domains map
+ * @param {string} url - The URL that was searched
+ * @param {Object} searchResult - Search result data
+ * @throws {Error} If parameters are invalid
+ */
+function addDryRunSearchString(matchedDomains, url, searchResult) {
+  if (!(matchedDomains instanceof Map)) {
+    throw new Error('matchedDomains must be a Map instance');
+  }
+  
+  if (!url || typeof url !== 'string') {
+    throw new Error('URL must be a non-empty string');
+  }
+  
+  if (!searchResult || typeof searchResult !== 'object') {
+    throw new Error('Search result must be an object');
+  }
+  
+  if (!matchedDomains.has(DRY_RUN_KEYS.SEARCH_STRING)) {
+    throw new Error('Dry run collections not initialized. Call initializeDryRunCollections first.');
+  }
+  
+  matchedDomains.get(DRY_RUN_KEYS.SEARCH_STRING).set(url, {
+    ...searchResult,
+    timestamp: new Date().toISOString()
+  });
+}
+
+/**
+ * Safely truncate long URLs for display
+ * @param {string} url - URL to truncate
+ * @param {number} maxLength - Maximum length to display
+ * @returns {string} Truncated URL with ellipsis if needed
+ */
+function truncateUrl(url, maxLength = 80) {
+  if (!url || url.length <= maxLength) {
+    return url;
+  }
+  return url.substring(0, maxLength - 3) + '...';
+}
+
+/**
+ * Format search string match information
+ * @param {Object} searchStringMatch - Search string match data
+ * @returns {string} Formatted match description
+ */
+function formatSearchStringMatch(searchStringMatch) {
+  if (!searchStringMatch) return null;
+  
+  const matchType = searchStringMatch.type || 'unknown';
+  const term = searchStringMatch.term || 'unknown';
+  return `${matchType} - "${term}"`;
+}
+
+/**
+ * Generate adblock rule from domain and resource type
+ * @param {string} domain - Domain name
+ * @param {string} resourceType - Resource type (optional)
+ * @returns {string} Formatted adblock rule
+ */
+function generateAdblockRule(domain, resourceType = null) {
+  if (!domain) return '';
+  
+  if (resourceType && resourceType !== 'other') {
+    return `||${domain}^${resourceType}`;
+  }
+  return `||${domain}^`;
+}
+
+/**
+ * Outputs dry run results to console with formatted display
+ * If outputFile is specified, also captures output for file writing
+ * @param {string} url - The URL being processed  
+ * @param {Array} matchedItems - Array of matched items with regex, domain, and resource type
+ * @param {Array} netToolsResults - Array of whois/dig results
+ * @param {string} pageTitle - Title of the page (if available)
+ * @param {string} outputFile - Output file path (optional)
+ * @param {Array} dryRunOutput - Array to collect output lines for file writing
+ */
+function outputDryRunResults(url, matchedItems = [], netToolsResults = [], pageTitle = '', outputFile = null, dryRunOutput = []) {
+  try {
+    const lines = [];
+    const truncatedUrl = truncateUrl(url);
+    
+    lines.push(`\n=== DRY RUN RESULTS === ${truncatedUrl}`);
+    console.log(`\n${messageColors.scanning('=== DRY RUN RESULTS ===')} ${truncatedUrl}`);
+    
+    if (pageTitle && pageTitle.trim()) {
+      const cleanTitle = pageTitle.trim().substring(0, 200); // Limit title length
+      lines.push(`Title: ${cleanTitle}`);
+      console.log(`${messageColors.info('Title:')} ${cleanTitle}`);
+    }
+    
+    const totalMatches = matchedItems.length + netToolsResults.length;
+    
+    if (totalMatches === 0) {
+      const noMatchMsg = `No matching rules found on ${truncatedUrl}`;
+      lines.push(noMatchMsg);
+      
+      if (outputFile) {
+        dryRunOutput.push(...lines);
+        dryRunOutput.push(''); // Add empty line
+      }
+      console.log(messageColors.warn(noMatchMsg));
+      return;
+    }
+    
+    lines.push(`Matches found: ${totalMatches}`);
+    console.log(`${messageColors.success('Matches found:')} ${totalMatches}`);
+    
+    // Process regex matches
+    matchedItems.forEach((item, index) => {
+      try {
+        lines.push('');
+        lines.push(`[${index + 1}] Regex Match:`);
+        lines.push(`  Pattern: ${item.regex || 'unknown'}`);
+        lines.push(`  Domain: ${item.domain || 'unknown'}`);
+        lines.push(`  Resource Type: ${item.resourceType || 'unknown'}`);
+        lines.push(`  Full URL: ${truncateUrl(item.fullUrl || '')}`);
+
+        console.log(`\n${messageColors.highlight(`[${index + 1}]`)} ${messageColors.match('Regex Match:')}`);
+        console.log(`  Pattern: ${item.regex || 'unknown'}`);
+        console.log(`  Domain: ${item.domain || 'unknown'}`);
+        console.log(`  Resource Type: ${item.resourceType || 'unknown'}`);
+        console.log(`  Full URL: ${truncateUrl(item.fullUrl || '')}`);
+        
+        // Show blocked status if applicable
+        if (item.wasBlocked) {
+          lines.push(`  Status: BLOCKED (even_blocked enabled)`);
+          console.log(`  ${messageColors.warn('Status:')} BLOCKED (even_blocked enabled)`);
+        }
+        
+        // Show searchstring results if available
+        if (item.searchStringMatch) {
+          const matchDesc = formatSearchStringMatch(item.searchStringMatch);
+          lines.push(`  ? Searchstring Match: ${matchDesc}`);
+          console.log(`  ${messageColors.success('? Searchstring Match:')} ${matchDesc}`);
+        } else if (item.searchStringChecked) {
+          lines.push(`  ? Searchstring: No matches found in content`);
+          console.log(`  ${messageColors.warn('? Searchstring:')} No matches found in content`);
+        }
+        
+        // Generate adblock rule
+        const adblockRule = generateAdblockRule(item.domain, item.resourceType);
+        lines.push(`  Adblock Rule: ${adblockRule}`);
+        console.log(`  ${messageColors.info('Adblock Rule:')} ${adblockRule}`);
+        
+      } catch (itemErr) {
+        const errorMsg = `Error processing match item ${index + 1}: ${itemErr.message}`;
+        lines.push(`  Error: ${errorMsg}`);
+        console.log(`  ${messageColors.warn('Error:')} ${errorMsg}`);
+      }
+    });
+    
+    // Process nettools results  
+    netToolsResults.forEach((result, index) => {
+      try {
+        const resultIndex = matchedItems.length + index + 1;
+        lines.push('');
+        lines.push(`[${resultIndex}] NetTools Match:`);
+        lines.push(`  Domain: ${result.domain || 'unknown'}`);
+        lines.push(`  Tool: ${(result.tool || 'unknown').toUpperCase()}`);
+        
+        const matchDesc = `${result.matchType || 'unknown'} - "${result.matchedTerm || 'unknown'}"`;
+        lines.push(`  ? Match: ${matchDesc}`);
+        
+        if (result.details) {
+          lines.push(`  Details: ${result.details}`);
+        }
+        
+        console.log(`\n${messageColors.highlight(`[${resultIndex}]`)} ${messageColors.match('NetTools Match:')}`);
+        console.log(`  Domain: ${result.domain || 'unknown'}`);
+        console.log(`  Tool: ${(result.tool || 'unknown').toUpperCase()}`);
+        console.log(`  ${messageColors.success('? Match:')} ${matchDesc}`);
+        
+        if (result.details) {
+          console.log(`  Details: ${result.details}`);
+        }
+        
+        // Generate adblock rule for nettools matches
+        const adblockRule = generateAdblockRule(result.domain);
+        lines.push(`  Adblock Rule: ${adblockRule}`);
+        console.log(`  ${messageColors.info('Adblock Rule:')} ${adblockRule}`);
+        
+      } catch (resultErr) {
+        const errorMsg = `Error processing nettools result ${index + 1}: ${resultErr.message}`;
+        lines.push(`  Error: ${errorMsg}`);
+        console.log(`  ${messageColors.warn('Error:')} ${errorMsg}`);
+      }
+    });
+
+    // Store output for file writing if outputFile is specified
+    if (outputFile) {
+      dryRunOutput.push(...lines);
+      dryRunOutput.push(''); // Add empty line between sites
+    }
+    
+  } catch (outputErr) {
+    const errorMsg = `Error in outputDryRunResults: ${outputErr.message}`;
+    console.error(messageColors.error(errorMsg));
+    if (outputFile) {
+      dryRunOutput.push(`Error: ${errorMsg}`);
+    }
+  }
+}
+
+/**
+ * Process dry run results for a URL and output them
+ * @param {string} currentUrl - The URL being processed
+ * @param {Map} matchedDomains - The matched domains map with dry run collections
+ * @param {Object} page - Puppeteer page object for getting title
+ * @param {string} outputFile - Output file path (optional)
+ * @param {Array} dryRunOutput - Array to collect output lines for file writing
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Object} Dry run result summary
+ */
+async function processDryRunResults(currentUrl, matchedDomains, page, outputFile = null, dryRunOutput = [], forceDebug = false) {
+  try {
+    // Validate inputs
+    if (!currentUrl || typeof currentUrl !== 'string') {
+      throw new Error('currentUrl must be a non-empty string');
+    }
+    
+    if (!(matchedDomains instanceof Map)) {
+      throw new Error('matchedDomains must be a Map instance');
+    }
+    
+    // Get page title for dry run output with error handling
+    let pageTitle = '';
+    try {
+      if (page && typeof page.title === 'function') {
+        pageTitle = await page.title();
+      }
+    } catch (titleErr) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `Failed to get page title for ${currentUrl}: ${titleErr.message}`));
+      }
+      pageTitle = 'Title unavailable';
+    }
+    
+    // Get collected matches with safe fallbacks
+    const dryRunMatches = matchedDomains.get(DRY_RUN_KEYS.MATCHES) || [];
+    const dryRunNetTools = matchedDomains.get(DRY_RUN_KEYS.NET_TOOLS) || [];
+    const dryRunSearchString = matchedDomains.get(DRY_RUN_KEYS.SEARCH_STRING) || new Map();
+    
+    // Enhance matches with searchstring results
+    const enhancedMatches = dryRunMatches.map((match, index) => {
+      try {
+        const searchResult = dryRunSearchString.get(match.fullUrl);
+        return {
+          ...match,
+          searchStringMatch: searchResult && searchResult.matched ? searchResult : null,
+          searchStringChecked: Boolean(match.needsSearchStringCheck)
+        };
+      } catch (enhanceErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `Error enhancing match ${index}: ${enhanceErr.message}`));
+        }
+        return {
+          ...match,
+          searchStringMatch: null,
+          searchStringChecked: false
+        };
+      }
+    });
+    
+    outputDryRunResults(currentUrl, enhancedMatches, dryRunNetTools, pageTitle, outputFile, dryRunOutput);
+    
+    const totalMatches = enhancedMatches.length + dryRunNetTools.length;
+    
+    return {
+      success: true,
+      matchCount: totalMatches,
+      enhancedMatches,
+      netToolsResults: dryRunNetTools,
+      pageTitle,
+      regexMatches: enhancedMatches.length,
+      netToolsMatches: dryRunNetTools.length
+    };
+    
+  } catch (processErr) {
+    const errorMsg = `Error processing dry run results for ${currentUrl}: ${processErr.message}`;
+    console.error(messageColors.error(errorMsg));
+    
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `Stack trace: ${processErr.stack}`));
+    }
+    
+    return {
+      success: false,
+      error: errorMsg,
+      matchCount: 0,
+      enhancedMatches: [],
+      netToolsResults: [],
+      pageTitle: '',
+      regexMatches: 0,
+      netToolsMatches: 0
+    };
+  }
+}
+
+/**
+ * Write dry run output to file with enhanced error handling
+ * @param {string} outputFile - Output file path
+ * @param {Array} dryRunOutput - Array of output lines
+ * @param {boolean} silentMode - Silent mode flag
+ * @returns {Object} Operation result with details
+ */
+function writeDryRunOutput(outputFile, dryRunOutput, silentMode = false) {
+  try {
+    if (!outputFile || typeof outputFile !== 'string') {
+      return { success: false, error: 'Invalid output file path' };
+    }
+    
+    if (!Array.isArray(dryRunOutput) || dryRunOutput.length === 0) {
+      if (!silentMode) {
+        console.log(messageColors.info('No dry run output to write'));
+      }
+      return { success: true, written: false, reason: 'No output to write' };
+    }
+    
+    const dryRunContent = dryRunOutput.join('\n');
+    
+    // Ensure output directory exists
+    const path = require('path');
+    const outputDir = path.dirname(outputFile);
+    if (outputDir !== '.' && !fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir, { recursive: true });
+    }
+    
+    fs.writeFileSync(outputFile, dryRunContent);
+    
+    if (!silentMode) {
+      console.log(`${messageColors.fileOp('?? Dry run results saved to:')} ${outputFile}`);
+    }
+    
+    return { 
+      success: true, 
+      written: true, 
+      file: outputFile, 
+      lines: dryRunOutput.length,
+      bytes: Buffer.byteLength(dryRunContent, 'utf8')
+    };
+    
+  } catch (writeErr) {
+    const errorMsg = `Failed to write dry run output to ${outputFile}: ${writeErr.message}`;
+    console.error(`? ${errorMsg}`);
+    
+    return { 
+      success: false, 
+      error: errorMsg,
+      written: false
+    };
+  }
+}
+
+/**
+ * Get statistics from dry run collections
+ * @param {Map} matchedDomains - The matched domains map
+ * @returns {Object} Statistics object
+ */
+function getDryRunStats(matchedDomains) {
+  if (!(matchedDomains instanceof Map)) {
+    return { error: 'Invalid matchedDomains Map' };
+  }
+  
+  const matches = matchedDomains.get(DRY_RUN_KEYS.MATCHES) || [];
+  const netTools = matchedDomains.get(DRY_RUN_KEYS.NET_TOOLS) || [];
+  const searchStrings = matchedDomains.get(DRY_RUN_KEYS.SEARCH_STRING) || new Map();
+  
+  return {
+    totalMatches: matches.length + netTools.length,
+    regexMatches: matches.length,
+    netToolsMatches: netTools.length,
+    searchStringResults: searchStrings.size,
+    domains: new Set([
+      ...matches.map(m => m.domain).filter(Boolean),
+      ...netTools.map(n => n.domain).filter(Boolean)
+    ]).size
+  };
+}
+
+module.exports = {
+  // Constants
+  DRY_RUN_KEYS,
+  
+  // Core functions
+  initializeDryRunCollections,
+  addDryRunMatch,
+  addDryRunNetTools,
+  addDryRunSearchString,
+  processDryRunResults,
+  writeDryRunOutput,
+  
+  // Utility functions
+  getDryRunStats,
+  validateMatchData,
+  validateNetToolsData,
+  truncateUrl,
+  formatSearchStringMatch,
+  generateAdblockRule,
+  outputDryRunResults
+};

package/nwss.1

@@ -303,7 +303,7 @@ Number of times to reload the page (default: 1).
 
 .TP
 .B forcereload
-Boolean. Force an additional reload after reloads.
+Boolean or Array. Force cache-clearing reload for all URLs or specific domains. Can be \fBtrue\fR/\fBfalse\fR or array of domain names like \fB["domain1.com", "domain2.com"]\fR. When set to \fBtrue\fR, applies to all URLs in the site configuration. When set to an array, applies only to URLs whose domains match the specified domains. Supports exact hostname matching and subdomain matching (e.g., "example.com" matches both "example.com" and "subdomain.example.com"). Domain matching is case-insensitive and automatically handles protocols, ports, and paths.
 
 .TP
 .B timeout
@@ -784,6 +784,37 @@ With default settings (\fBignore_similar_threshold: 80\fR):
 }
 .EE
 
+.SS Force reload examples:
+.EX
+# Force reload for all URLs in a site configuration
+{
+  "url": ["https://site1.com", "https://site2.com"],
+  "filterRegex": "ads|tracking",
+  "forcereload": true
+}
+
+# Force reload only for specific domains
+{
+  "url": [
+    "https://example.com/page1",
+    "https://test.org/page2", 
+    "https://demo.net/page3"
+  ],
+  "filterRegex": "\\\\.(space|website)\\\\b",
+  "forcereload": ["example.com", "demo.net"],
+  "comments": [
+    "Only example.com and demo.net URLs get force reload",
+    "test.org URLs use standard reload"
+  ]
+}
+
+# Mixed configuration with domain-specific force reload
+{
+  "url": "https://cdn.example.com/content/page.html",
+  "forcereload": ["example.com"]
+}
+.EE
+
 .SS Configuration with documentation comments:
 .EX
 {

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v2.0.20 ===
+// === Network scanner script (nwss.js) v2.0.22 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -44,6 +44,8 @@ const { performPageInteraction, createInteractionConfig } = require('./lib/inter
 const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
 const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
 const { clearPersistentCache } = require('./lib/smart-cache');
+// Dry run functionality
+const { initializeDryRunCollections, addDryRunMatch, addDryRunNetTools, processDryRunResults, writeDryRunOutput } = require('./lib/dry-run');
 // Enhanced site data clearing functionality
 const { clearSiteData } = require('./lib/clear_sitedata');
 
@@ -130,7 +132,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '2.0.20'; // Script version
+const VERSION = '2.0.22'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -516,7 +518,7 @@ Redirect Handling Options:
   interact_intensity: "low"|"medium"|"high"     Interaction simulation intensity (default: medium)
   delay: <milliseconds>                        Delay after load (default: 4000)
   reload: <number>                             Reload page n times after load (default: 1)
-  forcereload: true/false                      Force an additional reload after reloads
+  forcereload: true/false or ["domain1.com", "domain2.com"]  Force cache-clearing reload for all URLs or specific domains
   clear_sitedata: true/false                   Clear all cookies, cache, storage before each load (default: false)
   subDomains: 1/0                              Output full subdomains (default: 0)
   localhost: true/false                        Force localhost output (127.0.0.1)
@@ -953,103 +955,6 @@ function shouldEnableCDPForUrl(url, cdpSpecificList) {
   }
 }
 
-/**
- * Outputs dry run results to console with formatted display
- * If outputFile is specified, also captures output for file writing
- * @param {string} url - The URL being processed  
- * @param {Array} matchedItems - Array of matched items with regex, domain, and resource type
- * @param {Array} netToolsResults - Array of whois/dig results
- * @param {string} pageTitle - Title of the page (if available)
- */
-function outputDryRunResults(url, matchedItems, netToolsResults, pageTitle) {
-  const lines = [];
-  
-  lines.push(`\n=== DRY RUN RESULTS === ${url}`);
-
-  console.log(`\n${messageColors.scanning('=== DRY RUN RESULTS ===')} ${url}`);
-  
-  if (pageTitle && pageTitle.trim()) {
-    lines.push(`Title: ${pageTitle.trim()}`);
-    console.log(`${messageColors.info('Title:')} ${pageTitle.trim()}`);
-  }
-  
-  if (matchedItems.length === 0 && netToolsResults.length === 0) {
-    lines.push(`No matching rules found on ${url}`);
-    
-    // Store output for file writing if outputFile is specified
-    if (outputFile) {
-      dryRunOutput.push(...lines);
-      dryRunOutput.push(''); // Add empty line
-    }
-    console.log(messageColors.warn(`No matching rules found on ${url}`));
-    return;
-  }
-  
-  const totalMatches = matchedItems.length + netToolsResults.length;
-  lines.push(`Matches found: ${totalMatches}`);
-  console.log(`${messageColors.success('Matches found:')} ${totalMatches}`);
-  
-  matchedItems.forEach((item, index) => {
-    lines.push('');
-    lines.push(`[${index + 1}] Regex Match:`);
-    lines.push(`  Pattern: ${item.regex}`);
-    lines.push(`  Domain: ${item.domain}`);
-    lines.push(`  Resource Type: ${item.resourceType}`);
-    lines.push(`  Full URL: ${item.fullUrl}`);
-
-    console.log(`\n${messageColors.highlight(`[${index + 1}]`)} ${messageColors.match('Regex Match:')}`);
-    console.log(`  Pattern: ${item.regex}`);
-    console.log(`  Domain: ${item.domain}`);
-    console.log(`  Resource Type: ${item.resourceType}`);
-    console.log(`  Full URL: ${item.fullUrl}`);
-    
-    // Show searchstring results if available
-    if (item.searchStringMatch) {
-      lines.push(`  ✓ Searchstring Match: ${item.searchStringMatch.type} - "${item.searchStringMatch.term}"`);
-      console.log(`  ${messageColors.success('✓ Searchstring Match:')} ${item.searchStringMatch.type} - "${item.searchStringMatch.term}"`);
-    } else if (item.searchStringChecked) {
-      lines.push(`  ✗ Searchstring: No matches found in content`);
-      console.log(`  ${messageColors.warn('✗ Searchstring:')} No matches found in content`);
-    }
-    
-    // Generate adblock rule
-    const adblockRule = `||${item.domain}^$${item.resourceType}`;
-    lines.push(`  Adblock Rule: ${adblockRule}`);
-    console.log(`  ${messageColors.info('Adblock Rule:')} ${adblockRule}`);
-  });
-  
-  // Display nettools results
-  netToolsResults.forEach((result, index) => {
-    const resultIndex = matchedItems.length + index + 1;
-    lines.push('');
-    lines.push(`[${resultIndex}] NetTools Match:`);
-    lines.push(`  Domain: ${result.domain}`);
-    lines.push(`  Tool: ${result.tool.toUpperCase()}`);
-    lines.push(`  ✓ Match: ${result.matchType} - "${result.matchedTerm}"`);
-    if (result.details) {
-      lines.push(`  Details: ${result.details}`);
-    }
-    console.log(`\n${messageColors.highlight(`[${resultIndex}]`)} ${messageColors.match('NetTools Match:')}`);
-    console.log(`  Domain: ${result.domain}`);
-    console.log(`  Tool: ${result.tool.toUpperCase()}`);
-    console.log(`  ${messageColors.success('✓ Match:')} ${result.matchType} - "${result.matchedTerm}"`);
-    if (result.details) {
-      console.log(`  Details: ${result.details}`);
-    }
-    
-    // Generate adblock rule for nettools matches
-    const adblockRule = `||${result.domain}^`;
-    lines.push(`  Adblock Rule: ${adblockRule}`);
-    console.log(`  ${messageColors.info('Adblock Rule:')} ${adblockRule}`);
-  });
-
-  // Store output for file writing if outputFile is specified
-  if (outputFile) {
-    dryRunOutput.push(...lines);
-    dryRunOutput.push(''); // Add empty line between sites
-  }
-}
-
 /**
  * Helper function to check if a URL should be processed (valid HTTP/HTTPS)
  * @param {string} url - URL to validate
@@ -1543,9 +1448,7 @@ function setupFrameHandling(page, forceDebug) {
     
     // Initialize dry run matches collection
     if (dryRunMode) {
-      matchedDomains.set('dryRunMatches', []);
-      matchedDomains.set('dryRunNetTools', []);
-      matchedDomains.set('dryRunSearchString', new Map()); // Map URL to search results
+      initializeDryRunCollections(matchedDomains);
     }
     const timeout = siteConfig.timeout || TIMEOUTS.DEFAULT_PAGE;
 
@@ -2434,7 +2337,7 @@ function setupFrameHandling(page, forceDebug) {
                   const allowedResourceTypes = siteConfig.resourceTypes;
                   if (!allowedResourceTypes || !Array.isArray(allowedResourceTypes) || allowedResourceTypes.includes(resourceType)) {
                     if (dryRunMode) {
-                      matchedDomains.get('dryRunMatches').push({
+                      addDryRunMatch(matchedDomains, {
                         regex: matchedRegexPattern,
                         domain: reqDomain,
                         resourceType: resourceType,
@@ -2598,7 +2501,7 @@ function setupFrameHandling(page, forceDebug) {
            // If NO searchstring AND NO nettools are defined, match immediately (existing behavior)
            if (!hasSearchString && !hasSearchStringAnd && !hasNetTools) {
              if (dryRunMode) {
-               matchedDomains.get('dryRunMatches').push({
+               addDryRunMatch(matchedDomains, {
                  regex: matchedRegexPattern,
                  domain: reqDomain,
                  resourceType: resourceType,
@@ -2641,8 +2544,7 @@ function setupFrameHandling(page, forceDebug) {
              }
 
              if (dryRunMode) {
-               // For dry run, we'll collect the domain for nettools checking
-               matchedDomains.get('dryRunMatches').push({
+               addDryRunMatch(matchedDomains, {
                  regex: matchedRegexPattern,
                  domain: reqDomain,
                  resourceType: resourceType,
@@ -2722,7 +2624,7 @@ function setupFrameHandling(page, forceDebug) {
                console.log(formatLogMessage('debug', `${reqUrl} matched regex ${matchedRegexPattern} and resourceType ${resourceType}, queued for ${searchType} content search`));
              }
              if (dryRunMode) {
-               matchedDomains.get('dryRunMatches').push({
+               addDryRunMatch(matchedDomains, {
                  regex: matchedRegexPattern,
                  domain: reqDomain,
                  resourceType: resourceType,
@@ -3202,7 +3104,80 @@ function setupFrameHandling(page, forceDebug) {
       updatePageUsage(page, true);
 
       const totalReloads = (siteConfig.reload || 1) - 1; // Subtract 1 because initial load counts as first
-      const useForceReload = siteConfig.forcereload === true;
+      
+      // Enhanced forcereload logic: support boolean or domain array
+      let useForceReload = false;
+      if (siteConfig.forcereload === true) {
+        // Original behavior: force reload for all URLs
+        useForceReload = true;
+      } else if (Array.isArray(siteConfig.forcereload)) {
+        // Input validation: filter out invalid entries
+        const validDomains = siteConfig.forcereload.filter(domain => {
+          if (typeof domain !== 'string') {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Invalid forcereload entry (not string): ${typeof domain} - ${JSON.stringify(domain)}`));
+            }
+            return false;
+          }
+          
+          if (domain.trim() === '') {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Invalid forcereload entry (empty string)`));
+            }
+            return false;
+          }
+          
+          return true;
+        });
+        
+        if (validDomains.length === 0) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `No valid domains in forcereload array for ${currentUrl}`));
+          }
+          useForceReload = false;
+        } else {
+        // New behavior: force reload only for matching domains
+        const currentDomain = safeGetDomain(currentUrl, true); // Get full hostname
+        const currentRootDomain = safeGetDomain(currentUrl, false); // Get root domain
+        
+        useForceReload = validDomains.some(domain => {
+          // Enhanced domain cleaning: handle protocols, ports, paths, and normalize case
+          let cleanDomain = domain.trim();
+          cleanDomain = cleanDomain.replace(/^https?:\/\//, '');  // Remove protocol
+          cleanDomain = cleanDomain.replace(/:\d+$/, '');         // Remove port (e.g., :8080)
+          cleanDomain = cleanDomain.replace(/\/.*$/, '');         // Remove path
+          cleanDomain = cleanDomain.toLowerCase();               // Normalize case
+          
+          // Additional validation: basic domain format check
+          if (!/^[a-z0-9.-]+$/.test(cleanDomain)) {
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Skipping invalid domain format in forcereload: ${domain} -> ${cleanDomain}`));
+            }
+            return false;
+          }
+          
+          // Check if current URL matches this domain
+          // Support both exact hostname match and subdomain match
+          if (currentDomain.toLowerCase() === cleanDomain || currentRootDomain.toLowerCase() === cleanDomain) {
+            return true;
+          }
+          
+          // Check if current hostname ends with the domain (subdomain match)
+          if (currentDomain.toLowerCase().endsWith('.' + cleanDomain)) {
+            return true;
+          }
+          
+          return false;
+        });
+        }
+        
+        if (forceDebug && useForceReload) {
+          console.log(formatLogMessage('debug', `Force reload enabled for ${currentUrl} - matches domain in forcereload list`));
+        } else if (forceDebug && validDomains.length > 0) {
+          console.log(formatLogMessage('debug', `Force reload not applied for ${currentUrl} - no domain match in [${validDomains.join(', ')}]`));
+        }
+      }
+      // If forcereload is not specified, false, or any other value, useForceReload remains false
       
       if (useForceReload && forceDebug) {
         console.log(formatLogMessage('debug', `Using force reload mechanism for all ${totalReloads + 1} reload(s) on ${currentUrl}`));
@@ -3360,38 +3335,18 @@ function setupFrameHandling(page, forceDebug) {
     updatePageUsage(page, false);
 
       if (dryRunMode) {
-        // Get page title for dry run output
-        let pageTitle = '';
-        try {
-          pageTitle = await page.title();
-        } catch (titleErr) {
-          if (forceDebug) {
-            console.log(formatLogMessage('debug', `Failed to get page title for ${currentUrl}: ${titleErr.message}`));
-          }
+        // Process dry run results using the module
+        const dryRunResult = await processDryRunResults(currentUrl, matchedDomains, page, outputFile, dryRunOutput, forceDebug);
+
+        if (!dryRunResult.success) {
+          console.warn(messageColors.warn(`Dry run processing failed for ${currentUrl}: ${dryRunResult.error}`));
         }
         
-        // Get collected matches and enhance with searchstring results
-        const dryRunMatches = matchedDomains.get('dryRunMatches') || [];
-        const dryRunNetTools = matchedDomains.get('dryRunNetTools') || [];
-        const dryRunSearchString = matchedDomains.get('dryRunSearchString') || new Map();
-        
-        // Enhance matches with searchstring results
-        const enhancedMatches = dryRunMatches.map(match => {
-          const searchResult = dryRunSearchString.get(match.fullUrl);
-          return {
-            ...match,
-            searchStringMatch: searchResult && searchResult.matched ? searchResult : null,
-            searchStringChecked: match.needsSearchStringCheck
-          };
-        });
-        
         // Wait a moment for async nettools/searchstring operations to complete
         // Use fast timeout helper for Puppeteer 22.x compatibility
         await fastTimeout(TIMEOUTS.CURL_HANDLER_DELAY); // Wait for async operations
         
-        outputDryRunResults(currentUrl, enhancedMatches, dryRunNetTools, pageTitle);
-        
-        return { url: currentUrl, rules: [], success: true, dryRun: true, matchCount: dryRunMatches.length + dryRunNetTools.length };
+        return { url: currentUrl, rules: [], success: true, dryRun: true, matchCount: dryRunResult.matchCount };
       } else {
         // Format rules using the output module
         const globalOptions = {
@@ -3811,14 +3766,9 @@ function setupFrameHandling(page, forceDebug) {
 
   // Handle dry run output file writing
   if (dryRunMode && outputFile && dryRunOutput.length > 0) {
-    try {
-      const dryRunContent = dryRunOutput.join('\n');
-      fs.writeFileSync(outputFile, dryRunContent);
-      if (!silentMode) {
-        console.log(`${messageColors.fileOp('📄 Dry run results saved to:')} ${outputFile}`);
-      }
-    } catch (writeErr) {
-      console.error(`❌ Failed to write dry run output to ${outputFile}: ${writeErr.message}`);
+    const writeResult = writeDryRunOutput(outputFile, dryRunOutput, silentMode);
+    if (!writeResult.success && forceDebug) {
+      console.log(formatLogMessage('debug', `Dry run file write failed: ${writeResult.error}`));
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "2.0.20",
+  "version": "2.0.22",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {