@fanboynz/network-scanner 1.0.97 → 1.0.99

package/README.md

@@ -151,7 +151,10 @@ Example:
 | `blocked`            | Array | - | Domains or regexes to block during scanning |
 | `even_blocked`       | Boolean | `false` | Add matching rules even if requests are blocked |
 | `bypass_cache`       | Boolean | `false` | Skip all caching for this site's URLs |
-| `window_cleanup`     | Boolean | `false` | Close extra browser windows/tabs after entire URL group completes with 16s delay |
+| `window_cleanup`     | Boolean or String | `false` | Close old/unused browser windows/tabs after entire URL group completes |
+
+**Window cleanup modes:** `false` (disabled), `true` (conservative - closes obvious leftovers), `"all"` (aggressive - closes all content pages). Both active modes preserve the main Puppeteer window and wait 16 seconds before cleanup to avoid interfering with active operations.
+
 
 ### Redirect Handling Options
 
@@ -320,6 +323,39 @@ node nwss.js --max-concurrent 12 --cleanup-interval 300 -o rules.txt
 
 ### Stealth Configuration Examples
 
+#### Memory Management with Window Cleanup
+```json
+{
+  "url": [
+    "https://popup-heavy-site1.com",
+    "https://popup-heavy-site2.com", 
+    "https://popup-heavy-site3.com"
+  ],
+  "filterRegex": "\\.(space|website|tech)\\b",
+  "window_cleanup": "all",
+  "interact": true,
+  "reload": 2,
+  "resourceTypes": ["script", "fetch"],
+  "comments": "Aggressive cleanup for sites that open many popups"
+}
+```
+
+#### Conservative Memory Management
+```json
+{
+  "url": "https://complex-site.com",
+  "filterRegex": "analytics|tracking",
+  "window_cleanup": true,
+  "interact": true,
+  "delay": 8000,
+  "reload": 3,
+  "comments": [
+    "Conservative cleanup preserves potentially active content",
+    "Good for sites with complex iframe structures"
+  ]
+}
+```
+
 #### E-commerce Site Scanning
 ```json
 {
@@ -368,6 +404,22 @@ node nwss.js --max-concurrent 12 --cleanup-interval 300 -o rules.txt
 
 ---
 
+## Memory Management
+
+The scanner includes intelligent window management to prevent memory accumulation during long scans:
+
+- **Conservative cleanup** (`window_cleanup: true`): Selectively closes pages that appear to be leftovers from previous scans
+- **Aggressive cleanup** (`window_cleanup: "all"`): Closes all content pages from previous operations for maximum memory recovery  
+- **Main window preservation**: Both modes always preserve the main Puppeteer browser window to maintain stability
+- **Popup window handling**: Automatically detects and closes popup windows created by previous site scans
+- **Timing protection**: 16-second delay ensures no active operations are interrupted during cleanup
+- **Active page protection**: Never affects pages currently being processed by concurrent scanning operations
+- **Memory reporting**: Reports estimated memory freed from closed windows for performance monitoring
+
+Use aggressive cleanup for sites that open many popups or when processing large numbers of URLs. Use conservative cleanup when you want to preserve potentially active content but still free obvious leftovers.
+
+---
+
 ## INSTALL 
 
 #### (Ubuntu as example). NOTE: Use Chrome and not Chromium for best compatibility.

package/lib/browserhealth.js

@@ -7,7 +7,19 @@ const { formatLogMessage, messageColors } = require('./colorize');
 
 
 // Window cleanup delay constant
-const WINDOW_CLEANUP_DELAY_MS = 16000;
+const WINDOW_CLEANUP_DELAY_MS = 15000;
+// window_clean REALTIME
+const REALTIME_CLEANUP_BUFFER_MS = 15000; // Additional buffer time after site delay. Delay increased to fix missing hits.
+const REALTIME_CLEANUP_THRESHOLD = 8; // Default number of pages to keep
+const REALTIME_CLEANUP_MIN_PAGES = 3; // Minimum pages before cleanup kicks in
+
+// Track page creation order for realtime cleanup
+const pageCreationTracker = new Map(); // Maps page -> creation timestamp
+let pageCreationCounter = 0;
+
+// Track page usage for realtime cleanup safety
+const pageUsageTracker = new Map(); // Maps page -> { lastActivity: timestamp, isProcessing: boolean }
+const PAGE_IDLE_THRESHOLD = 15000; // 15 seconds of inactivity before considering page safe to clean
 
 /**
  * Performs group-level window cleanup after all URLs in a site group complete
@@ -15,33 +27,78 @@ const WINDOW_CLEANUP_DELAY_MS = 16000;
  * @param {import('puppeteer').Browser} browserInstance - Browser instance
  * @param {string} groupDescription - Description of the group for logging
  * @param {boolean} forceDebug - Debug logging flag
+ * @param {string|boolean} cleanupMode - Cleanup mode: true/"default" (conservative), "all" (aggressive)
  * @returns {Promise<Object>} Cleanup results
  */
-async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug) {
+async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug, cleanupMode = true) {
   try {
     // Wait before cleanup to allow any final operations to complete
+    const modeText = cleanupMode === "all" ? "aggressive cleanup of old windows" : "conservative cleanup of extra windows"
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before cleanup for group: ${groupDescription}`));
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before ${modeText} for group: ${groupDescription}`));
+
     }
     await new Promise(resolve => setTimeout(resolve, WINDOW_CLEANUP_DELAY_MS));
     
     const allPages = await browserInstance.pages();
-    const mainPage = allPages[0]; // Always keep the first page as main
-    const extraPages = allPages.slice(1); // All other pages can be closed
+    // Identify the main Puppeteer window (should be about:blank or the initial page)
+    let mainPuppeteerPage = null;
+    let pagesToClose = [];
     
-    if (extraPages.length === 0) {
+    // Find the main page - typically the first page that's about:blank or has been there longest
+    for (const page of allPages) {
+      const url = page.url();
+      if (url === 'about:blank' || url === '' || url.startsWith('chrome://')) {
+        if (!mainPuppeteerPage) {
+          mainPuppeteerPage = page; // First blank page is likely the main window
+        } else {
+          pagesToClose.push(page); // Additional blank pages can be closed
+        }
+      } else {
+        // Any page with actual content should be evaluated for closure
+        if (cleanupMode === "all") {
+          // Aggressive mode: close all content pages
+          pagesToClose.push(page);
+        } else {
+          // Conservative mode: only close pages that look like leftovers from previous scans
+          // Keep pages that might still be actively used
+          const isOldPage = await isPageFromPreviousScan(page, forceDebug);
+          if (isOldPage) {
+            pagesToClose.push(page);
+          }
+        }
+      }
+    }
+    
+    // Ensure we always have a main page
+    if (!mainPuppeteerPage && allPages.length > 0) {
+      mainPuppeteerPage = allPages[0]; // Fallback to first page
+      pagesToClose = allPages.slice(1);
       if (forceDebug) {
-        console.log(formatLogMessage('debug', `[group_window_cleanup] No extra windows to close for group: ${groupDescription}`));
+        console.log(formatLogMessage('debug', `[group_window_cleanup] No blank page found, using first page as main: ${mainPuppeteerPage.url()}`));
       }
-      return { success: true, closedCount: 0, totalPages: allPages.length, estimatedMemoryFreed: 0 };
+    }
+    
+    if (pagesToClose.length === 0) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[group_window_cleanup] No windows to close for group: ${groupDescription}`));
+      }
+      return { 
+        success: true, 
+        closedCount: 0, 
+        totalPages: allPages.length, 
+        estimatedMemoryFreed: 0,
+        mainPagePreserved: true,
+        cleanupMode: cleanupMode === "all" ? "all" : "default"
+      };
     }
     
     // Estimate memory usage before closing
     let totalEstimatedMemory = 0;
     const pageMemoryEstimates = [];
     
-    for (let i = 0; i < extraPages.length; i++) {
-      const page = extraPages[i];
+    for (let i = 0; i < pagesToClose.length; i++) {
+      const page = pagesToClose[i];
       let pageMemoryEstimate = 0;
       
       try {
@@ -78,17 +135,20 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
       totalEstimatedMemory += pageMemoryEstimate;
     }
     
-    // Close all extra pages since the entire group is complete
-    const closePromises = extraPages.map(async (page, index) => {
+    // Close identified old/unused pages
+    const closePromises = pagesToClose.map(async (page, index) => {
       try {
         if (!page.isClosed()) {
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${page.url()}`));
+          }
           await page.close();
           return { success: true, url: page.url() || `page-${index}`, estimatedMemory: pageMemoryEstimates[index] };
         }
         return { success: false, reason: 'already_closed', estimatedMemory: 0 };
       } catch (closeErr) {
         if (forceDebug) {
-          console.log(formatLogMessage('debug', `[group_window_cleanup] Failed to close page ${index + 1}: ${closeErr.message}`));
+          console.log(formatLogMessage('debug', `[group_window_cleanup] Failed to close old page ${index + 1}: ${closeErr.message}`));
         }
         return { success: false, error: closeErr.message, estimatedMemory: 0 };
       }
@@ -114,18 +174,22 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
     };
     
     if (forceDebug) {
-      console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${extraPages.length} windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
+      console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${pagesToClose.length} old windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
       console.log(formatLogMessage('debug', `[group_window_cleanup] Estimated memory freed: ${formatMemory(actualMemoryFreed)}`));
+      if (mainPuppeteerPage) {
+        console.log(formatLogMessage('debug', `[group_window_cleanup] Main Puppeteer window preserved: ${mainPuppeteerPage.url()}`));
+      }
     }
     
     return { 
       success: true, 
       closedCount: successfulCloses,
       totalPages: allPages.length,
-      mainPageKept: !mainPage.isClosed(),
+      mainPagePreserved: mainPuppeteerPage && !mainPuppeteerPage.isClosed(),
       delayUsed: WINDOW_CLEANUP_DELAY_MS,
       estimatedMemoryFreed: actualMemoryFreed,
-      estimatedMemoryFreedFormatted: formatMemory(actualMemoryFreed)
+      estimatedMemoryFreedFormatted: formatMemory(actualMemoryFreed),
+      cleanupMode: cleanupMode === "all" ? "all" : "default"
     };
   } catch (cleanupErr) {
     if (forceDebug) {
@@ -135,6 +199,237 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
   }
 }
 
+/**
+ * Checks if a page is safe to close (not actively processing)
+ * @param {import('puppeteer').Page} page - Page to check
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<boolean>} True if page is safe to close
+ */
+async function isPageSafeToClose(page, forceDebug) {
+  try {
+    if (page.isClosed()) {
+      return true; // Already closed
+    }
+
+    const usage = pageUsageTracker.get(page);
+    if (!usage) {
+      // No usage data - assume safe if page exists for a while
+      return true;
+    }
+
+    // Check if page is actively processing
+    if (usage.isProcessing) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[realtime_cleanup] Page still processing: ${page.url().substring(0, 50)}...`));
+      }
+      return false;
+    }
+
+    // Check if page has been idle long enough
+    const idleTime = Date.now() - usage.lastActivity;
+    const isSafe = idleTime >= PAGE_IDLE_THRESHOLD;
+    
+    if (!isSafe && forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Page not idle long enough: ${Math.round(idleTime/1000)}s < ${PAGE_IDLE_THRESHOLD/1000}s`));
+    }
+
+    return isSafe;
+  } catch (err) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Error checking page safety: ${err.message}`));
+    }
+    return true; // Assume safe if we can't check
+  }
+}
+
+/**
+ * Updates page usage tracking
+ * @param {import('puppeteer').Page} page - Page to update
+ * @param {boolean} isProcessing - Whether page is actively processing
+ */
+function updatePageUsage(page, isProcessing = false) {
+  try {
+    if (!page.isClosed()) {
+      pageUsageTracker.set(page, {
+        lastActivity: Date.now(),
+        isProcessing: isProcessing
+      });
+    }
+  } catch (err) {
+    // Ignore errors in usage tracking
+  }
+}
+
+/**
+ * Performs realtime window cleanup - removes oldest pages when threshold is exceeded
+ * Waits for site delay + 3 seconds before cleanup to ensure delayed requests are captured
+ * @param {import('puppeteer').Browser} browserInstance - Browser instance
+ * @param {number} threshold - Maximum number of pages to keep (default: 8)
+ * @param {boolean} forceDebug - Debug logging flag
+ * @param {number} siteDelay - Current site's delay value in milliseconds (default: 4000)
+ * @returns {Promise<Object>} Cleanup results
+ */
+async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIME_CLEANUP_THRESHOLD, forceDebug, siteDelay = 4000) {
+  try {
+    const allPages = await browserInstance.pages();
+    
+    // Skip cleanup if we don't have enough pages to warrant it
+    if (allPages.length <= Math.max(threshold, REALTIME_CLEANUP_MIN_PAGES)) {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[realtime_cleanup] Only ${allPages.length} pages open, threshold is ${threshold} - no cleanup needed`));
+      }
+      return { success: true, closedCount: 0, totalPages: allPages.length, reason: 'below_threshold' };
+    }
+
+    // Calculate cleanup delay: site delay + 3 second buffer
+    const cleanupDelay = siteDelay + REALTIME_CLEANUP_BUFFER_MS;
+    
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Waiting ${cleanupDelay}ms (site delay: ${siteDelay}ms + ${REALTIME_CLEANUP_BUFFER_MS}ms buffer) before cleanup (threshold: ${threshold})`));
+    }
+    await new Promise(resolve => setTimeout(resolve, cleanupDelay));
+    
+    const allPagesAfterDelay = await browserInstance.pages();
+    
+    // Find main Puppeteer page (usually about:blank)
+    let mainPage = allPagesAfterDelay.find(page => {
+      const url = page.url();
+      return url === 'about:blank' || url === '' || url.startsWith('chrome://');
+    }) || allPagesAfterDelay[0]; // Fallback to first page
+
+    // Get pages sorted by creation time (oldest first)
+    const sortedPages = allPagesAfterDelay
+      .filter(page => page !== mainPage && !page.isClosed())
+      .sort((a, b) => {
+        const timeA = pageCreationTracker.get(a) || 0;
+        const timeB = pageCreationTracker.get(b) || 0;
+        return timeA - timeB; // Oldest first
+      });
+
+    // Calculate how many pages to close
+    const pagesToKeep = threshold - 1; // -1 for main page
+    const pagesToClose = sortedPages.slice(0, Math.max(0, sortedPages.length - pagesToKeep));
+
+    // Filter out pages that are still being used
+    const safetyChecks = await Promise.all(
+      pagesToClose.map(page => isPageSafeToClose(page, forceDebug))
+    );
+    
+    const safePagesToClose = pagesToClose.filter((page, index) => safetyChecks[index]);
+    const unsafePagesCount = pagesToClose.length - safePagesToClose.length;
+    
+    if (unsafePagesCount > 0 && forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Skipping ${unsafePagesCount} active pages for safety`));
+    }
+
+    if (safePagesToClose.length === 0) {
+      if (forceDebug) {
+        const reason = pagesToClose.length === 0 ? 
+          `${sortedPages.length} content pages, keeping ${pagesToKeep}` :
+          `${pagesToClose.length} pages still active`;
+        console.log(formatLogMessage('debug', `[realtime_cleanup] No pages need closing (${reason})`));
+      }
+      return { success: true, closedCount: 0, totalPages: allPagesAfterDelay.length, reason: 'no_cleanup_needed' };
+    }
+
+    // Close oldest pages
+    let closedCount = 0;
+    for (const page of safePagesToClose) {
+      try {
+        if (!page.isClosed()) {
+          const pageUrl = page.url();
+          await page.close();
+          pageCreationTracker.delete(page); // Remove from tracker
+          closedCount++;
+          
+          if (forceDebug) {
+            console.log(formatLogMessage('debug', `[realtime_cleanup] Closed old page: ${pageUrl.substring(0, 50)}...`));
+          }
+        }
+      } catch (closeErr) {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `[realtime_cleanup] Failed to close page: ${closeErr.message}`));
+        }
+      }
+    }
+
+    const remainingPages = allPagesAfterDelay.length - closedCount;
+    
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Closed ${closedCount}/${pagesToClose.length} oldest pages (${unsafePagesCount} skipped for safety), ${remainingPages} pages remaining`));
+    }
+
+    return { 
+      success: true, 
+      closedCount, 
+      totalPages: allPagesAfterDelay.length,
+      remainingPages,
+      threshold,
+      cleanupDelay,
+      reason: 'cleanup_completed'
+    };
+  } catch (cleanupErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[realtime_cleanup] Cleanup failed: ${cleanupErr.message}`));
+    }
+    return { success: false, error: cleanupErr.message, closedCount: 0 };
+  }
+}
+
+/**
+ * Determines if a page appears to be from a previous scan and can be safely closed
+ * @param {import('puppeteer').Page} page - Page to evaluate
+ * @param {boolean} forceDebug - Debug logging flag
+ * @returns {Promise<boolean>} True if page appears to be from previous scan
+ */
+async function isPageFromPreviousScan(page, forceDebug) {
+  try {
+    const url = page.url();
+    
+    // Always consider these as old/closeable
+    if (url === 'about:blank' || 
+        url === '' || 
+        url.startsWith('chrome://') ||
+        url.startsWith('chrome-error://') ||
+        url.startsWith('data:')) {
+      return false; // Don't close blank pages here, handled separately
+    }
+    
+    // Check if page has been idle (no recent navigation)
+    // This is a heuristic - pages from previous scans are likely to be idle
+    try {
+      const title = await page.title();
+      // Pages with generic titles or error states are likely old
+      if (title.includes('404') || 
+          title.includes('Error') || 
+          title.includes('Not Found') ||
+          title === '') {
+        return true;
+      }
+    } catch (titleErr) {
+      // If we can't get title, page might be in bad state
+      return true;
+    }
+    
+    // Default: consider most content pages as potentially old in conservative mode
+    return false; // Conservative - don't close unless we're sure
+  } catch (err) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${page.url()}: ${err.message}`));
+    }
+    return false; // Conservative - don't close if we can't evaluate
+  }
+}
+
+/**
+ * Tracks a new page for realtime cleanup purposes
+ * @param {import('puppeteer').Page} page - Page to track
+ */
+function trackPageForRealtime(page) {
+  pageCreationTracker.set(page, ++pageCreationCounter);
+  updatePageUsage(page, false); // Initialize usage tracking
+}
+
 /**
  * Quick browser responsiveness test for use during page setup
  * Designed to catch browser degradation between operations
@@ -672,10 +967,28 @@ module.exports = {
   checkBrowserMemory,
   testBrowserConnectivity,
   performGroupWindowCleanup,
+  performRealtimeWindowCleanup,
+  trackPageForRealtime,
   testNetworkCapability,
   isQuicklyResponsive,
   performHealthAssessment,
   monitorBrowserHealth,
   isBrowserHealthy,
-  isCriticalProtocolError
+  isCriticalProtocolError,
+  updatePageUsage
 };
+
+// Clean up tracking maps when pages are closed
+const originalPageClose = require('puppeteer').Page.prototype.close;
+if (originalPageClose) {
+  require('puppeteer').Page.prototype.close = async function(...args) {
+    try {
+      // Clean up tracking data
+      pageCreationTracker.delete(this);
+      pageUsageTracker.delete(this);
+    } catch (err) {
+      // Ignore cleanup errors
+    }
+    return originalPageClose.apply(this, args);
+  };
+}

package/nwss.1

@@ -1,4 +1,4 @@
-.TH NWSS-SCRIPT 1 "2025" "scanner-script v1.0.32" "User Commands"
+.TH NWSS-SCRIPT 1 "2025" "scanner-script v1.0.98" "User Commands"
 .SH NAME
 NWSS scanner-script \- Network scanner for malware detection and domain analysis with advanced similarity filtering
 
@@ -27,6 +27,10 @@ Remove rules that already exist in \fIFILE\fR before output (requires \fB\-o\fR)
 .B \--append
 Append new rules to output file instead of overwriting (requires \fB\-o\fR).
 
+.TP
+.BR \--color ", " \--colour
+Enable colored console output for status messages.
+
 .SS Output Format Options
 .TP
 .B \--localhost
@@ -125,6 +129,14 @@ Console output only: show matching regex, titles, whois/dig/searchstring results
 .B \--remove-tempfiles
 Remove Chrome/Puppeteer temporary files before exit.
 
+.TP
+.BR \--max-concurrent " \fINUMBER\fR"
+Maximum concurrent site processing (1-50, overrides config/default).
+
+.TP
+.BR \--cleanup-interval " \fINUMBER\fR"
+Browser restart interval in URLs processed (1-1000, overrides config/default).
+
 .TP
 .BR \-h ", " \--help
 Show help message and exit.
@@ -134,6 +146,10 @@ Show help message and exit.
 Show version information and exit.
 
 .SS Validation Options
+.TP
+.B \--cache-requests
+Cache HTTP requests to avoid re-requesting same URLs within scan.
+
 .TP
 .B \--validate-config
 Validate config.json file and exit.
@@ -150,6 +166,14 @@ Clean rule files by removing invalid lines and optionally duplicates (uses \fB\-
 .B \--test-validation
 Run domain validation tests and exit.
 
+.TP
+.B \--clear-cache
+Clear persistent cache before scanning (improves fresh start performance).
+
+.TP
+.B \--ignore-cache
+Bypass all smart caching functionality during scanning.
+
 .SH CONFIGURATION
 
 Configuration is provided via JSON files. The default configuration file is \fBconfig.json\fR.
@@ -184,6 +208,30 @@ Number. Similarity threshold percentage for ignore_similar (default: 80).
 .B ignore_similar_ignored_domains
 Boolean. Ignore domains similar to ignoreDomains list (default: true).
 
+.TP
+.B max_concurrent_sites
+Number. Maximum concurrent site processing (1-50, default: 6).
+
+.TP
+.B resource_cleanup_interval
+Number. Browser restart interval in URLs processed (1-1000, default: 80).
+
+.TP
+.B cache_path
+String. Directory path for persistent cache storage (default: ".cache").
+
+.TP
+.B cache_max_size
+Number. Maximum number of entries in cache (default: 5000).
+
+.TP
+.B cache_autosave_minutes
+Number. Interval for automatic cache saves in minutes (default: 1).
+
+.TP
+.B cache_requests
+Boolean. Enable HTTP request response caching (default: false).
+
 .SS Per-Site Configuration Options
 
 .TP
@@ -194,6 +242,10 @@ Single URL string or array of URLs to scan.
 .B filterRegex
 Regex pattern(s) to match suspicious requests.
 
+.TP
+.B regex_and
+Boolean. Use AND logic for multiple filterRegex patterns - ALL patterns must match the same URL (default: false).
+
 .TP
 .B comments
 Documentation strings or notes - completely ignored by the scanner. Can be a single string or array of strings. Used for adding context, URLs, timestamps, or any documentation notes to configuration files.
@@ -234,6 +286,10 @@ Spoof User-Agent: \fB"chrome"\fR, \fB"firefox"\fR, or \fB"safari"\fR.
 .B interact
 Boolean. Simulate mouse movements and clicks.
 
+.TP
+.B interact_intensity
+String. Interaction simulation intensity: \fB"low"\fR, \fB"medium"\fR, \fB"high"\fR (default: "medium").
+
 .TP
 .B delay
 Milliseconds to wait after page load (default: 4000).
@@ -242,6 +298,10 @@ Milliseconds to wait after page load (default: 4000).
 .B reload
 Number of times to reload the page (default: 1).
 
+.TP
+.B forcereload
+Boolean. Force an additional reload after reloads.
+
 .TP
 .B timeout
 Request timeout in milliseconds (default: 30000).
@@ -258,6 +318,14 @@ Boolean. Allow third-party request matching (default: true).
 .B fingerprint_protection
 Boolean or \fB"random"\fR. Enable browser fingerprint spoofing.
 
+.TP
+.B referrer_headers
+String, Array, or Object. Set referrer header for realistic traffic sources.
+
+.TP
+.B custom_headers
+Object. Add custom HTTP headers to requests.
+
 .TP
 .B ignore_similar
 Boolean. Override global ignore_similar setting for this site.
@@ -274,6 +342,10 @@ Boolean. Override global ignore_similar_ignored_domains for this site.
 .B even_blocked
 Boolean. Add matching rules even if requests are blocked (default: false).
 
+.TP
+.B bypass_cache
+Boolean. Skip all caching for this site's URLs (default: false).
+
 .TP
 .B whois
 Array of terms that must ALL be found in WHOIS data (AND logic).
@@ -353,10 +425,6 @@ Object. Custom page.goto() options for Puppeteer navigation. Available options:
 .RE
 Example: \fB{"waitUntil": "networkidle2", "timeout": 60000}\fR
 
-.TP
-.B forcereload
-Boolean. Force an additional reload with cache disabled after normal reloads.
-
 .TP
 .B clear_sitedata
 Boolean. Clear all cookies, cache, and storage before each page load (default: false).
@@ -389,6 +457,45 @@ Boolean. Launch browser with GUI for this specific site.
 .B adblock_rules
 Boolean. Generate adblock filter rules with resource types for this site.
 
+.TP
+.B window_cleanup
+Boolean or String. Close old/unused browser windows and tabs after entire URL group completes (default: false).
+.RS
+.IP \(bu 4
+\fBfalse\fR - No cleanup performed
+.IP \(bu 4
+\fBtrue\fR - Conservative cleanup: closes pages that appear to be from previous scans while preserving main Puppeteer window
+.IP \(bu 4
+\fB"all"\fR - Aggressive cleanup: closes all content pages from previous operations while preserving main Puppeteer window
+.RE
+Both modes wait 16 seconds before cleanup to allow final operations to complete, and always preserve the main browser window to maintain browser instance stability. Cleanup targets popup windows, extra tabs, and pages from previously completed site groups, but never affects actively processing pages.
+
+.TP
+
+.SS Redirect Handling Options
+
+.TP
+.B follow_redirects
+Boolean. Follow redirects to new domains (default: true).
+
+.TP
+.B max_redirects
+Number. Maximum number of redirects to follow (default: 10).
+
+.TP
+.B js_redirect_timeout
+Milliseconds. Time to wait for JavaScript redirects (default: 5000).
+
+.TP
+.B detect_js_patterns
+Boolean. Analyze page source for redirect patterns (default: true).
+
+.TP
+.B redirect_timeout_multiplier
+Number. Increase timeout for redirected URLs (default: 1.5).
+
+.SS Cloudflare Protection Options
+
 .TP
 .B cloudflare_phish
 Boolean. Auto-click through Cloudflare phishing warnings (default: false).
@@ -397,6 +504,24 @@ Boolean. Auto-click through Cloudflare phishing warnings (default: false).
 .B cloudflare_bypass
 Boolean. Auto-solve Cloudflare "Verify you are human" challenges (default: false).
 
+.TP
+.B cloudflare_parallel_detection
+Boolean. Use parallel detection for faster Cloudflare checks (default: true).
+
+.TP
+.B cloudflare_max_retries
+Number. Maximum retry attempts for Cloudflare operations (default: 3).
+
+.TP
+.B cloudflare_cache_ttl
+Milliseconds. TTL for Cloudflare detection cache (default: 300000 - 5 minutes).
+
+.TP
+.B cloudflare_retry_on_error
+Boolean. Enable retry logic for Cloudflare operations (default: true).
+
+.SS FlowProxy Protection Options
+
 .TP
 .B flowproxy_detection
 Boolean. Enable flowProxy protection detection and handling (default: false).
@@ -421,6 +546,24 @@ Milliseconds. Delay for rate limiting (default: 30000).
 .B flowproxy_additional_delay
 Milliseconds. Additional processing delay (default: 5000).
 
+.SS Advanced Options
+
+.TP
+.B interact_duration
+Milliseconds. Duration of interaction simulation (default: 2000).
+
+.TP
+.B interact_scrolling
+Boolean. Enable scrolling simulation (default: true).
+
+.TP
+.B interact_clicks
+Boolean. Enable element clicking simulation (default: false).
+
+.TP
+.B interact_typing
+Boolean. Enable typing simulation (default: false).
+
 .TP
 .B verbose
 Boolean. Enable verbose output for this specific site.
@@ -461,6 +604,28 @@ Boolean. Force Pi-hole regex output format for this site.
 .B plain
 Boolean. Force plain domain output for this site.
 
+.SS Referrer Header Options
+
+.B Simple formats:
+.RS
+.IP \(bu 4
+\fB"referrer_headers": "https://google.com/search?q=example"\fR
+.IP \(bu 4
+\fB"referrer_headers": ["url1", "url2"]\fR
+.RE
+
+.B Smart modes:
+.RS
+.IP \(bu 4
+\fB"referrer_headers": {"mode": "random_search", "search_terms": ["reviews"]}\fR
+.IP \(bu 4
+\fB"referrer_headers": {"mode": "social_media"}\fR
+.IP \(bu 4
+\fB"referrer_headers": {"mode": "direct_navigation"}\fR
+.IP \(bu 4
+\fB"referrer_headers": {"mode": "custom", "custom": ["https://news.ycombinator.com/"]}\fR
+.RE
+
 .SH SIMILARITY FILTERING
 
 The scanner includes advanced similarity filtering to reduce noise and improve detection accuracy by automatically ignoring domains that are very similar to ones already found or explicitly ignored.
@@ -530,6 +695,16 @@ With default settings (\fBignore_similar_threshold: 80\fR):
 }
 .EE
 
+.SS Configuration with regex AND logic:
+.EX
+{
+  "url": "https://ad-network.com",
+  "filterRegex": ["tracking", "analytics"],
+  "regex_and": true,
+  "resourceTypes": ["script", "fetch"]
+}
+.EE
+
 .SS Configuration with similarity filtering:
 .EX
 {
@@ -661,6 +836,36 @@ node nwss.js -o dnsmasq.conf --dnsmasq --titles
 node nwss.js -o pihole_regex.txt --pihole --debug
 .EE
 
+.SS Performance tuning with concurrency:
+.EX
+node nwss.js --max-concurrent 12 --cleanup-interval 100 -o rules.txt
+.EE
+
+.SS Cache management:
+.EX
+node nwss.js --clear-cache --cache-requests -o rules.txt
+node nwss.js --ignore-cache --debug -o rules.txt
+.EE
+
+.SS Stealth configuration with referrer headers:
+.EX
+{
+  "url": "https://e-commerce-site.com",
+  "userAgent": "chrome",
+  "fingerprint_protection": "random",
+  "referrer_headers": {
+    "mode": "random_search",
+    "search_terms": ["product reviews", "best deals"]
+  },
+  "custom_headers": {
+    "Accept-Language": "en-US,en;q=0.9"
+  },
+  "interact": true,
+  "interact_intensity": "high",
+  "filterRegex": "analytics|tracking|ads"
+}
+.EE
+
 .SS Cloudflare bypass and fingerprint spoofing:
 .EX
 {
@@ -688,6 +893,35 @@ node nwss.js -o pihole_regex.txt --pihole --debug
 }
 .EE
 
+.SS Window cleanup for memory management:
+.EX
+{
+  "url": [
+    "https://site1.com",
+    "https://site2.com",
+    "https://site3.com"
+  ],
+  "filterRegex": "\\\\.(space|website)\\\\b",
+  "window_cleanup": "all",
+  "resourceTypes": ["script", "fetch"]
+}
+.EE
+
+.SS Conservative window cleanup:
+.EX
+{
+  "url": "https://popup-heavy-site.com",
+  "filterRegex": "\\\\.(top|buzz)\\\\b",
+  "window_cleanup": true,
+  "interact": true,
+  "reload": 2,
+  "comments": [
+    "Site opens many popup windows",
+    "Conservative cleanup preserves potentially active content"
+  ]
+}
+.EE
+
 .SH OUTPUT FORMATS
 
 The scanner supports multiple output formats for different blocking systems:
@@ -740,6 +974,10 @@ Default configuration file containing scan targets and rules.
 .B logs/
 Directory created for debug and matched URL logs when \fB\--debug\fR or \fB\--dumpurls\fR is used.
 
+.TP
+.B .cache/
+Default cache directory for smart caching functionality.
+
 .TP
 .B user.action
 Common Privoxy action file when using \fB\--privoxy\fR output.
@@ -747,7 +985,7 @@ Common Privoxy action file when using \fB\--privoxy\fR output.
 .SH DETECTION METHODS
 
 .SS URL Pattern Matching
-Uses regex patterns to identify suspicious domains and request URLs.
+Uses regex patterns to identify suspicious domains and request URLs with support for AND/OR logic.
 
 .SS Content Analysis
 Downloads page content with curl and searches for malicious strings using JavaScript or grep.
@@ -784,6 +1022,28 @@ Detects and handles FlowProxy protection systems.
 .SS Intelligent Domain Filtering
 Advanced similarity algorithms prevent duplicate detection across international domains and variations.
 
+.SS Smart Caching
+Persistent caching system for improved performance across multiple scans.
+
+.SS Window Management
+Intelligent browser window and tab cleanup to prevent memory accumulation:
+.RS
+.IP \(bu 4
+\fBConservative mode\fR (\fBwindow_cleanup: true\fR): Selectively closes pages that appear to be leftovers from previous scans based on URL patterns and page state analysis
+.IP \(bu 4
+\fBAggressive mode\fR (\fBwindow_cleanup: "all"\fR): Closes all content pages from previous operations for maximum memory recovery
+.IP \(bu 4
+\fBMain window preservation\fR: Both modes always preserve the main Puppeteer browser window (typically about:blank) to maintain browser instance stability
+.IP \(bu 4
+\fBPopup window handling\fR: Automatically detects and closes popup windows created by previous site scans
+.IP \(bu 4
+\fBTiming protection\fR: 16-second delay ensures no active operations are interrupted during cleanup
+.IP \(bu 4
+\fBMemory estimation\fR: Reports estimated memory freed from closed windows for performance monitoring
+.IP \(bu 4
+\fBActive page protection\fR: Never affects pages currently being processed by concurrent scanning operations
+.RE
+
 .SH EXIT STATUS
 .TP
 .B 0
@@ -821,4 +1081,4 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License along with
-this program. If not, see <https://www.gnu.org/licenses/>.
+this program. If not, see <https://www.gnu.org/licenses/>.

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.97 ===
+// === Network scanner script (nwss.js) v1.0.99 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -82,6 +82,8 @@ const CONCURRENCY_LIMITS = {
   HIGH_CONCURRENCY_THRESHOLD: 12  // Auto-enable aggressive caching above this
 };
 
+const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime cleanup
+
 /**
  * Detects the installed Puppeteer version dynamically
  * @returns {Object} Version info and compatibility settings
@@ -122,10 +124,10 @@ function detectPuppeteerVersion() {
 // Enhanced redirect handling
 const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
 // Ensure web browser is working correctly
-const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup } = require('./lib/browserhealth');
+const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants --- 
-const VERSION = '1.0.97'; // Script version
+const VERSION = '1.0.99'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -571,7 +573,11 @@ Advanced Options:
   dig_subdomain: true/false                    Use subdomain for dig lookup instead of root domain (default: false)
   digRecordType: "A"                          DNS record type for dig (default: A)
 
-  window_cleanup: true/false                   Close extra browser windows/tabs after entire URL group completes with 5s delay (default: false)
+  window_cleanup: true/false/"realtime"/"all"  Window cleanup mode:
+                                               true/false - Close extra windows after URL group completes (default: false)
+                                               "realtime" - Continuously cleanup oldest pages when threshold exceeded
+                                               "all" - Aggressive cleanup of all content pages after group
+  window_cleanup_threshold: <number>           For realtime mode: max pages to keep open (default: 8)
 
 Referrer Header Options:
   referrer_headers: "https://google.com"       Single referrer URL
@@ -1474,7 +1480,27 @@ function setupFrameHandling(page, forceDebug) {
         throw new Error('Failed to create valid page instance');
       }
 
-     
+      // Track page for realtime cleanup
+      trackPageForRealtime(page);
+
+      // Mark page as actively processing
+      updatePageUsage(page, true);
+
+      // Perform realtime cleanup if enabled
+      if (siteConfig.window_cleanup === "realtime") {
+        const threshold = typeof siteConfig.window_cleanup_threshold === 'number' 
+          ? siteConfig.window_cleanup_threshold 
+          : REALTIME_CLEANUP_THRESHOLD;
+        
+        // Get the site's delay value for cleanup timing
+        const siteDelay = siteConfig.delay || 4000;
+        
+        const realtimeResult = await performRealtimeWindowCleanup(browserInstance, threshold, forceDebug, siteDelay);
+        if (realtimeResult.success && realtimeResult.closedCount > 0 && forceDebug) {
+          console.log(formatLogMessage('debug', `[realtime_cleanup] Cleaned ${realtimeResult.closedCount} old pages, ${realtimeResult.remainingPages} remaining`));
+        }
+      } 
+    
       // Set aggressive timeouts for problematic operations
       // Optimized timeouts for Puppeteer 23.x responsiveness
       page.setDefaultTimeout(Math.min(timeout, TIMEOUTS.DEFAULT_PAGE_REDUCED));
@@ -1850,9 +1876,9 @@ function setupFrameHandling(page, forceDebug) {
    let curlUserAgent = '';
    if (useCurl && siteConfig.userAgent) {
      const userAgents = {
-       chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-       firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-       safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
+       chrome: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+       firefox: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/142.0",
+       safari: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"
      };
      curlUserAgent = userAgents[siteConfig.userAgent.toLowerCase()] || '';
    }
@@ -2612,6 +2638,9 @@ function setupFrameHandling(page, forceDebug) {
         request.continue();
       });
 
+      // Mark page as actively processing network requests
+      updatePageUsage(page, true);
+
      // Add response handler ONLY if searchstring/searchstring_and is defined AND neither curl nor grep is enabled
      if ((hasSearchString || hasSearchStringAnd) && !useCurl && !useGrep) {
        const responseHandler = createResponseHandler({
@@ -2649,6 +2678,9 @@ function setupFrameHandling(page, forceDebug) {
       // Create optimized interaction configuration for this site
       const interactionConfig = createInteractionConfig(currentUrl, siteConfig);
       
+      // Mark page as actively processing interactions
+      updatePageUsage(page, true);
+      
       // --- Runtime CSS Element Blocking (Fallback) ---
       // Apply CSS blocking after page load as a fallback in case evaluateOnNewDocument didn't work
       if (cssBlockedSelectors && Array.isArray(cssBlockedSelectors) && cssBlockedSelectors.length > 0) {
@@ -2875,6 +2907,9 @@ function setupFrameHandling(page, forceDebug) {
         console.log(formatLogMessage('info', `${messageColors.loaded('Loaded:')} (${siteCounter}/${totalUrls}) ${currentUrl}`));
         await page.evaluate(() => { console.log('Safe to evaluate on loaded page.'); });
         
+        // Mark page as processing frames
+        updatePageUsage(page, true);
+        
         // Wait for iframes to load and log them
         if (forceDebug) {
           try {
@@ -2899,6 +2934,9 @@ function setupFrameHandling(page, forceDebug) {
             console.log(formatLogMessage('debug', `Frame debugging failed: ${frameDebugErr.message}`));
           }
         }
+
+      // Page finished initial loading - mark as idle
+      updatePageUsage(page, false);
       } catch (err) {
         // Enhanced error handling for redirect timeouts using redirect module
         const timeoutResult = await handleRedirectTimeout(page, currentUrl, err, safeGetDomain, forceDebug, formatLogMessage);
@@ -2916,6 +2954,9 @@ function setupFrameHandling(page, forceDebug) {
 
       if (interactEnabled && !disableInteract) {
         if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
+        
+        // Mark page as processing during interactions
+        updatePageUsage(page, true);
         // Use enhanced interaction module
         await performPageInteraction(page, currentUrl, interactionConfig, forceDebug);
       }
@@ -2945,6 +2986,9 @@ function setupFrameHandling(page, forceDebug) {
       // Use fast timeout helper for consistent Puppeteer 23.x compatibility
 
       // Handle reloads - use force reload mechanism if forcereload is enabled
+      // Mark page as processing during reloads
+      updatePageUsage(page, true);
+
       const totalReloads = (siteConfig.reload || 1) - 1; // Subtract 1 because initial load counts as first
       const useForceReload = siteConfig.forcereload === true;
       
@@ -3049,6 +3093,9 @@ function setupFrameHandling(page, forceDebug) {
         await fastTimeout(delayMs);
       }
     }
+    
+    // Mark page as idle after all processing complete
+    updatePageUsage(page, false);
 
       if (dryRunMode) {
         // Get page title for dry run output
@@ -3172,6 +3219,10 @@ function setupFrameHandling(page, forceDebug) {
       // Guaranteed resource cleanup - this runs regardless of success or failure
       
       if (cdpSessionManager) {
+        // Mark page as idle when cleanup starts
+        if (page && !page.isClosed()) {
+          updatePageUsage(page, false);
+        }
         await cdpSessionManager.cleanup();
       }
       
@@ -3405,14 +3456,19 @@ function setupFrameHandling(page, forceDebug) {
     for (const [siteKey, siteTasks] of tasksBySite) {
       const siteConfig = siteTasks[0].config; // All tasks in group have same config
       
-      if (siteConfig.window_cleanup === true) {
+      if (siteConfig.window_cleanup === true || siteConfig.window_cleanup === "all" || siteConfig.window_cleanup === "realtime") {
         const urlCount = siteTasks.length;
         const groupDescription = `${urlCount} URLs from site group ${++siteGroupIndex}`;
+        const cleanupMode = siteConfig.window_cleanup === "realtime" ? true : siteConfig.window_cleanup; // Pass through the exact value, but don't pass "realtime" to group cleanup
         
         try {
-          const groupCleanupResult = await performGroupWindowCleanup(browser, groupDescription, forceDebug);
+          const groupCleanupResult = await performGroupWindowCleanup(browser, groupDescription, forceDebug, cleanupMode);
           if (!silentMode && groupCleanupResult.success && groupCleanupResult.closedCount > 0) {
-            console.log(`🗑️ Group cleanup: ${groupCleanupResult.closedCount} windows closed after completing ${groupDescription}`);
+            const modeText = cleanupMode === "all" ? "(aggressive)" : "(conservative)";
+            console.log(`🗑️ Group cleanup: ${groupCleanupResult.closedCount} old windows closed ${modeText} after completing ${groupDescription}`);
+            if (groupCleanupResult.mainPagePreserved) {
+              console.log(`✅ Main Puppeteer window preserved during cleanup`);
+            }
           }
         } catch (groupCleanupErr) {
           if (forceDebug) console.log(formatLogMessage('debug', `Group window cleanup failed: ${groupCleanupErr.message}`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.97",
+  "version": "1.0.99",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {