@fanboynz/network-scanner 1.0.46 → 1.0.48

package/.github/workflows/npm-publish.yml

@@ -1,33 +1,80 @@
-# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
-# For more information see: https://docs.github.com/en/actions/publishing-packages/publishing-nodejs-packages
-
-name: Node.js Package
+name: Publish to NPM
 
 on:
-  release:
-    types: [created]
+  push:
+    branches: [ main, master ]  # adjust to your default branch name
 
 jobs:
-  build:
+  check-version:
     runs-on: ubuntu-latest
+    outputs:
+      version-changed: ${{ steps.check.outputs.changed }}
+      version: ${{ steps.check.outputs.version }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
         with:
-          node-version: 20
-      - run: npm ci
-      - run: npm test
+          fetch-depth: 0  # fetch full history to compare versions
+          
+      - name: Check if version changed
+        id: check
+        run: |
+          # Get current version from package.json
+          CURRENT_VERSION=$(node -p "require('./package.json').version")
+          echo "Current version: $CURRENT_VERSION"
+          
+          # Get previous version from the last commit
+          git checkout HEAD~1 -- package.json 2>/dev/null || echo "No previous package.json found"
+          PREVIOUS_VERSION=$(node -p "require('./package.json').version" 2>/dev/null || echo "0.0.0")
+          echo "Previous version: $PREVIOUS_VERSION"
+          
+          # Restore current package.json
+          git checkout HEAD -- package.json
+          
+          # Check if version changed
+          if [ "$CURRENT_VERSION" != "$PREVIOUS_VERSION" ]; then
+            echo "Version changed from $PREVIOUS_VERSION to $CURRENT_VERSION"
+            echo "changed=true" >> $GITHUB_OUTPUT
+            echo "version=$CURRENT_VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Version unchanged"
+            echo "changed=false" >> $GITHUB_OUTPUT
+            echo "version=$CURRENT_VERSION" >> $GITHUB_OUTPUT
+          fi
 
-  publish-npm:
-    needs: build
+  publish:
+    needs: check-version
+    if: needs.check-version.outputs.version-changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
         with:
-          node-version: 20
-          registry-url: https://registry.npmjs.org/
-      - run: npm ci
-      - run: npm publish
+          node-version: '18'  # matches your package.json requirement
+          registry-url: 'https://registry.npmjs.org'
+          
+      - name: Install dependencies
+        run: npm ci
+        
+      - name: Run linting
+        run: npm run lint
+        
+      - name: Verify npm authentication
+        run: npm whoami
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          
+      - name: Publish to NPM
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          
+      - name: Create Git Tag
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git tag v${{ needs.check-version.outputs.version }}
+          git push origin v${{ needs.check-version.outputs.version }}
         env:
-          NODE_AUTH_TOKEN: ${{secrets.npm_token}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/eslint.config.mjs

@@ -0,0 +1,6 @@
+import globals from "globals";
+import { defineConfig } from "eslint/config";
+
+export default defineConfig([
+  { files: ["**/*.{js,mjs,cjs}"], languageOptions: { globals: globals.browser } },
+]);

package/lib/cdp.js

@@ -0,0 +1,302 @@
+// === Chrome DevTools Protocol (CDP) Module ===
+// Handles CDP session management and network request logging for enhanced browser monitoring
+//
+// INTEGRATION GUIDE FOR OTHER APPLICATIONS:
+// This module provides a clean interface for Chrome DevTools Protocol integration with Puppeteer.
+// It can be easily integrated into any Node.js application that uses Puppeteer for browser automation.
+//
+// BASIC USAGE:
+//   const { createCDPSession } = require('./lib/cdp');
+//   const cdpManager = await createCDPSession(page, url, options);
+//   // ... do your work ...
+//   await cdpManager.cleanup(); // Always cleanup when done
+//
+// DEPENDENCIES:
+//   - Puppeteer (any recent version)
+//   - ./colorize module (for logging) - can be replaced with console.log if needed
+//
+// PERFORMANCE CONSIDERATIONS:
+//   - CDP adds ~10-20% overhead to page processing
+//   - Use selectively on complex sites that need deep network visibility
+//   - Avoid on high-volume batch processing unless debugging
+//
+// COMPATIBILITY:
+//   - Works with Chrome/Chromium browsers
+//   - Compatible with headless and headful modes
+//   - Tested with Puppeteer 13+ but should work with older versions
+
+const { formatLogMessage } = require('./colorize');
+
+/**
+ * Creates and manages a CDP session for network monitoring
+ * 
+ * INTEGRATION EXAMPLE:
+ *   const cdpManager = await createCDPSession(page, 'https://example.com', {
+ *     enableCDP: true,        // Global CDP flag
+ *     siteSpecificCDP: true,  // Site-specific CDP flag  
+ *     forceDebug: false       // Enable debug logging
+ *   });
+ *   
+ *   // Your page automation code here...
+ *   await page.goto('https://example.com');
+ *   
+ *   // Always cleanup when done
+ *   await cdpManager.cleanup();
+ *
+ * WHAT IT MONITORS:
+ *   - All network requests (GET, POST, etc.)
+ *   - Request initiators (script, parser, user, etc.)
+ *   - Request/response timing
+ *   - Failed requests and errors
+ *
+ * ERROR HANDLING:
+ *   - Gracefully handles CDP connection failures
+ *   - Distinguishes between critical and non-critical errors
+ *   - Returns null session object if CDP setup fails
+ *   - Never throws on cleanup operations
+ *
+ * @param {import('puppeteer').Page} page - The Puppeteer page instance
+ * @param {string} currentUrl - The URL being processed (used for logging context)
+ * @param {object} options - Configuration options
+ * @param {boolean} options.enableCDP - Global CDP flag (from --cdp command line)
+ * @param {boolean} options.siteSpecificCDP - Site-specific CDP flag (from config)
+ * @param {boolean} options.forceDebug - Debug logging flag
+ * @returns {Promise<object>} CDP session object with cleanup method
+ */
+async function createCDPSession(page, currentUrl, options = {}) {
+  const { enableCDP, siteSpecificCDP, forceDebug } = options;
+  
+  // Determine if CDP logging is needed for this page
+  // You can customize this logic for your application's needs
+  const cdpLoggingNeeded = enableCDP || siteSpecificCDP === true;
+  
+  if (!cdpLoggingNeeded) {
+    // Return a null session with no-op cleanup for consistent API
+    return { session: null, cleanup: async () => {} };
+  }
+
+  // Log which CDP mode is being used
+  if (forceDebug) {
+    if (enableCDP) {
+      console.log(formatLogMessage('debug', `CDP logging globally enabled by --cdp, applying to page: ${currentUrl}`));
+    } else if (siteSpecificCDP === true) {
+      console.log(formatLogMessage('debug', `CDP logging enabled for page ${currentUrl} via site-specific 'cdp: true' config.`));
+    }
+  }
+
+  let cdpSession = null;
+
+  try {
+    // Create CDP session - this connects to Chrome's internal debugging interface
+    cdpSession = await page.target().createCDPSession();
+    
+    // Enable network domain - required for network event monitoring  
+    await cdpSession.send('Network.enable');
+    
+    // Set up network request monitoring
+    // This captures ALL network requests at the browser engine level
+    cdpSession.on('Network.requestWillBeSent', (params) => {
+      const { url: requestUrl, method } = params.request;
+      const initiator = params.initiator ? params.initiator.type : 'unknown';
+      
+      // Extract hostname for logging context (handles URL parsing errors gracefully)
+      let hostnameForLog = 'unknown-host';
+      try {
+        hostnameForLog = new URL(currentUrl).hostname;
+      } catch (_) { 
+        // Ignore URL parsing errors for logging context
+      }
+      
+      // Log the request with context - customize this for your needs
+      // Format: [cdp][hostname] METHOD url (initiator: type)
+      console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
+    });
+
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `CDP session created successfully for ${currentUrl}`));
+    }
+
+    return {
+      session: cdpSession,
+      cleanup: async () => {
+        // Safe cleanup that never throws errors
+        if (cdpSession) {
+          try {
+            await cdpSession.detach();
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `CDP session detached for ${currentUrl}`));
+            }
+          } catch (cdpCleanupErr) {
+            // Log cleanup errors but don't throw - cleanup should never fail the calling code
+            if (forceDebug) {
+              console.log(formatLogMessage('debug', `Failed to detach CDP session for ${currentUrl}: ${cdpCleanupErr.message}`));
+            }
+          }
+        }
+      }
+    };
+
+  } catch (cdpErr) {
+    cdpSession = null; // Reset on failure
+    
+    // Categorize CDP errors for proper handling
+    if (cdpErr.message.includes('Network.enable timed out') || 
+        cdpErr.message.includes('Protocol error')) {
+      // CRITICAL ERROR: Browser is broken and needs restart
+      // Re-throw these errors so calling code can handle browser restart
+      throw new Error(`Browser protocol broken: ${cdpErr.message}`);
+    }
+    
+    // NON-CRITICAL ERROR: CDP failed but browser is still usable
+    // Log warning but return working session object
+    console.warn(formatLogMessage('warn', `[cdp] Failed to attach CDP session for ${currentUrl}: ${cdpErr.message}`));
+    
+    // Return null session with no-op cleanup for consistent API
+    return {
+      session: null,
+      cleanup: async () => {}
+    };
+  }
+}
+
+/**
+ * Validates CDP availability and configuration
+ * 
+ * USAGE IN YOUR APPLICATION:
+ *   const validation = validateCDPConfig(siteConfig, globalCDPFlag);
+ *   if (!validation.isValid) {
+ *     console.warn('CDP configuration issues detected');
+ *   }
+ *   validation.recommendations.forEach(rec => console.log('Recommendation:', rec));
+ *
+ * @param {object} siteConfig - Site configuration object
+ * @param {boolean} globalCDP - Global CDP flag
+ * @returns {object} Validation result with recommendations
+ */
+function validateCDPConfig(siteConfig, globalCDP) {
+  const warnings = [];
+  const recommendations = [];
+  
+  // Check for conflicting configurations
+  if (globalCDP && siteConfig.cdp === false) {
+    warnings.push('Site-specific CDP disabled but global CDP is enabled - global setting will override');
+  }
+  
+  // Performance recommendations
+  if (globalCDP || siteConfig.cdp === true) {
+    recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
+    
+    if (siteConfig.timeout && siteConfig.timeout < 30000) {
+      recommendations.push('Consider increasing timeout when using CDP logging to avoid protocol timeouts');
+    }
+  }
+  
+  return {
+    isValid: true,
+    warnings,
+    recommendations
+  };
+}
+
+/**
+ * Enhanced CDP session with additional network monitoring features
+ * 
+ * ADVANCED FEATURES:
+ *   - JavaScript exception monitoring
+ *   - Security state change detection  
+ *   - Failed network request tracking
+ *   - Enhanced error reporting
+ *
+ * USE CASES:
+ *   - Security analysis requiring comprehensive monitoring
+ *   - Debugging complex single-page applications
+ *   - Performance analysis of web applications
+ *   - Research requiring detailed browser insights
+ *
+ * PERFORMANCE IMPACT:
+ *   - Adds additional CDP domain subscriptions
+ *   - Higher memory usage due to more event listeners
+ *   - Recommended only for detailed analysis scenarios
+ *
+ * @param {import('puppeteer').Page} page - The Puppeteer page instance
+ * @param {string} currentUrl - The URL being processed
+ * @param {object} options - Configuration options (same as createCDPSession)
+ * @returns {Promise<object>} Enhanced CDP session object with isEnhanced flag
+ */
+async function createEnhancedCDPSession(page, currentUrl, options = {}) {
+  const basicSession = await createCDPSession(page, currentUrl, options);
+  
+  if (!basicSession.session) {
+    return basicSession;
+  }
+
+  const { session } = basicSession;
+  const { forceDebug } = options;
+
+  try {
+    // Enable additional CDP domains for enhanced monitoring
+    await session.send('Runtime.enable');  // For JavaScript exceptions
+    await session.send('Security.enable'); // For security state changes
+    
+    // Monitor JavaScript exceptions - useful for debugging problematic sites
+    session.on('Runtime.exceptionThrown', (params) => {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[cdp][exception] ${params.exceptionDetails.text}`));
+      }
+    });
+
+    // Monitor security state changes - detect mixed content, certificate issues, etc.
+    session.on('Security.securityStateChanged', (params) => {
+      if (forceDebug && params.securityState !== 'secure') {
+        console.log(formatLogMessage('debug', `[cdp][security] Security state: ${params.securityState}`));
+      }
+    });
+
+    // Monitor failed network requests - useful for understanding site issues
+    session.on('Network.loadingFailed', (params) => {
+      if (forceDebug) {
+        console.log(formatLogMessage('debug', `[cdp][failed] ${params.errorText}: ${params.requestId}`));
+      }
+    });
+
+    return {
+      session,
+      cleanup: basicSession.cleanup,
+      isEnhanced: true // Flag to indicate enhanced features are active
+    };
+
+  } catch (enhancedErr) {
+    if (forceDebug) {
+      console.log(formatLogMessage('debug', `Enhanced CDP features failed, falling back to basic session: ${enhancedErr.message}`));
+    }
+    
+    // Graceful degradation: return basic session if enhanced features fail
+    // This ensures your application continues working even if advanced features break
+    return basicSession;
+  }
+}
+
+// EXPORT INTERFACE FOR OTHER APPLICATIONS:
+// This module provides a clean, reusable interface for CDP integration.
+// Simply require this module and use the exported functions.
+//
+// CUSTOMIZATION TIPS:
+// 1. Replace './colorize' import with your own logging system
+// 2. Modify the request logging format in the Network.requestWillBeSent handler
+// 3. Add additional CDP domain subscriptions in createEnhancedCDPSession
+// 4. Customize error categorization in the catch blocks
+//
+// TROUBLESHOOTING:
+// - If you get "Protocol error" frequently, the browser may be overloaded
+// - Timeout errors usually indicate the browser needs to be restarted
+// - "Target closed" means the page was closed while CDP was active
+//
+// BROWSER COMPATIBILITY:
+// - Chrome/Chromium 60+ (older versions may have limited CDP support)
+// - Works in both headless and headed modes
+// - Some features may not work in --no-sandbox mode
+module.exports = {
+  createCDPSession,
+  validateCDPConfig,
+  createEnhancedCDPSession
+};

package/nwss.js

@@ -1,4 +1,4 @@
-// === Network scanner script (nwss.js) v1.0.46 ===
+// === Network scanner script (nwss.js) v1.0.48 ===
 
 // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
 // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -25,6 +25,8 @@ const { handleBrowserExit, cleanupChromeTempFiles } = require('./lib/browserexit
 const { createNetToolsHandler, createEnhancedDryRunCallback, validateWhoisAvailability, validateDigAvailability } = require('./lib/nettools');
 // File compare
 const { loadComparisonRules, filterUniqueRules } = require('./lib/compare');
+// CDP functionality
+const { createCDPSession } = require('./lib/cdp');
 // Colorize various text when used
 const { colorize, colors, messageColors, tags, formatLogMessage } = require('./lib/colorize');
 // Enhanced mouse interaction and page simulation
@@ -37,7 +39,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
 const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
 
 // --- Script Configuration & Constants ---
-const VERSION = '1.0.46'; // Script version
+const VERSION = '1.0.48'; // Script version
 
 // get startTime
 const startTime = Date.now();
@@ -1149,6 +1151,7 @@ function setupFrameHandling(page, forceDebug) {
 
     let page = null;
     let cdpSession = null;
+    let cdpSessionManager = null;
     // Use Map to track domains and their resource types for --adblock-rules or --dry-run
     const matchedDomains = (adblockRulesMode || siteConfig.adblock_rules || dryRunMode) ? new Map() : new Set();
     
@@ -1274,36 +1277,19 @@ function setupFrameHandling(page, forceDebug) {
       // --- END: CSS Element Blocking Setup ---
 
       // --- Per-Page CDP Setup ---
-      const cdpLoggingNeededForPage = enableCDP || siteConfig.cdp === true;
-      if (cdpLoggingNeededForPage) {
-        if (forceDebug) {
-            if (enableCDP) {
-                console.log(formatLogMessage('debug', `CDP logging globally enabled by --cdp, applying to page: ${currentUrl}`));
-            } else if (siteConfig.cdp === true) {
-                console.log(formatLogMessage('debug', `CDP logging enabled for page ${currentUrl} via site-specific 'cdp: true' config.`));
-            }
-        }
-        try {
-            cdpSession = await page.target().createCDPSession();
-            await cdpSession.send('Network.enable');
-            cdpSession.on('Network.requestWillBeSent', (params) => {
-                const { url: requestUrl, method } = params.request;
-                const initiator = params.initiator ? params.initiator.type : 'unknown';
-                let hostnameForLog = 'unknown-host';
-                try {
-                    hostnameForLog = new URL(currentUrl).hostname;
-                } catch (_) { /* ignore if currentUrl is invalid for URL parsing */ }
-                console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
-            });
-        } catch (cdpErr) {
-            cdpSession = null; // Reset on failure
-            if (cdpErr.message.includes('Network.enable timed out') || 
-                cdpErr.message.includes('Protocol error')) {
-              // This indicates browser is completely broken
-              throw new Error(`Browser protocol broken: ${cdpErr.message}`);
-            }
-            console.warn(formatLogMessage('warn', `[cdp] Failed to attach CDP session for ${currentUrl}: ${cdpErr.message}`));
+      
+      try {
+        cdpSessionManager = await createCDPSession(page, currentUrl, {
+          enableCDP,
+          siteSpecificCDP: siteConfig.cdp,
+          forceDebug
+        });
+      } catch (cdpErr) {
+        if (cdpErr.message.includes('Browser protocol broken')) {
+          throw cdpErr; // Re-throw critical browser errors
         }
+        // Non-critical CDP errors are already handled in the module
+        cdpSessionManager = { session: null, cleanup: async () => {} };
       }
       // --- End of Per-Page CDP Setup ---
 
@@ -2267,19 +2253,8 @@ function setupFrameHandling(page, forceDebug) {
     } finally {
       // Guaranteed resource cleanup - this runs regardless of success or failure
       
-      if (cdpSession) {
-        try {
-          await cdpSession.detach();
-          if (forceDebug) console.log(formatLogMessage('debug', `CDP session detached for ${currentUrl}`));
-        } catch (cdpCleanupErr) {
-          if (forceDebug) console.log(formatLogMessage('debug', `Failed to detach CDP session for ${currentUrl}: ${cdpCleanupErr.message}`));
-        }
-      }
-      // Add small delay to allow cleanup to complete
-      try {
-        await new Promise(resolve => setTimeout(resolve, 100));
-      } catch (delayErr) {
-        // Ignore timeout errors
+      if (cdpSessionManager) {
+        await cdpSessionManager.cleanup();
       }
       
       if (page && !page.isClosed()) {

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@fanboynz/network-scanner",
-  "version": "1.0.46",
+  "version": "1.0.48",
   "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
   "main": "nwss.js",
   "scripts": {
     "start": "node nwss.js",
     "scan": "node nwss.js",
     "help": "node nwss.js --help",
-    "version": "node nwss.js --version"
+    "version": "node nwss.js --version",
+    "lint": "eslint *.js lib/*.js"
   },
   "dependencies": {
     "p-limit": "^4.0.0",
@@ -41,5 +42,9 @@
   "bugs": {
     "url": "https://github.com/ryanbr/network-scanner/issues"
   },
-  "homepage": "https://github.com/ryanbr/network-scanner"
+  "homepage": "https://github.com/ryanbr/network-scanner",
+  "devDependencies": {
+    "eslint": "^9.32.0",
+    "globals": "^16.3.0"
+  }
 }