crawlforge-mcp-server 5.5.4 → 5.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.5.4",
+  "version": "5.5.6",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.5.1",
+    "crawlforge-extractors": "^1.5.3",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.5.4",
+  version: "5.5.6",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/setup.js

@@ -68,8 +68,11 @@ function addToMcpConfig(configPath, clientName, apiKey) {
       }
     };
 
-    // Write updated config
-    fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+    // Write updated config. This file now embeds the API key, so keep it
+    // owner-only (0o600). The `mode` option only applies when writeFileSync
+    // creates the file, so chmod enforces it for a pre-existing config too.
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2), { mode: 0o600 });
+    fs.chmodSync(configPath, 0o600);
 
     return {
       success: true,
@@ -107,7 +110,7 @@ function configureMcpClients(apiKey) {
     if (!fs.existsSync(cursorConfigPath)) {
       // Create mcp.json if it doesn't exist
       try {
-        fs.writeFileSync(cursorConfigPath, JSON.stringify({ mcpServers: {} }, null, 2));
+        fs.writeFileSync(cursorConfigPath, JSON.stringify({ mcpServers: {} }, null, 2), { mode: 0o600 });
       } catch (e) {
         // Ignore creation errors
       }

package/src/core/ResearchOrchestrator.js

@@ -8,7 +8,7 @@ import { ResultRanker } from '../tools/search/ranking/ResultRanker.js';
 import { CacheManager } from './cache/CacheManager.js';
 import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
-import { safeFetch } from '../utils/ssrfGuard.js';
+import { safeFetch, safeGoto } from '../utils/ssrfGuard.js';
 import { preflightFetch, browserPreflight } from '../utils/robotsGate.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import {
@@ -989,7 +989,9 @@ export class ResearchOrchestrator extends EventEmitter {
       page = await this._stealthManager.createStealthPage(contextId);
     }
     try {
-      const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs });
+      // browserPreflight (robots + platform blocklist) ran in _stealthFetchHtml;
+      // safeGoto adds the SSRF/DNS-rebinding guard the browser path also needs.
+      const resp = await safeGoto(page, url, { waitUntil: 'domcontentloaded', timeout: this.stealthTimeoutMs });
       // Do NOT bail on the initial HTTP status: anti-bot challenges (Cloudflare
       // Turnstile) return 403 on the first response and only resolve to the
       // real page after their JS runs. Let it settle, then judge by the

package/src/core/StealthBrowserManager.js

@@ -13,6 +13,7 @@ import { z } from 'zod';
 import crypto from 'crypto';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
+import { safeGoto } from '../utils/ssrfGuard.js';
 
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
@@ -1776,7 +1777,9 @@ export class StealthBrowserManager {
     const { contextId } = await this.createStealthContext({ ...stealthConfig, engine });
     try {
       const page = await this.createStealthPage(contextId);
-      await page.goto(url, { waitUntil: 'domcontentloaded' });
+      // SSRF guard at the navigation boundary: the stealth engine resolves DNS
+      // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
+      await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
       if (wait_for > 0) await page.waitForTimeout(wait_for);
 
       const [title, html, text] = await Promise.all([

package/src/core/creatorMode.js

@@ -10,8 +10,21 @@
 import crypto from 'crypto';
 import dotenv from 'dotenv';
 
-// Load .env file early to check for creator secret
-dotenv.config({ path: '.env', quiet: true });
+// Load the cwd .env early to check for the creator secret, but into an isolated
+// object rather than straight into process.env. A .env sitting in whatever
+// directory the server is launched from is untrusted; seeding process.env from
+// it directly would let it silently flip the SSRF controls (this module is
+// imported before config.js runs its package-relative .env load). We copy the
+// benign keys through — preserving the historical "cwd .env is read" behavior,
+// including the creator secret — but never the SSRF/allowlist controls, which
+// must come from the real environment.
+const CWD_ENV_DENYLIST = new Set(['SSRF_PROTECTION_ENABLED', 'SSRF_STRICT', 'ALLOWED_DOMAINS']);
+const cwdEnv = {};
+dotenv.config({ path: '.env', quiet: true, processEnv: cwdEnv });
+for (const [key, value] of Object.entries(cwdEnv)) {
+  if (CWD_ENV_DENYLIST.has(key)) continue;
+  if (process.env[key] === undefined) process.env[key] = value; // don't override the real env
+}
 
 // SECURITY: Clear any externally-set creator mode env var to prevent bypass
 delete process.env.CRAWLFORGE_CREATOR_MODE;

package/src/core/processing/BrowserProcessor.js

@@ -9,6 +9,7 @@ import { z } from 'zod';
 import StealthBrowserManager from '../StealthBrowserManager.js';
 import HumanBehaviorSimulator from '../../utils/HumanBehaviorSimulator.js';
 import LocalizationManager from '../LocalizationManager.js';
+import { safeGoto } from '../../utils/ssrfGuard.js';
 
 const BrowserProcessorSchema = z.object({
   url: z.string().url(),
@@ -613,8 +614,10 @@ export class BrowserProcessor {
   async navigateAndWait(page, url, options) {
     const startTime = Date.now();
 
-    // Navigate to URL
-    await page.goto(url, {
+    // Navigate to URL. safeGoto adds the SSRF/DNS-rebinding guard on top of the
+    // robots/blocklist preflight the calling tool already ran, and re-checks the
+    // landed URL so a redirect can't carry us into a private/metadata range.
+    await safeGoto(page, url, {
       waitUntil: 'domcontentloaded',
       timeout: 30000
     });

package/src/server/transports/streamableHttp.js

@@ -151,7 +151,14 @@ function sendRpcError(res, status, code, message) {
  */
 export async function connectStreamableHttp(server, authManager, logger, options = {}) {
   const port = options.port ?? 3000;
-  const host = options.host ?? '0.0.0.0';
+  // Bind loopback by default so a local `--http` server is not exposed to the
+  // LAN. Managed hosts need 0.0.0.0 to be reachable by their router: Render sets
+  // $RENDER; $MCP_HTTP_HOST is an explicit override for any other platform.
+  const host = options.host ?? process.env.MCP_HTTP_HOST ?? (process.env.RENDER ? '0.0.0.0' : '127.0.0.1');
+  const hostIsLoopback = host === '127.0.0.1' || host === '::1' || host === 'localhost';
+  if (authManager.isCreatorMode() && !hostIsLoopback) {
+    console.error(`WARNING: creator mode is enabled but the server is bound to ${host} (non-loopback) — per-request auth will NOT be bypassed. Bind to 127.0.0.1 to use creator mode.`);
+  }
   const legacy = options.legacy === true;
   const oauthProvider = options.oauth ?? null;
   const metrics = options.metrics ?? null;
@@ -242,12 +249,14 @@ export async function connectStreamableHttp(server, authManager, logger, options
 
     // MCP endpoint
     if (req.url === '/mcp' || req.url === '/' || req.url?.startsWith('/mcp?')) {
-      // Per-request auth (bypassed in creator mode). `internal` marks a
-      // request from the website's REST proxy (INTERNAL_PROXY_SECRET): it is
-      // billing-exempt in withAuth because the website already charged the
-      // end user. Request-scoped only — never persisted on the session.
+      // Per-request auth. Creator mode bypasses it, but only on a loopback bind
+      // — never expose an unauthenticated MCP endpoint on a public interface.
+      // `internal` marks a request from the website's REST proxy
+      // (INTERNAL_PROXY_SECRET): it is billing-exempt in withAuth because the
+      // website already charged the end user. Request-scoped only — never
+      // persisted on the session.
       let internal = false;
-      if (!authManager.isCreatorMode()) {
+      if (!(authManager.isCreatorMode() && hostIsLoopback)) {
         const authResult = await authenticateRequest(req, authManager, oauthProvider);
         if (!authResult.ok) {
           logger.warn('Streamable HTTP request rejected', {

package/src/utils/ssrfGuard.js

@@ -323,6 +323,30 @@ export async function safeFetch(url, options = {}) {
   }
 }
 
+/**
+ * SSRF-safe wrapper around Playwright's `page.goto`. The navigate analog of
+ * safeFetch: a browser drives its own DNS resolution, so URL/host checks alone
+ * miss DNS-rebinding and private-IP targets — resolveDns:true closes that.
+ * Checks the URL before navigating, then re-checks the landed URL, because a
+ * redirect during navigation could carry us into a blocked range even when the
+ * original URL was safe. Returns the Response so callers can read the status.
+ * Options pass through unchanged; behaviour-preserving for allowed URLs.
+ *
+ * @param {import('playwright').Page} page
+ * @param {string} url
+ * @param {Parameters<import('playwright').Page['goto']>[1]} [options]
+ * @returns {Promise<import('playwright').Response|null>}
+ */
+export async function safeGoto(page, url, options = {}) {
+  await assertUrlAllowed(url, { resolveDns: true });
+  const response = await page.goto(url, options);
+  const landedUrl = page.url();
+  if (/^https?:\/\//i.test(landedUrl)) {
+    await assertUrlAllowed(landedUrl, { resolveDns: true });
+  }
+  return response;
+}
+
 // Exposed for unit tests.
 export const __ssrfInternals = {
   ssrfLookup,

package/src/security/security-patches.js

@@ -1,120 +0,0 @@
-/**
- * Critical Security Patches for Wave 3 Features
- * 
- * This file contains emergency patches for critical vulnerabilities
- * identified in the security audit. Apply these patches immediately
- * before any production deployment.
- */
-
-import Wave3Security from './wave3-security.js';
-const { SSRFProtection, PathSecurity, InputSecurity, CryptoSecurity, BrowserSecurity } = Wave3Security;
-
-/**
- * CRITICAL PATCH 1: SSRF Protection for Research Tool
- * Fixes: CVE-001 - Server-Side Request Forgery
- */
-export function patchResearchOrchestrator(ResearchOrchestrator) {
-  const originalConductResearch = ResearchOrchestrator.prototype.conductResearch;
-  
-  ResearchOrchestrator.prototype.conductResearch = async function(topic, options = {}) {
-    // Input validation
-    if (typeof topic !== 'string' || topic.length < 3 || topic.length > 500) {
-      throw new Error('Invalid topic: must be string between 3-500 characters');
-    }
-    
-    // Sanitize topic to prevent script injection
-    const sanitizedTopic = InputSecurity.sanitizeString(topic);
-    
-    // Validate and limit research parameters
-    const secureOptions = {
-      ...options,
-      maxUrls: Math.min(options.maxUrls || 10, 50),
-      timeLimit: Math.min(options.timeLimit || 60000, 120000),
-      maxDepth: Math.min(Math.max(options.maxDepth || 3, 1), 5),
-      concurrency: Math.min(options.concurrency || 3, 5)
-    };
-    
-    return originalConductResearch.call(this, sanitizedTopic, secureOptions);
-  };
-}
-
-/**
- * CRITICAL PATCH 2: Browser Security Hardening
- * Fixes: CVE-002, CVE-007 - Script Injection and Browser Security
- */
-export function patchStealthBrowserManager(StealthBrowserManager) {
-  // Override dangerous browser args
-  const originalLaunchStealthBrowser = StealthBrowserManager.prototype.launchStealthBrowser;
-  
-  StealthBrowserManager.prototype.launchStealthBrowser = async function(config = {}) {
-    // Force secure browser configuration
-    const secureConfig = {
-      ...config,
-      level: config.level === 'advanced' ? 'medium' : config.level
-    };
-    
-    // Override with secure browser args
-    this.secureArgs = BrowserSecurity.getSecureBrowserArgs();
-    
-    return originalLaunchStealthBrowser.call(this, secureConfig);
-  };
-}
-
-/**
- * CRITICAL PATCH 3: Path Traversal Protection
- * Fixes: CVE-003 - Arbitrary File System Access
- */
-export function patchSnapshotManager(SnapshotManager) {
-  // Secure file operations
-  const originalWriteSnapshotFile = SnapshotManager.prototype.writeSnapshotFile;
-  
-  SnapshotManager.prototype.writeSnapshotFile = async function(snapshotId, content) {
-    // Validate snapshot ID
-    const safeSnapshotId = PathSecurity.validateSnapshotId(snapshotId);
-    
-    // Validate content size
-    InputSecurity.validateContentSize(content);
-    
-    // Ensure safe file path
-    const safePath = PathSecurity.sanitizePath(`${safeSnapshotId}.snap`, this.options.storageDir);
-    
-    return originalWriteSnapshotFile.call(this, safeSnapshotId, content);
-  };
-  
-  // Override generateSnapshotId to use cryptographically secure generation
-  SnapshotManager.prototype.generateSnapshotId = function(url, timestamp) {
-    return CryptoSecurity.generateSecureId(16);
-  };
-}
-
-/**
- * Apply all critical patches
- */
-export function applyAllSecurityPatches(components) {
-  console.log('🔒 Applying Wave 3 security patches...');
-  
-  if (components.ResearchOrchestrator) {
-    patchResearchOrchestrator(components.ResearchOrchestrator);
-    console.log('✅ ResearchOrchestrator patched');
-  }
-  
-  if (components.StealthBrowserManager) {
-    patchStealthBrowserManager(components.StealthBrowserManager);
-    console.log('✅ StealthBrowserManager patched');  
-  }
-  
-  if (components.SnapshotManager) {
-    patchSnapshotManager(components.SnapshotManager);
-    console.log('✅ SnapshotManager patched');
-  }
-  
-  console.log('🛡️ All Wave 3 security patches applied successfully');
-  console.log('⚠️  Remember to run security tests before deployment');
-}
-
-export default {
-  patchResearchOrchestrator,
-  patchStealthBrowserManager, 
-  patchSnapshotManager,
-  applyAllSecurityPatches
-};

package/src/security/security-tests.js

@@ -1,355 +0,0 @@
-/**
- * Wave 3 Security Test Suite
- * Validates security patches and tests for vulnerabilities
- */
-
-import Wave3Security from './wave3-security.js';
-const { SecurityTesting, SSRFProtection, PathSecurity, InputSecurity } = Wave3Security;
-
-/**
- * Test SSRF Protection
- */
-export async function testSSRFProtection() {
-  console.log('🧪 Testing SSRF Protection...');
-  
-  const maliciousUrls = [
-    'http://localhost:3000/admin',
-    'http://127.0.0.1:22', 
-    'http://169.254.169.254/latest/meta-data/',
-    'file:///etc/passwd',
-    'ftp://internal.server.com',
-    'http://10.0.0.1/secret',
-    'https://metadata.google.internal/'
-  ];
-  
-  const results = SecurityTesting.testSecurityFunction(
-    SSRFProtection.validateUrl.bind(SSRFProtection),
-    maliciousUrls
-  );
-  
-  const blocked = results.filter(r => r.blocked).length;
-  const total = results.length;
-  
-  console.log(`✅ SSRF Test: ${blocked}/${total} malicious URLs blocked`);
-  
-  if (blocked !== total) {
-    console.error('❌ SSRF Protection insufficient!');
-    results.filter(r => !r.blocked).forEach(r => {
-      console.error(`  🚨 Not blocked: ${r.payload}`);
-    });
-  }
-  
-  return { passed: blocked === total, blocked, total };
-}
-
-/**
- * Test Path Traversal Protection  
- */
-export function testPathTraversal() {
-  console.log('🧪 Testing Path Traversal Protection...');
-  
-  const maliciousPaths = [
-    '../../../etc/passwd',
-    '..\\..\\..\\windows\\system32\\config\\sam',
-    '/etc/shadow', 
-    'C:\\Windows\\System32\\drivers\\etc\\hosts',
-    '....//....//....//etc/passwd',
-    '../../../../../../../etc/passwd',
-    '..\\..\\..\\..\\..\\..\\/etc/passwd'
-  ];
-  
-  const results = SecurityTesting.testSecurityFunction(
-    PathSecurity.validateSnapshotId.bind(PathSecurity),
-    maliciousPaths
-  );
-  
-  const blocked = results.filter(r => r.blocked).length;
-  const total = results.length;
-  
-  console.log(`✅ Path Traversal Test: ${blocked}/${total} malicious paths blocked`);
-  
-  if (blocked !== total) {
-    console.error('❌ Path Traversal Protection insufficient!');
-    results.filter(r => !r.blocked).forEach(r => {
-      console.error(`  🚨 Not blocked: ${r.payload}`);
-    });
-  }
-  
-  return { passed: blocked === total, blocked, total };
-}
-
-/**
- * Test Input Validation
- */
-export function testInputValidation() {
-  console.log('🧪 Testing Input Validation...');
-  
-  const maliciousInputs = [
-    '<script>alert("XSS")</script>',
-    'javascript:alert("XSS")',
-    '<img src="x" onerror="alert(\'XSS\')">',
-    '<svg onload="alert(\'XSS\')"></svg>',
-    '${7*7}',
-    '#{7*7}',
-    '{{7*7}}',
-    '<%= 7*7 %>'
-  ];
-  
-  const results = maliciousInputs.map(input => {
-    try {
-      const sanitized = InputSecurity.sanitizeHtml(input);
-      const isBlocked = !sanitized.includes('<script') && 
-                       !sanitized.includes('javascript:') &&
-                       !sanitized.includes('onerror=') &&
-                       !sanitized.includes('onload=');
-      return { input, blocked: isBlocked, sanitized };
-    } catch (error) {
-      return { input, blocked: true, error: error.message };
-    }
-  });
-  
-  const blocked = results.filter(r => r.blocked).length;
-  const total = results.length;
-  
-  console.log(`✅ Input Validation Test: ${blocked}/${total} malicious inputs neutralized`);
-  
-  return { passed: blocked === total, blocked, total };
-}
-
-/**
- * Test Resource Limits
- */
-export function testResourceLimits() {
-  console.log('🧪 Testing Resource Limits...');
-  
-  const tests = [
-    {
-      name: 'Large Content',
-      test: () => {
-        const largeContent = 'x'.repeat(100 * 1024 * 1024); // 100MB
-        try {
-          InputSecurity.validateContentSize(largeContent);
-          return false; // Should have thrown
-        } catch (error) {
-          return error.message.includes('Content too large');
-        }
-      }
-    },
-    {
-      name: 'Long String',
-      test: () => {
-        const longString = 'x'.repeat(20000);
-        try {
-          InputSecurity.sanitizeString(longString, 10000);
-          return false; // Should have thrown
-        } catch (error) {
-          return error.message.includes('String too long');
-        }
-      }
-    },
-    {
-      name: 'Large Array',
-      test: () => {
-        const largeArray = new Array(200).fill('item');
-        try {
-          InputSecurity.sanitizeArray(largeArray, 100);
-          return false; // Should have thrown
-        } catch (error) {
-          return error.message.includes('Array too long');
-        }
-      }
-    }
-  ];
-  
-  let passed = 0;
-  tests.forEach(test => {
-    if (test.test()) {
-      console.log(`  ✅ ${test.name}: PASS`);
-      passed++;
-    } else {
-      console.log(`  ❌ ${test.name}: FAIL`);
-    }
-  });
-  
-  console.log(`✅ Resource Limits Test: ${passed}/${tests.length} tests passed`);
-  
-  return { passed: passed === tests.length, passedCount: passed, total: tests.length };
-}
-
-/**
- * Test Cryptographic Functions
- */
-export function testCryptographicSecurity() {
-  console.log('🧪 Testing Cryptographic Security...');
-  
-  const tests = [
-    {
-      name: 'Secure ID Generation',
-      test: () => {
-        const id1 = Wave3Security.CryptoSecurity.generateSecureId();
-        const id2 = Wave3Security.CryptoSecurity.generateSecureId();
-        
-        // Should be different
-        if (id1 === id2) return false;
-        
-        // Should be hex and right length
-        if (!/^[a-f0-9]{64}$/.test(id1)) return false;
-        if (!/^[a-f0-9]{64}$/.test(id2)) return false;
-        
-        return true;
-      }
-    },
-    {
-      name: 'Timing Safe Comparison',
-      test: () => {
-        const hash1 = 'abcd1234';
-        const hash2 = 'abcd1234';
-        const hash3 = 'efgh5678';
-        
-        // Same hashes should match
-        if (!Wave3Security.CryptoSecurity.timingSafeCompare(hash1, hash2)) return false;
-        
-        // Different hashes should not match
-        if (Wave3Security.CryptoSecurity.timingSafeCompare(hash1, hash3)) return false;
-        
-        return true;
-      }
-    },
-    {
-      name: 'Webhook Signature',
-      test: () => {
-        const payload = JSON.stringify({ test: 'data' });
-        const secret = 'test-secret';
-        
-        const signature1 = Wave3Security.CryptoSecurity.generateWebhookSignature(payload, secret);
-        const signature2 = Wave3Security.CryptoSecurity.generateWebhookSignature(payload, secret);
-        
-        // Should be consistent
-        if (signature1 !== signature2) return false;
-        
-        // Should validate correctly
-        if (!Wave3Security.CryptoSecurity.validateWebhookSignature(payload, signature1, secret)) return false;
-        
-        // Should reject wrong signature
-        if (Wave3Security.CryptoSecurity.validateWebhookSignature(payload, 'wrong', secret)) return false;
-        
-        return true;
-      }
-    }
-  ];
-  
-  let passed = 0;
-  tests.forEach(test => {
-    if (test.test()) {
-      console.log(`  ✅ ${test.name}: PASS`);
-      passed++;
-    } else {
-      console.log(`  ❌ ${test.name}: FAIL`);
-    }
-  });
-  
-  console.log(`✅ Cryptographic Security Test: ${passed}/${tests.length} tests passed`);
-  
-  return { passed: passed === tests.length, passedCount: passed, total: tests.length };
-}
-
-/**
- * Run Full Security Test Suite
- */
-export async function runFullSecurityTests() {
-  console.log('🛡️  Running Wave 3 Security Test Suite...\n');
-  
-  const results = [];
-  
-  // Test SSRF Protection
-  results.push(await testSSRFProtection());
-  console.log('');
-  
-  // Test Path Traversal Protection
-  results.push(testPathTraversal());
-  console.log('');
-  
-  // Test Input Validation
-  results.push(testInputValidation());
-  console.log('');
-  
-  // Test Resource Limits
-  results.push(testResourceLimits());
-  console.log('');
-  
-  // Test Cryptographic Security
-  results.push(testCryptographicSecurity());
-  console.log('');
-  
-  // Summary
-  const totalPassed = results.filter(r => r.passed).length;
-  const totalTests = results.length;
-  
-  console.log('🎯 Security Test Summary:');
-  console.log(`   Passed: ${totalPassed}/${totalTests} test suites`);
-  
-  if (totalPassed === totalTests) {
-    console.log('✅ ALL SECURITY TESTS PASSED! Wave 3 appears secure.');
-  } else {
-    console.log('❌ SECURITY TESTS FAILED! Do not deploy to production.');
-    console.log('   Review failed tests and apply additional patches.');
-  }
-  
-  return {
-    allPassed: totalPassed === totalTests,
-    passed: totalPassed,
-    total: totalTests,
-    results
-  };
-}
-
-/**
- * Quick Security Health Check
- */
-export function quickSecurityCheck() {
-  console.log('🚀 Quick Security Health Check...');
-  
-  const checks = [
-    {
-      name: 'SSRF Protection Available',
-      check: () => typeof SSRFProtection.validateUrl === 'function'
-    },
-    {
-      name: 'Path Security Available', 
-      check: () => typeof PathSecurity.validateSnapshotId === 'function'
-    },
-    {
-      name: 'Input Sanitization Available',
-      check: () => typeof InputSecurity.sanitizeString === 'function'
-    },
-    {
-      name: 'Crypto Security Available',
-      check: () => typeof Wave3Security.CryptoSecurity.generateSecureId === 'function'
-    }
-  ];
-  
-  let passed = 0;
-  checks.forEach(check => {
-    if (check.check()) {
-      console.log(`  ✅ ${check.name}`);
-      passed++;
-    } else {
-      console.log(`  ❌ ${check.name}`);
-    }
-  });
-  
-  console.log(`\n🎯 Health Check: ${passed}/${checks.length} security functions available`);
-  
-  return passed === checks.length;
-}
-
-// Export test functions
-export default {
-  testSSRFProtection,
-  testPathTraversal,
-  testInputValidation,
-  testResourceLimits,
-  testCryptographicSecurity,
-  runFullSecurityTests,
-  quickSecurityCheck
-};

package/src/security/wave3-security.js

@@ -1,653 +0,0 @@
-/**
- * Wave 3 Security Validation Utilities
- * Provides security validation, sanitization, and protection functions
- * for Wave 3 features in the CrawlForge MCP Server project.
- */
-
-import { z } from 'zod';
-import { createHash, randomBytes, timingSafeEqual } from 'crypto';
-import path from 'path';
-import { URL } from 'url';
-import DOMPurify from 'isomorphic-dompurify';
-import { identityHeaders } from '../utils/fetchIdentity.js';
-
-// Security configuration
-const SECURITY_CONFIG = {
-  // SSRF Protection
-  allowedDomains: [
-    'google.com',
-    'bing.com',
-    'duckduckgo.com',
-    'wikipedia.org',
-    'archive.org',
-    '*.edu',
-    '*.gov'
-  ],
-  
-  blockedDomains: [
-    'localhost',
-    '127.0.0.1',
-    '0.0.0.0',
-    '10.*',
-    '172.16.*',
-    '192.168.*',
-    '169.254.*', // AWS metadata
-    'metadata.google.internal'
-  ],
-  
-  // Resource limits
-  maxContentSize: 50 * 1024 * 1024, // 50MB
-  maxSnapshotSize: 100 * 1024 * 1024, // 100MB  
-  maxResearchUrls: 50,
-  maxResearchTime: 120000, // 2 minutes
-  
-  // Path validation
-  allowedDirectories: [
-    './snapshots',
-    './cache',
-    './temp'
-  ],
-  
-  // Content validation
-  maxStringLength: 10000,
-  maxArrayLength: 100,
-  
-  // Rate limiting
-  defaultRateLimit: {
-    windowMs: 60 * 1000, // 1 minute
-    max: 100 // requests per window
-  }
-};
-
-/**
- * SSRF Protection Functions
- */
-export class SSRFProtection {
-  static validateUrl(url) {
-    try {
-      const parsedUrl = new URL(url);
-      
-      // Check protocol
-      if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
-        throw new Error('Invalid protocol. Only HTTP and HTTPS are allowed.');
-      }
-      
-      // Check for blocked domains
-      const hostname = parsedUrl.hostname.toLowerCase();
-      for (const blocked of SECURITY_CONFIG.blockedDomains) {
-        if (blocked.includes('*')) {
-          const pattern = blocked.replace('*', '.*');
-          if (new RegExp(pattern).test(hostname)) {
-            throw new Error(`Blocked domain: ${hostname}`);
-          }
-        } else if (hostname === blocked || hostname.endsWith(`.${blocked}`)) {
-          throw new Error(`Blocked domain: ${hostname}`);
-        }
-      }
-      
-      // Check for allowed domains (if whitelist is used)
-      if (SECURITY_CONFIG.allowedDomains.length > 0) {
-        let isAllowed = false;
-        for (const allowed of SECURITY_CONFIG.allowedDomains) {
-          if (allowed.includes('*')) {
-            const pattern = allowed.replace('*', '.*');
-            if (new RegExp(pattern).test(hostname)) {
-              isAllowed = true;
-              break;
-            }
-          } else if (hostname === allowed || hostname.endsWith(`.${allowed}`)) {
-            isAllowed = true;
-            break;
-          }
-        }
-        if (!isAllowed) {
-          throw new Error(`Domain not in whitelist: ${hostname}`);
-        }
-      }
-      
-      // Check for IP addresses (basic check)
-      const ipPattern = /^(\d{1,3}\.){3}\d{1,3}$/;
-      if (ipPattern.test(hostname)) {
-        throw new Error('Direct IP access not allowed');
-      }
-      
-      return parsedUrl;
-    } catch (error) {
-      if (error instanceof TypeError) {
-        throw new Error('Invalid URL format');
-      }
-      throw error;
-    }
-  }
-  
-  static async validateAndFetch(url, options = {}) {
-    const validatedUrl = this.validateUrl(url);
-    
-    const fetchOptions = {
-      timeout: options.timeout || 30000,
-      headers: {
-        ...identityHeaders({ role: 'health-check' }),
-        ...options.headers
-      },
-      ...options
-    };
-    
-    // Remove sensitive headers
-    delete fetchOptions.headers['Authorization'];
-    delete fetchOptions.headers['Cookie'];
-    
-    return fetch(validatedUrl.toString(), fetchOptions);
-  }
-}
-
-/**
- * Path Traversal Protection
- */
-export class PathSecurity {
-  static sanitizePath(userPath, baseDirectory) {
-    // Normalize the path to resolve any .. or . components
-    const normalizedPath = path.normalize(userPath);
-    
-    // Resolve the absolute path
-    const absolutePath = path.resolve(baseDirectory, normalizedPath);
-    const absoluteBase = path.resolve(baseDirectory);
-    
-    // Ensure the resolved path is within the base directory
-    if (!absolutePath.startsWith(absoluteBase + path.sep) && absolutePath !== absoluteBase) {
-      throw new Error('Path traversal detected');
-    }
-    
-    return absolutePath;
-  }
-  
-  static validateSnapshotId(snapshotId) {
-    // Only allow alphanumeric characters and hyphens
-    if (!/^[a-zA-Z0-9_-]{1,64}$/.test(snapshotId)) {
-      throw new Error('Invalid snapshot ID format');
-    }
-    
-    // Prevent path traversal attempts
-    if (snapshotId.includes('..') || snapshotId.includes('/') || snapshotId.includes('\\')) {
-      throw new Error('Invalid characters in snapshot ID');
-    }
-    
-    return snapshotId;
-  }
-  
-  static validateDirectory(directory) {
-    const normalizedDir = path.normalize(directory);
-    
-    // Check against allowed directories
-    for (const allowed of SECURITY_CONFIG.allowedDirectories) {
-      const absoluteAllowed = path.resolve(allowed);
-      const absoluteDir = path.resolve(normalizedDir);
-      
-      if (absoluteDir.startsWith(absoluteAllowed)) {
-        return absoluteDir;
-      }
-    }
-    
-    throw new Error('Directory not allowed');
-  }
-}
-
-/**
- * Input Validation and Sanitization
- */
-export class InputSecurity {
-  static sanitizeHtml(html) {
-    return DOMPurify.sanitize(html, {
-      ALLOWED_TAGS: ['p', 'br', 'strong', 'em', 'u', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'],
-      ALLOWED_ATTR: []
-    });
-  }
-  
-  static sanitizeString(str, maxLength = SECURITY_CONFIG.maxStringLength) {
-    if (typeof str !== 'string') {
-      throw new Error('Input must be a string');
-    }
-    
-    if (str.length > maxLength) {
-      throw new Error(`String too long. Maximum length is ${maxLength}`);
-    }
-    
-    // Remove null bytes and control characters
-    return str.replace(/[\x00-\x1F\x7F-\x9F]/g, '');
-  }
-  
-  static sanitizeArray(arr, maxLength = SECURITY_CONFIG.maxArrayLength) {
-    if (!Array.isArray(arr)) {
-      throw new Error('Input must be an array');
-    }
-    
-    if (arr.length > maxLength) {
-      throw new Error(`Array too long. Maximum length is ${maxLength}`);
-    }
-    
-    return arr;
-  }
-  
-  static validateContentSize(content) {
-    const size = Buffer.byteLength(content, 'utf8');
-    if (size > SECURITY_CONFIG.maxContentSize) {
-      throw new Error(`Content too large: ${size} bytes (max: ${SECURITY_CONFIG.maxContentSize})`);
-    }
-    return size;
-  }
-  
-  static sanitizeJavaScript(jsCode) {
-    // Remove potentially dangerous patterns
-    const dangerousPatterns = [
-      /eval\s*\(/gi,
-      /Function\s*\(/gi,
-      /setTimeout\s*\(/gi,
-      /setInterval\s*\(/gi,
-      /document\.write/gi,
-      /innerHTML/gi,
-      /outerHTML/gi,
-      /execScript/gi,
-      /script:/gi,
-      /javascript:/gi,
-      /data:/gi,
-      /vbscript:/gi
-    ];
-    
-    let sanitized = jsCode;
-    for (const pattern of dangerousPatterns) {
-      if (pattern.test(sanitized)) {
-        throw new Error('Dangerous JavaScript pattern detected');
-      }
-    }
-    
-    return sanitized;
-  }
-}
-
-/**
- * Cryptographic Security
- */
-export class CryptoSecurity {
-  static generateSecureId(length = 32) {
-    return randomBytes(length).toString('hex');
-  }
-  
-  static generateSessionId() {
-    return this.generateSecureId(16);
-  }
-  
-  static hashContent(content, algorithm = 'sha256') {
-    return createHash(algorithm).update(content).digest('hex');
-  }
-  
-  static timingSafeCompare(a, b) {
-    if (typeof a !== 'string' || typeof b !== 'string') {
-      throw new Error('Both values must be strings');
-    }
-    
-    if (a.length !== b.length) {
-      return false;
-    }
-    
-    const bufferA = Buffer.from(a);
-    const bufferB = Buffer.from(b);
-    
-    return timingSafeEqual(bufferA, bufferB);
-  }
-  
-  static generateWebhookSignature(payload, secret) {
-    return createHash('sha256')
-      .update(payload + secret)
-      .digest('hex');
-  }
-  
-  static validateWebhookSignature(payload, signature, secret) {
-    const expectedSignature = this.generateWebhookSignature(payload, secret);
-    return this.timingSafeCompare(signature, expectedSignature);
-  }
-}
-
-/**
- * Resource Management Security
- */
-export class ResourceSecurity {
-  static validateResearchLimits(options) {
-    const maxUrls = Math.min(options.maxUrls || 10, SECURITY_CONFIG.maxResearchUrls);
-    const timeLimit = Math.min(options.timeLimit || 60000, SECURITY_CONFIG.maxResearchTime);
-    const maxDepth = Math.min(Math.max(options.maxDepth || 3, 1), 5);
-    
-    return {
-      maxUrls,
-      timeLimit,
-      maxDepth,
-      concurrency: Math.min(options.concurrency || 3, 5)
-    };
-  }
-  
-  static validateSnapshotLimits(options) {
-    return {
-      maxSnapshots: Math.min(options.maxSnapshots || 100, 1000),
-      maxAge: Math.max(options.maxAge || 86400000, 3600000), // Min 1 hour
-      maxStorageSize: Math.min(options.maxStorageSize || 1073741824, 10737418240), // Max 10GB
-      compressionThreshold: Math.max(options.compressionThreshold || 1024, 1024)
-    };
-  }
-}
-
-/**
- * Browser Security
- */
-export class BrowserSecurity {
-  static getSecureBrowserArgs() {
-    return [
-      // Security-focused args (remove dangerous ones from original)
-      '--disable-blink-features=AutomationControlled',
-      '--disable-dev-shm-usage',
-      '--disable-extensions',
-      '--disable-plugins',
-      '--disable-images', // Reduce attack surface
-      '--disable-javascript-harmony-shipping',
-      '--disable-background-timer-throttling',
-      '--disable-backgrounding-occluded-windows',
-      '--disable-renderer-backgrounding',
-      '--disable-field-trial-config',
-      '--disable-back-forward-cache',
-      '--disable-hang-monitor',
-      '--disable-prompt-on-repost',
-      '--disable-sync',
-      '--disable-translate',
-      '--metrics-recording-only',
-      '--no-first-run',
-      '--safebrowsing-disable-auto-update',
-      '--password-store=basic',
-      '--use-mock-keychain',
-      
-      // Security enhancements
-      '--disable-file-system',
-      '--disable-databases',
-      '--disable-local-storage',
-      '--disable-session-storage',
-      '--disable-application-cache',
-      '--disable-notifications',
-      '--disable-geolocation',
-      '--disable-microphone',
-      '--disable-camera',
-      
-      // DO NOT include these dangerous args from original:
-      // --no-sandbox (removes critical security boundary)
-      // --disable-web-security (removes Same Origin Policy)
-      // --disable-features=VizDisplayCompositor (removes security features)
-    ];
-  }
-  
-  static validateStealthConfig(config) {
-    // Ensure stealth mode doesn't compromise security
-    const secureConfig = {
-      ...config,
-      hideWebDriver: true, // This is fine
-      blockWebRTC: true,   // This is fine for privacy
-      spoofTimezone: true, // This is fine for privacy
-      randomizeHeaders: true, // This is fine for privacy
-      
-      // Remove dangerous options
-      disableSecurity: false,
-      allowUnsafeInlineScripts: false,
-      bypassCSP: false
-    };
-    
-    return secureConfig;
-  }
-  
-  static sanitizeInjectedScript(script) {
-    // Validate that injected scripts don't contain dangerous patterns
-    const dangerousPatterns = [
-      /eval\(/gi,
-      /Function\(/gi,
-      /setTimeout\(/gi,
-      /setInterval\(/gi,
-      /XMLHttpRequest/gi,
-      /fetch\(/gi,
-      /import\(/gi,
-      /require\(/gi
-    ];
-    
-    for (const pattern of dangerousPatterns) {
-      if (pattern.test(script)) {
-        throw new Error('Dangerous script pattern detected');
-      }
-    }
-    
-    return script;
-  }
-}
-
-/**
- * Validation Schemas for Wave 3 Components
- */
-export const Wave3SecuritySchemas = {
-  // Research Tool Security Schema
-  researchRequest: z.object({
-    topic: z.string().min(3).max(500).refine(val => !/<script/i.test(val), 'No script tags allowed'),
-    maxDepth: z.number().min(1).max(5),
-    maxUrls: z.number().min(1).max(50),
-    timeLimit: z.number().min(30000).max(120000),
-    concurrency: z.number().min(1).max(5)
-  }),
-  
-  // Stealth Browser Security Schema  
-  stealthConfig: z.object({
-    level: z.enum(['basic', 'medium']), // Remove 'advanced' - too dangerous
-    randomizeFingerprint: z.boolean(),
-    hideWebDriver: z.boolean(),
-    blockWebRTC: z.boolean(),
-    spoofTimezone: z.boolean(),
-    customUserAgent: z.string().max(500).optional(),
-    customViewport: z.object({
-      width: z.number().min(800).max(1920),
-      height: z.number().min(600).max(1080)
-    }).optional()
-  }),
-  
-  // Localization Security Schema
-  localizationConfig: z.object({
-    countryCode: z.string().length(2).regex(/^[A-Z]{2}$/),
-    language: z.string().max(10).regex(/^[a-z]{2}-[A-Z]{2}$/),
-    timezone: z.string().max(50).regex(/^[A-Za-z_/]+$/),
-    currency: z.string().length(3).regex(/^[A-Z]{3}$/)
-  }),
-  
-  // Snapshot Security Schema
-  snapshotRequest: z.object({
-    url: z.string().url().refine(url => {
-      try {
-        SSRFProtection.validateUrl(url);
-        return true;
-      } catch {
-        return false;
-      }
-    }, 'Invalid or blocked URL'),
-    content: z.string().max(SECURITY_CONFIG.maxContentSize),
-    snapshotId: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/).optional()
-  }),
-  
-  // Change Tracking Security Schema
-  changeTrackingRequest: z.object({
-    url: z.string().url(),
-    granularity: z.enum(['page', 'section', 'element', 'text']),
-    customSelectors: z.array(z.string().max(100)).max(10).optional(),
-    excludeSelectors: z.array(z.string().max(100)).max(20).optional()
-  })
-};
-
-/**
- * Security Middleware Factory
- */
-export class SecurityMiddleware {
-  static rateLimiter(options = SECURITY_CONFIG.defaultRateLimit) {
-    const requests = new Map();
-    
-    return (identifier) => {
-      const now = Date.now();
-      const windowStart = now - options.windowMs;
-      
-      // Clean old requests
-      for (const [key, timestamps] of requests.entries()) {
-        requests.set(key, timestamps.filter(time => time > windowStart));
-      }
-      
-      // Check current requests
-      const userRequests = requests.get(identifier) || [];
-      if (userRequests.length >= options.max) {
-        throw new Error('Rate limit exceeded');
-      }
-      
-      // Add current request
-      userRequests.push(now);
-      requests.set(identifier, userRequests);
-    };
-  }
-  
-  static validateRequest(schema) {
-    return (data) => {
-      try {
-        return schema.parse(data);
-      } catch (error) {
-        throw new Error(`Validation failed: ${error.message}`);
-      }
-    };
-  }
-  
-  static auditLogger() {
-    return (operation, details) => {
-      const logEntry = {
-        timestamp: new Date().toISOString(),
-        operation,
-        details: typeof details === 'object' ? JSON.stringify(details) : details,
-        level: 'SECURITY'
-      };
-      
-      console.log('[SECURITY AUDIT]', JSON.stringify(logEntry));
-    };
-  }
-}
-
-/**
- * Security Testing Utilities
- */
-export class SecurityTesting {
-  static generateMaliciousPayloads() {
-    return {
-      ssrf: [
-        'http://localhost:3000/admin',
-        'http://127.0.0.1:22',
-        'http://169.254.169.254/latest/meta-data/',
-        'file:///etc/passwd',
-        'ftp://internal.server.com'
-      ],
-      
-      pathTraversal: [
-        '../../../etc/passwd',
-        '..\\..\\..\\windows\\system32\\config\\sam',
-        '/etc/shadow',
-        'C:\\Windows\\System32\\drivers\\etc\\hosts'
-      ],
-      
-      xss: [
-        '<script>alert("XSS")</script>',
-        'javascript:alert("XSS")',
-        '<img src="x" onerror="alert(\'XSS\')">',
-        '<svg onload="alert(\'XSS\')"></svg>'
-      ],
-      
-      injection: [
-        "'; DROP TABLE users; --",
-        '${7*7}',
-        '#{7*7}',
-        '{{7*7}}',
-        '<%= 7*7 %>'
-      ]
-    };
-  }
-  
-  static testSecurityFunction(securityFunction, maliciousPayloads) {
-    const results = [];
-    
-    for (const payload of maliciousPayloads) {
-      try {
-        securityFunction(payload);
-        results.push({ payload, blocked: false, error: null });
-      } catch (error) {
-        results.push({ payload, blocked: true, error: error.message });
-      }
-    }
-    
-    return results;
-  }
-}
-
-/**
- * Emergency Security Patches
- */
-export class EmergencyPatches {
-  // Patch for ResearchOrchestrator SSRF vulnerability
-  static patchResearchTool(researchTool) {
-    const originalExecute = researchTool.execute;
-    
-    researchTool.execute = async function(params) {
-      // Validate research limits
-      const secureParams = ResourceSecurity.validateResearchLimits(params);
-      
-      // Validate topic for script injection
-      secureParams.topic = InputSecurity.sanitizeString(params.topic, 500);
-      
-      return originalExecute.call(this, secureParams);
-    };
-  }
-  
-  // Patch for StealthBrowserManager script injection
-  static patchBrowserManager(browserManager) {
-    const originalCreateStealthContext = browserManager.createStealthContext;
-    
-    browserManager.createStealthContext = async function(config) {
-      // Validate and secure the config
-      const secureConfig = BrowserSecurity.validateStealthConfig(config);
-      
-      // Use secure browser args
-      secureConfig.browserArgs = BrowserSecurity.getSecureBrowserArgs();
-      
-      return originalCreateStealthContext.call(this, secureConfig);
-    };
-  }
-  
-  // Patch for SnapshotManager path traversal
-  static patchSnapshotManager(snapshotManager) {
-    const originalStoreSnapshot = snapshotManager.storeSnapshot;
-    
-    snapshotManager.storeSnapshot = async function(url, content, metadata, options) {
-      // Validate URL
-      SSRFProtection.validateUrl(url);
-      
-      // Validate content size
-      InputSecurity.validateContentSize(content);
-      
-      // Generate secure snapshot ID
-      const snapshotId = CryptoSecurity.generateSecureId();
-      
-      return originalStoreSnapshot.call(this, url, content, metadata, { ...options, snapshotId });
-    };
-  }
-}
-
-// Export default security instance
-export default {
-  SSRFProtection,
-  PathSecurity,
-  InputSecurity,
-  CryptoSecurity,
-  ResourceSecurity,
-  BrowserSecurity,
-  Wave3SecuritySchemas,
-  SecurityMiddleware,
-  SecurityTesting,
-  EmergencyPatches,
-  SECURITY_CONFIG
-};