crawlforge-mcp-server 3.0.0

package/src/utils/rateLimiter.js

@@ -0,0 +1,196 @@
+export class RateLimiter {
+  constructor(options = {}) {
+    const {
+      requestsPerSecond = 10,
+      requestsPerMinute = 100,
+      perDomain = true
+    } = options;
+
+    this.requestsPerSecond = requestsPerSecond;
+    this.requestsPerMinute = requestsPerMinute;
+    this.perDomain = perDomain;
+    this.windowMs = 1000; // 1 second window
+    this.limits = new Map(); // domain -> { count, resetTime }
+  }
+
+  async checkLimit(urlOrDomain) {
+    const domain = this.extractDomain(urlOrDomain);
+    const now = Date.now();
+    
+    const key = this.perDomain ? domain : 'global';
+    let limit = this.limits.get(key);
+    
+    if (!limit) {
+      limit = {
+        secondCount: 0,
+        secondReset: now + 1000,
+        minuteCount: 0,
+        minuteReset: now + 60000
+      };
+      this.limits.set(key, limit);
+    }
+
+    // Reset counters if windows have passed
+    if (now > limit.secondReset) {
+      limit.secondCount = 0;
+      limit.secondReset = now + 1000;
+    }
+    
+    if (now > limit.minuteReset) {
+      limit.minuteCount = 0;
+      limit.minuteReset = now + 60000;
+    }
+
+    // Check rate limits
+    if (limit.secondCount >= this.requestsPerSecond) {
+      const waitTime = limit.secondReset - now;
+      await this.delay(waitTime);
+      return this.checkLimit(urlOrDomain);
+    }
+    
+    if (limit.minuteCount >= this.requestsPerMinute) {
+      const waitTime = limit.minuteReset - now;
+      await this.delay(waitTime);
+      return this.checkLimit(urlOrDomain);
+    }
+
+    // Increment counters
+    limit.secondCount++;
+    limit.minuteCount++;
+    
+    return true;
+  }
+
+  extractDomain(urlOrDomain) {
+    try {
+      if (urlOrDomain.startsWith('http://') || urlOrDomain.startsWith('https://')) {
+        const url = new URL(urlOrDomain);
+        return url.hostname;
+      }
+      return urlOrDomain;
+    } catch {
+      return urlOrDomain;
+    }
+  }
+
+  delay(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  reset(domain) {
+    if (domain) {
+      this.limits.delete(domain);
+    } else {
+      this.limits.clear();
+    }
+  }
+
+  getStats() {
+    const stats = {};
+    for (const [domain, limit] of this.limits.entries()) {
+      stats[domain] = {
+        secondCount: limit.secondCount,
+        minuteCount: limit.minuteCount,
+        secondsUntilReset: Math.max(0, Math.ceil((limit.secondReset - Date.now()) / 1000)),
+        minutesUntilReset: Math.max(0, Math.ceil((limit.minuteReset - Date.now()) / 60000))
+      };
+    }
+    return stats;
+  }
+}
+
+export class CircuitBreaker {
+  constructor(options = {}) {
+    const {
+      threshold = 5,
+      timeout = 60000,
+      resetTimeout = 120000
+    } = options;
+
+    this.threshold = threshold;
+    this.timeout = timeout;
+    this.resetTimeout = resetTimeout;
+    this.failures = new Map(); // domain -> { count, state, nextAttempt }
+  }
+
+  async execute(domain, fn) {
+    const breaker = this.getBreaker(domain);
+    
+    if (breaker.state === 'OPEN') {
+      if (Date.now() < breaker.nextAttempt) {
+        throw new Error(`Circuit breaker is OPEN for ${domain}`);
+      }
+      breaker.state = 'HALF_OPEN';
+    }
+
+    try {
+      const result = await Promise.race([
+        fn(),
+        this.timeoutPromise()
+      ]);
+      
+      this.onSuccess(domain);
+      return result;
+    } catch (error) {
+      this.onFailure(domain);
+      throw error;
+    }
+  }
+
+  getBreaker(domain) {
+    if (!this.failures.has(domain)) {
+      this.failures.set(domain, {
+        count: 0,
+        state: 'CLOSED',
+        nextAttempt: Date.now()
+      });
+    }
+    return this.failures.get(domain);
+  }
+
+  onSuccess(domain) {
+    const breaker = this.getBreaker(domain);
+    breaker.count = 0;
+    breaker.state = 'CLOSED';
+  }
+
+  onFailure(domain) {
+    const breaker = this.getBreaker(domain);
+    breaker.count++;
+    
+    if (breaker.count >= this.threshold) {
+      breaker.state = 'OPEN';
+      breaker.nextAttempt = Date.now() + this.resetTimeout;
+    }
+  }
+
+  timeoutPromise() {
+    return new Promise((_, reject) => {
+      setTimeout(() => reject(new Error('Operation timeout')), this.timeout);
+    });
+  }
+
+  reset(domain) {
+    if (domain) {
+      this.failures.delete(domain);
+    } else {
+      this.failures.clear();
+    }
+  }
+
+  getStats() {
+    const stats = {};
+    for (const [domain, breaker] of this.failures.entries()) {
+      stats[domain] = {
+        failureCount: breaker.count,
+        state: breaker.state,
+        nextAttemptIn: breaker.state === 'OPEN' 
+          ? Math.max(0, Math.ceil((breaker.nextAttempt - Date.now()) / 1000))
+          : 0
+      };
+    }
+    return stats;
+  }
+}
+
+export default RateLimiter;

package/src/utils/robotsChecker.js

@@ -0,0 +1,91 @@
+import robotsParser from 'robots-parser';
+
+export class RobotsChecker {
+  constructor(userAgent = 'CrawlForge/1.0') {
+    this.userAgent = userAgent;
+    this.robotsCache = new Map();
+  }
+
+  async canFetch(url) {
+    try {
+      const urlObj = new URL(url);
+      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+      
+      let robots = this.robotsCache.get(robotsUrl);
+      
+      if (!robots) {
+        const robotsTxt = await this.fetchRobotsTxt(robotsUrl);
+        robots = robotsParser(robotsUrl, robotsTxt);
+        this.robotsCache.set(robotsUrl, robots);
+      }
+      
+      return robots.isAllowed(url, this.userAgent);
+    } catch (error) {
+      // If we can't fetch robots.txt, assume we can crawl
+      console.warn(`Failed to check robots.txt for ${url}:`, error.message);
+      return true;
+    }
+  }
+
+  async fetchRobotsTxt(robotsUrl) {
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), 5000);
+      
+      const response = await fetch(robotsUrl, {
+        signal: controller.signal,
+        headers: {
+          'User-Agent': this.userAgent
+        }
+      });
+      
+      clearTimeout(timeoutId);
+      
+      if (!response.ok) {
+        return ''; // Empty robots.txt means everything is allowed
+      }
+      
+      return await response.text();
+    } catch (error) {
+      return ''; // If we can't fetch, assume no restrictions
+    }
+  }
+
+  getCrawlDelay(url) {
+    try {
+      const urlObj = new URL(url);
+      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+      const robots = this.robotsCache.get(robotsUrl);
+      
+      if (robots) {
+        return robots.getCrawlDelay(this.userAgent) || 0;
+      }
+      
+      return 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  getSitemaps(url) {
+    try {
+      const urlObj = new URL(url);
+      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+      const robots = this.robotsCache.get(robotsUrl);
+      
+      if (robots) {
+        return robots.getSitemaps() || [];
+      }
+      
+      return [];
+    } catch {
+      return [];
+    }
+  }
+
+  clearCache() {
+    this.robotsCache.clear();
+  }
+}
+
+export default RobotsChecker;

package/src/utils/securityMiddleware.js

@@ -0,0 +1,416 @@
+/**
+ * Security Middleware for MCP WebScraper
+ * Integrates SSRF protection, input validation, and other security measures
+ */
+
+import { SSRFProtection } from './ssrfProtection.js';
+import { InputValidator } from './inputValidation.js';
+import { config } from '../constants/config.js';
+import { Logger } from './Logger.js';
+
+// Initialize security components
+const ssrfProtection = new SSRFProtection({
+  allowedProtocols: config.security.ssrfProtection.allowedProtocols,
+  maxRequestSize: config.security.ssrfProtection.maxRequestSize,
+  maxTimeout: config.security.ssrfProtection.maxTimeout,
+  maxRedirects: config.security.ssrfProtection.maxRedirects,
+  blockedHostnames: config.security.ssrfProtection.blockedDomains
+});
+
+const inputValidator = new InputValidator({
+  maxStringLength: config.security.inputValidation.maxStringLength,
+  maxArrayLength: config.security.inputValidation.maxArrayLength,
+  maxObjectDepth: config.security.inputValidation.maxObjectDepth,
+  maxRegexLength: config.security.inputValidation.maxRegexLength,
+  allowedHTMLTags: config.security.contentSecurity.allowedHTMLTags
+});
+
+const logger = new Logger();
+
+/**
+ * Security middleware class for MCP tools
+ */
+export class SecurityMiddleware {
+  constructor(options = {}) {
+    this.ssrfProtection = ssrfProtection;
+    this.inputValidator = inputValidator;
+    this.logger = logger;
+    this.config = config.security;
+    this.violationStats = {
+      totalViolations: 0,
+      blockedRequests: 0,
+      ssrfBlocked: 0,
+      injectionBlocked: 0,
+      validationErrors: 0
+    };
+  }
+
+  /**
+   * Validate URL parameter for SSRF protection
+   * @param {string} url - URL to validate
+   * @param {Object} context - Request context
+   * @returns {Promise<Object>} - Validation result
+   */
+  async validateURL(url, context = {}) {
+    if (!this.config.ssrfProtection.enabled) {
+      return { allowed: true, sanitizedURL: url };
+    }
+
+    try {
+      const result = await this.ssrfProtection.validateURL(url);
+      
+      if (!result.allowed) {
+        this.violationStats.ssrfBlocked++;
+        this.violationStats.blockedRequests++;
+        
+        this.logSecurityViolation('SSRF_BLOCKED', {
+          url,
+          violations: result.violations,
+          context
+        });
+      }
+
+      return {
+        allowed: result.allowed,
+        sanitizedURL: result.sanitizedURL || url,
+        violations: result.violations
+      };
+    } catch (error) {
+      this.logger.error('SSRF validation error:', error);
+      return { allowed: false, error: error.message };
+    }
+  }
+
+  /**
+   * Validate search query parameters
+   * @param {string} query - Search query to validate
+   * @param {Object} context - Request context
+   * @returns {Object} - Validation result
+   */
+  validateSearchQuery(query, context = {}) {
+    if (!this.config.inputValidation.enabled) {
+      return { isValid: true, sanitizedValue: query };
+    }
+
+    try {
+      const result = this.inputValidator.validateSearchQuery(query);
+      
+      if (!result.isValid) {
+        this.violationStats.injectionBlocked++;
+        this.violationStats.blockedRequests++;
+        
+        this.logSecurityViolation('INJECTION_BLOCKED', {
+          query,
+          violations: result.violations,
+          context
+        });
+      }
+
+      return result;
+    } catch (error) {
+      this.logger.error('Query validation error:', error);
+      this.violationStats.validationErrors++;
+      return { isValid: false, error: error.message };
+    }
+  }
+
+  /**
+   * Validate CSS selector parameters
+   * @param {string} selector - CSS selector to validate
+   * @param {Object} context - Request context
+   * @returns {Object} - Validation result
+   */
+  validateCSSSelector(selector, context = {}) {
+    if (!this.config.inputValidation.enabled) {
+      return { isValid: true, sanitizedValue: selector };
+    }
+
+    try {
+      const result = this.inputValidator.validateCSSSelector(selector);
+      
+      if (!result.isValid) {
+        this.violationStats.injectionBlocked++;
+        this.violationStats.blockedRequests++;
+        
+        this.logSecurityViolation('CSS_INJECTION_BLOCKED', {
+          selector,
+          violations: result.violations,
+          context
+        });
+      }
+
+      return result;
+    } catch (error) {
+      this.logger.error('CSS validation error:', error);
+      this.violationStats.validationErrors++;
+      return { isValid: false, error: error.message };
+    }
+  }
+
+  /**
+   * Validate object parameters
+   * @param {Object} obj - Object to validate
+   * @param {Object} context - Request context
+   * @returns {Object} - Validation result
+   */
+  validateObject(obj, context = {}) {
+    if (!this.config.inputValidation.enabled) {
+      return { isValid: true, sanitizedValue: obj };
+    }
+
+    try {
+      const result = this.inputValidator.validateObject(obj);
+      
+      if (!result.isValid) {
+        this.violationStats.validationErrors++;
+        
+        this.logSecurityViolation('OBJECT_VALIDATION_FAILED', {
+          objectKeys: Object.keys(obj || {}),
+          violations: result.violations,
+          context
+        });
+      }
+
+      return result;
+    } catch (error) {
+      this.logger.error('Object validation error:', error);
+      this.violationStats.validationErrors++;
+      return { isValid: false, error: error.message };
+    }
+  }
+
+  /**
+   * Validate HTML content
+   * @param {string} html - HTML content to validate
+   * @param {Object} context - Request context
+   * @returns {Object} - Validation result
+   */
+  validateHTML(html, context = {}) {
+    if (!this.config.contentSecurity.sanitizeHTML) {
+      return { isValid: true, sanitizedValue: html };
+    }
+
+    try {
+      const result = this.inputValidator.validateHTML(html);
+      
+      if (!result.isValid) {
+        this.violationStats.injectionBlocked++;
+        
+        this.logSecurityViolation('HTML_XSS_BLOCKED', {
+          htmlLength: html.length,
+          violations: result.violations,
+          context
+        });
+      }
+
+      return result;
+    } catch (error) {
+      this.logger.error('HTML validation error:', error);
+      this.violationStats.validationErrors++;
+      return { isValid: false, error: error.message };
+    }
+  }
+
+  /**
+   * Create secure fetch function with SSRF protection
+   * @param {Object} options - Fetch options
+   * @returns {Function} - Secure fetch function
+   */
+  createSecureFetch(options = {}) {
+    return this.ssrfProtection.createSecureFetch({
+      allowedDomains: this.config.ssrfProtection.allowedDomains,
+      maxRequestSize: this.config.ssrfProtection.maxRequestSize,
+      ...options
+    });
+  }
+
+  /**
+   * Validate tool parameters based on schema
+   * @param {Object} params - Tool parameters
+   * @param {string} toolName - Name of the tool
+   * @returns {Promise<Object>} - Validation result
+   */
+  async validateToolParameters(params, toolName) {
+    const results = {
+      isValid: true,
+      violations: [],
+      sanitizedParams: { ...params }
+    };
+
+    const context = { toolName, timestamp: new Date().toISOString() };
+
+    // URL validation for tools that accept URLs
+    if (params.url) {
+      const urlResult = await this.validateURL(params.url, context);
+      if (!urlResult.allowed) {
+        results.isValid = false;
+        results.violations.push(...(urlResult.violations || []));
+      } else {
+        results.sanitizedParams.url = urlResult.sanitizedURL;
+      }
+    }
+
+    // Search query validation
+    if (params.query) {
+      const queryResult = this.validateSearchQuery(params.query, context);
+      if (!queryResult.isValid) {
+        results.isValid = false;
+        results.violations.push(...(queryResult.violations || []));
+      } else {
+        results.sanitizedParams.query = queryResult.sanitizedValue;
+      }
+    }
+
+    // CSS selectors validation
+    if (params.selectors) {
+      for (const [key, selector] of Object.entries(params.selectors)) {
+        const selectorResult = this.validateCSSSelector(selector, context);
+        if (!selectorResult.isValid) {
+          results.isValid = false;
+          results.violations.push(...(selectorResult.violations || []));
+        } else {
+          results.sanitizedParams.selectors[key] = selectorResult.sanitizedValue;
+        }
+      }
+    }
+
+    // Object validation for complex parameters
+    if (params.options && typeof params.options === 'object') {
+      const objectResult = this.validateObject(params.options, context);
+      if (!objectResult.isValid) {
+        results.isValid = false;
+        results.violations.push(...(objectResult.violations || []));
+      } else {
+        results.sanitizedParams.options = objectResult.sanitizedValue;
+      }
+    }
+
+    // Validate arrays (like include_patterns, exclude_patterns)
+    ['include_patterns', 'exclude_patterns'].forEach(paramName => {
+      if (params[paramName] && Array.isArray(params[paramName])) {
+        for (const pattern of params[paramName]) {
+          if (typeof pattern === 'string') {
+            const regexResult = this.inputValidator.validateRegex(pattern);
+            if (!regexResult.isValid) {
+              results.isValid = false;
+              results.violations.push(...(regexResult.violations || []));
+              this.logSecurityViolation('REGEX_VALIDATION_FAILED', {
+                pattern,
+                violations: regexResult.violations,
+                context
+              });
+            }
+          }
+        }
+      }
+    });
+
+    return results;
+  }
+
+  /**
+   * Log security violations
+   * @param {string} type - Violation type
+   * @param {Object} details - Violation details
+   */
+  logSecurityViolation(type, details) {
+    this.violationStats.totalViolations++;
+
+    if (this.config.monitoring?.violationLogging !== false) {
+      this.logger.warn('Security violation detected', {
+        type,
+        details,
+        timestamp: new Date().toISOString(),
+        severity: this.getViolationSeverity(details.violations)
+      });
+    }
+
+    // Log high-severity violations to security log
+    if (this.config.monitoring?.securityLogging !== false) {
+      const severity = this.getViolationSeverity(details.violations);
+      if (severity === 'HIGH') {
+        this.logger.error('High-severity security violation', {
+          type,
+          details: {
+            ...details,
+            // Don't log full content for security
+            input: details.query ? details.query.substring(0, 100) : undefined,
+            url: details.url ? details.url.substring(0, 200) : undefined
+          }
+        });
+      }
+    }
+  }
+
+  /**
+   * Get violation severity level
+   * @param {Array} violations - Array of violations
+   * @returns {string} - Severity level
+   */
+  getViolationSeverity(violations = []) {
+    if (violations.some(v => v.severity === 'HIGH')) return 'HIGH';
+    if (violations.some(v => v.severity === 'MEDIUM')) return 'MEDIUM';
+    return 'LOW';
+  }
+
+  /**
+   * Get security statistics
+   * @returns {Object} - Security statistics
+   */
+  getSecurityStats() {
+    return {
+      violations: this.violationStats,
+      ssrfStats: this.ssrfProtection.getStats(),
+      validationStats: this.inputValidator.getStats(),
+      configEnabled: {
+        ssrfProtection: this.config.ssrfProtection.enabled,
+        inputValidation: this.config.inputValidation.enabled,
+        contentSecurity: this.config.contentSecurity.sanitizeHTML,
+        auditLogging: this.config.apiSecurity.auditLogging
+      }
+    };
+  }
+
+  /**
+   * Reset security statistics
+   */
+  resetStats() {
+    this.violationStats = {
+      totalViolations: 0,
+      blockedRequests: 0,
+      ssrfBlocked: 0,
+      injectionBlocked: 0,
+      validationErrors: 0
+    };
+    this.ssrfProtection.clearCache();
+    this.inputValidator.clearViolationLog();
+  }
+
+  /**
+   * Check if request should be authenticated
+   * @param {Object} request - Request object
+   * @returns {boolean} - Whether authentication is required
+   */
+  requiresAuthentication(request) {
+    return this.config.apiSecurity.requireAuthentication;
+  }
+
+  /**
+   * Validate API key
+   * @param {string} apiKey - API key to validate
+   * @returns {boolean} - Whether API key is valid
+   */
+  validateAPIKey(apiKey) {
+    if (!this.config.apiSecurity.requireAuthentication) {
+      return true;
+    }
+
+    return apiKey === this.config.apiSecurity.apiKey && 
+           this.config.apiSecurity.apiKey.length > 0;
+  }
+}
+
+// Export singleton instance
+export const securityMiddleware = new SecurityMiddleware();
+
+export default securityMiddleware;