@devilsdev/rag-pipeline-utils 2.2.0 → 2.3.0

package/src/utils/rate-limiter.js

@@ -0,0 +1,244 @@
+/**
+ * Rate Limiter Utility
+ * Protects against brute force attacks and abuse
+ *
+ * Features:
+ * - Sliding window rate limiting
+ * - Per-identifier tracking (IP, user, tenant)
+ * - Automatic cleanup of expired entries
+ * - Configurable thresholds and windows
+ */
+
+const crypto = require('crypto');
+
+class RateLimiter {
+  /**
+   * Create a rate limiter instance
+   * @param {Object} options - Configuration options
+   * @param {number} options.maxAttempts - Maximum attempts allowed (default: 5)
+   * @param {number} options.windowMs - Time window in milliseconds (default: 15 minutes)
+   * @param {number} options.blockDurationMs - How long to block after exceeding limit (default: 1 hour)
+   * @param {number} options.cleanupIntervalMs - How often to cleanup expired entries (default: 5 minutes)
+   */
+  constructor(options = {}) {
+    this.maxAttempts = options.maxAttempts || 5;
+    this.windowMs = options.windowMs || 15 * 60 * 1000; // 15 minutes
+    this.blockDurationMs = options.blockDurationMs || 60 * 60 * 1000; // 1 hour
+    this.cleanupIntervalMs = options.cleanupIntervalMs || 5 * 60 * 1000; // 5 minutes
+
+    // Map of identifier -> { attempts: [], blockedUntil: timestamp }
+    this.store = new Map();
+
+    // Start automatic cleanup
+    this._startCleanup();
+  }
+
+  /**
+   * Check if a request should be allowed
+   * @param {string} identifier - Unique identifier (IP, userId, tenantId, etc.)
+   * @returns {Object} { allowed: boolean, remaining: number, resetAt: Date, blockedUntil: Date|null }
+   */
+  allowRequest(identifier) {
+    const key = this._hashIdentifier(identifier);
+    const now = Date.now();
+
+    // Get or create record
+    let record = this.store.get(key);
+    if (!record) {
+      record = { attempts: [], blockedUntil: null };
+      this.store.set(key, record);
+    }
+
+    // Check if currently blocked
+    if (record.blockedUntil && record.blockedUntil > now) {
+      return {
+        allowed: false,
+        remaining: 0,
+        resetAt: new Date(record.blockedUntil),
+        blockedUntil: new Date(record.blockedUntil),
+        retryAfter: Math.ceil((record.blockedUntil - now) / 1000), // seconds
+      };
+    }
+
+    // Clear expired block
+    if (record.blockedUntil && record.blockedUntil <= now) {
+      record.blockedUntil = null;
+    }
+
+    // Remove expired attempts
+    const windowStart = now - this.windowMs;
+    record.attempts = record.attempts.filter(
+      (timestamp) => timestamp > windowStart,
+    );
+
+    // Check if exceeded limit
+    if (record.attempts.length >= this.maxAttempts) {
+      // Block the identifier
+      record.blockedUntil = now + this.blockDurationMs;
+      return {
+        allowed: false,
+        remaining: 0,
+        resetAt: new Date(record.blockedUntil),
+        blockedUntil: new Date(record.blockedUntil),
+        retryAfter: Math.ceil(this.blockDurationMs / 1000), // seconds
+      };
+    }
+
+    // Allow request and record attempt
+    record.attempts.push(now);
+
+    return {
+      allowed: true,
+      remaining: this.maxAttempts - record.attempts.length,
+      resetAt: new Date(now + this.windowMs),
+      blockedUntil: null,
+      retryAfter: 0,
+    };
+  }
+
+  /**
+   * Manually reset rate limit for an identifier
+   * @param {string} identifier - Identifier to reset
+   */
+  reset(identifier) {
+    const key = this._hashIdentifier(identifier);
+    this.store.delete(key);
+  }
+
+  /**
+   * Get current status for an identifier
+   * @param {string} identifier - Identifier to check
+   * @returns {Object} Current rate limit status
+   */
+  getStatus(identifier) {
+    const key = this._hashIdentifier(identifier);
+    const record = this.store.get(key);
+    const now = Date.now();
+
+    if (!record) {
+      return {
+        attempts: 0,
+        remaining: this.maxAttempts,
+        blocked: false,
+        blockedUntil: null,
+      };
+    }
+
+    // Remove expired attempts
+    const windowStart = now - this.windowMs;
+    const validAttempts = record.attempts.filter(
+      (timestamp) => timestamp > windowStart,
+    );
+
+    const blocked = !!(record.blockedUntil && record.blockedUntil > now);
+
+    return {
+      attempts: validAttempts.length,
+      remaining: Math.max(0, this.maxAttempts - validAttempts.length),
+      blocked,
+      blockedUntil: blocked ? new Date(record.blockedUntil) : null,
+    };
+  }
+
+  /**
+   * Clean up expired entries
+   */
+  cleanup() {
+    const now = Date.now();
+    const expiredKeys = [];
+
+    for (const [key, record] of this.store.entries()) {
+      // Check if block has expired and no recent attempts
+      const blockExpired = !record.blockedUntil || record.blockedUntil < now;
+      const windowStart = now - this.windowMs;
+      const hasRecentAttempts = record.attempts.some(
+        (timestamp) => timestamp > windowStart,
+      );
+
+      if (blockExpired && !hasRecentAttempts) {
+        expiredKeys.push(key);
+      }
+    }
+
+    // Delete expired entries
+    expiredKeys.forEach((key) => this.store.delete(key));
+
+    return {
+      cleaned: expiredKeys.length,
+      remaining: this.store.size,
+    };
+  }
+
+  /**
+   * Get statistics about rate limiter usage
+   * @returns {Object} Statistics
+   */
+  getStats() {
+    const now = Date.now();
+    let totalAttempts = 0;
+    let blockedIdentifiers = 0;
+    let activeIdentifiers = 0;
+
+    for (const record of this.store.values()) {
+      const windowStart = now - this.windowMs;
+      const validAttempts = record.attempts.filter(
+        (timestamp) => timestamp > windowStart,
+      );
+
+      if (validAttempts.length > 0) {
+        activeIdentifiers++;
+        totalAttempts += validAttempts.length;
+      }
+
+      if (record.blockedUntil && record.blockedUntil > now) {
+        blockedIdentifiers++;
+      }
+    }
+
+    return {
+      totalIdentifiers: this.store.size,
+      activeIdentifiers,
+      blockedIdentifiers,
+      totalAttempts,
+      averageAttemptsPerIdentifier:
+        activeIdentifiers > 0 ? totalAttempts / activeIdentifiers : 0,
+    };
+  }
+
+  /**
+   * Stop the rate limiter and cleanup
+   */
+  destroy() {
+    if (this._cleanupInterval) {
+      clearInterval(this._cleanupInterval);
+      this._cleanupInterval = null;
+    }
+    this.store.clear();
+  }
+
+  /**
+   * Hash identifier for storage (privacy)
+   * @private
+   */
+  _hashIdentifier(identifier) {
+    return crypto.createHash('sha256').update(identifier).digest('hex');
+  }
+
+  /**
+   * Start automatic cleanup timer
+   * @private
+   */
+  _startCleanup() {
+    this._cleanupInterval = setInterval(() => {
+      this.cleanup();
+    }, this.cleanupIntervalMs);
+
+    // Don't prevent process exit
+    if (this._cleanupInterval.unref) {
+      this._cleanupInterval.unref();
+    }
+  }
+}
+
+module.exports = RateLimiter;
+module.exports.default = module.exports;

package/src/utils/secure-logger-config.js

@@ -0,0 +1,352 @@
+'use strict';
+
+/**
+ * Secure Logger Configuration Loader
+ *
+ * Loads and applies redaction pattern configurations from JSON files
+ * or JavaScript objects to create configured SecureLogger instances.
+ *
+ * @module secure-logger-config
+ * @since 2.2.5
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { SecureLogger } = require('./secure-logger');
+
+/**
+ * Load configuration from JSON file
+ *
+ * @param {string} configPath - Path to configuration file
+ * @returns {Object} Configuration object
+ * @throws {Error} If file cannot be read or parsed
+ *
+ * @private
+ */
+function loadConfigFile(configPath) {
+  try {
+    const content = fs.readFileSync(configPath, 'utf8');
+    return JSON.parse(content);
+  } catch (error) {
+    throw new Error(
+      `Failed to load redaction config from ${configPath}: ${error.message}`,
+    );
+  }
+}
+
+/**
+ * Convert regex string to RegExp object
+ *
+ * @param {string} regexStr - Regex pattern as string
+ * @param {string} flags - Regex flags
+ * @returns {RegExp} Compiled regular expression
+ *
+ * @private
+ */
+function compileRegex(regexStr, flags = 'g') {
+  try {
+    return new RegExp(regexStr, flags);
+  } catch (error) {
+    throw new Error(`Invalid regex pattern: ${regexStr} - ${error.message}`);
+  }
+}
+
+/**
+ * Apply environment-specific configuration overrides
+ *
+ * @param {Object} config - Base configuration
+ * @param {string} [environment] - Environment name (defaults to NODE_ENV)
+ * @returns {Object} Configuration with environment overrides applied
+ *
+ * @private
+ */
+function applyEnvironmentOverrides(config, environment) {
+  const env = environment || process.env.NODE_ENV || 'development';
+
+  if (config.environments && config.environments[env]) {
+    return {
+      ...config,
+      ...config.environments[env],
+    };
+  }
+
+  return config;
+}
+
+/**
+ * Convert custom pattern configuration to SecureLogger format
+ *
+ * @param {Object} customPatterns - Custom patterns from config
+ * @returns {Object} Patterns in SecureLogger format
+ *
+ * @private
+ */
+function convertCustomPatterns(customPatterns) {
+  const converted = {};
+
+  for (const [name, pattern] of Object.entries(customPatterns)) {
+    converted[name] = {
+      regex: compileRegex(pattern.regex, pattern.flags || 'g'),
+      replacement: pattern.replacement,
+      description: pattern.description || '',
+      enabled: pattern.enabled !== false,
+    };
+  }
+
+  return converted;
+}
+
+/**
+ * Create SecureLogger from configuration
+ *
+ * @param {Object|string} config - Configuration object or path to config file
+ * @param {string} [environment] - Environment name for overrides
+ * @returns {SecureLogger} Configured SecureLogger instance
+ *
+ * @example
+ * // Load from file
+ * const logger = createLoggerFromConfig('./redaction-config.json');
+ *
+ * // Load from object
+ * const logger = createLoggerFromConfig({
+ *   enabled: true,
+ *   redactEmails: true,
+ *   customPatterns: {
+ *     myPattern: {
+ *       regex: '\\bMY-[A-Z0-9]+\\b',
+ *       replacement: '[MY_TOKEN]'
+ *     }
+ *   }
+ * });
+ *
+ * // Load with environment override
+ * const logger = createLoggerFromConfig(config, 'production');
+ */
+function createLoggerFromConfig(config, environment) {
+  let configuration;
+
+  // Load from file if string path provided
+  if (typeof config === 'string') {
+    configuration = loadConfigFile(config);
+  } else {
+    configuration = config;
+  }
+
+  // Apply environment-specific overrides
+  configuration = applyEnvironmentOverrides(configuration, environment);
+
+  // Build SecureLogger options
+  const options = {
+    enabled: configuration.enabled !== false,
+    trackStats: configuration.trackStats !== false,
+    redactionMarker: configuration.redactionMarker,
+    redactEmails: configuration.redactEmails || false,
+    redactIPs: configuration.redactIPs || false,
+  };
+
+  // Add custom patterns
+  if (configuration.customPatterns) {
+    options.patterns = convertCustomPatterns(configuration.customPatterns);
+  }
+
+  // Add sensitive fields
+  if (
+    configuration.sensitiveFields &&
+    Array.isArray(configuration.sensitiveFields)
+  ) {
+    options.sensitiveFields = configuration.sensitiveFields;
+  }
+
+  // Create logger instance
+  const logger = new SecureLogger(options);
+
+  // Handle enabled/disabled patterns
+  if (
+    configuration.disabledPatterns &&
+    Array.isArray(configuration.disabledPatterns)
+  ) {
+    configuration.disabledPatterns.forEach((patternName) => {
+      logger.removePattern(patternName);
+    });
+  }
+
+  if (
+    configuration.enabledPatterns &&
+    Array.isArray(configuration.enabledPatterns)
+  ) {
+    // If specific patterns are listed, disable all others
+    const activePatterns = logger.getActivePatterns();
+    activePatterns.forEach((pattern) => {
+      if (!configuration.enabledPatterns.includes(pattern.name)) {
+        logger.removePattern(pattern.name);
+      }
+    });
+  }
+
+  return logger;
+}
+
+/**
+ * Load default configuration
+ *
+ * Attempts to load configuration from default locations:
+ * 1. ./redaction-config.json (current directory)
+ * 2. ./config/redaction-patterns.json
+ * 3. src/config/redaction-patterns.json
+ *
+ * @param {string} [environment] - Environment name for overrides
+ * @returns {SecureLogger} Configured logger or default logger if no config found
+ *
+ * @example
+ * const logger = loadDefaultConfig();
+ * const prodLogger = loadDefaultConfig('production');
+ */
+function loadDefaultConfig(environment) {
+  const defaultPaths = [
+    path.join(process.cwd(), 'redaction-config.json'),
+    path.join(process.cwd(), 'config', 'redaction-patterns.json'),
+    path.join(__dirname, '..', 'config', 'redaction-patterns.json'),
+  ];
+
+  for (const configPath of defaultPaths) {
+    if (fs.existsSync(configPath)) {
+      try {
+        return createLoggerFromConfig(configPath, environment);
+      } catch (error) {
+        console.warn(
+          `Failed to load config from ${configPath}:`,
+          error.message,
+        );
+      }
+    }
+  }
+
+  // Return default logger if no config found
+  return new SecureLogger();
+}
+
+/**
+ * Validate configuration against schema
+ *
+ * @param {Object} config - Configuration to validate
+ * @returns {Object} Validation result { valid: boolean, errors: string[] }
+ *
+ * @example
+ * const result = validateConfig(myConfig);
+ * if (!result.valid) {
+ *   console.error('Config errors:', result.errors);
+ * }
+ */
+function validateConfig(config) {
+  const errors = [];
+
+  // Check required fields
+  if (config.enabled !== undefined && typeof config.enabled !== 'boolean') {
+    errors.push('enabled must be a boolean');
+  }
+
+  if (
+    config.trackStats !== undefined &&
+    typeof config.trackStats !== 'boolean'
+  ) {
+    errors.push('trackStats must be a boolean');
+  }
+
+  if (
+    config.redactionMarker !== undefined &&
+    typeof config.redactionMarker !== 'string'
+  ) {
+    errors.push('redactionMarker must be a string');
+  }
+
+  // Validate custom patterns
+  if (config.customPatterns && typeof config.customPatterns === 'object') {
+    for (const [name, pattern] of Object.entries(config.customPatterns)) {
+      if (!pattern.regex) {
+        errors.push(`Custom pattern '${name}' missing regex field`);
+      }
+      if (!pattern.replacement) {
+        errors.push(`Custom pattern '${name}' missing replacement field`);
+      }
+
+      // Try to compile regex
+      if (pattern.regex) {
+        try {
+          compileRegex(pattern.regex, pattern.flags || 'g');
+        } catch (error) {
+          errors.push(
+            `Custom pattern '${name}' has invalid regex: ${error.message}`,
+          );
+        }
+      }
+    }
+  }
+
+  // Validate sensitive fields
+  if (config.sensitiveFields !== undefined) {
+    if (!Array.isArray(config.sensitiveFields)) {
+      errors.push('sensitiveFields must be an array');
+    } else {
+      config.sensitiveFields.forEach((field, index) => {
+        if (typeof field !== 'string') {
+          errors.push(`sensitiveFields[${index}] must be a string`);
+        }
+      });
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Create configuration template
+ *
+ * @returns {Object} Template configuration object
+ *
+ * @example
+ * const template = createConfigTemplate();
+ * // Modify template as needed
+ * const logger = createLoggerFromConfig(template);
+ */
+function createConfigTemplate() {
+  return {
+    enabled: true,
+    trackStats: true,
+    redactionMarker: '[REDACTED]',
+    redactEmails: false,
+    redactIPs: false,
+    customPatterns: {},
+    sensitiveFields: [],
+    disabledPatterns: [],
+    enabledPatterns: [],
+    environments: {
+      development: {
+        enabled: false,
+        trackStats: false,
+      },
+      production: {
+        enabled: true,
+        trackStats: true,
+        redactEmails: true,
+        redactIPs: true,
+      },
+      test: {
+        enabled: false,
+      },
+    },
+  };
+}
+
+module.exports = {
+  createLoggerFromConfig,
+  loadDefaultConfig,
+  validateConfig,
+  createConfigTemplate,
+  loadConfigFile,
+  compileRegex,
+  applyEnvironmentOverrides,
+  convertCustomPatterns,
+};