@dynamicu/chromedebug-mcp 2.2.0

package/src/capture/memory-manager.js

@@ -0,0 +1,713 @@
+/**
+ * Unified Interaction Capture System - Memory Management & Performance
+ *
+ * This module provides memory management and performance optimization
+ * for the capture system. It handles:
+ *
+ * 1. Memory usage monitoring and limits
+ * 2. Automatic cleanup and garbage collection
+ * 3. Event buffering and streaming
+ * 4. Performance optimization strategies
+ * 5. Backpressure handling
+ *
+ * MEMORY STRATEGY:
+ * ================
+ * • Hot Path: Recent events in memory for instant access
+ * • Warm Path: Session events in database for quick queries
+ * • Cold Path: Archived events in file system for long-term storage
+ * • Size-based routing: Small events cached, large events streamed
+ *
+ * PERFORMANCE STRATEGY:
+ * =====================
+ * • Async pipelines with backpressure
+ * • Batch processing for efficiency
+ * • Circular buffers for high-frequency events
+ * • Lazy loading for large datasets
+ * • Compression for storage optimization
+ */
+
+import { ICaptureMemoryManager } from './interfaces.js';
+import { ARCHITECTURE_CONFIG } from './architecture.js';
+
+/**
+ * Memory-efficient circular buffer for high-frequency events
+ */
+export class CircularEventBuffer {
+  constructor(maxSize = 1000) {
+    this.buffer = new Array(maxSize);
+    this.maxSize = maxSize;
+    this.head = 0;
+    this.tail = 0;
+    this.count = 0;
+    this.overflowCount = 0;
+  }
+
+  /**
+   * Add event to buffer
+   * @param {CaptureEvent} event - Event to add
+   * @returns {boolean} True if added, false if buffer full
+   */
+  add(event) {
+    this.buffer[this.tail] = event;
+    this.tail = (this.tail + 1) % this.maxSize;
+
+    if (this.count < this.maxSize) {
+      this.count++;
+      return true;
+    } else {
+      // Buffer full, overwrite oldest
+      this.head = (this.head + 1) % this.maxSize;
+      this.overflowCount++;
+      return false;
+    }
+  }
+
+  /**
+   * Get events from buffer
+   * @param {number} [limit] - Maximum events to return
+   * @returns {CaptureEvent[]} Events from buffer
+   */
+  getEvents(limit = this.count) {
+    const events = [];
+    const actualLimit = Math.min(limit, this.count);
+
+    for (let i = 0; i < actualLimit; i++) {
+      const index = (this.head + i) % this.maxSize;
+      events.push(this.buffer[index]);
+    }
+
+    return events;
+  }
+
+  /**
+   * Remove events from buffer
+   * @param {number} count - Number of events to remove
+   */
+  remove(count) {
+    const actualCount = Math.min(count, this.count);
+    this.head = (this.head + actualCount) % this.maxSize;
+    this.count -= actualCount;
+  }
+
+  /**
+   * Clear all events from buffer
+   */
+  clear() {
+    this.head = 0;
+    this.tail = 0;
+    this.count = 0;
+  }
+
+  /**
+   * Get buffer statistics
+   * @returns {Object} Buffer stats
+   */
+  getStats() {
+    return {
+      count: this.count,
+      maxSize: this.maxSize,
+      utilization: this.count / this.maxSize,
+      overflowCount: this.overflowCount,
+      memoryUsageBytes: this.estimateMemoryUsage()
+    };
+  }
+
+  /**
+   * Estimate memory usage of buffer
+   * @returns {number} Estimated bytes
+   */
+  estimateMemoryUsage() {
+    // Rough estimate: 1KB per event on average
+    return this.count * 1024;
+  }
+}
+
+/**
+ * Event batch processor for efficient database writes
+ */
+export class EventBatchProcessor {
+  constructor(storage, batchSize = 100, flushInterval = 5000) {
+    this.storage = storage;
+    this.batchSize = batchSize;
+    this.flushInterval = flushInterval;
+    this.batch = [];
+    this.flushTimer = null;
+    this.processing = false;
+    this.stats = {
+      batchesProcessed: 0,
+      eventsProcessed: 0,
+      errors: 0,
+      avgBatchSize: 0
+    };
+  }
+
+  /**
+   * Add event to batch
+   * @param {CaptureEvent} event - Event to add
+   * @returns {Promise<void>}
+   */
+  async addEvent(event) {
+    this.batch.push(event);
+
+    // Flush if batch is full
+    if (this.batch.length >= this.batchSize) {
+      await this.flush();
+    } else if (!this.flushTimer) {
+      // Start flush timer if not already running
+      this.flushTimer = setTimeout(() => this.flush(), this.flushInterval);
+    }
+  }
+
+  /**
+   * Flush current batch to storage
+   * @returns {Promise<void>}
+   */
+  async flush() {
+    if (this.processing || this.batch.length === 0) {
+      return;
+    }
+
+    this.processing = true;
+
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    const currentBatch = this.batch.splice(0);
+
+    try {
+      await this.storage.storeEvents(currentBatch);
+
+      // Update statistics
+      this.stats.batchesProcessed++;
+      this.stats.eventsProcessed += currentBatch.length;
+      this.stats.avgBatchSize = this.stats.eventsProcessed / this.stats.batchesProcessed;
+
+    } catch (error) {
+      this.stats.errors++;
+      console.error('Error flushing event batch:', error);
+
+      // Re-add events to batch for retry
+      this.batch.unshift(...currentBatch);
+    } finally {
+      this.processing = false;
+    }
+  }
+
+  /**
+   * Stop processor and flush remaining events
+   * @returns {Promise<void>}
+   */
+  async stop() {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    await this.flush();
+  }
+
+  /**
+   * Get processor statistics
+   * @returns {Object} Processor stats
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      pendingEvents: this.batch.length,
+      processing: this.processing
+    };
+  }
+}
+
+/**
+ * Memory usage monitor
+ */
+export class MemoryMonitor {
+  constructor() {
+    this.components = new Map();
+    this.limits = {
+      maxMemoryMB: ARCHITECTURE_CONFIG.MEMORY.MAX_MEMORY_MB,
+      warningThresholdMB: ARCHITECTURE_CONFIG.MEMORY.WARNING_THRESHOLD_MB,
+      criticalThresholdMB: ARCHITECTURE_CONFIG.MEMORY.CRITICAL_THRESHOLD_MB
+    };
+    this.callbacks = {
+      warning: [],
+      critical: [],
+      cleanup: []
+    };
+    this.monitoring = false;
+    this.monitorInterval = null;
+  }
+
+  /**
+   * Register component for monitoring
+   * @param {string} componentId - Component identifier
+   * @param {Object} component - Component with getMemoryUsage method
+   */
+  registerComponent(componentId, component) {
+    this.components.set(componentId, component);
+  }
+
+  /**
+   * Unregister component
+   * @param {string} componentId - Component identifier
+   */
+  unregisterComponent(componentId) {
+    this.components.delete(componentId);
+  }
+
+  /**
+   * Start memory monitoring
+   * @param {number} [intervalMs=30000] - Monitoring interval
+   */
+  startMonitoring(intervalMs = 30000) {
+    if (this.monitoring) return;
+
+    this.monitoring = true;
+    this.monitorInterval = setInterval(() => {
+      this.checkMemoryUsage();
+    }, intervalMs);
+  }
+
+  /**
+   * Stop memory monitoring
+   */
+  stopMonitoring() {
+    if (!this.monitoring) return;
+
+    this.monitoring = false;
+    if (this.monitorInterval) {
+      clearInterval(this.monitorInterval);
+      this.monitorInterval = null;
+    }
+  }
+
+  /**
+   * Check current memory usage
+   * @returns {Promise<Object>} Memory usage statistics
+   */
+  async checkMemoryUsage() {
+    const usage = await this.getMemoryUsage();
+    const totalMB = usage.totalBytes / (1024 * 1024);
+
+    // Trigger callbacks based on usage
+    if (totalMB >= this.limits.criticalThresholdMB) {
+      this.triggerCallbacks('critical', usage);
+    } else if (totalMB >= this.limits.warningThresholdMB) {
+      this.triggerCallbacks('warning', usage);
+    }
+
+    return usage;
+  }
+
+  /**
+   * Get comprehensive memory usage
+   * @returns {Promise<Object>} Memory usage data
+   */
+  async getMemoryUsage() {
+    const componentUsage = new Map();
+    let totalBytes = 0;
+
+    // Get usage from registered components
+    for (const [id, component] of this.components) {
+      try {
+        let usage = 0;
+        if (typeof component.getMemoryUsage === 'function') {
+          usage = await component.getMemoryUsage();
+        } else if (typeof component.estimateMemoryUsage === 'function') {
+          usage = component.estimateMemoryUsage();
+        }
+
+        componentUsage.set(id, usage);
+        totalBytes += usage;
+      } catch (error) {
+        console.warn(`Error getting memory usage for component ${id}:`, error);
+        componentUsage.set(id, 0);
+      }
+    }
+
+    // Get Node.js process memory usage
+    const processUsage = process.memoryUsage();
+
+    return {
+      totalBytes,
+      totalMB: totalBytes / (1024 * 1024),
+      componentUsage: Object.fromEntries(componentUsage),
+      processUsage,
+      limits: this.limits,
+      timestamp: Date.now()
+    };
+  }
+
+  /**
+   * Register callback for memory events
+   * @param {string} event - Event type (warning, critical, cleanup)
+   * @param {Function} callback - Callback function
+   */
+  onMemoryEvent(event, callback) {
+    if (!this.callbacks[event]) {
+      this.callbacks[event] = [];
+    }
+    this.callbacks[event].push(callback);
+  }
+
+  /**
+   * Trigger callbacks for event type
+   * @param {string} event - Event type
+   * @param {Object} data - Event data
+   */
+  triggerCallbacks(event, data) {
+    const callbacks = this.callbacks[event] || [];
+    for (const callback of callbacks) {
+      try {
+        callback(data);
+      } catch (error) {
+        console.error(`Error in memory event callback for ${event}:`, error);
+      }
+    }
+  }
+
+  /**
+   * Request immediate cleanup
+   * @param {string} [priority='normal'] - Cleanup priority
+   * @returns {Promise<Object>} Cleanup results
+   */
+  async requestCleanup(priority = 'normal') {
+    const results = {
+      beforeUsage: await this.getMemoryUsage(),
+      componentsProcessed: 0,
+      bytesFreed: 0,
+      errors: []
+    };
+
+    // Trigger cleanup callbacks
+    this.triggerCallbacks('cleanup', { priority, results });
+
+    // Call cleanup on registered components
+    for (const [id, component] of this.components) {
+      try {
+        if (typeof component.cleanup === 'function') {
+          const beforeBytes = component.estimateMemoryUsage?.() || 0;
+          await component.cleanup(priority);
+          const afterBytes = component.estimateMemoryUsage?.() || 0;
+
+          results.componentsProcessed++;
+          results.bytesFreed += Math.max(0, beforeBytes - afterBytes);
+        }
+      } catch (error) {
+        console.error(`Error cleaning up component ${id}:`, error);
+        results.errors.push({ component: id, error: error.message });
+      }
+    }
+
+    results.afterUsage = await this.getMemoryUsage();
+
+    return results;
+  }
+}
+
+/**
+ * Backpressure manager for flow control
+ */
+export class BackpressureManager {
+  constructor() {
+    this.thresholds = {
+      warning: 0.7,   // 70% capacity
+      critical: 0.9,  // 90% capacity
+      max: 1.0        // 100% capacity
+    };
+    this.components = new Map();
+    this.state = 'normal'; // normal, warning, critical, blocked
+    this.callbacks = new Map();
+  }
+
+  /**
+   * Register component for backpressure monitoring
+   * @param {string} componentId - Component identifier
+   * @param {Object} component - Component with capacity info
+   */
+  registerComponent(componentId, component) {
+    this.components.set(componentId, {
+      component,
+      capacity: component.getCapacity?.() || 1.0,
+      utilization: 0
+    });
+  }
+
+  /**
+   * Update component utilization
+   * @param {string} componentId - Component identifier
+   * @param {number} utilization - Current utilization (0-1)
+   */
+  updateUtilization(componentId, utilization) {
+    const info = this.components.get(componentId);
+    if (info) {
+      info.utilization = utilization;
+      this.checkBackpressure();
+    }
+  }
+
+  /**
+   * Check if backpressure should be applied
+   */
+  checkBackpressure() {
+    let maxUtilization = 0;
+
+    for (const [id, info] of this.components) {
+      maxUtilization = Math.max(maxUtilization, info.utilization);
+    }
+
+    const newState = this.calculateState(maxUtilization);
+
+    if (newState !== this.state) {
+      const oldState = this.state;
+      this.state = newState;
+      this.notifyStateChange(oldState, newState, maxUtilization);
+    }
+  }
+
+  /**
+   * Calculate backpressure state
+   * @param {number} utilization - Maximum utilization
+   * @returns {string} New state
+   */
+  calculateState(utilization) {
+    if (utilization >= this.thresholds.max) {
+      return 'blocked';
+    } else if (utilization >= this.thresholds.critical) {
+      return 'critical';
+    } else if (utilization >= this.thresholds.warning) {
+      return 'warning';
+    } else {
+      return 'normal';
+    }
+  }
+
+  /**
+   * Notify components of state change
+   * @param {string} oldState - Previous state
+   * @param {string} newState - New state
+   * @param {number} utilization - Current utilization
+   */
+  notifyStateChange(oldState, newState, utilization) {
+    const callbacks = this.callbacks.get(newState) || [];
+    for (const callback of callbacks) {
+      try {
+        callback({ oldState, newState, utilization });
+      } catch (error) {
+        console.error('Error in backpressure callback:', error);
+      }
+    }
+  }
+
+  /**
+   * Register callback for state changes
+   * @param {string} state - State to listen for
+   * @param {Function} callback - Callback function
+   */
+  onStateChange(state, callback) {
+    if (!this.callbacks.has(state)) {
+      this.callbacks.set(state, []);
+    }
+    this.callbacks.get(state).push(callback);
+  }
+
+  /**
+   * Check if system should throttle input
+   * @returns {boolean} True if should throttle
+   */
+  shouldThrottle() {
+    return this.state === 'critical' || this.state === 'blocked';
+  }
+
+  /**
+   * Get suggested delay for throttling
+   * @returns {number} Delay in milliseconds
+   */
+  getThrottleDelay() {
+    switch (this.state) {
+      case 'warning': return 10;
+      case 'critical': return 100;
+      case 'blocked': return 1000;
+      default: return 0;
+    }
+  }
+}
+
+/**
+ * Main memory manager implementation
+ */
+export class CaptureMemoryManager extends ICaptureMemoryManager {
+  constructor() {
+    super();
+    this.monitor = new MemoryMonitor();
+    this.backpressure = new BackpressureManager();
+    this.buffers = new Map();
+    this.processors = new Map();
+    this.initialized = false;
+  }
+
+  /**
+   * Initialize the memory manager
+   * @param {Object} config - Memory management configuration
+   * @returns {Promise<void>}
+   */
+  async initialize(config = {}) {
+    if (this.initialized) return;
+
+    // Apply configuration
+    Object.assign(this.monitor.limits, config.limits || {});
+    Object.assign(this.backpressure.thresholds, config.thresholds || {});
+
+    // Set up memory monitoring
+    this.monitor.onMemoryEvent('critical', async (usage) => {
+      console.warn('Critical memory usage detected:', usage.totalMB, 'MB');
+      await this.requestCleanup('aggressive');
+    });
+
+    this.monitor.onMemoryEvent('warning', (usage) => {
+      console.info('High memory usage detected:', usage.totalMB, 'MB');
+    });
+
+    // Set up backpressure monitoring
+    this.backpressure.onStateChange('critical', ({ utilization }) => {
+      console.warn('Critical backpressure detected, utilization:', utilization);
+    });
+
+    this.backpressure.onStateChange('blocked', ({ utilization }) => {
+      console.error('System blocked due to backpressure, utilization:', utilization);
+    });
+
+    // Start monitoring
+    this.monitor.startMonitoring();
+
+    this.initialized = true;
+  }
+
+  /**
+   * Check current memory usage
+   * @returns {Promise<Object>} Memory usage statistics
+   */
+  async getMemoryUsage() {
+    return await this.monitor.getMemoryUsage();
+  }
+
+  /**
+   * Request memory cleanup
+   * @param {string} [priority='normal'] - Cleanup priority
+   * @returns {Promise<Object>} Cleanup results
+   */
+  async requestCleanup(priority = 'normal') {
+    return await this.monitor.requestCleanup(priority);
+  }
+
+  /**
+   * Set memory limits
+   * @param {Object} limits - Memory limits configuration
+   * @returns {Promise<void>}
+   */
+  async setLimits(limits) {
+    Object.assign(this.monitor.limits, limits);
+  }
+
+  /**
+   * Register component for memory monitoring
+   * @param {string} componentId - Component identifier
+   * @param {Object} component - Component to monitor
+   * @returns {Promise<void>}
+   */
+  async registerComponent(componentId, component) {
+    this.monitor.registerComponent(componentId, component);
+
+    if (component.getCapacity) {
+      this.backpressure.registerComponent(componentId, component);
+    }
+  }
+
+  /**
+   * Create optimized event buffer
+   * @param {string} bufferId - Buffer identifier
+   * @param {Object} options - Buffer options
+   * @returns {CircularEventBuffer} Created buffer
+   */
+  createBuffer(bufferId, options = {}) {
+    const buffer = new CircularEventBuffer(options.maxSize);
+    this.buffers.set(bufferId, buffer);
+    this.monitor.registerComponent(`buffer_${bufferId}`, buffer);
+    return buffer;
+  }
+
+  /**
+   * Create batch processor
+   * @param {string} processorId - Processor identifier
+   * @param {Object} storage - Storage backend
+   * @param {Object} options - Processor options
+   * @returns {EventBatchProcessor} Created processor
+   */
+  createBatchProcessor(processorId, storage, options = {}) {
+    const processor = new EventBatchProcessor(
+      storage,
+      options.batchSize,
+      options.flushInterval
+    );
+    this.processors.set(processorId, processor);
+    this.monitor.registerComponent(`processor_${processorId}`, processor);
+    return processor;
+  }
+
+  /**
+   * Get system performance metrics
+   * @returns {Promise<Object>} Performance metrics
+   */
+  async getPerformanceMetrics() {
+    const memoryUsage = await this.getMemoryUsage();
+    const bufferStats = {};
+    const processorStats = {};
+
+    for (const [id, buffer] of this.buffers) {
+      bufferStats[id] = buffer.getStats();
+    }
+
+    for (const [id, processor] of this.processors) {
+      processorStats[id] = processor.getStats();
+    }
+
+    return {
+      memory: memoryUsage,
+      backpressure: {
+        state: this.backpressure.state,
+        shouldThrottle: this.backpressure.shouldThrottle(),
+        throttleDelay: this.backpressure.getThrottleDelay()
+      },
+      buffers: bufferStats,
+      processors: processorStats,
+      timestamp: Date.now()
+    };
+  }
+
+  /**
+   * Shutdown memory manager
+   * @returns {Promise<void>}
+   */
+  async shutdown() {
+    this.monitor.stopMonitoring();
+
+    // Stop all processors
+    const stopPromises = [];
+    for (const processor of this.processors.values()) {
+      stopPromises.push(processor.stop());
+    }
+    await Promise.all(stopPromises);
+
+    // Clear all buffers
+    for (const buffer of this.buffers.values()) {
+      buffer.clear();
+    }
+
+    this.initialized = false;
+  }
+}
+
+export default CaptureMemoryManager;