@joystick.js/db-canary 0.0.0-canary.2270 → 0.0.0-canary.2271

package/src/server/lib/processing_lane.js

@@ -1,417 +0,0 @@
-/**
- * @fileoverview Processing lane for batched write operations.
- * Each lane processes operations independently to enable parallel processing
- * while maintaining operation ordering within each lane.
- */
-
-import create_logger from './logger.js';
-
-const { create_context_logger } = create_logger('processing_lane');
-
-/**
- * Processing lane that batches and processes write operations independently.
- * Provides batching, timeout handling, and transaction management per lane.
- */
-class ProcessingLane {
-  /**
-   * Creates a new ProcessingLane instance.
-   * @param {Object} options - Configuration options
-   * @param {number} [options.batch_size=100] - Maximum operations per batch
-   * @param {number} [options.batch_timeout=10] - Maximum wait time in milliseconds
-   * @param {number} [options.lane_id=0] - Unique identifier for this lane
-   */
-  constructor(options = {}) {
-    this.batch_size = options.batch_size || 100;
-    this.batch_timeout = options.batch_timeout || 10;
-    this.lane_id = options.lane_id || 0;
-    
-    /** @type {Array<Object>} Current batch of operations */
-    this.current_batch = [];
-    
-    /** @type {boolean} Whether lane is currently processing a batch */
-    this.processing = false;
-    
-    /** @type {boolean} Whether lane is shutting down */
-    this.shutting_down = false;
-    
-    /** @type {NodeJS.Timeout|null} Timeout handle for batch processing */
-    this.batch_timeout_handle = null;
-    
-    /** @type {Object} Lane-specific statistics */
-    this.stats = {
-      total_operations: 0,
-      completed_operations: 0,
-      failed_operations: 0,
-      batches_processed: 0,
-      current_batch_size: 0,
-      max_batch_size: 0,
-      total_batch_wait_time_ms: 0,
-      total_batch_processing_time_ms: 0
-    };
-    
-    this.log = create_context_logger(`lane_${this.lane_id}`);
-  }
-
-  /**
-   * Adds an operation to this lane's batch queue.
-   * @param {Object} operation - Operation to add to batch
-   * @param {function} operation.operation_fn - Async function that performs the write operation
-   * @param {Object} [operation.context={}] - Additional context for logging and debugging
-   * @returns {Promise<*>} Promise that resolves with the operation result
-   * @throws {Error} When lane is shutting down
-   */
-  async add_operation(operation) {
-    if (this.shutting_down) {
-      throw new Error('Processing lane shutting down');
-    }
-    
-    return new Promise((resolve, reject) => {
-      if (this.shutting_down) {
-        reject(new Error('Processing lane shutting down'));
-        return;
-      }
-      
-      const batch_item = {
-        ...operation,
-        resolve,
-        reject,
-        enqueued_at: Date.now(),
-        id: this.generate_operation_id()
-      };
-
-      this.current_batch.push(batch_item);
-      this.stats.total_operations++;
-      this.stats.current_batch_size = this.current_batch.length;
-      
-      if (this.stats.current_batch_size > this.stats.max_batch_size) {
-        this.stats.max_batch_size = this.stats.current_batch_size;
-      }
-
-      this.log.debug('Operation added to batch', {
-        lane_id: this.lane_id,
-        operation_id: batch_item.id,
-        batch_size: this.stats.current_batch_size,
-        context: operation.context
-      });
-
-      // Process batch if it reaches the configured size
-      if (this.current_batch.length >= this.batch_size) {
-        this.process_current_batch();
-      } else if (this.current_batch.length === 1) {
-        // Start timeout for first operation in batch
-        this.start_batch_timeout();
-      }
-    });
-  }
-
-  /**
-   * Starts the batch timeout to ensure batches are processed within time limit.
-   */
-  start_batch_timeout() {
-    if (this.batch_timeout_handle) {
-      clearTimeout(this.batch_timeout_handle);
-    }
-    
-    this.batch_timeout_handle = setTimeout(() => {
-      if (this.current_batch.length > 0 && !this.processing) {
-        this.log.debug('Batch timeout triggered', {
-          lane_id: this.lane_id,
-          batch_size: this.current_batch.length
-        });
-        this.process_current_batch();
-      }
-    }, this.batch_timeout);
-  }
-
-  /**
-   * Processes the current batch of operations in a single transaction.
-   * @returns {Promise<void>} Promise that resolves when batch processing is complete
-   */
-  async process_current_batch() {
-    if (this.processing || this.current_batch.length === 0 || this.shutting_down) {
-      return;
-    }
-
-    // Clear timeout since we're processing now
-    if (this.batch_timeout_handle) {
-      clearTimeout(this.batch_timeout_handle);
-      this.batch_timeout_handle = null;
-    }
-
-    this.processing = true;
-    const batch_to_process = [...this.current_batch];
-    this.current_batch = [];
-    this.stats.current_batch_size = 0;
-    
-    const batch_start_time = Date.now();
-    const oldest_operation_time = Math.min(...batch_to_process.map(op => op.enqueued_at));
-    const batch_wait_time_ms = batch_start_time - oldest_operation_time;
-    
-    this.stats.total_batch_wait_time_ms += batch_wait_time_ms;
-    this.stats.batches_processed++;
-
-    this.log.debug('Processing batch', {
-      lane_id: this.lane_id,
-      batch_size: batch_to_process.length,
-      batch_wait_time_ms
-    });
-
-    try {
-      // Execute all operations in the batch within a single transaction context
-      const results = await this.execute_batch_transaction(batch_to_process);
-      
-      const batch_processing_time_ms = Date.now() - batch_start_time;
-      this.stats.total_batch_processing_time_ms += batch_processing_time_ms;
-      this.stats.completed_operations += batch_to_process.length;
-
-      // Resolve all operations with their results
-      batch_to_process.forEach((operation, index) => {
-        operation.resolve(results[index]);
-      });
-
-      this.log.debug('Batch completed successfully', {
-        lane_id: this.lane_id,
-        batch_size: batch_to_process.length,
-        batch_wait_time_ms,
-        batch_processing_time_ms
-      });
-
-    } catch (error) {
-      const batch_processing_time_ms = Date.now() - batch_start_time;
-      this.stats.total_batch_processing_time_ms += batch_processing_time_ms;
-      this.stats.failed_operations += batch_to_process.length;
-
-      // Reject all operations with the batch error
-      batch_to_process.forEach(operation => {
-        operation.reject(error);
-      });
-
-      this.log.error('Batch processing failed', {
-        lane_id: this.lane_id,
-        batch_size: batch_to_process.length,
-        batch_wait_time_ms,
-        batch_processing_time_ms,
-        error: error.message
-      });
-    }
-
-    this.processing = false;
-
-    // Process next batch if operations are waiting
-    if (this.current_batch.length > 0) {
-      if (this.current_batch.length >= this.batch_size) {
-        setImmediate(() => this.process_current_batch());
-      } else {
-        this.start_batch_timeout();
-      }
-    }
-  }
-
-  /**
-   * Executes all operations in a batch within a single transaction context.
-   * @param {Array<Object>} batch_operations - Operations to execute in batch
-   * @returns {Promise<Array<*>>} Promise that resolves with array of operation results
-   */
-  async execute_batch_transaction(batch_operations) {
-    const results = [];
-    
-    // Execute each operation and collect results with retry logic
-    for (const operation of batch_operations) {
-      try {
-        const result = await this.execute_with_retry(operation.operation_fn, operation.context);
-        results.push(result);
-      } catch (error) {
-        // If any operation fails, the entire batch fails
-        throw error;
-      }
-    }
-    
-    return results;
-  }
-
-  /**
-   * Executes an operation with retry logic and exponential backoff.
-   * @param {function} operation_fn - Async function to execute
-   * @param {Object} context - Context for logging
-   * @param {number} [max_retries=3] - Maximum number of retry attempts
-   * @returns {Promise<*>} Promise that resolves with operation result
-   * @throws {Error} When all retry attempts are exhausted
-   */
-  async execute_with_retry(operation_fn, context, max_retries = 3) {
-    let last_error = null;
-    
-    for (let attempt = 1; attempt <= max_retries; attempt++) {
-      try {
-        return await operation_fn();
-      } catch (error) {
-        last_error = error;
-        
-        if (this.is_retryable_error(error) && attempt < max_retries) {
-          const delay_ms = this.calculate_backoff_delay(attempt);
-          
-          this.log.warn('Operation failed, retrying', {
-            lane_id: this.lane_id,
-            attempt,
-            max_retries,
-            delay_ms,
-            error: error.message,
-            context
-          });
-          
-          await this.sleep(delay_ms);
-          continue;
-        }
-        
-        break;
-      }
-    }
-    
-    throw last_error;
-  }
-
-  /**
-   * Determines if an error is retryable based on error patterns.
-   * @param {Error} error - Error to check
-   * @returns {boolean} True if error is retryable, false otherwise
-   */
-  is_retryable_error(error) {
-    const retryable_patterns = [
-      'MDB_MAP_FULL',
-      'MDB_TXN_FULL',
-      'MDB_READERS_FULL',
-      'EAGAIN',
-      'EBUSY'
-    ];
-    
-    return retryable_patterns.some(pattern => 
-      error.message.includes(pattern) || error.code === pattern
-    );
-  }
-
-  /**
-   * Calculates exponential backoff delay with jitter for retry attempts.
-   * @param {number} attempt - Current attempt number (1-based)
-   * @returns {number} Delay in milliseconds
-   */
-  calculate_backoff_delay(attempt) {
-    const base_delay = 100;
-    const max_delay = 5000;
-    const exponential_delay = base_delay * Math.pow(2, attempt - 1);
-    const jitter = Math.random() * 0.1 * exponential_delay;
-    
-    return Math.min(exponential_delay + jitter, max_delay);
-  }
-
-  /**
-   * Utility function to sleep for specified milliseconds.
-   * @param {number} ms - Milliseconds to sleep
-   * @returns {Promise<void>} Promise that resolves after delay
-   */
-  sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-
-  /**
-   * Generates a unique operation ID for tracking.
-   * @returns {string} Unique operation identifier
-   */
-  generate_operation_id() {
-    return `lane_${this.lane_id}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-  }
-
-  /**
-   * Gets comprehensive lane statistics including calculated averages.
-   * @returns {Object} Statistics object with performance metrics
-   */
-  get_stats() {
-    const avg_batch_wait_time = this.stats.batches_processed > 0 
-      ? Math.round(this.stats.total_batch_wait_time_ms / this.stats.batches_processed)
-      : 0;
-    
-    const avg_batch_processing_time = this.stats.batches_processed > 0
-      ? Math.round(this.stats.total_batch_processing_time_ms / this.stats.batches_processed)
-      : 0;
-
-    const avg_batch_size = this.stats.batches_processed > 0
-      ? Math.round(this.stats.completed_operations / this.stats.batches_processed)
-      : 0;
-
-    return {
-      lane_id: this.lane_id,
-      ...this.stats,
-      avg_batch_wait_time_ms: avg_batch_wait_time,
-      avg_batch_processing_time_ms: avg_batch_processing_time,
-      avg_batch_size,
-      success_rate: this.stats.total_operations > 0 
-        ? Math.round((this.stats.completed_operations / this.stats.total_operations) * 100)
-        : 100
-    };
-  }
-
-  /**
-   * Clears all statistics while preserving current batch size.
-   */
-  clear_stats() {
-    this.stats = {
-      total_operations: 0,
-      completed_operations: 0,
-      failed_operations: 0,
-      batches_processed: 0,
-      current_batch_size: this.current_batch.length,
-      max_batch_size: 0,
-      total_batch_wait_time_ms: 0,
-      total_batch_processing_time_ms: 0
-    };
-  }
-
-  /**
-   * Forces processing of current batch regardless of size or timeout.
-   * @returns {Promise<void>} Promise that resolves when batch is processed
-   */
-  async flush_batch() {
-    if (this.current_batch.length > 0 && !this.processing) {
-      await this.process_current_batch();
-    }
-  }
-
-  /**
-   * Gracefully shuts down the processing lane.
-   * Processes any remaining operations and rejects new ones.
-   * @returns {Promise<void>} Promise that resolves when shutdown is complete
-   */
-  async shutdown() {
-    this.log.info('Shutting down processing lane', {
-      lane_id: this.lane_id,
-      pending_operations: this.current_batch.length,
-      currently_processing: this.processing
-    });
-
-    this.shutting_down = true;
-
-    // Clear timeout
-    if (this.batch_timeout_handle) {
-      clearTimeout(this.batch_timeout_handle);
-      this.batch_timeout_handle = null;
-    }
-
-    // Process any remaining operations
-    if (this.current_batch.length > 0 && !this.processing) {
-      await this.process_current_batch();
-    }
-
-    // Wait for current processing to complete
-    while (this.processing) {
-      await new Promise(resolve => setTimeout(resolve, 10));
-    }
-
-    // Reject any remaining operations
-    this.current_batch.forEach(operation => {
-      operation.reject(new Error('Processing lane shutting down'));
-    });
-    
-    this.current_batch = [];
-    this.processing = false;
-  }
-}
-
-export default ProcessingLane;