@joystick.js/db-canary 0.0.0-canary.2268 → 0.0.0-canary.2270

package/src/server/lib/processing_lane.js

@@ -0,0 +1,417 @@
+/**
+ * @fileoverview Processing lane for batched write operations.
+ * Each lane processes operations independently to enable parallel processing
+ * while maintaining operation ordering within each lane.
+ */
+
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('processing_lane');
+
+/**
+ * Processing lane that batches and processes write operations independently.
+ * Provides batching, timeout handling, and transaction management per lane.
+ */
+class ProcessingLane {
+  /**
+   * Creates a new ProcessingLane instance.
+   * @param {Object} options - Configuration options
+   * @param {number} [options.batch_size=100] - Maximum operations per batch
+   * @param {number} [options.batch_timeout=10] - Maximum wait time in milliseconds
+   * @param {number} [options.lane_id=0] - Unique identifier for this lane
+   */
+  constructor(options = {}) {
+    this.batch_size = options.batch_size || 100;
+    this.batch_timeout = options.batch_timeout || 10;
+    this.lane_id = options.lane_id || 0;
+    
+    /** @type {Array<Object>} Current batch of operations */
+    this.current_batch = [];
+    
+    /** @type {boolean} Whether lane is currently processing a batch */
+    this.processing = false;
+    
+    /** @type {boolean} Whether lane is shutting down */
+    this.shutting_down = false;
+    
+    /** @type {NodeJS.Timeout|null} Timeout handle for batch processing */
+    this.batch_timeout_handle = null;
+    
+    /** @type {Object} Lane-specific statistics */
+    this.stats = {
+      total_operations: 0,
+      completed_operations: 0,
+      failed_operations: 0,
+      batches_processed: 0,
+      current_batch_size: 0,
+      max_batch_size: 0,
+      total_batch_wait_time_ms: 0,
+      total_batch_processing_time_ms: 0
+    };
+    
+    this.log = create_context_logger(`lane_${this.lane_id}`);
+  }
+
+  /**
+   * Adds an operation to this lane's batch queue.
+   * @param {Object} operation - Operation to add to batch
+   * @param {function} operation.operation_fn - Async function that performs the write operation
+   * @param {Object} [operation.context={}] - Additional context for logging and debugging
+   * @returns {Promise<*>} Promise that resolves with the operation result
+   * @throws {Error} When lane is shutting down
+   */
+  async add_operation(operation) {
+    if (this.shutting_down) {
+      throw new Error('Processing lane shutting down');
+    }
+    
+    return new Promise((resolve, reject) => {
+      if (this.shutting_down) {
+        reject(new Error('Processing lane shutting down'));
+        return;
+      }
+      
+      const batch_item = {
+        ...operation,
+        resolve,
+        reject,
+        enqueued_at: Date.now(),
+        id: this.generate_operation_id()
+      };
+
+      this.current_batch.push(batch_item);
+      this.stats.total_operations++;
+      this.stats.current_batch_size = this.current_batch.length;
+      
+      if (this.stats.current_batch_size > this.stats.max_batch_size) {
+        this.stats.max_batch_size = this.stats.current_batch_size;
+      }
+
+      this.log.debug('Operation added to batch', {
+        lane_id: this.lane_id,
+        operation_id: batch_item.id,
+        batch_size: this.stats.current_batch_size,
+        context: operation.context
+      });
+
+      // Process batch if it reaches the configured size
+      if (this.current_batch.length >= this.batch_size) {
+        this.process_current_batch();
+      } else if (this.current_batch.length === 1) {
+        // Start timeout for first operation in batch
+        this.start_batch_timeout();
+      }
+    });
+  }
+
+  /**
+   * Starts the batch timeout to ensure batches are processed within time limit.
+   */
+  start_batch_timeout() {
+    if (this.batch_timeout_handle) {
+      clearTimeout(this.batch_timeout_handle);
+    }
+    
+    this.batch_timeout_handle = setTimeout(() => {
+      if (this.current_batch.length > 0 && !this.processing) {
+        this.log.debug('Batch timeout triggered', {
+          lane_id: this.lane_id,
+          batch_size: this.current_batch.length
+        });
+        this.process_current_batch();
+      }
+    }, this.batch_timeout);
+  }
+
+  /**
+   * Processes the current batch of operations in a single transaction.
+   * @returns {Promise<void>} Promise that resolves when batch processing is complete
+   */
+  async process_current_batch() {
+    if (this.processing || this.current_batch.length === 0 || this.shutting_down) {
+      return;
+    }
+
+    // Clear timeout since we're processing now
+    if (this.batch_timeout_handle) {
+      clearTimeout(this.batch_timeout_handle);
+      this.batch_timeout_handle = null;
+    }
+
+    this.processing = true;
+    const batch_to_process = [...this.current_batch];
+    this.current_batch = [];
+    this.stats.current_batch_size = 0;
+    
+    const batch_start_time = Date.now();
+    const oldest_operation_time = Math.min(...batch_to_process.map(op => op.enqueued_at));
+    const batch_wait_time_ms = batch_start_time - oldest_operation_time;
+    
+    this.stats.total_batch_wait_time_ms += batch_wait_time_ms;
+    this.stats.batches_processed++;
+
+    this.log.debug('Processing batch', {
+      lane_id: this.lane_id,
+      batch_size: batch_to_process.length,
+      batch_wait_time_ms
+    });
+
+    try {
+      // Execute all operations in the batch within a single transaction context
+      const results = await this.execute_batch_transaction(batch_to_process);
+      
+      const batch_processing_time_ms = Date.now() - batch_start_time;
+      this.stats.total_batch_processing_time_ms += batch_processing_time_ms;
+      this.stats.completed_operations += batch_to_process.length;
+
+      // Resolve all operations with their results
+      batch_to_process.forEach((operation, index) => {
+        operation.resolve(results[index]);
+      });
+
+      this.log.debug('Batch completed successfully', {
+        lane_id: this.lane_id,
+        batch_size: batch_to_process.length,
+        batch_wait_time_ms,
+        batch_processing_time_ms
+      });
+
+    } catch (error) {
+      const batch_processing_time_ms = Date.now() - batch_start_time;
+      this.stats.total_batch_processing_time_ms += batch_processing_time_ms;
+      this.stats.failed_operations += batch_to_process.length;
+
+      // Reject all operations with the batch error
+      batch_to_process.forEach(operation => {
+        operation.reject(error);
+      });
+
+      this.log.error('Batch processing failed', {
+        lane_id: this.lane_id,
+        batch_size: batch_to_process.length,
+        batch_wait_time_ms,
+        batch_processing_time_ms,
+        error: error.message
+      });
+    }
+
+    this.processing = false;
+
+    // Process next batch if operations are waiting
+    if (this.current_batch.length > 0) {
+      if (this.current_batch.length >= this.batch_size) {
+        setImmediate(() => this.process_current_batch());
+      } else {
+        this.start_batch_timeout();
+      }
+    }
+  }
+
+  /**
+   * Executes all operations in a batch within a single transaction context.
+   * @param {Array<Object>} batch_operations - Operations to execute in batch
+   * @returns {Promise<Array<*>>} Promise that resolves with array of operation results
+   */
+  async execute_batch_transaction(batch_operations) {
+    const results = [];
+    
+    // Execute each operation and collect results with retry logic
+    for (const operation of batch_operations) {
+      try {
+        const result = await this.execute_with_retry(operation.operation_fn, operation.context);
+        results.push(result);
+      } catch (error) {
+        // If any operation fails, the entire batch fails
+        throw error;
+      }
+    }
+    
+    return results;
+  }
+
+  /**
+   * Executes an operation with retry logic and exponential backoff.
+   * @param {function} operation_fn - Async function to execute
+   * @param {Object} context - Context for logging
+   * @param {number} [max_retries=3] - Maximum number of retry attempts
+   * @returns {Promise<*>} Promise that resolves with operation result
+   * @throws {Error} When all retry attempts are exhausted
+   */
+  async execute_with_retry(operation_fn, context, max_retries = 3) {
+    let last_error = null;
+    
+    for (let attempt = 1; attempt <= max_retries; attempt++) {
+      try {
+        return await operation_fn();
+      } catch (error) {
+        last_error = error;
+        
+        if (this.is_retryable_error(error) && attempt < max_retries) {
+          const delay_ms = this.calculate_backoff_delay(attempt);
+          
+          this.log.warn('Operation failed, retrying', {
+            lane_id: this.lane_id,
+            attempt,
+            max_retries,
+            delay_ms,
+            error: error.message,
+            context
+          });
+          
+          await this.sleep(delay_ms);
+          continue;
+        }
+        
+        break;
+      }
+    }
+    
+    throw last_error;
+  }
+
+  /**
+   * Determines if an error is retryable based on error patterns.
+   * @param {Error} error - Error to check
+   * @returns {boolean} True if error is retryable, false otherwise
+   */
+  is_retryable_error(error) {
+    const retryable_patterns = [
+      'MDB_MAP_FULL',
+      'MDB_TXN_FULL',
+      'MDB_READERS_FULL',
+      'EAGAIN',
+      'EBUSY'
+    ];
+    
+    return retryable_patterns.some(pattern => 
+      error.message.includes(pattern) || error.code === pattern
+    );
+  }
+
+  /**
+   * Calculates exponential backoff delay with jitter for retry attempts.
+   * @param {number} attempt - Current attempt number (1-based)
+   * @returns {number} Delay in milliseconds
+   */
+  calculate_backoff_delay(attempt) {
+    const base_delay = 100;
+    const max_delay = 5000;
+    const exponential_delay = base_delay * Math.pow(2, attempt - 1);
+    const jitter = Math.random() * 0.1 * exponential_delay;
+    
+    return Math.min(exponential_delay + jitter, max_delay);
+  }
+
+  /**
+   * Utility function to sleep for specified milliseconds.
+   * @param {number} ms - Milliseconds to sleep
+   * @returns {Promise<void>} Promise that resolves after delay
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Generates a unique operation ID for tracking.
+   * @returns {string} Unique operation identifier
+   */
+  generate_operation_id() {
+    return `lane_${this.lane_id}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  /**
+   * Gets comprehensive lane statistics including calculated averages.
+   * @returns {Object} Statistics object with performance metrics
+   */
+  get_stats() {
+    const avg_batch_wait_time = this.stats.batches_processed > 0 
+      ? Math.round(this.stats.total_batch_wait_time_ms / this.stats.batches_processed)
+      : 0;
+    
+    const avg_batch_processing_time = this.stats.batches_processed > 0
+      ? Math.round(this.stats.total_batch_processing_time_ms / this.stats.batches_processed)
+      : 0;
+
+    const avg_batch_size = this.stats.batches_processed > 0
+      ? Math.round(this.stats.completed_operations / this.stats.batches_processed)
+      : 0;
+
+    return {
+      lane_id: this.lane_id,
+      ...this.stats,
+      avg_batch_wait_time_ms: avg_batch_wait_time,
+      avg_batch_processing_time_ms: avg_batch_processing_time,
+      avg_batch_size,
+      success_rate: this.stats.total_operations > 0 
+        ? Math.round((this.stats.completed_operations / this.stats.total_operations) * 100)
+        : 100
+    };
+  }
+
+  /**
+   * Clears all statistics while preserving current batch size.
+   */
+  clear_stats() {
+    this.stats = {
+      total_operations: 0,
+      completed_operations: 0,
+      failed_operations: 0,
+      batches_processed: 0,
+      current_batch_size: this.current_batch.length,
+      max_batch_size: 0,
+      total_batch_wait_time_ms: 0,
+      total_batch_processing_time_ms: 0
+    };
+  }
+
+  /**
+   * Forces processing of current batch regardless of size or timeout.
+   * @returns {Promise<void>} Promise that resolves when batch is processed
+   */
+  async flush_batch() {
+    if (this.current_batch.length > 0 && !this.processing) {
+      await this.process_current_batch();
+    }
+  }
+
+  /**
+   * Gracefully shuts down the processing lane.
+   * Processes any remaining operations and rejects new ones.
+   * @returns {Promise<void>} Promise that resolves when shutdown is complete
+   */
+  async shutdown() {
+    this.log.info('Shutting down processing lane', {
+      lane_id: this.lane_id,
+      pending_operations: this.current_batch.length,
+      currently_processing: this.processing
+    });
+
+    this.shutting_down = true;
+
+    // Clear timeout
+    if (this.batch_timeout_handle) {
+      clearTimeout(this.batch_timeout_handle);
+      this.batch_timeout_handle = null;
+    }
+
+    // Process any remaining operations
+    if (this.current_batch.length > 0 && !this.processing) {
+      await this.process_current_batch();
+    }
+
+    // Wait for current processing to complete
+    while (this.processing) {
+      await new Promise(resolve => setTimeout(resolve, 10));
+    }
+
+    // Reject any remaining operations
+    this.current_batch.forEach(operation => {
+      operation.reject(new Error('Processing lane shutting down'));
+    });
+    
+    this.current_batch = [];
+    this.processing = false;
+  }
+}
+
+export default ProcessingLane;

package/src/server/lib/write_queue.js

@@ -2,9 +2,13 @@
  * @fileoverview Write queue system for JoystickDB providing serialized write operations.
  * Ensures write operations are processed sequentially to maintain data consistency and ACID properties.
  * Includes retry logic, backoff strategies, performance monitoring, and graceful shutdown capabilities.
+ * 
+ * Now supports both traditional sequential processing and high-performance batched processing
+ * with automatic fallback and transparent integration.
  */
 
 import create_logger from './logger.js';
+import { get_batched_write_queue, shutdown_batched_write_queue } from './batched_write_queue.js';
 
 const { create_context_logger } = create_logger('write_queue');
 
@@ -312,13 +316,26 @@ class WriteQueue {
 /** @type {WriteQueue|null} Singleton instance of the write queue */
 let write_queue_instance = null;
 
+/** @type {boolean} Whether to use batched write queue for improved performance */
+let use_batched_queue = true;
+
 /**
  * Gets the singleton write queue instance, creating it if it doesn't exist.
+ * Automatically uses batched write queue for improved performance while maintaining
+ * complete backward compatibility.
+ * @param {Object} [options] - Configuration options for batched queue
  * @returns {WriteQueue} The write queue instance
  */
-export const get_write_queue = () => {
+export const get_write_queue = (options) => {
   if (!write_queue_instance) {
-    write_queue_instance = new WriteQueue();
+    if (use_batched_queue) {
+      // Use batched write queue with WriteQueue-compatible wrapper
+      const batched_queue = get_batched_write_queue(options);
+      write_queue_instance = new WriteQueueWrapper(batched_queue);
+    } else {
+      // Use traditional sequential write queue
+      write_queue_instance = new WriteQueue();
+    }
   }
   return write_queue_instance;
 };
@@ -332,4 +349,122 @@ export const shutdown_write_queue = async () => {
     await write_queue_instance.shutdown();
     write_queue_instance = null;
   }
+  
+  // Also shutdown batched queue if it was used
+  if (use_batched_queue) {
+    await shutdown_batched_write_queue();
+  }
 };
+
+/**
+ * Enables or disables batched write queue usage.
+ * @param {boolean} enabled - Whether to use batched queue
+ */
+export const set_batched_queue_enabled = (enabled) => {
+  use_batched_queue = enabled;
+};
+
+/**
+ * Wrapper class that provides WriteQueue-compatible API while using BatchedWriteQueue internally.
+ * Ensures complete backward compatibility with existing code.
+ */
+class WriteQueueWrapper {
+  /**
+   * Creates a new WriteQueueWrapper instance.
+   * @param {BatchedWriteQueue} batched_queue - The batched write queue instance to wrap
+   */
+  constructor(batched_queue) {
+    this.batched_queue = batched_queue;
+    this.log = create_context_logger('write_queue_wrapper');
+  }
+
+  /**
+   * Enqueues a write operation using the batched queue.
+   * Maintains identical API to original WriteQueue.
+   * @param {function} operation_fn - Async function that performs the write operation
+   * @param {Object} [context={}] - Additional context for logging and debugging
+   * @returns {Promise<*>} Promise that resolves with the operation result
+   */
+  async enqueue_write_operation(operation_fn, context = {}) {
+    return this.batched_queue.enqueue_write_operation(operation_fn, context);
+  }
+
+  /**
+   * Gets queue statistics with backward-compatible format.
+   * @returns {Object} Statistics object matching original WriteQueue format
+   */
+  get_stats() {
+    const batched_stats = this.batched_queue.get_stats();
+    
+    // Return stats in original WriteQueue format for backward compatibility
+    return {
+      total_operations: batched_stats.total_operations,
+      completed_operations: batched_stats.completed_operations,
+      failed_operations: batched_stats.failed_operations,
+      current_queue_depth: batched_stats.current_queue_depth,
+      max_queue_depth: batched_stats.max_queue_depth,
+      avg_wait_time_ms: batched_stats.avg_wait_time_ms,
+      avg_processing_time_ms: batched_stats.avg_processing_time_ms,
+      success_rate: batched_stats.success_rate
+    };
+  }
+
+  /**
+   * Clears all statistics.
+   */
+  clear_stats() {
+    this.batched_queue.clear_stats();
+  }
+
+  /**
+   * Gracefully shuts down the wrapper and underlying batched queue.
+   * @returns {Promise<void>} Promise that resolves when shutdown is complete
+   */
+  async shutdown() {
+    await this.batched_queue.shutdown();
+  }
+
+  /**
+   * Determines if an error is retryable based on error patterns.
+   * Exposed for backward compatibility with existing tests.
+   * @param {Error} error - Error to check
+   * @returns {boolean} True if error is retryable, false otherwise
+   */
+  is_retryable_error(error) {
+    const retryable_patterns = [
+      'MDB_MAP_FULL',
+      'MDB_TXN_FULL',
+      'MDB_READERS_FULL',
+      'EAGAIN',
+      'EBUSY'
+    ];
+    
+    return retryable_patterns.some(pattern => 
+      error.message.includes(pattern) || error.code === pattern
+    );
+  }
+
+  /**
+   * Calculates exponential backoff delay with jitter for retry attempts.
+   * Exposed for backward compatibility with existing tests.
+   * @param {number} attempt - Current attempt number (1-based)
+   * @returns {number} Delay in milliseconds
+   */
+  calculate_backoff_delay(attempt) {
+    const base_delay = 100;
+    const max_delay = 5000;
+    const exponential_delay = base_delay * Math.pow(2, attempt - 1);
+    const jitter = Math.random() * 0.1 * exponential_delay;
+    
+    return Math.min(exponential_delay + jitter, max_delay);
+  }
+
+  /**
+   * Generates a unique operation ID for tracking.
+   * Exposed for backward compatibility with existing tests.
+   * @returns {string} Unique operation identifier
+   */
+  generate_operation_id() {
+    return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+  }
+}