@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/lib/write_queue.js

@@ -0,0 +1,335 @@
+/**
+ * @fileoverview Write queue system for JoystickDB providing serialized write operations.
+ * Ensures write operations are processed sequentially to maintain data consistency and ACID properties.
+ * Includes retry logic, backoff strategies, performance monitoring, and graceful shutdown capabilities.
+ */
+
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('write_queue');
+
+/**
+ * Write queue class that serializes database write operations to ensure consistency.
+ * Provides queuing, retry logic, performance tracking, and graceful shutdown.
+ */
+class WriteQueue {
+  /**
+   * Creates a new WriteQueue instance.
+   * Initializes queue, processing state, statistics, and logging.
+   */
+  constructor() {
+    /** @type {Array<Object>} Queue of pending write operations */
+    this.queue = [];
+    
+    /** @type {boolean} Whether queue is currently processing operations */
+    this.processing = false;
+    
+    /** @type {boolean} Whether queue is shutting down */
+    this.shutting_down = false;
+    
+    /** @type {Object} Performance and operational statistics */
+    this.stats = {
+      total_operations: 0,
+      completed_operations: 0,
+      failed_operations: 0,
+      current_queue_depth: 0,
+      max_queue_depth: 0,
+      total_wait_time_ms: 0,
+      total_processing_time_ms: 0
+    };
+    
+    this.log = create_context_logger();
+  }
+
+  /**
+   * Enqueues a write operation to be processed sequentially.
+   * @param {function} operation_fn - Async function that performs the write operation
+   * @param {Object} [context={}] - Additional context for logging and debugging
+   * @returns {Promise<*>} Promise that resolves with the operation result
+   * @throws {Error} When server is shutting down
+   */
+  async enqueue_write_operation(operation_fn, context = {}) {
+    if (this.shutting_down) {
+      throw new Error('Server shutting down');
+    }
+    
+    return new Promise((resolve, reject) => {
+      if (this.shutting_down) {
+        reject(new Error('Server shutting down'));
+        return;
+      }
+      
+      const queue_item = {
+        operation_fn,
+        context,
+        resolve,
+        reject,
+        enqueued_at: Date.now(),
+        id: this.generate_operation_id()
+      };
+
+      this.queue.push(queue_item);
+      this.stats.total_operations++;
+      this.stats.current_queue_depth = this.queue.length;
+      
+      if (this.stats.current_queue_depth > this.stats.max_queue_depth) {
+        this.stats.max_queue_depth = this.stats.current_queue_depth;
+      }
+
+      this.log.debug('Write operation enqueued', {
+        operation_id: queue_item.id,
+        queue_depth: this.stats.current_queue_depth,
+        context: context
+      });
+
+      this.process_queue();
+    });
+  }
+
+  /**
+   * Processes queued write operations sequentially.
+   * Handles timing, error tracking, and statistics updates.
+   * @returns {Promise<void>} Promise that resolves when processing is complete
+   */
+  async process_queue() {
+    if (this.processing || this.queue.length === 0 || this.shutting_down) {
+      return;
+    }
+
+    this.processing = true;
+
+    while (this.queue.length > 0 && !this.shutting_down) {
+      const queue_item = this.queue.shift();
+      this.stats.current_queue_depth = this.queue.length;
+      
+      const wait_time_ms = Date.now() - queue_item.enqueued_at;
+      this.stats.total_wait_time_ms += wait_time_ms;
+      
+      const processing_start = Date.now();
+
+      try {
+        this.log.debug('Processing write operation', {
+          operation_id: queue_item.id,
+          wait_time_ms,
+          context: queue_item.context
+        });
+
+        const result = await this.execute_with_retry(queue_item.operation_fn, queue_item.context);
+        
+        const processing_time_ms = Date.now() - processing_start;
+        this.stats.total_processing_time_ms += processing_time_ms;
+        this.stats.completed_operations++;
+
+        this.log.debug('Write operation completed', {
+          operation_id: queue_item.id,
+          wait_time_ms,
+          processing_time_ms,
+          context: queue_item.context
+        });
+
+        queue_item.resolve(result);
+      } catch (error) {
+        const processing_time_ms = Date.now() - processing_start;
+        this.stats.total_processing_time_ms += processing_time_ms;
+        this.stats.failed_operations++;
+
+        this.log.error('Write operation failed', {
+          operation_id: queue_item.id,
+          wait_time_ms,
+          processing_time_ms,
+          error: error.message,
+          context: queue_item.context
+        });
+
+        queue_item.reject(error);
+      }
+    }
+
+    this.processing = false;
+  }
+
+  /**
+   * Executes an operation with retry logic and exponential backoff.
+   * @param {function} operation_fn - Async function to execute
+   * @param {Object} context - Context for logging
+   * @param {number} [max_retries=3] - Maximum number of retry attempts
+   * @returns {Promise<*>} Promise that resolves with operation result
+   * @throws {Error} When all retry attempts are exhausted
+   */
+  async execute_with_retry(operation_fn, context, max_retries = 3) {
+    let last_error = null;
+    
+    for (let attempt = 1; attempt <= max_retries; attempt++) {
+      try {
+        return await operation_fn();
+      } catch (error) {
+        last_error = error;
+        
+        if (this.is_retryable_error(error) && attempt < max_retries) {
+          const delay_ms = this.calculate_backoff_delay(attempt);
+          
+          this.log.warn('Write operation failed, retrying', {
+            attempt,
+            max_retries,
+            delay_ms,
+            error: error.message,
+            context
+          });
+          
+          await this.sleep(delay_ms);
+          continue;
+        }
+        
+        break;
+      }
+    }
+    
+    throw last_error;
+  }
+
+  /**
+   * Determines if an error is retryable based on error patterns.
+   * @param {Error} error - Error to check
+   * @returns {boolean} True if error is retryable, false otherwise
+   */
+  is_retryable_error(error) {
+    const retryable_patterns = [
+      'MDB_MAP_FULL',
+      'MDB_TXN_FULL',
+      'MDB_READERS_FULL',
+      'EAGAIN',
+      'EBUSY'
+    ];
+    
+    return retryable_patterns.some(pattern => 
+      error.message.includes(pattern) || error.code === pattern
+    );
+  }
+
+  /**
+   * Calculates exponential backoff delay with jitter for retry attempts.
+   * @param {number} attempt - Current attempt number (1-based)
+   * @returns {number} Delay in milliseconds
+   */
+  calculate_backoff_delay(attempt) {
+    const base_delay = 100;
+    const max_delay = 5000;
+    const exponential_delay = base_delay * Math.pow(2, attempt - 1);
+    const jitter = Math.random() * 0.1 * exponential_delay;
+    
+    return Math.min(exponential_delay + jitter, max_delay);
+  }
+
+  /**
+   * Utility function to sleep for specified milliseconds.
+   * @param {number} ms - Milliseconds to sleep
+   * @returns {Promise<void>} Promise that resolves after delay
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Generates a unique operation ID for tracking.
+   * @returns {string} Unique operation identifier
+   */
+  generate_operation_id() {
+    return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  /**
+   * Gets comprehensive queue statistics including calculated averages.
+   * @returns {Object} Statistics object with performance metrics
+   * @returns {number} returns.total_operations - Total operations enqueued
+   * @returns {number} returns.completed_operations - Successfully completed operations
+   * @returns {number} returns.failed_operations - Failed operations
+   * @returns {number} returns.current_queue_depth - Current number of queued operations
+   * @returns {number} returns.max_queue_depth - Maximum queue depth reached
+   * @returns {number} returns.avg_wait_time_ms - Average wait time in milliseconds
+   * @returns {number} returns.avg_processing_time_ms - Average processing time in milliseconds
+   * @returns {number} returns.success_rate - Success rate as percentage
+   */
+  get_stats() {
+    const avg_wait_time = this.stats.completed_operations > 0 
+      ? Math.round(this.stats.total_wait_time_ms / this.stats.completed_operations)
+      : 0;
+    
+    const avg_processing_time = this.stats.completed_operations > 0
+      ? Math.round(this.stats.total_processing_time_ms / this.stats.completed_operations)
+      : 0;
+
+    return {
+      ...this.stats,
+      avg_wait_time_ms: avg_wait_time,
+      avg_processing_time_ms: avg_processing_time,
+      success_rate: this.stats.total_operations > 0 
+        ? Math.round((this.stats.completed_operations / this.stats.total_operations) * 100)
+        : 100
+    };
+  }
+
+  /**
+   * Clears all statistics while preserving current queue depth.
+   */
+  clear_stats() {
+    this.stats = {
+      total_operations: 0,
+      completed_operations: 0,
+      failed_operations: 0,
+      current_queue_depth: this.queue.length,
+      max_queue_depth: 0,
+      total_wait_time_ms: 0,
+      total_processing_time_ms: 0
+    };
+  }
+
+  /**
+   * Gracefully shuts down the write queue.
+   * Waits for current operation to complete and rejects all pending operations.
+   * @returns {Promise<void>} Promise that resolves when shutdown is complete
+   */
+  async shutdown() {
+    this.log.info('Shutting down write queue', {
+      pending_operations: this.queue.length,
+      currently_processing: this.processing
+    });
+
+    this.shutting_down = true;
+
+    while (this.processing) {
+      await this.sleep(50);
+    }
+
+    this.queue.forEach(queue_item => {
+      queue_item.reject(new Error('Server shutting down'));
+    });
+    
+    this.queue = [];
+    this.processing = false;
+  }
+}
+
+/** @type {WriteQueue|null} Singleton instance of the write queue */
+let write_queue_instance = null;
+
+/**
+ * Gets the singleton write queue instance, creating it if it doesn't exist.
+ * @returns {WriteQueue} The write queue instance
+ */
+export const get_write_queue = () => {
+  if (!write_queue_instance) {
+    write_queue_instance = new WriteQueue();
+  }
+  return write_queue_instance;
+};
+
+/**
+ * Shuts down the write queue and clears the singleton instance.
+ * @returns {Promise<void>} Promise that resolves when shutdown is complete
+ */
+export const shutdown_write_queue = async () => {
+  if (write_queue_instance) {
+    await write_queue_instance.shutdown();
+    write_queue_instance = null;
+  }
+};