@joystick.js/db-canary 0.0.0-canary.2275 → 0.0.0-canary.2276

package/src/server/lib/replication_manager.js

@@ -1,727 +0,0 @@
-/**
- * @fileoverview Database replication manager for JoystickDB master-secondary architecture.
- * Provides asynchronous replication of write operations to secondary nodes with authentication,
- * batching, retry logic, and comprehensive monitoring. Supports dynamic secondary management,
- * health checking, and graceful failover handling with detailed statistics tracking.
- */
-
-import net from 'net';
-import crypto from 'crypto';
-import { get_settings } from './load_settings.js';
-import { encode_message, create_message_parser } from './tcp_protocol.js';
-import create_logger from './logger.js';
-
-const { create_context_logger } = create_logger('replication');
-
-/**
- * Manages database replication to secondary nodes with comprehensive monitoring and control.
- * Handles connection management, authentication, operation queuing, batching, and statistics.
- * Provides both asynchronous and synchronous replication modes with configurable parameters.
- */
-class ReplicationManager {
-  /**
-   * Creates a new ReplicationManager instance with default configuration.
-   * Initializes connection tracking, operation queuing, and statistics collection.
-   */
-  constructor() {
-    /** @type {Map<string, Object>} Map of secondary node connections by ID */
-    this.secondary_connections = new Map();
-    
-    /** @type {Array<Object>} Queue of operations pending replication */
-    this.replication_queue = [];
-    
-    /** @type {boolean} Flag indicating if replication processing is active */
-    this.processing_replication = false;
-    
-    /** @type {number} Monotonic sequence number for operation ordering */
-    this.sequence_number = 0;
-    
-    /** @type {boolean} Whether replication is enabled */
-    this.enabled = false;
-    
-    /** @type {string} Replication mode ('async' or 'sync') */
-    this.mode = 'async';
-    
-    /** @type {number} Timeout for replication operations in milliseconds */
-    this.timeout_ms = 5000;
-    
-    /** @type {number} Number of retry attempts for failed operations */
-    this.retry_attempts = 3;
-    
-    /** @type {number} Maximum operations per replication batch */
-    this.batch_size = 100;
-    
-    /** @type {Object} Logger instance for replication events */
-    this.log = create_context_logger();
-    
-    /** @type {Object} Replication statistics and metrics */
-    this.stats = {
-      total_operations_replicated: 0,
-      successful_replications: 0,
-      failed_replications: 0,
-      connected_secondaries: 0,
-      avg_replication_latency_ms: 0,
-      total_replication_time_ms: 0
-    };
-  }
-
-  /**
-   * Initializes replication manager with configuration from settings.
-   * Loads replication configuration, connects to secondary nodes, and starts processing.
-   * Gracefully handles missing or invalid configuration by disabling replication.
-   * @returns {void}
-   */
-  initialize() {
-    try {
-      const settings = get_settings();
-      const replication_config = settings.replication;
-      
-      if (!replication_config || !replication_config.enabled) {
-        this.log.info('Replication disabled in settings');
-        return;
-      }
-      
-      this.enabled = replication_config.enabled;
-      this.mode = replication_config.mode || 'async';
-      this.timeout_ms = replication_config.timeout_ms || 5000;
-      this.retry_attempts = replication_config.retry_attempts || 3;
-      this.batch_size = replication_config.batch_size || 100;
-      
-      const secondaries = replication_config.secondaries || [];
-      
-      this.log.info('Initializing replication manager', {
-        enabled: this.enabled,
-        mode: this.mode,
-        secondary_count: secondaries.length,
-        timeout_ms: this.timeout_ms,
-        batch_size: this.batch_size
-      });
-      
-      // Connect to all enabled secondary nodes
-      for (const secondary of secondaries) {
-        if (secondary.enabled) {
-          this.connect_to_secondary(secondary);
-        }
-      }
-      
-      // Start replication processing
-      this.start_replication_processing();
-      
-    } catch (error) {
-      this.log.warn('Could not initialize replication - settings not loaded', {
-        error: error.message
-      });
-    }
-  }
-
-  /**
-   * Establishes connection to a secondary node with authentication and error handling.
-   * Sets up TCP connection, message parsing, event handlers, and automatic reconnection.
-   * Tracks connection state and updates statistics for monitoring purposes.
-   * @param {Object} secondary - Secondary node configuration
-   * @param {string} secondary.id - Unique identifier for the secondary node
-   * @param {string} secondary.ip - IP address of the secondary node
-   * @param {number} secondary.port - Port number of the secondary node
-   * @param {string} secondary.private_key - Base64 encoded private key for authentication
-   * @returns {Promise<void>}
-   */
-  async connect_to_secondary(secondary) {
-    const { id, ip, port, private_key } = secondary;
-    
-    this.log.info('Connecting to secondary node', { id, ip, port });
-    
-    try {
-      const socket = new net.Socket();
-      const message_parser = create_message_parser();
-      
-      socket.connect(port, ip, () => {
-        this.log.info('Connected to secondary node', { id, ip, port });
-        
-        // Send authentication message
-        this.authenticate_with_secondary(socket, id, private_key);
-      });
-      
-      socket.on('data', (data) => {
-        try {
-          const messages = message_parser.parse_messages(data);
-          for (const message of messages) {
-            this.handle_secondary_response(id, message);
-          }
-        } catch (error) {
-          this.log.error('Failed to parse secondary response', {
-            secondary_id: id,
-            error: error.message
-          });
-        }
-      });
-      
-      socket.on('error', (error) => {
-        this.log.error('Secondary connection error', {
-          secondary_id: id,
-          error: error.message
-        });
-        
-        this.secondary_connections.delete(id);
-        this.stats.connected_secondaries = this.secondary_connections.size;
-        
-        // Attempt reconnection after delay
-        setTimeout(() => {
-          this.connect_to_secondary(secondary);
-        }, 5000);
-      });
-      
-      socket.on('close', () => {
-        this.log.warn('Secondary connection closed', { secondary_id: id });
-        this.secondary_connections.delete(id);
-        this.stats.connected_secondaries = this.secondary_connections.size;
-        
-        // Attempt reconnection after delay
-        setTimeout(() => {
-          this.connect_to_secondary(secondary);
-        }, 5000);
-      });
-      
-      this.secondary_connections.set(id, {
-        socket,
-        id,
-        ip,
-        port,
-        authenticated: false,
-        last_ping: Date.now(),
-        pending_operations: new Map()
-      });
-      
-      this.stats.connected_secondaries = this.secondary_connections.size;
-      
-    } catch (error) {
-      this.log.error('Failed to connect to secondary', {
-        secondary_id: id,
-        ip,
-        port,
-        error: error.message
-      });
-    }
-  }
-
-  /**
-   * Authenticates with a secondary node using HMAC-SHA256 signature.
-   * Creates timestamped signature using private key and sends authentication message.
-   * Provides secure authentication to prevent unauthorized replication access.
-   * @param {net.Socket} socket - Socket connection to secondary
-   * @param {string} secondary_id - Secondary node ID
-   * @param {string} private_key - Base64 encoded private key for signing
-   * @returns {void}
-   */
-  authenticate_with_secondary(socket, secondary_id, private_key) {
-    try {
-      const timestamp = Date.now();
-      const message_to_sign = `${secondary_id}:${timestamp}`;
-      
-      // Create signature using private key
-      const signature = crypto
-        .createHmac('sha256', Buffer.from(private_key, 'base64'))
-        .update(message_to_sign)
-        .digest('base64');
-      
-      const auth_message = {
-        type: 'replication_auth',
-        node_id: secondary_id,
-        timestamp,
-        signature
-      };
-      
-      const encoded_message = encode_message(auth_message);
-      socket.write(encoded_message);
-      
-      this.log.debug('Sent authentication to secondary', { secondary_id });
-      
-    } catch (error) {
-      this.log.error('Failed to authenticate with secondary', {
-        secondary_id,
-        error: error.message
-      });
-    }
-  }
-
-  /**
-   * Processes responses from secondary nodes including authentication and acknowledgments.
-   * Routes different message types to appropriate handlers and maintains connection state.
-   * Handles authentication results, replication acknowledgments, and ping responses.
-   * @param {string} secondary_id - Secondary node ID that sent the response
-   * @param {Object|string} message - Parsed message from secondary node
-   * @returns {void}
-   */
-  handle_secondary_response(secondary_id, message) {
-    const connection = this.secondary_connections.get(secondary_id);
-    if (!connection) return;
-    
-    try {
-      const parsed_message = typeof message === 'string' ? JSON.parse(message) : message;
-      
-      switch (parsed_message.type) {
-        case 'auth_success':
-          connection.authenticated = true;
-          this.log.info('Secondary authentication successful', { secondary_id });
-          break;
-          
-        case 'auth_failed':
-          this.log.error('Secondary authentication failed', { secondary_id });
-          connection.socket.end();
-          break;
-          
-        case 'replication_ack':
-          this.handle_replication_acknowledgment(secondary_id, parsed_message);
-          break;
-          
-        case 'ping_response':
-          connection.last_ping = Date.now();
-          break;
-          
-        default:
-          this.log.debug('Unknown message from secondary', {
-            secondary_id,
-            type: parsed_message.type
-          });
-      }
-    } catch (error) {
-      this.log.error('Failed to handle secondary response', {
-        secondary_id,
-        error: error.message
-      });
-    }
-  }
-
-  /**
-   * Processes replication acknowledgments and updates statistics.
-   * Tracks operation latency, success/failure rates, and removes pending operations.
-   * Updates average replication latency and logs replication results for monitoring.
-   * @param {string} secondary_id - Secondary node ID that sent acknowledgment
-   * @param {Object} ack_message - Acknowledgment message with status and sequence number
-   * @param {number} ack_message.sequence_number - Sequence number of acknowledged operation
-   * @param {string} ack_message.status - Status of replication ('success' or 'error')
-   * @param {string} [ack_message.error] - Error message if replication failed
-   * @returns {void}
-   */
-  handle_replication_acknowledgment(secondary_id, ack_message) {
-    const connection = this.secondary_connections.get(secondary_id);
-    if (!connection) return;
-    
-    const { sequence_number, status, error } = ack_message;
-    const pending_op = connection.pending_operations.get(sequence_number);
-    
-    if (pending_op) {
-      const latency = Date.now() - pending_op.sent_at;
-      this.stats.total_replication_time_ms += latency;
-      
-      if (status === 'success') {
-        this.stats.successful_replications++;
-        this.log.debug('Replication acknowledged', {
-          secondary_id,
-          sequence_number,
-          latency_ms: latency
-        });
-      } else {
-        this.stats.failed_replications++;
-        this.log.error('Replication failed on secondary', {
-          secondary_id,
-          sequence_number,
-          error,
-          latency_ms: latency
-        });
-      }
-      
-      connection.pending_operations.delete(sequence_number);
-      
-      // Update average latency
-      const total_ops = this.stats.successful_replications + this.stats.failed_replications;
-      this.stats.avg_replication_latency_ms = total_ops > 0 
-        ? Math.round(this.stats.total_replication_time_ms / total_ops)
-        : 0;
-    }
-  }
-
-  /**
-   * Queues a write operation for replication to secondary nodes.
-   * Creates replication item with sequence number and timestamps, adds to queue,
-   * and triggers processing if not already active. Skips if replication disabled.
-   * @param {string} operation - Type of operation (insert, update, delete, etc.)
-   * @param {string} collection - Name of the collection being modified
-   * @param {Object} data - Operation data including document and filter information
-   * @param {string|null} [transaction_id=null] - Optional transaction identifier
-   * @returns {void}
-   */
-  queue_replication(operation, collection, data, transaction_id = null) {
-    if (!this.enabled || this.secondary_connections.size === 0) {
-      return;
-    }
-    
-    const replication_item = {
-      operation,
-      collection,
-      data,
-      transaction_id,
-      timestamp: Date.now(),
-      sequence_number: ++this.sequence_number
-    };
-    
-    this.replication_queue.push(replication_item);
-    this.stats.total_operations_replicated++;
-    
-    this.log.debug('Queued operation for replication', {
-      operation,
-      collection,
-      sequence_number: replication_item.sequence_number,
-      queue_length: this.replication_queue.length
-    });
-    
-    // Process queue if not already processing
-    if (!this.processing_replication) {
-      setImmediate(() => this.process_replication_queue());
-    }
-  }
-
-  /**
-   * Initiates replication queue processing if not already active.
-   * Sets processing flag and begins queue processing loop to handle pending operations.
-   * @returns {void}
-   */
-  start_replication_processing() {
-    if (this.processing_replication) return;
-    
-    this.processing_replication = true;
-    this.process_replication_queue();
-  }
-
-  /**
-   * Processes replication queue in batches with controlled timing.
-   * Continuously processes batches until queue is empty, with small delays
-   * between batches to prevent overwhelming secondary nodes.
-   * @returns {Promise<void>}
-   */
-  async process_replication_queue() {
-    while (this.replication_queue.length > 0 && this.enabled) {
-      const batch = this.replication_queue.splice(0, this.batch_size);
-      
-      if (batch.length === 0) break;
-      
-      await this.replicate_batch(batch);
-      
-      // Small delay between batches to prevent overwhelming secondaries
-      if (this.replication_queue.length > 0) {
-        await new Promise(resolve => setTimeout(resolve, 10));
-      }
-    }
-    
-    // Continue processing if more items were added
-    if (this.replication_queue.length > 0) {
-      setImmediate(() => this.process_replication_queue());
-    }
-  }
-
-  /**
-   * Sends a batch of operations to all authenticated secondary nodes.
-   * Creates replication message, encodes it, and sends to all connected secondaries.
-   * Tracks pending operations for acknowledgment and handles send failures gracefully.
-   * @param {Array<Object>} batch - Array of operations to replicate
-   * @returns {Promise<void>}
-   */
-  async replicate_batch(batch) {
-    const replication_message = {
-      type: 'replication',
-      timestamp: Date.now(),
-      sequence_number: batch[0].sequence_number,
-      operations: batch
-    };
-    
-    const encoded_message = encode_message(replication_message);
-    const authenticated_secondaries = Array.from(this.secondary_connections.values())
-      .filter(conn => conn.authenticated);
-    
-    if (authenticated_secondaries.length === 0) {
-      this.log.warn('No authenticated secondaries available for replication');
-      return;
-    }
-    
-    this.log.debug('Replicating batch to secondaries', {
-      batch_size: batch.length,
-      secondary_count: authenticated_secondaries.length,
-      sequence_number: batch[0].sequence_number
-    });
-    
-    // Send to all authenticated secondaries
-    for (const connection of authenticated_secondaries) {
-      try {
-        connection.socket.write(encoded_message);
-        
-        // Track pending operations for acknowledgment
-        for (const operation of batch) {
-          connection.pending_operations.set(operation.sequence_number, {
-            operation,
-            sent_at: Date.now()
-          });
-        }
-        
-      } catch (error) {
-        this.log.error('Failed to send replication to secondary', {
-          secondary_id: connection.id,
-          error: error.message
-        });
-      }
-    }
-  }
-
-  /**
-   * Dynamically adds a new secondary node to the replication cluster.
-   * Validates that secondary doesn't already exist, establishes connection,
-   * and returns success confirmation with secondary ID.
-   * @param {Object} secondary - Secondary node configuration
-   * @param {string} secondary.id - Unique identifier for the secondary node
-   * @param {string} secondary.ip - IP address of the secondary node
-   * @param {number} secondary.port - Port number of the secondary node
-   * @param {string} secondary.private_key - Base64 encoded private key
-   * @returns {Promise<Object>} Operation result with success status and secondary ID
-   * @throws {Error} When secondary node with same ID already exists
-   */
-  async add_secondary(secondary) {
-    const { id } = secondary;
-    
-    if (this.secondary_connections.has(id)) {
-      throw new Error(`Secondary node ${id} already exists`);
-    }
-    
-    this.log.info('Adding new secondary node', { id });
-    
-    await this.connect_to_secondary(secondary);
-    
-    return {
-      success: true,
-      message: `Secondary node ${id} added successfully`,
-      secondary_id: id
-    };
-  }
-
-  /**
-   * Removes a secondary node from the replication cluster.
-   * Closes connection, removes from tracking, and updates statistics.
-   * Provides clean disconnection and resource cleanup.
-   * @param {string} secondary_id - Unique identifier of secondary node to remove
-   * @returns {Object} Operation result with success status and secondary ID
-   * @throws {Error} When secondary node with specified ID is not found
-   */
-  remove_secondary(secondary_id) {
-    const connection = this.secondary_connections.get(secondary_id);
-    
-    if (!connection) {
-      throw new Error(`Secondary node ${secondary_id} not found`);
-    }
-    
-    this.log.info('Removing secondary node', { secondary_id });
-    
-    connection.socket.end();
-    this.secondary_connections.delete(secondary_id);
-    this.stats.connected_secondaries = this.secondary_connections.size;
-    
-    return {
-      success: true,
-      message: `Secondary node ${secondary_id} removed successfully`,
-      secondary_id
-    };
-  }
-
-  /**
-   * Forces synchronization with all secondary nodes and checks connectivity.
-   * Processes pending replication queue and sends ping messages to verify
-   * secondary node health and responsiveness.
-   * @returns {Promise<Object>} Synchronization result with status for each secondary
-   * @throws {Error} When replication is not enabled
-   */
-  async sync_secondaries() {
-    if (!this.enabled) {
-      throw new Error('Replication is not enabled');
-    }
-    
-    this.log.info('Forcing secondary synchronization');
-    
-    // Process any pending replication queue
-    await this.process_replication_queue();
-    
-    // Send ping to all secondaries to check connectivity
-    const ping_message = {
-      type: 'ping',
-      timestamp: Date.now()
-    };
-    
-    const encoded_ping = encode_message(ping_message);
-    const results = [];
-    
-    for (const [id, connection] of this.secondary_connections) {
-      try {
-        if (connection.authenticated) {
-          connection.socket.write(encoded_ping);
-          results.push({ secondary_id: id, status: 'ping_sent' });
-        } else {
-          results.push({ secondary_id: id, status: 'not_authenticated' });
-        }
-      } catch (error) {
-        results.push({ 
-          secondary_id: id, 
-          status: 'error', 
-          error: error.message 
-        });
-      }
-    }
-    
-    return {
-      success: true,
-      message: 'Secondary synchronization initiated',
-      results
-    };
-  }
-
-  /**
-   * Retrieves comprehensive replication status and statistics.
-   * Returns current configuration, connection states, queue status, and
-   * detailed statistics for monitoring and debugging purposes.
-   * @returns {Object} Complete replication status information
-   * @returns {boolean} returns.enabled - Whether replication is enabled
-   * @returns {string} returns.mode - Replication mode (async/sync)
-   * @returns {number} returns.connected_secondaries - Number of connected secondaries
-   * @returns {number} returns.queue_length - Current replication queue length
-   * @returns {boolean} returns.processing - Whether queue processing is active
-   * @returns {Object} returns.stats - Detailed replication statistics
-   * @returns {Array<Object>} returns.secondaries - Status of each secondary node
-   */
-  get_replication_status() {
-    const secondary_status = [];
-    
-    for (const [id, connection] of this.secondary_connections) {
-      secondary_status.push({
-        id,
-        ip: connection.ip,
-        port: connection.port,
-        authenticated: connection.authenticated,
-        last_ping: connection.last_ping,
-        pending_operations: connection.pending_operations.size,
-        connected: !connection.socket.destroyed
-      });
-    }
-    
-    return {
-      enabled: this.enabled,
-      mode: this.mode,
-      connected_secondaries: this.stats.connected_secondaries,
-      queue_length: this.replication_queue.length,
-      processing: this.processing_replication,
-      stats: this.stats,
-      secondaries: secondary_status
-    };
-  }
-
-  /**
-   * Evaluates health status of all secondary nodes.
-   * Checks authentication status, connection state, and ping responsiveness
-   * to determine overall health of each secondary node.
-   * @returns {Object} Health status summary and detailed node information
-   * @returns {number} returns.total_secondaries - Total number of secondary nodes
-   * @returns {number} returns.healthy_secondaries - Number of healthy secondary nodes
-   * @returns {Array<Object>} returns.secondaries - Detailed health status for each node
-   */
-  get_secondary_health() {
-    const health_status = [];
-    const current_time = Date.now();
-    
-    for (const [id, connection] of this.secondary_connections) {
-      const last_ping_age = current_time - connection.last_ping;
-      const is_healthy = connection.authenticated && 
-                        !connection.socket.destroyed && 
-                        last_ping_age < 30000; // 30 seconds
-      
-      health_status.push({
-        id,
-        ip: connection.ip,
-        port: connection.port,
-        healthy: is_healthy,
-        authenticated: connection.authenticated,
-        connected: !connection.socket.destroyed,
-        last_ping_age_ms: last_ping_age,
-        pending_operations: connection.pending_operations.size
-      });
-    }
-    
-    return {
-      total_secondaries: health_status.length,
-      healthy_secondaries: health_status.filter(s => s.healthy).length,
-      secondaries: health_status
-    };
-  }
-
-  /**
-   * Gracefully shuts down replication manager and closes all connections.
-   * Disables replication, closes secondary connections, clears queues,
-   * and resets all state for clean shutdown.
-   * @returns {Promise<void>}
-   */
-  async shutdown() {
-    this.log.info('Shutting down replication manager');
-    
-    this.enabled = false;
-    this.processing_replication = false;
-    
-    // Close all secondary connections
-    for (const [id, connection] of this.secondary_connections) {
-      try {
-        connection.socket.end();
-      } catch (error) {
-        this.log.warn('Error closing secondary connection', {
-          secondary_id: id,
-          error: error.message
-        });
-      }
-    }
-    
-    this.secondary_connections.clear();
-    this.replication_queue = [];
-    this.stats.connected_secondaries = 0;
-    
-    this.log.info('Replication manager shutdown complete');
-  }
-}
-
-/** @type {ReplicationManager|null} Singleton replication manager instance */
-let replication_manager_instance = null;
-
-/**
- * Gets the replication manager singleton instance.
- * Creates new instance if one doesn't exist, ensuring single manager per process.
- * @returns {ReplicationManager} Replication manager singleton instance
- */
-export const get_replication_manager = () => {
-  if (!replication_manager_instance) {
-    replication_manager_instance = new ReplicationManager();
-  }
-  return replication_manager_instance;
-};
-
-/**
- * Initializes the replication manager singleton with configuration.
- * Gets or creates manager instance and calls initialize to load settings
- * and establish secondary connections.
- * @returns {void}
- */
-export const initialize_replication_manager = () => {
-  const manager = get_replication_manager();
-  manager.initialize();
-};
-
-/**
- * Shuts down the replication manager singleton and cleans up resources.
- * Calls shutdown on existing instance and resets singleton reference
- * for clean process termination.
- * @returns {Promise<void>}
- */
-export const shutdown_replication_manager = async () => {
-  if (replication_manager_instance) {
-    await replication_manager_instance.shutdown();
-    replication_manager_instance = null;
-  }
-};