@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/cluster/master.js

@@ -0,0 +1,774 @@
+import cluster from 'cluster';
+import os from 'os';
+import { EventEmitter } from 'events';
+import { writeFileSync } from 'fs';
+import { load_settings, get_port_configuration } from '../lib/load_settings.js';
+import { 
+  restore_backup, 
+  start_backup_schedule, 
+  stop_backup_schedule 
+} from '../lib/backup_manager.js';
+import create_logger from '../lib/logger.js';
+import { initialize_database, check_and_grow_map_size, cleanup_database } from '../lib/query_engine.js';
+import { 
+  setup_authentication, 
+  verify_password, 
+  initialize_auth_manager
+} from '../lib/auth_manager.js';
+import insert_one from '../lib/operations/insert_one.js';
+import update_one from '../lib/operations/update_one.js';
+import delete_one from '../lib/operations/delete_one.js';
+import bulk_write from '../lib/operations/bulk_write.js';
+import find_one from '../lib/operations/find_one.js';
+import find from '../lib/operations/find.js';
+import create_index_operation from '../lib/operations/create_index.js';
+import drop_index_operation from '../lib/operations/drop_index.js';
+import get_indexes_operation from '../lib/operations/get_indexes.js';
+import { start_http_server, stop_http_server, is_setup_required } from '../lib/http_server.js';
+
+/**
+ * @typedef {Object} ClusterMasterOptions
+ * @property {number} [worker_count] - Number of worker processes to spawn
+ * @property {number} [port=1983] - Server port
+ * @property {string} [settings_file='settings.db.json'] - Settings file path
+ */
+
+/**
+ * Cluster master process that manages worker processes and coordinates database writes.
+ * @extends EventEmitter
+ * @fires ClusterMaster#writes_completed
+ */
+class ClusterMaster extends EventEmitter {
+  /**
+   * Creates a new cluster master instance.
+   * @param {ClusterMasterOptions} [options={}] - Master configuration options
+   */
+  constructor(options = {}) {
+    super();
+    
+    this.workers = new Map();
+    this.write_queue = [];
+    this.processing_writes = false;
+    this.authenticated_sessions = new Map();
+    this.worker_count = options.worker_count || os.cpus().length;
+    this.port = options.port || 1983;
+    this.settings_file = options.settings_file || 'settings.db.json';
+    this.settings = null;
+    this.pending_writes = new Map();
+    this.write_id_counter = 0;
+    this.shutting_down = false;
+    this.master_id = `master_${Date.now()}_${Math.random()}`;
+    
+    const { create_context_logger } = create_logger('master');
+    this.log = create_context_logger({ port: this.port, worker_count: this.worker_count, master_id: this.master_id });
+    
+    this.setup_master();
+  }
+  
+  /**
+   * Sets up the cluster master configuration and event handlers.
+   */
+  setup_master() {
+    cluster.setupPrimary({
+      exec: new URL('./index.js', import.meta.url).pathname,
+      args: [],
+      silent: false
+    });
+    
+    cluster.on('exit', (worker, code, signal) => {
+      this.log.warn('Worker died', { 
+        worker_pid: worker.process.pid, 
+        exit_code: code, 
+        signal: signal 
+      });
+      this.handle_worker_death(worker);
+    });
+    
+    cluster.on('message', (worker, message) => {
+      this.handle_worker_message(worker, message);
+    });
+  }
+  
+  /**
+   * Starts the cluster master and spawns worker processes.
+   * @throws {Error} When startup fails
+   */
+  async start() {
+    const start_time = Date.now();
+    
+    try {
+      this.settings = load_settings(this.settings_file);
+      this.log.info('Settings loaded successfully', { settings_file: this.settings_file });
+      
+      // NOTE: Initialize database and auth manager.
+      initialize_database();
+      initialize_auth_manager();
+      this.log.info('Database and auth manager initialized');
+      
+      // NOTE: Handle startup restore if configured.
+      if (this.settings?.restore_from) {
+        try {
+          this.log.info('Startup restore requested', { backup_filename: this.settings.restore_from });
+          
+          const restore_result = await restore_backup(this.settings.restore_from);
+          
+          this.log.info('Startup restore completed', {
+            backup_filename: this.settings.restore_from,
+            duration_ms: restore_result.duration_ms
+          });
+          
+          // NOTE: Remove restore_from from settings after successful restore.
+          const updated_settings = { ...this.settings };
+          delete updated_settings.restore_from;
+          
+          writeFileSync(this.settings_file, JSON.stringify(updated_settings, null, 2));
+          
+          // NOTE: Reload settings to reflect the change.
+          this.settings = load_settings(this.settings_file);
+          
+          this.log.info('Removed restore_from from settings after successful restore');
+        } catch (restore_error) {
+          this.log.error('Startup restore failed', {
+            backup_filename: this.settings.restore_from,
+            error: restore_error.message
+          });
+          
+          // NOTE: Continue with existing database if restore fails.
+          this.log.info('Continuing with existing database after restore failure');
+        }
+      }
+      
+      // NOTE: Start backup scheduling if S3 is configured.
+      if (this.settings?.s3) {
+        try {
+          start_backup_schedule();
+          this.log.info('Backup scheduling started');
+        } catch (backup_schedule_error) {
+          this.log.warn('Failed to start backup scheduling', { error: backup_schedule_error.message });
+        }
+      }
+      
+      // NOTE: Start HTTP server for setup if authentication is not configured.
+      if (is_setup_required()) {
+        try {
+          const { http_port } = get_port_configuration();
+          const http_server_instance = await start_http_server(http_port);
+          if (http_server_instance) {
+            this.log.info('HTTP setup server started', { http_port });
+          }
+        } catch (http_error) {
+          this.log.warn('Failed to start HTTP setup server', { error: http_error.message });
+        }
+      }
+      
+      for (let i = 0; i < this.worker_count; i++) {
+        this.spawn_worker();
+      }
+      
+      const duration_ms = Date.now() - start_time;
+      this.log.info('Master process started successfully', {
+        workers_spawned: this.worker_count,
+        startup_duration_ms: duration_ms
+      });
+      
+    } catch (error) {
+      this.log.error('Failed to start master process', { error: error.message });
+      process.exit(1);
+    }
+  }
+  
+  /**
+   * Spawns a new worker process.
+   * @returns {cluster.Worker} The spawned worker instance
+   */
+  spawn_worker() {
+    const spawn_start = Date.now();
+    this.log.info('Spawning worker');
+    
+    const worker = cluster.fork({
+      WORKER_PORT: this.port,
+      WORKER_SETTINGS: JSON.stringify(this.settings)
+    });
+    
+    this.workers.set(worker.id, {
+      worker,
+      connections: 0,
+      last_heartbeat: Date.now(),
+      status: 'starting'
+    });
+    
+    const spawn_duration = Date.now() - spawn_start;
+    this.log.info('Worker spawned successfully', {
+      worker_id: worker.id,
+      worker_pid: worker.process.pid,
+      spawn_duration_ms: spawn_duration
+    });
+    
+    return worker;
+  }
+  
+  /**
+   * Handles worker process death and respawning.
+   * @param {cluster.Worker} dead_worker - The worker that died
+   */
+  handle_worker_death(dead_worker) {
+    this.workers.delete(dead_worker.id);
+    
+    if (!this.shutting_down) {
+      this.log.info('Respawning worker after death', { 
+        dead_worker_id: dead_worker.id,
+        respawn_delay_ms: 1000
+      });
+      setTimeout(() => {
+        this.spawn_worker();
+      }, 1000);
+    }
+  }
+  
+  /**
+   * Handles messages from worker processes.
+   * @param {cluster.Worker} worker - Worker that sent the message
+   * @param {Object} message - Message from worker
+   */
+  handle_worker_message(worker, message) {
+    switch (message.type) {
+      case 'worker_ready':
+        this.handle_worker_ready_for_config(worker, message);
+        break;
+        
+      case 'server_ready':
+        this.handle_worker_server_ready(worker, message);
+        break;
+        
+      case 'write_request':
+        this.handle_write_request(worker, message);
+        break;
+        
+      case 'auth_request':
+        this.handle_auth_request(worker, message);
+        break;
+        
+      case 'setup_request':
+        this.handle_setup_request(worker, message);
+        break;
+        
+      case 'connection_count':
+        this.update_worker_connections(worker, message);
+        break;
+        
+      case 'heartbeat':
+        this.handle_worker_heartbeat(worker, message);
+        break;
+        
+      default:
+        this.log.warn('Unknown message type received', { 
+          message_type: message.type,
+          worker_id: worker.id 
+        });
+    }
+  }
+  
+  handle_worker_ready_for_config(worker, message) {
+    this.log.info('Worker ready for config, sending configuration', {
+      worker_id: worker.id,
+      worker_pid: worker.process.pid,
+      master_id: this.master_id
+    });
+    worker.send({
+      type: 'config',
+      data: {
+        port: this.port,
+        settings: this.settings,
+        master_id: this.master_id
+      }
+    });
+  }
+  
+  handle_worker_server_ready(worker, message) {
+    const worker_info = this.workers.get(worker.id);
+    if (worker_info) {
+      worker_info.status = 'ready';
+      this.log.info('Worker server ready', {
+        worker_id: worker.id,
+        worker_pid: worker.process.pid
+      });
+    }
+  }
+  
+  /**
+   * Handles write requests from workers, queuing them for serialized execution.
+   * @param {cluster.Worker} worker - Worker requesting the write
+   * @param {Object} message - Write request message
+   */
+  async handle_write_request(worker, message) {
+    if (this.shutting_down) {
+      worker.send({
+        type: 'write_response',
+        data: {
+          write_id: message.data.write_id,
+          success: false,
+          error: 'Server is shutting down'
+        }
+      });
+      return;
+    }
+    
+    const { write_id, op_type, data, socket_id } = message.data;
+    
+    try {
+      const write_operation = {
+        write_id,
+        worker_id: worker.id,
+        op_type,
+        data,
+        socket_id,
+        timestamp: Date.now()
+      };
+      
+      this.write_queue.push(write_operation);
+      this.process_write_queue();
+      
+    } catch (error) {
+      worker.send({
+        type: 'write_response',
+        data: {
+          write_id,
+          success: false,
+          error: error.message
+        }
+      });
+    }
+  }
+  
+  /**
+   * Processes the write queue, executing operations serially.
+   */
+  async process_write_queue() {
+    if (this.processing_writes || this.write_queue.length === 0) {
+      return;
+    }
+    
+    this.processing_writes = true;
+    
+    while (this.write_queue.length > 0) {
+      const write_op = this.write_queue.shift();
+      await this.execute_write_operation(write_op);
+    }
+    
+    this.processing_writes = false;
+    
+    if (this.shutting_down && this.write_queue.length === 0) {
+      this.emit('writes_completed');
+    }
+  }
+  
+  /**
+   * Executes a single write operation and sends response to worker.
+   * @param {Object} write_op - Write operation to execute
+   */
+  async execute_write_operation(write_op) {
+    const { write_id, worker_id, op_type, data, socket_id } = write_op;
+    const worker_info = this.workers.get(worker_id);
+    
+    if (!worker_info) {
+      this.log.error('Worker not found for write operation', { worker_id });
+      return;
+    }
+    
+    try {
+      const result = await this.perform_database_operation(op_type, data);
+      
+      worker_info.worker.send({
+        type: 'write_response',
+        data: {
+          write_id,
+          success: true,
+          result
+        }
+      });
+      
+      this.broadcast_write_notification(op_type, data, worker_id);
+      
+    } catch (error) {
+      this.log.error('Write operation failed', {
+        write_id,
+        op_type,
+        worker_id,
+        error_message: error.message
+      });
+      
+      worker_info.worker.send({
+        type: 'write_response',
+        data: {
+          write_id,
+          success: false,
+          error: error.message
+        }
+      });
+    }
+  }
+  
+  /**
+   * Performs database operations (both read and write operations).
+   * @param {string} op_type - Type of database operation
+   * @param {Object} data - Operation data
+   * @returns {Promise<Object>} Operation result
+   * @throws {Error} When operation fails or is unsupported
+   */
+  async perform_database_operation(op_type, data) {
+    const operation_start = Date.now();
+    this.log.info('Executing database operation', { op_type });
+    
+    try {
+      let result;
+      const database_name = data.database || 'default';
+      
+      switch (op_type) {
+        case 'insert_one':
+          result = await insert_one(database_name, data.collection, data.document, data.options);
+          break;
+          
+        case 'update_one':
+          result = await update_one(database_name, data.collection, data.filter, data.update, data.options);
+          break;
+          
+        case 'delete_one':
+          result = await delete_one(database_name, data.collection, data.filter, data.options);
+          break;
+          
+        case 'bulk_write':
+          result = await bulk_write(database_name, data.collection, data.operations, data.options);
+          break;
+          
+        case 'find_one':
+          result = await find_one(database_name, data.collection, data.filter, data.options);
+          break;
+          
+        case 'find':
+          result = await find(database_name, data.collection, data.filter, data.options);
+          break;
+          
+        case 'create_index':
+          result = await create_index_operation(database_name, data.collection, data.field, data.options);
+          break;
+          
+        case 'drop_index':
+          result = await drop_index_operation(database_name, data.collection, data.field);
+          break;
+          
+        case 'get_indexes':
+          result = await get_indexes_operation(database_name, data.collection);
+          break;
+          
+        default:
+          throw new Error(`Unsupported database operation: ${op_type}`);
+      }
+      
+      const duration_ms = Date.now() - operation_start;
+      this.log.log_operation(op_type, duration_ms, { result });
+      
+      // NOTE: Trigger map size check after write operations (not read operations).
+      const read_operations = ['find_one', 'find', 'get_indexes'];
+      if (!read_operations.includes(op_type)) {
+        setImmediate(() => check_and_grow_map_size());
+      }
+      
+      return result;
+    } catch (error) {
+      const duration_ms = Date.now() - operation_start;
+      this.log.error('Database operation failed', {
+        op_type,
+        duration_ms,
+        error_message: error.message
+      });
+      throw error;
+    }
+  }
+  
+  /**
+   * Broadcasts write notifications to all workers except the originating one.
+   * @param {string} op_type - Type of write operation
+   * @param {Object} data - Operation data
+   * @param {number} exclude_worker_id - Worker ID to exclude from broadcast
+   */
+  broadcast_write_notification(op_type, data, exclude_worker_id) {
+    const notification = {
+      type: 'write_notification',
+      data: {
+        op_type,
+        data,
+        timestamp: Date.now()
+      }
+    };
+    
+    for (const [worker_id, worker_info] of this.workers) {
+      if (worker_id !== exclude_worker_id && worker_info.status === 'ready') {
+        worker_info.worker.send(notification);
+      }
+    }
+  }
+  
+  async handle_auth_request(worker, message) {
+    const { auth_id, socket_id, password } = message.data;
+    
+    try {
+      const is_authenticated = await verify_password(password, 'cluster_client');
+      
+      if (is_authenticated) {
+        this.authenticated_sessions.set(socket_id, {
+          authenticated_at: Date.now(),
+          worker_id: worker.id
+        });
+      }
+      
+      worker.send({
+        type: 'auth_response',
+        data: {
+          auth_id,
+          success: is_authenticated,
+          message: is_authenticated ? 'Authentication successful' : 'Authentication failed'
+        }
+      });
+    } catch (error) {
+      worker.send({
+        type: 'auth_response',
+        data: {
+          auth_id,
+          success: false,
+          message: `Authentication error: ${error.message}`
+        }
+      });
+    }
+  }
+  
+  handle_setup_request(worker, message) {
+    const { setup_id, socket_id } = message.data;
+    
+    try {
+      const password = setup_authentication();
+      
+      const instructions = `===
+JoystickDB Setup
+
+Your database has been setup. Follow the instructions below carefully to avoid issues.
+
+Password:
+${password}
+
+Store this password in your environment settings or another secure location. When connecting a client, make sure to provide this password via the client's options object like this:
+
+import joystickdb from '@joystickdb/client';
+
+const client = joystickdb.client({
+  host: 'localhost',
+  port: 1983,
+  password: '${password}'
+});
+
+await client.ping();
+===`;
+      
+      worker.send({
+        type: 'setup_response',
+        data: {
+          setup_id,
+          success: true,
+          password,
+          instructions,
+          message: 'Authentication setup completed successfully'
+        }
+      });
+    } catch (error) {
+      worker.send({
+        type: 'setup_response',
+        data: {
+          setup_id,
+          success: false,
+          error: error.message
+        }
+      });
+    }
+  }
+  
+  update_worker_connections(worker, message) {
+    const worker_info = this.workers.get(worker.id);
+    if (worker_info) {
+      worker_info.connections = message.data.count;
+    }
+  }
+  
+  handle_worker_heartbeat(worker, message) {
+    const worker_info = this.workers.get(worker.id);
+    if (worker_info) {
+      worker_info.last_heartbeat = Date.now();
+    }
+  }
+  
+  /**
+   * Gets comprehensive cluster statistics.
+   * @returns {Object} Cluster statistics including worker info and connection counts
+   */
+  get_cluster_stats() {
+    const stats = {
+      master_pid: process.pid,
+      worker_count: this.workers.size,
+      total_connections: 0,
+      write_queue_length: this.write_queue.length,
+      authenticated_sessions: this.authenticated_sessions.size,
+      workers: []
+    };
+    
+    for (const [worker_id, worker_info] of this.workers) {
+      stats.total_connections += worker_info.connections;
+      stats.workers.push({
+        id: worker_id,
+        pid: worker_info.worker.process.pid,
+        connections: worker_info.connections,
+        status: worker_info.status,
+        last_heartbeat: worker_info.last_heartbeat
+      });
+    }
+    
+    return stats;
+  }
+  
+  /**
+   * Performs graceful shutdown of the cluster master and all workers.
+   */
+  async shutdown() {
+    const shutdown_start = Date.now();
+    this.log.info('Initiating graceful shutdown');
+    this.shutting_down = true;
+    
+    // NOTE: Stop HTTP server.
+    try {
+      await stop_http_server();
+      this.log.info('HTTP server stopped');
+    } catch (http_stop_error) {
+      this.log.warn('Failed to stop HTTP server', { error: http_stop_error.message });
+    }
+    
+    // NOTE: Stop backup scheduling.
+    try {
+      stop_backup_schedule();
+      this.log.info('Backup scheduling stopped');
+    } catch (backup_stop_error) {
+      this.log.warn('Failed to stop backup scheduling', { error: backup_stop_error.message });
+    }
+    
+    // NOTE: Send shutdown signal to all workers.
+    for (const [worker_id, worker_info] of this.workers) {
+      try {
+        worker_info.worker.send({ type: 'shutdown' });
+      } catch (error) {
+        this.log.warn('Error sending shutdown signal to worker', { worker_id, error: error.message });
+      }
+    }
+    
+    // NOTE: Wait for pending writes to complete.
+    if (this.write_queue.length > 0) {
+      this.log.info('Waiting for pending writes to complete', { 
+        pending_writes: this.write_queue.length 
+      });
+      await new Promise((resolve) => {
+        const timeout = setTimeout(() => {
+          this.log.warn('Timeout waiting for writes to complete, proceeding with shutdown');
+          resolve();
+        }, process.env.NODE_ENV === 'test' ? 1000 : 5000);
+        
+        this.once('writes_completed', () => {
+          clearTimeout(timeout);
+          resolve();
+        });
+      });
+    }
+    
+    this.log.info('All writes completed, disconnecting workers');
+    
+    // NOTE: Disconnect all workers.
+    for (const [worker_id, worker_info] of this.workers) {
+      try {
+        worker_info.worker.disconnect();
+      } catch (error) {
+        this.log.warn('Error disconnecting worker', { worker_id, error: error.message });
+      }
+    }
+    
+    // NOTE: Wait for workers to exit gracefully, but with a timeout for forced termination.
+    const graceful_timeout = process.env.NODE_ENV === 'test' ? 500 : 3000;
+    
+    await new Promise((resolve) => {
+      const timeout = setTimeout(() => {
+        // NOTE: Force kill any remaining workers.
+        for (const [worker_id, worker_info] of this.workers) {
+          this.log.warn('Force killing worker after timeout', { worker_id });
+          try {
+            worker_info.worker.kill('SIGKILL');
+          } catch (error) {
+            this.log.warn('Error force killing worker', { worker_id, error: error.message });
+          }
+        }
+        this.workers.clear();
+        resolve();
+      }, graceful_timeout);
+      
+      const check_workers = () => {
+        if (this.workers.size === 0) {
+          clearTimeout(timeout);
+          resolve();
+        } else {
+          setTimeout(check_workers, 50);
+        }
+      };
+      check_workers();
+    });
+    
+    // NOTE: Clean up database connections in master process.
+    try {
+      cleanup_database();
+      this.log.info('Database cleanup completed');
+    } catch (cleanup_error) {
+      this.log.warn('Error during database cleanup', { error: cleanup_error.message });
+    }
+    
+    // NOTE: Clear all internal state.
+    this.authenticated_sessions.clear();
+    this.write_queue.length = 0;
+    this.pending_writes.clear();
+    
+    // NOTE: In test environment, aggressively clean up cluster state to prevent worker reuse.
+    if (process.env.NODE_ENV === 'test') {
+      try {
+        // Force kill any remaining workers in the global cluster state
+        for (const worker_id in cluster.workers) {
+          const worker = cluster.workers[worker_id];
+          if (worker && !worker.isDead()) {
+            this.log.info('Force killing remaining cluster worker', { worker_id, worker_pid: worker.process.pid });
+            try {
+              worker.kill('SIGKILL');
+            } catch (kill_error) {
+              this.log.warn('Error force killing remaining worker', { worker_id, error: kill_error.message });
+            }
+          }
+        }
+        
+        // Clear the global cluster workers registry
+        for (const worker_id in cluster.workers) {
+          delete cluster.workers[worker_id];
+        }
+        
+        // Remove all cluster event listeners to prevent memory leaks
+        cluster.removeAllListeners();
+        
+        this.log.info('Aggressive cluster cleanup completed for test environment');
+      } catch (cleanup_error) {
+        this.log.warn('Error during aggressive cluster cleanup', { error: cleanup_error.message });
+      }
+    }
+    
+    const shutdown_duration = Date.now() - shutdown_start;
+    this.log.info('Shutdown complete', { shutdown_duration_ms: shutdown_duration });
+  }
+}
+
+export default ClusterMaster;