@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/lib/load_settings.js

@@ -0,0 +1,143 @@
+/**
+ * @fileoverview Settings management utilities for loading and caching JoystickDB configuration.
+ * Provides functions to load, cache, reload, and clear settings from process.env.JOYSTICK_DB_SETTINGS.
+ * This allows the database server to be started as a child process with configuration passed via environment variables.
+ */
+
+/** @type {Object|null} Cached settings object */
+let cached_settings = null;
+
+/**
+ * Validates port configuration in settings.
+ * @param {Object} settings - Settings object to validate
+ * @throws {Error} When port configuration is invalid
+ */
+const validate_port_configuration = (settings) => {
+  if (settings.port !== undefined) {
+    if (!Number.isInteger(settings.port)) {
+      throw new Error('Port must be an integer');
+    }
+    
+    if (settings.port < 1024) {
+      throw new Error('Port must be between 1024 and 65534 (HTTP port will be port + 1)');
+    }
+    
+    const http_port = settings.port + 1;
+    if (http_port > 65535) {
+      throw new Error(`HTTP port (${http_port}) would exceed maximum port number (65535). Use a lower TCP port.`);
+    }
+    
+    if (settings.port > 65534) {
+      throw new Error('Port must be between 1024 and 65534 (HTTP port will be port + 1)');
+    }
+  }
+};
+
+/**
+ * Loads JoystickDB settings from process.env.JOYSTICK_DB_SETTINGS and caches them.
+ * @returns {Object} Parsed JoystickDB settings object
+ * @throws {Error} When environment variable is missing or contains invalid JSON
+ */
+export const load_settings = () => {
+  try {
+    const raw_settings = process.env.JOYSTICK_DB_SETTINGS;
+    
+    if (!raw_settings) {
+      throw new Error('JOYSTICK_DB_SETTINGS environment variable is not set');
+    }
+    
+    const parsed_settings = JSON.parse(raw_settings);
+    
+    // NOTE: Validate port configuration if present.
+    validate_port_configuration(parsed_settings);
+    
+    cached_settings = parsed_settings;
+    
+    return cached_settings;
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      throw new Error(`Invalid JSON in JOYSTICK_DB_SETTINGS environment variable: ${error.message}`);
+    }
+    
+    throw new Error(`Failed to load JoystickDB settings: ${error.message}`);
+  }
+};
+
+/**
+ * Gets the cached settings object.
+ * @returns {Object} Cached settings object
+ * @throws {Error} When settings haven't been loaded yet
+ */
+export const get_settings = () => {
+  if (!cached_settings) {
+    throw new Error('Settings not loaded. Call load_settings() first.');
+  }
+  
+  return cached_settings;
+};
+
+/**
+ * Reloads settings from process.env.JOYSTICK_DB_SETTINGS, clearing cache first.
+ * @returns {Object} Reloaded settings object
+ * @throws {Error} When environment variable loading fails
+ */
+export const reload_settings = () => {
+  cached_settings = null;
+  return load_settings();
+};
+
+/**
+ * Clears the cached settings, forcing next load_settings call to read from environment variable.
+ */
+export const clear_settings_cache = () => {
+  cached_settings = null;
+};
+
+/**
+ * Gets the current environment name from NODE_ENV.
+ * @returns {string} Current environment name
+ */
+export const get_current_environment = () => {
+  return process.env.NODE_ENV || 'development';
+};
+
+/**
+ * Checks if JoystickDB settings are available in the environment.
+ * @returns {boolean} True if JOYSTICK_DB_SETTINGS environment variable is set
+ */
+export const has_settings = () => {
+  return !!process.env.JOYSTICK_DB_SETTINGS;
+};
+
+/**
+ * Gets the configured TCP and HTTP ports from settings.
+ * @returns {Object} Port configuration with tcp_port and http_port
+ * @returns {number} returns.tcp_port - TCP server port
+ * @returns {number} returns.http_port - HTTP server port (tcp_port + 1)
+ */
+export const get_port_configuration = () => {
+  let tcp_port = 1983; // Default TCP port
+  
+  try {
+    // NOTE: Try to load settings if they haven't been loaded yet.
+    let settings;
+    try {
+      settings = get_settings();
+    } catch (error) {
+      // NOTE: Settings not loaded yet, try to load them.
+      load_settings();
+      settings = get_settings();
+    }
+    
+    if (settings.port !== undefined) {
+      tcp_port = settings.port;
+    }
+  } catch (error) {
+    // NOTE: Settings not available or failed to load, use default port.
+  }
+  
+  return {
+    tcp_port,
+    http_port: tcp_port + 1
+  };
+};

package/src/server/lib/logger.js

@@ -0,0 +1,135 @@
+import winston from 'winston';
+import DailyRotateFile from 'winston-daily-rotate-file';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const log_directory = path.join(__dirname, '../../../logs');
+
+/**
+ * @typedef {Object} ContextLogger
+ * @property {function(string, Object): void} info - Log info message with context
+ * @property {function(string, Object): void} warn - Log warning message with context
+ * @property {function(string, Error|Object): void} error - Log error message with context or error object
+ * @property {function(string, Object): void} debug - Log debug message with context
+ * @property {function(string, number, Object): void} log_operation - Log operation completion with duration
+ * @property {function(string, string, Object): void} log_request - Log request with ID and operation
+ */
+
+/**
+ * Creates a Winston logger instance with file rotation and console output.
+ * @param {string} [component='server'] - Component name for log identification
+ * @returns {Object} Object containing logger instance and context logger factory
+ * @returns {winston.Logger} returns.logger - Winston logger instance
+ * @returns {function(Object): ContextLogger} returns.create_context_logger - Factory for context loggers
+ */
+const create_logger = (component = 'server') => {
+  const logger = winston.createLogger({
+    level: process.env.LOG_LEVEL || 'info',
+    format: winston.format.combine(
+      winston.format.timestamp({
+        format: 'YYYY-MM-DDTHH:mm:ss.SSSZ'
+      }),
+      winston.format.errors({ stack: true }),
+      winston.format.json(),
+      winston.format.printf((info) => {
+        const log_entry = {
+          timestamp: info.timestamp,
+          level: info.level,
+          component: component,
+          pid: process.pid,
+          message: info.message,
+          ...info.context && { context: info.context },
+          ...info.duration_ms && { duration_ms: info.duration_ms },
+          ...info.request_id && { request_id: info.request_id },
+          ...info.worker_id && { worker_id: info.worker_id },
+          ...info.stack && { stack: info.stack }
+        };
+        
+        return JSON.stringify(log_entry);
+      })
+    ),
+    transports: [
+      new winston.transports.Console({
+        format: winston.format.printf((info) => {
+          const log_entry = {
+            level: info.level,
+            message: info.message,
+            ...info.context && { context: info.context },
+            ...info.duration_ms && { duration_ms: info.duration_ms },
+            ...info.request_id && { request_id: info.request_id },
+            ...info.worker_id && { worker_id: info.worker_id },
+            timestamp: info.timestamp
+          };
+          
+          return JSON.stringify(log_entry);
+        })
+      }),
+      new DailyRotateFile({
+        filename: path.join(log_directory, `${component}-%DATE%.log`),
+        datePattern: 'YYYY-MM-DD',
+        zippedArchive: true,
+        maxSize: '20m',
+        maxFiles: '14d',
+        format: winston.format.json()
+      }),
+      new DailyRotateFile({
+        filename: path.join(log_directory, `${component}-error-%DATE%.log`),
+        datePattern: 'YYYY-MM-DD',
+        zippedArchive: true,
+        maxSize: '20m',
+        maxFiles: '30d',
+        level: 'error',
+        format: winston.format.json()
+      })
+    ]
+  });
+
+  /**
+   * Creates a context logger with predefined base context.
+   * @param {Object} [base_context={}] - Base context to include in all log messages
+   * @returns {ContextLogger} Context logger with info, warn, error, debug methods
+   */
+  const create_context_logger = (base_context = {}) => {
+    return {
+      info: (message, context = {}) => {
+        logger.info(message, { context: { ...base_context, ...context } });
+      },
+      warn: (message, context = {}) => {
+        logger.warn(message, { context: { ...base_context, ...context } });
+      },
+      error: (message, error_or_context = {}) => {
+        if (error_or_context instanceof Error) {
+          logger.error(message, { 
+            context: base_context,
+            stack: error_or_context.stack,
+            error_message: error_or_context.message
+          });
+        } else {
+          logger.error(message, { context: { ...base_context, ...error_or_context } });
+        }
+      },
+      debug: (message, context = {}) => {
+        logger.debug(message, { context: { ...base_context, ...context } });
+      },
+      log_operation: (operation, duration_ms, context = {}) => {
+        logger.info(`Operation completed: ${operation}`, {
+          context: { ...base_context, ...context, operation },
+          duration_ms
+        });
+      },
+      log_request: (request_id, operation, context = {}) => {
+        logger.info(`Request: ${operation}`, {
+          request_id,
+          context: { ...base_context, ...context, operation }
+        });
+      }
+    };
+  };
+
+  return { logger, create_context_logger };
+};
+
+export default create_logger;

package/src/server/lib/op_types.js

@@ -0,0 +1,29 @@
+/**
+ * @fileoverview Supported operation types for JoystickDB server validation.
+ * 
+ * Defines the complete list of valid operation types that the server can handle,
+ * used for request validation and operation routing.
+ */
+
+/**
+ * Array of supported operation types for the JoystickDB server.
+ * @type {string[]}
+ */
+const op_types = [
+  "authentication",
+  "setup",
+  "find_one",
+  "find",
+  "insert_one",
+  "update_one",
+  "delete_one",
+  "bulk_write",
+  "create_index",
+  "drop_index",
+  "get_indexes",
+  "admin",
+  "ping",
+  "reload",
+];
+
+export default op_types;

package/src/server/lib/operation_dispatcher.js

@@ -0,0 +1,268 @@
+/**
+ * @fileoverview Central operation dispatcher for routing and handling database and admin operations.
+ * 
+ * Provides unified handling for all database operations (CRUD, indexing) and admin operations
+ * with authentication, performance monitoring, replication, and write forwarding support.
+ * Manages operation routing, error handling, and response formatting.
+ */
+
+import { encode_message } from './tcp_protocol.js';
+import { get_write_forwarder } from './write_forwarder.js';
+import { get_replication_manager } from './replication_manager.js';
+import { check_and_grow_map_size } from './query_engine.js';
+import { performance_monitor } from './performance_monitor.js';
+import create_logger from './logger.js';
+import insert_one from './operations/insert_one.js';
+import update_one from './operations/update_one.js';
+import delete_one from './operations/delete_one.js';
+import bulk_write from './operations/bulk_write.js';
+import find_one from './operations/find_one.js';
+import find from './operations/find.js';
+import create_index_operation from './operations/create_index.js';
+import drop_index_operation from './operations/drop_index.js';
+import get_indexes_operation from './operations/get_indexes.js';
+import admin_operation from './operations/admin.js';
+
+const { create_context_logger } = create_logger('operation_dispatcher');
+
+/**
+ * Validates database name according to naming rules.
+ * @param {string} database_name - Database name to validate
+ * @returns {boolean} True if valid, false otherwise
+ */
+const validate_database_name = (database_name) => {
+  if (!database_name || typeof database_name !== 'string') {
+    return false;
+  }
+  
+  // NOTE: Check length limit.
+  if (database_name.length > 64) {
+    return false;
+  }
+  
+  // NOTE: Check for reserved names.
+  const reserved_names = ['admin', 'config', 'local'];
+  if (reserved_names.includes(database_name.toLowerCase())) {
+    return false;
+  }
+  
+  // NOTE: Check naming convention (alphanumeric, underscores, hyphens).
+  const valid_name_pattern = /^[a-zA-Z0-9_-]+$/;
+  return valid_name_pattern.test(database_name);
+};
+
+/**
+ * Handles database operations (CRUD, indexing) with authentication, logging, and performance monitoring.
+ * @param {net.Socket} socket - Client socket connection
+ * @param {string} op_type - Type of database operation
+ * @param {Object} data - Operation data
+ * @param {Function} check_authentication - Function to check if socket is authenticated
+ * @param {number} [raw_data_size=0] - Size of raw request data for monitoring
+ * @param {Object} [connection_manager] - Connection manager instance (optional)
+ * @param {Set} [authenticated_clients] - Set of authenticated client IDs (optional)
+ */
+export const handle_database_operation = async (socket, op_type, data, check_authentication, raw_data_size = 0, connection_manager = null, authenticated_clients = null) => {
+  const log = create_context_logger();
+  const start_time = Date.now();
+  
+  if (!check_authentication(socket)) {
+    const response = { ok: 0, error: 'Authentication required' };
+    const encoded_response = encode_message(response);
+    socket.write(encoded_response);
+    performance_monitor.log_structured_operation(socket.id, op_type, null, 0, 'error', 'Authentication required', raw_data_size, 0);
+    return;
+  }
+  
+  // NOTE: Extract and validate database name from request.
+  const database_name = data.database || 'default';
+  
+  if (!validate_database_name(database_name)) {
+    const response = { ok: 0, error: 'Invalid database name. Database names must be alphanumeric with underscores/hyphens, max 64 characters, and cannot be reserved names (admin, config, local).' };
+    const encoded_response = encode_message(response);
+    socket.write(encoded_response);
+    performance_monitor.log_structured_operation(socket.id, op_type, data.collection, 0, 'error', 'Invalid database name', raw_data_size, 0);
+    return;
+  }
+  
+  // NOTE: Check if this is a secondary node and should forward write operations.
+  const write_forwarder = get_write_forwarder();
+  const forwarded = await write_forwarder.forward_operation(socket, op_type, data);
+  
+  if (forwarded) {
+    // Operation was forwarded to primary, response will be handled by forwarder
+    return;
+  }
+  
+  try {
+    let result;
+    
+    switch (op_type) {
+      case 'insert_one':
+        result = await insert_one(database_name, data.collection, data.document, data.options);
+        break;
+        
+      case 'update_one':
+        result = await update_one(database_name, data.collection, data.filter, data.update, data.options);
+        break;
+        
+      case 'delete_one':
+        result = await delete_one(database_name, data.collection, data.filter, data.options);
+        break;
+        
+      case 'bulk_write':
+        result = await bulk_write(database_name, data.collection, data.operations, data.options);
+        break;
+        
+      case 'find_one':
+        result = await find_one(database_name, data.collection, data.filter, data.options);
+        break;
+        
+      case 'find':
+        result = await find(database_name, data.collection, data.filter, data.options);
+        break;
+        
+      case 'create_index':
+        result = await create_index_operation(database_name, data.collection, data.field, data.options);
+        break;
+        
+      case 'drop_index':
+        result = await drop_index_operation(database_name, data.collection, data.field);
+        break;
+        
+      case 'get_indexes':
+        result = await get_indexes_operation(database_name, data.collection);
+        break;
+        
+      default:
+        throw new Error(`Unsupported operation: ${op_type}`);
+    }
+    
+    const duration_ms = Date.now() - start_time;
+    
+    // NOTE: All operations now return structured responses for consistency.
+    let response;
+    if (op_type === 'find_one') {
+      response = { ok: 1, document: result };
+    } else if (op_type === 'find') {
+      response = { ok: 1, documents: result };
+    } else {
+      response = { ok: 1, ...result };
+    }
+    
+    const encoded_response = encode_message(response);
+    const response_size = encoded_response.length;
+    
+    socket.write(encoded_response);
+    
+    // NOTE: Log structured operation with performance monitoring.
+    performance_monitor.log_structured_operation(
+      socket.id, 
+      op_type, 
+      data.collection, 
+      duration_ms, 
+      'success', 
+      null, 
+      raw_data_size, 
+      response_size
+    );
+    
+    log.info('Database operation completed', {
+      client_id: socket.id,
+      op: op_type,
+      collection: data.collection,
+      duration_ms,
+      status: 'success',
+      request_size: raw_data_size,
+      response_size
+    });
+    
+    // NOTE: Queue write operations for replication to secondary nodes.
+    if (op_type !== 'find' && op_type !== 'find_one' && op_type !== 'get_indexes') {
+      const replication_manager = get_replication_manager();
+      replication_manager.queue_replication(op_type, data.collection, data);
+      
+      setImmediate(() => check_and_grow_map_size());
+    }
+  } catch (error) {
+    const duration_ms = Date.now() - start_time;
+    
+    // NOTE: Log structured operation with error.
+    performance_monitor.log_structured_operation(
+      socket.id, 
+      op_type, 
+      data.collection, 
+      duration_ms, 
+      'error', 
+      error.message, 
+      raw_data_size, 
+      0
+    );
+    
+    log.error('Database operation failed', {
+      client_id: socket.id,
+      op: op_type,
+      collection: data.collection,
+      duration_ms,
+      status: 'error',
+      error: error.message,
+      request_size: raw_data_size
+    });
+    
+    const error_response = { ok: 0, error: error.message };
+    const encoded_error_response = encode_message(error_response);
+    socket.write(encoded_error_response);
+  }
+};
+
+/**
+ * Handles admin operations with authentication and error handling.
+ * @param {net.Socket} socket - Client socket connection
+ * @param {Object} data - Admin operation data
+ * @param {Function} check_authentication - Function to check if socket is authenticated
+ * @param {Object} [connection_manager] - Connection manager instance (optional)
+ * @param {Set} [authenticated_clients] - Set of authenticated client IDs (optional)
+ */
+export const handle_admin_operation = async (socket, data, check_authentication, connection_manager = null, authenticated_clients = null) => {
+  if (!check_authentication(socket)) {
+    // NOTE: Unauthenticated admin requests should return ok: false.
+    const response = { ok: false, error: 'Authentication required' };
+    const encoded_response = encode_message(response);
+    socket.write(encoded_response);
+    return;
+  }
+  
+  try {
+    const admin_action = data?.admin_action;
+    const admin_data = data || {};
+    
+    const result = await admin_operation(admin_action, admin_data, connection_manager, authenticated_clients);
+    
+    // NOTE: Handle different response formats for admin operations.
+    if (!admin_action) {
+      // NOTE: Default admin operation (backward compatibility) - return result directly with ok: true.
+      const response = { ok: true, ...result };
+      const encoded_response = encode_message(response);
+      socket.write(encoded_response);
+    } else {
+      // NOTE: Specific admin actions - return result directly with ok: 1.
+      const response = { ok: 1, ...result };
+      const encoded_response = encode_message(response);
+      socket.write(encoded_response);
+    }
+  } catch (error) {
+    // NOTE: Admin operation errors use ok: 0.
+    const response = { ok: 0, error: `Admin operation failed: ${error.message}` };
+    const encoded_response = encode_message(response);
+    socket.write(encoded_response);
+  }
+};
+
+/**
+ * Handles ping operations.
+ * @param {net.Socket} socket - Client socket connection
+ */
+export const handle_ping_operation = (socket) => {
+  const ping_response = { ok: 1 };
+  const encoded_ping_response = encode_message(ping_response);
+  socket.write(encoded_ping_response);
+};