@joystick.js/db-canary 0.0.0-canary.2300 → 0.0.0-canary.2302

package/debug_storage_engine.js

@@ -0,0 +1,56 @@
+/**
+ * Debug script to identify what object is being passed as storage engine
+ */
+
+import { initialize_database, get_database } from './src/server/lib/query_engine.js';
+
+async function debug_storage_engine() {
+  console.log('=== DEBUGGING STORAGE ENGINE ===');
+  
+  try {
+    // Initialize database
+    const db_path = `./debug_test_${Date.now()}`;
+    const db = await initialize_database(db_path);
+    
+    console.log('Database instance type:', typeof db);
+    console.log('Database constructor name:', db.constructor?.name);
+    
+    // Get all methods and properties
+    const methods = Object.getOwnPropertyNames(db).filter(prop => typeof db[prop] === 'function');
+    const properties = Object.getOwnPropertyNames(db).filter(prop => typeof db[prop] !== 'function');
+    
+    console.log('Available methods:', methods);
+    console.log('Available properties:', properties);
+    
+    // Check specific LMDB methods
+    const lmdb_methods = ['get', 'put', 'del', 'getRange', 'transaction'];
+    console.log('\nLMDB method availability:');
+    for (const method of lmdb_methods) {
+      console.log(`  ${method}:`, typeof db[method] === 'function' ? '✓ available' : '✗ missing');
+    }
+    
+    // Check problematic methods
+    const problematic_methods = ['writeKey', 'readKey'];
+    console.log('\nProblematic method check:');
+    for (const method of problematic_methods) {
+      console.log(`  ${method}:`, typeof db[method] === 'function' ? '✗ present' : '✓ not present');
+    }
+    
+    // Check prototype chain
+    console.log('\nPrototype chain:');
+    let current = db;
+    let level = 0;
+    while (current && level < 5) {
+      console.log(`  Level ${level}:`, current.constructor?.name || 'Unknown');
+      current = Object.getPrototypeOf(current);
+      level++;
+    }
+    
+    console.log('\n=== END DEBUG ===');
+    
+  } catch (error) {
+    console.error('Debug failed:', error);
+  }
+}
+
+debug_storage_engine().catch(console.error);

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@joystick.js/db-canary",
   "type": "module",
-  "version": "0.0.0-canary.2300",
-  "canary_version": "0.0.0-canary.2299",
+  "version": "0.0.0-canary.2302",
+  "canary_version": "0.0.0-canary.2301",
   "description": "JoystickDB - A minimalist database server for the Joystick framework",
   "main": "./dist/server/index.js",
   "scripts": {

package/src/client/index.js

@@ -135,11 +135,11 @@ const create_client_options = (options = {}) => ({
  * @param {number} timeout_ms - Timeout duration in milliseconds
  * @returns {NodeJS.Timeout} Timeout reference
  */
-const create_connection_timeout = (socket, error_handler, timeout_ms) => {
+const create_connection_timeout = (socket, error_handler, timeout_ms, host, port) => {
   return setTimeout(() => {
     if (socket && !socket.destroyed) {
       socket.destroy();
-      error_handler(new Error('Connection timeout'));
+      error_handler(new Error(`Connection timeout after ${timeout_ms}ms to ${host}:${port}`));
     }
   }, timeout_ms);
 };
@@ -151,12 +151,12 @@ const create_connection_timeout = (socket, error_handler, timeout_ms) => {
  * @param {number} timeout_ms - Timeout duration in milliseconds
  * @returns {NodeJS.Timeout} Timeout reference
  */
-const create_request_timeout = (pending_requests, request_id, timeout_ms) => {
+const create_request_timeout = (pending_requests, request_id, timeout_ms, operation = 'unknown') => {
   return setTimeout(() => {
     const request = pending_requests.get(request_id);
     if (request) {
       pending_requests.delete(request_id);
-      request.reject(new Error('Request timeout'));
+      request.reject(new Error(`Request timeout after ${timeout_ms}ms for operation: ${operation}`));
     }
   }, timeout_ms);
 };
@@ -273,10 +273,15 @@ class JoystickDBClient extends EventEmitter {
     this.socket = new net.Socket();
     this.message_parser = create_message_parser();
     
+    this.socket.setNoDelay(true);
+    this.socket.setKeepAlive(true, 30000);
+    
     const connection_timeout = create_connection_timeout(
       this.socket, 
       this.handle_connection_error.bind(this), 
-      this.timeout
+      this.timeout,
+      this.host,
+      this.port
     );
     
     this.setup_socket_handlers(connection_timeout);
@@ -494,16 +499,38 @@ class JoystickDBClient extends EventEmitter {
   send_request_now(request) {
     const { message, resolve, reject, request_id } = request;
     
-    const timeout = create_request_timeout(this.pending_requests, request_id, this.timeout);
+    if (!this.socket || this.socket.destroyed) {
+      reject(new Error('Socket not available'));
+      return;
+    }
+    
+    const timeout = create_request_timeout(
+      this.pending_requests, 
+      request_id, 
+      this.timeout,
+      message.op
+    );
     this.pending_requests.set(request_id, { resolve, reject, timeout });
     
     try {
       const encoded_message = encode_message(message);
-      this.socket.write(encoded_message);
+      
+      const write_success = this.socket.write(encoded_message, (error) => {
+        if (error) {
+          clearTimeout(timeout);
+          this.pending_requests.delete(request_id);
+          reject(new Error(`Socket write error: ${error.message}`));
+        }
+      });
+      
+      if (!write_success) {
+        this.socket.once('drain', () => {});
+      }
+      
     } catch (error) {
       clearTimeout(timeout);
       this.pending_requests.delete(request_id);
-      reject(error);
+      reject(new Error(`Message encoding error: ${error.message}`));
     }
   }
   
@@ -522,9 +549,82 @@ class JoystickDBClient extends EventEmitter {
       this.reconnect_timeout = null;
     }
     
-    if (this.socket) {
+    return new Promise((resolve) => {
+      if (!this.socket || this.socket.destroyed) {
+        resolve();
+        return;
+      }
+      
+      const cleanup_timeout = setTimeout(() => {
+        if (this.socket && !this.socket.destroyed) {
+          this.socket.destroy();
+        }
+        resolve();
+      }, 1000);
+      
+      this.socket.once('close', () => {
+        clearTimeout(cleanup_timeout);
+        resolve();
+      });
+      
       this.socket.end();
-    }
+    });
+  }
+  
+  /**
+   * Waits for client to be fully ready (connected and authenticated if needed).
+   * @param {number} timeout_ms - Maximum time to wait in milliseconds
+   * @returns {Promise<void>} Resolves when client is ready
+   */
+  async wait_for_ready(timeout_ms = 5000) {
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        reject(new Error(`Client connection timeout after ${timeout_ms}ms`));
+      }, timeout_ms);
+      
+      const check_ready = () => {
+        if (this.is_connected && (!this.authentication || this.is_authenticated)) {
+          clearTimeout(timeout);
+          resolve();
+          return true;
+        }
+        return false;
+      };
+      
+      // Check if already ready
+      if (check_ready()) {
+        return;
+      }
+      
+      // Wait for connection events
+      const cleanup_listeners = () => {
+        this.removeListener('connect', on_connect);
+        this.removeListener('authenticated', on_authenticated);
+        this.removeListener('error', on_error);
+      };
+      
+      const on_connect = () => {
+        if (check_ready()) {
+          cleanup_listeners();
+        }
+      };
+      
+      const on_authenticated = () => {
+        if (check_ready()) {
+          cleanup_listeners();
+        }
+      };
+      
+      const on_error = (error) => {
+        clearTimeout(timeout);
+        cleanup_listeners();
+        reject(error);
+      };
+      
+      this.on('connect', on_connect);
+      this.on('authenticated', on_authenticated);
+      this.on('error', on_error);
+    });
   }
   
   // NOTE: Backup Operations.
@@ -594,6 +694,44 @@ class JoystickDBClient extends EventEmitter {
   }
   
   // NOTE: Database Operations.
+  async insert(database, collection, document, options = {}) {
+    return this.send_request('insert_one', { 
+      database, 
+      collection, 
+      document, 
+      options 
+    });
+  }
+  
+  async insert_one(database, collection, document, options = {}) {
+    return this.send_request('insert_one', { 
+      database, 
+      collection, 
+      document, 
+      options 
+    });
+  }
+  
+  async find_one(database, collection, filter = {}, options = {}) {
+    const result = await this.send_request('find_one', { 
+      database, 
+      collection, 
+      filter, 
+      options 
+    });
+    return result.document;
+  }
+  
+  async find(database, collection, filter = {}, options = {}) {
+    const result = await this.send_request('find', { 
+      database, 
+      collection, 
+      filter, 
+      options 
+    });
+    return result.documents || [];
+  }
+  
   async delete_many(collection, filter = {}, options = {}) {
     return this.send_request('delete_many', { 
       database: 'default',

package/src/server/cluster/index.js

@@ -45,7 +45,7 @@ const start_cluster = (options = {}) => {
 };
 
 // NOTE: Auto-execute cluster logic when this file is run directly by cluster.fork().
-if (process.argv[1].includes('cluster/index.js')) {
+if (process.argv && process.argv[1] && process.argv[1].includes('cluster/index.js')) {
   start_cluster();
 }
 

package/src/server/cluster/master.js

@@ -230,6 +230,19 @@ class ClusterMaster extends EventEmitter {
     }
   }
 
+  /**
+   * Loads server settings with fallback for missing files.
+   * @returns {Object|null} Settings object or null if not available
+   */
+  load_server_settings() {
+    try {
+      load_settings();
+      return get_settings();
+    } catch (error) {
+      return null;
+    }
+  }
+
   /**
    * Starts the cluster master and spawns worker processes.
    * @throws {Error} When startup fails
@@ -238,8 +251,12 @@ class ClusterMaster extends EventEmitter {
     const start_time = Date.now();
     
     try {
-      this.settings = load_settings(this.settings_file);
-      this.log.info('Settings loaded successfully', { settings_file: this.settings_file });
+      this.settings = this.load_server_settings();
+      if (this.settings) {
+        this.log.info('Settings loaded successfully');
+      } else {
+        this.log.info('No settings found, using defaults');
+      }
       
       await this.initialize_core_systems();
       await this.handle_startup_restore();
@@ -256,6 +273,13 @@ class ClusterMaster extends EventEmitter {
       
     } catch (error) {
       this.log.error('Failed to start master process', { error: error.message });
+      
+      // For test environment, throw error instead of exiting process
+      if (process.env.NODE_ENV === 'test') {
+        throw error;
+      }
+      
+      // In production, exit the process
       process.exit(1);
     }
   }
@@ -268,9 +292,23 @@ class ClusterMaster extends EventEmitter {
     const spawn_start = Date.now();
     this.log.info('Spawning worker');
     
+    // NOTE: Ensure settings is always an object for worker environment, and handle serialization safely
+    const worker_settings = this.settings || {};
+    let serialized_settings;
+    
+    try {
+      // NOTE: Create a clean serializable version of settings
+      const clean_settings = JSON.parse(JSON.stringify(worker_settings));
+      serialized_settings = JSON.stringify(clean_settings);
+    } catch (json_error) {
+      // NOTE: If JSON serialization fails, use empty object
+      this.log.warn('Settings serialization failed, using empty settings', { error: json_error.message });
+      serialized_settings = JSON.stringify({});
+    }
+    
     const worker = cluster.fork({
       WORKER_PORT: this.port,
-      WORKER_SETTINGS: JSON.stringify(this.settings)
+      WORKER_SETTINGS: serialized_settings
     });
     
     this.workers.set(worker.id, {

package/src/server/index.js

@@ -70,6 +70,29 @@ import { has_settings } from './lib/load_settings.js';
 const authenticated_clients = new Set();
 let connection_manager = null;
 
+/**
+ * Resets server singleton state for testing.
+ * Clears authenticated clients and connection manager.
+ */
+const reset_server_state = () => {
+  authenticated_clients.clear();
+  if (connection_manager) {
+    connection_manager.shutdown();
+  }
+  connection_manager = null;
+};
+
+/**
+ * Gets current server state for verification.
+ * @returns {Object} Current server state
+ */
+const get_server_state = () => {
+  return {
+    authenticated_clients_count: authenticated_clients.size,
+    has_connection_manager: !!connection_manager
+  };
+};
+
 /**
  * Generates a cryptographically secure random password.
  * @returns {string} 32-character hexadecimal password
@@ -282,12 +305,12 @@ export const parse_data = (raw_data) => {
 
 /**
  * Checks if a socket connection is authenticated.
- * In development mode, authentication is bypassed for TCP connections.
+ * In development or test mode, authentication is bypassed for TCP connections.
  * @param {net.Socket} socket - Socket connection to check
- * @returns {boolean} True if socket is authenticated or in development mode
+ * @returns {boolean} True if socket is authenticated or in development/test mode
  */
 const check_authentication = (socket) => {
-  if (is_development_mode()) {
+  if (is_development_mode() || process.env.NODE_ENV === 'test') {
     return true;
   }
   
@@ -354,16 +377,18 @@ const initialize_server_components = async (settings) => {
   const { tcp_port } = get_port_configuration();
   const database_path = settings?.data_path || `./.joystick/data/joystickdb_${tcp_port}`;
   
-  const db = initialize_database(database_path);
-  initialize_auth_manager();
+  const db = await initialize_database(database_path);
   
-  // NOTE: Initialize user authentication manager and set storage engine reference
+  // NOTE: Always initialize user authentication manager for username/password authentication
   initialize_user_auth_manager();
+  
+  // Set the actual LMDB database instance as storage engine
   set_storage_engine(db);
   
+  initialize_auth_manager();
+  
   if (is_development_mode()) {
-    // NOTE: Development mode: Use user_auth_manager only, skip api_key_manager complexity
-    // Only auto-create admin user if none exists and we're not in test mode
+    // NOTE: Development mode: Auto-create admin user for local development
     if (process.env.NODE_ENV !== 'test') {
       try {
         const auth_stats = await get_auth_stats();
@@ -380,7 +405,8 @@ const initialize_server_components = async (settings) => {
       }
     }
   } else {
-    // NOTE: Production mode: Use api_key_manager for secure initial setup
+    // NOTE: Production mode: Initialize API key manager for secure initial user setup
+    // Users created via API key will be stored in same location as user_auth_manager expects
     await initialize_api_key_manager();
   }
   
@@ -719,7 +745,10 @@ const handle_new_socket_connection = (socket, log) => {
 const create_server_cleanup_function = () => {
   return async () => {
     try {
-      await stop_http_server();
+      // NOTE: In test environment, don't let HTTP server shutdown affect TCP server
+      if (process.env.NODE_ENV !== 'test') {
+        await stop_http_server();
+      }
       stop_backup_schedule();
       await shutdown_simple_sync_manager();
       await shutdown_sync_receiver();
@@ -737,6 +766,11 @@ const create_server_cleanup_function = () => {
       reset_auth_state();
       reset_user_auth_state();
       reset_recovery_state();
+      
+      // NOTE: Stop HTTP server last in test environment
+      if (process.env.NODE_ENV === 'test') {
+        await stop_http_server();
+      }
     } catch (error) {
       // NOTE: Ignore cleanup errors in tests.
     }
@@ -841,6 +875,9 @@ const log_server_startup_info = (options, log) => {
   });
 };
 
+// Export reset functions for testing
+export { reset_server_state, get_server_state };
+
 if (import.meta.url === `file://${process.argv[1]}`) {
   const { create_context_logger } = create_logger('main');
   const log = create_context_logger();

package/src/server/lib/api_key_manager.js

@@ -18,7 +18,10 @@ const log = create_context_logger();
 /** @type {string} Path to API key file */
 const API_KEY_FILE_PATH = './API_KEY';
 
-/** @type {string} Collection name for storing users */
+/** @type {string} Admin database name */
+const ADMIN_DATABASE = '_admin';
+
+/** @type {string} Users collection name */
 const USERS_COLLECTION = '_users';
 
 /** @type {number} BCrypt salt rounds for password hashing */
@@ -240,7 +243,7 @@ const create_user = async (user_data, is_development_default = false) => {
   };
   
   // Store user in database using transaction
-  const user_key = build_collection_key(USERS_COLLECTION, username);
+  const user_key = build_collection_key(ADMIN_DATABASE, USERS_COLLECTION, username);
   let stored_user = null;
   
   await db.transaction(() => {
@@ -278,7 +281,7 @@ const get_all_users = () => {
   const db = get_database();
   const users = [];
   
-  for (const { key, value } of db.getRange({ start: `${USERS_COLLECTION}:`, end: `${USERS_COLLECTION}:\xFF` })) {
+  for (const { key, value } of db.getRange({ start: `${ADMIN_DATABASE}:${USERS_COLLECTION}:`, end: `${ADMIN_DATABASE}:${USERS_COLLECTION}:\xFF` })) {
     const parsed_user = JSON.parse(value);
     const user_data = {
       username: parsed_user.username,
@@ -299,7 +302,7 @@ const get_all_users = () => {
  */
 const get_user = (username) => {
   const db = get_database();
-  const user_key = build_collection_key(USERS_COLLECTION, username);
+  const user_key = build_collection_key(ADMIN_DATABASE, USERS_COLLECTION, username);
   const user_data = db.get(user_key);
   
   if (!user_data) {
@@ -327,7 +330,7 @@ const get_user = (username) => {
  */
 const update_user = async (username, updates) => {
   const db = get_database();
-  const user_key = build_collection_key(USERS_COLLECTION, username);
+  const user_key = build_collection_key(ADMIN_DATABASE, USERS_COLLECTION, username);
   
   // Validate password first if provided (before transaction)
   let password_hash = null;
@@ -385,7 +388,7 @@ const update_user = async (username, updates) => {
  */
 const delete_user = (username) => {
   const db = get_database();
-  const user_key = build_collection_key(USERS_COLLECTION, username);
+  const user_key = build_collection_key(ADMIN_DATABASE, USERS_COLLECTION, username);
   
   // Check if user exists first
   const existing_user = db.get(user_key);
@@ -408,7 +411,7 @@ const delete_user = (username) => {
  */
 const verify_user_password = async (username, password) => {
   const db = get_database();
-  const user_key = build_collection_key(USERS_COLLECTION, username);
+  const user_key = build_collection_key(ADMIN_DATABASE, USERS_COLLECTION, username);
   const user_data = db.get(user_key);
   
   if (!user_data) {
@@ -441,9 +444,17 @@ const check_admin_user_exists = () => {
   
   const db = get_database();
   
-  for (const { value } of db.getRange({ start: `${USERS_COLLECTION}:`, end: `${USERS_COLLECTION}:\xFF` })) {
-    const parsed_user = JSON.parse(value);
-    if (parsed_user.role === 'read_write') {
+  for (const { value } of db.getRange({ start: `${ADMIN_DATABASE}:${USERS_COLLECTION}:`, end: `${ADMIN_DATABASE}:${USERS_COLLECTION}:\xFF` })) {
+    let parsed_user;
+    
+    // Handle both string and object values
+    if (typeof value === 'string') {
+      parsed_user = JSON.parse(value);
+    } else {
+      parsed_user = value;
+    }
+    
+    if (parsed_user && parsed_user.role === 'read_write') {
       admin_user_exists = true;
       return true;
     }

package/src/server/lib/development_mode.js

@@ -12,10 +12,10 @@ const log = create_context_logger();
 
 /**
  * Checks if the server is running in development mode.
- * @returns {boolean} True if NODE_ENV is set to 'development' or 'test'
+ * @returns {boolean} True if NODE_ENV is set to 'development'
  */
 const is_development_mode = () => {
-  return process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test';
+  return process.env.NODE_ENV === 'development';
 };
 
 /**