@joystick.js/db-canary 0.0.0-canary.2274 → 0.0.0-canary.2276

package/package.json

@@ -1,21 +1,24 @@
 {
   "name": "@joystick.js/db-canary",
   "type": "module",
-  "version": "0.0.0-canary.2274",
-  "canary_version": "0.0.0-canary.2273",
+  "version": "0.0.0-canary.2276",
+  "canary_version": "0.0.0-canary.2275",
   "description": "JoystickDB - A minimalist database server for the Joystick framework",
   "main": "./dist/server/index.js",
   "scripts": {
     "build": "node ./.build/index.js",
     "release": "node increment_version.js && npm run build && npm publish",
     "start": "node src/server/index.js",
-    "test": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=8192' ava --serial --timeout=10m",
+    "test": "NODE_ENV=test node test_runner.js core",
     "test:watch": "NODE_ENV=test ava --watch",
-    "test:performance": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/*.test.js",
-    "test:enterprise": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/bulk_insert_enterprise_*.test.js",
-    "test:benchmarks": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/bulk_insert_benchmarks.test.js",
-    "test:1m": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=8192' ava --serial --timeout=15m tests/performance/bulk_insert_1m_test.js",
+    "test:performance": "NODE_ENV=test node test_runner.js bulk",
+    "test:enterprise": "NODE_ENV=test node test_runner.js enterprise",
+    "test:benchmarks": "NODE_ENV=test node test_runner.js benchmarks",
+    "test:1m": "NODE_ENV=test node test_runner.js individual",
     "test:runner": "node test_runner.js",
+    "test:safe": "NODE_ENV=test node test_runner.js safe",
+    "test:standard": "NODE_ENV=test node test_runner.js standard",
+    "test:isolated": "NODE_ENV=test node test_runner.js isolated",
     "build:types": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types src/**/*.js",
     "build:types:client": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types/client src/client/*.js",
     "build:types:server": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types/server src/server/**/*.js"

package/src/client/index.js

@@ -587,6 +587,7 @@ class JoystickDBClient extends EventEmitter {
   // NOTE: Database Operations.
   async delete_many(collection, filter = {}, options = {}) {
     return this.send_request('delete_many', { 
+      database: 'default',
       collection, 
       filter, 
       options 

package/src/server/cluster/master.js

@@ -102,14 +102,20 @@ class ClusterMaster extends EventEmitter {
    * @returns {string} Database path
    */
   get_database_path() {
-    let database_path = './data';
+    let database_path;
     try {
       const settings = get_settings();
       if (settings?.data_path) {
         database_path = settings.data_path;
+      } else {
+        // NOTE: Use proper .joystick/data path with port number as fallback, matching other database providers
+        const { tcp_port } = get_port_configuration();
+        database_path = `./.joystick/data/joystickdb_${tcp_port}`;
       }
     } catch (error) {
-      // Settings not available, use default path
+      // NOTE: Settings not available, use default path with port from this.port
+      const { tcp_port } = get_port_configuration();
+      database_path = `./.joystick/data/joystickdb_${tcp_port}`;
     }
     return database_path;
   }

package/src/server/cluster/worker.js

@@ -99,7 +99,7 @@ class ClusterWorker {
     }
   }
   
-  handle_config(message) {
+  async handle_config(message) {
     const incoming_master_id = message.data.master_id;
     
     // NOTE: Only handle config from the first master to avoid conflicts between multiple test masters.
@@ -134,14 +134,20 @@ class ClusterWorker {
     // NOTE: Initialize database for read operations in worker process.
     try {
       // NOTE: Initialize database with data_path from settings if available.
-      let database_path = './data'; // Default path
+      let database_path;
       try {
         const settings = get_settings();
         if (settings?.data_path) {
           database_path = settings.data_path;
+        } else {
+          // NOTE: Use proper .joystick/data path with port number as fallback, matching other database providers
+          const { get_port_configuration } = await import('../lib/load_settings.js');
+          const { tcp_port } = get_port_configuration();
+          database_path = `./.joystick/data/joystickdb_${tcp_port}`;
         }
       } catch (error) {
-        // NOTE: Settings not available, use default path.
+        // NOTE: Settings not available, use default path with port from this.port
+        database_path = `./.joystick/data/joystickdb_${this.port}`;
       }
       
       initialize_database(database_path);

package/src/server/index.js

@@ -42,13 +42,13 @@ import {
   stop_backup_schedule 
 } from './lib/backup_manager.js';
 import { 
-  initialize_replication_manager, 
-  shutdown_replication_manager
-} from './lib/replication_manager.js';
+  initialize_simple_sync_manager, 
+  shutdown_simple_sync_manager
+} from './lib/simple_sync_manager.js';
 import { 
-  initialize_write_forwarder, 
-  shutdown_write_forwarder
-} from './lib/write_forwarder.js';
+  initialize_sync_receiver, 
+  shutdown_sync_receiver
+} from './lib/sync_receiver.js';
 import { handle_database_operation, handle_admin_operation, handle_ping_operation } from './lib/operation_dispatcher.js';
 import { start_http_server, stop_http_server } from './lib/http_server.js';
 import { 
@@ -326,29 +326,30 @@ const initialize_server_components = async (settings) => {
 };
 
 /**
- * Initializes replication manager with error handling.
+ * Initializes simple sync manager with error handling.
  * @param {Function} log - Logger function
  */
-const initialize_replication_with_logging = (log) => {
+const initialize_sync_manager_with_logging = (log) => {
   try {
-    initialize_replication_manager();
-    log.info('Replication manager initialized');
-  } catch (replication_error) {
-    log.warn('Failed to initialize replication manager', { error: replication_error.message });
+    initialize_simple_sync_manager();
+    log.info('Simple sync manager initialized');
+  } catch (sync_error) {
+    log.warn('Failed to initialize simple sync manager', { error: sync_error.message });
   }
 };
 
 /**
- * Initializes write forwarder with error handling.
+ * Initializes sync receiver with error handling.
  * @param {Function} log - Logger function
  */
-const initialize_write_forwarder_with_logging = (log) => {
-  try {
-    initialize_write_forwarder();
-    log.info('Write forwarder initialized');
-  } catch (forwarder_error) {
-    log.warn('Failed to initialize write forwarder', { error: forwarder_error.message });
-  }
+const initialize_sync_receiver_with_logging = (log) => {
+  initialize_sync_receiver()
+    .then(() => {
+      log.info('Sync receiver initialized');
+    })
+    .catch((receiver_error) => {
+      log.warn('Failed to initialize sync receiver', { error: receiver_error.message });
+    });
 };
 
 /**
@@ -658,8 +659,8 @@ const create_server_cleanup_function = () => {
     try {
       await stop_http_server();
       stop_backup_schedule();
-      await shutdown_replication_manager();
-      await shutdown_write_forwarder();
+      await shutdown_simple_sync_manager();
+      await shutdown_sync_receiver();
       
       if (connection_manager) {
         connection_manager.shutdown();
@@ -699,8 +700,8 @@ export const create_server = async () => {
   await attempt_startup_restore(settings, log);
   await initialize_server_components(settings);
   
-  initialize_replication_with_logging(log);
-  initialize_write_forwarder_with_logging(log);
+  initialize_sync_manager_with_logging(log);
+  initialize_sync_receiver_with_logging(log);
   start_backup_scheduling(settings, log);
   
   connection_manager = create_server_connection_manager();

package/src/server/lib/auto_index_manager.js

@@ -140,10 +140,15 @@ const extract_query_fields = (filter) => {
     return fields;
   }
   
-  for (const [field, value] of Object.entries(filter)) {
-    if (should_monitor_field(field)) {
-      fields.push(field);
+  try {
+    for (const [field, value] of Object.entries(filter)) {
+      if (should_monitor_field(field)) {
+        fields.push(field);
+      }
     }
+  } catch (error) {
+    // Return empty array if filter processing fails
+    return [];
   }
   
   return fields;

package/src/server/lib/bulk_insert_optimizer.js

@@ -306,6 +306,45 @@ const rebuild_collection_indexes = async (database_name, collection_name) => {
   });
 };
 
+/**
+ * Fast path for small document sets without optimization overhead.
+ * @param {string} database_name - Database name
+ * @param {string} collection_name - Collection name
+ * @param {Array<Object>} documents - Documents to insert
+ * @returns {Promise<Object>} Bulk insert results
+ */
+const bulk_insert_fast_path = async (database_name, collection_name, documents) => {
+  const db = get_database();
+  const current_timestamp = new Date().toISOString();
+  const inserted_ids = [];
+  
+  await db.transaction(() => {
+    for (const doc of documents) {
+      const document_id = doc._id || generate_sequential_id();
+      
+      const document_with_timestamps = {
+        ...doc,
+        _id: document_id,
+        _created_at: doc._created_at || current_timestamp,
+        _updated_at: doc._updated_at || current_timestamp
+      };
+      
+      const key = build_collection_key(database_name, collection_name, document_id);
+      
+      // Check if document already exists
+      const existing = db.get(key);
+      if (existing) {
+        throw new Error(`Document with _id ${document_id} already exists`);
+      }
+      
+      db.put(key, JSON.stringify(document_with_timestamps));
+      inserted_ids.push(document_id);
+    }
+  });
+  
+  return inserted_ids;
+};
+
 /**
  * Optimized bulk insert implementation with all performance optimizations.
  * @param {string} database_name - Database name
@@ -336,6 +375,46 @@ const bulk_insert_optimized = async (database_name, collection_name, documents,
     throw new Error('Documents must be a non-empty array');
   }
   
+  // For small datasets (< 5000 docs), use fast path to avoid optimization overhead
+  const use_fast_path = documents.length < 5000;
+  
+  if (use_fast_path) {
+    log.debug('Using fast path for small dataset', {
+      database: database_name,
+      collection: collection_name,
+      document_count: documents.length
+    });
+    
+    try {
+      const inserted_ids = await bulk_insert_fast_path(database_name, collection_name, documents);
+      
+      const end_time = Date.now();
+      const end_memory = process.memoryUsage();
+      
+      const performance_metrics = {
+        duration_ms: end_time - start_time,
+        documents_per_second: Math.round(documents.length / ((end_time - start_time) / 1000)),
+        memory_delta_mb: Math.round((end_memory.heapUsed - start_memory.heapUsed) / (1024 * 1024)),
+        peak_memory_mb: Math.round(end_memory.heapUsed / (1024 * 1024))
+      };
+      
+      return {
+        acknowledged: true,
+        inserted_count: inserted_ids.length,
+        inserted_ids: inserted_ids,
+        performance: performance_metrics
+      };
+    } catch (error) {
+      log.error('Fast path bulk insert failed', {
+        database: database_name,
+        collection: collection_name,
+        error: error.message
+      });
+      throw error;
+    }
+  }
+  
+  // Use optimized path for larger datasets
   log.info('Starting optimized bulk insert', {
     database: database_name,
     collection: collection_name,

package/src/server/lib/http_server.js

@@ -1084,6 +1084,13 @@ const start_http_server = (port = 1984) => {
   return new Promise((resolve, reject) => {
     // NOTE: Set up error handler before calling listen.
     server.once('error', (error) => {
+      // NOTE: In test environment, silently fail HTTP server startup to avoid port conflicts.
+      if (process.env.NODE_ENV === 'test') {
+        log.warn('Failed to start HTTP server', { error: error.message });
+        resolve(null);
+        return;
+      }
+      
       // NOTE: Clean up on startup failure.
       if (setup_required) {
         setup_token = null;

package/src/server/lib/operation_dispatcher.js

@@ -7,8 +7,8 @@
  */
 
 import { encode_message } from './tcp_protocol.js';
-import { get_write_forwarder } from './write_forwarder.js';
-import { get_replication_manager } from './replication_manager.js';
+import { get_simple_sync_manager } from './simple_sync_manager.js';
+import { get_sync_receiver } from './sync_receiver.js';
 import { check_and_grow_map_size } from './query_engine.js';
 import { performance_monitor } from './performance_monitor.js';
 import create_logger from './logger.js';
@@ -236,17 +236,17 @@ const is_write_operation = (op_type) => {
 };
 
 /**
- * Queues operation for replication if it's a write operation.
+ * Queues operation for sync if it's a write operation.
  * @param {string} op_type - Operation type
  * @param {Object} data - Operation data
  */
-const queue_replication_if_write_operation = (op_type, data) => {
+const queue_sync_if_write_operation = (op_type, data) => {
   if (!is_write_operation(op_type)) {
     return;
   }
 
-  const replication_manager = get_replication_manager();
-  replication_manager.queue_replication(op_type, data.collection, data);
+  const sync_manager = get_simple_sync_manager();
+  sync_manager.queue_sync(op_type, data.collection, data);
   
   setImmediate(() => check_and_grow_map_size());
 };
@@ -292,7 +292,7 @@ const handle_successful_operation = (socket, op_type, data, result, start_time,
     response_size
   );
   
-  queue_replication_if_write_operation(op_type, data);
+  queue_sync_if_write_operation(op_type, data);
 };
 
 /**
@@ -364,10 +364,16 @@ export const handle_database_operation = async (socket, op_type, data, check_aut
     return;
   }
   
-  const write_forwarder = get_write_forwarder();
-  const forwarded = await write_forwarder.forward_operation(socket, op_type, data);
+  const sync_receiver = get_sync_receiver();
+  const should_block = sync_receiver.should_block_client_operation(op_type);
   
-  if (forwarded) {
+  if (should_block) {
+    const response = {
+      ok: 0,
+      error: 'Write operations not allowed on secondary node. Use primary node for write operations.'
+    };
+    send_encoded_response(socket, response);
+    log_operation_performance(socket.id, op_type, data.collection, 0, 'error', 'Write operation blocked on secondary', raw_data_size, 0);
     return;
   }
   

package/src/server/lib/operations/admin.js

@@ -28,8 +28,8 @@ import {
   restore_backup,
   cleanup_old_backups
 } from '../backup_manager.js';
-import { get_replication_manager } from '../replication_manager.js';
-import { get_write_forwarder } from '../write_forwarder.js';
+import { get_simple_sync_manager } from '../simple_sync_manager.js';
+import { get_sync_receiver } from '../sync_receiver.js';
 import create_logger from '../logger.js';
 import { performance_monitor } from '../performance_monitor.js';
 
@@ -869,46 +869,79 @@ export default async (admin_action, data = {}, connection_manager, authenticated
         result = { indexes: get_indexes(data.database || 'default', data.collection) };
         break;
         
-      case 'get_replication_status':
-        const replication_manager = get_replication_manager();
-        result = replication_manager.get_replication_status();
+      case 'get_sync_status':
+        const sync_manager = get_simple_sync_manager();
+        const sync_receiver = get_sync_receiver();
+        result = {
+          sync_manager: sync_manager.get_sync_status(),
+          sync_receiver: sync_receiver.get_sync_status()
+        };
         break;
         
-      case 'add_secondary':
-        if (!data.id || !data.ip || !data.port || !data.private_key) {
-          throw new Error('id, ip, port, and private_key are required for add_secondary operation');
+      case 'update_secondary_nodes':
+        if (!Array.isArray(data.secondary_nodes)) {
+          throw new Error('secondary_nodes array is required for update_secondary_nodes operation');
         }
-        const add_replication_manager = get_replication_manager();
-        result = await add_replication_manager.add_secondary({ 
-          id: data.id, 
-          ip: data.ip, 
-          port: data.port, 
-          private_key: data.private_key, 
-          enabled: true 
-        });
+        const update_sync_manager = get_simple_sync_manager();
+        update_sync_manager.update_secondary_nodes(data.secondary_nodes);
+        result = {
+          success: true,
+          message: 'Secondary nodes updated successfully',
+          secondary_nodes: data.secondary_nodes
+        };
         break;
         
-      case 'remove_secondary':
-        if (!data.secondary_id) {
-          throw new Error('secondary_id is required for remove_secondary operation');
-        }
-        const remove_replication_manager = get_replication_manager();
-        result = remove_replication_manager.remove_secondary(data.secondary_id);
+      case 'force_sync':
+        const force_sync_manager = get_simple_sync_manager();
+        result = await force_sync_manager.force_sync();
         break;
         
-      case 'sync_secondaries':
-        const sync_replication_manager = get_replication_manager();
-        result = await sync_replication_manager.sync_secondaries();
+      case 'set_primary_role':
+        if (typeof data.primary !== 'boolean') {
+          throw new Error('primary boolean value is required for set_primary_role operation');
+        }
+        
+        if (data.primary) {
+          // Promoting to primary - stop sync receiver, start sync manager
+          const receiver = get_sync_receiver();
+          receiver.promote_to_primary();
+          result = {
+            success: true,
+            message: 'Node promoted to primary successfully',
+            role: 'primary'
+          };
+        } else {
+          // Demoting to secondary - this would require restart with new config
+          result = {
+            success: false,
+            message: 'Demoting primary to secondary requires server restart with updated configuration',
+            role: 'primary'
+          };
+        }
         break;
         
-      case 'get_secondary_health':
-        const health_replication_manager = get_replication_manager();
-        result = health_replication_manager.get_secondary_health();
+      case 'reload_sync_key':
+        const key_sync_receiver = get_sync_receiver();
+        if (!key_sync_receiver.is_secondary) {
+          throw new Error('reload_sync_key can only be used on secondary nodes');
+        }
+        await key_sync_receiver.reload_api_key();
+        result = {
+          success: true,
+          message: 'API_KEY reloaded successfully'
+        };
         break;
         
-      case 'get_forwarder_status':
-        const write_forwarder = get_write_forwarder();
-        result = write_forwarder.get_forwarder_status();
+      case 'get_secondary_auth_status':
+        const auth_sync_manager = get_simple_sync_manager();
+        const auth_status = auth_sync_manager.get_sync_status();
+        result = {
+          secondary_count: auth_status.secondary_count,
+          auth_failures: auth_status.stats.auth_failures,
+          successful_syncs: auth_status.stats.successful_syncs,
+          failed_syncs: auth_status.stats.failed_syncs,
+          secondaries: auth_status.secondaries
+        };
         break;
         
       default: