@joystick.js/db-canary 0.0.0-canary.2274 → 0.0.0-canary.2276

package/src/server/lib/sync_receiver.js

@@ -0,0 +1,461 @@
+/**
+ * @fileoverview Sync receiver for secondary nodes in JoystickDB.
+ * Receives and processes sync messages from primary nodes with API_KEY authentication.
+ * Secondary nodes are read-only except for authenticated sync operations.
+ */
+
+import net from 'net';
+import fs from 'fs/promises';
+import { get_settings } from './load_settings.js';
+import { create_message_parser, encode_message } from './tcp_protocol.js';
+import create_logger from './logger.js';
+import insert_one from './operations/insert_one.js';
+import update_one from './operations/update_one.js';
+import delete_one from './operations/delete_one.js';
+import delete_many from './operations/delete_many.js';
+import bulk_write from './operations/bulk_write.js';
+import create_index_operation from './operations/create_index.js';
+import drop_index_operation from './operations/drop_index.js';
+
+const { create_context_logger } = create_logger('sync_receiver');
+
+/**
+ * Sync receiver that processes authenticated sync messages from primary nodes.
+ * Validates API_KEY and applies operations to local database.
+ */
+class SyncReceiver {
+  /**
+   * Creates a new SyncReceiver instance.
+   */
+  constructor() {
+    /** @type {boolean} Whether this node is configured as secondary */
+    this.is_secondary = false;
+    
+    /** @type {string|null} API_KEY loaded from file for authentication */
+    this.api_key = null;
+    
+    /** @type {string|null} Path to API_KEY file */
+    this.api_key_file_path = null;
+    
+    /** @type {net.Server|null} TCP server for receiving sync messages */
+    this.server = null;
+    
+    /** @type {number} TCP port for sync operations */
+    this.sync_port = 1985;
+    
+    /** @type {Object} Logger instance */
+    this.log = create_context_logger();
+    
+    /** @type {Object} Sync statistics */
+    this.stats = {
+      total_received: 0,
+      successful_syncs: 0,
+      failed_syncs: 0,
+      auth_failures: 0,
+      operations_applied: 0
+    };
+  }
+
+  /**
+   * Initializes the sync receiver with settings configuration.
+   */
+  async initialize() {
+    try {
+      const settings = get_settings();
+      
+      if (settings.primary === true) {
+        this.log.info('Node configured as primary - sync receiver disabled');
+        return;
+      }
+      
+      if (settings.primary === false) {
+        this.is_secondary = true;
+        this.api_key_file_path = settings.secondary_sync_key;
+        this.sync_port = settings.sync_port || 1985;
+        
+        if (!this.api_key_file_path) {
+          this.log.error('Secondary node missing secondary_sync_key configuration');
+          return;
+        }
+        
+        await this.load_api_key();
+        
+        if (!this.api_key) {
+          this.log.error('Failed to load API_KEY - sync receiver disabled');
+          return;
+        }
+        
+        this.log.info('Initializing sync receiver for secondary node', {
+          api_key_file: this.api_key_file_path,
+          sync_port: this.sync_port
+        });
+        
+        this.start_server();
+      }
+      
+    } catch (error) {
+      this.log.warn('Could not initialize sync receiver - settings not loaded', {
+        error: error.message
+      });
+    }
+  }
+
+  /**
+   * Loads API_KEY from configured file path.
+   */
+  async load_api_key() {
+    try {
+      const key_content = await fs.readFile(this.api_key_file_path, 'utf8');
+      this.api_key = key_content.trim();
+      
+      this.log.info('API_KEY loaded successfully', {
+        file_path: this.api_key_file_path,
+        key_length: this.api_key.length
+      });
+      
+    } catch (error) {
+      this.log.error('Failed to load API_KEY from file', {
+        file_path: this.api_key_file_path,
+        error: error.message
+      });
+    }
+  }
+
+  /**
+   * Starts the TCP server to receive sync messages.
+   */
+  start_server() {
+    try {
+      this.server = net.createServer((socket) => {
+        this.log.debug('Sync connection established', {
+          remote_address: socket.remoteAddress,
+          remote_port: socket.remotePort
+        });
+        
+        const message_parser = create_message_parser();
+        
+        socket.on('data', (data) => {
+          try {
+            const messages = message_parser.parse_messages(data);
+            for (const message of messages) {
+              this.handle_sync_message(socket, message).catch((error) => {
+                this.log.error('Failed to handle sync message', {
+                  error: error.message,
+                  remote_address: socket.remoteAddress
+                });
+              });
+            }
+          } catch (error) {
+            this.log.error('Failed to parse sync message', {
+              error: error.message,
+              remote_address: socket.remoteAddress
+            });
+          }
+        });
+        
+        socket.on('error', (error) => {
+          this.log.error('Sync connection error', {
+            error: error.message,
+            remote_address: socket.remoteAddress
+          });
+        });
+        
+        socket.on('close', () => {
+          this.log.debug('Sync connection closed', {
+            remote_address: socket.remoteAddress
+          });
+        });
+      });
+      
+      this.server.listen(this.sync_port, () => {
+        this.log.info('Sync receiver server started', {
+          port: this.sync_port
+        });
+      });
+      
+      this.server.on('error', (error) => {
+        this.log.error('Sync receiver server error', {
+          error: error.message,
+          port: this.sync_port
+        });
+      });
+    } catch (error) {
+      this.log.error('Failed to start sync receiver server', {
+        error: error.message,
+        port: this.sync_port
+      });
+    }
+  }
+
+  /**
+   * Handles incoming sync message from primary node.
+   * @param {net.Socket} socket - Connection socket
+   * @param {Object|string} raw_message - Raw sync message
+   */
+  async handle_sync_message(socket, raw_message) {
+    this.stats.total_received++;
+    
+    let message;
+    try {
+      message = typeof raw_message === 'string' ? JSON.parse(raw_message) : raw_message;
+    } catch (error) {
+      this.send_sync_response(socket, null, 'error', 'Invalid JSON message');
+      return;
+    }
+    
+    const { type, api_key, sequence, operation, collection, data } = message;
+    
+    if (type !== 'operation_sync') {
+      this.send_sync_response(socket, sequence, 'error', 'Invalid message type');
+      return;
+    }
+    
+    // Validate API_KEY
+    if (!this.validate_api_key(api_key)) {
+      this.stats.auth_failures++;
+      this.log.error('Sync authentication failed', {
+        sequence,
+        operation,
+        remote_address: socket.remoteAddress
+      });
+      this.send_sync_response(socket, sequence, 'auth_failed', 'Invalid API_KEY');
+      return;
+    }
+    
+    // Apply operation to local database
+    try {
+      await this.apply_sync_operation(operation, collection, data);
+      this.stats.successful_syncs++;
+      this.stats.operations_applied++;
+      
+      this.log.debug('Sync operation applied successfully', {
+        sequence,
+        operation,
+        collection
+      });
+      
+      this.send_sync_response(socket, sequence, 'success', null);
+      
+    } catch (error) {
+      this.stats.failed_syncs++;
+      
+      this.log.error('Failed to apply sync operation', {
+        sequence,
+        operation,
+        collection,
+        error: error.message
+      });
+      
+      this.send_sync_response(socket, sequence, 'error', error.message);
+    }
+  }
+
+  /**
+   * Validates API_KEY from sync message.
+   * @param {string} provided_key - API_KEY from sync message
+   * @returns {boolean} True if key is valid
+   */
+  validate_api_key(provided_key) {
+    if (!provided_key || !this.api_key) {
+      return false;
+    }
+    
+    return provided_key === this.api_key;
+  }
+
+  /**
+   * Applies sync operation to local database.
+   * @param {string} operation - Operation type
+   * @param {string} collection - Collection name
+   * @param {Object} data - Operation data
+   */
+  async apply_sync_operation(operation, collection, data) {
+    const database_name = data.database || 'default';
+    
+    switch (operation) {
+      case 'insert_one':
+        return await insert_one(database_name, collection, data.document, data.options);
+        
+      case 'update_one':
+        return await update_one(database_name, collection, data.filter, data.update, data.options);
+        
+      case 'delete_one':
+        return await delete_one(database_name, collection, data.filter, data.options);
+        
+      case 'delete_many':
+        return await delete_many(database_name, collection, data.filter, data.options);
+        
+      case 'bulk_write':
+        return await bulk_write(database_name, collection, data.operations, data.options);
+        
+      case 'create_index':
+        return await create_index_operation(database_name, collection, data.field, data.options);
+        
+      case 'drop_index':
+        return await drop_index_operation(database_name, collection, data.field);
+        
+      default:
+        throw new Error(`Unsupported sync operation: ${operation}`);
+    }
+  }
+
+  /**
+   * Sends sync response back to primary node.
+   * @param {net.Socket} socket - Connection socket
+   * @param {number|null} sequence - Sequence number
+   * @param {string} status - Response status
+   * @param {string|null} error - Error message if any
+   */
+  send_sync_response(socket, sequence, status, error) {
+    const response = {
+      type: 'sync_acknowledged',
+      sequence,
+      status,
+      timestamp: Date.now()
+    };
+    
+    if (error) {
+      response.error = error;
+    }
+    
+    try {
+      const encoded_response = encode_message(response);
+      socket.write(encoded_response);
+    } catch (write_error) {
+      this.log.error('Failed to send sync response', {
+        sequence,
+        status,
+        error: write_error.message
+      });
+    }
+  }
+
+  /**
+   * Checks if a client operation should be blocked on secondary.
+   * @param {string} operation_type - Type of operation
+   * @returns {boolean} True if operation should be blocked
+   */
+  should_block_client_operation(operation_type) {
+    if (!this.is_secondary) {
+      return false;
+    }
+    
+    // Allow read operations on secondary
+    const read_operations = ['find', 'find_one', 'count_documents', 'get_indexes'];
+    
+    if (read_operations.includes(operation_type)) {
+      return false;
+    }
+    
+    // Block all write operations for direct clients
+    return true;
+  }
+
+  /**
+   * Gets current sync receiver status and statistics.
+   * @returns {Object} Sync receiver status
+   */
+  get_sync_status() {
+    return {
+      is_secondary: this.is_secondary,
+      api_key_loaded: !!this.api_key,
+      api_key_file: this.api_key_file_path,
+      server_running: !!this.server && this.server.listening,
+      sync_port: this.sync_port,
+      stats: this.stats
+    };
+  }
+
+  /**
+   * Reloads API_KEY from file (for key rotation).
+   */
+  async reload_api_key() {
+    if (!this.api_key_file_path) {
+      throw new Error('No API_KEY file path configured');
+    }
+    
+    const old_key_length = this.api_key ? this.api_key.length : 0;
+    await this.load_api_key();
+    
+    this.log.info('API_KEY reloaded', {
+      old_key_length,
+      new_key_length: this.api_key ? this.api_key.length : 0
+    });
+  }
+
+  /**
+   * Promotes secondary to primary (manual failover support).
+   */
+  promote_to_primary() {
+    if (!this.is_secondary) {
+      throw new Error('Node is not configured as secondary');
+    }
+    
+    this.log.info('Promoting secondary to primary');
+    
+    // Stop sync receiver server
+    if (this.server) {
+      this.server.close(() => {
+        this.log.info('Sync receiver server stopped for primary promotion');
+      });
+      this.server = null;
+    }
+    
+    this.is_secondary = false;
+    
+    this.log.info('Node promoted to primary - sync receiver disabled');
+  }
+
+  /**
+   * Shuts down the sync receiver and closes server.
+   * @returns {Promise<void>}
+   */
+  async shutdown() {
+    this.log.info('Shutting down sync receiver');
+    
+    if (this.server) {
+      return new Promise((resolve) => {
+        this.server.close(() => {
+          this.log.info('Sync receiver server closed');
+          resolve();
+        });
+      });
+    }
+    
+    this.log.info('Sync receiver shutdown complete');
+  }
+}
+
+/** @type {SyncReceiver|null} Singleton instance */
+let sync_receiver_instance = null;
+
+/**
+ * Gets the sync receiver singleton instance.
+ * @returns {SyncReceiver} Sync receiver instance
+ */
+export const get_sync_receiver = () => {
+  if (!sync_receiver_instance) {
+    sync_receiver_instance = new SyncReceiver();
+  }
+  return sync_receiver_instance;
+};
+
+/**
+ * Initializes the sync receiver singleton.
+ * @returns {Promise<void>}
+ */
+export const initialize_sync_receiver = async () => {
+  const receiver = get_sync_receiver();
+  await receiver.initialize();
+};
+
+/**
+ * Shuts down the sync receiver singleton.
+ * @returns {Promise<void>}
+ */
+export const shutdown_sync_receiver = async () => {
+  if (sync_receiver_instance) {
+    await sync_receiver_instance.shutdown();
+    sync_receiver_instance = null;
+  }
+};

package/tests/client/index.test.js

@@ -18,6 +18,13 @@ test.beforeEach(async () => {
   
   // Clean up database files to ensure test isolation
   try {
+    if (existsSync('./.joystick/data/joystickdb_1983/data.mdb')) {
+      unlinkSync('./.joystick/data/joystickdb_1983/data.mdb');
+    }
+    if (existsSync('./.joystick/data/joystickdb_1983/lock.mdb')) {
+      unlinkSync('./.joystick/data/joystickdb_1983/lock.mdb');
+    }
+    // Also clean up old location for backward compatibility
     if (existsSync('./data/data.mdb')) {
       unlinkSync('./data/data.mdb');
     }

package/tests/performance/isolated_5000000_test.js

@@ -0,0 +1,184 @@
+
+import { rmSync, existsSync } from 'fs';
+import { initialize_database, cleanup_database } from '../../src/server/lib/query_engine.js';
+import { bulk_insert_with_metrics } from '../../src/server/lib/bulk_insert_optimizer.js';
+
+const TEST_DB_PATH = './test_data/isolated_5000000_test';
+const TEST_DATABASE = 'isolated_db_5000000';
+const TEST_COLLECTION = 'isolated_collection';
+
+// Generate minimal test documents
+const generate_documents = (count) => {
+  const documents = [];
+  const test_id = Date.now().toString(36);
+  
+  for (let i = 0; i < count; i++) {
+    documents.push({
+      _id: `iso_${test_id}_${i.toString().padStart(8, '0')}`,
+      idx: i,
+      cat: i % 50,
+      val: i % 1000,
+      ts: Date.now() + i
+    });
+  }
+  
+  return documents;
+};
+
+// Aggressive memory management
+const force_cleanup = async () => {
+  await cleanup_database(true);
+  
+  // Force multiple GC cycles
+  if (global.gc) {
+    for (let i = 0; i < 5; i++) {
+      global.gc();
+      await new Promise(resolve => setTimeout(resolve, 50));
+    }
+  }
+  
+  // Wait for LMDB resources to be released
+  await new Promise(resolve => setTimeout(resolve, 200));
+};
+
+// Main test execution
+const run_test = async () => {
+  try {
+    console.log('🚀 Starting 5M Document Enterprise Scale Test (5,000,000 documents)');
+    
+    // Clean setup
+    if (existsSync(TEST_DB_PATH)) {
+      rmSync(TEST_DB_PATH, { recursive: true, force: true });
+    }
+    
+    // Initial memory state
+    const initial_memory = process.memoryUsage();
+    console.log(`Initial Memory: ${Math.round(initial_memory.heapUsed / (1024 * 1024))}MB heap used`);
+    
+    initialize_database(TEST_DB_PATH);
+    
+    // Generate documents
+    console.log('Generating documents...');
+    const documents = generate_documents(5000000);
+    
+    // Run test with optimal settings for isolation
+    const start_time = Date.now();
+    const result = await bulk_insert_with_metrics(TEST_DATABASE, TEST_COLLECTION, documents, {
+      disable_indexing: true,
+      pre_allocate_map_size: true,
+      sort_keys: true,
+      stream_processing: true,
+      batch_size: 250 // Smaller batches for very large datasets
+    });
+    
+    const total_duration = Date.now() - start_time;
+    const duration_seconds = total_duration / 1000;
+    
+    // Output results in parseable format
+    console.log(`\n✅ 5M DOCUMENT ENTERPRISE SCALE TEST RESULTS:`);
+    console.log(`Duration: ${duration_seconds.toFixed(2)} seconds`);
+    console.log(`Throughput: ${result.performance.documents_per_second.toLocaleString()} docs/sec`);
+    console.log(`Memory Delta: ${result.performance.memory_usage.delta_heap_mb}MB`);
+    console.log(`Peak Memory: ${result.performance.memory_usage.peak_heap_mb}MB`);
+    console.log(`Success Rate: 100%`);
+    
+    // Validate results
+    if (!result.acknowledged) {
+      throw new Error('Insert not acknowledged');
+    }
+    if (result.inserted_count !== 5000000) {
+      throw new Error(`Expected ${5000000} inserts, got ${result.inserted_count}`);
+    }
+    
+    // Performance validation
+    const max_duration = 180;
+    const min_throughput = 25000;
+    const max_memory = 2048;
+    
+    if (duration_seconds > max_duration) {
+      throw new Error(`Duration ${duration_seconds}s exceeds ${max_duration}s limit`);
+    }
+    if (result.performance.documents_per_second < min_throughput) {
+      throw new Error(`Throughput ${result.performance.documents_per_second} below ${min_throughput} docs/sec target`);
+    }
+    if (result.performance.memory_usage.peak_heap_mb > max_memory) {
+      throw new Error(`Memory ${result.performance.memory_usage.peak_heap_mb}MB exceeds ${max_memory}MB limit`);
+    }
+    
+    console.log(`\n📈 5M DOCUMENT ENTERPRISE SCALE TEST VALIDATION:`);
+    console.log(`✅ Performance targets met`);
+    console.log(`✅ Memory usage within limits`);
+    console.log(`✅ All ${5000000} documents inserted successfully`);
+    
+    // Cleanup
+    await force_cleanup();
+    
+    const final_memory = process.memoryUsage();
+    console.log(`Final Memory: ${Math.round(final_memory.heapUsed / (1024 * 1024))}MB heap used`);
+    
+    console.log('\n🎉 Test completed successfully');
+    process.exit(0);
+    
+  } catch (error) {
+    console.error(`\n❌ Test failed: ${error.message}`);
+    console.error(error.stack);
+    
+    try {
+      await force_cleanup();
+    } catch (cleanupError) {
+      console.error('Cleanup error:', cleanupError.message);
+    }
+    
+    process.exit(1);
+  }
+};
+
+// Handle process signals
+process.on('SIGTERM', async () => {
+  console.log('Received SIGTERM, cleaning up...');
+  try {
+    await force_cleanup();
+  } catch (error) {
+    console.error('Cleanup error:', error.message);
+  }
+  process.exit(1);
+});
+
+process.on('SIGINT', async () => {
+  console.log('Received SIGINT, cleaning up...');
+  try {
+    await force_cleanup();
+  } catch (error) {
+    console.error('Cleanup error:', error.message);
+  }
+  process.exit(1);
+});
+
+// Add uncaught exception handlers
+process.on('uncaughtException', async (error) => {
+  console.error('\n💥 Uncaught Exception:', error.message);
+  console.error(error.stack);
+  
+  try {
+    await force_cleanup();
+  } catch (cleanupError) {
+    console.error('Cleanup error:', cleanupError.message);
+  }
+  
+  process.exit(1);
+});
+
+process.on('unhandledRejection', async (reason, promise) => {
+  console.error('\n💥 Unhandled Rejection at:', promise, 'reason:', reason);
+  
+  try {
+    await force_cleanup();
+  } catch (cleanupError) {
+    console.error('Cleanup error:', cleanupError.message);
+  }
+  
+  process.exit(1);
+});
+
+// Run the test
+run_test();

package/tests/server/lib/http_server.test.js

@@ -290,18 +290,9 @@ test('HTTP server handles port conflicts gracefully', async (t) => {
   // NOTE: Wait a moment to ensure the first server is fully established.
   await new Promise(resolve => setTimeout(resolve, 100));
   
-  // NOTE: Try to start second server on same port - this should fail.
-  let error_caught = false;
-  try {
-    await start_http_server(test_port);
-    t.fail('Should throw error for port conflict');
-  } catch (error) {
-    error_caught = true;
-    t.true(error.message.includes('EADDRINUSE') || error.code === 'EADDRINUSE');
-  }
-  
-  // NOTE: Ensure we caught the expected error.
-  t.true(error_caught, 'Expected EADDRINUSE error was not caught');
+  // NOTE: Try to start second server on same port - in test environment this should return null.
+  const server2 = await start_http_server(test_port);
+  t.is(server2, null, 'Second server should return null due to port conflict in test environment');
   
   // NOTE: Verify first server is still running and accessible.
   const setup_info = get_setup_info();