@joystick.js/db-canary 0.0.0-canary.2275 → 0.0.0-canary.2277

package/dist/server/lib/sync_receiver.js

@@ -0,0 +1 @@
+import y from"net";import h from"fs/promises";import{get_settings as d}from"./load_settings.js";import{create_message_parser as f,encode_message as m}from"./tcp_protocol.js";import g from"./logger.js";import u from"./operations/insert_one.js";import v from"./operations/update_one.js";import w from"./operations/delete_one.js";import k from"./operations/delete_many.js";import S from"./operations/bulk_write.js";import b from"./operations/create_index.js";import A from"./operations/drop_index.js";const{create_context_logger:I}=g("sync_receiver");class x{constructor(){this.is_secondary=!1,this.api_key=null,this.api_key_file_path=null,this.server=null,this.sync_port=1985,this.log=I(),this.stats={total_received:0,successful_syncs:0,failed_syncs:0,auth_failures:0,operations_applied:0}}async initialize(){try{const e=d();if(e.primary===!0){this.log.info("Node configured as primary - sync receiver disabled");return}if(e.primary===!1){if(this.is_secondary=!0,this.api_key_file_path=e.secondary_sync_key,this.sync_port=e.sync_port||1985,!this.api_key_file_path){this.log.error("Secondary node missing secondary_sync_key configuration");return}if(await this.load_api_key(),!this.api_key){this.log.error("Failed to load API_KEY - sync receiver disabled");return}this.log.info("Initializing sync receiver for secondary node",{api_key_file:this.api_key_file_path,sync_port:this.sync_port}),this.start_server()}}catch(e){this.log.warn("Could not initialize sync receiver - settings not loaded",{error:e.message})}}async load_api_key(){try{const e=await h.readFile(this.api_key_file_path,"utf8");this.api_key=e.trim(),this.log.info("API_KEY loaded successfully",{file_path:this.api_key_file_path,key_length:this.api_key.length})}catch(e){this.log.error("Failed to load API_KEY from file",{file_path:this.api_key_file_path,error:e.message})}}start_server(){try{this.server=y.createServer(e=>{this.log.debug("Sync connection established",{remote_address:e.remoteAddress,remote_port:e.remotePort});const s=f();e.on("data",r=>{try{const t=s.parse_messages(r);for(const o of t)this.handle_sync_message(e,o).catch(i=>{this.log.error("Failed to handle sync message",{error:i.message,remote_address:e.remoteAddress})})}catch(t){this.log.error("Failed to parse sync message",{error:t.message,remote_address:e.remoteAddress})}}),e.on("error",r=>{this.log.error("Sync connection error",{error:r.message,remote_address:e.remoteAddress})}),e.on("close",()=>{this.log.debug("Sync connection closed",{remote_address:e.remoteAddress})})}),this.server.listen(this.sync_port,()=>{this.log.info("Sync receiver server started",{port:this.sync_port})}),this.server.on("error",e=>{this.log.error("Sync receiver server error",{error:e.message,port:this.sync_port})})}catch(e){this.log.error("Failed to start sync receiver server",{error:e.message,port:this.sync_port})}}async handle_sync_message(e,s){this.stats.total_received++;let r;try{r=typeof s=="string"?JSON.parse(s):s}catch{this.send_sync_response(e,null,"error","Invalid JSON message");return}const{type:t,api_key:o,sequence:i,operation:a,collection:_,data:p}=r;if(t!=="operation_sync"){this.send_sync_response(e,i,"error","Invalid message type");return}if(!this.validate_api_key(o)){this.stats.auth_failures++,this.log.error("Sync authentication failed",{sequence:i,operation:a,remote_address:e.remoteAddress}),this.send_sync_response(e,i,"auth_failed","Invalid API_KEY");return}try{await this.apply_sync_operation(a,_,p),this.stats.successful_syncs++,this.stats.operations_applied++,this.log.debug("Sync operation applied successfully",{sequence:i,operation:a,collection:_}),this.send_sync_response(e,i,"success",null)}catch(c){this.stats.failed_syncs++,this.log.error("Failed to apply sync operation",{sequence:i,operation:a,collection:_,error:c.message}),this.send_sync_response(e,i,"error",c.message)}}validate_api_key(e){return!e||!this.api_key?!1:e===this.api_key}async apply_sync_operation(e,s,r){const t=r.database||"default";switch(e){case"insert_one":return await u(t,s,r.document,r.options);case"update_one":return await v(t,s,r.filter,r.update,r.options);case"delete_one":return await w(t,s,r.filter,r.options);case"delete_many":return await k(t,s,r.filter,r.options);case"bulk_write":return await S(t,s,r.operations,r.options);case"create_index":return await b(t,s,r.field,r.options);case"drop_index":return await A(t,s,r.field);default:throw new Error(`Unsupported sync operation: ${e}`)}}send_sync_response(e,s,r,t){const o={type:"sync_acknowledged",sequence:s,status:r,timestamp:Date.now()};t&&(o.error=t);try{const i=m(o);e.write(i)}catch(i){this.log.error("Failed to send sync response",{sequence:s,status:r,error:i.message})}}should_block_client_operation(e){return!(!this.is_secondary||["find","find_one","count_documents","get_indexes"].includes(e))}get_sync_status(){return{is_secondary:this.is_secondary,api_key_loaded:!!this.api_key,api_key_file:this.api_key_file_path,server_running:!!this.server&&this.server.listening,sync_port:this.sync_port,stats:this.stats}}async reload_api_key(){if(!this.api_key_file_path)throw new Error("No API_KEY file path configured");const e=this.api_key?this.api_key.length:0;await this.load_api_key(),this.log.info("API_KEY reloaded",{old_key_length:e,new_key_length:this.api_key?this.api_key.length:0})}promote_to_primary(){if(!this.is_secondary)throw new Error("Node is not configured as secondary");this.log.info("Promoting secondary to primary"),this.server&&(this.server.close(()=>{this.log.info("Sync receiver server stopped for primary promotion")}),this.server=null),this.is_secondary=!1,this.log.info("Node promoted to primary - sync receiver disabled")}async shutdown(){if(this.log.info("Shutting down sync receiver"),this.server)return new Promise(e=>{this.server.close(()=>{this.log.info("Sync receiver server closed"),e()})});this.log.info("Sync receiver shutdown complete")}}let n=null;const E=()=>(n||(n=new x),n),$=async()=>{await E().initialize()},j=async()=>{n&&(await n.shutdown(),n=null)};export{E as get_sync_receiver,$ as initialize_sync_receiver,j as shutdown_sync_receiver};

package/full_debug_test_runner.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview Full debug test runner that mimics the original test runner
+ * but with enhanced exception tracking to identify problematic tests.
+ */
+
+import { spawn } from 'child_process';
+
+// Track uncaught exceptions with detailed context
+const uncaught_exceptions = [];
+let current_test_phase = 'startup';
+let test_start_time = null;
+
+// Enhanced exception handlers that log but don't exit
+process.on('uncaughtException', (error) => {
+  const exception_info = {
+    type: 'uncaughtException',
+    phase: current_test_phase,
+    message: error.message,
+    stack: error.stack,
+    timestamp: new Date().toISOString(),
+    elapsed_ms: test_start_time ? Date.now() - test_start_time : 0
+  };
+  
+  uncaught_exceptions.push(exception_info);
+  
+  console.error(`\n🔥 UNCAUGHT EXCEPTION #${uncaught_exceptions.length}:`);
+  console.error(`📍 Phase: ${current_test_phase}`);
+  console.error(`⏰ Elapsed: ${exception_info.elapsed_ms}ms`);
+  console.error(`💥 Error: ${error.message}`);
+  console.error(`📚 Stack (first 5 lines):`);
+  const stack_lines = error.stack.split('\n').slice(0, 5);
+  stack_lines.forEach(line => console.error(`    ${line}`));
+  console.error(`⏰ Time: ${exception_info.timestamp}\n`);
+});
+
+process.on('unhandledRejection', (reason, promise) => {
+  const exception_info = {
+    type: 'unhandledRejection',
+    phase: current_test_phase,
+    reason: reason?.toString() || 'Unknown reason',
+    stack: reason?.stack || 'No stack available',
+    timestamp: new Date().toISOString(),
+    elapsed_ms: test_start_time ? Date.now() - test_start_time : 0
+  };
+  
+  uncaught_exceptions.push(exception_info);
+  
+  console.error(`\n🔥 UNHANDLED REJECTION #${uncaught_exceptions.length}:`);
+  console.error(`📍 Phase: ${current_test_phase}`);
+  console.error(`⏰ Elapsed: ${exception_info.elapsed_ms}ms`);
+  console.error(`💥 Reason: ${reason}`);
+  console.error(`📚 Stack (first 5 lines):`);
+  const stack_lines = (reason?.stack || 'No stack available').split('\n').slice(0, 5);
+  stack_lines.forEach(line => console.error(`    ${line}`));
+  console.error(`⏰ Time: ${exception_info.timestamp}\n`);
+});
+
+/**
+ * Runs the full test suite exactly like the original test runner.
+ * @returns {Promise<number>} Exit code
+ */
+const run_full_test_suite_debug = () => {
+  return new Promise((resolve) => {
+    current_test_phase = 'full-test-suite';
+    test_start_time = Date.now();
+    
+    console.log(`🔍 Running FULL TEST SUITE with debug tracking`);
+    console.log(`📝 This mimics the exact command: npm test`);
+    console.log(`💻 Command: ./node_modules/.bin/ava --serial --verbose tests/client/**/*.test.js tests/server/**/*.test.js`);
+    console.log(`🔧 NODE_OPTIONS: --expose-gc --max-old-space-size=4096`);
+    console.log(`⏰ Started at: ${new Date().toISOString()}\n`);
+    
+    const command = './node_modules/.bin/ava';
+    const args = ['--serial', '--verbose', 'tests/client/**/*.test.js', 'tests/server/**/*.test.js'];
+    
+    const child = spawn(command, args, {
+      stdio: 'pipe', // Capture output so we can track progress
+      env: {
+        ...process.env,
+        NODE_ENV: 'test',
+        NODE_OPTIONS: '--expose-gc --max-old-space-size=4096'
+      }
+    });
+    
+    let output_buffer = '';
+    let test_count = 0;
+    
+    child.stdout.on('data', (data) => {
+      const text = data.toString();
+      output_buffer += text;
+      process.stdout.write(text);
+      
+      // Track test progress
+      const test_matches = text.match(/✔/g);
+      if (test_matches) {
+        test_count += test_matches.length;
+        current_test_phase = `test-${test_count}`;
+      }
+    });
+    
+    child.stderr.on('data', (data) => {
+      const text = data.toString();
+      output_buffer += text;
+      process.stderr.write(text);
+    });
+    
+    child.on('close', (code) => {
+      const elapsed = Date.now() - test_start_time;
+      
+      console.log(`\n✅ Full test suite completed with exit code: ${code}`);
+      console.log(`⏰ Total elapsed: ${elapsed}ms`);
+      console.log(`📊 Total tests detected: ${test_count}`);
+      
+      // Exception analysis
+      console.log(`\n📊 UNCAUGHT EXCEPTION ANALYSIS:`);
+      console.log(`Total exceptions detected: ${uncaught_exceptions.length}`);
+      
+      if (uncaught_exceptions.length > 0) {
+        console.log('\n🔥 Exception Timeline:');
+        
+        uncaught_exceptions.forEach((exc, index) => {
+          console.log(`\nException #${index + 1}:`);
+          console.log(`  Type: ${exc.type}`);
+          console.log(`  Phase: ${exc.phase}`);
+          console.log(`  Elapsed: ${exc.elapsed_ms}ms`);
+          console.log(`  Message: ${exc.message}`);
+          console.log(`  Time: ${exc.timestamp}`);
+          
+          if (exc.stack) {
+            console.log(`  Key Stack Lines:`);
+            const stack_lines = exc.stack.split('\n')
+              .filter(line => line.includes('db/src/') || line.includes('db/tests/'))
+              .slice(0, 3);
+            stack_lines.forEach(line => console.log(`    ${line.trim()}`));
+          }
+        });
+        
+        // Try to correlate with test timing
+        console.log('\n🕒 Exception Timing Analysis:');
+        uncaught_exceptions.forEach((exc, index) => {
+          const test_number_estimate = Math.floor((exc.elapsed_ms / elapsed) * test_count);
+          console.log(`  Exception #${index + 1} occurred around test #${test_number_estimate} (${exc.elapsed_ms}ms elapsed)`);
+        });
+      }
+      
+      resolve(code);
+    });
+    
+    child.on('error', (error) => {
+      console.error(`\n❌ Full test suite failed: ${error.message}`);
+      resolve(1);
+    });
+  });
+};
+
+/**
+ * Main execution function.
+ */
+const main = async () => {
+  const args = process.argv.slice(2);
+  
+  if (args.includes('--help') || args.includes('-h')) {
+    console.log(`
+🔍 Full Debug Test Runner for JoystickDB
+
+Usage: node full_debug_test_runner.js
+
+This runner executes the complete test suite exactly like 'npm test'
+but captures uncaught exceptions and unhandled rejections with detailed
+timing and context information to identify problematic tests.
+`);
+    process.exit(0);
+  }
+  
+  console.log('🎯 Selected strategy: Full Test Suite Debug Analysis');
+  
+  const exit_code = await run_full_test_suite_debug();
+  
+  if (exit_code === 0 && uncaught_exceptions.length === 0) {
+    console.log(`\n🎉 All tests passed with NO uncaught exceptions!`);
+  } else if (exit_code === 0 && uncaught_exceptions.length > 0) {
+    console.log(`\n⚠️  All tests passed but ${uncaught_exceptions.length} uncaught exceptions detected`);
+  } else {
+    console.log(`\n💥 Tests failed with exit code: ${exit_code}`);
+  }
+  
+  process.exit(exit_code);
+};
+
+// Run the main function
+main().catch(error => {
+  console.error(`\n💥 Debug runner error: ${error.message}`);
+  console.error(error.stack);
+  process.exit(1);
+});

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@joystick.js/db-canary",
   "type": "module",
-  "version": "0.0.0-canary.2275",
-  "canary_version": "0.0.0-canary.2274",
+  "version": "0.0.0-canary.2277",
+  "canary_version": "0.0.0-canary.2276",
   "description": "JoystickDB - A minimalist database server for the Joystick framework",
   "main": "./dist/server/index.js",
   "scripts": {

package/src/server/index.js

@@ -42,13 +42,13 @@ import {
   stop_backup_schedule 
 } from './lib/backup_manager.js';
 import { 
-  initialize_replication_manager, 
-  shutdown_replication_manager
-} from './lib/replication_manager.js';
+  initialize_simple_sync_manager, 
+  shutdown_simple_sync_manager
+} from './lib/simple_sync_manager.js';
 import { 
-  initialize_write_forwarder, 
-  shutdown_write_forwarder
-} from './lib/write_forwarder.js';
+  initialize_sync_receiver, 
+  shutdown_sync_receiver
+} from './lib/sync_receiver.js';
 import { handle_database_operation, handle_admin_operation, handle_ping_operation } from './lib/operation_dispatcher.js';
 import { start_http_server, stop_http_server } from './lib/http_server.js';
 import { 
@@ -326,29 +326,30 @@ const initialize_server_components = async (settings) => {
 };
 
 /**
- * Initializes replication manager with error handling.
+ * Initializes simple sync manager with error handling.
  * @param {Function} log - Logger function
  */
-const initialize_replication_with_logging = (log) => {
+const initialize_sync_manager_with_logging = (log) => {
   try {
-    initialize_replication_manager();
-    log.info('Replication manager initialized');
-  } catch (replication_error) {
-    log.warn('Failed to initialize replication manager', { error: replication_error.message });
+    initialize_simple_sync_manager();
+    log.info('Simple sync manager initialized');
+  } catch (sync_error) {
+    log.warn('Failed to initialize simple sync manager', { error: sync_error.message });
   }
 };
 
 /**
- * Initializes write forwarder with error handling.
+ * Initializes sync receiver with error handling.
  * @param {Function} log - Logger function
  */
-const initialize_write_forwarder_with_logging = (log) => {
-  try {
-    initialize_write_forwarder();
-    log.info('Write forwarder initialized');
-  } catch (forwarder_error) {
-    log.warn('Failed to initialize write forwarder', { error: forwarder_error.message });
-  }
+const initialize_sync_receiver_with_logging = (log) => {
+  initialize_sync_receiver()
+    .then(() => {
+      log.info('Sync receiver initialized');
+    })
+    .catch((receiver_error) => {
+      log.warn('Failed to initialize sync receiver', { error: receiver_error.message });
+    });
 };
 
 /**
@@ -658,8 +659,8 @@ const create_server_cleanup_function = () => {
     try {
       await stop_http_server();
       stop_backup_schedule();
-      await shutdown_replication_manager();
-      await shutdown_write_forwarder();
+      await shutdown_simple_sync_manager();
+      await shutdown_sync_receiver();
       
       if (connection_manager) {
         connection_manager.shutdown();
@@ -699,8 +700,8 @@ export const create_server = async () => {
   await attempt_startup_restore(settings, log);
   await initialize_server_components(settings);
   
-  initialize_replication_with_logging(log);
-  initialize_write_forwarder_with_logging(log);
+  initialize_sync_manager_with_logging(log);
+  initialize_sync_receiver_with_logging(log);
   start_backup_scheduling(settings, log);
   
   connection_manager = create_server_connection_manager();

package/src/server/lib/operation_dispatcher.js

@@ -7,8 +7,8 @@
  */
 
 import { encode_message } from './tcp_protocol.js';
-import { get_write_forwarder } from './write_forwarder.js';
-import { get_replication_manager } from './replication_manager.js';
+import { get_simple_sync_manager } from './simple_sync_manager.js';
+import { get_sync_receiver } from './sync_receiver.js';
 import { check_and_grow_map_size } from './query_engine.js';
 import { performance_monitor } from './performance_monitor.js';
 import create_logger from './logger.js';
@@ -236,17 +236,17 @@ const is_write_operation = (op_type) => {
 };
 
 /**
- * Queues operation for replication if it's a write operation.
+ * Queues operation for sync if it's a write operation.
  * @param {string} op_type - Operation type
  * @param {Object} data - Operation data
  */
-const queue_replication_if_write_operation = (op_type, data) => {
+const queue_sync_if_write_operation = (op_type, data) => {
   if (!is_write_operation(op_type)) {
     return;
   }
 
-  const replication_manager = get_replication_manager();
-  replication_manager.queue_replication(op_type, data.collection, data);
+  const sync_manager = get_simple_sync_manager();
+  sync_manager.queue_sync(op_type, data.collection, data);
   
   setImmediate(() => check_and_grow_map_size());
 };
@@ -292,7 +292,7 @@ const handle_successful_operation = (socket, op_type, data, result, start_time,
     response_size
   );
   
-  queue_replication_if_write_operation(op_type, data);
+  queue_sync_if_write_operation(op_type, data);
 };
 
 /**
@@ -364,10 +364,16 @@ export const handle_database_operation = async (socket, op_type, data, check_aut
     return;
   }
   
-  const write_forwarder = get_write_forwarder();
-  const forwarded = await write_forwarder.forward_operation(socket, op_type, data);
+  const sync_receiver = get_sync_receiver();
+  const should_block = sync_receiver.should_block_client_operation(op_type);
   
-  if (forwarded) {
+  if (should_block) {
+    const response = {
+      ok: 0,
+      error: 'Write operations not allowed on secondary node. Use primary node for write operations.'
+    };
+    send_encoded_response(socket, response);
+    log_operation_performance(socket.id, op_type, data.collection, 0, 'error', 'Write operation blocked on secondary', raw_data_size, 0);
     return;
   }
   

package/src/server/lib/operations/admin.js

@@ -28,8 +28,8 @@ import {
   restore_backup,
   cleanup_old_backups
 } from '../backup_manager.js';
-import { get_replication_manager } from '../replication_manager.js';
-import { get_write_forwarder } from '../write_forwarder.js';
+import { get_simple_sync_manager } from '../simple_sync_manager.js';
+import { get_sync_receiver } from '../sync_receiver.js';
 import create_logger from '../logger.js';
 import { performance_monitor } from '../performance_monitor.js';
 
@@ -869,46 +869,79 @@ export default async (admin_action, data = {}, connection_manager, authenticated
         result = { indexes: get_indexes(data.database || 'default', data.collection) };
         break;
         
-      case 'get_replication_status':
-        const replication_manager = get_replication_manager();
-        result = replication_manager.get_replication_status();
+      case 'get_sync_status':
+        const sync_manager = get_simple_sync_manager();
+        const sync_receiver = get_sync_receiver();
+        result = {
+          sync_manager: sync_manager.get_sync_status(),
+          sync_receiver: sync_receiver.get_sync_status()
+        };
         break;
         
-      case 'add_secondary':
-        if (!data.id || !data.ip || !data.port || !data.private_key) {
-          throw new Error('id, ip, port, and private_key are required for add_secondary operation');
+      case 'update_secondary_nodes':
+        if (!Array.isArray(data.secondary_nodes)) {
+          throw new Error('secondary_nodes array is required for update_secondary_nodes operation');
         }
-        const add_replication_manager = get_replication_manager();
-        result = await add_replication_manager.add_secondary({ 
-          id: data.id, 
-          ip: data.ip, 
-          port: data.port, 
-          private_key: data.private_key, 
-          enabled: true 
-        });
+        const update_sync_manager = get_simple_sync_manager();
+        update_sync_manager.update_secondary_nodes(data.secondary_nodes);
+        result = {
+          success: true,
+          message: 'Secondary nodes updated successfully',
+          secondary_nodes: data.secondary_nodes
+        };
         break;
         
-      case 'remove_secondary':
-        if (!data.secondary_id) {
-          throw new Error('secondary_id is required for remove_secondary operation');
-        }
-        const remove_replication_manager = get_replication_manager();
-        result = remove_replication_manager.remove_secondary(data.secondary_id);
+      case 'force_sync':
+        const force_sync_manager = get_simple_sync_manager();
+        result = await force_sync_manager.force_sync();
         break;
         
-      case 'sync_secondaries':
-        const sync_replication_manager = get_replication_manager();
-        result = await sync_replication_manager.sync_secondaries();
+      case 'set_primary_role':
+        if (typeof data.primary !== 'boolean') {
+          throw new Error('primary boolean value is required for set_primary_role operation');
+        }
+        
+        if (data.primary) {
+          // Promoting to primary - stop sync receiver, start sync manager
+          const receiver = get_sync_receiver();
+          receiver.promote_to_primary();
+          result = {
+            success: true,
+            message: 'Node promoted to primary successfully',
+            role: 'primary'
+          };
+        } else {
+          // Demoting to secondary - this would require restart with new config
+          result = {
+            success: false,
+            message: 'Demoting primary to secondary requires server restart with updated configuration',
+            role: 'primary'
+          };
+        }
         break;
         
-      case 'get_secondary_health':
-        const health_replication_manager = get_replication_manager();
-        result = health_replication_manager.get_secondary_health();
+      case 'reload_sync_key':
+        const key_sync_receiver = get_sync_receiver();
+        if (!key_sync_receiver.is_secondary) {
+          throw new Error('reload_sync_key can only be used on secondary nodes');
+        }
+        await key_sync_receiver.reload_api_key();
+        result = {
+          success: true,
+          message: 'API_KEY reloaded successfully'
+        };
         break;
         
-      case 'get_forwarder_status':
-        const write_forwarder = get_write_forwarder();
-        result = write_forwarder.get_forwarder_status();
+      case 'get_secondary_auth_status':
+        const auth_sync_manager = get_simple_sync_manager();
+        const auth_status = auth_sync_manager.get_sync_status();
+        result = {
+          secondary_count: auth_status.secondary_count,
+          auth_failures: auth_status.stats.auth_failures,
+          successful_syncs: auth_status.stats.successful_syncs,
+          failed_syncs: auth_status.stats.failed_syncs,
+          secondaries: auth_status.secondaries
+        };
         break;
         
       default: