@joystick.js/db-canary 0.0.0-canary.2209

package/tests/server/integration/backup_integration.test.js

@@ -0,0 +1,513 @@
+import test from 'ava';
+import sinon from 'sinon';
+import net from 'net';
+import { encode as encode_messagepack, decode as decode_messagepack } from 'msgpackr';
+import { encode_message } from '../../../src/server/lib/tcp_protocol.js';
+import { create_server } from '../../../src/server/index.js';
+
+// Shared server instance
+let shared_server = null;
+let shared_port = null;
+
+test.before(async () => {
+  // Set test environment
+  process.env.NODE_ENV = 'test';
+  
+  // Create shared server instance
+  shared_server = await create_server();
+  shared_port = 0; // Use random available port
+  
+  // Start server
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error('Server start timeout'));
+    }, 5000);
+    
+    shared_server.listen(shared_port, (error) => {
+      clearTimeout(timeout);
+      if (error) {
+        reject(error);
+      } else {
+        shared_port = shared_server.address().port;
+        resolve();
+      }
+    });
+  });
+});
+
+test.after.always(async () => {
+  // Clean up shared server
+  if (shared_server) {
+    try {
+      if (shared_server.cleanup) {
+        await shared_server.cleanup();
+      }
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    
+    await new Promise((resolve) => {
+      const timeout = setTimeout(() => {
+        try {
+          shared_server.close();
+        } catch (error) {
+          // Ignore errors
+        }
+        resolve();
+      }, 2000);
+      
+      try {
+        shared_server.close(() => {
+          clearTimeout(timeout);
+          resolve();
+        });
+      } catch (error) {
+        clearTimeout(timeout);
+        resolve();
+      }
+    });
+    
+    shared_server = null;
+  }
+  
+  // Restore sinon mocks
+  sinon.restore();
+  
+  // Force garbage collection if available
+  if (global.gc) {
+    global.gc();
+  }
+});
+
+const create_client = () => {
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection({ port: shared_port }, () => {
+      // Set socket options for better cleanup
+      client.setKeepAlive(false);
+      client.setTimeout(5000);
+      
+      resolve(client);
+    });
+    
+    client.on('error', (error) => {
+      client.destroy();
+      reject(error);
+    });
+    
+    client.on('timeout', () => {
+      client.destroy();
+      reject(new Error('Client connection timeout'));
+    });
+  });
+};
+
+const send_message = (client, message) => {
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error('Message timeout'));
+    }, 5000);
+    
+    const encoded_message = encode_message(message);
+    
+    client.once('data', (data) => {
+      clearTimeout(timeout);
+      try {
+        // Parse the MessagePack response with length prefix
+        if (data.length < 4) {
+          reject(new Error('Response too short'));
+          return;
+        }
+        
+        const length = data.readUInt32BE(0);
+        if (data.length < 4 + length) {
+          reject(new Error('Incomplete response'));
+          return;
+        }
+        
+        const messagepack_data = data.slice(4, 4 + length);
+        const response = decode_messagepack(messagepack_data);
+        resolve(response);
+      } catch (error) {
+        reject(error);
+      }
+    });
+    
+    client.once('error', (error) => {
+      clearTimeout(timeout);
+      reject(error);
+    });
+    
+    client.write(encoded_message);
+  });
+};
+
+test('admin backup operations - should handle test_s3_connection', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // Test S3 connection
+    const s3_test_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'test_s3_connection' }
+    });
+    
+    // Should fail without proper S3 configuration
+    t.is(s3_test_response.ok, 0);
+    t.truthy(s3_test_response.error);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('admin backup operations - should handle backup_now', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // Try to create backup
+    const backup_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'backup_now' }
+    });
+    
+    // Should fail without proper S3 configuration
+    t.is(backup_response.ok, 0);
+    t.truthy(backup_response.error);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('admin backup operations - should handle list_backups', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // List backups
+    const list_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'list_backups' }
+    });
+    
+    // Should fail without proper S3 configuration
+    t.is(list_response.ok, 0);
+    t.truthy(list_response.error);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('admin backup operations - should handle restore_backup', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // Try to restore backup
+    const restore_response = await send_message(client, {
+      op: 'admin',
+      data: { 
+        admin_action: 'restore_backup',
+        backup_filename: 'joystickdb-backup-2025-08-31T12-00-00.tar.gz'
+      }
+    });
+    
+    // Should fail without proper S3 configuration
+    t.is(restore_response.ok, 0);
+    t.truthy(restore_response.error);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('admin backup operations - should require backup_filename for restore', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // Try to restore without backup_filename
+    const restore_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'restore_backup' }
+    });
+    
+    // Should fail due to missing backup_filename
+    t.is(restore_response.ok, 0);
+    t.truthy(restore_response.error);
+    t.regex(restore_response.error, /backup_filename is required/);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('admin backup operations - should handle cleanup_backups', async t => {
+  const client = await create_client();
+  
+  try {
+    // First authenticate
+    const auth_response = await send_message(client, {
+      op: 'authentication',
+      data: { password: 'test-password' }
+    });
+    
+    // Skip if authentication fails (expected in test environment)
+    if (auth_response.ok !== 1) {
+      t.pass('Authentication failed as expected in test environment');
+      return;
+    }
+    
+    // Cleanup backups
+    const cleanup_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'cleanup_backups' }
+    });
+    
+    // Should fail without proper S3 configuration
+    t.is(cleanup_response.ok, 0);
+    t.truthy(cleanup_response.error);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('backup operations - should require authentication', async t => {
+  const client = await create_client();
+  
+  try {
+    // Try backup operation without authentication
+    const backup_response = await send_message(client, {
+      op: 'admin',
+      data: { admin_action: 'backup_now' }
+    });
+    
+    // Should fail due to lack of authentication
+    t.is(backup_response.ok, false);
+    t.truthy(backup_response.error);
+    
+    // Handle both string and object error formats
+    const error_message = typeof backup_response.error === 'string' 
+      ? backup_response.error 
+      : backup_response.error.message || JSON.stringify(backup_response.error);
+    
+    t.regex(error_message, /Authentication required|Invalid message format/);
+    
+  } finally {
+    try {
+      client.end();
+      await new Promise(resolve => setTimeout(resolve, 100));
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+    try {
+      client.destroy();
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test('backup filename validation - should accept valid backup filenames', t => {
+  const valid_filenames = [
+    'joystickdb-backup-2025-08-31T12-00-00.tar.gz',
+    'joystickdb-backup-2025-12-31T23-59-59-999Z.tar.gz',
+    'joystickdb-backup-2025-01-01T00-00-00-000Z.tar.gz'
+  ];
+  
+  const filename_pattern = /^joystickdb-backup-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}.*\.tar\.gz$/;
+  
+  for (const filename of valid_filenames) {
+    t.regex(filename, filename_pattern, `${filename} should match pattern`);
+  }
+});
+
+test('backup filename validation - should reject invalid backup filenames', t => {
+  const invalid_filenames = [
+    'invalid-backup.tar.gz',
+    'joystickdb-backup-invalid-date.tar.gz',
+    'joystickdb-backup-2025-08-31.zip',
+    'backup-2025-08-31T12-00-00.tar.gz'
+  ];
+  
+  const filename_pattern = /^joystickdb-backup-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}.*\.tar\.gz$/;
+  
+  for (const filename of invalid_filenames) {
+    t.notRegex(filename, filename_pattern, `${filename} should not match pattern`);
+  }
+});
+
+test('S3 configuration validation - should validate required fields', t => {
+  const valid_s3_config = {
+    bucket: 'test-bucket',
+    region: 'us-east-1',
+    access_key: 'AKIA...',
+    secret_key: 'secret...'
+  };
+  
+  // Test that all required fields are present
+  t.truthy(valid_s3_config.bucket);
+  t.truthy(valid_s3_config.region);
+  t.truthy(valid_s3_config.access_key);
+  t.truthy(valid_s3_config.secret_key);
+});
+
+test('backup schedule validation - should accept valid schedules', t => {
+  const valid_schedules = ['hourly', 'daily', 'weekly'];
+  
+  for (const schedule of valid_schedules) {
+    t.true(valid_schedules.includes(schedule));
+  }
+});
+
+test('backup schedule validation - should handle invalid schedules', t => {
+  const invalid_schedules = ['minutely', 'monthly', 'yearly', 'invalid'];
+  const valid_schedules = ['hourly', 'daily', 'weekly'];
+  
+  for (const schedule of invalid_schedules) {
+    t.false(valid_schedules.includes(schedule));
+  }
+});
+
+test('backup retention calculation - should calculate correct retention periods', t => {
+  const now = new Date('2025-08-31T12:00:00Z');
+  const one_hour = 60 * 60 * 1000;
+  const one_day = 24 * one_hour;
+  
+  // Test different backup ages
+  const backup_ages = [
+    { age_hours: 1, should_keep_hourly: true, should_keep_daily: false },
+    { age_hours: 25, should_keep_hourly: false, should_keep_daily: true },
+    { age_hours: 31 * 24, should_keep_hourly: false, should_keep_daily: false }
+  ];
+  
+  for (const { age_hours, should_keep_hourly, should_keep_daily } of backup_ages) {
+    const backup_date = new Date(now.getTime() - age_hours * one_hour);
+    const age_ms = now - backup_date;
+    
+    const keep_hourly = age_ms <= 24 * one_hour;
+    const keep_daily = age_ms > 24 * one_hour && age_ms <= 30 * one_day;
+    
+    t.is(keep_hourly, should_keep_hourly, `${age_hours}h backup hourly retention`);
+    t.is(keep_daily, should_keep_daily, `${age_hours}h backup daily retention`);
+  }
+});

package/tests/server/integration/indexing_integration.test.js

@@ -0,0 +1,126 @@
+import test from 'ava';
+import { initialize_database, get_database, cleanup_database } from '../../../src/server/lib/query_engine.js';
+import { initialize_index_database, get_index_database, cleanup_index_database } from '../../../src/server/lib/index_manager.js';
+import insert_one from '../../../src/server/lib/operations/insert_one.js';
+import find from '../../../src/server/lib/operations/find.js';
+import find_one from '../../../src/server/lib/operations/find_one.js';
+import create_index_operation from '../../../src/server/lib/operations/create_index.js';
+import get_indexes_operation from '../../../src/server/lib/operations/get_indexes.js';
+import drop_index_operation from '../../../src/server/lib/operations/drop_index.js';
+
+test.beforeEach(async () => {
+  // Ensure clean state
+  await cleanup_database();
+  
+  // Initialize fresh databases
+  initialize_database('./test_data');
+  initialize_index_database();
+  
+  const main_db = get_database();
+  const index_db = get_index_database();
+  
+  // Clear both databases
+  main_db.clearSync();
+  index_db.clearSync();
+});
+
+test.afterEach(async () => {
+  // cleanup_index_database is called automatically by cleanup_database
+  await cleanup_database();
+});
+
+test('integration - indexing improves query performance', async (t) => {
+  await create_index_operation('default', 'users', 'email');
+  
+  await insert_one('default', 'users', { email: 'john@example.com', name: 'John Doe' });
+  await insert_one('default', 'users', { email: 'jane@example.com', name: 'Jane Smith' });
+  await insert_one('default', 'users', { email: 'bob@example.com', name: 'Bob Johnson' });
+  
+  const result = await find_one('default', 'users', { email: 'jane@example.com' });
+  
+  t.truthy(result);
+  t.is(result.email, 'jane@example.com');
+  t.is(result.name, 'Jane Smith');
+});
+
+test('integration - index management operations work correctly', async (t) => {
+  await create_index_operation('default', 'users', 'email');
+  await create_index_operation('default', 'users', 'status');
+  
+  let indexes_result = await get_indexes_operation('default', 'users');
+  t.is(indexes_result.indexes.length, 2);
+  
+  await drop_index_operation('default', 'users', 'email');
+  
+  indexes_result = await get_indexes_operation('default', 'users');
+  t.is(indexes_result.indexes.length, 1);
+  t.is(indexes_result.indexes[0].field, 'status');
+});
+
+test('integration - unique index prevents duplicate values', async (t) => {
+  await create_index_operation('default', 'users', 'email', { unique: true });
+  
+  await insert_one('default', 'users', { email: 'unique@example.com', name: 'User 1' });
+  
+  await t.throwsAsync(
+    () => insert_one('default', 'users', { email: 'unique@example.com', name: 'User 2' }),
+    { message: /Duplicate value for unique index/ }
+  );
+});
+
+test('integration - sparse index handles null values correctly', async (t) => {
+  await create_index_operation('default', 'users', 'email', { sparse: true });
+  
+  await insert_one('default', 'users', { name: 'User Without Email' });
+  await insert_one('default', 'users', { email: 'user@example.com', name: 'User With Email' });
+  
+  const users_with_email = await find('default', 'users', { email: { $exists: true } });
+  t.is(users_with_email.length, 1);
+  t.is(users_with_email[0].email, 'user@example.com');
+});
+
+test('integration - nested field indexing works', async (t) => {
+  await create_index_operation('default', 'users', 'profile.email');
+  
+  await insert_one('default', 'users', {
+    name: 'John Doe',
+    profile: {
+      email: 'john@example.com',
+      age: 30
+    }
+  });
+  
+  const result = await find_one('default', 'users', { 'profile.email': 'john@example.com' });
+  
+  t.truthy(result);
+  t.is(result.name, 'John Doe');
+  t.is(result.profile.email, 'john@example.com');
+});
+
+test('integration - $in operator uses index', async (t) => {
+  await create_index_operation('default', 'users', 'status');
+  
+  await insert_one('default', 'users', { name: 'User 1', status: 'active' });
+  await insert_one('default', 'users', { name: 'User 2', status: 'inactive' });
+  await insert_one('default', 'users', { name: 'User 3', status: 'pending' });
+  await insert_one('default', 'users', { name: 'User 4', status: 'suspended' });
+  
+  const results = await find('default', 'users', { status: { $in: ['active', 'pending'] } });
+  
+  t.is(results.length, 2);
+  const statuses = results.map(user => user.status).sort();
+  t.deepEqual(statuses, ['active', 'pending']);
+});
+
+test('integration - fallback to collection scan when no index available', async (t) => {
+  await create_index_operation('default', 'users', 'email');
+  
+  await insert_one('default', 'users', { email: 'user1@example.com', name: 'User 1', age: 25 });
+  await insert_one('default', 'users', { email: 'user2@example.com', name: 'User 2', age: 30 });
+  
+  const results = await find('default', 'users', { age: 30 });
+  
+  t.is(results.length, 1);
+  t.is(results[0].name, 'User 2');
+  t.is(results[0].age, 30);
+});