@joystick.js/db-canary 0.0.0-canary.2251 → 0.0.0-canary.2253

package/src/server/lib/auto_index_manager.js

@@ -334,45 +334,107 @@ const is_auto_created_index = (collection_name, field_name) => {
   }
 };
 
+/**
+ * Checks if collection has reached maximum auto-index limit.
+ * @param {string} collection - Collection name
+ * @param {Object} config - Auto-index configuration
+ * @returns {boolean} True if at maximum limit
+ */
+const has_reached_auto_index_limit = (collection, config) => {
+  const existing_indexes = get_indexes('default', collection);
+  const auto_index_count = existing_indexes.filter(index => 
+    is_auto_created_index(collection, index.field)
+  ).length;
+  
+  return auto_index_count >= config.max_auto_indexes_per_collection;
+};
+
+/**
+ * Checks if field is within monitoring window.
+ * @param {Object} stats - Field statistics
+ * @param {number} window_ms - Monitoring window in milliseconds
+ * @returns {boolean} True if within window
+ */
+const is_within_monitoring_window = (stats, window_ms) => {
+  const now = new Date();
+  const time_since_last_query = now - stats.last_queried;
+  return time_since_last_query <= window_ms;
+};
+
+/**
+ * Checks if field already has an index.
+ * @param {string} field - Field name
+ * @param {Array} existing_indexes - Array of existing indexes
+ * @returns {boolean} True if field has existing index
+ */
+const has_existing_index = (field, existing_indexes) => {
+  return existing_indexes.some(index => index.field === field);
+};
+
+/**
+ * Determines if field meets criteria for auto-indexing.
+ * @param {Object} stats - Field statistics
+ * @param {Object} config - Auto-index configuration
+ * @returns {boolean} True if field meets criteria
+ */
+const meets_auto_index_criteria = (stats, config) => {
+  const meets_frequency_threshold = stats.query_count >= config.frequency_threshold;
+  const meets_performance_threshold = stats.avg_time_ms >= config.performance_threshold_ms;
+  const has_slow_queries = stats.slow_query_count > 0;
+  
+  return meets_frequency_threshold || (meets_performance_threshold && has_slow_queries);
+};
+
+/**
+ * Calculates priority score for index candidate.
+ * @param {Object} stats - Field statistics
+ * @param {Object} config - Auto-index configuration
+ * @returns {number} Priority score
+ */
+const calculate_candidate_priority = (stats, config) => {
+  return stats.slow_query_count * 2 + (stats.query_count / config.frequency_threshold);
+};
+
+/**
+ * Creates index candidate object.
+ * @param {string} collection - Collection name
+ * @param {string} field - Field name
+ * @param {Object} stats - Field statistics
+ * @param {Object} config - Auto-index configuration
+ * @returns {Object} Index candidate
+ */
+const create_index_candidate = (collection, field, stats, config) => {
+  return {
+    collection,
+    field,
+    stats: { ...stats },
+    priority: calculate_candidate_priority(stats, config)
+  };
+};
+
 const get_auto_index_candidates = () => {
   const config = get_auto_index_config();
   const candidates = [];
-  const now = new Date();
   const window_ms = config.monitoring_window_hours * 60 * 60 * 1000;
   
   for (const [collection, fields] of query_stats.entries()) {
-    const existing_indexes = get_indexes('default', collection);
-    const auto_index_count = existing_indexes.filter(index => 
-      is_auto_created_index(collection, index.field)
-    ).length;
-    
-    if (auto_index_count >= config.max_auto_indexes_per_collection) {
+    if (has_reached_auto_index_limit(collection, config)) {
       continue;
     }
     
+    const existing_indexes = get_indexes('default', collection);
+    
     for (const [field, stats] of fields.entries()) {
-      const time_since_last_query = now - stats.last_queried;
-      
-      if (time_since_last_query > window_ms) {
+      if (!is_within_monitoring_window(stats, window_ms)) {
         continue;
       }
       
-      const has_existing_index = existing_indexes.some(index => index.field === field);
-      if (has_existing_index) {
+      if (has_existing_index(field, existing_indexes)) {
         continue;
       }
       
-      const meets_frequency_threshold = stats.query_count >= config.frequency_threshold;
-      const meets_performance_threshold = stats.avg_time_ms >= config.performance_threshold_ms;
-      const has_slow_queries = stats.slow_query_count > 0;
-      
-      if (meets_frequency_threshold || (meets_performance_threshold && has_slow_queries)) {
-        candidates.push({
-          collection,
-          field,
-          stats: { ...stats },
-          priority: stats.slow_query_count * 2 + (stats.query_count / config.frequency_threshold)
-        });
+      if (meets_auto_index_criteria(stats, config)) {
+        candidates.push(create_index_candidate(collection, field, stats, config));
       }
     }
   }

package/src/server/lib/backup_manager.js

@@ -117,6 +117,117 @@ const calculate_sha256 = (file_path) => {
   });
 };
 
+/**
+ * Generates a timestamped backup filename.
+ * @returns {string} Backup filename with timestamp
+ */
+const generate_backup_filename = () => {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  return `joystickdb-backup-${timestamp}.tar.gz`;
+};
+
+/**
+ * Creates temporary directory for backup operations.
+ * @returns {Promise<string>} Path to temporary directory
+ */
+const create_temp_backup_directory = async () => {
+  const temp_dir = resolve('./temp_backup');
+  await mkdir(temp_dir, { recursive: true });
+  return temp_dir;
+};
+
+/**
+ * Creates compressed tar archive of database data.
+ * @param {string} temp_backup_path - Path for the backup file
+ * @returns {Promise<void>} Promise that resolves when compression completes
+ * @throws {Error} When tar process fails
+ */
+const create_compressed_archive = async (temp_backup_path) => {
+  const tar_process = spawn('tar', [
+    '--sparse',
+    '-czf',
+    temp_backup_path,
+    '-C',
+    './data',
+    '.'
+  ], {
+    stdio: ['pipe', 'pipe', 'pipe']
+  });
+  
+  let tar_output = '';
+  let tar_error = '';
+  
+  tar_process.stdout.on('data', (data) => {
+    tar_output += data.toString();
+  });
+  
+  tar_process.stderr.on('data', (data) => {
+    tar_error += data.toString();
+  });
+  
+  const tar_exit_code = await new Promise((resolve) => {
+    tar_process.on('close', resolve);
+  });
+  
+  if (tar_exit_code !== 0) {
+    throw new Error(`Tar process failed with exit code ${tar_exit_code}: ${tar_error}`);
+  }
+};
+
+/**
+ * Uploads backup file to S3 with metadata.
+ * @param {string} temp_backup_path - Path to backup file
+ * @param {string} backup_filename - Name of backup file
+ * @param {string} checksum - SHA256 checksum
+ * @param {number} size_bytes - File size in bytes
+ * @param {Object} client - S3 client
+ * @param {string} bucket - S3 bucket name
+ * @returns {Promise<void>} Promise that resolves when upload completes
+ */
+const upload_backup_to_s3 = async (temp_backup_path, backup_filename, checksum, size_bytes, client, bucket) => {
+  const upload_stream = createReadStream(temp_backup_path);
+  
+  const upload_command = new PutObjectCommand({
+    Bucket: bucket,
+    Key: backup_filename,
+    Body: upload_stream,
+    Metadata: {
+      checksum,
+      created_at: new Date().toISOString(),
+      size_bytes: size_bytes.toString()
+    }
+  });
+  
+  await client.send(upload_command);
+};
+
+/**
+ * Cleans up temporary backup files and directory.
+ * @param {string} temp_backup_path - Path to backup file
+ * @param {string} temp_dir - Path to temporary directory
+ * @returns {Promise<void>} Promise that resolves when cleanup completes
+ */
+const cleanup_temp_backup_files = async (temp_backup_path, temp_dir) => {
+  unlinkSync(temp_backup_path);
+  await rm(temp_dir, { recursive: true, force: true });
+};
+
+/**
+ * Handles cleanup on backup failure.
+ * @param {Object} log - Logger instance
+ * @returns {Promise<void>} Promise that resolves when cleanup completes
+ */
+const handle_backup_failure_cleanup = async (log) => {
+  try {
+    const temp_dir = resolve('./temp_backup');
+    if (existsSync(temp_dir)) {
+      await rm(temp_dir, { recursive: true, force: true });
+    }
+  } catch (cleanup_error) {
+    log.warn('Failed to clean up temporary backup files', { error: cleanup_error.message });
+  }
+};
+
 /**
  * Creates a compressed backup of the database and uploads it to S3.
  * Generates a timestamped tar.gz archive with sparse file support, calculates checksums,
@@ -135,52 +246,15 @@ const create_backup = async () => {
   const backup_start = Date.now();
   
   try {
-    // NOTE: Ensure S3 is configured.
     const { client, bucket } = get_s3_client();
-    
-    // NOTE: Create temporary directory.
-    const temp_dir = resolve('./temp_backup');
-    await mkdir(temp_dir, { recursive: true });
-    
-    // NOTE: Generate backup filename.
-    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-    const backup_filename = `joystickdb-backup-${timestamp}.tar.gz`;
+    const temp_dir = await create_temp_backup_directory();
+    const backup_filename = generate_backup_filename();
     const temp_backup_path = join(temp_dir, backup_filename);
     
     log.info('Starting backup creation', { backup_filename, temp_backup_path });
     
-    // NOTE: Create compressed sparse backup.
-    const tar_process = spawn('tar', [
-      '--sparse',
-      '-czf',
-      temp_backup_path,
-      '-C',
-      './data',
-      '.'
-    ], {
-      stdio: ['pipe', 'pipe', 'pipe']
-    });
+    await create_compressed_archive(temp_backup_path);
     
-    let tar_output = '';
-    let tar_error = '';
-    
-    tar_process.stdout.on('data', (data) => {
-      tar_output += data.toString();
-    });
-    
-    tar_process.stderr.on('data', (data) => {
-      tar_error += data.toString();
-    });
-    
-    const tar_exit_code = await new Promise((resolve) => {
-      tar_process.on('close', resolve);
-    });
-    
-    if (tar_exit_code !== 0) {
-      throw new Error(`Tar process failed with exit code ${tar_exit_code}: ${tar_error}`);
-    }
-    
-    // NOTE: Calculate checksum.
     const checksum = await calculate_sha256(temp_backup_path);
     const backup_stats = statSync(temp_backup_path);
     
@@ -191,25 +265,8 @@ const create_backup = async () => {
       checksum
     });
     
-    // NOTE: Upload to S3.
-    const upload_stream = createReadStream(temp_backup_path);
-    
-    const upload_command = new PutObjectCommand({
-      Bucket: bucket,
-      Key: backup_filename,
-      Body: upload_stream,
-      Metadata: {
-        checksum,
-        created_at: new Date().toISOString(),
-        size_bytes: backup_stats.size.toString()
-      }
-    });
-    
-    await client.send(upload_command);
-    
-    // NOTE: Clean up temporary file.
-    unlinkSync(temp_backup_path);
-    await rm(temp_dir, { recursive: true, force: true });
+    await upload_backup_to_s3(temp_backup_path, backup_filename, checksum, backup_stats.size, client, bucket);
+    await cleanup_temp_backup_files(temp_backup_path, temp_dir);
     
     const backup_duration = Date.now() - backup_start;
     
@@ -231,17 +288,7 @@ const create_backup = async () => {
     };
   } catch (error) {
     log.error('Backup creation failed', { error: error.message });
-    
-    // NOTE: Clean up on failure.
-    try {
-      const temp_dir = resolve('./temp_backup');
-      if (existsSync(temp_dir)) {
-        await rm(temp_dir, { recursive: true, force: true });
-      }
-    } catch (cleanup_error) {
-      log.warn('Failed to clean up temporary backup files', { error: cleanup_error.message });
-    }
-    
+    await handle_backup_failure_cleanup(log);
     throw error;
   }
 };

package/src/server/lib/index_manager.js

@@ -124,50 +124,64 @@ const compare_index_definitions = (existing, new_options) => {
 };
 
 /**
- * Rebuilds an index by clearing existing entries and re-indexing all documents.
+ * Clears existing index entries for a field.
  * @param {string} database_name - Name of the database
  * @param {string} collection_name - Name of the collection
- * @param {string} field_name - Name of the field to index
- * @param {Object} options - Index options (unique, sparse)
+ * @param {string} field_name - Name of the field
  * @param {Object} index_db - Index database instance
- * @param {Object} main_db - Main database instance
- * @throws {Error} When unique constraint violations are found
  */
-const rebuild_index = (database_name, collection_name, field_name, options, index_db, main_db) => {
+const clear_existing_index_entries = (database_name, collection_name, field_name, index_db) => {
   const index_prefix = `index:${database_name}:${collection_name}:${field_name}:`;
   const range = index_db.getRange({ start: index_prefix, end: index_prefix + '\xFF' });
   
   for (const { key } of range) {
     index_db.remove(key);
   }
-  
+};
+
+/**
+ * Validates unique constraint for index rebuild.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the field
+ * @param {Object} main_db - Main database instance
+ * @throws {Error} When duplicate values found for unique index
+ */
+const validate_unique_constraint = (database_name, collection_name, field_name, main_db) => {
   const collection_prefix = `${database_name}:${collection_name}:`;
   const document_range = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
+  const value_counts = new Map();
   
-  // First pass: check for unique constraint violations if unique index
-  if (options.unique) {
-    const value_counts = new Map();
+  for (const { key, value: document_data } of document_range) {
+    const document = JSON.parse(document_data);
+    const field_value = get_field_value(document, field_name);
     
-    for (const { key, value: document_data } of document_range) {
-      const document = JSON.parse(document_data);
-      const field_value = get_field_value(document, field_name);
+    if (field_value !== undefined && field_value !== null) {
+      const value_str = typeof field_value === 'object' ? JSON.stringify(field_value) : String(field_value);
+      const count = value_counts.get(value_str) || 0;
+      value_counts.set(value_str, count + 1);
       
-      if (field_value !== undefined && field_value !== null) {
-        const value_str = typeof field_value === 'object' ? JSON.stringify(field_value) : String(field_value);
-        const count = value_counts.get(value_str) || 0;
-        value_counts.set(value_str, count + 1);
-        
-        if (count >= 1) {
-          throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${field_name}: ${field_value}`);
-        }
+      if (count >= 1) {
+        throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${field_name}: ${field_value}`);
       }
     }
   }
+};
+
+/**
+ * Rebuilds index entries for all documents in collection.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the field
+ * @param {Object} options - Index options
+ * @param {Object} index_db - Index database instance
+ * @param {Object} main_db - Main database instance
+ */
+const rebuild_index_entries = (database_name, collection_name, field_name, options, index_db, main_db) => {
+  const collection_prefix = `${database_name}:${collection_name}:`;
+  const document_range = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
   
-  // Second pass: rebuild the index
-  const document_range_rebuild = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
-  
-  for (const { key, value: document_data } of document_range_rebuild) {
+  for (const { key, value: document_data } of document_range) {
     const document = JSON.parse(document_data);
     const field_value = get_field_value(document, field_name);
     
@@ -188,6 +202,26 @@ const rebuild_index = (database_name, collection_name, field_name, options, inde
   }
 };
 
+/**
+ * Rebuilds an index by clearing existing entries and re-indexing all documents.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the field to index
+ * @param {Object} options - Index options (unique, sparse)
+ * @param {Object} index_db - Index database instance
+ * @param {Object} main_db - Main database instance
+ * @throws {Error} When unique constraint violations are found
+ */
+const rebuild_index = (database_name, collection_name, field_name, options, index_db, main_db) => {
+  clear_existing_index_entries(database_name, collection_name, field_name, index_db);
+  
+  if (options.unique) {
+    validate_unique_constraint(database_name, collection_name, field_name, main_db);
+  }
+  
+  rebuild_index_entries(database_name, collection_name, field_name, options, index_db, main_db);
+};
+
 /**
  * Creates or updates an index on a collection field with support for unique and sparse options.
  * @param {string} database_name - Name of the database
@@ -434,6 +468,10 @@ const update_indexes_on_insert = async (database_name, collection_name, document
   const log = create_context_logger();
   const index_db = get_index_database();
   
+  if (!document) {
+    throw new Error('Document is required for index update');
+  }
+  
   try {
     const indexes = get_indexes(database_name, collection_name);