@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/lib/auto_index_manager.js

@@ -0,0 +1,681 @@
+/**
+ * @fileoverview Automatic index management system for JoystickDB with intelligent query analysis.
+ * Monitors query patterns, analyzes performance metrics, and automatically creates/removes indexes
+ * based on usage patterns, frequency thresholds, and performance improvements. Provides comprehensive
+ * statistics tracking, configurable thresholds, and cleanup of unused indexes.
+ */
+
+import { get_database } from './query_engine.js';
+import { create_index, get_indexes, drop_index } from './index_manager.js';
+import { get_settings } from './load_settings.js';
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('auto_index_manager');
+
+/** @type {Object|null} LMDB database instance for storing auto-indexing data */
+let auto_index_db = null;
+
+/** @type {Map<string, Map<string, Object>>} Query statistics by collection and field */
+let query_stats = new Map();
+
+/** @type {Map<string, Map<string, Object>>} Auto-created index metadata by collection and field */
+let auto_index_metadata = new Map();
+
+/** @type {NodeJS.Timeout|null} Timer for periodic index evaluation */
+let evaluation_timer = null;
+
+/**
+ * Initializes the auto-index database and loads existing metadata and statistics.
+ * Sets up the evaluation timer for periodic index analysis.
+ * @returns {Object} LMDB database instance for auto-indexing data
+ */
+const initialize_auto_index_database = () => {
+  if (!auto_index_db) {
+    const main_db = get_database();
+    auto_index_db = main_db.openDB('auto_indexes', { create: true });
+    
+    load_auto_index_metadata();
+    load_query_stats();
+    start_evaluation_timer();
+  }
+  return auto_index_db;
+};
+
+/**
+ * Gets the initialized auto-index database instance.
+ * @returns {Object} LMDB database instance for auto-indexing data
+ * @throws {Error} When auto-index database is not initialized
+ */
+const get_auto_index_database = () => {
+  if (!auto_index_db) {
+    throw new Error('Auto index database not initialized. Call initialize_auto_index_database first.');
+  }
+  return auto_index_db;
+};
+
+/**
+ * Gets auto-indexing configuration from settings with fallback defaults.
+ * @returns {Object} Auto-indexing configuration object
+ * @returns {boolean} returns.enabled - Whether auto-indexing is enabled
+ * @returns {number} returns.frequency_threshold - Minimum query count for index creation
+ * @returns {number} returns.performance_threshold_ms - Performance threshold in milliseconds
+ * @returns {number} returns.max_auto_indexes_per_collection - Maximum auto-indexes per collection
+ * @returns {number} returns.monitoring_window_hours - Query monitoring window in hours
+ * @returns {number} returns.cleanup_unused_after_hours - Hours before cleaning unused indexes
+ * @returns {Array<string>} returns.excluded_fields - Fields to exclude from auto-indexing
+ * @returns {Array<string>} returns.included_collections - Collections to include (or ['*'] for all)
+ * @returns {Array<string>} returns.excluded_collections - Collections to exclude
+ */
+const get_auto_index_config = () => {
+  try {
+    const settings = get_settings();
+    return settings.auto_indexing || {
+      enabled: true,
+      frequency_threshold: 100,
+      performance_threshold_ms: 50,
+      max_auto_indexes_per_collection: 10,
+      monitoring_window_hours: 24,
+      cleanup_unused_after_hours: 168,
+      excluded_fields: ['_id', 'created_at'],
+      included_collections: ['*'],
+      excluded_collections: []
+    };
+  } catch (error) {
+    return {
+      enabled: true,
+      frequency_threshold: 100,
+      performance_threshold_ms: 50,
+      max_auto_indexes_per_collection: 10,
+      monitoring_window_hours: 24,
+      cleanup_unused_after_hours: 168,
+      excluded_fields: ['_id', 'created_at'],
+      included_collections: ['*'],
+      excluded_collections: []
+    };
+  }
+};
+
+/**
+ * Determines if a collection should be monitored for auto-indexing.
+ * @param {string} collection_name - Name of the collection to check
+ * @returns {boolean} True if collection should be monitored
+ */
+const should_monitor_collection = (collection_name) => {
+  const config = get_auto_index_config();
+  
+  if (config.excluded_collections.includes(collection_name)) {
+    return false;
+  }
+  
+  if (config.included_collections.includes('*')) {
+    return true;
+  }
+  
+  return config.included_collections.includes(collection_name);
+};
+
+/**
+ * Determines if a field should be monitored for auto-indexing.
+ * @param {string} field_name - Name of the field to check
+ * @returns {boolean} True if field should be monitored
+ */
+const should_monitor_field = (field_name) => {
+  const config = get_auto_index_config();
+  return !config.excluded_fields.includes(field_name);
+};
+
+/**
+ * Extracts monitorable field names from a query filter.
+ * @param {Object} filter - Query filter object
+ * @returns {Array<string>} Array of field names that should be monitored
+ */
+const extract_query_fields = (filter) => {
+  const fields = [];
+  
+  if (!filter || typeof filter !== 'object') {
+    return fields;
+  }
+  
+  for (const [field, value] of Object.entries(filter)) {
+    if (should_monitor_field(field)) {
+      fields.push(field);
+    }
+  }
+  
+  return fields;
+};
+
+/**
+ * Records a query for auto-indexing analysis, tracking performance and usage patterns.
+ * @param {string} collection_name - Name of the collection queried
+ * @param {Object} filter - Query filter used
+ * @param {number} execution_time_ms - Query execution time in milliseconds
+ * @param {boolean} [used_index=false] - Whether query used an existing index
+ * @param {string|null} [indexed_field=null] - Specific field that used an index
+ */
+const record_query = (collection_name, filter, execution_time_ms, used_index = false, indexed_field = null) => {
+  const log = create_context_logger();
+  const config = get_auto_index_config();
+  
+  if (!config.enabled || !should_monitor_collection(collection_name)) {
+    return;
+  }
+  
+  // Only record if auto index database is initialized
+  if (!auto_index_db) {
+    return;
+  }
+  
+  const fields = extract_query_fields(filter);
+  const now = new Date();
+  
+  if (!query_stats.has(collection_name)) {
+    query_stats.set(collection_name, new Map());
+  }
+  
+  const collection_stats = query_stats.get(collection_name);
+  
+  for (const field of fields) {
+    if (!collection_stats.has(field)) {
+      collection_stats.set(field, {
+        query_count: 0,
+        total_time_ms: 0,
+        avg_time_ms: 0,
+        last_queried: now,
+        slow_query_count: 0,
+        used_index_count: 0
+      });
+    }
+    
+    const field_stats = collection_stats.get(field);
+    field_stats.query_count++;
+    field_stats.total_time_ms += execution_time_ms;
+    field_stats.avg_time_ms = field_stats.total_time_ms / field_stats.query_count;
+    field_stats.last_queried = now;
+    
+    if (execution_time_ms > config.performance_threshold_ms) {
+      field_stats.slow_query_count++;
+    }
+    
+    // Increment used_index_count for the specific field that used the index
+    // If no indexed_field is specified but used_index is true, assume all fields used indexes
+    if (used_index && (indexed_field === field || indexed_field === null)) {
+      field_stats.used_index_count++;
+    }
+  }
+  
+  log.debug('Query recorded for auto-indexing analysis', {
+    collection: collection_name,
+    fields,
+    execution_time_ms,
+    used_index,
+    indexed_field
+  });
+};
+
+const save_query_stats = () => {
+  const log = create_context_logger();
+  
+  try {
+    const auto_index_db = get_auto_index_database();
+    const stats_data = {};
+    
+    for (const [collection, fields] of query_stats.entries()) {
+      stats_data[collection] = {};
+      for (const [field, stats] of fields.entries()) {
+        stats_data[collection][field] = {
+          ...stats,
+          last_queried: stats.last_queried.toISOString()
+        };
+      }
+    }
+    
+    auto_index_db.put('query_stats', stats_data);
+    
+    log.debug('Query statistics saved to database');
+  } catch (error) {
+    log.error('Failed to save query statistics', { error: error.message });
+  }
+};
+
+const load_query_stats = () => {
+  const log = create_context_logger();
+  
+  try {
+    const auto_index_db = get_auto_index_database();
+    const stats_data = auto_index_db.get('query_stats');
+    
+    if (stats_data) {
+      query_stats.clear();
+      
+      for (const [collection, fields] of Object.entries(stats_data)) {
+        const collection_stats = new Map();
+        
+        for (const [field, stats] of Object.entries(fields)) {
+          collection_stats.set(field, {
+            ...stats,
+            last_queried: new Date(stats.last_queried)
+          });
+        }
+        
+        query_stats.set(collection, collection_stats);
+      }
+      
+      log.debug('Query statistics loaded from database');
+    }
+  } catch (error) {
+    log.error('Failed to load query statistics', { error: error.message });
+  }
+};
+
+const save_auto_index_metadata = () => {
+  const log = create_context_logger();
+  
+  try {
+    const auto_index_db = get_auto_index_database();
+    const metadata_data = {};
+    
+    for (const [collection, fields] of auto_index_metadata.entries()) {
+      metadata_data[collection] = {};
+      for (const [field, metadata] of fields.entries()) {
+        metadata_data[collection][field] = {
+          ...metadata,
+          created_at: metadata.created_at.toISOString(),
+          last_used: metadata.last_used ? metadata.last_used.toISOString() : null
+        };
+      }
+    }
+    
+    auto_index_db.put('auto_index_metadata', metadata_data);
+    
+    log.debug('Auto index metadata saved to database');
+  } catch (error) {
+    log.error('Failed to save auto index metadata', { error: error.message });
+  }
+};
+
+const load_auto_index_metadata = () => {
+  const log = create_context_logger();
+  
+  try {
+    const auto_index_db = get_auto_index_database();
+    const metadata_data = auto_index_db.get('auto_index_metadata');
+    
+    if (metadata_data) {
+      auto_index_metadata.clear();
+      
+      for (const [collection, fields] of Object.entries(metadata_data)) {
+        const collection_metadata = new Map();
+        
+        for (const [field, metadata] of Object.entries(fields)) {
+          collection_metadata.set(field, {
+            ...metadata,
+            created_at: new Date(metadata.created_at),
+            last_used: metadata.last_used ? new Date(metadata.last_used) : null
+          });
+        }
+        
+        auto_index_metadata.set(collection, collection_metadata);
+      }
+      
+      log.debug('Auto index metadata loaded from database');
+    }
+  } catch (error) {
+    log.error('Failed to load auto index metadata', { error: error.message });
+  }
+};
+
+const is_auto_created_index = (collection_name, field_name) => {
+  try {
+    const collection_metadata = auto_index_metadata.get(collection_name);
+    return !!(collection_metadata && collection_metadata.has(field_name));
+  } catch (error) {
+    return false;
+  }
+};
+
+const get_auto_index_candidates = () => {
+  const config = get_auto_index_config();
+  const candidates = [];
+  const now = new Date();
+  const window_ms = config.monitoring_window_hours * 60 * 60 * 1000;
+  
+  for (const [collection, fields] of query_stats.entries()) {
+    const existing_indexes = get_indexes('default', collection);
+    const auto_index_count = existing_indexes.filter(index => 
+      is_auto_created_index(collection, index.field)
+    ).length;
+    
+    if (auto_index_count >= config.max_auto_indexes_per_collection) {
+      continue;
+    }
+    
+    for (const [field, stats] of fields.entries()) {
+      const time_since_last_query = now - stats.last_queried;
+      
+      if (time_since_last_query > window_ms) {
+        continue;
+      }
+      
+      const has_existing_index = existing_indexes.some(index => index.field === field);
+      if (has_existing_index) {
+        continue;
+      }
+      
+      const meets_frequency_threshold = stats.query_count >= config.frequency_threshold;
+      const meets_performance_threshold = stats.avg_time_ms >= config.performance_threshold_ms;
+      const has_slow_queries = stats.slow_query_count > 0;
+      
+      if (meets_frequency_threshold || (meets_performance_threshold && has_slow_queries)) {
+        candidates.push({
+          collection,
+          field,
+          stats: { ...stats },
+          priority: stats.slow_query_count * 2 + (stats.query_count / config.frequency_threshold)
+        });
+      }
+    }
+  }
+  
+  return candidates.sort((a, b) => b.priority - a.priority);
+};
+
+const create_automatic_index = async (collection_name, field_name, stats) => {
+  const log = create_context_logger();
+  
+  try {
+    await create_index('default', collection_name, field_name, { sparse: true });
+    
+    if (!auto_index_metadata.has(collection_name)) {
+      auto_index_metadata.set(collection_name, new Map());
+    }
+    
+    const collection_metadata = auto_index_metadata.get(collection_name);
+    collection_metadata.set(field_name, {
+      created_at: new Date(),
+      query_count_at_creation: stats.query_count,
+      avg_performance_improvement_ms: 0,
+      last_used: null,
+      usage_count: 0,
+      auto_created: true
+    });
+    
+    save_auto_index_metadata();
+    
+    log.info('Automatic index created', {
+      collection: collection_name,
+      field: field_name,
+      query_count: stats.query_count,
+      avg_time_ms: stats.avg_time_ms,
+      slow_query_count: stats.slow_query_count
+    });
+    
+    return true;
+  } catch (error) {
+    log.error('Failed to create automatic index', {
+      collection: collection_name,
+      field: field_name,
+      error: error.message
+    });
+    return false;
+  }
+};
+
+const evaluate_and_create_indexes = async () => {
+  const log = create_context_logger();
+  const config = get_auto_index_config();
+  
+  if (!config.enabled) {
+    return;
+  }
+  
+  try {
+    const candidates = get_auto_index_candidates();
+    
+    if (candidates.length === 0) {
+      log.debug('No automatic index candidates found');
+      return;
+    }
+    
+    log.info('Evaluating automatic index candidates', {
+      candidate_count: candidates.length
+    });
+    
+    for (const candidate of candidates.slice(0, 5)) {
+      const success = await create_automatic_index(
+        candidate.collection,
+        candidate.field,
+        candidate.stats
+      );
+      
+      if (success) {
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+    }
+  } catch (error) {
+    log.error('Failed to evaluate automatic indexes', { error: error.message });
+  }
+};
+
+const cleanup_unused_indexes = async () => {
+  const log = create_context_logger();
+  const config = get_auto_index_config();
+  const now = new Date();
+  const cleanup_threshold_ms = config.cleanup_unused_after_hours * 60 * 60 * 1000;
+  
+  try {
+    for (const [collection, fields] of auto_index_metadata.entries()) {
+      for (const [field, metadata] of fields.entries()) {
+        if (!metadata.last_used) {
+          const time_since_creation = now - metadata.created_at;
+          if (time_since_creation > cleanup_threshold_ms) {
+            await drop_index('default', collection, field);
+            fields.delete(field);
+            
+            log.info('Removed unused automatic index', {
+              collection,
+              field,
+              created_at: metadata.created_at,
+              usage_count: metadata.usage_count
+            });
+          }
+        } else {
+          const time_since_last_use = now - metadata.last_used;
+          if (time_since_last_use > cleanup_threshold_ms) {
+            await drop_index('default', collection, field);
+            fields.delete(field);
+            
+            log.info('Removed unused automatic index', {
+              collection,
+              field,
+              last_used: metadata.last_used,
+              usage_count: metadata.usage_count
+            });
+          }
+        }
+      }
+    }
+    
+    save_auto_index_metadata();
+  } catch (error) {
+    log.error('Failed to cleanup unused indexes', { error: error.message });
+  }
+};
+
+const record_index_usage = (collection_name, field_name) => {
+  const collection_metadata = auto_index_metadata.get(collection_name);
+  
+  if (collection_metadata && collection_metadata.has(field_name)) {
+    const metadata = collection_metadata.get(field_name);
+    metadata.last_used = new Date();
+    metadata.usage_count++;
+  }
+};
+
+const start_evaluation_timer = () => {
+  const config = get_auto_index_config();
+  
+  if (evaluation_timer) {
+    clearInterval(evaluation_timer);
+  }
+  
+  if (config.enabled) {
+    evaluation_timer = setInterval(async () => {
+      save_query_stats();
+      await evaluate_and_create_indexes();
+      await cleanup_unused_indexes();
+    }, 60000);
+  }
+};
+
+const stop_evaluation_timer = () => {
+  if (evaluation_timer) {
+    clearInterval(evaluation_timer);
+    evaluation_timer = null;
+  }
+};
+
+const get_query_statistics = (collection_name = null) => {
+  if (collection_name) {
+    const collection_stats = query_stats.get(collection_name);
+    if (!collection_stats) {
+      return {};
+    }
+    
+    const stats = {};
+    for (const [field, field_stats] of collection_stats.entries()) {
+      stats[field] = { ...field_stats };
+    }
+    return stats;
+  }
+  
+  const all_stats = {};
+  for (const [collection, fields] of query_stats.entries()) {
+    all_stats[collection] = {};
+    for (const [field, field_stats] of fields.entries()) {
+      all_stats[collection][field] = { ...field_stats };
+    }
+  }
+  
+  return all_stats;
+};
+
+const get_auto_index_statistics = () => {
+  const stats = {
+    total_auto_indexes: 0,
+    collections: {}
+  };
+  
+  for (const [collection, fields] of auto_index_metadata.entries()) {
+    stats.collections[collection] = {};
+    
+    for (const [field, metadata] of fields.entries()) {
+      stats.total_auto_indexes++;
+      stats.collections[collection][field] = { ...metadata };
+    }
+  }
+  
+  return stats;
+};
+
+const force_index_evaluation = async (collection_name = null) => {
+  const log = create_context_logger();
+  
+  try {
+    if (collection_name) {
+      const candidates = get_auto_index_candidates().filter(
+        candidate => candidate.collection === collection_name
+      );
+      
+      for (const candidate of candidates) {
+        await create_automatic_index(
+          candidate.collection,
+          candidate.field,
+          candidate.stats
+        );
+      }
+      
+      log.info('Forced index evaluation completed', {
+        collection: collection_name,
+        candidates_processed: candidates.length
+      });
+    } else {
+      await evaluate_and_create_indexes();
+      log.info('Forced index evaluation completed for all collections');
+    }
+    
+    return { acknowledged: true };
+  } catch (error) {
+    log.error('Failed to force index evaluation', { error: error.message });
+    throw error;
+  }
+};
+
+const remove_automatic_indexes = async (collection_name, field_names = null) => {
+  const log = create_context_logger();
+  
+  try {
+    const collection_metadata = auto_index_metadata.get(collection_name);
+    if (!collection_metadata) {
+      return { acknowledged: true, removed_count: 0 };
+    }
+    
+    const fields_to_remove = field_names || Array.from(collection_metadata.keys());
+    let removed_count = 0;
+    
+    for (const field_name of fields_to_remove) {
+      if (collection_metadata.has(field_name)) {
+        await drop_index('default', collection_name, field_name);
+        collection_metadata.delete(field_name);
+        removed_count++;
+        
+        log.info('Removed automatic index', {
+          collection: collection_name,
+          field: field_name
+        });
+      }
+    }
+    
+    save_auto_index_metadata();
+    
+    return { acknowledged: true, removed_count };
+  } catch (error) {
+    log.error('Failed to remove automatic indexes', {
+      collection: collection_name,
+      field_names,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+const cleanup_auto_index_database = () => {
+  stop_evaluation_timer();
+  if (auto_index_db) {
+    try {
+      // Clear database entries instead of saving them
+      auto_index_db.remove('query_stats');
+      auto_index_db.remove('auto_index_metadata');
+    } catch (error) {
+      // Ignore errors during cleanup
+    }
+  }
+  query_stats.clear();
+  auto_index_metadata.clear();
+  auto_index_db = null;
+};
+
+export {
+  initialize_auto_index_database,
+  get_auto_index_database,
+  record_query,
+  record_index_usage,
+  get_query_statistics,
+  get_auto_index_statistics,
+  force_index_evaluation,
+  remove_automatic_indexes,
+  is_auto_created_index,
+  start_evaluation_timer,
+  stop_evaluation_timer,
+  cleanup_auto_index_database
+};