@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/lib/index_manager.js

@@ -0,0 +1,756 @@
+/**
+ * @fileoverview Index management system for JoystickDB providing secondary indexing capabilities.
+ * Handles creation, maintenance, and querying of database indexes with support for unique and sparse indexes.
+ * Manages index metadata, automatic index updates on document changes, and index-based query optimization.
+ */
+
+import { get_database } from './query_engine.js';
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('index_manager');
+
+/** @type {Object|null} LMDB database instance for storing index data */
+let index_db = null;
+
+/**
+ * Initializes the index database by opening the 'indexes' sub-database.
+ * Always reinitializes to ensure fresh reference after database restarts.
+ * @returns {Object} LMDB database instance for indexes
+ * @throws {Error} When main database is not initialized
+ */
+const initialize_index_database = () => {
+  // Always reinitialize to ensure we have a fresh reference
+  // This handles cases where the main database was closed and reopened
+  try {
+    const main_db = get_database();
+    index_db = main_db.openDB('indexes', { create: true });
+  } catch (error) {
+    // If main database is not initialized, throw a clear error
+    throw new Error('Main database not initialized. Call initialize_database first.');
+  }
+  return index_db;
+};
+
+/**
+ * Gets the initialized index database instance.
+ * @returns {Object} LMDB database instance for indexes
+ * @throws {Error} When index database is not initialized
+ */
+const get_index_database = () => {
+  if (!index_db) {
+    throw new Error('Index database not initialized. Call initialize_index_database first.');
+  }
+  return index_db;
+};
+
+/**
+ * Builds a standardized index key for storing index entries.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the indexed field
+ * @param {*} field_value - Value to index (will be stringified)
+ * @returns {string} Formatted index key
+ */
+const build_index_key = (database_name, collection_name, field_name, field_value) => {
+  const value_str = typeof field_value === 'object' ? JSON.stringify(field_value) : String(field_value);
+  return `index:${database_name}:${collection_name}:${field_name}:${value_str}`;
+};
+
+/**
+ * Parses an index key to extract database, collection, field, and value components.
+ * @param {string} key - Index key to parse
+ * @returns {Object|null} Parsed components or null if invalid format
+ * @returns {string} returns.database - Database name
+ * @returns {string} returns.collection - Collection name
+ * @returns {string} returns.field - Field name
+ * @returns {string} returns.value - Field value as string
+ */
+const parse_index_key = (key) => {
+  const parts = key.split(':');
+  if (parts.length < 5 || parts[0] !== 'index') {
+    return null;
+  }
+  
+  return {
+    database: parts[1],
+    collection: parts[2],
+    field: parts[3],
+    value: parts.slice(4).join(':')
+  };
+};
+
+/**
+ * Builds a metadata key for storing index configuration.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the indexed field
+ * @returns {string} Formatted metadata key
+ */
+const build_index_metadata_key = (database_name, collection_name, field_name) => {
+  return `meta:${database_name}:${collection_name}:${field_name}`;
+};
+
+/**
+ * Extracts field value from document using dot notation path.
+ * @param {Object} document - Document to extract value from
+ * @param {string} field_path - Dot-separated field path (e.g., 'user.name')
+ * @returns {*} Field value or undefined if path doesn't exist
+ */
+const get_field_value = (document, field_path) => {
+  const parts = field_path.split('.');
+  let value = document;
+  
+  for (const part of parts) {
+    if (value === null || value === undefined) {
+      return undefined;
+    }
+    value = value[part];
+  }
+  
+  return value;
+};
+
+/**
+ * Compares existing index definition with new options to detect changes.
+ * @param {Object} existing - Existing index metadata
+ * @param {Object} new_options - New index options
+ * @returns {boolean} True if definitions match, false otherwise
+ */
+const compare_index_definitions = (existing, new_options) => {
+  return (
+    existing.unique === (new_options.unique || false) &&
+    existing.sparse === (new_options.sparse || false)
+  );
+};
+
+/**
+ * Rebuilds an index by clearing existing entries and re-indexing all documents.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the field to index
+ * @param {Object} options - Index options (unique, sparse)
+ * @param {Object} index_db - Index database instance
+ * @param {Object} main_db - Main database instance
+ * @throws {Error} When unique constraint violations are found
+ */
+const rebuild_index = (database_name, collection_name, field_name, options, index_db, main_db) => {
+  const index_prefix = `index:${database_name}:${collection_name}:${field_name}:`;
+  const range = index_db.getRange({ start: index_prefix, end: index_prefix + '\xFF' });
+  
+  for (const { key } of range) {
+    index_db.remove(key);
+  }
+  
+  const collection_prefix = `${database_name}:${collection_name}:`;
+  const document_range = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
+  
+  // First pass: check for unique constraint violations if unique index
+  if (options.unique) {
+    const value_counts = new Map();
+    
+    for (const { key, value: document_data } of document_range) {
+      const document = JSON.parse(document_data);
+      const field_value = get_field_value(document, field_name);
+      
+      if (field_value !== undefined && field_value !== null) {
+        const value_str = typeof field_value === 'object' ? JSON.stringify(field_value) : String(field_value);
+        const count = value_counts.get(value_str) || 0;
+        value_counts.set(value_str, count + 1);
+        
+        if (count >= 1) {
+          throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${field_name}: ${field_value}`);
+        }
+      }
+    }
+  }
+  
+  // Second pass: rebuild the index
+  const document_range_rebuild = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
+  
+  for (const { key, value: document_data } of document_range_rebuild) {
+    const document = JSON.parse(document_data);
+    const field_value = get_field_value(document, field_name);
+    
+    if (field_value === undefined || field_value === null) {
+      if (!options.sparse) {
+        const index_key = build_index_key(database_name, collection_name, field_name, null);
+        const existing_ids = index_db.get(index_key) || [];
+        existing_ids.push(document._id);
+        index_db.put(index_key, existing_ids);
+      }
+      continue;
+    }
+    
+    const index_key = build_index_key(database_name, collection_name, field_name, field_value);
+    const existing_ids = index_db.get(index_key) || [];
+    existing_ids.push(document._id);
+    index_db.put(index_key, existing_ids);
+  }
+};
+
+/**
+ * Creates or updates an index on a collection field with support for unique and sparse options.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection to index
+ * @param {string} field_name - Name of the field to index
+ * @param {Object} [options={}] - Index creation options
+ * @param {boolean} [options.unique=false] - Whether index should enforce uniqueness
+ * @param {boolean} [options.sparse=false] - Whether to exclude null/undefined values
+ * @param {boolean} [options.upsert=false] - Whether to update existing index if it exists
+ * @returns {Promise<Object>} Result object with acknowledgment and operation type
+ * @returns {boolean} returns.acknowledged - Whether operation was acknowledged
+ * @returns {string} returns.operation_type - Type of operation performed (created/updated/unchanged)
+ * @throws {Error} When database/collection/field name missing, index exists without upsert, or unique constraint violations
+ */
+const create_index = async (database_name, collection_name, field_name, options = {}) => {
+  const log = create_context_logger();
+  
+  if (!database_name) {
+    throw new Error('Database name is required');
+  }
+  
+  if (!collection_name) {
+    throw new Error('Collection name is required');
+  }
+  
+  if (!field_name) {
+    throw new Error('Field name is required');
+  }
+  
+  const index_db = get_index_database();
+  const main_db = get_database();
+  const metadata_key = build_index_metadata_key(database_name, collection_name, field_name);
+  const upsert = options.upsert || false;
+  
+  try {
+    let operation_type = 'created';
+    
+    await index_db.transaction(() => {
+      const existing_index = index_db.get(metadata_key);
+      
+      if (existing_index) {
+        if (!upsert) {
+          throw new Error(`Index on ${database_name}.${collection_name}.${field_name} already exists`);
+        }
+        
+        const definitions_match = compare_index_definitions(existing_index, options);
+        
+        if (definitions_match) {
+          operation_type = 'unchanged';
+          return;
+        }
+        
+        operation_type = 'updated';
+        try {
+          rebuild_index(database_name, collection_name, field_name, options, index_db, main_db);
+        } catch (rebuild_error) {
+          throw rebuild_error;
+        }
+      } else {
+        const collection_prefix = `${database_name}:${collection_name}:`;
+        const range = main_db.getRange({ start: collection_prefix, end: collection_prefix + '\xFF' });
+        
+        for (const { key, value: document_data } of range) {
+          const document = JSON.parse(document_data);
+          const field_value = get_field_value(document, field_name);
+          
+          if (field_value === undefined || field_value === null) {
+            if (!options.sparse) {
+              const index_key = build_index_key(database_name, collection_name, field_name, null);
+              const existing_ids = index_db.get(index_key) || [];
+              existing_ids.push(document._id);
+              index_db.put(index_key, existing_ids);
+            }
+            continue;
+          }
+          
+          if (options.unique) {
+            const index_key = build_index_key(database_name, collection_name, field_name, field_value);
+            const existing_ids = index_db.get(index_key) || [];
+            
+            if (existing_ids.length > 0) {
+              throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${field_name}: ${field_value}`);
+            }
+          }
+          
+          const index_key = build_index_key(database_name, collection_name, field_name, field_value);
+          const existing_ids = index_db.get(index_key) || [];
+          existing_ids.push(document._id);
+          index_db.put(index_key, existing_ids);
+        }
+      }
+      
+      const index_metadata = {
+        database: database_name,
+        collection: collection_name,
+        field: field_name,
+        unique: options.unique || false,
+        sparse: options.sparse || false,
+        created_at: existing_index ? existing_index.created_at : new Date().toISOString(),
+        updated_at: operation_type === 'updated' ? new Date().toISOString() : undefined
+      };
+      
+      index_db.put(metadata_key, index_metadata);
+    });
+    
+    log.info(`Index ${operation_type} successfully`, {
+      database: database_name,
+      collection: collection_name,
+      field: field_name,
+      options,
+      operation_type
+    });
+    
+    return { acknowledged: true, operation_type };
+  } catch (error) {
+    log.error('Failed to create/upsert index', {
+      database: database_name,
+      collection: collection_name,
+      field: field_name,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Drops an existing index from a collection field.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the indexed field
+ * @returns {Promise<Object>} Result object with acknowledgment
+ * @returns {boolean} returns.acknowledged - Whether operation was acknowledged
+ * @throws {Error} When database/collection/field name missing or index doesn't exist
+ */
+const drop_index = async (database_name, collection_name, field_name) => {
+  const log = create_context_logger();
+  
+  if (!database_name) {
+    throw new Error('Database name is required');
+  }
+  
+  if (!collection_name) {
+    throw new Error('Collection name is required');
+  }
+  
+  if (!field_name) {
+    throw new Error('Field name is required');
+  }
+  
+  const index_db = get_index_database();
+  const metadata_key = build_index_metadata_key(database_name, collection_name, field_name);
+  
+  try {
+    await index_db.transaction(() => {
+      const existing_index = index_db.get(metadata_key);
+      
+      if (!existing_index) {
+        throw new Error(`Index on ${database_name}.${collection_name}.${field_name} does not exist`);
+      }
+      
+      index_db.remove(metadata_key);
+      
+      const index_prefix = `index:${database_name}:${collection_name}:${field_name}:`;
+      const range = index_db.getRange({ start: index_prefix, end: index_prefix + '\xFF' });
+      
+      for (const { key } of range) {
+        index_db.remove(key);
+      }
+    });
+    
+    log.info('Index dropped successfully', {
+      database: database_name,
+      collection: collection_name,
+      field: field_name
+    });
+    
+    return { acknowledged: true };
+  } catch (error) {
+    log.error('Failed to drop index', {
+      database: database_name,
+      collection: collection_name,
+      field: field_name,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Retrieves all indexes for a given collection.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @returns {Array<Object>} Array of index metadata objects
+ * @throws {Error} When database/collection name is missing or retrieval fails
+ */
+const get_indexes = (database_name, collection_name) => {
+  const log = create_context_logger();
+  
+  if (!database_name) {
+    throw new Error('Database name is required');
+  }
+  
+  if (!collection_name) {
+    throw new Error('Collection name is required');
+  }
+  
+  const index_db = get_index_database();
+  const indexes = [];
+  
+  try {
+    const metadata_prefix = `meta:${database_name}:${collection_name}:`;
+    const range = index_db.getRange({ start: metadata_prefix, end: metadata_prefix + '\xFF' });
+    
+    for (const { key, value: metadata } of range) {
+      indexes.push(metadata);
+    }
+    
+    log.info('Retrieved indexes', {
+      database: database_name,
+      collection: collection_name,
+      count: indexes.length
+    });
+    
+    return indexes;
+  } catch (error) {
+    log.error('Failed to get indexes', {
+      database: database_name,
+      collection: collection_name,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Updates all relevant indexes when a new document is inserted.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {Object} document - Document being inserted
+ * @returns {Promise<void>} Promise that resolves when indexes are updated
+ * @throws {Error} When unique constraint violations occur or update fails
+ */
+const update_indexes_on_insert = async (database_name, collection_name, document) => {
+  const log = create_context_logger();
+  const index_db = get_index_database();
+  
+  try {
+    const indexes = get_indexes(database_name, collection_name);
+    
+    await index_db.transaction(() => {
+      for (const index_metadata of indexes) {
+        const field_value = get_field_value(document, index_metadata.field);
+        
+        if (field_value === undefined || field_value === null) {
+          if (!index_metadata.sparse) {
+            const index_key = build_index_key(database_name, collection_name, index_metadata.field, null);
+            const existing_ids = index_db.get(index_key) || [];
+            existing_ids.push(document._id);
+            index_db.put(index_key, existing_ids);
+          }
+          continue;
+        }
+        
+        if (index_metadata.unique) {
+          const index_key = build_index_key(database_name, collection_name, index_metadata.field, field_value);
+          const existing_ids = index_db.get(index_key) || [];
+          
+          if (existing_ids.length > 0) {
+            throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${index_metadata.field}: ${field_value}`);
+          }
+        }
+        
+        const index_key = build_index_key(database_name, collection_name, index_metadata.field, field_value);
+        const existing_ids = index_db.get(index_key) || [];
+        existing_ids.push(document._id);
+        index_db.put(index_key, existing_ids);
+      }
+    });
+  } catch (error) {
+    log.error('Failed to update indexes on insert', {
+      database: database_name,
+      collection: collection_name,
+      document_id: document._id,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Updates all relevant indexes when a document is modified.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {Object} old_document - Document before update
+ * @param {Object} new_document - Document after update
+ * @returns {Promise<void>} Promise that resolves when indexes are updated
+ * @throws {Error} When unique constraint violations occur or update fails
+ */
+const update_indexes_on_update = async (database_name, collection_name, old_document, new_document) => {
+  const log = create_context_logger();
+  const index_db = get_index_database();
+  
+  try {
+    const indexes = get_indexes(database_name, collection_name);
+    
+    await index_db.transaction(() => {
+      for (const index_metadata of indexes) {
+        const old_value = get_field_value(old_document, index_metadata.field);
+        const new_value = get_field_value(new_document, index_metadata.field);
+        
+        if (old_value === new_value) {
+          continue;
+        }
+        
+        if (old_value !== undefined && old_value !== null) {
+          const old_index_key = build_index_key(database_name, collection_name, index_metadata.field, old_value);
+          const old_ids = index_db.get(old_index_key) || [];
+          const filtered_ids = old_ids.filter(id => id !== old_document._id);
+          
+          if (filtered_ids.length === 0) {
+            index_db.remove(old_index_key);
+          } else {
+            index_db.put(old_index_key, filtered_ids);
+          }
+        } else if (!index_metadata.sparse) {
+          const old_index_key = build_index_key(database_name, collection_name, index_metadata.field, null);
+          const old_ids = index_db.get(old_index_key) || [];
+          const filtered_ids = old_ids.filter(id => id !== old_document._id);
+          
+          if (filtered_ids.length === 0) {
+            index_db.remove(old_index_key);
+          } else {
+            index_db.put(old_index_key, filtered_ids);
+          }
+        }
+        
+        if (new_value !== undefined && new_value !== null) {
+          if (index_metadata.unique) {
+            const new_index_key = build_index_key(database_name, collection_name, index_metadata.field, new_value);
+            const existing_ids = index_db.get(new_index_key) || [];
+            
+            if (existing_ids.length > 0) {
+              throw new Error(`Duplicate value for unique index on ${database_name}.${collection_name}.${index_metadata.field}: ${new_value}`);
+            }
+          }
+          
+          const new_index_key = build_index_key(database_name, collection_name, index_metadata.field, new_value);
+          const existing_ids = index_db.get(new_index_key) || [];
+          existing_ids.push(new_document._id);
+          index_db.put(new_index_key, existing_ids);
+        } else if (!index_metadata.sparse) {
+          const new_index_key = build_index_key(database_name, collection_name, index_metadata.field, null);
+          const existing_ids = index_db.get(new_index_key) || [];
+          existing_ids.push(new_document._id);
+          index_db.put(new_index_key, existing_ids);
+        }
+      }
+    });
+  } catch (error) {
+    log.error('Failed to update indexes on update', {
+      database: database_name,
+      collection: collection_name,
+      document_id: old_document._id,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Updates all relevant indexes when a document is deleted.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {Object} document - Document being deleted
+ * @returns {Promise<void>} Promise that resolves when indexes are updated
+ * @throws {Error} When index update fails
+ */
+const update_indexes_on_delete = async (database_name, collection_name, document) => {
+  const log = create_context_logger();
+  const index_db = get_index_database();
+  
+  try {
+    const indexes = get_indexes(database_name, collection_name);
+    
+    await index_db.transaction(() => {
+      for (const index_metadata of indexes) {
+        const field_value = get_field_value(document, index_metadata.field);
+        
+        let index_key;
+        if (field_value === undefined || field_value === null) {
+          if (!index_metadata.sparse) {
+            index_key = build_index_key(database_name, collection_name, index_metadata.field, null);
+          } else {
+            continue;
+          }
+        } else {
+          index_key = build_index_key(database_name, collection_name, index_metadata.field, field_value);
+        }
+        
+        const existing_ids = index_db.get(index_key) || [];
+        const filtered_ids = existing_ids.filter(id => id !== document._id);
+        
+        if (filtered_ids.length === 0) {
+          index_db.remove(index_key);
+        } else {
+          index_db.put(index_key, filtered_ids);
+        }
+      }
+    });
+  } catch (error) {
+    log.error('Failed to update indexes on delete', {
+      database: database_name,
+      collection: collection_name,
+      document_id: document._id,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Finds document IDs using an index for the specified field and operator.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {string} field_name - Name of the indexed field
+ * @param {string} operator - Query operator ($eq, $in, $exists, or 'eq')
+ * @param {*} value - Value to search for
+ * @returns {Array<string>|null} Array of document IDs or null if no matches/unsupported operator
+ * @throws {Error} When $in operator used with non-array value
+ */
+const find_documents_by_index = (database_name, collection_name, field_name, operator, value) => {
+  const log = create_context_logger();
+  const index_db = get_index_database();
+  const document_ids = new Set();
+  
+  try {
+    switch (operator) {
+      case '$eq':
+      case 'eq': {
+        const index_key = build_index_key(database_name, collection_name, field_name, value);
+        const ids = index_db.get(index_key) || [];
+        ids.forEach(id => document_ids.add(id));
+        break;
+      }
+      
+      case '$in': {
+        if (!Array.isArray(value)) {
+          throw new Error('$in operator requires an array value');
+        }
+        
+        for (const val of value) {
+          const index_key = build_index_key(database_name, collection_name, field_name, val);
+          const ids = index_db.get(index_key) || [];
+          ids.forEach(id => document_ids.add(id));
+        }
+        break;
+      }
+      
+      case '$exists': {
+        const index_prefix = `index:${database_name}:${collection_name}:${field_name}:`;
+        const range = index_db.getRange({ start: index_prefix, end: index_prefix + '\xFF' });
+        
+        for (const { key, value: ids } of range) {
+          const parsed = parse_index_key(key);
+          const has_value = parsed && parsed.value !== 'null';
+          
+          if ((value && has_value) || (!value && !has_value)) {
+            ids.forEach(id => document_ids.add(id));
+          }
+        }
+        break;
+      }
+      
+      default:
+        return null;
+    }
+    
+    const result = Array.from(document_ids);
+    
+    log.info('Index query completed', {
+      database: database_name,
+      collection: collection_name,
+      field: field_name,
+      operator,
+      document_ids_found: result.length
+    });
+    
+    return result.length > 0 ? result : null;
+  } catch (error) {
+    log.error('Failed to query index', {
+      database: database_name,
+      collection: collection_name,
+      field: field_name,
+      operator,
+      error: error.message
+    });
+    return null;
+  }
+};
+
+/**
+ * Determines if an index can be used for the given filter and returns optimization info.
+ * @param {string} database_name - Name of the database
+ * @param {string} collection_name - Name of the collection
+ * @param {Object} filter - Query filter object
+ * @returns {Object|null} Index usage info or null if no suitable index found
+ * @returns {string} returns.field - Field name that can use index
+ * @returns {Array<string>} returns.operators - Supported operators for the field
+ */
+const can_use_index = (database_name, collection_name, filter) => {
+  const indexes = get_indexes(database_name, collection_name);
+  
+  for (const [field, value] of Object.entries(filter)) {
+    const has_index = indexes.some(index => index.field === field);
+    
+    if (has_index) {
+      if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+        const operators = Object.keys(value);
+        const supported_operators = ['$eq', '$in', '$exists'];
+        
+        if (operators.some(op => supported_operators.includes(op))) {
+          return { field, operators: operators.filter(op => supported_operators.includes(op)) };
+        }
+      } else {
+        return { field, operators: ['eq'] };
+      }
+    }
+  }
+  
+  return null;
+};
+
+/**
+ * Cleans up the index database by removing all entries and resetting the reference.
+ * Used during database cleanup and shutdown procedures.
+ */
+const cleanup_index_database = () => {
+  if (index_db) {
+    try {
+      // Clear all index metadata and index entries
+      const range = index_db.getRange();
+      for (const { key } of range) {
+        index_db.remove(key);
+      }
+    } catch (error) {
+      // Ignore errors during cleanup
+    }
+  }
+  index_db = null;
+};
+
+export {
+  initialize_index_database,
+  get_index_database,
+  create_index,
+  drop_index,
+  get_indexes,
+  update_indexes_on_insert,
+  update_indexes_on_update,
+  update_indexes_on_delete,
+  find_documents_by_index,
+  can_use_index,
+  build_index_key,
+  parse_index_key,
+  cleanup_index_database
+};