@push.rocks/smartmongo 2.0.14 → 2.1.0

package/ts/congodb/engine/IndexEngine.ts

@@ -0,0 +1,479 @@
+import * as plugins from '../congodb.plugins.js';
+import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
+import type {
+  Document,
+  IStoredDocument,
+  IIndexSpecification,
+  IIndexInfo,
+  ICreateIndexOptions,
+} from '../types/interfaces.js';
+import { CongoDuplicateKeyError, CongoIndexError } from '../errors/CongoErrors.js';
+import { QueryEngine } from './QueryEngine.js';
+
+/**
+ * Index data structure for fast lookups
+ */
+interface IIndexData {
+  name: string;
+  key: Record<string, 1 | -1 | string>;
+  unique: boolean;
+  sparse: boolean;
+  expireAfterSeconds?: number;
+  // Map from index key value to document _id(s)
+  entries: Map<string, Set<string>>;
+}
+
+/**
+ * Index engine for managing indexes and query optimization
+ */
+export class IndexEngine {
+  private dbName: string;
+  private collName: string;
+  private storage: IStorageAdapter;
+  private indexes: Map<string, IIndexData> = new Map();
+  private initialized = false;
+
+  constructor(dbName: string, collName: string, storage: IStorageAdapter) {
+    this.dbName = dbName;
+    this.collName = collName;
+    this.storage = storage;
+  }
+
+  /**
+   * Initialize indexes from storage
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    const storedIndexes = await this.storage.getIndexes(this.dbName, this.collName);
+    const documents = await this.storage.findAll(this.dbName, this.collName);
+
+    for (const indexSpec of storedIndexes) {
+      const indexData: IIndexData = {
+        name: indexSpec.name,
+        key: indexSpec.key,
+        unique: indexSpec.unique || false,
+        sparse: indexSpec.sparse || false,
+        expireAfterSeconds: indexSpec.expireAfterSeconds,
+        entries: new Map(),
+      };
+
+      // Build index entries
+      for (const doc of documents) {
+        const keyValue = this.extractKeyValue(doc, indexSpec.key);
+        if (keyValue !== null || !indexData.sparse) {
+          const keyStr = JSON.stringify(keyValue);
+          if (!indexData.entries.has(keyStr)) {
+            indexData.entries.set(keyStr, new Set());
+          }
+          indexData.entries.get(keyStr)!.add(doc._id.toHexString());
+        }
+      }
+
+      this.indexes.set(indexSpec.name, indexData);
+    }
+
+    this.initialized = true;
+  }
+
+  /**
+   * Create a new index
+   */
+  async createIndex(
+    key: Record<string, 1 | -1 | 'text' | '2dsphere'>,
+    options?: ICreateIndexOptions
+  ): Promise<string> {
+    await this.initialize();
+
+    // Generate index name if not provided
+    const name = options?.name || this.generateIndexName(key);
+
+    // Check if index already exists
+    if (this.indexes.has(name)) {
+      return name;
+    }
+
+    // Create index data structure
+    const indexData: IIndexData = {
+      name,
+      key: key as Record<string, 1 | -1 | string>,
+      unique: options?.unique || false,
+      sparse: options?.sparse || false,
+      expireAfterSeconds: options?.expireAfterSeconds,
+      entries: new Map(),
+    };
+
+    // Build index from existing documents
+    const documents = await this.storage.findAll(this.dbName, this.collName);
+
+    for (const doc of documents) {
+      const keyValue = this.extractKeyValue(doc, key);
+
+      if (keyValue === null && indexData.sparse) {
+        continue;
+      }
+
+      const keyStr = JSON.stringify(keyValue);
+
+      if (indexData.unique && indexData.entries.has(keyStr)) {
+        throw new CongoDuplicateKeyError(
+          `E11000 duplicate key error index: ${this.dbName}.${this.collName}.$${name}`,
+          key as Record<string, 1>,
+          keyValue
+        );
+      }
+
+      if (!indexData.entries.has(keyStr)) {
+        indexData.entries.set(keyStr, new Set());
+      }
+      indexData.entries.get(keyStr)!.add(doc._id.toHexString());
+    }
+
+    // Store index
+    this.indexes.set(name, indexData);
+    await this.storage.saveIndex(this.dbName, this.collName, name, {
+      key,
+      unique: options?.unique,
+      sparse: options?.sparse,
+      expireAfterSeconds: options?.expireAfterSeconds,
+    });
+
+    return name;
+  }
+
+  /**
+   * Drop an index
+   */
+  async dropIndex(name: string): Promise<void> {
+    await this.initialize();
+
+    if (name === '_id_') {
+      throw new CongoIndexError('cannot drop _id index');
+    }
+
+    if (!this.indexes.has(name)) {
+      throw new CongoIndexError(`index not found: ${name}`);
+    }
+
+    this.indexes.delete(name);
+    await this.storage.dropIndex(this.dbName, this.collName, name);
+  }
+
+  /**
+   * Drop all indexes except _id
+   */
+  async dropAllIndexes(): Promise<void> {
+    await this.initialize();
+
+    const names = Array.from(this.indexes.keys()).filter(n => n !== '_id_');
+    for (const name of names) {
+      this.indexes.delete(name);
+      await this.storage.dropIndex(this.dbName, this.collName, name);
+    }
+  }
+
+  /**
+   * List all indexes
+   */
+  async listIndexes(): Promise<IIndexInfo[]> {
+    await this.initialize();
+
+    return Array.from(this.indexes.values()).map(idx => ({
+      v: 2,
+      key: idx.key,
+      name: idx.name,
+      unique: idx.unique || undefined,
+      sparse: idx.sparse || undefined,
+      expireAfterSeconds: idx.expireAfterSeconds,
+    }));
+  }
+
+  /**
+   * Check if an index exists
+   */
+  async indexExists(name: string): Promise<boolean> {
+    await this.initialize();
+    return this.indexes.has(name);
+  }
+
+  /**
+   * Update index entries after document insert
+   */
+  async onInsert(doc: IStoredDocument): Promise<void> {
+    await this.initialize();
+
+    for (const [name, indexData] of this.indexes) {
+      const keyValue = this.extractKeyValue(doc, indexData.key);
+
+      if (keyValue === null && indexData.sparse) {
+        continue;
+      }
+
+      const keyStr = JSON.stringify(keyValue);
+
+      // Check unique constraint
+      if (indexData.unique) {
+        const existing = indexData.entries.get(keyStr);
+        if (existing && existing.size > 0) {
+          throw new CongoDuplicateKeyError(
+            `E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
+            indexData.key as Record<string, 1>,
+            keyValue
+          );
+        }
+      }
+
+      if (!indexData.entries.has(keyStr)) {
+        indexData.entries.set(keyStr, new Set());
+      }
+      indexData.entries.get(keyStr)!.add(doc._id.toHexString());
+    }
+  }
+
+  /**
+   * Update index entries after document update
+   */
+  async onUpdate(oldDoc: IStoredDocument, newDoc: IStoredDocument): Promise<void> {
+    await this.initialize();
+
+    for (const [name, indexData] of this.indexes) {
+      const oldKeyValue = this.extractKeyValue(oldDoc, indexData.key);
+      const newKeyValue = this.extractKeyValue(newDoc, indexData.key);
+      const oldKeyStr = JSON.stringify(oldKeyValue);
+      const newKeyStr = JSON.stringify(newKeyValue);
+
+      // Remove old entry if key changed
+      if (oldKeyStr !== newKeyStr) {
+        if (oldKeyValue !== null || !indexData.sparse) {
+          const oldSet = indexData.entries.get(oldKeyStr);
+          if (oldSet) {
+            oldSet.delete(oldDoc._id.toHexString());
+            if (oldSet.size === 0) {
+              indexData.entries.delete(oldKeyStr);
+            }
+          }
+        }
+
+        // Add new entry
+        if (newKeyValue !== null || !indexData.sparse) {
+          // Check unique constraint
+          if (indexData.unique) {
+            const existing = indexData.entries.get(newKeyStr);
+            if (existing && existing.size > 0) {
+              throw new CongoDuplicateKeyError(
+                `E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
+                indexData.key as Record<string, 1>,
+                newKeyValue
+              );
+            }
+          }
+
+          if (!indexData.entries.has(newKeyStr)) {
+            indexData.entries.set(newKeyStr, new Set());
+          }
+          indexData.entries.get(newKeyStr)!.add(newDoc._id.toHexString());
+        }
+      }
+    }
+  }
+
+  /**
+   * Update index entries after document delete
+   */
+  async onDelete(doc: IStoredDocument): Promise<void> {
+    await this.initialize();
+
+    for (const indexData of this.indexes.values()) {
+      const keyValue = this.extractKeyValue(doc, indexData.key);
+
+      if (keyValue === null && indexData.sparse) {
+        continue;
+      }
+
+      const keyStr = JSON.stringify(keyValue);
+      const set = indexData.entries.get(keyStr);
+      if (set) {
+        set.delete(doc._id.toHexString());
+        if (set.size === 0) {
+          indexData.entries.delete(keyStr);
+        }
+      }
+    }
+  }
+
+  /**
+   * Find the best index for a query
+   */
+  selectIndex(filter: Document): { name: string; data: IIndexData } | null {
+    if (!filter || Object.keys(filter).length === 0) {
+      return null;
+    }
+
+    // Get filter fields
+    const filterFields = new Set(this.getFilterFields(filter));
+
+    // Score each index
+    let bestIndex: { name: string; data: IIndexData } | null = null;
+    let bestScore = 0;
+
+    for (const [name, indexData] of this.indexes) {
+      const indexFields = Object.keys(indexData.key);
+      let score = 0;
+
+      // Count how many index fields are in the filter
+      for (const field of indexFields) {
+        if (filterFields.has(field)) {
+          score++;
+        } else {
+          break; // Index fields must be contiguous
+        }
+      }
+
+      // Prefer unique indexes
+      if (indexData.unique && score > 0) {
+        score += 0.5;
+      }
+
+      if (score > bestScore) {
+        bestScore = score;
+        bestIndex = { name, data: indexData };
+      }
+    }
+
+    return bestIndex;
+  }
+
+  /**
+   * Use index to find candidate document IDs
+   */
+  async findCandidateIds(filter: Document): Promise<Set<string> | null> {
+    await this.initialize();
+
+    const index = this.selectIndex(filter);
+    if (!index) return null;
+
+    // Try to use the index for equality matches
+    const indexFields = Object.keys(index.data.key);
+    const equalityValues: Record<string, any> = {};
+
+    for (const field of indexFields) {
+      const filterValue = this.getFilterValue(filter, field);
+      if (filterValue === undefined) break;
+
+      // Only use equality matches for index lookup
+      if (typeof filterValue === 'object' && filterValue !== null) {
+        if (filterValue.$eq !== undefined) {
+          equalityValues[field] = filterValue.$eq;
+        } else if (filterValue.$in !== undefined) {
+          // Handle $in with multiple lookups
+          const results = new Set<string>();
+          for (const val of filterValue.$in) {
+            equalityValues[field] = val;
+            const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
+            const ids = index.data.entries.get(keyStr);
+            if (ids) {
+              for (const id of ids) {
+                results.add(id);
+              }
+            }
+          }
+          return results;
+        } else {
+          break; // Non-equality operator, stop here
+        }
+      } else {
+        equalityValues[field] = filterValue;
+      }
+    }
+
+    if (Object.keys(equalityValues).length === 0) {
+      return null;
+    }
+
+    const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
+    return index.data.entries.get(keyStr) || new Set();
+  }
+
+  // ============================================================================
+  // Helper Methods
+  // ============================================================================
+
+  private generateIndexName(key: Record<string, any>): string {
+    return Object.entries(key)
+      .map(([field, dir]) => `${field}_${dir}`)
+      .join('_');
+  }
+
+  private extractKeyValue(doc: Document, key: Record<string, any>): any {
+    const values: any[] = [];
+
+    for (const field of Object.keys(key)) {
+      const value = QueryEngine.getNestedValue(doc, field);
+      values.push(value === undefined ? null : value);
+    }
+
+    // For single-field index, return the value directly
+    if (values.length === 1) {
+      return values[0];
+    }
+
+    return values;
+  }
+
+  private buildKeyValue(values: Record<string, any>, key: Record<string, any>): any {
+    const result: any[] = [];
+
+    for (const field of Object.keys(key)) {
+      result.push(values[field] !== undefined ? values[field] : null);
+    }
+
+    if (result.length === 1) {
+      return result[0];
+    }
+
+    return result;
+  }
+
+  private getFilterFields(filter: Document, prefix = ''): string[] {
+    const fields: string[] = [];
+
+    for (const [key, value] of Object.entries(filter)) {
+      if (key.startsWith('$')) {
+        // Logical operator
+        if (key === '$and' || key === '$or' || key === '$nor') {
+          for (const subFilter of value as Document[]) {
+            fields.push(...this.getFilterFields(subFilter, prefix));
+          }
+        }
+      } else {
+        const fullKey = prefix ? `${prefix}.${key}` : key;
+        fields.push(fullKey);
+
+        // Check for nested filters
+        if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+          const subKeys = Object.keys(value);
+          if (subKeys.length > 0 && !subKeys[0].startsWith('$')) {
+            fields.push(...this.getFilterFields(value, fullKey));
+          }
+        }
+      }
+    }
+
+    return fields;
+  }
+
+  private getFilterValue(filter: Document, field: string): any {
+    // Handle dot notation
+    const parts = field.split('.');
+    let current: any = filter;
+
+    for (const part of parts) {
+      if (current === null || current === undefined) {
+        return undefined;
+      }
+      current = current[part];
+    }
+
+    return current;
+  }
+}