@push.rocks/smartmongo 3.0.0 → 4.1.1

package/ts/tsmdb/engine/IndexEngine.ts

@@ -1,5 +1,89 @@
 import * as plugins from '../tsmdb.plugins.js';
 import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
+
+// Simple B-Tree implementation for range queries
+// Since sorted-btree has ESM/CJS interop issues, we use a simple custom implementation
+class SimpleBTree<K, V> {
+  private entries: Map<string, { key: K; value: V }> = new Map();
+  private sortedKeys: K[] = [];
+  private comparator: (a: K, b: K) => number;
+
+  constructor(_unused?: undefined, comparator?: (a: K, b: K) => number) {
+    this.comparator = comparator || ((a: K, b: K) => {
+      if (a < b) return -1;
+      if (a > b) return 1;
+      return 0;
+    });
+  }
+
+  private keyToString(key: K): string {
+    return JSON.stringify(key);
+  }
+
+  set(key: K, value: V): boolean {
+    const keyStr = this.keyToString(key);
+    const existed = this.entries.has(keyStr);
+    this.entries.set(keyStr, { key, value });
+
+    if (!existed) {
+      // Insert in sorted order
+      const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) > 0);
+      if (idx === -1) {
+        this.sortedKeys.push(key);
+      } else {
+        this.sortedKeys.splice(idx, 0, key);
+      }
+    }
+    return !existed;
+  }
+
+  get(key: K): V | undefined {
+    const entry = this.entries.get(this.keyToString(key));
+    return entry?.value;
+  }
+
+  delete(key: K): boolean {
+    const keyStr = this.keyToString(key);
+    if (this.entries.has(keyStr)) {
+      this.entries.delete(keyStr);
+      const idx = this.sortedKeys.findIndex(k => this.comparator(k, key) === 0);
+      if (idx !== -1) {
+        this.sortedKeys.splice(idx, 1);
+      }
+      return true;
+    }
+    return false;
+  }
+
+  forRange(
+    lowKey: K | undefined,
+    highKey: K | undefined,
+    lowInclusive: boolean,
+    highInclusive: boolean,
+    callback: (value: V, key: K) => void
+  ): void {
+    for (const key of this.sortedKeys) {
+      // Check low bound
+      if (lowKey !== undefined) {
+        const cmp = this.comparator(key, lowKey);
+        if (cmp < 0) continue;
+        if (cmp === 0 && !lowInclusive) continue;
+      }
+
+      // Check high bound
+      if (highKey !== undefined) {
+        const cmp = this.comparator(key, highKey);
+        if (cmp > 0) break;
+        if (cmp === 0 && !highInclusive) break;
+      }
+
+      const entry = this.entries.get(this.keyToString(key));
+      if (entry) {
+        callback(entry.value, key);
+      }
+    }
+  }
+}
 import type {
   Document,
   IStoredDocument,
@@ -11,7 +95,61 @@ import { TsmdbDuplicateKeyError, TsmdbIndexError } from '../errors/TsmdbErrors.j
 import { QueryEngine } from './QueryEngine.js';
 
 /**
- * Index data structure for fast lookups
+ * Comparator for B-Tree that handles mixed types consistently
+ */
+function indexKeyComparator(a: any, b: any): number {
+  // Handle null/undefined
+  if (a === null || a === undefined) {
+    if (b === null || b === undefined) return 0;
+    return -1;
+  }
+  if (b === null || b === undefined) return 1;
+
+  // Handle arrays (compound keys)
+  if (Array.isArray(a) && Array.isArray(b)) {
+    for (let i = 0; i < Math.max(a.length, b.length); i++) {
+      const cmp = indexKeyComparator(a[i], b[i]);
+      if (cmp !== 0) return cmp;
+    }
+    return 0;
+  }
+
+  // Handle ObjectId
+  if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
+    return a.toHexString().localeCompare(b.toHexString());
+  }
+
+  // Handle Date
+  if (a instanceof Date && b instanceof Date) {
+    return a.getTime() - b.getTime();
+  }
+
+  // Handle different types - use type ordering (null < number < string < object)
+  const typeOrder = (v: any): number => {
+    if (v === null || v === undefined) return 0;
+    if (typeof v === 'number') return 1;
+    if (typeof v === 'string') return 2;
+    if (typeof v === 'boolean') return 3;
+    if (v instanceof Date) return 4;
+    if (v instanceof plugins.bson.ObjectId) return 5;
+    return 6;
+  };
+
+  const typeA = typeOrder(a);
+  const typeB = typeOrder(b);
+  if (typeA !== typeB) return typeA - typeB;
+
+  // Same type comparison
+  if (typeof a === 'number') return a - b;
+  if (typeof a === 'string') return a.localeCompare(b);
+  if (typeof a === 'boolean') return (a ? 1 : 0) - (b ? 1 : 0);
+
+  // Fallback to string comparison
+  return String(a).localeCompare(String(b));
+}
+
+/**
+ * Index data structure using B-Tree for range queries
  */
 interface IIndexData {
   name: string;
@@ -19,8 +157,10 @@ interface IIndexData {
   unique: boolean;
   sparse: boolean;
   expireAfterSeconds?: number;
-  // Map from index key value to document _id(s)
-  entries: Map<string, Set<string>>;
+  // B-Tree for ordered index lookups (supports range queries)
+  btree: SimpleBTree<any, Set<string>>;
+  // Hash map for fast equality lookups
+  hashMap: Map<string, Set<string>>;
 }
 
 /**
@@ -55,7 +195,8 @@ export class IndexEngine {
         unique: indexSpec.unique || false,
         sparse: indexSpec.sparse || false,
         expireAfterSeconds: indexSpec.expireAfterSeconds,
-        entries: new Map(),
+        btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
+        hashMap: new Map(),
       };
 
       // Build index entries
@@ -63,10 +204,20 @@ export class IndexEngine {
         const keyValue = this.extractKeyValue(doc, indexSpec.key);
         if (keyValue !== null || !indexData.sparse) {
           const keyStr = JSON.stringify(keyValue);
-          if (!indexData.entries.has(keyStr)) {
-            indexData.entries.set(keyStr, new Set());
+
+          // Add to hash map
+          if (!indexData.hashMap.has(keyStr)) {
+            indexData.hashMap.set(keyStr, new Set());
+          }
+          indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
+
+          // Add to B-tree
+          const existing = indexData.btree.get(keyValue);
+          if (existing) {
+            existing.add(doc._id.toHexString());
+          } else {
+            indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
           }
-          indexData.entries.get(keyStr)!.add(doc._id.toHexString());
         }
       }
 
@@ -100,7 +251,8 @@ export class IndexEngine {
       unique: options?.unique || false,
       sparse: options?.sparse || false,
       expireAfterSeconds: options?.expireAfterSeconds,
-      entries: new Map(),
+      btree: new SimpleBTree<any, Set<string>>(undefined, indexKeyComparator),
+      hashMap: new Map(),
     };
 
     // Build index from existing documents
@@ -115,7 +267,7 @@ export class IndexEngine {
 
       const keyStr = JSON.stringify(keyValue);
 
-      if (indexData.unique && indexData.entries.has(keyStr)) {
+      if (indexData.unique && indexData.hashMap.has(keyStr)) {
         throw new TsmdbDuplicateKeyError(
           `E11000 duplicate key error index: ${this.dbName}.${this.collName}.$${name}`,
           key as Record<string, 1>,
@@ -123,10 +275,19 @@ export class IndexEngine {
         );
       }
 
-      if (!indexData.entries.has(keyStr)) {
-        indexData.entries.set(keyStr, new Set());
+      // Add to hash map
+      if (!indexData.hashMap.has(keyStr)) {
+        indexData.hashMap.set(keyStr, new Set());
+      }
+      indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
+
+      // Add to B-tree
+      const existing = indexData.btree.get(keyValue);
+      if (existing) {
+        existing.add(doc._id.toHexString());
+      } else {
+        indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
       }
-      indexData.entries.get(keyStr)!.add(doc._id.toHexString());
     }
 
     // Store index
@@ -213,7 +374,7 @@ export class IndexEngine {
 
       // Check unique constraint
       if (indexData.unique) {
-        const existing = indexData.entries.get(keyStr);
+        const existing = indexData.hashMap.get(keyStr);
         if (existing && existing.size > 0) {
           throw new TsmdbDuplicateKeyError(
             `E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
@@ -223,10 +384,19 @@ export class IndexEngine {
         }
       }
 
-      if (!indexData.entries.has(keyStr)) {
-        indexData.entries.set(keyStr, new Set());
+      // Add to hash map
+      if (!indexData.hashMap.has(keyStr)) {
+        indexData.hashMap.set(keyStr, new Set());
+      }
+      indexData.hashMap.get(keyStr)!.add(doc._id.toHexString());
+
+      // Add to B-tree
+      const btreeSet = indexData.btree.get(keyValue);
+      if (btreeSet) {
+        btreeSet.add(doc._id.toHexString());
+      } else {
+        indexData.btree.set(keyValue, new Set([doc._id.toHexString()]));
       }
-      indexData.entries.get(keyStr)!.add(doc._id.toHexString());
     }
   }
 
@@ -245,11 +415,21 @@ export class IndexEngine {
       // Remove old entry if key changed
       if (oldKeyStr !== newKeyStr) {
         if (oldKeyValue !== null || !indexData.sparse) {
-          const oldSet = indexData.entries.get(oldKeyStr);
-          if (oldSet) {
-            oldSet.delete(oldDoc._id.toHexString());
-            if (oldSet.size === 0) {
-              indexData.entries.delete(oldKeyStr);
+          // Remove from hash map
+          const oldHashSet = indexData.hashMap.get(oldKeyStr);
+          if (oldHashSet) {
+            oldHashSet.delete(oldDoc._id.toHexString());
+            if (oldHashSet.size === 0) {
+              indexData.hashMap.delete(oldKeyStr);
+            }
+          }
+
+          // Remove from B-tree
+          const oldBtreeSet = indexData.btree.get(oldKeyValue);
+          if (oldBtreeSet) {
+            oldBtreeSet.delete(oldDoc._id.toHexString());
+            if (oldBtreeSet.size === 0) {
+              indexData.btree.delete(oldKeyValue);
             }
           }
         }
@@ -258,7 +438,7 @@ export class IndexEngine {
         if (newKeyValue !== null || !indexData.sparse) {
           // Check unique constraint
           if (indexData.unique) {
-            const existing = indexData.entries.get(newKeyStr);
+            const existing = indexData.hashMap.get(newKeyStr);
             if (existing && existing.size > 0) {
               throw new TsmdbDuplicateKeyError(
                 `E11000 duplicate key error collection: ${this.dbName}.${this.collName} index: ${name}`,
@@ -268,10 +448,19 @@ export class IndexEngine {
             }
           }
 
-          if (!indexData.entries.has(newKeyStr)) {
-            indexData.entries.set(newKeyStr, new Set());
+          // Add to hash map
+          if (!indexData.hashMap.has(newKeyStr)) {
+            indexData.hashMap.set(newKeyStr, new Set());
+          }
+          indexData.hashMap.get(newKeyStr)!.add(newDoc._id.toHexString());
+
+          // Add to B-tree
+          const newBtreeSet = indexData.btree.get(newKeyValue);
+          if (newBtreeSet) {
+            newBtreeSet.add(newDoc._id.toHexString());
+          } else {
+            indexData.btree.set(newKeyValue, new Set([newDoc._id.toHexString()]));
           }
-          indexData.entries.get(newKeyStr)!.add(newDoc._id.toHexString());
         }
       }
     }
@@ -291,11 +480,22 @@ export class IndexEngine {
       }
 
       const keyStr = JSON.stringify(keyValue);
-      const set = indexData.entries.get(keyStr);
-      if (set) {
-        set.delete(doc._id.toHexString());
-        if (set.size === 0) {
-          indexData.entries.delete(keyStr);
+
+      // Remove from hash map
+      const hashSet = indexData.hashMap.get(keyStr);
+      if (hashSet) {
+        hashSet.delete(doc._id.toHexString());
+        if (hashSet.size === 0) {
+          indexData.hashMap.delete(keyStr);
+        }
+      }
+
+      // Remove from B-tree
+      const btreeSet = indexData.btree.get(keyValue);
+      if (btreeSet) {
+        btreeSet.delete(doc._id.toHexString());
+        if (btreeSet.size === 0) {
+          indexData.btree.delete(keyValue);
         }
       }
     }
@@ -309,8 +509,8 @@ export class IndexEngine {
       return null;
     }
 
-    // Get filter fields
-    const filterFields = new Set(this.getFilterFields(filter));
+    // Get filter fields and operators
+    const filterInfo = this.analyzeFilter(filter);
 
     // Score each index
     let bestIndex: { name: string; data: IIndexData } | null = null;
@@ -320,12 +520,21 @@ export class IndexEngine {
       const indexFields = Object.keys(indexData.key);
       let score = 0;
 
-      // Count how many index fields are in the filter
+      // Count how many index fields can be used
       for (const field of indexFields) {
-        if (filterFields.has(field)) {
-          score++;
+        const info = filterInfo.get(field);
+        if (!info) break;
+
+        // Equality is best
+        if (info.equality) {
+          score += 2;
+        } else if (info.range) {
+          // Range queries can use B-tree
+          score += 1;
+        } else if (info.in) {
+          score += 1.5;
         } else {
-          break; // Index fields must be contiguous
+          break;
         }
       }
 
@@ -344,7 +553,46 @@ export class IndexEngine {
   }
 
   /**
-   * Use index to find candidate document IDs
+   * Analyze filter to extract field operators
+   */
+  private analyzeFilter(filter: Document): Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }> {
+    const result = new Map<string, { equality: boolean; range: boolean; in: boolean; ops: Record<string, any> }>();
+
+    for (const [key, value] of Object.entries(filter)) {
+      if (key.startsWith('$')) continue;
+
+      const info = { equality: false, range: false, in: false, ops: {} as Record<string, any> };
+
+      if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
+        info.equality = true;
+        info.ops['$eq'] = value;
+      } else {
+        const ops = value as Record<string, any>;
+        if (ops.$eq !== undefined) {
+          info.equality = true;
+          info.ops['$eq'] = ops.$eq;
+        }
+        if (ops.$in !== undefined) {
+          info.in = true;
+          info.ops['$in'] = ops.$in;
+        }
+        if (ops.$gt !== undefined || ops.$gte !== undefined || ops.$lt !== undefined || ops.$lte !== undefined) {
+          info.range = true;
+          if (ops.$gt !== undefined) info.ops['$gt'] = ops.$gt;
+          if (ops.$gte !== undefined) info.ops['$gte'] = ops.$gte;
+          if (ops.$lt !== undefined) info.ops['$lt'] = ops.$lt;
+          if (ops.$lte !== undefined) info.ops['$lte'] = ops.$lte;
+        }
+      }
+
+      result.set(key, info);
+    }
+
+    return result;
+  }
+
+  /**
+   * Use index to find candidate document IDs (supports range queries with B-tree)
    */
   async findCandidateIds(filter: Document): Promise<Set<string> | null> {
     await this.initialize();
@@ -352,25 +600,58 @@ export class IndexEngine {
     const index = this.selectIndex(filter);
     if (!index) return null;
 
-    // Try to use the index for equality matches
+    const filterInfo = this.analyzeFilter(filter);
     const indexFields = Object.keys(index.data.key);
-    const equalityValues: Record<string, any> = {};
 
-    for (const field of indexFields) {
-      const filterValue = this.getFilterValue(filter, field);
-      if (filterValue === undefined) break;
+    // For single-field indexes with range queries, use B-tree
+    if (indexFields.length === 1) {
+      const field = indexFields[0];
+      const info = filterInfo.get(field);
+
+      if (info) {
+        // Handle equality using hash map (faster)
+        if (info.equality) {
+          const keyStr = JSON.stringify(info.ops['$eq']);
+          return index.data.hashMap.get(keyStr) || new Set();
+        }
 
-      // Only use equality matches for index lookup
-      if (typeof filterValue === 'object' && filterValue !== null) {
-        if (filterValue.$eq !== undefined) {
-          equalityValues[field] = filterValue.$eq;
-        } else if (filterValue.$in !== undefined) {
+        // Handle $in using hash map
+        if (info.in) {
+          const results = new Set<string>();
+          for (const val of info.ops['$in']) {
+            const keyStr = JSON.stringify(val);
+            const ids = index.data.hashMap.get(keyStr);
+            if (ids) {
+              for (const id of ids) {
+                results.add(id);
+              }
+            }
+          }
+          return results;
+        }
+
+        // Handle range queries using B-tree
+        if (info.range) {
+          return this.findRangeCandidates(index.data, info.ops);
+        }
+      }
+    } else {
+      // For compound indexes, use hash map with partial key matching
+      const equalityValues: Record<string, any> = {};
+
+      for (const field of indexFields) {
+        const info = filterInfo.get(field);
+        if (!info) break;
+
+        if (info.equality) {
+          equalityValues[field] = info.ops['$eq'];
+        } else if (info.in) {
           // Handle $in with multiple lookups
           const results = new Set<string>();
-          for (const val of filterValue.$in) {
+          for (const val of info.ops['$in']) {
             equalityValues[field] = val;
             const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
-            const ids = index.data.entries.get(keyStr);
+            const ids = index.data.hashMap.get(keyStr);
             if (ids) {
               for (const id of ids) {
                 results.add(id);
@@ -379,19 +660,57 @@ export class IndexEngine {
           }
           return results;
         } else {
-          break; // Non-equality operator, stop here
+          break; // Non-equality/in operator, stop here
         }
-      } else {
-        equalityValues[field] = filterValue;
+      }
+
+      if (Object.keys(equalityValues).length > 0) {
+        const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
+        return index.data.hashMap.get(keyStr) || new Set();
       }
     }
 
-    if (Object.keys(equalityValues).length === 0) {
-      return null;
+    return null;
+  }
+
+  /**
+   * Find candidates using B-tree range scan
+   */
+  private findRangeCandidates(indexData: IIndexData, ops: Record<string, any>): Set<string> {
+    const results = new Set<string>();
+
+    let lowKey: any = undefined;
+    let highKey: any = undefined;
+    let lowInclusive = true;
+    let highInclusive = true;
+
+    if (ops['$gt'] !== undefined) {
+      lowKey = ops['$gt'];
+      lowInclusive = false;
+    }
+    if (ops['$gte'] !== undefined) {
+      lowKey = ops['$gte'];
+      lowInclusive = true;
     }
+    if (ops['$lt'] !== undefined) {
+      highKey = ops['$lt'];
+      highInclusive = false;
+    }
+    if (ops['$lte'] !== undefined) {
+      highKey = ops['$lte'];
+      highInclusive = true;
+    }
+
+    // Use B-tree range iteration
+    indexData.btree.forRange(lowKey, highKey, lowInclusive, highInclusive, (value, key) => {
+      if (value) {
+        for (const id of value) {
+          results.add(id);
+        }
+      }
+    });
 
-    const keyStr = JSON.stringify(this.buildKeyValue(equalityValues, index.data.key));
-    return index.data.entries.get(keyStr) || new Set();
+    return results;
   }
 
   // ============================================================================