@push.rocks/smartdb 1.0.1 → 2.0.0

package/ts/ts_smartdb/engine/QueryEngine.ts

@@ -1,301 +0,0 @@
-import * as plugins from '../plugins.js';
-import type { Document, IStoredDocument, ISortSpecification, ISortDirection } from '../types/interfaces.js';
-
-// Import mingo Query class
-import { Query } from 'mingo';
-
-/**
- * Query engine using mingo for MongoDB-compatible query matching
- */
-export class QueryEngine {
-  /**
-   * Filter documents by a MongoDB query filter
-   */
-  static filter(documents: IStoredDocument[], filter: Document): IStoredDocument[] {
-    if (!filter || Object.keys(filter).length === 0) {
-      return documents;
-    }
-
-    const query = new Query(filter);
-    return documents.filter(doc => query.test(doc));
-  }
-
-  /**
-   * Test if a single document matches a filter
-   */
-  static matches(document: Document, filter: Document): boolean {
-    if (!filter || Object.keys(filter).length === 0) {
-      return true;
-    }
-
-    const query = new Query(filter);
-    return query.test(document);
-  }
-
-  /**
-   * Find a single document matching the filter
-   */
-  static findOne(documents: IStoredDocument[], filter: Document): IStoredDocument | null {
-    if (!filter || Object.keys(filter).length === 0) {
-      return documents[0] || null;
-    }
-
-    const query = new Query(filter);
-    for (const doc of documents) {
-      if (query.test(doc)) {
-        return doc;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Sort documents by a sort specification
-   */
-  static sort(documents: IStoredDocument[], sort: ISortSpecification): IStoredDocument[] {
-    if (!sort) {
-      return documents;
-    }
-
-    // Normalize sort specification to array of [field, direction] pairs
-    const sortFields: Array<[string, number]> = [];
-
-    if (Array.isArray(sort)) {
-      for (const [field, direction] of sort) {
-        sortFields.push([field, this.normalizeDirection(direction)]);
-      }
-    } else {
-      for (const [field, direction] of Object.entries(sort)) {
-        sortFields.push([field, this.normalizeDirection(direction)]);
-      }
-    }
-
-    return [...documents].sort((a, b) => {
-      for (const [field, direction] of sortFields) {
-        const aVal = this.getNestedValue(a, field);
-        const bVal = this.getNestedValue(b, field);
-
-        const comparison = this.compareValues(aVal, bVal);
-        if (comparison !== 0) {
-          return comparison * direction;
-        }
-      }
-      return 0;
-    });
-  }
-
-  /**
-   * Apply projection to documents
-   */
-  static project(documents: IStoredDocument[], projection: Document): Document[] {
-    if (!projection || Object.keys(projection).length === 0) {
-      return documents;
-    }
-
-    // Determine if this is inclusion or exclusion projection
-    const keys = Object.keys(projection);
-    const hasInclusion = keys.some(k => k !== '_id' && projection[k] === 1);
-    const hasExclusion = keys.some(k => k !== '_id' && projection[k] === 0);
-
-    // Can't mix inclusion and exclusion (except for _id)
-    if (hasInclusion && hasExclusion) {
-      throw new Error('Cannot mix inclusion and exclusion in projection');
-    }
-
-    return documents.map(doc => {
-      if (hasInclusion) {
-        // Inclusion projection
-        const result: Document = {};
-
-        // Handle _id
-        if (projection._id !== 0 && projection._id !== false) {
-          result._id = doc._id;
-        }
-
-        for (const key of keys) {
-          if (key === '_id') continue;
-          if (projection[key] === 1 || projection[key] === true) {
-            const value = this.getNestedValue(doc, key);
-            if (value !== undefined) {
-              this.setNestedValue(result, key, value);
-            }
-          }
-        }
-
-        return result;
-      } else {
-        // Exclusion projection - start with copy and remove fields
-        const result = { ...doc };
-
-        for (const key of keys) {
-          if (projection[key] === 0 || projection[key] === false) {
-            this.deleteNestedValue(result, key);
-          }
-        }
-
-        return result;
-      }
-    });
-  }
-
-  /**
-   * Get distinct values for a field
-   */
-  static distinct(documents: IStoredDocument[], field: string, filter?: Document): any[] {
-    let docs = documents;
-    if (filter && Object.keys(filter).length > 0) {
-      docs = this.filter(documents, filter);
-    }
-
-    const values = new Set<any>();
-    for (const doc of docs) {
-      const value = this.getNestedValue(doc, field);
-      if (value !== undefined) {
-        if (Array.isArray(value)) {
-          // For arrays, add each element
-          for (const v of value) {
-            values.add(this.toComparable(v));
-          }
-        } else {
-          values.add(this.toComparable(value));
-        }
-      }
-    }
-
-    return Array.from(values);
-  }
-
-  /**
-   * Normalize sort direction to 1 or -1
-   */
-  private static normalizeDirection(direction: ISortDirection): number {
-    if (typeof direction === 'number') {
-      return direction > 0 ? 1 : -1;
-    }
-    if (direction === 'asc' || direction === 'ascending') {
-      return 1;
-    }
-    return -1;
-  }
-
-  /**
-   * Get a nested value from an object using dot notation
-   */
-  static getNestedValue(obj: any, path: string): any {
-    const parts = path.split('.');
-    let current = obj;
-
-    for (const part of parts) {
-      if (current === null || current === undefined) {
-        return undefined;
-      }
-      if (Array.isArray(current)) {
-        // Handle array access
-        const index = parseInt(part, 10);
-        if (!isNaN(index)) {
-          current = current[index];
-        } else {
-          // Get the field from all array elements
-          return current.map(item => this.getNestedValue(item, part)).flat();
-        }
-      } else {
-        current = current[part];
-      }
-    }
-
-    return current;
-  }
-
-  /**
-   * Set a nested value in an object using dot notation
-   */
-  private static setNestedValue(obj: any, path: string, value: any): void {
-    const parts = path.split('.');
-    let current = obj;
-
-    for (let i = 0; i < parts.length - 1; i++) {
-      const part = parts[i];
-      if (!(part in current)) {
-        current[part] = {};
-      }
-      current = current[part];
-    }
-
-    current[parts[parts.length - 1]] = value;
-  }
-
-  /**
-   * Delete a nested value from an object using dot notation
-   */
-  private static deleteNestedValue(obj: any, path: string): void {
-    const parts = path.split('.');
-    let current = obj;
-
-    for (let i = 0; i < parts.length - 1; i++) {
-      const part = parts[i];
-      if (!(part in current)) {
-        return;
-      }
-      current = current[part];
-    }
-
-    delete current[parts[parts.length - 1]];
-  }
-
-  /**
-   * Compare two values for sorting
-   */
-  private static compareValues(a: any, b: any): number {
-    // Handle undefined/null
-    if (a === undefined && b === undefined) return 0;
-    if (a === undefined) return -1;
-    if (b === undefined) return 1;
-    if (a === null && b === null) return 0;
-    if (a === null) return -1;
-    if (b === null) return 1;
-
-    // Handle ObjectId
-    if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
-      return a.toHexString().localeCompare(b.toHexString());
-    }
-
-    // Handle dates
-    if (a instanceof Date && b instanceof Date) {
-      return a.getTime() - b.getTime();
-    }
-
-    // Handle numbers
-    if (typeof a === 'number' && typeof b === 'number') {
-      return a - b;
-    }
-
-    // Handle strings
-    if (typeof a === 'string' && typeof b === 'string') {
-      return a.localeCompare(b);
-    }
-
-    // Handle booleans
-    if (typeof a === 'boolean' && typeof b === 'boolean') {
-      return (a ? 1 : 0) - (b ? 1 : 0);
-    }
-
-    // Fall back to string comparison
-    return String(a).localeCompare(String(b));
-  }
-
-  /**
-   * Convert a value to a comparable form (for distinct)
-   */
-  private static toComparable(value: any): any {
-    if (value instanceof plugins.bson.ObjectId) {
-      return value.toHexString();
-    }
-    if (value instanceof Date) {
-      return value.toISOString();
-    }
-    if (typeof value === 'object' && value !== null) {
-      return JSON.stringify(value);
-    }
-    return value;
-  }
-}

package/ts/ts_smartdb/engine/QueryPlanner.ts

@@ -1,393 +0,0 @@
-import * as plugins from '../plugins.js';
-import type { Document, IStoredDocument } from '../types/interfaces.js';
-import { IndexEngine } from './IndexEngine.js';
-
-/**
- * Query execution plan types
- */
-export type TQueryPlanType = 'IXSCAN' | 'COLLSCAN' | 'FETCH' | 'IXSCAN_RANGE';
-
-/**
- * Represents a query execution plan
- */
-export interface IQueryPlan {
-  /** The type of scan used */
-  type: TQueryPlanType;
-  /** Index name if using an index */
-  indexName?: string;
-  /** Index key specification */
-  indexKey?: Record<string, 1 | -1 | string>;
-  /** Whether the query can be fully satisfied by the index */
-  indexCovering: boolean;
-  /** Estimated selectivity (0-1, lower is more selective) */
-  selectivity: number;
-  /** Whether range operators are used */
-  usesRange: boolean;
-  /** Fields used from the index */
-  indexFieldsUsed: string[];
-  /** Filter conditions that must be applied post-index lookup */
-  residualFilter?: Document;
-  /** Explanation for debugging */
-  explanation: string;
-}
-
-/**
- * Filter operator analysis
- */
-interface IFilterOperatorInfo {
-  field: string;
-  operators: string[];
-  equality: boolean;
-  range: boolean;
-  in: boolean;
-  exists: boolean;
-  regex: boolean;
-  values: Record<string, any>;
-}
-
-/**
- * QueryPlanner - Analyzes queries and selects optimal execution plans
- */
-export class QueryPlanner {
-  private indexEngine: IndexEngine;
-
-  constructor(indexEngine: IndexEngine) {
-    this.indexEngine = indexEngine;
-  }
-
-  /**
-   * Generate an execution plan for a query filter
-   */
-  async plan(filter: Document): Promise<IQueryPlan> {
-    await this.indexEngine['initialize']();
-
-    // Empty filter = full collection scan
-    if (!filter || Object.keys(filter).length === 0) {
-      return {
-        type: 'COLLSCAN',
-        indexCovering: false,
-        selectivity: 1.0,
-        usesRange: false,
-        indexFieldsUsed: [],
-        explanation: 'No filter specified, full collection scan required',
-      };
-    }
-
-    // Analyze the filter
-    const operatorInfo = this.analyzeFilter(filter);
-
-    // Get available indexes
-    const indexes = await this.indexEngine.listIndexes();
-
-    // Score each index
-    let bestPlan: IQueryPlan | null = null;
-    let bestScore = -1;
-
-    for (const index of indexes) {
-      const plan = this.scoreIndex(index, operatorInfo, filter);
-      if (plan.selectivity < 1.0) {
-        const score = this.calculateScore(plan);
-        if (score > bestScore) {
-          bestScore = score;
-          bestPlan = plan;
-        }
-      }
-    }
-
-    // If no suitable index found, fall back to collection scan
-    if (!bestPlan || bestScore <= 0) {
-      return {
-        type: 'COLLSCAN',
-        indexCovering: false,
-        selectivity: 1.0,
-        usesRange: false,
-        indexFieldsUsed: [],
-        explanation: 'No suitable index found for this query',
-      };
-    }
-
-    return bestPlan;
-  }
-
-  /**
-   * Analyze filter to extract operator information per field
-   */
-  private analyzeFilter(filter: Document, prefix = ''): Map<string, IFilterOperatorInfo> {
-    const result = new Map<string, IFilterOperatorInfo>();
-
-    for (const [key, value] of Object.entries(filter)) {
-      // Skip logical operators at the top level
-      if (key.startsWith('$')) {
-        if (key === '$and' && Array.isArray(value)) {
-          // Merge $and conditions
-          for (const subFilter of value) {
-            const subInfo = this.analyzeFilter(subFilter, prefix);
-            for (const [field, info] of subInfo) {
-              if (result.has(field)) {
-                // Merge operators
-                const existing = result.get(field)!;
-                existing.operators.push(...info.operators);
-                existing.equality = existing.equality || info.equality;
-                existing.range = existing.range || info.range;
-                existing.in = existing.in || info.in;
-                Object.assign(existing.values, info.values);
-              } else {
-                result.set(field, info);
-              }
-            }
-          }
-        }
-        continue;
-      }
-
-      const fullKey = prefix ? `${prefix}.${key}` : key;
-      const info: IFilterOperatorInfo = {
-        field: fullKey,
-        operators: [],
-        equality: false,
-        range: false,
-        in: false,
-        exists: false,
-        regex: false,
-        values: {},
-      };
-
-      if (typeof value !== 'object' || value === null || value instanceof plugins.bson.ObjectId || value instanceof Date) {
-        // Direct equality
-        info.equality = true;
-        info.operators.push('$eq');
-        info.values['$eq'] = value;
-      } else if (Array.isArray(value)) {
-        // Array equality (rare, but possible)
-        info.equality = true;
-        info.operators.push('$eq');
-        info.values['$eq'] = value;
-      } else {
-        // Operator object
-        for (const [op, opValue] of Object.entries(value)) {
-          if (op.startsWith('$')) {
-            info.operators.push(op);
-            info.values[op] = opValue;
-
-            switch (op) {
-              case '$eq':
-                info.equality = true;
-                break;
-              case '$ne':
-              case '$not':
-                // These can use indexes but with low selectivity
-                break;
-              case '$in':
-                info.in = true;
-                break;
-              case '$nin':
-                // Can't efficiently use indexes
-                break;
-              case '$gt':
-              case '$gte':
-              case '$lt':
-              case '$lte':
-                info.range = true;
-                break;
-              case '$exists':
-                info.exists = true;
-                break;
-              case '$regex':
-                info.regex = true;
-                break;
-            }
-          } else {
-            // Nested object - recurse
-            const nestedInfo = this.analyzeFilter({ [op]: opValue }, fullKey);
-            for (const [nestedField, nestedFieldInfo] of nestedInfo) {
-              result.set(nestedField, nestedFieldInfo);
-            }
-          }
-        }
-      }
-
-      if (info.operators.length > 0) {
-        result.set(fullKey, info);
-      }
-    }
-
-    return result;
-  }
-
-  /**
-   * Score an index for the given filter
-   */
-  private scoreIndex(
-    index: { name: string; key: Record<string, any>; unique?: boolean; sparse?: boolean },
-    operatorInfo: Map<string, IFilterOperatorInfo>,
-    filter: Document
-  ): IQueryPlan {
-    const indexFields = Object.keys(index.key);
-    const usedFields: string[] = [];
-    let usesRange = false;
-    let canUseIndex = true;
-    let selectivity = 1.0;
-    let residualFilter: Document | undefined;
-
-    // Check each index field in order
-    for (const field of indexFields) {
-      const info = operatorInfo.get(field);
-      if (!info) {
-        // Index field not in filter - stop here
-        break;
-      }
-
-      usedFields.push(field);
-
-      // Calculate selectivity based on operator
-      if (info.equality) {
-        // Equality has high selectivity
-        selectivity *= 0.01; // Assume 1% match
-      } else if (info.in) {
-        // $in selectivity depends on array size
-        const inValues = info.values['$in'];
-        if (Array.isArray(inValues)) {
-          selectivity *= Math.min(0.5, inValues.length * 0.01);
-        } else {
-          selectivity *= 0.1;
-        }
-      } else if (info.range) {
-        // Range queries have moderate selectivity
-        selectivity *= 0.25;
-        usesRange = true;
-        // After range, can't use more index fields efficiently
-        break;
-      } else if (info.exists) {
-        // $exists can use sparse indexes
-        selectivity *= 0.5;
-      } else {
-        // Other operators may not be indexable
-        canUseIndex = false;
-        break;
-      }
-    }
-
-    if (!canUseIndex || usedFields.length === 0) {
-      return {
-        type: 'COLLSCAN',
-        indexCovering: false,
-        selectivity: 1.0,
-        usesRange: false,
-        indexFieldsUsed: [],
-        explanation: `Index ${index.name} cannot be used for this query`,
-      };
-    }
-
-    // Build residual filter for conditions not covered by index
-    const coveredFields = new Set(usedFields);
-    const residualConditions: Record<string, any> = {};
-    for (const [field, info] of operatorInfo) {
-      if (!coveredFields.has(field)) {
-        // This field isn't covered by the index
-        if (info.equality) {
-          residualConditions[field] = info.values['$eq'];
-        } else {
-          residualConditions[field] = info.values;
-        }
-      }
-    }
-
-    if (Object.keys(residualConditions).length > 0) {
-      residualFilter = residualConditions;
-    }
-
-    // Unique indexes have better selectivity for equality
-    if (index.unique && usedFields.length === indexFields.length) {
-      selectivity = Math.min(selectivity, 0.001); // At most 1 document
-    }
-
-    return {
-      type: usesRange ? 'IXSCAN_RANGE' : 'IXSCAN',
-      indexName: index.name,
-      indexKey: index.key,
-      indexCovering: Object.keys(residualConditions).length === 0,
-      selectivity,
-      usesRange,
-      indexFieldsUsed: usedFields,
-      residualFilter,
-      explanation: `Using index ${index.name} on fields [${usedFields.join(', ')}]`,
-    };
-  }
-
-  /**
-   * Calculate overall score for a plan (higher is better)
-   */
-  private calculateScore(plan: IQueryPlan): number {
-    let score = 0;
-
-    // Lower selectivity is better (fewer documents to fetch)
-    score += (1 - plan.selectivity) * 100;
-
-    // Index covering queries are best
-    if (plan.indexCovering) {
-      score += 50;
-    }
-
-    // More index fields used is better
-    score += plan.indexFieldsUsed.length * 10;
-
-    // Equality scans are better than range scans
-    if (!plan.usesRange) {
-      score += 20;
-    }
-
-    return score;
-  }
-
-  /**
-   * Explain a query - returns detailed plan information
-   */
-  async explain(filter: Document): Promise<{
-    queryPlanner: {
-      plannerVersion: number;
-      namespace: string;
-      indexFilterSet: boolean;
-      winningPlan: IQueryPlan;
-      rejectedPlans: IQueryPlan[];
-    };
-  }> {
-    await this.indexEngine['initialize']();
-
-    // Analyze the filter
-    const operatorInfo = this.analyzeFilter(filter);
-
-    // Get available indexes
-    const indexes = await this.indexEngine.listIndexes();
-
-    // Score all indexes
-    const plans: IQueryPlan[] = [];
-
-    for (const index of indexes) {
-      const plan = this.scoreIndex(index, operatorInfo, filter);
-      plans.push(plan);
-    }
-
-    // Add collection scan as fallback
-    plans.push({
-      type: 'COLLSCAN',
-      indexCovering: false,
-      selectivity: 1.0,
-      usesRange: false,
-      indexFieldsUsed: [],
-      explanation: 'Full collection scan',
-    });
-
-    // Sort by score (best first)
-    plans.sort((a, b) => this.calculateScore(b) - this.calculateScore(a));
-
-    return {
-      queryPlanner: {
-        plannerVersion: 1,
-        namespace: `${this.indexEngine['dbName']}.${this.indexEngine['collName']}`,
-        indexFilterSet: false,
-        winningPlan: plans[0],
-        rejectedPlans: plans.slice(1),
-      },
-    };
-  }
-}