@objectql/driver-memory 4.0.1 → 4.0.3

package/src/index.ts

@@ -1,8 +1,7 @@
-import { Data, System } from '@objectstack/spec';
+import { Data, System as SystemSpec } from '@objectstack/spec';
 type QueryAST = Data.QueryAST;
-type FilterNode = Data.FilterNode;
 type SortNode = Data.SortNode;
-type DriverInterface = System.DriverInterface;
+type DriverInterface = Data.DriverInterface;
 /**
  * ObjectQL
  * Copyright (c) 2026-present ObjectStack Inc.
@@ -39,7 +38,7 @@ type DriverInterface = System.DriverInterface;
  */
 
 import { Driver, ObjectQLError } from '@objectql/types';
-import { Query } from 'mingo';
+import { Query, Aggregator } from 'mingo';
 
 /**
  * Command interface for executeCommand method
@@ -73,6 +72,24 @@ export interface MemoryDriverConfig {
     initialData?: Record<string, any[]>;
     /** Optional: Enable strict mode (throw on missing objects) */
     strictMode?: boolean;
+    /** Optional: Enable persistence to file system */
+    persistence?: {
+        /** File path to persist data */
+        filePath: string;
+        /** Auto-save interval in milliseconds (default: 5000) */
+        autoSaveInterval?: number;
+    };
+    /** Optional: Fields to index for faster queries */
+    indexes?: Record<string, string[]>; // objectName -> field names
+}
+
+/**
+ * In-memory transaction
+ */
+interface MemoryTransaction {
+    id: string;
+    snapshot: Map<string, any>;
+    operations: Array<{ type: 'set' | 'delete'; key: string; value?: any }>;
 }
 
 /**
@@ -88,32 +105,47 @@ export class MemoryDriver implements Driver {
     public readonly name = 'MemoryDriver';
     public readonly version = '4.0.0';
     public readonly supports = {
-        transactions: false,
+        transactions: true,
         joins: false,
         fullTextSearch: false,
         jsonFields: true,
         arrayFields: true,
         queryFilters: true,
-        queryAggregations: false,
+        queryAggregations: true,
         querySorting: true,
         queryPagination: true,
         queryWindowFunctions: false,
         querySubqueries: false
     };
 
-    private store: Map<string, any>;
-    private config: MemoryDriverConfig;
-    private idCounters: Map<string, number>;
+    protected store: Map<string, any>;
+    protected config: MemoryDriverConfig;
+    protected idCounters: Map<string, number>;
+    protected transactions: Map<string, MemoryTransaction>;
+    protected indexes: Map<string, Map<string, Set<string>>>; // objectName -> field -> Set<recordIds>
+    protected persistenceTimer?: NodeJS.Timeout;
 
     constructor(config: MemoryDriverConfig = {}) {
         this.config = config;
         this.store = new Map<string, any>();
         this.idCounters = new Map<string, number>();
+        this.transactions = new Map();
+        this.indexes = new Map();
         
         // Load initial data if provided
         if (config.initialData) {
             this.loadInitialData(config.initialData);
         }
+        
+        // Set up persistence if configured
+        if (config.persistence) {
+            this.setupPersistence();
+        }
+        
+        // Build indexes if configured
+        if (config.indexes) {
+            this.buildIndexes(config.indexes);
+        }
     }
 
     /**
@@ -135,7 +167,7 @@ export class MemoryDriver implements Driver {
     /**
      * Load initial data into the store.
      */
-    private loadInitialData(data: Record<string, any[]>): void {
+    protected loadInitialData(data: Record<string, any[]>): void {
         for (const [objectName, records] of Object.entries(data)) {
             for (const record of records) {
                 const id = record.id || this.generateId(objectName);
@@ -150,9 +182,6 @@ export class MemoryDriver implements Driver {
      * Supports filtering, sorting, pagination, and field projection using Mingo.
      */
     async find(objectName: string, query: any = {}, options?: any): Promise<any[]> {
-        // Normalize query to support both legacy and QueryAST formats
-        const normalizedQuery = this.normalizeQuery(query);
-        
         // Get all records for this object type
         const pattern = `${objectName}:`;
         let records: any[] = [];
@@ -164,7 +193,7 @@ export class MemoryDriver implements Driver {
         }
         
         // Convert ObjectQL filters to MongoDB query format
-        const mongoQuery = this.convertToMongoQuery(normalizedQuery.filters);
+        const mongoQuery = this.convertToMongoQuery(query.where);
         
         // Apply filters using Mingo
         if (mongoQuery && Object.keys(mongoQuery).length > 0) {
@@ -173,21 +202,21 @@ export class MemoryDriver implements Driver {
         }
         
         // Apply sorting manually (Mingo's sort has issues with CJS builds)
-        if (normalizedQuery.sort && Array.isArray(normalizedQuery.sort) && normalizedQuery.sort.length > 0) {
-            records = this.applyManualSort(records, normalizedQuery.sort);
+        if (query.orderBy && Array.isArray(query.orderBy) && query.orderBy.length > 0) {
+            records = this.applyManualSort(records, query.orderBy);
         }
         
         // Apply pagination
-        if (normalizedQuery.skip) {
-            records = records.slice(normalizedQuery.skip);
+        if (query.offset) {
+            records = records.slice(query.offset);
         }
-        if (normalizedQuery.limit) {
-            records = records.slice(0, normalizedQuery.limit);
+        if (query.limit) {
+            records = records.slice(0, query.limit);
         }
         
         // Apply field projection
-        if (normalizedQuery.fields && Array.isArray(normalizedQuery.fields)) {
-            records = records.map(doc => this.projectFields(doc, normalizedQuery.fields));
+        if (query.fields && Array.isArray(query.fields)) {
+            records = records.map(doc => this.projectFields(doc, query.fields));
         }
         
         return records;
@@ -239,6 +268,10 @@ export class MemoryDriver implements Driver {
         };
         
         this.store.set(key, doc);
+        
+        // Update indexes
+        this.updateIndex(objectName, id, doc);
+        
         return { ...doc };
     }
 
@@ -269,6 +302,10 @@ export class MemoryDriver implements Driver {
         };
         
         this.store.set(key, doc);
+        
+        // Update indexes
+        this.updateIndex(objectName, id.toString(), doc);
+        
         return { ...doc };
     }
 
@@ -279,6 +316,11 @@ export class MemoryDriver implements Driver {
         const key = `${objectName}:${id}`;
         const deleted = this.store.delete(key);
         
+        // Remove from indexes
+        if (deleted) {
+            this.removeFromIndex(objectName, id.toString());
+        }
+        
         if (!deleted && this.config.strictMode) {
             throw new ObjectQLError({
                 code: 'RECORD_NOT_FOUND',
@@ -296,10 +338,10 @@ export class MemoryDriver implements Driver {
     async count(objectName: string, filters: any, options?: any): Promise<number> {
         const pattern = `${objectName}:`;
         
-        // Extract actual filters from query object if needed
-        let actualFilters = filters;
-        if (filters && !Array.isArray(filters) && filters.filters) {
-            actualFilters = filters.filters;
+        // Extract where condition from query object if needed
+        let whereCondition = filters;
+        if (filters && !Array.isArray(filters) && filters.where) {
+            whereCondition = filters.where;
         }
         
         // Get all records for this object type
@@ -311,12 +353,12 @@ export class MemoryDriver implements Driver {
         }
         
         // If no filters, return total count
-        if (!actualFilters || (Array.isArray(actualFilters) && actualFilters.length === 0)) {
+        if (!whereCondition || (Array.isArray(whereCondition) && whereCondition.length === 0)) {
             return records.length;
         }
         
         // Convert to MongoDB query and use Mingo to count
-        const mongoQuery = this.convertToMongoQuery(actualFilters);
+        const mongoQuery = this.convertToMongoQuery(whereCondition);
         if (mongoQuery && Object.keys(mongoQuery).length > 0) {
             const mingoQuery = new Query(mongoQuery);
             const matchedRecords = mingoQuery.find(records).all();
@@ -472,62 +514,327 @@ export class MemoryDriver implements Driver {
     }
 
     /**
-     * Disconnect (no-op for memory driver).
+     * Disconnect and cleanup resources.
      */
     async disconnect(): Promise<void> {
-        // No-op: Memory driver doesn't need cleanup
+        // Save data to disk if persistence is enabled
+        if (this.config.persistence) {
+            this.saveToDisk();
+            
+            // Clear the auto-save timer
+            if (this.persistenceTimer) {
+                clearInterval(this.persistenceTimer);
+                this.persistenceTimer = undefined;
+            }
+        }
     }
 
-    // ========== Helper Methods ==========
-
     /**
-     * Normalizes query format to support both legacy UnifiedQuery and QueryAST formats.
-     * This ensures backward compatibility while supporting the new @objectstack/spec interface.
+     * Perform aggregation operations using Mingo
+     * @param objectName - The object type to aggregate
+     * @param pipeline - MongoDB-style aggregation pipeline
+     * @param options - Optional query options
+     * @returns Aggregation results
      * 
-     * QueryAST format uses 'top' for limit, while UnifiedQuery uses 'limit'.
-     * QueryAST sort is array of {field, order}, while UnifiedQuery is array of [field, order].
+     * @example
+     * // Group by status and count
+     * const results = await driver.aggregate('orders', [
+     *   { $match: { status: 'completed' } },
+     *   { $group: { _id: '$customer', totalAmount: { $sum: '$amount' } } }
+     * ]);
+     * 
+     * @example
+     * // Calculate average with filter
+     * const results = await driver.aggregate('products', [
+     *   { $match: { category: 'electronics' } },
+     *   { $group: { _id: null, avgPrice: { $avg: '$price' } } }
+     * ]);
+     */
+    async aggregate(objectName: string, pipeline: any[], options?: any): Promise<any[]> {
+        const pattern = `${objectName}:`;
+        
+        // Get all records for this object type
+        let records: any[] = [];
+        for (const [key, value] of this.store.entries()) {
+            if (key.startsWith(pattern)) {
+                records.push({ ...value });
+            }
+        }
+        
+        // Use Mingo to execute the aggregation pipeline
+        const aggregator = new Aggregator(pipeline);
+        const results = aggregator.run(records);
+        
+        return results;
+    }
+
+    /**
+     * Begin a new transaction
+     * @returns Transaction object that can be passed to other methods
      */
-    private normalizeQuery(query: any): any {
-        if (!query) return {};
+    async beginTransaction(): Promise<any> {
+        const txId = `tx_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+        
+        // Create a deep snapshot of the current store to ensure complete transaction isolation
+        // This prevents issues with nested objects and arrays being modified during the transaction
+        const snapshot = new Map<string, any>();
+        for (const [key, value] of this.store.entries()) {
+            snapshot.set(key, JSON.parse(JSON.stringify(value)));
+        }
+        
+        const transaction: MemoryTransaction = {
+            id: txId,
+            snapshot,
+            operations: []
+        };
+        
+        this.transactions.set(txId, transaction);
+        
+        return { id: txId };
+    }
+
+    /**
+     * Commit a transaction
+     * @param transaction - Transaction object returned by beginTransaction()
+     */
+    async commitTransaction(transaction: any): Promise<void> {
+        const txId = transaction?.id;
+        if (!txId) {
+            throw new ObjectQLError({
+                code: 'INVALID_TRANSACTION',
+                message: 'Invalid transaction object'
+            });
+        }
+        
+        const tx = this.transactions.get(txId);
+        if (!tx) {
+            throw new ObjectQLError({
+                code: 'TRANSACTION_NOT_FOUND',
+                message: `Transaction ${txId} not found`
+            });
+        }
+        
+        // Operations are already applied to the store during the transaction
+        // Just clean up the transaction record
+        this.transactions.delete(txId);
+    }
+
+    /**
+     * Rollback a transaction
+     * @param transaction - Transaction object returned by beginTransaction()
+     */
+    async rollbackTransaction(transaction: any): Promise<void> {
+        const txId = transaction?.id;
+        if (!txId) {
+            throw new ObjectQLError({
+                code: 'INVALID_TRANSACTION',
+                message: 'Invalid transaction object'
+            });
+        }
+        
+        const tx = this.transactions.get(txId);
+        if (!tx) {
+            throw new ObjectQLError({
+                code: 'TRANSACTION_NOT_FOUND',
+                message: `Transaction ${txId} not found`
+            });
+        }
+        
+        // Restore the snapshot
+        this.store = new Map(tx.snapshot);
+        
+        // Clean up the transaction
+        this.transactions.delete(txId);
+    }
+
+    /**
+     * Set up persistence to file system
+     * @private
+     */
+    protected setupPersistence(): void {
+        if (!this.config.persistence) return;
+        
+        const { filePath, autoSaveInterval = 5000 } = this.config.persistence;
+        
+        // Try to load existing data from file
+        try {
+            const fs = require('fs');
+            if (fs.existsSync(filePath)) {
+                const fileData = fs.readFileSync(filePath, 'utf8');
+                const data = JSON.parse(fileData);
+                
+                // Load data into store
+                for (const [key, value] of Object.entries(data.store || {})) {
+                    this.store.set(key, value);
+                }
+                
+                // Load ID counters
+                if (data.idCounters) {
+                    for (const [key, value] of Object.entries(data.idCounters)) {
+                        this.idCounters.set(key, value as number);
+                    }
+                }
+            }
+        } catch (error) {
+            console.warn('[MemoryDriver] Failed to load persisted data:', error);
+        }
         
-        const normalized: any = { ...query };
+        // Set up auto-save timer
+        // Use unref() to allow Node.js process to exit gracefully when idle
+        this.persistenceTimer = setInterval(() => {
+            this.saveToDisk();
+        }, autoSaveInterval);
+        this.persistenceTimer.unref();
+    }
+
+    /**
+     * Save current state to disk
+     * @private
+     */
+    protected saveToDisk(): void {
+        if (!this.config.persistence) return;
         
-        // Normalize limit/top
-        if (normalized.top !== undefined && normalized.limit === undefined) {
-            normalized.limit = normalized.top;
+        try {
+            const fs = require('fs');
+            const data = {
+                store: Object.fromEntries(this.store),
+                idCounters: Object.fromEntries(this.idCounters),
+                timestamp: new Date().toISOString()
+            };
+            
+            fs.writeFileSync(this.config.persistence.filePath, JSON.stringify(data, null, 2), 'utf8');
+        } catch (error) {
+            console.error('[MemoryDriver] Failed to save data to disk:', error);
+        }
+    }
+
+    /**
+     * Build indexes for faster queries
+     * @private
+     */
+    protected buildIndexes(indexConfig: Record<string, string[]>): void {
+        for (const [objectName, fields] of Object.entries(indexConfig)) {
+            const objectIndexes = new Map<string, Set<string>>();
+            
+            for (const field of fields) {
+                const fieldIndex = new Set<string>();
+                
+                // Scan all records of this object type
+                const pattern = `${objectName}:`;
+                for (const [key, record] of this.store.entries()) {
+                    if (key.startsWith(pattern)) {
+                        const fieldValue = record[field];
+                        if (fieldValue !== undefined && fieldValue !== null) {
+                            // Store the record ID in the index
+                            fieldIndex.add(record.id);
+                        }
+                    }
+                }
+                
+                objectIndexes.set(field, fieldIndex);
+            }
+            
+            this.indexes.set(objectName, objectIndexes);
         }
+    }
+
+    /**
+     * Update index when a record is created or updated
+     * @private
+     */
+    protected updateIndex(objectName: string, recordId: string, record: any): void {
+        const objectIndexes = this.indexes.get(objectName);
+        if (!objectIndexes) return;
         
-        // Normalize sort format
-        if (normalized.sort && Array.isArray(normalized.sort)) {
-            // Check if it's already in the array format [field, order]
-            const firstSort = normalized.sort[0];
-            if (firstSort && typeof firstSort === 'object' && !Array.isArray(firstSort)) {
-                // Convert from QueryAST format {field, order} to internal format [field, order]
-                normalized.sort = normalized.sort.map((item: any) => [
-                    item.field,
-                    item.order || item.direction || item.dir || 'asc'
-                ]);
+        for (const [field, indexSet] of objectIndexes.entries()) {
+            const fieldValue = record[field];
+            if (fieldValue !== undefined && fieldValue !== null) {
+                indexSet.add(recordId);
+            } else {
+                indexSet.delete(recordId);
             }
         }
+    }
+
+    /**
+     * Remove record from indexes
+     * @private
+     */
+    protected removeFromIndex(objectName: string, recordId: string): void {
+        const objectIndexes = this.indexes.get(objectName);
+        if (!objectIndexes) return;
         
-        return normalized;
+        for (const indexSet of objectIndexes.values()) {
+            indexSet.delete(recordId);
+        }
     }
 
+    // ========== Helper Methods ==========
+
+    /**
+     * Normalizes query format to support both legacy UnifiedQuery and QueryAST formats.
+     * This ensures backward compatibility while supporting the new @objectstack/spec interface.
+     * 
+     * QueryAST format uses 'top' for limit, while UnifiedQuery uses 'limit'.
+     * QueryAST sort is array of {field, order}, while UnifiedQuery is array of [field, order].
+     */
     /**
      * Convert ObjectQL filters to MongoDB query format for Mingo.
      * 
-     * Supports ObjectQL filter format:
-     * [
-     *   ['field', 'operator', value],
-     *   'or',
-     *   ['field2', 'operator', value2]
-     * ]
+     * Supports both:
+     * 1. Legacy ObjectQL filter format (array):
+     *    [['field', 'operator', value], 'or', ['field2', 'operator', value2']]
+     * 2. New FilterCondition format (object - already MongoDB-like):
+     *    { $and: [{ field: { $eq: value }}, { field2: { $gt: value2 }}] }
      * 
      * Converts to MongoDB query format:
      * { $or: [{ field: { $operator: value }}, { field2: { $operator: value2 }}] }
      */
-    private convertToMongoQuery(filters?: any[]): Record<string, any> {
-        if (!filters || filters.length === 0) {
+    /**
+     * Convert ObjectQL filter format to MongoDB query format.
+     * Supports three input formats:
+     * 
+     * 1. AST Comparison Node: { type: 'comparison', field: string, operator: string, value: any }
+     * 2. AST Logical Node: { type: 'logical', operator: 'and' | 'or', conditions: Node[] }
+     * 3. Legacy Array Format: [field, operator, value, 'and', field2, operator2, value2]
+     * 4. MongoDB Format: { field: value } or { field: { $eq: value } } (passthrough)
+     * 
+     * @param filters - Filter in any supported format
+     * @returns MongoDB query object
+     */
+    protected convertToMongoQuery(filters?: any[] | Record<string, any>): Record<string, any> {
+        if (!filters) {
+            return {};
+        }
+        
+        // Handle AST node format (ObjectQL QueryAST)
+        if (!Array.isArray(filters) && typeof filters === 'object') {
+            // AST Comparison Node: { type: 'comparison', field, operator, value }
+            if (filters.type === 'comparison') {
+                const mongoCondition = this.convertConditionToMongo(filters.field, filters.operator, filters.value);
+                return mongoCondition || {};
+            } 
+            
+            // AST Logical Node: { type: 'logical', operator: 'and' | 'or', conditions: [...] }
+            else if (filters.type === 'logical') {
+                const conditions = filters.conditions?.map((cond: any) => this.convertToMongoQuery(cond)) || [];
+                if (conditions.length === 0) {
+                    return {};
+                }
+                if (conditions.length === 1) {
+                    return conditions[0];
+                }
+                const operator = filters.operator === 'or' ? '$or' : '$and';
+                return { [operator]: conditions };
+            }
+            
+            // MongoDB format (passthrough): { field: value } or { field: { $eq: value } }
+            // This handles cases where filters is already a proper MongoDB query
+            return filters;
+        }
+        
+        // Handle legacy array format
+        if (filters.length === 0) {
             return {};
         }
         
@@ -606,7 +913,7 @@ export class MemoryDriver implements Driver {
     /**
      * Convert a single ObjectQL condition to MongoDB operator format.
      */
-    private convertConditionToMongo(field: string, operator: string, value: any): Record<string, any> | null {
+    protected convertConditionToMongo(field: string, operator: string, value: any): Record<string, any> | null {
         switch (operator) {
             case '=':
             case '==':
@@ -667,7 +974,7 @@ export class MemoryDriver implements Driver {
      * Escape special regex characters to prevent ReDoS and ensure literal matching.
      * This is crucial for security when using user input in regex patterns.
      */
-    private escapeRegex(str: string): string {
+    protected escapeRegex(str: string): string {
         return String(str).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
     }
 
@@ -675,7 +982,7 @@ export class MemoryDriver implements Driver {
      * Apply manual sorting to an array of records.
      * This is used instead of Mingo's sort to avoid CJS build issues.
      */
-    private applyManualSort(records: any[], sort: any[]): any[] {
+    protected applyManualSort(records: any[], sort: any[]): any[] {
         const sorted = [...records];
         
         // Apply sorts in reverse order for correct multi-field precedence
@@ -716,7 +1023,7 @@ export class MemoryDriver implements Driver {
     /**
      * Project specific fields from a document.
      */
-    private projectFields(doc: any, fields: string[]): any {
+    protected projectFields(doc: any, fields: string[]): any {
         const result: any = {};
         for (const field of fields) {
             if (doc[field] !== undefined) {
@@ -729,7 +1036,7 @@ export class MemoryDriver implements Driver {
     /**
      * Generate a unique ID for a record.
      */
-    private generateId(objectName: string): string {
+    protected generateId(objectName: string): string {
         const counter = (this.idCounters.get(objectName) || 0) + 1;
         this.idCounters.set(objectName, counter);
         
@@ -751,17 +1058,8 @@ export class MemoryDriver implements Driver {
     async executeQuery(ast: QueryAST, options?: any): Promise<{ value: any[]; count?: number }> {
         const objectName = ast.object || '';
         
-        // Convert QueryAST to legacy query format
-        const legacyQuery: any = {
-            fields: ast.fields,
-            filters: this.convertFilterNodeToLegacy(ast.filters),
-            sort: ast.sort?.map((s: SortNode) => [s.field, s.order]),
-            limit: ast.top,
-            offset: ast.skip,
-        };
-        
-        // Use existing find method
-        const results = await this.find(objectName, legacyQuery, options);
+        // Use existing find method with QueryAST directly
+        const results = await this.find(objectName, ast, options);
         
         return {
             value: results,
@@ -873,63 +1171,6 @@ export class MemoryDriver implements Driver {
         }
     }
 
-    /**
-     * Convert FilterNode (QueryAST format) to legacy filter array format
-     * This allows reuse of existing filter logic while supporting new QueryAST
-     * 
-     * @private
-     */
-    private convertFilterNodeToLegacy(node?: FilterNode): any {
-        if (!node) return undefined;
-        
-        switch (node.type) {
-            case 'comparison':
-                // Convert comparison node to [field, operator, value] format
-                const operator = node.operator || '=';
-                return [[node.field, operator, node.value]];
-            
-            case 'and':
-                // Convert AND node to array with 'and' separator
-                if (!node.children || node.children.length === 0) return undefined;
-                const andResults: any[] = [];
-                for (const child of node.children) {
-                    const converted = this.convertFilterNodeToLegacy(child);
-                    if (converted) {
-                        if (andResults.length > 0) {
-                            andResults.push('and');
-                        }
-                        andResults.push(...(Array.isArray(converted) ? converted : [converted]));
-                    }
-                }
-                return andResults.length > 0 ? andResults : undefined;
-            
-            case 'or':
-                // Convert OR node to array with 'or' separator
-                if (!node.children || node.children.length === 0) return undefined;
-                const orResults: any[] = [];
-                for (const child of node.children) {
-                    const converted = this.convertFilterNodeToLegacy(child);
-                    if (converted) {
-                        if (orResults.length > 0) {
-                            orResults.push('or');
-                        }
-                        orResults.push(...(Array.isArray(converted) ? converted : [converted]));
-                    }
-                }
-                return orResults.length > 0 ? orResults : undefined;
-            
-            case 'not':
-                // NOT is complex - we'll just process the first child for now
-                if (node.children && node.children.length > 0) {
-                    return this.convertFilterNodeToLegacy(node.children[0]);
-                }
-                return undefined;
-            
-            default:
-                return undefined;
-        }
-    }
-
     /**
      * Execute command (alternative signature for compatibility)
      *