@objectql/driver-memory 4.0.2 → 4.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectql/driver-memory",
-  "version": "4.0.2",
+  "version": "4.0.3",
   "description": "In-memory driver for ObjectQL - Fast MongoDB-like query engine powered by Mingo with DriverInterface v4.0 compliance",
   "keywords": [
     "objectql",
@@ -16,9 +16,9 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
-    "@objectstack/spec": "^0.3.3",
+    "@objectstack/spec": "^0.8.0",
     "mingo": "^7.1.1",
-    "@objectql/types": "4.0.2"
+    "@objectql/types": "4.0.3"
   },
   "devDependencies": {
     "@types/jest": "^29.0.0",

package/src/index.ts

@@ -1,7 +1,7 @@
-import { Data, Driver as DriverSpec } from '@objectstack/spec';
+import { Data, System as SystemSpec } from '@objectstack/spec';
 type QueryAST = Data.QueryAST;
 type SortNode = Data.SortNode;
-type DriverInterface = DriverSpec.DriverInterface;
+type DriverInterface = Data.DriverInterface;
 /**
  * ObjectQL
  * Copyright (c) 2026-present ObjectStack Inc.
@@ -38,7 +38,7 @@ type DriverInterface = DriverSpec.DriverInterface;
  */
 
 import { Driver, ObjectQLError } from '@objectql/types';
-import { Query } from 'mingo';
+import { Query, Aggregator } from 'mingo';
 
 /**
  * Command interface for executeCommand method
@@ -72,6 +72,24 @@ export interface MemoryDriverConfig {
     initialData?: Record<string, any[]>;
     /** Optional: Enable strict mode (throw on missing objects) */
     strictMode?: boolean;
+    /** Optional: Enable persistence to file system */
+    persistence?: {
+        /** File path to persist data */
+        filePath: string;
+        /** Auto-save interval in milliseconds (default: 5000) */
+        autoSaveInterval?: number;
+    };
+    /** Optional: Fields to index for faster queries */
+    indexes?: Record<string, string[]>; // objectName -> field names
+}
+
+/**
+ * In-memory transaction
+ */
+interface MemoryTransaction {
+    id: string;
+    snapshot: Map<string, any>;
+    operations: Array<{ type: 'set' | 'delete'; key: string; value?: any }>;
 }
 
 /**
@@ -87,32 +105,47 @@ export class MemoryDriver implements Driver {
     public readonly name = 'MemoryDriver';
     public readonly version = '4.0.0';
     public readonly supports = {
-        transactions: false,
+        transactions: true,
         joins: false,
         fullTextSearch: false,
         jsonFields: true,
         arrayFields: true,
         queryFilters: true,
-        queryAggregations: false,
+        queryAggregations: true,
         querySorting: true,
         queryPagination: true,
         queryWindowFunctions: false,
         querySubqueries: false
     };
 
-    private store: Map<string, any>;
-    private config: MemoryDriverConfig;
-    private idCounters: Map<string, number>;
+    protected store: Map<string, any>;
+    protected config: MemoryDriverConfig;
+    protected idCounters: Map<string, number>;
+    protected transactions: Map<string, MemoryTransaction>;
+    protected indexes: Map<string, Map<string, Set<string>>>; // objectName -> field -> Set<recordIds>
+    protected persistenceTimer?: NodeJS.Timeout;
 
     constructor(config: MemoryDriverConfig = {}) {
         this.config = config;
         this.store = new Map<string, any>();
         this.idCounters = new Map<string, number>();
+        this.transactions = new Map();
+        this.indexes = new Map();
         
         // Load initial data if provided
         if (config.initialData) {
             this.loadInitialData(config.initialData);
         }
+        
+        // Set up persistence if configured
+        if (config.persistence) {
+            this.setupPersistence();
+        }
+        
+        // Build indexes if configured
+        if (config.indexes) {
+            this.buildIndexes(config.indexes);
+        }
     }
 
     /**
@@ -134,7 +167,7 @@ export class MemoryDriver implements Driver {
     /**
      * Load initial data into the store.
      */
-    private loadInitialData(data: Record<string, any[]>): void {
+    protected loadInitialData(data: Record<string, any[]>): void {
         for (const [objectName, records] of Object.entries(data)) {
             for (const record of records) {
                 const id = record.id || this.generateId(objectName);
@@ -235,6 +268,10 @@ export class MemoryDriver implements Driver {
         };
         
         this.store.set(key, doc);
+        
+        // Update indexes
+        this.updateIndex(objectName, id, doc);
+        
         return { ...doc };
     }
 
@@ -265,6 +302,10 @@ export class MemoryDriver implements Driver {
         };
         
         this.store.set(key, doc);
+        
+        // Update indexes
+        this.updateIndex(objectName, id.toString(), doc);
+        
         return { ...doc };
     }
 
@@ -275,6 +316,11 @@ export class MemoryDriver implements Driver {
         const key = `${objectName}:${id}`;
         const deleted = this.store.delete(key);
         
+        // Remove from indexes
+        if (deleted) {
+            this.removeFromIndex(objectName, id.toString());
+        }
+        
         if (!deleted && this.config.strictMode) {
             throw new ObjectQLError({
                 code: 'RECORD_NOT_FOUND',
@@ -468,10 +514,259 @@ export class MemoryDriver implements Driver {
     }
 
     /**
-     * Disconnect (no-op for memory driver).
+     * Disconnect and cleanup resources.
      */
     async disconnect(): Promise<void> {
-        // No-op: Memory driver doesn't need cleanup
+        // Save data to disk if persistence is enabled
+        if (this.config.persistence) {
+            this.saveToDisk();
+            
+            // Clear the auto-save timer
+            if (this.persistenceTimer) {
+                clearInterval(this.persistenceTimer);
+                this.persistenceTimer = undefined;
+            }
+        }
+    }
+
+    /**
+     * Perform aggregation operations using Mingo
+     * @param objectName - The object type to aggregate
+     * @param pipeline - MongoDB-style aggregation pipeline
+     * @param options - Optional query options
+     * @returns Aggregation results
+     * 
+     * @example
+     * // Group by status and count
+     * const results = await driver.aggregate('orders', [
+     *   { $match: { status: 'completed' } },
+     *   { $group: { _id: '$customer', totalAmount: { $sum: '$amount' } } }
+     * ]);
+     * 
+     * @example
+     * // Calculate average with filter
+     * const results = await driver.aggregate('products', [
+     *   { $match: { category: 'electronics' } },
+     *   { $group: { _id: null, avgPrice: { $avg: '$price' } } }
+     * ]);
+     */
+    async aggregate(objectName: string, pipeline: any[], options?: any): Promise<any[]> {
+        const pattern = `${objectName}:`;
+        
+        // Get all records for this object type
+        let records: any[] = [];
+        for (const [key, value] of this.store.entries()) {
+            if (key.startsWith(pattern)) {
+                records.push({ ...value });
+            }
+        }
+        
+        // Use Mingo to execute the aggregation pipeline
+        const aggregator = new Aggregator(pipeline);
+        const results = aggregator.run(records);
+        
+        return results;
+    }
+
+    /**
+     * Begin a new transaction
+     * @returns Transaction object that can be passed to other methods
+     */
+    async beginTransaction(): Promise<any> {
+        const txId = `tx_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+        
+        // Create a deep snapshot of the current store to ensure complete transaction isolation
+        // This prevents issues with nested objects and arrays being modified during the transaction
+        const snapshot = new Map<string, any>();
+        for (const [key, value] of this.store.entries()) {
+            snapshot.set(key, JSON.parse(JSON.stringify(value)));
+        }
+        
+        const transaction: MemoryTransaction = {
+            id: txId,
+            snapshot,
+            operations: []
+        };
+        
+        this.transactions.set(txId, transaction);
+        
+        return { id: txId };
+    }
+
+    /**
+     * Commit a transaction
+     * @param transaction - Transaction object returned by beginTransaction()
+     */
+    async commitTransaction(transaction: any): Promise<void> {
+        const txId = transaction?.id;
+        if (!txId) {
+            throw new ObjectQLError({
+                code: 'INVALID_TRANSACTION',
+                message: 'Invalid transaction object'
+            });
+        }
+        
+        const tx = this.transactions.get(txId);
+        if (!tx) {
+            throw new ObjectQLError({
+                code: 'TRANSACTION_NOT_FOUND',
+                message: `Transaction ${txId} not found`
+            });
+        }
+        
+        // Operations are already applied to the store during the transaction
+        // Just clean up the transaction record
+        this.transactions.delete(txId);
+    }
+
+    /**
+     * Rollback a transaction
+     * @param transaction - Transaction object returned by beginTransaction()
+     */
+    async rollbackTransaction(transaction: any): Promise<void> {
+        const txId = transaction?.id;
+        if (!txId) {
+            throw new ObjectQLError({
+                code: 'INVALID_TRANSACTION',
+                message: 'Invalid transaction object'
+            });
+        }
+        
+        const tx = this.transactions.get(txId);
+        if (!tx) {
+            throw new ObjectQLError({
+                code: 'TRANSACTION_NOT_FOUND',
+                message: `Transaction ${txId} not found`
+            });
+        }
+        
+        // Restore the snapshot
+        this.store = new Map(tx.snapshot);
+        
+        // Clean up the transaction
+        this.transactions.delete(txId);
+    }
+
+    /**
+     * Set up persistence to file system
+     * @private
+     */
+    protected setupPersistence(): void {
+        if (!this.config.persistence) return;
+        
+        const { filePath, autoSaveInterval = 5000 } = this.config.persistence;
+        
+        // Try to load existing data from file
+        try {
+            const fs = require('fs');
+            if (fs.existsSync(filePath)) {
+                const fileData = fs.readFileSync(filePath, 'utf8');
+                const data = JSON.parse(fileData);
+                
+                // Load data into store
+                for (const [key, value] of Object.entries(data.store || {})) {
+                    this.store.set(key, value);
+                }
+                
+                // Load ID counters
+                if (data.idCounters) {
+                    for (const [key, value] of Object.entries(data.idCounters)) {
+                        this.idCounters.set(key, value as number);
+                    }
+                }
+            }
+        } catch (error) {
+            console.warn('[MemoryDriver] Failed to load persisted data:', error);
+        }
+        
+        // Set up auto-save timer
+        // Use unref() to allow Node.js process to exit gracefully when idle
+        this.persistenceTimer = setInterval(() => {
+            this.saveToDisk();
+        }, autoSaveInterval);
+        this.persistenceTimer.unref();
+    }
+
+    /**
+     * Save current state to disk
+     * @private
+     */
+    protected saveToDisk(): void {
+        if (!this.config.persistence) return;
+        
+        try {
+            const fs = require('fs');
+            const data = {
+                store: Object.fromEntries(this.store),
+                idCounters: Object.fromEntries(this.idCounters),
+                timestamp: new Date().toISOString()
+            };
+            
+            fs.writeFileSync(this.config.persistence.filePath, JSON.stringify(data, null, 2), 'utf8');
+        } catch (error) {
+            console.error('[MemoryDriver] Failed to save data to disk:', error);
+        }
+    }
+
+    /**
+     * Build indexes for faster queries
+     * @private
+     */
+    protected buildIndexes(indexConfig: Record<string, string[]>): void {
+        for (const [objectName, fields] of Object.entries(indexConfig)) {
+            const objectIndexes = new Map<string, Set<string>>();
+            
+            for (const field of fields) {
+                const fieldIndex = new Set<string>();
+                
+                // Scan all records of this object type
+                const pattern = `${objectName}:`;
+                for (const [key, record] of this.store.entries()) {
+                    if (key.startsWith(pattern)) {
+                        const fieldValue = record[field];
+                        if (fieldValue !== undefined && fieldValue !== null) {
+                            // Store the record ID in the index
+                            fieldIndex.add(record.id);
+                        }
+                    }
+                }
+                
+                objectIndexes.set(field, fieldIndex);
+            }
+            
+            this.indexes.set(objectName, objectIndexes);
+        }
+    }
+
+    /**
+     * Update index when a record is created or updated
+     * @private
+     */
+    protected updateIndex(objectName: string, recordId: string, record: any): void {
+        const objectIndexes = this.indexes.get(objectName);
+        if (!objectIndexes) return;
+        
+        for (const [field, indexSet] of objectIndexes.entries()) {
+            const fieldValue = record[field];
+            if (fieldValue !== undefined && fieldValue !== null) {
+                indexSet.add(recordId);
+            } else {
+                indexSet.delete(recordId);
+            }
+        }
+    }
+
+    /**
+     * Remove record from indexes
+     * @private
+     */
+    protected removeFromIndex(objectName: string, recordId: string): void {
+        const objectIndexes = this.indexes.get(objectName);
+        if (!objectIndexes) return;
+        
+        for (const indexSet of objectIndexes.values()) {
+            indexSet.delete(recordId);
+        }
     }
 
     // ========== Helper Methods ==========
@@ -495,13 +790,46 @@ export class MemoryDriver implements Driver {
      * Converts to MongoDB query format:
      * { $or: [{ field: { $operator: value }}, { field2: { $operator: value2 }}] }
      */
-    private convertToMongoQuery(filters?: any[] | Record<string, any>): Record<string, any> {
+    /**
+     * Convert ObjectQL filter format to MongoDB query format.
+     * Supports three input formats:
+     * 
+     * 1. AST Comparison Node: { type: 'comparison', field: string, operator: string, value: any }
+     * 2. AST Logical Node: { type: 'logical', operator: 'and' | 'or', conditions: Node[] }
+     * 3. Legacy Array Format: [field, operator, value, 'and', field2, operator2, value2]
+     * 4. MongoDB Format: { field: value } or { field: { $eq: value } } (passthrough)
+     * 
+     * @param filters - Filter in any supported format
+     * @returns MongoDB query object
+     */
+    protected convertToMongoQuery(filters?: any[] | Record<string, any>): Record<string, any> {
         if (!filters) {
             return {};
         }
         
-        // If filters is already an object (FilterCondition format), return it directly
-        if (!Array.isArray(filters)) {
+        // Handle AST node format (ObjectQL QueryAST)
+        if (!Array.isArray(filters) && typeof filters === 'object') {
+            // AST Comparison Node: { type: 'comparison', field, operator, value }
+            if (filters.type === 'comparison') {
+                const mongoCondition = this.convertConditionToMongo(filters.field, filters.operator, filters.value);
+                return mongoCondition || {};
+            } 
+            
+            // AST Logical Node: { type: 'logical', operator: 'and' | 'or', conditions: [...] }
+            else if (filters.type === 'logical') {
+                const conditions = filters.conditions?.map((cond: any) => this.convertToMongoQuery(cond)) || [];
+                if (conditions.length === 0) {
+                    return {};
+                }
+                if (conditions.length === 1) {
+                    return conditions[0];
+                }
+                const operator = filters.operator === 'or' ? '$or' : '$and';
+                return { [operator]: conditions };
+            }
+            
+            // MongoDB format (passthrough): { field: value } or { field: { $eq: value } }
+            // This handles cases where filters is already a proper MongoDB query
             return filters;
         }
         
@@ -585,7 +913,7 @@ export class MemoryDriver implements Driver {
     /**
      * Convert a single ObjectQL condition to MongoDB operator format.
      */
-    private convertConditionToMongo(field: string, operator: string, value: any): Record<string, any> | null {
+    protected convertConditionToMongo(field: string, operator: string, value: any): Record<string, any> | null {
         switch (operator) {
             case '=':
             case '==':
@@ -646,7 +974,7 @@ export class MemoryDriver implements Driver {
      * Escape special regex characters to prevent ReDoS and ensure literal matching.
      * This is crucial for security when using user input in regex patterns.
      */
-    private escapeRegex(str: string): string {
+    protected escapeRegex(str: string): string {
         return String(str).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
     }
 
@@ -654,7 +982,7 @@ export class MemoryDriver implements Driver {
      * Apply manual sorting to an array of records.
      * This is used instead of Mingo's sort to avoid CJS build issues.
      */
-    private applyManualSort(records: any[], sort: any[]): any[] {
+    protected applyManualSort(records: any[], sort: any[]): any[] {
         const sorted = [...records];
         
         // Apply sorts in reverse order for correct multi-field precedence
@@ -695,7 +1023,7 @@ export class MemoryDriver implements Driver {
     /**
      * Project specific fields from a document.
      */
-    private projectFields(doc: any, fields: string[]): any {
+    protected projectFields(doc: any, fields: string[]): any {
         const result: any = {};
         for (const field of fields) {
             if (doc[field] !== undefined) {
@@ -708,7 +1036,7 @@ export class MemoryDriver implements Driver {
     /**
      * Generate a unique ID for a record.
      */
-    private generateId(objectName: string): string {
+    protected generateId(objectName: string): string {
         const counter = (this.idCounters.get(objectName) || 0) + 1;
         this.idCounters.set(objectName, counter);