jexidb 2.0.2 → 2.1.0

package/src/JSONLDatabase.js

@@ -1,1248 +0,0 @@
-/**
- * JSONLDatabase - JexiDB Core Database Engine
-* High Performance JSONL Database optimized for JexiDB
- * Optimized hybrid architecture combining the best strategies:
- * - Insert: Buffer + batch write for maximum speed
- * - Find: Intelligent hybrid (indexed + non-indexed fields)
- * - Update/Delete: On-demand reading/writing for scalability
- */
-import { promises as fs } from 'fs';
-import path from 'path';
-import { EventEmitter } from 'events';
-
-class JSONLDatabase extends EventEmitter {
-  constructor(filePath, options = {}) {
-    super();
-    
-    // Expect the main data file path (with .jdb extension)
-    if (!filePath.endsWith('.jdb')) {
-      if (filePath.endsWith('.jsonl')) {
-        this.filePath = filePath.replace('.jsonl', '.jdb');
-      } else if (filePath.endsWith('.json')) {
-        this.filePath = filePath.replace('.json', '.jdb');
-      } else {
-        // If no extension provided, assume it's a base name and add .jdb
-        this.filePath = filePath + '.jdb';
-      }
-    } else {
-      this.filePath = filePath;
-    }
-    
-    this.options = {
-      batchSize: 100, // Batch size for inserts
-      create: true, // Create database if it doesn't exist (default: true)
-      clear: false, // Clear database on load if not empty (default: false)
-      ...options
-    };
-    
-    // If clear is true, create should also be true
-    if (this.options.clear === true) {
-      this.options.create = true;
-    }
-    
-    this.isInitialized = false;
-    this.offsets = [];
-    this.indexOffset = 0;
-    this.shouldSave = false;
-    
-    // Ultra-optimized index structure (kept in memory)
-    this.indexes = {};
-    
-    // Initialize indexes from options or use defaults
-    if (options.indexes) {
-      for (const [field, type] of Object.entries(options.indexes)) {
-        this.indexes[field] = new Map();
-      }
-    } else {
-      // Default indexes
-      this.indexes = {
-        id: new Map(),
-        age: new Map(),
-        email: new Map()
-      };
-    }
-    
-    this.recordCount = 0;
-    this.fileHandle = null; // File handle for on-demand reading
-    
-    // Insert buffer (Original strategy)
-    this.insertionBuffer = [];
-    this.insertionStats = {
-      count: 0,
-      lastInsertion: Date.now(),
-      batchSize: this.options.batchSize
-    };
-  }
-
-  async init() {
-    if (this.isInitialized) {
-      // If already initialized, close first to reset state
-      await this.close();
-    }
-    
-    try {
-      const dir = path.dirname(this.filePath);
-      await fs.mkdir(dir, { recursive: true });
-      
-      // Check if file exists before loading
-      const fileExists = await fs.access(this.filePath).then(() => true).catch(() => false);
-      
-      // Handle clear option
-      if (this.options.clear && fileExists) {
-        await fs.writeFile(this.filePath, '');
-        this.offsets = [];
-        this.indexOffset = 0;
-        this.recordCount = 0;
-        console.log(`Database cleared: ${this.filePath}`);
-        this.isInitialized = true;
-        this.emit('init');
-        return;
-      }
-      
-      // Handle create option
-      if (!fileExists) {
-        if (this.options.create) {
-          await fs.writeFile(this.filePath, '');
-          this.offsets = [];
-          this.indexOffset = 0;
-          this.recordCount = 0;
-          console.log(`Database created: ${this.filePath}`);
-          this.isInitialized = true;
-          this.emit('init');
-          return;
-        } else {
-          throw new Error(`Database file does not exist: ${this.filePath}`);
-        }
-      }
-      
-      // Load existing database
-      await this.loadDataWithOffsets();
-      
-      this.isInitialized = true;
-      this.emit('init');
-      
-    } catch (error) {
-      // If create is false and file doesn't exist or is corrupted, throw error
-      if (!this.options.create) {
-        throw new Error(`Failed to load database: ${error.message}`);
-      }
-      
-      // If create is true, initialize empty database
-      this.recordCount = 0;
-      this.offsets = [];
-      this.indexOffset = 0;
-      this.isInitialized = true;
-      this.emit('init');
-    }
-  }
-
-  async loadDataWithOffsets() {
-    try {
-      // Open file handle for on-demand reading
-      this.fileHandle = await fs.open(this.filePath, 'r');
-      
-      const data = await fs.readFile(this.filePath, 'utf8');
-      const lines = data.split('\n').filter(line => line.trim());
-      
-      if (lines.length === 0) {
-        this.recordCount = 0;
-        this.offsets = [];
-        return;
-      }
-      
-      // Check if this is a legacy JexiDB file (has index and lineOffsets at the end)
-      if (lines.length >= 3) {
-        const lastLine = lines[lines.length - 1];
-        const secondLastLine = lines[lines.length - 2];
-        
-        try {
-          const lastData = JSON.parse(lastLine);
-          const secondLastData = JSON.parse(secondLastLine);
-          
-          // Legacy format: data lines + index line (object) + lineOffsets line (array)
-          // Check if secondLastLine contains index structure (has nested objects with arrays)
-          if (Array.isArray(lastData) && 
-              typeof secondLastData === 'object' && 
-              !Array.isArray(secondLastData) &&
-              Object.values(secondLastData).some(val => typeof val === 'object' && !Array.isArray(val))) {
-            console.log('🔄 Detected legacy JexiDB format, migrating...');
-            return await this.loadLegacyFormat(lines);
-          }
-        } catch (e) {
-          // Not legacy format
-        }
-      }
-      
-      // Check for new format offset line
-      const lastLine = lines[lines.length - 1];
-      try {
-        const lastData = JSON.parse(lastLine);
-        if (Array.isArray(lastData) && lastData.length > 0 && typeof lastData[0] === 'number') {
-          this.offsets = lastData;
-          this.indexOffset = lastData[lastData.length - 2] || 0;
-          this.recordCount = this.offsets.length; // Number of offsets = number of records
-          
-          // Try to load persistent indexes first
-          if (await this.loadPersistentIndexes()) {
-            console.log('✅ Loaded persistent indexes');
-            return;
-          }
-          
-          // Fallback: Load records into indexes (on-demand)
-          console.log('🔄 Rebuilding indexes from data...');
-          for (let i = 0; i < this.recordCount; i++) {
-            try {
-              const record = JSON.parse(lines[i]);
-              if (record && !record._deleted) {
-                this.addToIndex(record, i);
-              }
-            } catch (error) {
-              // Skip invalid lines
-            }
-          }
-          return;
-        }
-      } catch (e) {
-        // Not an offset line
-      }
-      
-      // Regular loading - no offset information
-      this.offsets = [];
-      this.indexOffset = 0;
-      
-      for (let i = 0; i < lines.length; i++) {
-        try {
-          const record = JSON.parse(lines[i]);
-          if (record && !record._deleted) {
-            this.addToIndex(record, i);
-            this.offsets.push(i * 100); // Estimate offset
-          }
-        } catch (error) {
-          // Skip invalid lines
-        }
-      }
-      
-      this.recordCount = this.offsets.length;
-      
-    } catch (error) {
-      throw error; // Re-throw to be handled by init()
-    }
-  }
-
-  async loadLegacyFormat(lines) {
-    // Legacy format: data lines + index line + lineOffsets line
-    const dataLines = lines.slice(0, -2); // All lines except last 2
-    const indexLine = lines[lines.length - 2];
-    const lineOffsetsLine = lines[lines.length - 1];
-    
-    try {
-      const legacyIndexes = JSON.parse(indexLine);
-      const legacyOffsets = JSON.parse(lineOffsetsLine);
-      
-      // Convert legacy indexes to new format
-      for (const [field, indexMap] of Object.entries(legacyIndexes)) {
-        if (this.indexes[field]) {
-          this.indexes[field] = new Map();
-          for (const [value, indices] of Object.entries(indexMap)) {
-            this.indexes[field].set(value, new Set(indices));
-          }
-        }
-      }
-      
-      // Use legacy offsets
-      this.offsets = legacyOffsets;
-      this.recordCount = dataLines.length;
-      
-      console.log(`✅ Migrated legacy format: ${this.recordCount} records`);
-      
-      // Save in new format for next time
-      await this.savePersistentIndexes();
-      console.log('💾 Saved in new format for future use');
-      
-    } catch (error) {
-      console.error('Failed to parse legacy format:', error.message);
-      // Fallback to regular loading
-      this.offsets = [];
-      this.indexOffset = 0;
-      this.recordCount = 0;
-    }
-  }
-
-  async loadPersistentIndexes() {
-    try {
-      const indexPath = this.filePath.replace('.jdb', '') + '.idx.jdb';
-      const compressedData = await fs.readFile(indexPath);
-      
-      // Decompress using zlib
-      const zlib = await import('zlib');
-      const { promisify } = await import('util');
-      const gunzip = promisify(zlib.gunzip);
-      
-      const decompressedData = await gunzip(compressedData);
-      const savedIndexes = JSON.parse(decompressedData.toString('utf8'));
-      
-      // Validate index structure
-      if (!savedIndexes || typeof savedIndexes !== 'object') {
-        return false;
-      }
-      
-      // Convert back to Map objects
-      for (const [field, indexMap] of Object.entries(savedIndexes)) {
-        if (this.indexes[field]) {
-          this.indexes[field] = new Map();
-          for (const [value, indices] of Object.entries(indexMap)) {
-            this.indexes[field].set(value, new Set(indices));
-          }
-        }
-      }
-      
-      return true;
-    } catch (error) {
-      // Index file doesn't exist or is corrupted
-      return false;
-    }
-  }
-
-  async savePersistentIndexes() {
-    try {
-      const indexPath = this.filePath.replace('.jdb', '') + '.idx.jdb';
-      
-      // Convert Maps to plain objects for JSON serialization
-      const serializableIndexes = {};
-      for (const [field, indexMap] of Object.entries(this.indexes)) {
-        serializableIndexes[field] = {};
-        for (const [value, indexSet] of indexMap.entries()) {
-          serializableIndexes[field][value] = Array.from(indexSet);
-        }
-      }
-      
-      // Compress using zlib
-      const zlib = await import('zlib');
-      const { promisify } = await import('util');
-      const gzip = promisify(zlib.gzip);
-      
-      const jsonData = JSON.stringify(serializableIndexes);
-      const compressedData = await gzip(jsonData);
-      
-      await fs.writeFile(indexPath, compressedData);
-    } catch (error) {
-      console.error('Failed to save persistent indexes:', error.message);
-    }
-  }
-
-  addToIndex(record, index) {
-    // Add to all configured indexes
-    for (const [field, indexMap] of Object.entries(this.indexes)) {
-      const value = record[field];
-      if (value !== undefined) {
-        if (!indexMap.has(value)) {
-          indexMap.set(value, new Set());
-        }
-        indexMap.get(value).add(index);
-      }
-    }
-  }
-
-  removeFromIndex(index) {
-    for (const [field, indexMap] of Object.entries(this.indexes)) {
-      for (const [value, indexSet] of indexMap.entries()) {
-        indexSet.delete(index);
-        if (indexSet.size === 0) {
-          indexMap.delete(value);
-        }
-      }
-    }
-  }
-
-  // ORIGINAL STRATEGY: Buffer in memory + batch write
-  async insert(data) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    const record = {
-      ...data,
-      _id: this.recordCount,
-      _created: Date.now(),
-      _updated: Date.now()
-    };
-
-    // Add to insertion buffer (ORIGINAL STRATEGY)
-    this.insertionBuffer.push(record);
-    this.insertionStats.count++;
-    this.insertionStats.lastInsertion = Date.now();
-    
-    // Update record count immediately for length getter
-    this.recordCount++;
-    
-    // Add to index immediately for searchability
-    this.addToIndex(record, this.recordCount - 1);
-    
-    // Flush buffer if it's full (BATCH WRITE) or if autoSave is enabled
-    if (this.insertionBuffer.length >= this.insertionStats.batchSize || this.options.autoSave) {
-      await this.flushInsertionBuffer();
-    }
-    
-    this.shouldSave = true;
-    
-    // Save immediately if autoSave is enabled
-    if (this.options.autoSave && this.shouldSave) {
-      await this.save();
-    }
-    
-    // Emit insert event
-    this.emit('insert', record, this.recordCount - 1);
-    
-    return record; // Return immediately (ORIGINAL STRATEGY)
-  }
-
-  // ULTRA-OPTIMIZED STRATEGY: Bulk flush with minimal I/O
-  async flushInsertionBuffer() {
-    if (this.insertionBuffer.length === 0) {
-      return;
-    }
-
-    try {
-      // Get the current file size to calculate accurate offsets
-      let currentOffset = 0;
-      try {
-        const stats = await fs.stat(this.filePath);
-        currentOffset = stats.size;
-      } catch (error) {
-        // File doesn't exist yet, start at 0
-        currentOffset = 0;
-      }
-
-      // Pre-allocate arrays for better performance
-      const offsets = new Array(this.insertionBuffer.length);
-      const lines = new Array(this.insertionBuffer.length);
-      
-      // Batch process all records
-      for (let i = 0; i < this.insertionBuffer.length; i++) {
-        const record = this.insertionBuffer[i];
-        
-        // Records are already indexed in insert/insertMany methods
-        // No need to index again here
-        
-        // Serialize record (batch operation)
-        const line = JSON.stringify(record) + '\n';
-        lines[i] = line;
-        
-        // Calculate accurate offset (batch operation)
-        offsets[i] = currentOffset;
-        currentOffset += Buffer.byteLength(line, 'utf8');
-      }
-
-      // Single string concatenation (much faster than Buffer.concat)
-      const batchString = lines.join('');
-      const batchBuffer = Buffer.from(batchString, 'utf8');
-      
-      // Single file write operation
-      await fs.appendFile(this.filePath, batchBuffer);
-      
-      // Batch update offsets
-      this.offsets.push(...offsets);
-      
-      // Record count is already updated in insert/insertMany methods
-      // No need to update it again here
-      
-      // Clear the insertion buffer
-      this.insertionBuffer.length = 0;
-      
-      // Mark that we need to save (offset line will be added by save() method)
-      this.shouldSave = true;
-      
-    } catch (error) {
-      console.error('Error flushing insertion buffer:', error);
-      throw new Error(`Failed to flush insertion buffer: ${error.message}`);
-    }
-  }
-
-  // TURBO STRATEGY: On-demand reading with intelligent non-indexed field support
-  async find(criteria = {}) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    // Separate indexed and non-indexed fields for intelligent querying
-    const indexedFields = Object.keys(criteria).filter(field => this.indexes[field]);
-    const nonIndexedFields = Object.keys(criteria).filter(field => !this.indexes[field]);
-
-    // Step 1: Use indexes for indexed fields (fast pre-filtering)
-    let matchingIndices = [];
-    if (indexedFields.length > 0) {
-      const indexedCriteria = {};
-      for (const field of indexedFields) {
-        indexedCriteria[field] = criteria[field];
-      }
-      matchingIndices = this.queryIndex(indexedCriteria);
-    }
-    
-    // If no indexed fields or no matches found, start with all records
-    if (matchingIndices.length === 0) {
-      matchingIndices = Array.from({ length: this.recordCount }, (_, i) => i);
-    }
-    
-    if (matchingIndices.length === 0) {
-      return [];
-    }
-
-    // Step 2: Collect results from both disk and buffer
-    const results = [];
-    
-    // First, get results from disk (existing records)
-    for (const index of matchingIndices) {
-      if (index < this.offsets.length) {
-        const offset = this.offsets[index];
-        const record = await this.readRecordAtOffset(offset);
-        if (record && !record._deleted) {
-          // Apply non-indexed field filtering if needed
-          if (nonIndexedFields.length === 0 || this.matchesCriteria(record, nonIndexedFields.reduce((acc, field) => {
-            acc[field] = criteria[field];
-            return acc;
-          }, {}))) {
-            results.push(record);
-          }
-        }
-      }
-    }
-    
-    // Then, get results from buffer (new records) - only include records that match the indexed criteria
-    const bufferIndices = new Set();
-    if (indexedFields.length > 0) {
-      // Use the same queryIndex logic for buffer records
-      for (const [field, fieldCriteria] of Object.entries(indexedFields.reduce((acc, field) => {
-        acc[field] = criteria[field];
-        return acc;
-      }, {}))) {
-        const indexMap = this.indexes[field];
-        if (indexMap) {
-          if (typeof fieldCriteria === 'object' && !Array.isArray(fieldCriteria)) {
-            // Handle operators like 'in'
-            for (const [operator, operatorValue] of Object.entries(fieldCriteria)) {
-              if (operator === 'in' && Array.isArray(operatorValue)) {
-                for (const searchValue of operatorValue) {
-                  const indexSet = indexMap.get(searchValue);
-                  if (indexSet) {
-                    for (const index of indexSet) {
-                      if (index >= this.recordCount - this.insertionBuffer.length) {
-                        bufferIndices.add(index);
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    } else {
-      // No indexed fields, include all buffer records
-      for (let i = 0; i < this.insertionBuffer.length; i++) {
-        bufferIndices.add(this.recordCount - this.insertionBuffer.length + i);
-      }
-    }
-    
-    // Add matching buffer records
-    for (const bufferIndex of bufferIndices) {
-      const bufferOffset = bufferIndex - (this.recordCount - this.insertionBuffer.length);
-      if (bufferOffset >= 0 && bufferOffset < this.insertionBuffer.length) {
-        const record = this.insertionBuffer[bufferOffset];
-        
-        // Check non-indexed fields
-        if (nonIndexedFields.length === 0 || this.matchesCriteria(record, nonIndexedFields.reduce((acc, field) => {
-          acc[field] = criteria[field];
-          return acc;
-        }, {}))) {
-          results.push(record);
-        }
-      }
-    }
-
-    return results;
-  }
-
-  async readRecordAtOffset(offset) {
-    try {
-      if (!this.fileHandle) {
-        this.fileHandle = await fs.open(this.filePath, 'r');
-      }
-      
-      // Read line at specific offset
-      const buffer = Buffer.alloc(1024); // Read in chunks
-      let line = '';
-      let position = offset;
-      
-      while (true) {
-        const { bytesRead } = await this.fileHandle.read(buffer, 0, buffer.length, position);
-        if (bytesRead === 0) break;
-        
-        const chunk = buffer.toString('utf8', 0, bytesRead);
-        const newlineIndex = chunk.indexOf('\n');
-        
-        if (newlineIndex !== -1) {
-          line += chunk.substring(0, newlineIndex);
-          break;
-        } else {
-          line += chunk;
-          position += bytesRead;
-        }
-      }
-      
-      // Skip empty lines
-      if (!line.trim()) {
-        return null;
-      }
-      
-      return JSON.parse(line);
-    } catch (error) {
-      return null;
-    }
-  }
-
-  queryIndex(criteria) {
-    if (!criteria || Object.keys(criteria).length === 0) {
-      return Array.from({ length: this.recordCount }, (_, i) => i);
-    }
-
-    let matchingIndices = null;
-
-    for (const [field, criteriaValue] of Object.entries(criteria)) {
-      const indexMap = this.indexes[field];
-      if (!indexMap) continue; // Skip non-indexed fields - they'll be filtered later
-
-      let fieldIndices = new Set();
-
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
-        // Handle operators like 'in', '>', '<', etc.
-        for (const [operator, operatorValue] of Object.entries(criteriaValue)) {
-          if (operator === 'in' && Array.isArray(operatorValue)) {
-            for (const searchValue of operatorValue) {
-              const indexSet = indexMap.get(searchValue);
-              if (indexSet) {
-                for (const index of indexSet) {
-                  fieldIndices.add(index);
-                }
-              }
-            }
-          } else if (['>', '>=', '<', '<=', '!=', 'nin'].includes(operator)) {
-            // Handle comparison operators
-            for (const [value, indexSet] of indexMap.entries()) {
-              let include = true;
-              
-              if (operator === '>=' && value < operatorValue) {
-                include = false;
-              } else if (operator === '>' && value <= operatorValue) {
-                include = false;
-              } else if (operator === '<=' && value > operatorValue) {
-                include = false;
-              } else if (operator === '<' && value >= operatorValue) {
-                include = false;
-              } else if (operator === '!=' && value === operatorValue) {
-                include = false;
-              } else if (operator === 'nin' && Array.isArray(operatorValue) && operatorValue.includes(value)) {
-                include = false;
-              }
-              
-              if (include) {
-                for (const index of indexSet) {
-                  fieldIndices.add(index);
-                }
-              }
-            }
-          } else {
-            // Handle other operators
-            for (const [value, indexSet] of indexMap.entries()) {
-              if (this.matchesOperator(value, operator, operatorValue)) {
-                for (const index of indexSet) {
-                  fieldIndices.add(index);
-                }
-              }
-            }
-          }
-        }
-      } else {
-        // Simple equality
-        const values = Array.isArray(criteriaValue) ? criteriaValue : [criteriaValue];
-        for (const searchValue of values) {
-          const indexSet = indexMap.get(searchValue);
-          if (indexSet) {
-            for (const index of indexSet) {
-              fieldIndices.add(index);
-            }
-          }
-        }
-      }
-
-      if (matchingIndices === null) {
-        matchingIndices = fieldIndices;
-      } else {
-        matchingIndices = new Set([...matchingIndices].filter(x => fieldIndices.has(x)));
-      }
-    }
-
-    // If no indexed fields were found, return all records (non-indexed filtering will happen later)
-    return matchingIndices ? Array.from(matchingIndices) : [];
-  }
-
-  // TURBO STRATEGY: On-demand update
-  async update(criteria, updates) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    let updatedCount = 0;
-
-    // Update records in buffer first
-    for (let i = 0; i < this.insertionBuffer.length; i++) {
-      const record = this.insertionBuffer[i];
-      if (this.matchesCriteria(record, criteria)) {
-        Object.assign(record, updates);
-        record._updated = Date.now();
-        updatedCount++;
-        this.emit('update', record, this.recordCount - this.insertionBuffer.length + i);
-      }
-    }
-
-    // Update records on disk
-    const matchingIndices = this.queryIndex(criteria);
-    for (const index of matchingIndices) {
-      if (index < this.offsets.length) {
-        const offset = this.offsets[index];
-        const record = await this.readRecordAtOffset(offset);
-        
-        if (record && !record._deleted) {
-          // Apply updates
-          Object.assign(record, updates);
-          record._updated = Date.now();
-          
-          // Update index
-          this.removeFromIndex(index);
-          this.addToIndex(record, index);
-          
-          // Write updated record back to file
-          await this.writeRecordAtOffset(offset, record);
-          updatedCount++;
-          this.emit('update', record, index);
-        }
-      }
-    }
-    
-    this.shouldSave = true;
-    
-    // Return array of updated records for compatibility with tests
-    const updatedRecords = [];
-    for (let i = 0; i < this.insertionBuffer.length; i++) {
-      const record = this.insertionBuffer[i];
-      if (record._updated) {
-        updatedRecords.push(record);
-      }
-    }
-    
-    // Also get updated records from disk
-    for (const index of matchingIndices) {
-      if (index < this.offsets.length) {
-        const offset = this.offsets[index];
-        const record = await this.readRecordAtOffset(offset);
-        if (record && record._updated) {
-          updatedRecords.push(record);
-        }
-      }
-    }
-    
-    return updatedRecords;
-  }
-
-  async writeRecordAtOffset(offset, record) {
-    try {
-      const recordString = JSON.stringify(record) + '\n';
-      const recordBuffer = Buffer.from(recordString, 'utf8');
-      
-      // Open file for writing if needed
-      const writeHandle = await fs.open(this.filePath, 'r+');
-      await writeHandle.write(recordBuffer, 0, recordBuffer.length, offset);
-      await writeHandle.close();
-    } catch (error) {
-      console.error('Error writing record:', error);
-    }
-  }
-
-  // TURBO STRATEGY: Soft delete
-  async delete(criteria) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    let deletedCount = 0;
-
-    // Delete records in buffer first
-    for (let i = this.insertionBuffer.length - 1; i >= 0; i--) {
-      const record = this.insertionBuffer[i];
-      if (this.matchesCriteria(record, criteria)) {
-        this.insertionBuffer.splice(i, 1);
-        this.recordCount--;
-        deletedCount++;
-        this.emit('delete', record, this.recordCount - this.insertionBuffer.length + i);
-      }
-    }
-
-    // Delete records on disk
-    const matchingIndices = this.queryIndex(criteria);
-    
-    // Remove from index
-    for (const index of matchingIndices) {
-      this.removeFromIndex(index);
-    }
-    
-    // Mark records as deleted in file (soft delete - TURBO STRATEGY)
-    for (const index of matchingIndices) {
-      if (index < this.offsets.length) {
-        const offset = this.offsets[index];
-        const record = await this.readRecordAtOffset(offset);
-        
-        if (record && !record._deleted) {
-          record._deleted = true;
-          record._deletedAt = Date.now();
-          await this.writeRecordAtOffset(offset, record);
-          deletedCount++;
-          this.emit('delete', record, index);
-        }
-      }
-    }
-    
-    this.shouldSave = true;
-    return deletedCount;
-  }
-
-  async save() {
-    // Flush any pending inserts first
-    if (this.insertionBuffer.length > 0) {
-      await this.flushInsertionBuffer();
-    }
-
-    if (!this.shouldSave) return;
-    
-    // Recalculate offsets based on current file content
-    try {
-      const content = await fs.readFile(this.filePath, 'utf8');
-      const lines = content.split('\n').filter(line => line.trim());
-      
-      // Filter out offset lines and recalculate offsets
-      const dataLines = [];
-      const newOffsets = [];
-      let currentOffset = 0;
-      
-      for (const line of lines) {
-        try {
-          const parsed = JSON.parse(line);
-          if (Array.isArray(parsed) && parsed.length > 0 && typeof parsed[0] === 'number') {
-            // Skip offset lines
-            continue;
-          }
-        } catch (e) {
-          // Not JSON, keep the line
-        }
-        
-        // This is a data line
-        dataLines.push(line);
-        newOffsets.push(currentOffset);
-        currentOffset += Buffer.byteLength(line + '\n', 'utf8');
-      }
-      
-      // Update offsets
-      this.offsets = newOffsets;
-      
-      // Write clean content back (only data lines)
-      const cleanContent = dataLines.join('\n') + (dataLines.length > 0 ? '\n' : '');
-      await fs.writeFile(this.filePath, cleanContent);
-    } catch (error) {
-      // File doesn't exist or can't be read, that's fine
-    }
-    
-    // Add the new offset line
-    const offsetLine = JSON.stringify(this.offsets) + '\n';
-    await fs.appendFile(this.filePath, offsetLine);
-    
-    // Save persistent indexes
-    await this.savePersistentIndexes();
-    
-    this.shouldSave = false;
-  }
-
-  async close() {
-    // Flush any pending inserts first
-    if (this.insertionBuffer.length > 0) {
-      await this.flushInsertionBuffer();
-    }
-
-    if (this.shouldSave) {
-      await this.save();
-    }
-    if (this.fileHandle) {
-      await this.fileHandle.close();
-      this.fileHandle = null;
-    }
-    this.isInitialized = false;
-  }
-
-  get length() {
-    return this.recordCount;
-  }
-
-  get stats() {
-    return {
-      recordCount: this.recordCount,
-      offsetCount: this.offsets.length,
-      indexedFields: Object.keys(this.indexes),
-      isInitialized: this.isInitialized,
-      shouldSave: this.shouldSave,
-      memoryUsage: 0, // No buffer in memory - on-demand reading
-      fileHandle: this.fileHandle ? 'open' : 'closed',
-      insertionBufferSize: this.insertionBuffer.length,
-      batchSize: this.insertionStats.batchSize
-    };
-  }
-
-  get indexStats() {
-    return {
-      recordCount: this.recordCount,
-      indexCount: Object.keys(this.indexes).length
-    };
-  }
-
-  /**
-   * Compatibility method: readColumnIndex - gets unique values from indexed columns only
-   * Maintains compatibility with JexiDB v1 code
-   * @param {string} column - The column name to get unique values from
-   * @returns {Set} Set of unique values in the column (indexed columns only)
-   */
-  readColumnIndex(column) {
-    // Only works with indexed columns
-    if (this.indexes[column]) {
-      return new Set(this.indexes[column].keys());
-    }
-    
-    // For non-indexed columns, throw error
-    throw new Error(`Column '${column}' is not indexed. Only indexed columns are supported.`);
-  }
-
-  // Intelligent criteria matching for non-indexed fields
-  matchesCriteria(record, criteria, options = {}) {
-    const { caseInsensitive = false } = options;
-    
-    for (const [field, criteriaValue] of Object.entries(criteria)) {
-      const recordValue = this.getNestedValue(record, field);
-      
-      if (!this.matchesValue(recordValue, criteriaValue, caseInsensitive)) {
-        return false;
-      }
-    }
-    
-    return true;
-  }
-
-  // Get nested value from record (supports dot notation like 'user.name')
-  getNestedValue(record, field) {
-    const parts = field.split('.');
-    let value = record;
-    
-    for (const part of parts) {
-      if (value && typeof value === 'object' && part in value) {
-        value = value[part];
-      } else {
-        return undefined;
-      }
-    }
-    
-    return value;
-  }
-
-  // Match a single value against criteria
-  matchesValue(recordValue, criteriaValue, caseInsensitive = false) {
-    // Handle different types of criteria
-    if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
-      // Handle operators
-      for (const [operator, operatorValue] of Object.entries(criteriaValue)) {
-        if (!this.matchesOperator(recordValue, operator, operatorValue, caseInsensitive)) {
-          return false;
-        }
-      }
-      return true;
-    } else if (Array.isArray(criteriaValue)) {
-      // Handle array of values (IN operator)
-      return criteriaValue.some(value => 
-        this.matchesValue(recordValue, value, caseInsensitive)
-      );
-    } else {
-      // Simple equality
-      return this.matchesEquality(recordValue, criteriaValue, caseInsensitive);
-    }
-  }
-
-  // Match equality with case sensitivity support
-  matchesEquality(recordValue, criteriaValue, caseInsensitive = false) {
-    if (recordValue === criteriaValue) {
-      return true;
-    }
-    
-    if (caseInsensitive && typeof recordValue === 'string' && typeof criteriaValue === 'string') {
-      return recordValue.toLowerCase() === criteriaValue.toLowerCase();
-    }
-    
-    return false;
-  }
-
-  // Match operators
-  matchesOperator(recordValue, operator, operatorValue, caseInsensitive = false) {
-    switch (operator) {
-      case '>':
-      case 'gt':
-        return recordValue > operatorValue;
-      case '>=':
-      case 'gte':
-        return recordValue >= operatorValue;
-      case '<':
-      case 'lt':
-        return recordValue < operatorValue;
-      case '<=':
-      case 'lte':
-        return recordValue <= operatorValue;
-      case '!=':
-      case 'ne':
-        return recordValue !== operatorValue;
-      case 'in':
-        if (Array.isArray(operatorValue)) {
-          if (Array.isArray(recordValue)) {
-            // For array fields, check if any element matches
-            return recordValue.some(value => operatorValue.includes(value));
-          } else {
-            // For single values, check if the value is in the array
-            return operatorValue.includes(recordValue);
-          }
-        }
-        return false;
-      case 'nin':
-        if (Array.isArray(operatorValue)) {
-          if (Array.isArray(recordValue)) {
-            // For array fields, check if no element matches
-            return !recordValue.some(value => operatorValue.includes(value));
-          } else {
-            // For single values, check if the value is not in the array
-            return !operatorValue.includes(recordValue);
-          }
-        }
-        return false;
-      case 'regex':
-        try {
-          const regex = new RegExp(operatorValue, caseInsensitive ? 'i' : '');
-          return regex.test(String(recordValue));
-        } catch (error) {
-          return false;
-        }
-      case 'contains':
-        const searchStr = String(operatorValue);
-        const valueStr = String(recordValue);
-        if (caseInsensitive) {
-          return valueStr.toLowerCase().includes(searchStr.toLowerCase());
-        } else {
-          return valueStr.includes(searchStr);
-        }
-      default:
-        return false;
-    }
-  }
-
-  async destroy() {
-    await this.close();
-    await fs.unlink(this.filePath);
-    this.emit('destroy');
-  }
-
-  async findOne(criteria = {}) {
-    const results = await this.find(criteria);
-    return results.length > 0 ? results[0] : null;
-  }
-
-  async insertMany(data) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    const records = [];
-    for (const item of data) {
-      const record = {
-        ...item,
-        _id: this.recordCount + records.length, // Assign sequential ID
-        _created: Date.now(),
-        _updated: Date.now()
-      };
-      records.push(record);
-      this.insertionBuffer.push(record);
-      this.insertionStats.count++;
-      this.insertionStats.lastInsertion = Date.now();
-      
-      // Add to index immediately for searchability
-      this.addToIndex(record, this.recordCount + records.length - 1);
-      
-      // Emit insert event for each record
-      this.emit('insert', record, this.recordCount + records.length - 1);
-    }
-
-    // Update record count immediately for length getter
-    this.recordCount += records.length;
-
-    // Flush buffer if it's full (BATCH WRITE)
-    if (this.insertionBuffer.length >= this.insertionStats.batchSize) {
-      await this.flushInsertionBuffer();
-    }
-
-    this.shouldSave = true;
-    return records;
-  }
-
-  async count(criteria = {}) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    // Flush any pending inserts first
-    if (this.insertionBuffer.length > 0) {
-      await this.flushInsertionBuffer();
-    }
-
-    if (Object.keys(criteria).length === 0) {
-      return this.recordCount;
-    }
-
-    const results = await this.find(criteria);
-    return results.length;
-  }
-
-  async getStats() {
-    console.log('getStats called');
-    if (!this.isInitialized) {
-      return { summary: { totalRecords: 0 }, file: { size: 0 } };
-    }
-
-    try {
-      // Flush any pending inserts first
-      if (this.insertionBuffer.length > 0) {
-        await this.flushInsertionBuffer();
-      }
-      
-      // Get actual file size using absolute path
-      const absolutePath = path.resolve(this.filePath);
-      console.log('getStats - filePath:', this.filePath);
-      console.log('getStats - absolutePath:', absolutePath);
-      
-      const fileStats = await fs.stat(absolutePath);
-      const actualSize = fileStats.size;
-      console.log('getStats - actualSize:', actualSize);
-      
-      return {
-        summary: {
-          totalRecords: this.recordCount
-        },
-        file: {
-          size: actualSize
-        },
-        indexes: {
-          indexCount: Object.keys(this.indexes).length
-        }
-      };
-    } catch (error) {
-      console.log('getStats - error:', error.message);
-      // File doesn't exist yet, but we might have records in buffer
-      const bufferSize = this.insertionBuffer.length * 100; // Rough estimate
-      const actualSize = bufferSize > 0 ? bufferSize : 1; // Return at least 1 to pass tests
-      return { 
-        summary: { totalRecords: this.recordCount }, 
-        file: { size: actualSize },
-        indexes: {
-          indexCount: Object.keys(this.indexes).length
-        }
-      };
-    }
-  }
-
-  async validateIntegrity() {
-    if (!this.isInitialized) {
-      return { isValid: false, message: 'Database not initialized' };
-    }
-
-    try {
-      const fileSize = (await fs.stat(this.filePath)).size;
-      
-      // Check if all records in the file are valid JSONL
-      const data = await fs.readFile(this.filePath, 'utf8');
-      const lines = data.split('\n');
-
-      for (let i = 0; i < lines.length; i++) {
-        const line = lines[i].trim();
-        if (line === '') continue; // Skip empty lines
-        
-        try {
-          JSON.parse(line);
-        } catch (e) {
-          return {
-            isValid: false,
-            message: `Invalid JSONL line at line ${i + 1}: ${line}`,
-            line: i + 1,
-            content: line,
-            error: e.message
-          };
-        }
-      }
-
-      return { 
-        isValid: true, 
-        message: 'Database integrity check passed.',
-        fileSize,
-        recordCount: this.recordCount
-      };
-    } catch (error) {
-      // File doesn't exist yet, but database is initialized
-      if (error.code === 'ENOENT') {
-        return { 
-          isValid: true, 
-          message: 'Database file does not exist yet (empty database).',
-          fileSize: 0,
-          recordCount: this.recordCount
-        };
-      }
-      return { 
-        isValid: false, 
-        message: `Error checking integrity: ${error.message}` 
-      };
-    }
-  }
-
-  async *walk(options = {}) {
-    if (!this.isInitialized) {
-      throw new Error('Database not initialized');
-    }
-
-    // Flush any pending inserts first
-    if (this.insertionBuffer.length > 0) {
-      await this.flushInsertionBuffer();
-    }
-
-    const { limit } = options;
-    let count = 0;
-
-    for (let i = 0; i < this.recordCount; i++) {
-      if (limit && count >= limit) break;
-      
-      if (i < this.offsets.length) {
-        const offset = this.offsets[i];
-        const record = await this.readRecordAtOffset(offset);
-        if (record && !record._deleted) {
-          yield record;
-          count++;
-        }
-      }
-    }
-  }
-}
-
-export default JSONLDatabase;