jexidb 2.1.3 → 2.1.4

package/dist/Database.cjs

@@ -1241,11 +1241,11 @@ class IndexManager {
         // This will be handled by the QueryManager's streaming strategy
         continue;
       }
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
+      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
         const fieldIndex = data[field];
 
         // Handle $in operator for array queries
-        if (criteriaValue.$in !== undefined) {
+        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
           const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
 
           // PERFORMANCE: Cache term mapping field check once
@@ -2137,6 +2137,14 @@ class IndexManager {
       // Keep the current index with initialized fields
       return;
     }
+
+    // Restore totalLines from saved data
+    if (index.totalLines !== undefined) {
+      this.totalLines = index.totalLines;
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`);
+      }
+    }
     this.index = processedIndex;
   }
 
@@ -2177,7 +2185,8 @@ class IndexManager {
    */
   toJSON() {
     const serializable = {
-      data: {}
+      data: {},
+      totalLines: this.totalLines
     };
 
     // Check if this is a term mapping field for conversion
@@ -2408,35 +2417,10 @@ class SchemaManager {
     const obj = {};
     const idIndex = this.schema.indexOf('id');
 
-    // CRITICAL FIX: Handle schema migration where 'id' was first field in old schema
-    // but is not in current schema. Check if first element looks like an ID.
-    // Only do this if:
-    // 1. 'id' is not in current schema
-    // 2. Array has significantly more elements than current schema (2+ extra elements)
-    //    This suggests the old schema had more fields, and 'id' was likely the first
-    // 3. First element is a very short string (max 20 chars) that looks like a generated ID
-    //    (typically alphanumeric, often starting with letters like 'mit...' or similar patterns)
-    // 4. First field in current schema is not 'id' (to avoid false positives)
-    // 5. First element is not an array (to avoid false positives with array fields)
+    // DISABLED: Schema migration detection was causing field mapping corruption
+    // The logic was incorrectly assuming ID was in first position when it's appended at the end
+    // This caused fields to be shifted incorrectly during object-to-array-to-object conversion
     let arrayOffset = 0;
-    if (idIndex === -1 && arr.length >= this.schema.length + 2 && this.schema.length > 0) {
-      // Only apply if array has at least 2 extra elements (suggests old schema had more fields)
-      const firstElement = arr[0];
-      const firstFieldName = this.schema[0];
-
-      // Only apply shift if:
-      // - First field is not 'id'
-      // - First element is a very short string (max 20 chars) that looks like a generated ID
-      // - First element is not an array (to avoid false positives)
-      // - Array has at least 2 extra elements (strong indicator of schema migration)
-      if (firstFieldName !== 'id' && typeof firstElement === 'string' && !Array.isArray(firstElement) && firstElement.length > 0 && firstElement.length <= 20 &&
-      // Very conservative: max 20 chars (typical ID length)
-      /^[a-zA-Z0-9_-]+$/.test(firstElement)) {
-        // First element is likely the ID from old schema
-        obj.id = firstElement;
-        arrayOffset = 1;
-      }
-    }
 
     // Map array values to object properties
     // Only include fields that are in the schema
@@ -6042,7 +6026,7 @@ class QueryManager {
         }
         return false;
       }
-      if (typeof condition === 'object' && !Array.isArray(condition)) {
+      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
         const operators = Object.keys(condition).map(op => normalizeOperator(op));
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' has operators:`, operators);
@@ -6339,7 +6323,7 @@ class QueryManager {
       if (field.startsWith('$')) continue;
       if (termMappingFields.includes(field)) {
         // For term mapping fields, simple equality or $in queries work well
-        if (typeof condition === 'string' || typeof condition === 'object' && condition.$in && Array.isArray(condition.$in)) {
+        if (typeof condition === 'string' || typeof condition === 'object' && condition !== null && condition.$in && Array.isArray(condition.$in)) {
           return true;
         }
       }
@@ -8237,6 +8221,14 @@ class Database extends events.EventEmitter {
 
       // Manual save is now the default behavior
 
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length);
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`);
+        }
+      }
       this.initialized = true;
       this.emit('initialized');
       if (this.opts.debugMode) {
@@ -9042,6 +9034,7 @@ class Database extends events.EventEmitter {
       });
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`);
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`);
         console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? {
           id: String(r.id),
           price: r.price,
@@ -9065,6 +9058,9 @@ class Database extends events.EventEmitter {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200));
         }
       }
+
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = [];
       let currentOffset = 0;
       for (let i = 0; i < lines.length; i++) {
@@ -9074,6 +9070,9 @@ class Database extends events.EventEmitter {
         const lineWithNewline = lines[i] + '\n';
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8');
       }
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`);
+      }
 
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset;
@@ -9093,11 +9092,15 @@ class Database extends events.EventEmitter {
       this.shouldSave = false;
       this.lastSaveTime = Date.now();
 
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
+      const hadDeletedRecords = deletedIdsSnapshot.size > 0;
+      const hadUpdatedRecords = writeBufferSnapshot.length > 0;
+
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
         // Rebuild index when records were deleted or updated to maintain consistency
-        const hadDeletedRecords = deletedIdsSnapshot.size > 0;
-        const hadUpdatedRecords = writeBufferSnapshot.length > 0;
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
           if (hadDeletedRecords || hadUpdatedRecords) {
             // Clear the index and rebuild it from the saved records
@@ -9154,8 +9157,19 @@ class Database extends events.EventEmitter {
               }
               await this.indexManager.add(record, i);
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length;
+            if (indexedRecordCount !== this.offsets.length) {
+              console.warn(`⚠️ Index inconsistency detected: indexed ${indexedRecordCount} records but offsets has ${this.offsets.length} entries`);
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length);
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length);
+            }
             if (this.opts.debugMode) {
-              console.log(`💾 Save: Index rebuilt with ${allData.length} records`);
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`);
             }
           }
         }
@@ -9176,6 +9190,22 @@ class Database extends events.EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId);
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear();
+          this.indexManager.setTotalLines(0);
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`);
+          }
+        }
+
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId);
+        }
 
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -9675,6 +9705,42 @@ class Database extends events.EventEmitter {
       console.log(`🔍 FIND START: criteria=${JSON.stringify(criteria)}, writeBuffer=${this.writeBuffer.length}`);
     }
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // Check if index and offsets are synchronized
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0;
+        const offsetsLength = this.offsets.length;
+        if (indexTotalLines !== offsetsLength) {
+          console.warn(`⚠️ Data integrity issue detected: index.totalLines=${indexTotalLines}, offsets.length=${offsetsLength}`);
+          // Auto-correct by updating index totalLines to match offsets
+          this.indexManager.setTotalLines(offsetsLength);
+          if (this.opts.debugMode) {
+            console.log(`🔧 Auto-corrected index totalLines to ${offsetsLength}`);
+          }
+
+          // CRITICAL FIX: Also save the corrected index to prevent persistence of inconsistency
+          // This ensures the .idx.jdb file contains the correct totalLines value
+          try {
+            await this._saveIndexDataToFile();
+            if (this.opts.debugMode) {
+              console.log(`💾 Saved corrected index data to prevent future inconsistencies`);
+            }
+          } catch (error) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Failed to save corrected index: ${error.message}`);
+            }
+          }
+
+          // Verify the fix worked
+          const newIndexTotalLines = this.indexManager.totalLines || 0;
+          if (newIndexTotalLines === offsetsLength) {
+            console.log(`✅ Data integrity successfully corrected: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`);
+          } else {
+            console.error(`❌ Data integrity correction failed: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`);
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options);
@@ -9691,31 +9757,23 @@ class Database extends events.EventEmitter {
       const writeBufferResultsWithTerms = options.restoreTerms !== false ? writeBufferResults.map(record => this.restoreTermIdsAfterDeserialization(record)) : writeBufferResults;
 
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults;
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))), Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))]);
 
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id)));
-        allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms];
-
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id);
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record;
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record);
-          }
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map();
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record);
         }
-      }
+      });
+
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record => record && record.id && !writeBufferMap.has(record.id));
+
+      // Combine results: file results (filtered) + all writeBuffer results
+      allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
 
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id));
@@ -9963,19 +10021,6 @@ class Database extends events.EventEmitter {
 
         // CRITICAL FIX: Validate state before update operation
         this.validateState();
-
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`);
-          }
-          const saveStart = Date.now();
-          await this.save(false); // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`);
-          }
-        }
         if (this.opts.debugMode) {
           console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`);
         }
@@ -9988,6 +10033,13 @@ class Database extends events.EventEmitter {
           console.log(`🔄 UPDATE: Find completed in ${Date.now() - findStart}ms, found ${records.length} records`);
         }
         const updatedRecords = [];
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: About to process ${records.length} records`);
+          console.log(`🔄 UPDATE: Records:`, records.map(r => ({
+            id: r.id,
+            value: r.value
+          })));
+        }
         for (const record of records) {
           const recordStart = Date.now();
           if (this.opts.debugMode) {
@@ -10026,12 +10078,18 @@ class Database extends events.EventEmitter {
           // For records in the file, we need to ensure they are properly marked for replacement
           const index = this.writeBuffer.findIndex(r => r.id === record.id);
           let lineNumber = null;
+          if (this.opts.debugMode) {
+            console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`);
+            console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`);
+            console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id));
+          }
           if (index !== -1) {
             // Record is already in writeBuffer, update it
             this.writeBuffer[index] = updated;
             lineNumber = this._getAbsoluteLineNumber(index);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           } else {
             // Record is in file, add updated version to writeBuffer
@@ -10041,6 +10099,7 @@ class Database extends events.EventEmitter {
             lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           }
           const indexUpdateStart = Date.now();
@@ -10076,6 +10135,26 @@ class Database extends events.EventEmitter {
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState();
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, {
+            operation: 'delete'
+          });
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {});
+          const queryFields = this._extractQueryFields(criteria);
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field));
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`);
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`);
+            }
+          }
+        }
         const records = await this.find(criteria);
         const deletedIds = [];
         if (this.opts.debugMode) {
@@ -11628,14 +11707,30 @@ class Database extends events.EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine);
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
-            if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
-              recordId = arrayData[2];
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema();
+              const idIndex = schema.indexOf('id');
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex];
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             } else {
-              // For arrays without ID field, use first element as fallback
-              recordId = arrayData[0];
+              // No schema available, try common positions
+              if (arrayData.length > 2) {
+                // Try position 2 (common in older formats)
+                recordId = arrayData[2];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId);
@@ -11717,7 +11812,7 @@ class Database extends events.EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`);
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`);
           }
           return {
             type: 'kept',
@@ -11728,7 +11823,9 @@ class Database extends events.EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`);
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`);
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot));
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`);
           }
           return {
             type: 'deleted',
@@ -11771,6 +11868,54 @@ class Database extends events.EventEmitter {
       const offset = parseInt(rangeKey);
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null;
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line);
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema();
+                  const idIndex = schema.indexOf('id');
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex]);
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line);
+              unchangedRecordId = obj.id ? String(obj.id) : null;
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`);
+            }
+            continue;
+          }
+
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`);
+            }
+            deletedOffsets.add(offset);
+            break;
+          }
+
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line);
           keptRecords.push({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jexidb",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "type": "module",
   "description": "JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.",
   "main": "./dist/Database.cjs",
@@ -36,15 +36,9 @@
     "rollup": "^4.48.1"
   },
   "dependencies": {
-    "@valentech/sializer": "^0.3.9",
     "async-mutex": "^0.5.0",
     "p-limit": "^6.1.0"
   },
-  "optionalDependencies": {
-    "fast-deep-equal": "^3.1.3",
-    "fast-json-stringify": "^5.7.0",
-    "msgpack": "^1.0.3"
-  },
   "directories": {
     "test": "test"
   },

package/src/Database.mjs

@@ -770,10 +770,19 @@ class Database extends EventEmitter {
       }
       
       // Manual save is now the default behavior
-      
+
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length)
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`)
+        }
+      }
+
       this.initialized = true
       this.emit('initialized')
-      
+
       if (this.opts.debugMode) {
         console.log(`✅ Database initialized with ${this.writeBuffer.length} records`)
       }
@@ -1606,6 +1615,7 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`)
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`)
         console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null')
       }
@@ -1624,6 +1634,8 @@ class Database extends EventEmitter {
         }
       }
       
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = []
       let currentOffset = 0
       for (let i = 0; i < lines.length; i++) {
@@ -1634,6 +1646,10 @@ class Database extends EventEmitter {
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8')
       }
       
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`)
+      }
+      
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset
       
@@ -1655,11 +1671,15 @@ class Database extends EventEmitter {
       this.shouldSave = false
       this.lastSaveTime = Date.now()
       
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
-        // Rebuild index when records were deleted or updated to maintain consistency
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
         const hadDeletedRecords = deletedIdsSnapshot.size > 0
         const hadUpdatedRecords = writeBufferSnapshot.length > 0
+      
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
+        // Rebuild index when records were deleted or updated to maintain consistency
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
           if (hadDeletedRecords || hadUpdatedRecords) {
             // Clear the index and rebuild it from the saved records
@@ -1713,8 +1733,20 @@ class Database extends EventEmitter {
               
               await this.indexManager.add(record, i)
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length
+            if (indexedRecordCount !== this.offsets.length) {
+              console.warn(`⚠️ Index inconsistency detected: indexed ${indexedRecordCount} records but offsets has ${this.offsets.length} entries`)
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length)
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length)
+            }
+
             if (this.opts.debugMode) {
-              console.log(`💾 Save: Index rebuilt with ${allData.length} records`)
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`)
             }
           }
         }
@@ -1738,6 +1770,22 @@ class Database extends EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId)
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear()
+          this.indexManager.setTotalLines(0)
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`)
+          }
+        }
+        
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId)
+        }
         
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -2257,6 +2305,43 @@ class Database extends EventEmitter {
     }
     
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // Check if index and offsets are synchronized
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0
+        const offsetsLength = this.offsets.length
+
+        if (indexTotalLines !== offsetsLength) {
+          console.warn(`⚠️ Data integrity issue detected: index.totalLines=${indexTotalLines}, offsets.length=${offsetsLength}`)
+          // Auto-correct by updating index totalLines to match offsets
+          this.indexManager.setTotalLines(offsetsLength)
+          if (this.opts.debugMode) {
+            console.log(`🔧 Auto-corrected index totalLines to ${offsetsLength}`)
+          }
+
+          // CRITICAL FIX: Also save the corrected index to prevent persistence of inconsistency
+          // This ensures the .idx.jdb file contains the correct totalLines value
+          try {
+            await this._saveIndexDataToFile()
+            if (this.opts.debugMode) {
+              console.log(`💾 Saved corrected index data to prevent future inconsistencies`)
+            }
+          } catch (error) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Failed to save corrected index: ${error.message}`)
+            }
+          }
+
+          // Verify the fix worked
+          const newIndexTotalLines = this.indexManager.totalLines || 0
+          if (newIndexTotalLines === offsetsLength) {
+            console.log(`✅ Data integrity successfully corrected: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`)
+          } else {
+            console.error(`❌ Data integrity correction failed: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`)
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options)
@@ -2279,36 +2364,25 @@ class Database extends EventEmitter {
       
       
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([
-          Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))),
-          Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))
-        ])
+
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map()
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record)
+        }
+      })
         
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(
-          fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id))
-        )
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record =>
+        record && record.id && !writeBufferMap.has(record.id)
+      )
+
+      // Combine results: file results (filtered) + all writeBuffer results
         allResults = [...filteredFileResults, ...writeBufferResultsWithTerms]
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms]
-        
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id)
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record)
-          }
-        }
-      }
       
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id))
@@ -2566,19 +2640,6 @@ class Database extends EventEmitter {
         
         // CRITICAL FIX: Validate state before update operation
         this.validateState()
-        
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`)
-          }
-          const saveStart = Date.now()
-          await this.save(false) // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`)
-          }
-        }
       
       if (this.opts.debugMode) {
         console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`)
@@ -2591,7 +2652,12 @@ class Database extends EventEmitter {
       }
       
       const updatedRecords = []
-      
+
+      if (this.opts.debugMode) {
+        console.log(`🔄 UPDATE: About to process ${records.length} records`)
+        console.log(`🔄 UPDATE: Records:`, records.map(r => ({ id: r.id, value: r.value })))
+      }
+
       for (const record of records) {
         const recordStart = Date.now()
         if (this.opts.debugMode) {
@@ -2628,13 +2694,20 @@ class Database extends EventEmitter {
         // For records in the file, we need to ensure they are properly marked for replacement
         const index = this.writeBuffer.findIndex(r => r.id === record.id)
         let lineNumber = null
-        
+
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`)
+          console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`)
+          console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id))
+        }
+
         if (index !== -1) {
           // Record is already in writeBuffer, update it
           this.writeBuffer[index] = updated
           lineNumber = this._getAbsoluteLineNumber(index)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         } else {
           // Record is in file, add updated version to writeBuffer
@@ -2644,6 +2717,7 @@ class Database extends EventEmitter {
           lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         }
         
@@ -2679,13 +2753,32 @@ class Database extends EventEmitter {
    */
   async delete(criteria) {
     this._validateInitialization('delete')
-    
+
     return this.operationQueue.enqueue(async () => {
       this.isInsideOperationQueue = true
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState()
-      
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, { operation: 'delete' })
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {})
+          const queryFields = this._extractQueryFields(criteria)
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field))
+
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`)
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`)
+            }
+          }
+        }
+
       const records = await this.find(criteria)
       const deletedIds = []
       
@@ -4289,14 +4382,30 @@ class Database extends EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine)
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema()
+              const idIndex = schema.indexOf('id')
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex]
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length]
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0]
+              }
+            } else {
+              // No schema available, try common positions
             if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
+                // Try position 2 (common in older formats)
               recordId = arrayData[2]
             } else {
-              // For arrays without ID field, use first element as fallback
+                // Fallback: use first element
               recordId = arrayData[0]
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId)
@@ -4369,7 +4478,7 @@ class Database extends EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`)
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`)
           }
           return { 
             type: 'kept', 
@@ -4380,7 +4489,9 @@ class Database extends EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`)
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`)
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot))
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`)
           }
           return { 
             type: 'deleted', 
@@ -4426,6 +4537,54 @@ class Database extends EventEmitter {
       
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line)
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema()
+                  const idIndex = schema.indexOf('id')
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex])
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line)
+              unchangedRecordId = obj.id ? String(obj.id) : null
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`)
+            }
+            continue
+          }
+          
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`)
+            }
+            deletedOffsets.add(offset)
+            break
+          }
+          
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line)
           keptRecords.push({ offset, type: 'unchanged', line: result.line })

package/src/SchemaManager.mjs

@@ -163,38 +163,10 @@ export default class SchemaManager {
     const obj = {}
     const idIndex = this.schema.indexOf('id')
     
-    // CRITICAL FIX: Handle schema migration where 'id' was first field in old schema
-    // but is not in current schema. Check if first element looks like an ID.
-    // Only do this if:
-    // 1. 'id' is not in current schema
-    // 2. Array has significantly more elements than current schema (2+ extra elements)
-    //    This suggests the old schema had more fields, and 'id' was likely the first
-    // 3. First element is a very short string (max 20 chars) that looks like a generated ID
-    //    (typically alphanumeric, often starting with letters like 'mit...' or similar patterns)
-    // 4. First field in current schema is not 'id' (to avoid false positives)
-    // 5. First element is not an array (to avoid false positives with array fields)
+    // DISABLED: Schema migration detection was causing field mapping corruption
+    // The logic was incorrectly assuming ID was in first position when it's appended at the end
+    // This caused fields to be shifted incorrectly during object-to-array-to-object conversion
     let arrayOffset = 0
-    if (idIndex === -1 && arr.length >= this.schema.length + 2 && this.schema.length > 0) {
-      // Only apply if array has at least 2 extra elements (suggests old schema had more fields)
-      const firstElement = arr[0]
-      const firstFieldName = this.schema[0]
-      
-      // Only apply shift if:
-      // - First field is not 'id'
-      // - First element is a very short string (max 20 chars) that looks like a generated ID
-      // - First element is not an array (to avoid false positives)
-      // - Array has at least 2 extra elements (strong indicator of schema migration)
-      if (firstFieldName !== 'id' && 
-          typeof firstElement === 'string' && 
-          !Array.isArray(firstElement) &&
-          firstElement.length > 0 && 
-          firstElement.length <= 20 && // Very conservative: max 20 chars (typical ID length)
-          /^[a-zA-Z0-9_-]+$/.test(firstElement)) {
-        // First element is likely the ID from old schema
-        obj.id = firstElement
-        arrayOffset = 1
-      }
-    }
     
     // Map array values to object properties
     // Only include fields that are in the schema

package/src/managers/IndexManager.mjs

@@ -1022,11 +1022,11 @@ export default class IndexManager {
         continue;
       }
 
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
+      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
         const fieldIndex = data[field];
-        
+
         // Handle $in operator for array queries
-        if (criteriaValue.$in !== undefined) {
+        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
           const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
           
           // PERFORMANCE: Cache term mapping field check once
@@ -1969,6 +1969,14 @@ export default class IndexManager {
       return
     }
     
+    // Restore totalLines from saved data
+    if (index.totalLines !== undefined) {
+      this.totalLines = index.totalLines
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`)
+      }
+    }
+
     this.index = processedIndex
   }
 
@@ -2008,7 +2016,10 @@ export default class IndexManager {
    * This resolves the issue where Sets appear as empty objects in JSON.stringify
    */
   toJSON() {
-    const serializable = { data: {} }
+    const serializable = {
+      data: {},
+      totalLines: this.totalLines
+    }
     
     // Check if this is a term mapping field for conversion
     const isTermMappingField = (field) => {

package/src/managers/QueryManager.mjs

@@ -1215,7 +1215,7 @@ export class QueryManager {
         return false;
       }
       
-      if (typeof condition === 'object' && !Array.isArray(condition)) {
+      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
         const operators = Object.keys(condition).map(op => normalizeOperator(op));
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' has operators:`, operators)
@@ -1532,8 +1532,8 @@ export class QueryManager {
       
       if (termMappingFields.includes(field)) {
         // For term mapping fields, simple equality or $in queries work well
-        if (typeof condition === 'string' || 
-            (typeof condition === 'object' && condition.$in && Array.isArray(condition.$in))) {
+        if (typeof condition === 'string' ||
+            (typeof condition === 'object' && condition !== null && condition.$in && Array.isArray(condition.$in))) {
           return true;
         }
       }