jexidb 2.1.2 → 2.1.3

package/dist/Database.cjs

@@ -2257,9 +2257,9 @@ class IndexManager {
   }
 }
 
-/**
- * SchemaManager - Manages field schemas for optimized array-based serialization
- * This replaces the need for repeating field names in JSON objects
+/**
+ * SchemaManager - Manages field schemas for optimized array-based serialization
+ * This replaces the need for repeating field names in JSON objects
  */
 class SchemaManager {
   constructor(opts = {}) {
@@ -2277,8 +2277,8 @@ class SchemaManager {
     this.isInitialized = false;
   }
 
-  /**
-   * Initialize schema from options or auto-detect from data
+  /**
+   * Initialize schema from options or auto-detect from data
    */
   initializeSchema(schemaOrData, autoDetect = false) {
     if (this.isInitialized && this.opts.strictSchema) {
@@ -2303,8 +2303,8 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Set explicit schema
+  /**
+   * Set explicit schema
    */
   setSchema(fieldNames) {
     this.schema = [...fieldNames]; // Create copy
@@ -2319,8 +2319,8 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Auto-detect schema from sample data
+  /**
+   * Auto-detect schema from sample data
    */
   autoDetectSchema(sampleData) {
     if (Array.isArray(sampleData)) {
@@ -2341,9 +2341,9 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Initialize schema from database options
-   * Note: schema option is no longer supported, use fields instead
+  /**
+   * Initialize schema from database options
+   * Note: schema option is no longer supported, use fields instead
    */
   initializeFromOptions(opts) {
     // Schema option is no longer supported - fields should be used instead
@@ -2351,8 +2351,8 @@ class SchemaManager {
     // Schema initialization is handled by Database.initializeSchema() using fields
   }
 
-  /**
-   * Add new field to schema (for schema evolution)
+  /**
+   * Add new field to schema (for schema evolution)
    */
   addField(fieldName) {
     if (this.fieldToIndex.has(fieldName)) {
@@ -2368,8 +2368,8 @@ class SchemaManager {
     return newIndex;
   }
 
-  /**
-   * Convert object to array using schema with strict field enforcement
+  /**
+   * Convert object to array using schema with strict field enforcement
    */
   objectToArray(obj) {
     if (!this.isInitialized || !this.opts.enableArraySerialization) {
@@ -2395,8 +2395,8 @@ class SchemaManager {
     return result;
   }
 
-  /**
-   * Convert array back to object using schema
+  /**
+   * Convert array back to object using schema
    */
   arrayToObject(arr) {
     if (!this.isInitialized || !this.opts.enableArraySerialization) {
@@ -2406,31 +2406,61 @@ class SchemaManager {
       return arr; // Don't convert non-arrays
     }
     const obj = {};
+    const idIndex = this.schema.indexOf('id');
+
+    // CRITICAL FIX: Handle schema migration where 'id' was first field in old schema
+    // but is not in current schema. Check if first element looks like an ID.
+    // Only do this if:
+    // 1. 'id' is not in current schema
+    // 2. Array has significantly more elements than current schema (2+ extra elements)
+    //    This suggests the old schema had more fields, and 'id' was likely the first
+    // 3. First element is a very short string (max 20 chars) that looks like a generated ID
+    //    (typically alphanumeric, often starting with letters like 'mit...' or similar patterns)
+    // 4. First field in current schema is not 'id' (to avoid false positives)
+    // 5. First element is not an array (to avoid false positives with array fields)
+    let arrayOffset = 0;
+    if (idIndex === -1 && arr.length >= this.schema.length + 2 && this.schema.length > 0) {
+      // Only apply if array has at least 2 extra elements (suggests old schema had more fields)
+      const firstElement = arr[0];
+      const firstFieldName = this.schema[0];
+
+      // Only apply shift if:
+      // - First field is not 'id'
+      // - First element is a very short string (max 20 chars) that looks like a generated ID
+      // - First element is not an array (to avoid false positives)
+      // - Array has at least 2 extra elements (strong indicator of schema migration)
+      if (firstFieldName !== 'id' && typeof firstElement === 'string' && !Array.isArray(firstElement) && firstElement.length > 0 && firstElement.length <= 20 &&
+      // Very conservative: max 20 chars (typical ID length)
+      /^[a-zA-Z0-9_-]+$/.test(firstElement)) {
+        // First element is likely the ID from old schema
+        obj.id = firstElement;
+        arrayOffset = 1;
+      }
+    }
 
     // Map array values to object properties
     // Only include fields that are in the schema
-    for (let i = 0; i < Math.min(arr.length, this.schema.length); i++) {
+    for (let i = 0; i < Math.min(arr.length - arrayOffset, this.schema.length); i++) {
       const fieldName = this.schema[i];
+      const arrayIndex = i + arrayOffset;
       // Only include non-undefined values to avoid cluttering the object
-      if (arr[i] !== undefined) {
-        obj[fieldName] = arr[i];
+      if (arr[arrayIndex] !== undefined) {
+        obj[fieldName] = arr[arrayIndex];
       }
     }
 
     // CRITICAL FIX: Always preserve 'id' field if it exists in the original object
     // The 'id' field may not be in the schema but must be preserved
-    // Check if 'id' is in the schema first
-    const idIndex = this.schema.indexOf('id');
     if (idIndex !== -1 && arr[idIndex] !== undefined) {
       // 'id' is in schema and has a value
       obj.id = arr[idIndex];
-    } else if (idIndex === -1 && arr.length > this.schema.length) {
+    } else if (!obj.id && arr.length > this.schema.length + arrayOffset) {
       // 'id' is not in schema but array has extra element(s) - check if last element could be ID
       // This handles cases where ID was added after schema initialization
-      for (let i = this.schema.length; i < arr.length; i++) {
+      for (let i = this.schema.length + arrayOffset; i < arr.length; i++) {
         // Try to infer if this is an ID (string that looks like an ID)
         const potentialId = arr[i];
-        if (potentialId !== undefined && potentialId !== null && typeof potentialId === 'string' && potentialId.length > 0) {
+        if (potentialId !== undefined && potentialId !== null && typeof potentialId === 'string' && potentialId.length > 0 && potentialId.length < 100) {
           obj.id = potentialId;
           break; // Use first potential ID found
         }
@@ -2439,43 +2469,43 @@ class SchemaManager {
     return obj;
   }
 
-  /**
-   * Get field index by name
+  /**
+   * Get field index by name
    */
   getFieldIndex(fieldName) {
     return this.fieldToIndex.get(fieldName);
   }
 
-  /**
-   * Get field name by index
+  /**
+   * Get field name by index
    */
   getFieldName(index) {
     return this.indexToField.get(index);
   }
 
-  /**
-   * Check if field exists in schema
+  /**
+   * Check if field exists in schema
    */
   hasField(fieldName) {
     return this.fieldToIndex.has(fieldName);
   }
 
-  /**
-   * Get schema as array of field names
+  /**
+   * Get schema as array of field names
    */
   getSchema() {
     return [...this.schema]; // Return copy
   }
 
-  /**
-   * Get schema size
+  /**
+   * Get schema size
    */
   getSchemaSize() {
     return this.schema.length;
   }
 
-  /**
-   * Validate that object conforms to schema
+  /**
+   * Validate that object conforms to schema
    */
   validateObject(obj) {
     if (!this.isInitialized || !this.opts.strictSchema) {
@@ -2497,8 +2527,8 @@ class SchemaManager {
     return true;
   }
 
-  /**
-   * Get schema metadata for serialization
+  /**
+   * Get schema metadata for serialization
    */
   getSchemaMetadata() {
     return {
@@ -2509,8 +2539,8 @@ class SchemaManager {
     };
   }
 
-  /**
-   * Reset schema
+  /**
+   * Reset schema
    */
   reset() {
     this.schema = [];
@@ -2520,8 +2550,8 @@ class SchemaManager {
     this.schemaVersion++;
   }
 
-  /**
-   * Get performance statistics
+  /**
+   * Get performance statistics
    */
   getStats() {
     return {
@@ -5134,9 +5164,23 @@ class QueryManager {
       }
     }
 
+    // Handle $not operator - include it if it can be processed by IndexManager
+    if (criteria.$not && typeof criteria.$not === 'object') {
+      // Check if $not condition contains only indexable fields
+      const notFields = Object.keys(criteria.$not);
+      const allNotFieldsIndexed = notFields.every(field => this.indexManager.opts.indexes && this.indexManager.opts.indexes[field]);
+      if (allNotFieldsIndexed && notFields.length > 0) {
+        // Extract indexable criteria from $not condition
+        const indexableNotCriteria = this._extractIndexableCriteria(criteria.$not);
+        if (Object.keys(indexableNotCriteria).length > 0) {
+          indexableCriteria.$not = indexableNotCriteria;
+        }
+      }
+    }
+
     // Handle regular field conditions
     for (const [field, condition] of Object.entries(criteria)) {
-      if (field.startsWith('$')) continue; // Skip logical operators
+      if (field.startsWith('$')) continue; // Skip logical operators (already handled above)
 
       // RegExp conditions cannot be pre-filtered using indices
       if (condition instanceof RegExp) {
@@ -5484,47 +5528,82 @@ class QueryManager {
     // Read specific records using the line numbers
     if (lineNumbers.size > 0) {
       const lineNumbersArray = Array.from(lineNumbers);
-      const ranges = this.database.getRanges(lineNumbersArray);
-      const groupedRanges = await this.database.fileHandler.groupedRanges(ranges);
-      const fs = await import('fs');
-      const fd = await fs.promises.open(this.database.fileHandler.file, 'r');
-      try {
-        for (const groupedRange of groupedRanges) {
-          var _iteratorAbruptCompletion2 = false;
-          var _didIteratorError2 = false;
-          var _iteratorError2;
+      const persistedCount = Array.isArray(this.database.offsets) ? this.database.offsets.length : 0;
+
+      // Separate lineNumbers into file records and writeBuffer records
+      const fileLineNumbers = [];
+      const writeBufferLineNumbers = [];
+      for (const lineNumber of lineNumbersArray) {
+        if (lineNumber >= persistedCount) {
+          // This lineNumber points to writeBuffer
+          writeBufferLineNumbers.push(lineNumber);
+        } else {
+          // This lineNumber points to file
+          fileLineNumbers.push(lineNumber);
+        }
+      }
+
+      // Read records from file
+      if (fileLineNumbers.length > 0) {
+        const ranges = this.database.getRanges(fileLineNumbers);
+        if (ranges.length > 0) {
+          const groupedRanges = await this.database.fileHandler.groupedRanges(ranges);
+          const fs = await import('fs');
+          const fd = await fs.promises.open(this.database.fileHandler.file, 'r');
           try {
-            for (var _iterator2 = _asyncIterator(this.database.fileHandler.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = await _iterator2.next()).done; _iteratorAbruptCompletion2 = false) {
-              const row = _step2.value;
-              {
+            for (const groupedRange of groupedRanges) {
+              var _iteratorAbruptCompletion2 = false;
+              var _didIteratorError2 = false;
+              var _iteratorError2;
+              try {
+                for (var _iterator2 = _asyncIterator(this.database.fileHandler.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = await _iterator2.next()).done; _iteratorAbruptCompletion2 = false) {
+                  const row = _step2.value;
+                  {
+                    try {
+                      const record = this.database.serializer.deserialize(row.line);
+                      const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
+                      results.push(recordWithTerms);
+                      if (limit && results.length >= limit) break;
+                    } catch (error) {
+                      // Skip invalid lines
+                    }
+                  }
+                }
+              } catch (err) {
+                _didIteratorError2 = true;
+                _iteratorError2 = err;
+              } finally {
                 try {
-                  const record = this.database.serializer.deserialize(row.line);
-                  const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
-                  results.push(recordWithTerms);
-                  if (limit && results.length >= limit) break;
-                } catch (error) {
-                  // Skip invalid lines
+                  if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
+                    await _iterator2.return();
+                  }
+                } finally {
+                  if (_didIteratorError2) {
+                    throw _iteratorError2;
+                  }
                 }
               }
+              if (limit && results.length >= limit) break;
             }
-          } catch (err) {
-            _didIteratorError2 = true;
-            _iteratorError2 = err;
           } finally {
-            try {
-              if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
-                await _iterator2.return();
-              }
-            } finally {
-              if (_didIteratorError2) {
-                throw _iteratorError2;
-              }
-            }
+            await fd.close();
           }
+        }
+      }
+
+      // Read records from writeBuffer
+      if (writeBufferLineNumbers.length > 0 && this.database.writeBuffer) {
+        for (const lineNumber of writeBufferLineNumbers) {
           if (limit && results.length >= limit) break;
+          const writeBufferIndex = lineNumber - persistedCount;
+          if (writeBufferIndex >= 0 && writeBufferIndex < this.database.writeBuffer.length) {
+            const record = this.database.writeBuffer[writeBufferIndex];
+            if (record) {
+              const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
+              results.push(recordWithTerms);
+            }
+          }
         }
-      } finally {
-        await fd.close();
       }
     }
     if (options.orderBy) {
@@ -5944,8 +6023,8 @@ class QueryManager {
       }
     }
     const allFieldsIndexed = Object.keys(criteria).every(field => {
-      // Skip $and as it's handled separately above
-      if (field === '$and') return true;
+      // Skip $and and $not as they're handled separately above
+      if (field === '$and' || field === '$not') return true;
       if (!this.opts.indexes || !this.opts.indexes[field]) {
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' not indexed. Available indexes:`, Object.keys(this.opts.indexes || {}));
@@ -8654,39 +8733,21 @@ class Database extends events.EventEmitter {
         this.pendingIndexUpdates = [];
       }
 
-      // CRITICAL FIX: Flush write buffer completely after capturing snapshot
-      await this._flushWriteBufferCompletely();
-
-      // CRITICAL FIX: Wait for all I/O operations to complete before clearing writeBuffer
-      await this._waitForIOCompletion();
-
-      // CRITICAL FIX: Verify write buffer is empty after I/O completion
-      // But allow for ongoing insertions during high-volume scenarios
-      if (this.writeBuffer.length > 0) {
-        if (this.opts.debugMode) {
-          console.log(`💾 Save: WriteBuffer still has ${this.writeBuffer.length} items after flush - this may indicate ongoing insertions`);
-        }
-
-        // If we have a reasonable number of items, continue processing
-        if (this.writeBuffer.length < 10000) {
-          // Reasonable threshold
-          if (this.opts.debugMode) {
-            console.log(`💾 Save: Continuing to process remaining ${this.writeBuffer.length} items`);
-          }
-          // Continue with the save process - the remaining items will be included in the final save
-        } else {
-          // Too many items remaining - likely a real problem
-          throw new Error(`WriteBuffer has too many items after flush: ${this.writeBuffer.length} items remaining (threshold: 10000)`);
-        }
+      // CRITICAL FIX: DO NOT flush writeBuffer before processing existing records
+      // This prevents duplicating updated records in the file.
+      // The _streamExistingRecords() will handle replacing old records with updated ones from writeBufferSnapshot.
+      // After processing, all records (existing + updated + new) will be written to file in one operation.
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: writeBufferSnapshot captured with ${writeBufferSnapshot.length} records (will be processed with existing records)`);
       }
 
       // OPTIMIZATION: Parallel operations - cleanup and data preparation
       let allData = [];
       let orphanedCount = 0;
 
-      // Check if there are new records to save (after flush, writeBuffer should be empty)
-      // CRITICAL FIX: Also check writeBufferSnapshot.length > 0 to handle updates/deletes
-      // that were in writeBuffer before flush but are now in snapshot
+      // Check if there are records to save from writeBufferSnapshot
+      // CRITICAL FIX: Process writeBufferSnapshot records (both new and updated) with existing records
+      // Updated records will replace old ones via _streamExistingRecords, new records will be added
       if (this.opts.debugMode) {
         console.log(`💾 Save: writeBuffer.length=${this.writeBuffer.length}, writeBufferSnapshot.length=${writeBufferSnapshot.length}`);
       }
@@ -8729,11 +8790,60 @@ class Database extends events.EventEmitter {
             // CRITICAL FIX: Normalize IDs to strings for consistent comparison
             const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)));
 
+            // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+            const existingRecordsById = new Map();
+            existingRecords.forEach(r => {
+              if (r && r.id) {
+                existingRecordsById.set(String(r.id), r);
+              }
+            });
+
             // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+            // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
             for (const record of writeBufferSnapshot) {
-              if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(String(record.id))) {
+              if (!record || !record.id) continue;
+              if (deletedIdsSnapshot.has(String(record.id))) continue;
+              const recordIdStr = String(record.id);
+              const existingRecord = existingRecordsById.get(recordIdStr);
+              if (!existingRecord) {
                 // This is a new record, not an update
                 allData.push(record);
+                if (this.opts.debugMode) {
+                  console.log(`💾 Save: Adding NEW record to allData:`, {
+                    id: recordIdStr,
+                    price: record.price,
+                    app_id: record.app_id,
+                    currency: record.currency
+                  });
+                }
+              } else {
+                // This is an update - verify that existingRecords contains the updated version
+                // If not, replace it (this handles edge cases where substitution might have failed)
+                const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr);
+                if (existingIndex !== -1) {
+                  // Verify if the existing record is actually the updated one
+                  // Compare key fields to detect if replacement is needed
+                  const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record);
+                  if (needsReplacement) {
+                    if (this.opts.debugMode) {
+                      console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                        old: {
+                          id: String(allData[existingIndex].id),
+                          price: allData[existingIndex].price
+                        },
+                        new: {
+                          id: recordIdStr,
+                          price: record.price
+                        }
+                      });
+                    }
+                    allData[existingIndex] = record;
+                  } else if (this.opts.debugMode) {
+                    console.log(`💾 Save: Record already correctly updated in allData:`, {
+                      id: recordIdStr
+                    });
+                  }
+                }
               }
             }
           }));
@@ -8780,30 +8890,88 @@ class Database extends events.EventEmitter {
               allData = [...existingRecords];
 
               // OPTIMIZATION: Use Set for faster lookups of existing record IDs
-              const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => r.id));
+              // CRITICAL FIX: Normalize IDs to strings for consistent comparison
+              const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)));
+
+              // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+              const existingRecordsById = new Map();
+              existingRecords.forEach(r => {
+                if (r && r.id) {
+                  existingRecordsById.set(String(r.id), r);
+                }
+              });
 
               // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+              // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
               for (const record of writeBufferSnapshot) {
-                if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(record.id)) {
+                if (!record || !record.id) continue;
+                if (deletedIdsSnapshot.has(String(record.id))) continue;
+                const recordIdStr = String(record.id);
+                const existingRecord = existingRecordsById.get(recordIdStr);
+                if (!existingRecord) {
                   // This is a new record, not an update
                   allData.push(record);
+                  if (this.opts.debugMode) {
+                    console.log(`💾 Save: Adding NEW record to allData:`, {
+                      id: recordIdStr,
+                      price: record.price,
+                      app_id: record.app_id,
+                      currency: record.currency
+                    });
+                  }
+                } else {
+                  // This is an update - verify that existingRecords contains the updated version
+                  // If not, replace it (this handles edge cases where substitution might have failed)
+                  const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr);
+                  if (existingIndex !== -1) {
+                    // Verify if the existing record is actually the updated one
+                    // Compare key fields to detect if replacement is needed
+                    const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record);
+                    if (needsReplacement) {
+                      if (this.opts.debugMode) {
+                        console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                          old: {
+                            id: String(allData[existingIndex].id),
+                            price: allData[existingIndex].price
+                          },
+                          new: {
+                            id: recordIdStr,
+                            price: record.price
+                          }
+                        });
+                      }
+                      allData[existingIndex] = record;
+                    } else if (this.opts.debugMode) {
+                      console.log(`💾 Save: Record already correctly updated in allData:`, {
+                        id: recordIdStr
+                      });
+                    }
+                  }
                 }
               }
               if (this.opts.debugMode) {
                 const updatedCount = writeBufferSnapshot.filter(r => r && r.id && existingRecordIds.has(String(r.id))).length;
                 const newCount = writeBufferSnapshot.filter(r => r && r.id && !existingRecordIds.has(String(r.id))).length;
                 console.log(`💾 Save: Combined data - existingRecords: ${existingRecords.length}, updatedFromBuffer: ${updatedCount}, newFromBuffer: ${newCount}, total: ${allData.length}`);
-                console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? r.id : 'no-id'));
+                console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? String(r.id) : 'no-id'));
                 console.log(`💾 Save: Existing record IDs:`, Array.from(existingRecordIds));
+                console.log(`💾 Save: All records in allData:`, allData.map(r => r && r.id ? {
+                  id: String(r.id),
+                  price: r.price,
+                  app_id: r.app_id,
+                  currency: r.currency
+                } : 'no-id'));
                 console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? {
-                  id: existingRecords[0].id,
-                  name: existingRecords[0].name,
-                  tags: existingRecords[0].tags
+                  id: String(existingRecords[0].id),
+                  price: existingRecords[0].price,
+                  app_id: existingRecords[0].app_id,
+                  currency: existingRecords[0].currency
                 } : 'null');
                 console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? {
-                  id: writeBufferSnapshot[0].id,
-                  name: writeBufferSnapshot[0].name,
-                  tags: writeBufferSnapshot[0].tags
+                  id: String(writeBufferSnapshot[0].id),
+                  price: writeBufferSnapshot[0].price,
+                  app_id: writeBufferSnapshot[0].app_id,
+                  currency: writeBufferSnapshot[0].currency
                 } : 'null');
               }
             }).catch(error => {
@@ -8874,6 +9042,12 @@ class Database extends events.EventEmitter {
       });
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`);
+        console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? {
+          id: String(r.id),
+          price: r.price,
+          app_id: r.app_id,
+          currency: r.currency
+        } : 'no-id'));
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null');
       }
       const jsonlData = cleanedData.length > 0 ? this.serializer.serializeBatch(cleanedData) : '';
@@ -8881,6 +9055,12 @@ class Database extends events.EventEmitter {
       const lines = jsonlString.split('\n').filter(line => line.trim());
       if (this.opts.debugMode) {
         console.log(`💾 Save: Serialized ${lines.length} lines`);
+        console.log(`💾 Save: All records in allData after serialization check:`, allData.map(r => r && r.id ? {
+          id: String(r.id),
+          price: r.price,
+          app_id: r.app_id,
+          currency: r.currency
+        } : 'no-id'));
         if (lines.length > 0) {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200));
         }
@@ -8901,51 +9081,9 @@ class Database extends events.EventEmitter {
         console.log(`💾 Save: Calculated indexOffset: ${this.indexOffset}, allData.length: ${allData.length}`);
       }
 
-      // OPTIMIZATION: Parallel operations - file writing and index data preparation
-      const parallelWriteOperations = [];
-
-      // Add main file write operation
-      parallelWriteOperations.push(this.fileHandler.writeBatch([jsonlData]));
-
-      // Add index file operations - ALWAYS save offsets, even without indexed fields
-      if (this.indexManager) {
-        const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb');
-
-        // OPTIMIZATION: Parallel data preparation
-        const indexDataPromise = Promise.resolve({
-          index: this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {},
-          offsets: this.offsets,
-          // Save actual offsets for efficient file operations
-          indexOffset: this.indexOffset // Save file size for proper range calculations
-        });
-
-        // Add term mapping data if needed
-        const termMappingFields = this.getTermMappingFields();
-        if (termMappingFields.length > 0 && this.termManager) {
-          const termDataPromise = this.termManager.saveTerms();
-
-          // Combine index data and term data
-          const combinedDataPromise = Promise.all([indexDataPromise, termDataPromise]).then(([indexData, termData]) => {
-            indexData.termMapping = termData;
-            return indexData;
-          });
-
-          // Add index file write operation
-          parallelWriteOperations.push(combinedDataPromise.then(indexData => {
-            const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts);
-            return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2));
-          }));
-        } else {
-          // Add index file write operation without term mapping
-          parallelWriteOperations.push(indexDataPromise.then(indexData => {
-            const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts);
-            return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2));
-          }));
-        }
-      }
-
-      // Execute parallel write operations
-      await Promise.all(parallelWriteOperations);
+      // CRITICAL FIX: Write main data file first
+      // Index will be saved AFTER reconstruction to ensure it contains correct data
+      await this.fileHandler.writeBatch([jsonlData]);
       if (this.opts.debugMode) {
         console.log(`💾 Saved ${allData.length} records to ${this.normalizedFile}`);
       }
@@ -8957,20 +9095,39 @@ class Database extends events.EventEmitter {
 
       // Clear writeBuffer and deletedIds after successful save only if we had data to save
       if (allData.length > 0) {
-        // Rebuild index when records were deleted to maintain consistency
+        // Rebuild index when records were deleted or updated to maintain consistency
         const hadDeletedRecords = deletedIdsSnapshot.size > 0;
+        const hadUpdatedRecords = writeBufferSnapshot.length > 0;
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
-          if (hadDeletedRecords) {
-            // Clear the index and rebuild it from the remaining records
+          if (hadDeletedRecords || hadUpdatedRecords) {
+            // Clear the index and rebuild it from the saved records
+            // This ensures that lineNumbers point to the correct positions in the file
             this.indexManager.clear();
             if (this.opts.debugMode) {
-              console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records`);
+              if (hadDeletedRecords && hadUpdatedRecords) {
+                console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records and updating ${writeBufferSnapshot.length} records`);
+              } else if (hadDeletedRecords) {
+                console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records`);
+              } else {
+                console.log(`🧹 Rebuilding index after updating ${writeBufferSnapshot.length} records`);
+              }
             }
 
             // Rebuild index from the saved records
             // CRITICAL: Process term mapping for records loaded from file to ensure ${field}Ids are available
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Rebuilding index from ${allData.length} records in allData`);
+            }
             for (let i = 0; i < allData.length; i++) {
               let record = allData[i];
+              if (this.opts.debugMode && i < 3) {
+                console.log(`💾 Save: Rebuilding index record[${i}]:`, {
+                  id: String(record.id),
+                  price: record.price,
+                  app_id: record.app_id,
+                  currency: record.currency
+                });
+              }
 
               // CRITICAL FIX: Ensure records have ${field}Ids for term mapping fields
               // Records from writeBuffer already have ${field}Ids from processTermMapping
@@ -8997,6 +9154,9 @@ class Database extends events.EventEmitter {
               }
               await this.indexManager.add(record, i);
             }
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records`);
+            }
           }
         }
 
@@ -9077,12 +9237,21 @@ class Database extends events.EventEmitter {
               this.termManager.decrementTermCount(termId);
             }
           } else if (oldRecord[field] && Array.isArray(oldRecord[field])) {
-            // Use terms to decrement (fallback for backward compatibility)
-            for (const term of oldRecord[field]) {
-              const termId = this.termManager.termToId.get(term);
-              if (termId) {
+            // Check if field contains term IDs (numbers) or terms (strings)
+            const firstValue = oldRecord[field][0];
+            if (typeof firstValue === 'number') {
+              // Field contains term IDs (from find with restoreTerms: false)
+              for (const termId of oldRecord[field]) {
                 this.termManager.decrementTermCount(termId);
               }
+            } else if (typeof firstValue === 'string') {
+              // Field contains terms (strings) - convert to term IDs
+              for (const term of oldRecord[field]) {
+                const termId = this.termManager.termToId.get(term);
+                if (termId) {
+                  this.termManager.decrementTermCount(termId);
+                }
+              }
             }
           }
         }
@@ -9323,6 +9492,7 @@ class Database extends events.EventEmitter {
         }
 
         // Apply schema enforcement - convert to array format and back to enforce schema
+        // This will discard any fields not in the schema
         const schemaEnforcedRecord = this.applySchemaEnforcement(record);
 
         // Don't store in this.data - only use writeBuffer and index
@@ -9828,19 +9998,12 @@ class Database extends events.EventEmitter {
             ...updateData
           };
 
-          // CRITICAL FIX: Update schema if new fields are added during update
-          // This ensures new fields (like 'status') are included in the serialized format
-          if (this.serializer && this.serializer.schemaManager) {
-            const newFields = Object.keys(updateData).filter(field => !record.hasOwnProperty(field));
-            for (const field of newFields) {
-              if (field !== 'id' && field !== '_') {
-                // Skip special fields
-                this.serializer.schemaManager.addField(field);
-                if (this.opts.debugMode) {
-                  console.log(`🔄 UPDATE: Added new field '${field}' to schema`);
-                }
-              }
-            }
+          // DEBUG: Log the update operation details
+          if (this.opts.debugMode) {
+            console.log(`🔄 UPDATE: Original record ID: ${record.id}, type: ${typeof record.id}`);
+            console.log(`🔄 UPDATE: Updated record ID: ${updated.id}, type: ${typeof updated.id}`);
+            console.log(`🔄 UPDATE: Update data keys:`, Object.keys(updateData));
+            console.log(`🔄 UPDATE: Updated record keys:`, Object.keys(updated));
           }
 
           // Process term mapping for update
@@ -9848,6 +10011,7 @@ class Database extends events.EventEmitter {
           this.processTermMapping(updated, true, record);
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Term mapping completed in ${Date.now() - termMappingStart}ms`);
+            console.log(`🔄 UPDATE: After term mapping - ID: ${updated.id}, type: ${typeof updated.id}`);
           }
 
           // CRITICAL FIX: Remove old terms from index before adding new ones
@@ -10821,64 +10985,112 @@ class Database extends events.EventEmitter {
     // Fetch actual records
     const lineNumbers = limitedEntries.map(([lineNumber]) => lineNumber);
     const scoresByLineNumber = new Map(limitedEntries);
-
-    // Use getRanges and fileHandler to read records
-    const ranges = this.getRanges(lineNumbers);
-    const groupedRanges = await this.fileHandler.groupedRanges(ranges);
-    const fs = await import('fs');
-    const fd = await fs.promises.open(this.fileHandler.file, 'r');
+    const persistedCount = Array.isArray(this.offsets) ? this.offsets.length : 0;
+
+    // Separate lineNumbers into file records and writeBuffer records
+    const fileLineNumbers = [];
+    const writeBufferLineNumbers = [];
+    for (const lineNumber of lineNumbers) {
+      if (lineNumber >= persistedCount) {
+        // This lineNumber points to writeBuffer
+        writeBufferLineNumbers.push(lineNumber);
+      } else {
+        // This lineNumber points to file
+        fileLineNumbers.push(lineNumber);
+      }
+    }
     const results = [];
-    try {
-      for (const groupedRange of groupedRanges) {
-        var _iteratorAbruptCompletion3 = false;
-        var _didIteratorError3 = false;
-        var _iteratorError3;
+
+    // Read records from file
+    if (fileLineNumbers.length > 0) {
+      const ranges = this.getRanges(fileLineNumbers);
+      if (ranges.length > 0) {
+        // Create a map from start offset to lineNumber for accurate mapping
+        const startToLineNumber = new Map();
+        for (const range of ranges) {
+          if (range.index !== undefined) {
+            startToLineNumber.set(range.start, range.index);
+          }
+        }
+        const groupedRanges = await this.fileHandler.groupedRanges(ranges);
+        const fs = await import('fs');
+        const fd = await fs.promises.open(this.fileHandler.file, 'r');
         try {
-          for (var _iterator3 = _asyncIterator(this.fileHandler.readGroupedRange(groupedRange, fd)), _step3; _iteratorAbruptCompletion3 = !(_step3 = await _iterator3.next()).done; _iteratorAbruptCompletion3 = false) {
-            const row = _step3.value;
-            {
-              try {
-                const record = this.serializer.deserialize(row.line);
+          for (const groupedRange of groupedRanges) {
+            var _iteratorAbruptCompletion3 = false;
+            var _didIteratorError3 = false;
+            var _iteratorError3;
+            try {
+              for (var _iterator3 = _asyncIterator(this.fileHandler.readGroupedRange(groupedRange, fd)), _step3; _iteratorAbruptCompletion3 = !(_step3 = await _iterator3.next()).done; _iteratorAbruptCompletion3 = false) {
+                const row = _step3.value;
+                {
+                  try {
+                    const record = this.serializer.deserialize(row.line);
 
-                // Get line number from the row
-                const lineNumber = row._ || 0;
+                    // Get line number from the row, fallback to start offset mapping
+                    let lineNumber = row._ !== null && row._ !== undefined ? row._ : startToLineNumber.get(row.start) ?? 0;
 
-                // Restore term IDs to terms
-                const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
+                    // Restore term IDs to terms
+                    const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
 
-                // Add line number
-                recordWithTerms._ = lineNumber;
+                    // Add line number
+                    recordWithTerms._ = lineNumber;
 
-                // Add score if includeScore is true
-                if (opts.includeScore) {
-                  recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
+                    // Add score if includeScore is true (default is true)
+                    if (opts.includeScore !== false) {
+                      recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
+                    }
+                    results.push(recordWithTerms);
+                  } catch (error) {
+                    // Skip invalid lines
+                    if (this.opts.debugMode) {
+                      console.error('Error deserializing record in score():', error);
+                    }
+                  }
                 }
-                results.push(recordWithTerms);
-              } catch (error) {
-                // Skip invalid lines
-                if (this.opts.debugMode) {
-                  console.error('Error deserializing record in score():', error);
+              }
+            } catch (err) {
+              _didIteratorError3 = true;
+              _iteratorError3 = err;
+            } finally {
+              try {
+                if (_iteratorAbruptCompletion3 && _iterator3.return != null) {
+                  await _iterator3.return();
+                }
+              } finally {
+                if (_didIteratorError3) {
+                  throw _iteratorError3;
                 }
               }
             }
           }
-        } catch (err) {
-          _didIteratorError3 = true;
-          _iteratorError3 = err;
         } finally {
-          try {
-            if (_iteratorAbruptCompletion3 && _iterator3.return != null) {
-              await _iterator3.return();
-            }
-          } finally {
-            if (_didIteratorError3) {
-              throw _iteratorError3;
+          await fd.close();
+        }
+      }
+    }
+
+    // Read records from writeBuffer
+    if (writeBufferLineNumbers.length > 0 && this.writeBuffer) {
+      for (const lineNumber of writeBufferLineNumbers) {
+        const writeBufferIndex = lineNumber - persistedCount;
+        if (writeBufferIndex >= 0 && writeBufferIndex < this.writeBuffer.length) {
+          const record = this.writeBuffer[writeBufferIndex];
+          if (record) {
+            // Restore term IDs to terms
+            const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
+
+            // Add line number
+            recordWithTerms._ = lineNumber;
+
+            // Add score if includeScore is true
+            if (opts.includeScore) {
+              recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
             }
+            results.push(recordWithTerms);
           }
         }
       }
-    } finally {
-      await fd.close();
     }
 
     // Re-sort results to maintain score order (since reads might be out of order)
@@ -11251,13 +11463,24 @@ class Database extends events.EventEmitter {
     // Create a map of updated records for quick lookup
     // CRITICAL FIX: Normalize IDs to strings for consistent comparison
     const updatedRecordsMap = new Map();
-    writeBufferSnapshot.forEach(record => {
+    writeBufferSnapshot.forEach((record, index) => {
       if (record && record.id !== undefined && record.id !== null) {
         // Normalize ID to string for consistent comparison
         const normalizedId = String(record.id);
         updatedRecordsMap.set(normalizedId, record);
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Added to updatedRecordsMap: ID=${normalizedId} (original: ${record.id}, type: ${typeof record.id}), index=${index}`);
+        }
+      } else if (this.opts.debugMode) {
+        console.log(`⚠️ Save: Skipped record in writeBufferSnapshot[${index}] - missing or invalid ID:`, record ? {
+          id: record.id,
+          keys: Object.keys(record)
+        } : 'null');
       }
     });
+    if (this.opts.debugMode) {
+      console.log(`💾 Save: updatedRecordsMap size: ${updatedRecordsMap.size}, keys:`, Array.from(updatedRecordsMap.keys()));
+    }
 
     // OPTIMIZATION: Cache file stats to avoid repeated stat() calls
     let fileSize = 0;
@@ -11460,11 +11683,30 @@ class Database extends events.EventEmitter {
 
         // CRITICAL FIX: Normalize ID to string for consistent comparison
         const normalizedId = String(recordWithIds.id);
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Checking record ID=${normalizedId} (original: ${recordWithIds.id}, type: ${typeof recordWithIds.id}) in updatedRecordsMap`);
+          console.log(`💾 Save: updatedRecordsMap.has(${normalizedId}): ${updatedRecordsMap.has(normalizedId)}`);
+          if (!updatedRecordsMap.has(normalizedId)) {
+            console.log(`💾 Save: Record ${normalizedId} NOT found in updatedRecordsMap. Available keys:`, Array.from(updatedRecordsMap.keys()));
+          }
+        }
         if (updatedRecordsMap.has(normalizedId)) {
           // Replace with updated version
           const updatedRecord = updatedRecordsMap.get(normalizedId);
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Updated record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`);
+            console.log(`💾 Save: ✅ REPLACING record ${recordWithIds.id} with updated version`);
+            console.log(`💾 Save: Old record:`, {
+              id: recordWithIds.id,
+              price: recordWithIds.price,
+              app_id: recordWithIds.app_id,
+              currency: recordWithIds.currency
+            });
+            console.log(`💾 Save: New record:`, {
+              id: updatedRecord.id,
+              price: updatedRecord.price,
+              app_id: updatedRecord.app_id,
+              currency: updatedRecord.currency
+            });
           }
           return {
             type: 'updated',