jexidb 2.1.2 → 2.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jexidb",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "type": "module",
   "description": "JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.",
   "main": "./dist/Database.cjs",

package/src/Database.mjs

@@ -1319,38 +1319,21 @@ class Database extends EventEmitter {
         this.pendingIndexUpdates = []
       }
       
-      // CRITICAL FIX: Flush write buffer completely after capturing snapshot
-      await this._flushWriteBufferCompletely()
-      
-      // CRITICAL FIX: Wait for all I/O operations to complete before clearing writeBuffer
-      await this._waitForIOCompletion()
-      
-      // CRITICAL FIX: Verify write buffer is empty after I/O completion
-      // But allow for ongoing insertions during high-volume scenarios
-      if (this.writeBuffer.length > 0) {
-        if (this.opts.debugMode) {
-          console.log(`💾 Save: WriteBuffer still has ${this.writeBuffer.length} items after flush - this may indicate ongoing insertions`)
-        }
-        
-        // If we have a reasonable number of items, continue processing
-        if (this.writeBuffer.length < 10000) { // Reasonable threshold
-          if (this.opts.debugMode) {
-            console.log(`💾 Save: Continuing to process remaining ${this.writeBuffer.length} items`)
-          }
-          // Continue with the save process - the remaining items will be included in the final save
-        } else {
-          // Too many items remaining - likely a real problem
-          throw new Error(`WriteBuffer has too many items after flush: ${this.writeBuffer.length} items remaining (threshold: 10000)`)
-        }
+      // CRITICAL FIX: DO NOT flush writeBuffer before processing existing records
+      // This prevents duplicating updated records in the file.
+      // The _streamExistingRecords() will handle replacing old records with updated ones from writeBufferSnapshot.
+      // After processing, all records (existing + updated + new) will be written to file in one operation.
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: writeBufferSnapshot captured with ${writeBufferSnapshot.length} records (will be processed with existing records)`)
       }
       
       // OPTIMIZATION: Parallel operations - cleanup and data preparation
       let allData = []
       let orphanedCount = 0
       
-      // Check if there are new records to save (after flush, writeBuffer should be empty)
-      // CRITICAL FIX: Also check writeBufferSnapshot.length > 0 to handle updates/deletes
-      // that were in writeBuffer before flush but are now in snapshot
+      // Check if there are records to save from writeBufferSnapshot
+      // CRITICAL FIX: Process writeBufferSnapshot records (both new and updated) with existing records
+      // Updated records will replace old ones via _streamExistingRecords, new records will be added
       if (this.opts.debugMode) {
         console.log(`💾 Save: writeBuffer.length=${this.writeBuffer.length}, writeBufferSnapshot.length=${writeBufferSnapshot.length}`)
       }
@@ -1397,11 +1380,49 @@ class Database extends EventEmitter {
               // CRITICAL FIX: Normalize IDs to strings for consistent comparison
               const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)))
               
+              // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+              const existingRecordsById = new Map()
+              existingRecords.forEach(r => {
+                if (r && r.id) {
+                  existingRecordsById.set(String(r.id), r)
+                }
+              })
+              
               // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+              // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
               for (const record of writeBufferSnapshot) {
-                if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(String(record.id))) {
+                if (!record || !record.id) continue
+                if (deletedIdsSnapshot.has(String(record.id))) continue
+                
+                const recordIdStr = String(record.id)
+                const existingRecord = existingRecordsById.get(recordIdStr)
+                
+                if (!existingRecord) {
                   // This is a new record, not an update
                   allData.push(record)
+                  if (this.opts.debugMode) {
+                    console.log(`💾 Save: Adding NEW record to allData:`, { id: recordIdStr, price: record.price, app_id: record.app_id, currency: record.currency })
+                  }
+                } else {
+                  // This is an update - verify that existingRecords contains the updated version
+                  // If not, replace it (this handles edge cases where substitution might have failed)
+                  const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr)
+                  if (existingIndex !== -1) {
+                    // Verify if the existing record is actually the updated one
+                    // Compare key fields to detect if replacement is needed
+                    const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record)
+                    if (needsReplacement) {
+                      if (this.opts.debugMode) {
+                        console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                          old: { id: String(allData[existingIndex].id), price: allData[existingIndex].price },
+                          new: { id: recordIdStr, price: record.price }
+                        })
+                      }
+                      allData[existingIndex] = record
+                    } else if (this.opts.debugMode) {
+                      console.log(`💾 Save: Record already correctly updated in allData:`, { id: recordIdStr })
+                    }
+                  }
                 }
               }
             })
@@ -1452,13 +1473,52 @@ class Database extends EventEmitter {
                 allData = [...existingRecords]
                 
                 // OPTIMIZATION: Use Set for faster lookups of existing record IDs
-                const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => r.id))
+                // CRITICAL FIX: Normalize IDs to strings for consistent comparison
+                const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)))
+                
+                // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+                const existingRecordsById = new Map()
+                existingRecords.forEach(r => {
+                  if (r && r.id) {
+                    existingRecordsById.set(String(r.id), r)
+                  }
+                })
                 
                 // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+                // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
                 for (const record of writeBufferSnapshot) {
-                  if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(record.id)) {
+                  if (!record || !record.id) continue
+                  if (deletedIdsSnapshot.has(String(record.id))) continue
+                  
+                  const recordIdStr = String(record.id)
+                  const existingRecord = existingRecordsById.get(recordIdStr)
+                  
+                  if (!existingRecord) {
                     // This is a new record, not an update
                     allData.push(record)
+                    if (this.opts.debugMode) {
+                      console.log(`💾 Save: Adding NEW record to allData:`, { id: recordIdStr, price: record.price, app_id: record.app_id, currency: record.currency })
+                    }
+                  } else {
+                    // This is an update - verify that existingRecords contains the updated version
+                    // If not, replace it (this handles edge cases where substitution might have failed)
+                    const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr)
+                    if (existingIndex !== -1) {
+                      // Verify if the existing record is actually the updated one
+                      // Compare key fields to detect if replacement is needed
+                      const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record)
+                      if (needsReplacement) {
+                        if (this.opts.debugMode) {
+                          console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                            old: { id: String(allData[existingIndex].id), price: allData[existingIndex].price },
+                            new: { id: recordIdStr, price: record.price }
+                          })
+                        }
+                        allData[existingIndex] = record
+                      } else if (this.opts.debugMode) {
+                        console.log(`💾 Save: Record already correctly updated in allData:`, { id: recordIdStr })
+                      }
+                    }
                   }
                 }
                 
@@ -1466,10 +1526,11 @@ class Database extends EventEmitter {
                   const updatedCount = writeBufferSnapshot.filter(r => r && r.id && existingRecordIds.has(String(r.id))).length
                   const newCount = writeBufferSnapshot.filter(r => r && r.id && !existingRecordIds.has(String(r.id))).length
                   console.log(`💾 Save: Combined data - existingRecords: ${existingRecords.length}, updatedFromBuffer: ${updatedCount}, newFromBuffer: ${newCount}, total: ${allData.length}`)
-                  console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? r.id : 'no-id'))
+                  console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? String(r.id) : 'no-id'))
                   console.log(`💾 Save: Existing record IDs:`, Array.from(existingRecordIds))
-                  console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? { id: existingRecords[0].id, name: existingRecords[0].name, tags: existingRecords[0].tags } : 'null')
-                  console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? { id: writeBufferSnapshot[0].id, name: writeBufferSnapshot[0].name, tags: writeBufferSnapshot[0].tags } : 'null')
+                  console.log(`💾 Save: All records in allData:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
+                  console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? { id: String(existingRecords[0].id), price: existingRecords[0].price, app_id: existingRecords[0].app_id, currency: existingRecords[0].currency } : 'null')
+                  console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? { id: String(writeBufferSnapshot[0].id), price: writeBufferSnapshot[0].price, app_id: writeBufferSnapshot[0].app_id, currency: writeBufferSnapshot[0].currency } : 'null')
                 }
               }).catch(error => {
                 if (this.opts.debugMode) {
@@ -1545,6 +1606,7 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`)
+        console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null')
       }
       
@@ -1556,6 +1618,7 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: Serialized ${lines.length} lines`)
+        console.log(`💾 Save: All records in allData after serialization check:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         if (lines.length > 0) {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200))
         }
@@ -1578,56 +1641,9 @@ class Database extends EventEmitter {
         console.log(`💾 Save: Calculated indexOffset: ${this.indexOffset}, allData.length: ${allData.length}`)
       }
       
-      // OPTIMIZATION: Parallel operations - file writing and index data preparation
-      const parallelWriteOperations = []
-      
-      // Add main file write operation
-      parallelWriteOperations.push(
-        this.fileHandler.writeBatch([jsonlData])
-      )
-      
-      // Add index file operations - ALWAYS save offsets, even without indexed fields
-      if (this.indexManager) {
-        const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
-        
-        // OPTIMIZATION: Parallel data preparation
-        const indexDataPromise = Promise.resolve({
-          index: this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {},
-          offsets: this.offsets, // Save actual offsets for efficient file operations
-          indexOffset: this.indexOffset // Save file size for proper range calculations
-        })
-        
-        // Add term mapping data if needed
-        const termMappingFields = this.getTermMappingFields()
-        if (termMappingFields.length > 0 && this.termManager) {
-          const termDataPromise = this.termManager.saveTerms()
-          
-          // Combine index data and term data
-          const combinedDataPromise = Promise.all([indexDataPromise, termDataPromise]).then(([indexData, termData]) => {
-            indexData.termMapping = termData
-            return indexData
-          })
-          
-          // Add index file write operation
-          parallelWriteOperations.push(
-            combinedDataPromise.then(indexData => {
-              const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
-              return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2))
-            })
-          )
-        } else {
-          // Add index file write operation without term mapping
-          parallelWriteOperations.push(
-            indexDataPromise.then(indexData => {
-              const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
-              return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2))
-            })
-          )
-        }
-      }
-      
-      // Execute parallel write operations
-      await Promise.all(parallelWriteOperations)
+      // CRITICAL FIX: Write main data file first
+      // Index will be saved AFTER reconstruction to ensure it contains correct data
+      await this.fileHandler.writeBatch([jsonlData])
       
       if (this.opts.debugMode) {
         console.log(`💾 Saved ${allData.length} records to ${this.normalizedFile}`)
@@ -1661,9 +1677,16 @@ class Database extends EventEmitter {
             
             // Rebuild index from the saved records
             // CRITICAL: Process term mapping for records loaded from file to ensure ${field}Ids are available
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Rebuilding index from ${allData.length} records in allData`)
+            }
             for (let i = 0; i < allData.length; i++) {
               let record = allData[i]
               
+              if (this.opts.debugMode && i < 3) {
+                console.log(`💾 Save: Rebuilding index record[${i}]:`, { id: String(record.id), price: record.price, app_id: record.app_id, currency: record.currency })
+              }
+              
               // CRITICAL FIX: Ensure records have ${field}Ids for term mapping fields
               // Records from writeBuffer already have ${field}Ids from processTermMapping
               // Records from file need to be processed to restore ${field}Ids
@@ -1690,6 +1713,9 @@ class Database extends EventEmitter {
               
               await this.indexManager.add(record, i)
             }
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records`)
+            }
           }
         }
         
@@ -2574,11 +2600,20 @@ class Database extends EventEmitter {
         
         const updated = { ...record, ...updateData }
         
+        // DEBUG: Log the update operation details
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: Original record ID: ${record.id}, type: ${typeof record.id}`)
+          console.log(`🔄 UPDATE: Updated record ID: ${updated.id}, type: ${typeof updated.id}`)
+          console.log(`🔄 UPDATE: Update data keys:`, Object.keys(updateData))
+          console.log(`🔄 UPDATE: Updated record keys:`, Object.keys(updated))
+        }
+        
         // Process term mapping for update
         const termMappingStart = Date.now()
         this.processTermMapping(updated, true, record)
         if (this.opts.debugMode) {
           console.log(`🔄 UPDATE: Term mapping completed in ${Date.now() - termMappingStart}ms`)
+          console.log(`🔄 UPDATE: After term mapping - ID: ${updated.id}, type: ${typeof updated.id}`)
         }
         
         // CRITICAL FIX: Remove old terms from index before adding new ones
@@ -4084,14 +4119,23 @@ class Database extends EventEmitter {
     // Create a map of updated records for quick lookup
     // CRITICAL FIX: Normalize IDs to strings for consistent comparison
     const updatedRecordsMap = new Map()
-    writeBufferSnapshot.forEach(record => {
+    writeBufferSnapshot.forEach((record, index) => {
       if (record && record.id !== undefined && record.id !== null) {
         // Normalize ID to string for consistent comparison
         const normalizedId = String(record.id)
         updatedRecordsMap.set(normalizedId, record)
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Added to updatedRecordsMap: ID=${normalizedId} (original: ${record.id}, type: ${typeof record.id}), index=${index}`)
+        }
+      } else if (this.opts.debugMode) {
+        console.log(`⚠️ Save: Skipped record in writeBufferSnapshot[${index}] - missing or invalid ID:`, record ? { id: record.id, keys: Object.keys(record) } : 'null')
       }
     })
     
+    if (this.opts.debugMode) {
+      console.log(`💾 Save: updatedRecordsMap size: ${updatedRecordsMap.size}, keys:`, Array.from(updatedRecordsMap.keys()))
+    }
+    
     // OPTIMIZATION: Cache file stats to avoid repeated stat() calls
     let fileSize = 0
     if (this._cachedFileStats && this._cachedFileStats.timestamp > Date.now() - 1000) {
@@ -4301,11 +4345,20 @@ class Database extends EventEmitter {
         
         // CRITICAL FIX: Normalize ID to string for consistent comparison
         const normalizedId = String(recordWithIds.id)
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Checking record ID=${normalizedId} (original: ${recordWithIds.id}, type: ${typeof recordWithIds.id}) in updatedRecordsMap`)
+          console.log(`💾 Save: updatedRecordsMap.has(${normalizedId}): ${updatedRecordsMap.has(normalizedId)}`)
+          if (!updatedRecordsMap.has(normalizedId)) {
+            console.log(`💾 Save: Record ${normalizedId} NOT found in updatedRecordsMap. Available keys:`, Array.from(updatedRecordsMap.keys()))
+          }
+        }
         if (updatedRecordsMap.has(normalizedId)) {
           // Replace with updated version
           const updatedRecord = updatedRecordsMap.get(normalizedId)
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Updated record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`)
+            console.log(`💾 Save: ✅ REPLACING record ${recordWithIds.id} with updated version`)
+            console.log(`💾 Save: Old record:`, { id: recordWithIds.id, price: recordWithIds.price, app_id: recordWithIds.app_id, currency: recordWithIds.currency })
+            console.log(`💾 Save: New record:`, { id: updatedRecord.id, price: updatedRecord.price, app_id: updatedRecord.app_id, currency: updatedRecord.currency })
           }
           return { 
             type: 'updated',