jexidb 1.0.5 → 1.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jexidb",
-  "version": "1.0.5",
+  "version": "1.0.8",
   "description": "JexiDB is a pure JS NPM library for managing data on disk using JSONL efficiently, without the need for a server.",
   "main": "./dist/Database.cjs",
   "module": "./src/Database.mjs",
@@ -23,6 +23,7 @@
     "@babel/preset-env": "^7.25.4"
   },
   "dependencies": {
+    "async-mutex": "^0.5.0",
     "p-limit": "^6.1.0"
   },
   "directories": {

package/src/Database.mjs

@@ -2,14 +2,15 @@ import { EventEmitter } from 'events'
 import FileHandler from './FileHandler.mjs'
 import IndexManager from './IndexManager.mjs'
 import Serializer from './Serializer.mjs'
+import { Mutex } from 'async-mutex'
 import fs from 'fs'
 
 export class Database extends EventEmitter {
-  constructor(file, opts={}) {
+  constructor(file, opts = {}) {
     super()
     this.opts = Object.assign({
       v8: false,
-      index: {data: {}},
+      index: { data: {} },
       indexes: {},
       compress: false,
       compressIndex: false,
@@ -22,29 +23,30 @@ export class Database extends EventEmitter {
     this.indexManager = new IndexManager(this.opts)
     this.indexOffset = 0
     this.writeBuffer = []
+    this.mutex = new Mutex()
   }
 
   use(plugin) {
-    if(this.destroyed) throw new Error('Database is destroyed')
+    if (this.destroyed) throw new Error('Database is destroyed')
     plugin(this)
   }
 
   async init() {
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(this.initialized) return
-    if(this.initlializing) return await new Promise(resolve => this.once('init', resolve))
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (this.initialized) return
+    if (this.initlializing) return await new Promise(resolve => this.once('init', resolve))
     this.initializing = true
     try {
-      if(this.opts.clear) {
+      if (this.opts.clear) {
         await this.fileHandler.truncate(0).catch(console.error)
         throw new Error('Cleared, empty file')
       }
       const lastLine = await this.fileHandler.readLastLine()
-      if(!lastLine || !lastLine.length) {
+      if (!lastLine || !lastLine.length) {
         throw new Error('File does not exists or is a empty file')
       }
-      const offsets = await this.serializer.deserialize(lastLine, {compress: this.opts.compressIndex})
-      if(!Array.isArray(offsets)) {
+      const offsets = await this.serializer.deserialize(lastLine, { compress: this.opts.compressIndex })
+      if (!Array.isArray(offsets)) {
         throw new Error('File to parse offsets, expected an array')
       }
       this.indexOffset = offsets[offsets.length - 2]
@@ -53,14 +55,14 @@ export class Database extends EventEmitter {
       this.offsets = this.offsets.slice(0, -2)
       this.shouldTruncate = true
       let indexLine = await this.fileHandler.readRange(...ptr)
-      const index = await this.serializer.deserialize(indexLine, {compress: this.opts.compressIndex})
+      const index = await this.serializer.deserialize(indexLine, { compress: this.opts.compressIndex })
       index && this.indexManager.load(index)
     } catch (e) {
-      if(Array.isArray(this.offsets)) {
+      if (Array.isArray(this.offsets)) {
         this.offsets = []
       }
       this.indexOffset = 0
-      if(!String(e).includes('empty file')) {
+      if (!String(e).includes('empty file')) {
         console.error('Error loading database:', e)
       }
     } finally {
@@ -71,30 +73,30 @@ export class Database extends EventEmitter {
   }
 
   async save() {
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(!this.initialized) throw new Error('Database not initialized')
-    if(this.saving) return new Promise(resolve => this.once('save', resolve))
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (!this.initialized) throw new Error('Database not initialized')
+    if (this.saving) return new Promise(resolve => this.once('save', resolve))
     this.saving = true
     await this.flush()
     if (!this.shouldSave) return
     this.emit('before-save')
-    const index = Object.assign({data: {}}, this.indexManager.index)
-    for(const field in this.indexManager.index.data) {
-      for(const term in this.indexManager.index.data[field]) {
+    const index = Object.assign({ data: {} }, this.indexManager.index)
+    for (const field in this.indexManager.index.data) {
+      for (const term in this.indexManager.index.data[field]) {
         index.data[field][term] = [...this.indexManager.index.data[field][term]] // set to array 
       }
     }
     const offsets = this.offsets.slice(0)
-    const indexString = await this.serializer.serialize(index, {compress: this.opts.compressIndex, linebreak: true}) // force linebreak here to allow 'init' to read last line as offsets correctly
-    for(const field in this.indexManager.index.data) {
-      for(const term in this.indexManager.index.data[field]) {
+    const indexString = await this.serializer.serialize(index, { compress: this.opts.compressIndex, linebreak: true }) // force linebreak here to allow 'init' to read last line as offsets correctly
+    for (const field in this.indexManager.index.data) {
+      for (const term in this.indexManager.index.data[field]) {
         this.indexManager.index.data[field][term] = new Set(index.data[field][term]) // set back to set because of serialization
       }
     }
     offsets.push(this.indexOffset)
     offsets.push(this.indexOffset + indexString.length)
     // save offsets as JSON always to prevent linebreaks on last line, which breaks 'init()'
-    const offsetsString = await this.serializer.serialize(offsets, {json: true, compress: false, linebreak: false})
+    const offsetsString = await this.serializer.serialize(offsets, { json: true, compress: false, linebreak: false })
     this.writeBuffer.push(indexString)
     this.writeBuffer.push(offsetsString)
     await this.flush() // write the index and offsets
@@ -112,7 +114,7 @@ export class Database extends EventEmitter {
 
   locate(n) {
     if (this.offsets[n] === undefined) {
-      if(this.offsets[n - 1]) {
+      if (this.offsets[n - 1]) {
         return [this.indexOffset, Number.MAX_SAFE_INTEGER]
       }
       return
@@ -120,58 +122,58 @@ export class Database extends EventEmitter {
     let end = (this.offsets[n + 1] || this.indexOffset || Number.MAX_SAFE_INTEGER)
     return [this.offsets[n], end]
   }
-  
+
   getRanges(map) {
     return (map || Array.from(this.offsets.keys())).map(n => {
-        const ret = this.locate(n)
-        if(ret !== undefined) return {start: ret[0], end: ret[1], index: n}
+      const ret = this.locate(n)
+      if (ret !== undefined) return { start: ret[0], end: ret[1], index: n }
     }).filter(n => n !== undefined)
   }
 
   async readLines(map, ranges) {
-    if(!ranges) ranges = this.getRanges(map)
+    if (!ranges) ranges = this.getRanges(map)
     const results = await this.fileHandler.readRanges(ranges, this.serializer.deserialize.bind(this.serializer))
     let i = 0
-    for(const start in results) {
-      if(!results[start] || results[start]._ !== undefined) continue
-      while(this.offsets[i] != start && i < map.length) i++ // weak comparison as 'start' is a string
+    for (const start in results) {
+      if (!results[start] || results[start]._ !== undefined) continue
+      while (this.offsets[i] != start && i < map.length) i++ // weak comparison as 'start' is a string
       results[start]._ = map[i++]
     }
     return Object.values(results).filter(r => r !== undefined)
   }
 
   async insert(data) {
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(!this.initialized) await this.init()
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (!this.initialized) await this.init()
     if (this.shouldTruncate) {
-        this.writeBuffer.push(this.indexOffset)
-        this.shouldTruncate = false
+      this.writeBuffer.push(this.indexOffset)
+      this.shouldTruncate = false
     }
-    const line = await this.serializer.serialize(data, {compress: this.opts.compress}) // using Buffer for offsets accuracy
+    const line = await this.serializer.serialize(data, { compress: this.opts.compress, v8: this.opts.v8 }) // using Buffer for offsets accuracy
     const position = this.offsets.length
     this.offsets.push(this.indexOffset)
     this.indexOffset += line.length
-    this.indexManager.add(data, position)
     this.emit('insert', data, position)
     this.writeBuffer.push(line)
-    if(!this.flushing && this.currentWriteBufferSize() > this.opts.maxMemoryUsage) {
+    if (!this.flushing && this.currentWriteBufferSize() > this.opts.maxMemoryUsage) {
       await this.flush()
     }
+    this.indexManager.add(data, position)
     this.shouldSave = true
   }
 
-  currentWriteBufferSize(){
+  currentWriteBufferSize() {
     const lengths = this.writeBuffer.filter(b => Buffer.isBuffer(b)).map(b => b.length)
     return lengths.reduce((a, b) => a + b, 0)
   }
 
   flush() {
-    if(this.flushing) {
+    if (this.flushing) {
       return this.flushing
     }
     return this.flushing = new Promise((resolve, reject) => {
-      if(this.destroyed) return reject(new Error('Database is destroyed'))
-      if(!this.writeBuffer.length) return resolve()
+      if (this.destroyed) return reject(new Error('Database is destroyed'))
+      if (!this.writeBuffer.length) return resolve()
       let err
       this._flush().catch(e => err = e).finally(() => {
         err ? reject(err) : resolve()
@@ -181,17 +183,18 @@ export class Database extends EventEmitter {
   }
 
   async _flush() {
+    const release = await this.mutex.acquire()
     let fd = await fs.promises.open(this.fileHandler.file, 'a')
     try {
-      while(this.writeBuffer.length) {
+      while (this.writeBuffer.length) {
         let data
         const pos = this.writeBuffer.findIndex(b => typeof b === 'number')
-        if(pos === 0) {
+        if (pos === 0) {
           await fd.close()
           await this.fileHandler.truncate(this.writeBuffer.shift())
           fd = await fs.promises.open(this.fileHandler.file, 'a')
           continue
-        } else if(pos === -1) {
+        } else if (pos === -1) {
           data = Buffer.concat(this.writeBuffer)
           this.writeBuffer.length = 0
         } else {
@@ -201,76 +204,72 @@ export class Database extends EventEmitter {
         await fd.write(data)
       }
       this.shouldSave = true
-    } catch(err) {
+    } catch (err) {
       console.error('Error flushing:', err)
     } finally {
-      await fd.close()
+      let err
+      await fd.close().catch(e => err = e)
+      release()
+      err && console.error('Error closing file:', err)
     }
   }
 
-  async *walk(map, options={}) {
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(!this.initialized) await this.init()
+  async *walk(map, options = {}) {
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (!this.initialized) await this.init()
     this.shouldSave && await this.save().catch(console.error)
-    if(this.indexOffset === 0) return
-    if(!Array.isArray(map)) {
+    if (this.indexOffset === 0) return
+    if (!Array.isArray(map)) {
       if (map instanceof Set) {
         map = [...map]
-      } else if(map && typeof map === 'object') {
+      } else if (map && typeof map === 'object') {
         map = [...this.indexManager.query(map, options)]
       } else {
         map = [...Array(this.offsets.length).keys()]
       }
     }
     const ranges = this.getRanges(map)
-    const partitionedRanges = [], currentPartition = 0
-    for (const line in ranges) {
-      if (partitionedRanges[currentPartition] === undefined) {
-        partitionedRanges[currentPartition] = []
-      }
-      partitionedRanges[currentPartition].push(ranges[line])
-      if (partitionedRanges[currentPartition].length >= this.opts.maxMemoryUsage) {
-        currentPartition++
-      }
-    }
-    let m = 0
-    for (const ranges of partitionedRanges) {
-      const lines = await this.fileHandler.readRanges(ranges)
-      for (const line in lines) {
-        let err
-        const entry = await this.serializer.deserialize(lines[line]).catch(e => console.error(err = e))
-        if (err) continue
-        if (entry._ === undefined) {
-          while(this.offsets[m] != line && m < map.length) m++ // weak comparison as 'start' is a string
-          entry._ = m++
+    const groupedRanges = await this.fileHandler.groupedRanges(ranges)
+    const fd = await fs.promises.open(this.fileHandler.file, 'r')
+    try {
+      for (const groupedRange of groupedRanges) {
+        for await (const row of this.fileHandler.readGroupedRange(groupedRange, fd)) {
+          const entry = await this.serializer.deserialize(row.line, { compress: this.opts.compress, v8: this.opts.v8 })
+          if (entry === null) continue
+          if (options.includeOffsets) {
+            yield { entry, start: row.start }
+          } else {
+            yield entry
+          }
         }
-        yield entry
       }
+    } finally {
+      await fd.close()
     }
   }
 
-  async query(criteria, options={}) {
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(!this.initialized) await this.init()
+  async query(criteria, options = {}) {
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (!this.initialized) await this.init()
     this.shouldSave && await this.save().catch(console.error)
-    if(Array.isArray(criteria)) {
+    if (Array.isArray(criteria)) {
       let results = await this.readLines(criteria)
       if (options.orderBy) {
-          const [field, direction = 'asc'] = options.orderBy.split(' ')
-          results.sort((a, b) => {
-              if (a[field] > b[field]) return direction === 'asc' ? 1 : -1
-              if (a[field] < b[field]) return direction === 'asc' ? -1 : 1
-              return 0;
-          })
+        const [field, direction = 'asc'] = options.orderBy.split(' ')
+        results.sort((a, b) => {
+          if (a[field] > b[field]) return direction === 'asc' ? 1 : -1
+          if (a[field] < b[field]) return direction === 'asc' ? -1 : 1
+          return 0;
+        })
       }
       if (options.limit) {
-          results = results.slice(0, options.limit);
+        results = results.slice(0, options.limit);
       }
       return results
     } else {
       const matchingLines = await this.indexManager.query(criteria, options)
       if (!matchingLines || !matchingLines.size) {
-          return []
+        return []
       }
       return await this.query([...matchingLines], options)
     }
@@ -323,17 +322,17 @@ export class Database extends EventEmitter {
     return entries
   }
 
-  async delete(criteria, options={}) {
+  async delete(criteria, options = {}) {
     if (this.shouldTruncate) {
-        this.writeBuffer.push(this.indexOffset)
-        this.shouldTruncate = false
+      this.writeBuffer.push(this.indexOffset)
+      this.shouldTruncate = false
     }
-    if(this.destroyed) throw new Error('Database is destroyed')
-    if(!this.initialized) await this.init()
+    if (this.destroyed) throw new Error('Database is destroyed')
+    if (!this.initialized) await this.init()
     this.shouldSave && await this.save().catch(console.error)
     const matchingLines = await this.indexManager.query(criteria, options)
     if (!matchingLines || !matchingLines.size) {
-        return 0
+      return 0
     }
     const ranges = this.getRanges([...matchingLines])
     const validMatchingLines = new Set(ranges.map(r => r.index))

package/src/FileHandler.mjs

@@ -24,22 +24,19 @@ export default class FileHandler {
     if(buffer.length > bytesRead) return buffer.subarray(0, bytesRead)
     return buffer
   }
-
+  
   async readRanges(ranges, mapper) {
     const lines = {}, limit = pLimit(4)
     const fd = await fs.promises.open(this.file, 'r')
+    const groupedRanges = await this.groupedRanges(ranges)
     try {
-      const tasks = ranges.map(r => {
-        return async () => {
-          let err
-          const length = r.end - r.start
-          let buffer = Buffer.alloc(length)
-          const { bytesRead } = await fd.read(buffer, 0, length, r.start).catch(e => err = e)
-          if (buffer.length > bytesRead) buffer = buffer.subarray(0, bytesRead)
-          lines[r.start] = mapper ? (await mapper(buffer, r)) : buffer
-        }
-      })
-      await Promise.allSettled(tasks.map(limit))
+      await Promise.allSettled(groupedRanges.map(async (groupedRange) => {
+        await limit(async () => {
+          for await (const row of this.readGroupedRange(groupedRange, fd)) {
+            lines[row.start] = mapper ? (await mapper(row.line, groupedRange)) : row.line
+          }
+        })
+      }))
     } catch (e) {
       console.error('Error reading ranges:', e)
     } finally {
@@ -48,43 +45,114 @@ export default class FileHandler {
     return lines
   }
 
+  async groupedRanges(ranges) { // expects ordered ranges from Database.getRanges()
+    const readSize = 512 * 1024 // 512KB  
+    const groupedRanges = []
+    let currentGroup = []
+    let currentSize = 0
+
+    // each range is a {start: number, end: number} object
+    for(const range of ranges) {
+      const rangeSize = range.end - range.start
+      
+      if(currentGroup.length > 0) {
+        const lastRange = currentGroup[currentGroup.length - 1]
+        if(lastRange.end !== range.start || currentSize + rangeSize > readSize) {
+          groupedRanges.push(currentGroup)
+          currentGroup = []
+          currentSize = 0
+        }
+      }
+    
+      currentGroup.push(range)
+      currentSize += rangeSize
+    }
+
+    if(currentGroup.length > 0) {
+      groupedRanges.push(currentGroup)
+    }
+
+    return groupedRanges
+  }
+
+  async *readGroupedRange(groupedRange, fd) {
+    const options = {start: groupedRange[0].start, end: groupedRange[groupedRange.length - 1].end}
+    
+    let i = 0, buffer = Buffer.alloc(options.end - options.start)
+    const results = {}, { bytesRead } = await fd.read(buffer, 0, options.end - options.start, options.start)
+    if(buffer.length > bytesRead) buffer = buffer.subarray(0, bytesRead)
+
+    for (const range of groupedRange) {
+      const startOffset = range.start - options.start;
+      let endOffset = range.end - options.start;
+      if (endOffset > buffer.length) {
+        endOffset = buffer.length;
+      }
+      if (startOffset >= buffer.length) {
+        continue;
+      }
+      const line = buffer.subarray(startOffset, endOffset);
+      if (line.length === 0) continue;
+      yield { line, start: range.start };
+    }
+
+
+    return results
+  }
+
+  async *walk(ranges) {
+    const fd = await fs.promises.open(this.file, 'r')
+    try {
+      const groupedRanges = await this.groupedRanges(ranges)
+      for(const groupedRange of groupedRanges) {
+        for await (const row of this.readGroupedRange(groupedRange, fd)) {
+          yield row
+        }
+      }
+    } finally {
+      await fd.close()
+    }
+  }
+
   async replaceLines(ranges, lines) {
-    let closed
-    const tmpFile = this.file + '.tmp'
-    const writer = await fs.promises.open(tmpFile, 'w+')
-    const reader = await fs.promises.open(this.file, 'r')
+    const tmpFile = this.file + '.tmp';
+    const writer = await fs.promises.open(tmpFile, 'w+');
+    const reader = await fs.promises.open(this.file, 'r');
     try {
-      let i = 0, start = 0
-      for (const r of ranges) {
-        const length = r.start - start
-        const buffer = Buffer.alloc(length)
-        await reader.read(buffer, 0, length, start)
-        start = r.end
-        buffer.length && await writer.write(buffer)
-        if (lines[i]) {
-          await writer.write(lines[i])
+      let position = 0;
+      let lineIndex = 0;
+
+      for (const range of ranges) {
+        if (position < range.start) {
+          const buffer = await this.readRange(position, range.start);
+          await writer.write(buffer);
         }
-        i++
+        if (lineIndex < lines.length && lines[lineIndex]) {
+          await writer.write(lines[lineIndex]);
+        }
+        position = range.end;
+        lineIndex++;
       }
-      const size = (await reader.stat()).size
-      const length = size - start
-      const buffer = Buffer.alloc(length)
-      await reader.read(buffer, 0, length, start)
-      await writer.write(buffer)
-      await reader.close()
-      await writer.close()
-      closed = true
-      await fs.promises.copyFile(tmpFile, this.file)
+
+      const { size } = await reader.stat();
+      if (position < size) {
+        const buffer = await this.readRange(position, size);
+        await writer.write(buffer);
+      }
+
+      await reader.close();
+      await writer.close();
+      await fs.promises.rename(tmpFile, this.file);
     } catch (e) {
-      console.error('Error replacing lines:', e)
+      console.error('Erro ao substituir linhas:', e);
+      throw e;
     } finally {
-      if(!closed) {
-        await reader.close()
-        await writer.close()
-      }
-      await fs.promises.unlink(tmpFile).catch(() => {})
+      await reader.close().catch(() => { });
+      await writer.close().catch(() => { });
+      await fs.promises.unlink(tmpFile).catch(() => { });
     }
   }
+
   async writeData(data, immediate, fd) {
     await fd.write(data)
   }
@@ -100,18 +168,19 @@ export default class FileHandler {
       if (size < 1) throw 'empty file'
       this.size = size
       const bufferSize = 16384
-      let buffer, lastReadSize, readPosition = Math.max(size - bufferSize, 0)
+      let buffer, isFirstRead = true, lastReadSize, readPosition = Math.max(size - bufferSize, 0)
       while (readPosition >= 0) {
         const readSize = Math.min(bufferSize, size - readPosition)
         if (readSize !== lastReadSize) {
           lastReadSize = readSize
           buffer = Buffer.alloc(readSize)
         }
-        const { bytesRead } = await reader.read(buffer, 0, readSize, readPosition)
+        const { bytesRead } = await reader.read(buffer, 0, isFirstRead ? (readSize - 1) : readSize, readPosition)
+        if (isFirstRead) isFirstRead = false
         if (bytesRead === 0) break
-        const newlineIndex = buffer.lastIndexOf(10, size - 4) // 0x0A is the ASCII code for '\n'
+        const newlineIndex = buffer.lastIndexOf(10)
+        const start = readPosition + newlineIndex + 1
         if (newlineIndex !== -1) {
-          const start = readPosition + newlineIndex + 1
           const lastLine = Buffer.alloc(size - start)
           await reader.read(lastLine, 0, size - start, start)
           if (!lastLine || !lastLine.length) {

package/src/Serializer.mjs

@@ -23,7 +23,7 @@ export default class Serializer extends EventEmitter {
       header |= 0x02 // set V8
       line = v8.serialize(data)
     } else {
-      let json = JSON.stringify(data)
+      const json = JSON.stringify(data)
       line = Buffer.from(json, 'utf-8')
     }
     if (compress) {
@@ -46,6 +46,9 @@ export default class Serializer extends EventEmitter {
   }  
 
   async deserialize(data) {
+    if(data.length === 0) {
+      return null
+    }
     let line, isCompressed, isV8
     const header = data.readUInt8(0)
     const valid = header === 0x00 || header === 0x01 || header === 0x02 || header === 0x03
@@ -58,7 +61,7 @@ export default class Serializer extends EventEmitter {
       try {
         return JSON.parse(data.toString('utf-8').trim())
       } catch (e) {
-        throw new Error('Failed to deserialize legacy JSON data')
+        throw new Error('Failed to deserialize JSON data')
       }
     }
     if (isCompressed) {