jexidb 2.1.0 → 2.1.2

package/src/FileHandler.mjs

@@ -6,10 +6,12 @@ import pLimit from 'p-limit'
 export default class FileHandler {
   constructor(file, fileMutex = null, opts = {}) {
     this.file = file
-    this.indexFile = file.replace(/\.jdb$/, '.idx.jdb')
+    this.indexFile = file ? file.replace(/\.jdb$/, '.idx.jdb') : null
     this.fileMutex = fileMutex
     this.opts = opts
     this.maxBufferSize = opts.maxBufferSize || 4 * 1024 * 1024 // 4MB default
+    // Global I/O limiter to prevent file descriptor exhaustion in concurrent operations
+    this.readLimiter = pLimit(opts.maxConcurrentReads || 4)
   }
 
   async truncate(offset) {
@@ -174,7 +176,7 @@ export default class FileHandler {
   }
   
   async readRanges(ranges, mapper) {
-    const lines = {}, limit = pLimit(4)
+    const lines = {}
     
     // Check if file exists before trying to read it
     if (!await this.exists()) {
@@ -185,7 +187,7 @@ export default class FileHandler {
     const groupedRanges = await this.groupedRanges(ranges)
     try {
       await Promise.allSettled(groupedRanges.map(async (groupedRange) => {
-        await limit(async () => {
+        await this.readLimiter(async () => {
           for await (const row of this.readGroupedRange(groupedRange, fd)) {
             lines[row.start] = mapper ? (await mapper(row.line, { start: row.start, end: row.start + row.line.length })) : row.line
           }
@@ -255,6 +257,10 @@ export default class FileHandler {
         lineString = actualBuffer.toString('utf8', { replacement: '?' })
       }
       
+      // CRITICAL FIX: Remove trailing newlines and whitespace for single range too
+      // Optimized: Use trimEnd() which efficiently removes all trailing whitespace (faster than manual checks)
+      lineString = lineString.trimEnd()
+      
       yield { 
         line: lineString, 
         start: range.start,
@@ -301,14 +307,42 @@ export default class FileHandler {
         }
       }
     } else {
-      // Original logic for non-adjacent ranges
+      // CRITICAL FIX: For non-adjacent ranges, use the range.end directly
+      // because range.end already excludes the newline (calculated as offsets[n+1] - 1)
+      // We just need to find the line start (beginning of the line in the buffer)
       for (let i = 0; i < groupedRange.length; i++) {
         const range = groupedRange[i]
         const relativeStart = range.start - firstRange.start
         const relativeEnd = range.end - firstRange.start
         
-        // Extract the specific range content
-        const rangeContent = content.substring(relativeStart, relativeEnd)
+        // OPTIMIZATION 2: Find line start only if necessary
+        // Check if we're already at a line boundary to avoid unnecessary backwards search
+        let lineStart = relativeStart
+        if (relativeStart > 0 && content[relativeStart - 1] !== '\n') {
+          // Only search backwards if we're not already at a line boundary
+          while (lineStart > 0 && content[lineStart - 1] !== '\n') {
+            lineStart--
+          }
+        }
+        
+        // OPTIMIZATION 3: Use slice() instead of substring() for better performance
+        // CRITICAL FIX: range.end = offsets[n+1] - 1 points to the newline character
+        // slice(start, end) includes characters from start to end-1 (end is exclusive)
+        // So if relativeEnd points to the newline, slice will include it
+        let rangeContent = content.slice(lineStart, relativeEnd)
+        
+        // OPTIMIZATION 4: Direct character check instead of regex/trimEnd
+        // Remove trailing newlines and whitespace efficiently
+        // trimEnd() is actually optimized in V8, but we can check if there's anything to trim first
+        const len = rangeContent.length
+        if (len > 0) {
+          // Quick check: if last char is not whitespace, skip trimEnd
+          const lastChar = rangeContent[len - 1]
+          if (lastChar === '\n' || lastChar === '\r' || lastChar === ' ' || lastChar === '\t') {
+            // Only call trimEnd if we detected trailing whitespace
+            rangeContent = rangeContent.trimEnd()
+          }
+        }
         
         if (rangeContent.length === 0) continue
         
@@ -543,45 +577,48 @@ export default class FileHandler {
   }
 
   async readLastLine() {
-    // Check if file exists before trying to read it
-    if (!await this.exists()) {
-      return null // Return null if file doesn't exist
-    }
-    
-    const reader = await fs.promises.open(this.file, 'r')
-    try {
-      const { size } = await reader.stat()
-      if (size < 1) throw 'empty file'
-      this.size = size
-      const bufferSize = 16384
-      let buffer, isFirstRead = true, lastReadSize, readPosition = Math.max(size - bufferSize, 0)
-      while (readPosition >= 0) {
-        const readSize = Math.min(bufferSize, size - readPosition)
-        if (readSize !== lastReadSize) {
-          lastReadSize = readSize
-          buffer = Buffer.alloc(readSize)
-        }
-        const { bytesRead } = await reader.read(buffer, 0, isFirstRead ? (readSize - 1) : readSize, readPosition)
-        if (isFirstRead) isFirstRead = false
-        if (bytesRead === 0) break
-        const newlineIndex = buffer.lastIndexOf(10)
-        const start = readPosition + newlineIndex + 1
-        if (newlineIndex !== -1) {
-          const lastLine = Buffer.alloc(size - start)
-          await reader.read(lastLine, 0, size - start, start)
-          if (!lastLine || !lastLine.length) {
-            throw 'no metadata or empty file'
+    // Use global read limiter to prevent file descriptor exhaustion
+    return this.readLimiter(async () => {
+      // Check if file exists before trying to read it
+      if (!await this.exists()) {
+        return null // Return null if file doesn't exist
+      }
+      
+      const reader = await fs.promises.open(this.file, 'r')
+      try {
+        const { size } = await reader.stat()
+        if (size < 1) throw 'empty file'
+        this.size = size
+        const bufferSize = 16384
+        let buffer, isFirstRead = true, lastReadSize, readPosition = Math.max(size - bufferSize, 0)
+        while (readPosition >= 0) {
+          const readSize = Math.min(bufferSize, size - readPosition)
+          if (readSize !== lastReadSize) {
+            lastReadSize = readSize
+            buffer = Buffer.alloc(readSize)
+          }
+          const { bytesRead } = await reader.read(buffer, 0, isFirstRead ? (readSize - 1) : readSize, readPosition)
+          if (isFirstRead) isFirstRead = false
+          if (bytesRead === 0) break
+          const newlineIndex = buffer.lastIndexOf(10)
+          const start = readPosition + newlineIndex + 1
+          if (newlineIndex !== -1) {
+            const lastLine = Buffer.alloc(size - start)
+            await reader.read(lastLine, 0, size - start, start)
+            if (!lastLine || !lastLine.length) {
+              throw 'no metadata or empty file'
+            }
+            return lastLine
+          } else {
+            readPosition -= bufferSize
           }
-          return lastLine
-        } else {
-          readPosition -= bufferSize
         }
+      } catch (e) {
+        String(e).includes('empty file') || console.error('Error reading last line:', e)
+      } finally {
+        reader.close()
       }
-    } catch (e) {
-      String(e).includes('empty file') || console.error('Error reading last line:', e)
-    } finally {
-      reader.close()
-    }
+    })
   }
 
   /**
@@ -597,10 +634,12 @@ export default class FileHandler {
       return this.fileMutex.runExclusive(async () => {
         // Add a small delay to ensure any pending operations complete
         await new Promise(resolve => setTimeout(resolve, 5));
-        return this._readWithStreamingInternal(criteria, options, matchesCriteria, serializer);
+        // Use global read limiter to prevent file descriptor exhaustion
+        return this.readLimiter(() => this._readWithStreamingInternal(criteria, options, matchesCriteria, serializer));
       });
     } else {
-      return this._readWithStreamingInternal(criteria, options, matchesCriteria, serializer);
+      // Use global read limiter to prevent file descriptor exhaustion
+      return this.readLimiter(() => this._readWithStreamingInternal(criteria, options, matchesCriteria, serializer));
     }
   }
 

package/src/OperationQueue.mjs

@@ -1,6 +1,6 @@
 /**
- * OperationQueue - Sistema de fila para operações do banco de dados
- * Resolve race conditions entre operações concorrentes
+ * OperationQueue - Queue system for database operations
+ * Resolves race conditions between concurrent operations
  */
 
 export class OperationQueue {
@@ -20,9 +20,9 @@ export class OperationQueue {
   }
   
   /**
-   * Adiciona uma operação à fila
-   * @param {Function} operation - Função assíncrona a ser executada
-   * @returns {Promise} - Promise que resolve quando a operação é concluída
+   * Adds an operation to the queue
+   * @param {Function} operation - Asynchronous function to be executed
+   * @returns {Promise} - Promise that resolves when the operation is completed
    */
   async enqueue(operation) {
     const id = ++this.operationId
@@ -48,13 +48,13 @@ export class OperationQueue {
         startTime: Date.now()
       })
       
-      // Processar imediatamente se não estiver processando
+      // Process immediately if not already processing
       this.process().catch(reject)
     })
   }
   
   /**
-   * Processa todas as operações na fila sequencialmente
+   * Processes all operations in the queue sequentially
    */
   async process() {
     if (this.processing || this.queue.length === 0) {
@@ -116,9 +116,9 @@ export class OperationQueue {
   }
   
   /**
-   * Aguarda todas as operações pendentes serem processadas
-   * @param {number|null} maxWaitTime - Tempo máximo de espera em ms (null = wait indefinitely)
-   * @returns {Promise<boolean>} - true se todas foram processadas, false se timeout
+   * Waits for all pending operations to be processed
+   * @param {number|null} maxWaitTime - Maximum wait time in ms (null = wait indefinitely)
+   * @returns {Promise<boolean>} - true if all operations were processed, false if a timeout occurred
    */
   async waitForCompletion(maxWaitTime = 5000) {
     const startTime = Date.now()
@@ -167,21 +167,21 @@ export class OperationQueue {
   }
   
   /**
-   * Retorna o tamanho atual da fila
+   * Returns the current queue length
    */
   getQueueLength() {
     return this.queue.length
   }
   
   /**
-   * Verifica se está processando operações
+   * Checks whether operations are currently being processed
    */
   isProcessing() {
     return this.processing
   }
   
   /**
-   * Retorna estatísticas da fila
+   * Returns queue statistics
    */
   getStats() {
     return {
@@ -194,7 +194,7 @@ export class OperationQueue {
   }
   
   /**
-   * Limpa a fila (para casos de emergência)
+   * Clears the queue (for emergency situations)
    */
   clear() {
     const clearedCount = this.queue.length
@@ -208,9 +208,9 @@ export class OperationQueue {
   }
 
   /**
-   * Detecta operações travadas e retorna informações detalhadas
-   * @param {number} stuckThreshold - Tempo em ms para considerar uma operação travada
-   * @returns {Array} - Lista de operações travadas com stack traces
+   * Detects stuck operations and returns detailed information
+   * @param {number} stuckThreshold - Time in ms to consider an operation stuck
+   * @returns {Array} - List of stuck operations with stack traces
    */
   detectStuckOperations(stuckThreshold = 10000) {
     const now = Date.now()
@@ -225,15 +225,15 @@ export class OperationQueue {
   }
 
   /**
-   * Força a limpeza de operações travadas (último recurso)
-   * @param {number} stuckThreshold - Tempo em ms para considerar uma operação travada
-   * @returns {number} - Número de operações removidas
+   * Force-cleans stuck operations (last resort)
+   * @param {number} stuckThreshold - Time in ms to consider an operation stuck
+   * @returns {number} - Number of operations removed
    */
   forceCleanupStuckOperations(stuckThreshold = 10000) {
     const stuckOps = this.detectStuckOperations(stuckThreshold)
     
     if (stuckOps.length > 0) {
-      // Rejeitar todas as operações travadas
+      // Reject all stuck operations
       stuckOps.forEach(stuckOp => {
         const opIndex = this.queue.findIndex(op => op.id === stuckOp.id)
         if (opIndex !== -1) {
@@ -255,14 +255,14 @@ export class OperationQueue {
   }
   
   /**
-   * Verifica se a fila está vazia
+   * Checks whether the queue is empty
    */
   isEmpty() {
     return this.queue.length === 0
   }
   
   /**
-   * Retorna informações sobre a próxima operação na fila
+   * Returns information about the next operation in the queue
    */
   peekNext() {
     if (this.queue.length === 0) {