jexidb 2.0.3 → 2.1.1

package/src/OperationQueue.mjs

@@ -0,0 +1,279 @@
+/**
+ * OperationQueue - Queue system for database operations
+ * Resolves race conditions between concurrent operations
+ */
+
+export class OperationQueue {
+  constructor(debugMode = false) {
+    this.queue = []
+    this.processing = false
+    this.operationId = 0
+    this.debugMode = debugMode
+    this.stats = {
+      totalOperations: 0,
+      completedOperations: 0,
+      failedOperations: 0,
+      averageProcessingTime: 0,
+      maxProcessingTime: 0,
+      totalProcessingTime: 0
+    }
+  }
+  
+  /**
+   * Adds an operation to the queue
+   * @param {Function} operation - Asynchronous function to be executed
+   * @returns {Promise} - Promise that resolves when the operation is completed
+   */
+  async enqueue(operation) {
+    const id = ++this.operationId
+    const startTime = Date.now()
+    
+    if (this.debugMode) {
+      console.log(`🔄 Queue: Enqueuing operation ${id}, queue length: ${this.queue.length}`)
+    }
+    
+    this.stats.totalOperations++
+    
+    return new Promise((resolve, reject) => {
+      // Capture stack trace for debugging stuck operations
+      const stackTrace = new Error().stack
+      
+      this.queue.push({ 
+        id, 
+        operation, 
+        resolve, 
+        reject,
+        timestamp: startTime,
+        stackTrace: stackTrace,
+        startTime: Date.now()
+      })
+      
+      // Process immediately if not already processing
+      this.process().catch(reject)
+    })
+  }
+  
+  /**
+   * Processes all operations in the queue sequentially
+   */
+  async process() {
+    if (this.processing || this.queue.length === 0) {
+      return
+    }
+    
+    this.processing = true
+    
+    if (this.debugMode) {
+      console.log(`🔄 Queue: Starting to process ${this.queue.length} operations`)
+    }
+    
+    try {
+      while (this.queue.length > 0) {
+        const { id, operation, resolve, reject, timestamp } = this.queue.shift()
+        
+        if (this.debugMode) {
+          console.log(`🔄 Queue: Processing operation ${id}`)
+        }
+        
+        try {
+          const result = await operation()
+          const processingTime = Date.now() - timestamp
+          
+          // Atualizar estatísticas
+          this.stats.completedOperations++
+          this.stats.totalProcessingTime += processingTime
+          this.stats.averageProcessingTime = this.stats.totalProcessingTime / this.stats.completedOperations
+          this.stats.maxProcessingTime = Math.max(this.stats.maxProcessingTime, processingTime)
+          
+          resolve(result)
+          
+          if (this.debugMode) {
+            console.log(`✅ Queue: Operation ${id} completed in ${processingTime}ms`)
+          }
+        } catch (error) {
+          const processingTime = Date.now() - timestamp
+          
+          // Atualizar estatísticas
+          this.stats.failedOperations++
+          this.stats.totalProcessingTime += processingTime
+          this.stats.averageProcessingTime = this.stats.totalProcessingTime / (this.stats.completedOperations + this.stats.failedOperations)
+          this.stats.maxProcessingTime = Math.max(this.stats.maxProcessingTime, processingTime)
+          
+          reject(error)
+          
+          if (this.debugMode) {
+            console.error(`❌ Queue: Operation ${id} failed in ${processingTime}ms:`, error)
+          }
+        }
+      }
+    } finally {
+      this.processing = false
+      
+      if (this.debugMode) {
+        console.log(`🔄 Queue: Finished processing, remaining: ${this.queue.length}`)
+      }
+    }
+  }
+  
+  /**
+   * Waits for all pending operations to be processed
+   * @param {number|null} maxWaitTime - Maximum wait time in ms (null = wait indefinitely)
+   * @returns {Promise<boolean>} - true if all operations were processed, false if a timeout occurred
+   */
+  async waitForCompletion(maxWaitTime = 5000) {
+    const startTime = Date.now()
+    
+    // CRITICAL FIX: Support infinite wait when maxWaitTime is null
+    const hasTimeout = maxWaitTime !== null && maxWaitTime !== undefined
+    
+    while (this.queue.length > 0) {
+      // Check timeout only if we have one
+      if (hasTimeout && (Date.now() - startTime) >= maxWaitTime) {
+        break
+      }
+      
+      await new Promise(resolve => setTimeout(resolve, 1))
+    }
+    
+    const completed = this.queue.length === 0
+    if (!completed && hasTimeout) {
+      // CRITICAL: Don't leave operations hanging - fail fast with detailed error
+      const pendingOperations = this.queue.map(op => ({
+        id: op.id,
+        stackTrace: op.stackTrace,
+        startTime: op.startTime,
+        waitTime: Date.now() - op.startTime
+      }))
+      
+      // Clear the queue to prevent memory leaks
+      this.queue = []
+      
+      const error = new Error(`OperationQueue: Operations timed out after ${maxWaitTime}ms. ${pendingOperations.length} operations were stuck and have been cleared.`)
+      error.pendingOperations = pendingOperations
+      error.queueStats = this.getStats()
+      
+      if (this.debugMode) {
+        console.error(`❌ Queue: Operations timed out, clearing ${pendingOperations.length} stuck operations:`)
+        pendingOperations.forEach(op => {
+          console.error(`  - Operation ${op.id} (waiting ${op.waitTime}ms):`)
+          console.error(`    Stack: ${op.stackTrace}`)
+        })
+      }
+      
+      throw error
+    }
+    
+    return completed
+  }
+  
+  /**
+   * Returns the current queue length
+   */
+  getQueueLength() {
+    return this.queue.length
+  }
+  
+  /**
+   * Checks whether operations are currently being processed
+   */
+  isProcessing() {
+    return this.processing
+  }
+  
+  /**
+   * Returns queue statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      queueLength: this.queue.length,
+      isProcessing: this.processing,
+      successRate: this.stats.totalOperations > 0 ? 
+        (this.stats.completedOperations / this.stats.totalOperations) * 100 : 0
+    }
+  }
+  
+  /**
+   * Clears the queue (for emergency situations)
+   */
+  clear() {
+    const clearedCount = this.queue.length
+    this.queue = []
+    
+    if (this.debugMode) {
+      console.log(`🧹 Queue: Cleared ${clearedCount} pending operations`)
+    }
+    
+    return clearedCount
+  }
+
+  /**
+   * Detects stuck operations and returns detailed information
+   * @param {number} stuckThreshold - Time in ms to consider an operation stuck
+   * @returns {Array} - List of stuck operations with stack traces
+   */
+  detectStuckOperations(stuckThreshold = 10000) {
+    const now = Date.now()
+    const stuckOperations = this.queue.filter(op => (now - op.startTime) > stuckThreshold)
+    
+    return stuckOperations.map(op => ({
+      id: op.id,
+      waitTime: now - op.startTime,
+      stackTrace: op.stackTrace,
+      timestamp: op.timestamp
+    }))
+  }
+
+  /**
+   * Force-cleans stuck operations (last resort)
+   * @param {number} stuckThreshold - Time in ms to consider an operation stuck
+   * @returns {number} - Number of operations removed
+   */
+  forceCleanupStuckOperations(stuckThreshold = 10000) {
+    const stuckOps = this.detectStuckOperations(stuckThreshold)
+    
+    if (stuckOps.length > 0) {
+      // Reject all stuck operations
+      stuckOps.forEach(stuckOp => {
+        const opIndex = this.queue.findIndex(op => op.id === stuckOp.id)
+        if (opIndex !== -1) {
+          const op = this.queue[opIndex]
+          op.reject(new Error(`Operation ${op.id} was stuck for ${stuckOp.waitTime}ms and has been force-cleaned. Stack: ${stuckOp.stackTrace}`))
+          this.queue.splice(opIndex, 1)
+        }
+      })
+      
+      if (this.debugMode) {
+        console.error(`🧹 Queue: Force-cleaned ${stuckOps.length} stuck operations`)
+        stuckOps.forEach(op => {
+          console.error(`  - Operation ${op.id} (stuck for ${op.waitTime}ms)`)
+        })
+      }
+    }
+    
+    return stuckOps.length
+  }
+  
+  /**
+   * Checks whether the queue is empty
+   */
+  isEmpty() {
+    return this.queue.length === 0
+  }
+  
+  /**
+   * Returns information about the next operation in the queue
+   */
+  peekNext() {
+    if (this.queue.length === 0) {
+      return null
+    }
+    
+    const next = this.queue[0]
+    return {
+      id: next.id,
+      timestamp: next.timestamp,
+      waitTime: Date.now() - next.timestamp
+    }
+  }
+}

package/src/SchemaManager.mjs

@@ -0,0 +1,268 @@
+/**
+ * SchemaManager - Manages field schemas for optimized array-based serialization
+ * This replaces the need for repeating field names in JSON objects
+ */
+export default class SchemaManager {
+  constructor(opts = {}) {
+    this.opts = Object.assign({
+      enableArraySerialization: true,
+      strictSchema: true,
+      debugMode: false
+    }, opts)
+    
+    // Schema definition: array of field names in order
+    this.schema = []
+    this.fieldToIndex = new Map() // field name -> index
+    this.indexToField = new Map() // index -> field name
+    this.schemaVersion = 1
+    this.isInitialized = false
+  }
+
+  /**
+   * Initialize schema from options or auto-detect from data
+   */
+  initializeSchema(schemaOrData, autoDetect = false) {
+    if (this.isInitialized && this.opts.strictSchema) {
+      if (this.opts.debugMode) {
+        console.log('SchemaManager: Schema already initialized, skipping')
+      }
+      return
+    }
+
+    if (Array.isArray(schemaOrData)) {
+      // Explicit schema provided
+      this.setSchema(schemaOrData)
+    } else if (autoDetect && typeof schemaOrData === 'object') {
+      // Auto-detect schema from data
+      this.autoDetectSchema(schemaOrData)
+    } else if (schemaOrData && typeof schemaOrData === 'object') {
+      // Initialize from database options
+      this.initializeFromOptions(schemaOrData)
+    }
+
+    this.isInitialized = true
+    if (this.opts.debugMode) {
+      console.log('SchemaManager: Schema initialized:', this.schema)
+    }
+  }
+
+  /**
+   * Set explicit schema
+   */
+  setSchema(fieldNames) {
+    this.schema = [...fieldNames] // Create copy
+    this.fieldToIndex.clear()
+    this.indexToField.clear()
+    
+    this.schema.forEach((field, index) => {
+      this.fieldToIndex.set(field, index)
+      this.indexToField.set(index, field)
+    })
+
+    if (this.opts.debugMode) {
+      console.log('SchemaManager: Schema set:', this.schema)
+    }
+  }
+
+  /**
+   * Auto-detect schema from sample data
+   */
+  autoDetectSchema(sampleData) {
+    if (Array.isArray(sampleData)) {
+      // Use first record as template
+      if (sampleData.length > 0) {
+        this.autoDetectSchema(sampleData[0])
+      }
+      return
+    }
+
+    if (typeof sampleData === 'object' && sampleData !== null) {
+      const fields = Object.keys(sampleData).sort() // Sort for consistency
+      
+      // CRITICAL FIX: Always include 'id' field in schema for proper array format
+      if (!fields.includes('id')) {
+        fields.push('id')
+      }
+      
+      this.setSchema(fields)
+    }
+  }
+
+  /**
+   * Initialize schema from database options
+   */
+  initializeFromOptions(opts) {
+    if (opts.schema && Array.isArray(opts.schema)) {
+      this.setSchema(opts.schema)
+    }
+    // CRITICAL FIX: Don't auto-initialize schema from indexes
+    // This was causing data loss because only indexed fields were preserved
+    // Let schema be auto-detected from actual data instead
+  }
+
+  /**
+   * Add new field to schema (for schema evolution)
+   */
+  addField(fieldName) {
+    if (this.fieldToIndex.has(fieldName)) {
+      return this.fieldToIndex.get(fieldName)
+    }
+
+    const newIndex = this.schema.length
+    this.schema.push(fieldName)
+    this.fieldToIndex.set(fieldName, newIndex)
+    this.indexToField.set(newIndex, fieldName)
+
+    if (this.opts.debugMode) {
+      console.log('SchemaManager: Added field:', fieldName, 'at index:', newIndex)
+    }
+
+    return newIndex
+  }
+
+  /**
+   * Convert object to array using schema with strict field enforcement
+   */
+  objectToArray(obj) {
+    if (!this.isInitialized || !this.opts.enableArraySerialization) {
+      return obj // Fallback to object format
+    }
+
+    if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+      return obj // Don't convert non-objects or arrays
+    }
+
+    const result = new Array(this.schema.length)
+    
+    // Fill array with values in schema order
+    // Missing fields become undefined, extra fields are ignored
+    for (let i = 0; i < this.schema.length; i++) {
+      const fieldName = this.schema[i]
+      result[i] = obj[fieldName] !== undefined ? obj[fieldName] : undefined
+    }
+
+    return result
+  }
+
+  /**
+   * Convert array back to object using schema
+   */
+  arrayToObject(arr) {
+    if (!this.isInitialized || !this.opts.enableArraySerialization) {
+      return arr // Fallback to array format
+    }
+
+    if (!Array.isArray(arr)) {
+      return arr // Don't convert non-arrays
+    }
+
+    const obj = {}
+    
+    // Map array values to object properties
+    // Only include fields that are in the schema
+    for (let i = 0; i < Math.min(arr.length, this.schema.length); i++) {
+      const fieldName = this.schema[i]
+      // Only include non-undefined values to avoid cluttering the object
+      if (arr[i] !== undefined) {
+        obj[fieldName] = arr[i]
+      }
+    }
+
+    return obj
+  }
+
+  /**
+   * Get field index by name
+   */
+  getFieldIndex(fieldName) {
+    return this.fieldToIndex.get(fieldName)
+  }
+
+  /**
+   * Get field name by index
+   */
+  getFieldName(index) {
+    return this.indexToField.get(index)
+  }
+
+  /**
+   * Check if field exists in schema
+   */
+  hasField(fieldName) {
+    return this.fieldToIndex.has(fieldName)
+  }
+
+  /**
+   * Get schema as array of field names
+   */
+  getSchema() {
+    return [...this.schema] // Return copy
+  }
+
+  /**
+   * Get schema size
+   */
+  getSchemaSize() {
+    return this.schema.length
+  }
+
+  /**
+   * Validate that object conforms to schema
+   */
+  validateObject(obj) {
+    if (!this.isInitialized || !this.opts.strictSchema) {
+      return true
+    }
+
+    if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+      return false
+    }
+
+    // Check if object has all required fields
+    for (const field of this.schema) {
+      if (!(field in obj)) {
+        if (this.opts.debugMode) {
+          console.warn('SchemaManager: Missing required field:', field)
+        }
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Get schema metadata for serialization
+   */
+  getSchemaMetadata() {
+    return {
+      version: this.schemaVersion,
+      fields: [...this.schema],
+      fieldCount: this.schema.length,
+      isInitialized: this.isInitialized
+    }
+  }
+
+  /**
+   * Reset schema
+   */
+  reset() {
+    this.schema = []
+    this.fieldToIndex.clear()
+    this.indexToField.clear()
+    this.isInitialized = false
+    this.schemaVersion++
+  }
+
+  /**
+   * Get performance statistics
+   */
+  getStats() {
+    return {
+      schemaSize: this.schema.length,
+      isInitialized: this.isInitialized,
+      version: this.schemaVersion,
+      enableArraySerialization: this.opts.enableArraySerialization
+    }
+  }
+}