jexidb 2.0.3 → 2.1.0

package/src/managers/StatisticsManager.mjs

@@ -0,0 +1,262 @@
+/**
+ * StatisticsManager - Handles all statistics and metrics collection
+ * 
+ * Responsibilities:
+ * - getJournalStats()
+ * - Performance metrics
+ * - Usage statistics
+ */
+
+export class StatisticsManager {
+  constructor(database) {
+    this.database = database
+    this.opts = database.opts
+    this.usageStats = database.usageStats || {
+      totalQueries: 0,
+      streamingQueries: 0,
+      indexedQueries: 0,
+      streamingAverageTime: 0,
+      indexedAverageTime: 0
+    }
+    this.performanceMetrics = {
+      startTime: Date.now(),
+      lastResetTime: Date.now(),
+      totalOperations: 0,
+      totalErrors: 0,
+      averageOperationTime: 0,
+      peakMemoryUsage: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    }
+  }
+
+  /**
+   * Get journal statistics
+   * @returns {Object} - Journal statistics
+   */
+  getJournalStats() {
+    return {
+      enabled: false,
+      message: 'Journal mode has been removed'
+    }
+  }
+
+  /**
+   * Get performance metrics
+   * @returns {Object} - Performance metrics
+   */
+  getPerformanceMetrics() {
+    const now = Date.now()
+    const uptime = now - this.performanceMetrics.startTime
+    
+    return {
+      uptime: uptime,
+      totalOperations: this.performanceMetrics.totalOperations,
+      totalErrors: this.performanceMetrics.totalErrors,
+      averageOperationTime: this.performanceMetrics.averageOperationTime,
+      operationsPerSecond: this.performanceMetrics.totalOperations / (uptime / 1000),
+      errorRate: this.performanceMetrics.totalErrors / Math.max(1, this.performanceMetrics.totalOperations),
+      peakMemoryUsage: this.performanceMetrics.peakMemoryUsage,
+      cacheHitRate: this.performanceMetrics.cacheHits / Math.max(1, this.performanceMetrics.cacheHits + this.performanceMetrics.cacheMisses),
+      lastResetTime: this.performanceMetrics.lastResetTime
+    }
+  }
+
+  /**
+   * Get usage statistics
+   * @returns {Object} - Usage statistics
+   */
+  getUsageStats() {
+    return {
+      totalQueries: this.usageStats.totalQueries,
+      streamingQueries: this.usageStats.streamingQueries,
+      indexedQueries: this.usageStats.indexedQueries,
+      streamingAverageTime: this.usageStats.streamingAverageTime,
+      indexedAverageTime: this.usageStats.indexedAverageTime,
+      queryDistribution: {
+        streaming: this.usageStats.streamingQueries / Math.max(1, this.usageStats.totalQueries),
+        indexed: this.usageStats.indexedQueries / Math.max(1, this.usageStats.totalQueries)
+      }
+    }
+  }
+
+  /**
+   * Get database statistics
+   * @returns {Object} - Database statistics
+   */
+  getDatabaseStats() {
+    return {
+      totalRecords: this.database.offsets?.length || 0,
+      indexOffset: this.database.indexOffset || 0,
+      writeBufferSize: this.database.writeBuffer?.length || 0,
+      indexedFields: Object.keys(this.database.indexManager?.index?.data || {}),
+      totalIndexedFields: Object.keys(this.database.indexManager?.index?.data || {}).length,
+      isInitialized: this.database.initialized || false,
+      isDestroyed: this.database.destroyed || false
+    }
+  }
+
+  /**
+   * Get comprehensive statistics
+   * @returns {Object} - All statistics combined
+   */
+  getComprehensiveStats() {
+    return {
+      database: this.getDatabaseStats(),
+      performance: this.getPerformanceMetrics(),
+      usage: this.getUsageStats(),
+      journal: this.getJournalStats(),
+      timestamp: Date.now()
+    }
+  }
+
+  /**
+   * Record operation performance
+   * @param {string} operation - Operation name
+   * @param {number} duration - Duration in milliseconds
+   * @param {boolean} success - Whether operation was successful
+   */
+  recordOperation(operation, duration, success = true) {
+    this.performanceMetrics.totalOperations++
+    
+    if (!success) {
+      this.performanceMetrics.totalErrors++
+    }
+    
+    // Update average operation time
+    const currentAverage = this.performanceMetrics.averageOperationTime
+    const totalOps = this.performanceMetrics.totalOperations
+    this.performanceMetrics.averageOperationTime = (currentAverage * (totalOps - 1) + duration) / totalOps
+    
+    // Update peak memory usage (if available)
+    if (typeof process !== 'undefined' && process.memoryUsage) {
+      const memoryUsage = process.memoryUsage()
+      this.performanceMetrics.peakMemoryUsage = Math.max(
+        this.performanceMetrics.peakMemoryUsage,
+        memoryUsage.heapUsed
+      )
+    }
+  }
+
+  /**
+   * Record cache hit
+   */
+  recordCacheHit() {
+    this.performanceMetrics.cacheHits++
+  }
+
+  /**
+   * Record cache miss
+   */
+  recordCacheMiss() {
+    this.performanceMetrics.cacheMisses++
+  }
+
+  /**
+   * Update query statistics
+   * @param {string} type - Query type ('streaming' or 'indexed')
+   * @param {number} duration - Query duration in milliseconds
+   */
+  updateQueryStats(type, duration) {
+    this.usageStats.totalQueries++
+    
+    if (type === 'streaming') {
+      this.usageStats.streamingQueries++
+      this.updateAverageTime('streaming', duration)
+    } else if (type === 'indexed') {
+      this.usageStats.indexedQueries++
+      this.updateAverageTime('indexed', duration)
+    }
+  }
+
+  /**
+   * Update average time for a query type
+   * @param {string} type - Query type
+   * @param {number} time - Time taken
+   */
+  updateAverageTime(type, time) {
+    const key = `${type}AverageTime`
+    if (!this.usageStats[key]) {
+      this.usageStats[key] = 0
+    }
+    
+    const currentAverage = this.usageStats[key]
+    const count = this.usageStats[`${type}Queries`] || 1
+    
+    // Calculate running average
+    this.usageStats[key] = (currentAverage * (count - 1) + time) / count
+  }
+
+  /**
+   * Reset all statistics
+   */
+  resetStats() {
+    this.usageStats = {
+      totalQueries: 0,
+      streamingQueries: 0,
+      indexedQueries: 0,
+      streamingAverageTime: 0,
+      indexedAverageTime: 0
+    }
+    
+    this.performanceMetrics = {
+      startTime: Date.now(),
+      lastResetTime: Date.now(),
+      totalOperations: 0,
+      totalErrors: 0,
+      averageOperationTime: 0,
+      peakMemoryUsage: 0,
+      cacheHits: 0,
+      cacheMisses: 0
+    }
+    
+    if (this.opts.debugMode) {
+      console.log('📊 Statistics reset')
+    }
+  }
+
+  /**
+   * Export statistics to JSON
+   * @returns {string} - JSON string of statistics
+   */
+  exportStats() {
+    return JSON.stringify(this.getComprehensiveStats(), null, 2)
+  }
+
+  /**
+   * Get statistics summary for logging
+   * @returns {string} - Summary string
+   */
+  getStatsSummary() {
+    const stats = this.getComprehensiveStats()
+    return `
+📊 Database Statistics Summary:
+  Records: ${stats.database.totalRecords}
+  Queries: ${stats.usage.totalQueries} (${Math.round(stats.usage.queryDistribution.streaming * 100)}% streaming, ${Math.round(stats.usage.queryDistribution.indexed * 100)}% indexed)
+  Operations: ${stats.performance.totalOperations}
+  Errors: ${stats.performance.totalErrors}
+  Uptime: ${Math.round(stats.performance.uptime / 1000)}s
+  Cache Hit Rate: ${Math.round(stats.performance.cacheHitRate * 100)}%
+    `.trim()
+  }
+
+  /**
+   * Check if statistics collection is enabled
+   * @returns {boolean} - True if enabled
+   */
+  isEnabled() {
+    return this.opts.collectStatistics !== false
+  }
+
+  /**
+   * Enable or disable statistics collection
+   * @param {boolean} enabled - Whether to enable statistics
+   */
+  setEnabled(enabled) {
+    this.opts.collectStatistics = enabled
+    
+    if (this.opts.debugMode) {
+      console.log(`📊 Statistics collection ${enabled ? 'enabled' : 'disabled'}`)
+    }
+  }
+}

package/src/managers/StreamingProcessor.mjs

@@ -0,0 +1,429 @@
+import { EventEmitter } from 'events'
+import fs from 'fs'
+import readline from 'readline'
+
+/**
+ * StreamingProcessor - Efficient streaming processing for large datasets
+ * 
+ * Features:
+ * - Memory-efficient processing of large files
+ * - Configurable batch sizes
+ * - Progress tracking
+ * - Error handling and recovery
+ * - Transform pipelines
+ * - Backpressure control
+ */
+export class StreamingProcessor extends EventEmitter {
+  constructor(opts = {}) {
+    super()
+    
+    this.opts = {
+      batchSize: opts.batchSize || 1000,
+      maxConcurrency: opts.maxConcurrency || 5,
+      bufferSize: opts.bufferSize || 64 * 1024, // 64KB
+      enableProgress: opts.enableProgress !== false,
+      progressInterval: opts.progressInterval || 1000, // 1 second
+      enableBackpressure: opts.enableBackpressure !== false,
+      maxPendingBatches: opts.maxPendingBatches || 10,
+      ...opts
+    }
+    
+    this.isProcessing = false
+    this.currentBatch = 0
+    this.totalBatches = 0
+    this.processedItems = 0
+    this.totalItems = 0
+    this.pendingBatches = 0
+    this.stats = {
+      startTime: 0,
+      endTime: 0,
+      totalProcessingTime: 0,
+      averageBatchTime: 0,
+      itemsPerSecond: 0,
+      memoryUsage: 0
+    }
+    
+    this.progressTimer = null
+    this.transformPipeline = []
+  }
+
+  /**
+   * Add a transform function to the pipeline
+   */
+  addTransform(transformFn) {
+    this.transformPipeline.push(transformFn)
+    return this
+  }
+
+  /**
+   * Process a file stream
+   */
+  async processFileStream(filePath, processorFn) {
+    if (this.isProcessing) {
+      throw new Error('Streaming processor is already running')
+    }
+    
+    this.isProcessing = true
+    this.stats.startTime = Date.now()
+    this.currentBatch = 0
+    this.processedItems = 0
+    
+    try {
+      // Get file size for progress tracking
+      const stats = await fs.promises.stat(filePath)
+      this.totalItems = Math.ceil(stats.size / this.opts.bufferSize)
+      
+      // Start progress tracking
+      if (this.opts.enableProgress) {
+        this._startProgressTracking()
+      }
+      
+      // Create read stream
+      const fileStream = fs.createReadStream(filePath, {
+        encoding: 'utf8',
+        highWaterMark: this.opts.bufferSize
+      })
+      
+      // Create readline interface
+      const rl = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity
+      })
+      
+      let batch = []
+      let lineCount = 0
+      
+      // Process lines in batches
+      for await (const line of rl) {
+        if (line.trim()) {
+          batch.push(line)
+          lineCount++
+          
+          // Process batch when it reaches the configured size
+          if (batch.length >= this.opts.batchSize) {
+            await this._processBatch(batch, processorFn)
+            batch = []
+          }
+        }
+      }
+      
+      // Process remaining items in the last batch
+      if (batch.length > 0) {
+        await this._processBatch(batch, processorFn)
+      }
+      
+      this.stats.endTime = Date.now()
+      this.stats.totalProcessingTime = this.stats.endTime - this.stats.startTime
+      this.stats.itemsPerSecond = this.processedItems / (this.stats.totalProcessingTime / 1000)
+      
+      this.emit('complete', {
+        totalItems: this.processedItems,
+        totalBatches: this.currentBatch,
+        processingTime: this.stats.totalProcessingTime,
+        itemsPerSecond: this.stats.itemsPerSecond
+      })
+      
+    } catch (error) {
+      this.emit('error', error)
+      throw error
+    } finally {
+      this.isProcessing = false
+      this._stopProgressTracking()
+    }
+  }
+
+  /**
+   * Process an array of items in streaming fashion
+   */
+  async processArray(items, processorFn) {
+    if (this.isProcessing) {
+      throw new Error('Streaming processor is already running')
+    }
+    
+    this.isProcessing = true
+    this.stats.startTime = Date.now()
+    this.currentBatch = 0
+    this.processedItems = 0
+    this.totalItems = items.length
+    this.totalBatches = Math.ceil(items.length / this.opts.batchSize)
+    
+    try {
+      // Start progress tracking
+      if (this.opts.enableProgress) {
+        this._startProgressTracking()
+      }
+      
+      // Process items in batches
+      for (let i = 0; i < items.length; i += this.opts.batchSize) {
+        const batch = items.slice(i, i + this.opts.batchSize)
+        await this._processBatch(batch, processorFn)
+      }
+      
+      this.stats.endTime = Date.now()
+      this.stats.totalProcessingTime = this.stats.endTime - this.stats.startTime
+      this.stats.itemsPerSecond = this.processedItems / (this.stats.totalProcessingTime / 1000)
+      
+      this.emit('complete', {
+        totalItems: this.processedItems,
+        totalBatches: this.currentBatch,
+        processingTime: this.stats.totalProcessingTime,
+        itemsPerSecond: this.stats.itemsPerSecond
+      })
+      
+    } catch (error) {
+      this.emit('error', error)
+      throw error
+    } finally {
+      this.isProcessing = false
+      this._stopProgressTracking()
+    }
+  }
+
+  /**
+   * Process a generator function
+   */
+  async processGenerator(generatorFn, processorFn) {
+    if (this.isProcessing) {
+      throw new Error('Streaming processor is already running')
+    }
+    
+    this.isProcessing = true
+    this.stats.startTime = Date.now()
+    this.currentBatch = 0
+    this.processedItems = 0
+    this.totalItems = 0 // Unknown for generators
+    
+    try {
+      // Start progress tracking
+      if (this.opts.enableProgress) {
+        this._startProgressTracking()
+      }
+      
+      const generator = generatorFn()
+      let batch = []
+      
+      for await (const item of generator) {
+        batch.push(item)
+        this.totalItems++
+        
+        // Process batch when it reaches the configured size
+        if (batch.length >= this.opts.batchSize) {
+          await this._processBatch(batch, processorFn)
+          batch = []
+        }
+      }
+      
+      // Process remaining items in the last batch
+      if (batch.length > 0) {
+        await this._processBatch(batch, processorFn)
+      }
+      
+      this.stats.endTime = Date.now()
+      this.stats.totalProcessingTime = this.stats.endTime - this.stats.startTime
+      this.stats.itemsPerSecond = this.processedItems / (this.stats.totalProcessingTime / 1000)
+      
+      this.emit('complete', {
+        totalItems: this.processedItems,
+        totalBatches: this.currentBatch,
+        processingTime: this.stats.totalProcessingTime,
+        itemsPerSecond: this.stats.itemsPerSecond
+      })
+      
+    } catch (error) {
+      this.emit('error', error)
+      throw error
+    } finally {
+      this.isProcessing = false
+      this._stopProgressTracking()
+    }
+  }
+
+  /**
+   * Process a single batch
+   */
+  async _processBatch(batch, processorFn) {
+    if (this.opts.enableBackpressure && this.pendingBatches >= this.opts.maxPendingBatches) {
+      // Wait for backpressure to reduce
+      await this._waitForBackpressure()
+    }
+    
+    this.pendingBatches++
+    this.currentBatch++
+    
+    try {
+      const startTime = Date.now()
+      
+      // Apply transform pipeline
+      let transformedBatch = batch
+      for (const transform of this.transformPipeline) {
+        transformedBatch = await transform(transformedBatch)
+      }
+      
+      // Process the batch
+      const result = await processorFn(transformedBatch, this.currentBatch)
+      
+      const batchTime = Date.now() - startTime
+      this.stats.averageBatchTime = 
+        (this.stats.averageBatchTime + batchTime) / 2
+      
+      this.processedItems += batch.length
+      
+      this.emit('batchComplete', {
+        batchNumber: this.currentBatch,
+        batchSize: batch.length,
+        processingTime: batchTime,
+        result
+      })
+      
+    } catch (error) {
+      this.emit('batchError', {
+        batchNumber: this.currentBatch,
+        batchSize: batch.length,
+        error
+      })
+      throw error
+    } finally {
+      this.pendingBatches--
+    }
+  }
+
+  /**
+   * Wait for backpressure to reduce
+   */
+  async _waitForBackpressure() {
+    return new Promise((resolve) => {
+      const checkBackpressure = () => {
+        if (this.pendingBatches < this.opts.maxPendingBatches) {
+          resolve()
+        } else {
+          setTimeout(checkBackpressure, 10)
+        }
+      }
+      checkBackpressure()
+    })
+  }
+
+  /**
+   * Start progress tracking
+   */
+  _startProgressTracking() {
+    this.progressTimer = setInterval(() => {
+      const progress = {
+        currentBatch: this.currentBatch,
+        totalBatches: this.totalBatches,
+        processedItems: this.processedItems,
+        totalItems: this.totalItems,
+        percentage: this.totalItems > 0 ? (this.processedItems / this.totalItems) * 100 : 0,
+        itemsPerSecond: this.stats.itemsPerSecond,
+        averageBatchTime: this.stats.averageBatchTime,
+        memoryUsage: process.memoryUsage().heapUsed / 1024 / 1024 // MB
+      }
+      
+      this.emit('progress', progress)
+    }, this.opts.progressInterval)
+    this.progressTimer.unref(); // Allow process to exit without waiting for this timer
+  }
+
+  /**
+   * Stop progress tracking
+   */
+  _stopProgressTracking() {
+    if (this.progressTimer) {
+      clearInterval(this.progressTimer)
+      this.progressTimer = null
+    }
+  }
+
+  /**
+   * Get current statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      isProcessing: this.isProcessing,
+      currentBatch: this.currentBatch,
+      totalBatches: this.totalBatches,
+      processedItems: this.processedItems,
+      totalItems: this.totalItems,
+      pendingBatches: this.pendingBatches,
+      transformPipelineLength: this.transformPipeline.length
+    }
+  }
+
+  /**
+   * Stop processing
+   */
+  stop() {
+    this.isProcessing = false
+    this._stopProgressTracking()
+    this.emit('stopped')
+  }
+
+  /**
+   * Reset the processor
+   */
+  reset() {
+    this.stop()
+    this.currentBatch = 0
+    this.totalBatches = 0
+    this.processedItems = 0
+    this.totalItems = 0
+    this.pendingBatches = 0
+    this.stats = {
+      startTime: 0,
+      endTime: 0,
+      totalProcessingTime: 0,
+      averageBatchTime: 0,
+      itemsPerSecond: 0,
+      memoryUsage: 0
+    }
+    this.transformPipeline = []
+  }
+}
+
+/**
+ * Predefined transform functions
+ */
+export const Transforms = {
+  // Parse JSON lines
+  parseJSON: (batch) => {
+    return batch.map(line => {
+      try {
+        return JSON.parse(line)
+      } catch (error) {
+        console.warn('Failed to parse JSON line:', line)
+        return null
+      }
+    }).filter(item => item !== null)
+  },
+  
+  // Filter out null/undefined values
+  filterNull: (batch) => {
+    return batch.filter(item => item !== null && item !== undefined)
+  },
+  
+  // Transform to specific format
+  toFormat: (format) => (batch) => {
+    switch (format) {
+      case 'string':
+        return batch.map(item => String(item))
+      case 'number':
+        return batch.map(item => Number(item))
+      case 'object':
+        return batch.map(item => typeof item === 'object' ? item : { value: item })
+      default:
+        return batch
+    }
+  },
+  
+  // Add metadata
+  addMetadata: (metadata) => (batch) => {
+    return batch.map(item => ({
+      ...item,
+      ...metadata,
+      processedAt: Date.now()
+    }))
+  }
+}
+
+export default StreamingProcessor