@joystick.js/db-canary 0.0.0-canary.2270 → 0.0.0-canary.2272

package/src/server/lib/memory_efficient_bulk_insert.js

@@ -0,0 +1,262 @@
+/**
+ * @fileoverview Memory-efficient bulk insert utilities for very large datasets.
+ * Provides streaming document generation and processing to minimize memory usage.
+ */
+
+import { bulk_insert_optimized } from './bulk_insert_optimizer.js';
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('memory_efficient_bulk_insert');
+
+/**
+ * Memory-efficient document generator that yields documents in batches.
+ * @param {number} total_count - Total number of documents to generate
+ * @param {Object} [options={}] - Generation options
+ * @returns {AsyncGenerator<Array<Object>>} Generator yielding document batches
+ */
+const generate_documents_streaming = async function* (total_count, options = {}) {
+  const {
+    batch_size = 1000,
+    document_template = 'minimal',
+    test_id = Date.now().toString(36)
+  } = options;
+  
+  for (let i = 0; i < total_count; i += batch_size) {
+    const current_batch_size = Math.min(batch_size, total_count - i);
+    const batch = [];
+    
+    for (let j = 0; j < current_batch_size; j++) {
+      const doc_index = i + j;
+      
+      let document;
+      if (document_template === 'minimal') {
+        document = {
+          _id: `mem_${test_id}_${doc_index.toString().padStart(8, '0')}`,
+          idx: doc_index,
+          cat: doc_index % 50,
+          val: doc_index % 1000
+        };
+      } else if (document_template === 'medium') {
+        document = {
+          _id: `mem_${test_id}_${doc_index.toString().padStart(8, '0')}`,
+          name: `Document ${doc_index}`,
+          index: doc_index,
+          category: `category_${doc_index % 100}`,
+          active: doc_index % 2 === 0,
+          priority: doc_index % 5,
+          score: Math.random() * 100,
+          created_timestamp: Date.now() + doc_index
+        };
+      } else if (document_template === 'large') {
+        document = {
+          _id: `mem_${test_id}_${doc_index.toString().padStart(8, '0')}`,
+          name: `Large Document ${doc_index}`,
+          index: doc_index,
+          category: `category_${doc_index % 100}`,
+          subcategory: `subcategory_${doc_index % 20}`,
+          active: doc_index % 2 === 0,
+          priority: doc_index % 5,
+          score: Math.random() * 100,
+          created_timestamp: Date.now() + doc_index,
+          description: `This is a large document with index ${doc_index} for performance testing purposes.`,
+          metadata: {
+            created_by: `user_${doc_index % 1000}`,
+            department: `dept_${doc_index % 50}`,
+            project: `project_${doc_index % 200}`,
+            tags: [`tag_${doc_index % 10}`, `tag_${(doc_index + 1) % 10}`]
+          },
+          measurements: Array.from({ length: 5 }, (_, k) => ({
+            timestamp: Date.now() + doc_index + k,
+            value: Math.random() * 1000
+          }))
+        };
+      }
+      
+      batch.push(document);
+    }
+    
+    yield batch;
+    
+    // Yield to event loop every batch to prevent blocking
+    await new Promise(resolve => setImmediate(resolve));
+  }
+};
+
+/**
+ * Memory-efficient bulk insert that processes documents in streaming fashion.
+ * @param {string} database_name - Database name
+ * @param {string} collection_name - Collection name
+ * @param {number} document_count - Number of documents to insert
+ * @param {Object} [options={}] - Options
+ * @returns {Promise<Object>} Bulk insert results
+ */
+const memory_efficient_bulk_insert = async (database_name, collection_name, document_count, options = {}) => {
+  const {
+    generation_batch_size = 1000,
+    insert_batch_size = 250,
+    document_template = 'minimal',
+    disable_indexing = true,
+    pre_allocate_map_size = true,
+    sort_keys = true
+  } = options;
+  
+  const log = create_context_logger();
+  const start_time = Date.now();
+  const start_memory = process.memoryUsage();
+  
+  log.info('Starting memory-efficient bulk insert', {
+    database: database_name,
+    collection: collection_name,
+    document_count,
+    generation_batch_size,
+    insert_batch_size,
+    document_template
+  });
+  
+  const all_inserted_ids = [];
+  let processed_count = 0;
+  let batch_number = 0;
+  
+  try {
+    // Process documents in streaming fashion
+    for await (const document_batch of generate_documents_streaming(document_count, {
+      batch_size: generation_batch_size,
+      document_template
+    })) {
+      
+      // Insert the batch using optimized bulk insert
+      const result = await bulk_insert_optimized(database_name, collection_name, document_batch, {
+        disable_indexing,
+        pre_allocate_map_size: batch_number === 0 ? pre_allocate_map_size : false, // Only pre-allocate on first batch
+        sort_keys,
+        stream_processing: true,
+        batch_size: insert_batch_size
+      });
+      
+      all_inserted_ids.push(...result.inserted_ids);
+      processed_count += result.inserted_count;
+      batch_number++;
+      
+      // Clear the batch to help GC
+      document_batch.length = 0;
+      
+      // Log progress every 10 batches
+      if (batch_number % 10 === 0) {
+        const current_memory = process.memoryUsage();
+        log.info('Memory-efficient bulk insert progress', {
+          processed: processed_count,
+          total: document_count,
+          percentage: Math.round((processed_count / document_count) * 100),
+          current_heap_mb: Math.round(current_memory.heapUsed / (1024 * 1024)),
+          batches_processed: batch_number
+        });
+      }
+      
+      // Optimized memory management for very large datasets
+      if (document_count >= 10000000) {
+        // For 10M+ documents, force GC every 20 batches with minimal delay
+        if (batch_number % 20 === 0 && global.gc) {
+          global.gc();
+          await new Promise(resolve => setTimeout(resolve, 25));
+        }
+        // Yield less frequently for 10M+ to improve performance
+        if (batch_number % 5 === 0) {
+          await new Promise(resolve => setImmediate(resolve));
+        }
+      } else if (document_count >= 5000000) {
+        // For 5M+ documents, force GC every 10 batches
+        if (batch_number % 10 === 0 && global.gc) {
+          global.gc();
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+        // Yield every other batch
+        if (batch_number % 2 === 0) {
+          await new Promise(resolve => setImmediate(resolve));
+        }
+      } else if (document_count >= 1000000) {
+        // For 1M+ documents, force GC every 10 batches
+        if (batch_number % 10 === 0 && global.gc) {
+          global.gc();
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+        // Always yield to event loop
+        await new Promise(resolve => setImmediate(resolve));
+      } else {
+        // For smaller datasets, yield every batch
+        await new Promise(resolve => setImmediate(resolve));
+      }
+    }
+    
+    const end_time = Date.now();
+    const end_memory = process.memoryUsage();
+    
+    const performance_metrics = {
+      duration_ms: end_time - start_time,
+      documents_per_second: Math.round(document_count / ((end_time - start_time) / 1000)),
+      memory_usage: {
+        start_heap_mb: Math.round(start_memory.heapUsed / (1024 * 1024)),
+        end_heap_mb: Math.round(end_memory.heapUsed / (1024 * 1024)),
+        delta_heap_mb: Math.round((end_memory.heapUsed - start_memory.heapUsed) / (1024 * 1024)),
+        peak_heap_mb: Math.round(end_memory.heapUsed / (1024 * 1024))
+      }
+    };
+    
+    log.info('Memory-efficient bulk insert completed', {
+      database: database_name,
+      collection: collection_name,
+      inserted_count: all_inserted_ids.length,
+      performance: performance_metrics
+    });
+    
+    return {
+      acknowledged: true,
+      inserted_count: all_inserted_ids.length,
+      inserted_ids: all_inserted_ids,
+      performance: performance_metrics
+    };
+    
+  } catch (error) {
+    log.error('Memory-efficient bulk insert failed', {
+      database: database_name,
+      collection: collection_name,
+      error: error.message
+    });
+    throw error;
+  }
+};
+
+/**
+ * Estimates memory usage for a bulk insert operation.
+ * @param {number} document_count - Number of documents
+ * @param {string} document_template - Document template type
+ * @param {number} batch_size - Batch size for processing
+ * @returns {Object} Memory usage estimates
+ */
+const estimate_memory_usage = (document_count, document_template = 'minimal', batch_size = 1000) => {
+  const doc_sizes = {
+    minimal: 50,   // ~50 bytes per document
+    medium: 200,   // ~200 bytes per document
+    large: 500     // ~500 bytes per document
+  };
+  
+  const avg_doc_size = doc_sizes[document_template] || doc_sizes.minimal;
+  const batch_memory_mb = Math.round((batch_size * avg_doc_size) / (1024 * 1024));
+  const total_data_size_mb = Math.round((document_count * avg_doc_size) / (1024 * 1024));
+  
+  // Estimate peak memory usage (batch + overhead + LMDB buffers)
+  const estimated_peak_mb = batch_memory_mb * 3 + 100; // 3x batch size + 100MB overhead
+  
+  return {
+    avg_document_size_bytes: avg_doc_size,
+    total_data_size_mb,
+    batch_memory_mb,
+    estimated_peak_memory_mb: estimated_peak_mb,
+    recommended_batch_size: document_count >= 10000000 ? 2000 : document_count >= 5000000 ? 1000 : document_count >= 1000000 ? 750 : 1000
+  };
+};
+
+export {
+  memory_efficient_bulk_insert,
+  generate_documents_streaming,
+  estimate_memory_usage
+};

package/src/server/lib/write_queue.js

@@ -2,13 +2,9 @@
  * @fileoverview Write queue system for JoystickDB providing serialized write operations.
  * Ensures write operations are processed sequentially to maintain data consistency and ACID properties.
  * Includes retry logic, backoff strategies, performance monitoring, and graceful shutdown capabilities.
- * 
- * Now supports both traditional sequential processing and high-performance batched processing
- * with automatic fallback and transparent integration.
  */
 
 import create_logger from './logger.js';
-import { get_batched_write_queue, shutdown_batched_write_queue } from './batched_write_queue.js';
 
 const { create_context_logger } = create_logger('write_queue');
 
@@ -316,26 +312,13 @@ class WriteQueue {
 /** @type {WriteQueue|null} Singleton instance of the write queue */
 let write_queue_instance = null;
 
-/** @type {boolean} Whether to use batched write queue for improved performance */
-let use_batched_queue = true;
-
 /**
  * Gets the singleton write queue instance, creating it if it doesn't exist.
- * Automatically uses batched write queue for improved performance while maintaining
- * complete backward compatibility.
- * @param {Object} [options] - Configuration options for batched queue
  * @returns {WriteQueue} The write queue instance
  */
-export const get_write_queue = (options) => {
+export const get_write_queue = () => {
   if (!write_queue_instance) {
-    if (use_batched_queue) {
-      // Use batched write queue with WriteQueue-compatible wrapper
-      const batched_queue = get_batched_write_queue(options);
-      write_queue_instance = new WriteQueueWrapper(batched_queue);
-    } else {
-      // Use traditional sequential write queue
-      write_queue_instance = new WriteQueue();
-    }
+    write_queue_instance = new WriteQueue();
   }
   return write_queue_instance;
 };
@@ -349,122 +332,4 @@ export const shutdown_write_queue = async () => {
     await write_queue_instance.shutdown();
     write_queue_instance = null;
   }
-  
-  // Also shutdown batched queue if it was used
-  if (use_batched_queue) {
-    await shutdown_batched_write_queue();
-  }
 };
-
-/**
- * Enables or disables batched write queue usage.
- * @param {boolean} enabled - Whether to use batched queue
- */
-export const set_batched_queue_enabled = (enabled) => {
-  use_batched_queue = enabled;
-};
-
-/**
- * Wrapper class that provides WriteQueue-compatible API while using BatchedWriteQueue internally.
- * Ensures complete backward compatibility with existing code.
- */
-class WriteQueueWrapper {
-  /**
-   * Creates a new WriteQueueWrapper instance.
-   * @param {BatchedWriteQueue} batched_queue - The batched write queue instance to wrap
-   */
-  constructor(batched_queue) {
-    this.batched_queue = batched_queue;
-    this.log = create_context_logger('write_queue_wrapper');
-  }
-
-  /**
-   * Enqueues a write operation using the batched queue.
-   * Maintains identical API to original WriteQueue.
-   * @param {function} operation_fn - Async function that performs the write operation
-   * @param {Object} [context={}] - Additional context for logging and debugging
-   * @returns {Promise<*>} Promise that resolves with the operation result
-   */
-  async enqueue_write_operation(operation_fn, context = {}) {
-    return this.batched_queue.enqueue_write_operation(operation_fn, context);
-  }
-
-  /**
-   * Gets queue statistics with backward-compatible format.
-   * @returns {Object} Statistics object matching original WriteQueue format
-   */
-  get_stats() {
-    const batched_stats = this.batched_queue.get_stats();
-    
-    // Return stats in original WriteQueue format for backward compatibility
-    return {
-      total_operations: batched_stats.total_operations,
-      completed_operations: batched_stats.completed_operations,
-      failed_operations: batched_stats.failed_operations,
-      current_queue_depth: batched_stats.current_queue_depth,
-      max_queue_depth: batched_stats.max_queue_depth,
-      avg_wait_time_ms: batched_stats.avg_wait_time_ms,
-      avg_processing_time_ms: batched_stats.avg_processing_time_ms,
-      success_rate: batched_stats.success_rate
-    };
-  }
-
-  /**
-   * Clears all statistics.
-   */
-  clear_stats() {
-    this.batched_queue.clear_stats();
-  }
-
-  /**
-   * Gracefully shuts down the wrapper and underlying batched queue.
-   * @returns {Promise<void>} Promise that resolves when shutdown is complete
-   */
-  async shutdown() {
-    await this.batched_queue.shutdown();
-  }
-
-  /**
-   * Determines if an error is retryable based on error patterns.
-   * Exposed for backward compatibility with existing tests.
-   * @param {Error} error - Error to check
-   * @returns {boolean} True if error is retryable, false otherwise
-   */
-  is_retryable_error(error) {
-    const retryable_patterns = [
-      'MDB_MAP_FULL',
-      'MDB_TXN_FULL',
-      'MDB_READERS_FULL',
-      'EAGAIN',
-      'EBUSY'
-    ];
-    
-    return retryable_patterns.some(pattern => 
-      error.message.includes(pattern) || error.code === pattern
-    );
-  }
-
-  /**
-   * Calculates exponential backoff delay with jitter for retry attempts.
-   * Exposed for backward compatibility with existing tests.
-   * @param {number} attempt - Current attempt number (1-based)
-   * @returns {number} Delay in milliseconds
-   */
-  calculate_backoff_delay(attempt) {
-    const base_delay = 100;
-    const max_delay = 5000;
-    const exponential_delay = base_delay * Math.pow(2, attempt - 1);
-    const jitter = Math.random() * 0.1 * exponential_delay;
-    
-    return Math.min(exponential_delay + jitter, max_delay);
-  }
-
-  /**
-   * Generates a unique operation ID for tracking.
-   * Exposed for backward compatibility with existing tests.
-   * @returns {string} Unique operation identifier
-   */
-  generate_operation_id() {
-    return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-  }
-}