@joystick.js/db-canary 0.0.0-canary.2270 → 0.0.0-canary.2272

package/tests/server/integration/development_mode_authentication.test.js

@@ -48,7 +48,9 @@ test.afterEach(async () => {
   
   if (server) {
     await server.cleanup();
-    server.close();
+    await new Promise((resolve) => {
+      server.close(resolve);
+    });
     server = null;
   }
   

package/tests/server/integration/production_safety_integration.test.js

@@ -57,7 +57,9 @@ test.before(async () => {
 test.after.always(async () => {
   if (server) {
     await server.cleanup();
-    server.close();
+    await new Promise((resolve) => {
+      server.close(resolve);
+    });
   }
 });
 

package/tests/server/lib/bulk_insert_optimizer.test.js

@@ -0,0 +1,523 @@
+/**
+ * @fileoverview Comprehensive tests for JoystickDB bulk insert optimizer.
+ * Tests performance optimizations, safety guarantees, and concurrent read capabilities.
+ */
+
+import test from 'ava';
+import { rmSync, existsSync } from 'fs';
+import { initialize_database, cleanup_database } from '../../../src/server/lib/query_engine.js';
+import {
+  bulk_insert_optimized,
+  bulk_insert_with_metrics,
+  non_blocking_bulk_insert,
+  calculate_average_document_size,
+  calculate_bulk_map_size,
+  create_size_based_batches,
+  sort_documents_by_key,
+  pre_encode_documents
+} from '../../../src/server/lib/bulk_insert_optimizer.js';
+import find_one from '../../../src/server/lib/operations/find_one.js';
+import find from '../../../src/server/lib/operations/find.js';
+
+const TEST_DB_PATH = './test_data/bulk_optimizer_test';
+const TEST_DATABASE = 'test_db';
+const TEST_COLLECTION = 'test_collection';
+
+/**
+ * Generates test documents for bulk insert testing.
+ * @param {number} count - Number of documents to generate
+ * @param {Object} [options={}] - Generation options
+ * @returns {Array<Object>} Array of test documents
+ */
+const generate_test_documents = (count, options = {}) => {
+  const { 
+    include_id = false,
+    base_size = 100,
+    variable_size = false 
+  } = options;
+  
+  const documents = [];
+  
+  for (let i = 0; i < count; i++) {
+    const doc = {
+      name: `Test Document ${i}`,
+      index: i,
+      category: `category_${i % 10}`,
+      active: i % 2 === 0,
+      metadata: {
+        created_by: 'test_user',
+        tags: [`tag_${i % 5}`, `tag_${(i + 1) % 5}`],
+        priority: i % 3
+      }
+    };
+    
+    if (include_id) {
+      doc._id = `test_doc_${i.toString().padStart(6, '0')}`;
+    }
+    
+    // Add variable size content for testing
+    if (variable_size) {
+      const extra_content_size = Math.floor(Math.random() * base_size);
+      doc.content = 'x'.repeat(extra_content_size);
+    }
+    
+    documents.push(doc);
+  }
+  
+  return documents;
+};
+
+/**
+ * Sets up test database before each test.
+ */
+test.beforeEach(async () => {
+  // Clean up any existing test database
+  if (existsSync(TEST_DB_PATH)) {
+    rmSync(TEST_DB_PATH, { recursive: true, force: true });
+  }
+  
+  // Initialize fresh database
+  initialize_database(TEST_DB_PATH);
+});
+
+/**
+ * Cleans up test database after each test.
+ */
+test.afterEach(async () => {
+  await cleanup_database(true);
+});
+
+// Utility function tests
+test('calculate_average_document_size should calculate correct average', t => {
+  const documents = [
+    { name: 'doc1', data: 'small' },
+    { name: 'doc2', data: 'medium_sized_content' },
+    { name: 'doc3', data: 'large_content_with_more_data' }
+  ];
+  
+  const avg_size = calculate_average_document_size(documents);
+  
+  t.true(avg_size > 0);
+  t.true(typeof avg_size === 'number');
+});
+
+test('calculate_bulk_map_size should return appropriate map size', t => {
+  const document_count = 100000;
+  const avg_document_size = 500;
+  
+  const map_size = calculate_bulk_map_size(document_count, avg_document_size);
+  
+  // Should be at least 2x the estimated size
+  const estimated_size = document_count * avg_document_size;
+  t.true(map_size >= estimated_size * 2);
+  
+  // Should be at least 10GB minimum
+  const minimum_size = 1024 * 1024 * 1024 * 10;
+  t.true(map_size >= minimum_size);
+});
+
+test('create_size_based_batches should create appropriate batches', t => {
+  const documents = generate_test_documents(1000);
+  const target_size = 50 * 1024; // 50KB target
+  
+  const batches = create_size_based_batches(documents, target_size);
+  
+  t.true(batches.length > 0);
+  t.true(Array.isArray(batches));
+  
+  // Verify all documents are included
+  const total_docs = batches.reduce((sum, batch) => sum + batch.length, 0);
+  t.is(total_docs, documents.length);
+});
+
+test('sort_documents_by_key should sort documents correctly', t => {
+  const documents = generate_test_documents(100);
+  
+  const sorted_docs = sort_documents_by_key(documents, TEST_DATABASE, TEST_COLLECTION);
+  
+  t.is(sorted_docs.length, documents.length);
+  
+  // Verify all documents have IDs
+  sorted_docs.forEach(doc => {
+    t.truthy(doc._id);
+  });
+  
+  // Verify sorting (keys should be in ascending order)
+  for (let i = 1; i < sorted_docs.length; i++) {
+    const key_a = `${TEST_DATABASE}:${TEST_COLLECTION}:${sorted_docs[i-1]._id}`;
+    const key_b = `${TEST_DATABASE}:${TEST_COLLECTION}:${sorted_docs[i]._id}`;
+    t.true(key_a.localeCompare(key_b) <= 0);
+  }
+});
+
+test('pre_encode_documents should encode documents correctly', t => {
+  const documents = generate_test_documents(10, { include_id: true });
+  
+  const encoded_docs = pre_encode_documents(documents, TEST_DATABASE, TEST_COLLECTION);
+  
+  t.is(encoded_docs.length, documents.length);
+  
+  encoded_docs.forEach((encoded, index) => {
+    t.truthy(encoded.key);
+    t.truthy(encoded.value);
+    t.truthy(encoded.document_id);
+    t.is(encoded.document_id, documents[index]._id);
+    
+    // Verify key format
+    t.true(encoded.key.startsWith(`${TEST_DATABASE}:${TEST_COLLECTION}:`));
+    
+    // Verify value is valid JSON
+    const parsed = JSON.parse(encoded.value);
+    t.truthy(parsed._created_at);
+    t.truthy(parsed._updated_at);
+  });
+});
+
+// Basic bulk insert tests
+test('bulk_insert_optimized should insert small dataset successfully', async t => {
+  const documents = generate_test_documents(100);
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 100);
+  t.is(result.inserted_ids.length, 100);
+  t.truthy(result.performance);
+  t.true(result.performance.duration_ms > 0);
+  t.true(result.performance.documents_per_second > 0);
+});
+
+test('bulk_insert_optimized should insert medium dataset successfully', async t => {
+  const documents = generate_test_documents(10000);
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 10000);
+  t.is(result.inserted_ids.length, 10000);
+  t.truthy(result.performance);
+  
+  // Performance should be reasonable
+  t.true(result.performance.documents_per_second > 1000);
+});
+
+test('bulk_insert_optimized should handle large dataset efficiently', async t => {
+  const documents = generate_test_documents(100000);
+  
+  const start_time = Date.now();
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents);
+  const duration = Date.now() - start_time;
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 100000);
+  t.is(result.inserted_ids.length, 100000);
+  
+  // Should complete in reasonable time (less than 30 seconds)
+  t.true(duration < 30000);
+  
+  // Should achieve good throughput
+  t.true(result.performance.documents_per_second > 3000);
+});
+
+// Performance optimization tests
+test('bulk_insert_optimized with streaming should handle memory efficiently', async t => {
+  const documents = generate_test_documents(50000, { variable_size: true, base_size: 1000 });
+  
+  const start_memory = process.memoryUsage().heapUsed;
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents, {
+    stream_processing: true,
+    batch_size: 1000
+  });
+  
+  const end_memory = process.memoryUsage().heapUsed;
+  const memory_delta = end_memory - start_memory;
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 50000);
+  
+  // Memory usage should be reasonable (less than 500MB delta)
+  t.true(memory_delta < 500 * 1024 * 1024);
+});
+
+test('bulk_insert_optimized with key sorting should improve performance', async t => {
+  const documents = generate_test_documents(10000);
+  
+  // Test with sorting enabled
+  const sorted_result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents, {
+    sort_keys: true
+  });
+  
+  // Clean up for second test
+  await cleanup_database(true);
+  initialize_database(TEST_DB_PATH);
+  
+  // Test with sorting disabled
+  const unsorted_result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents, {
+    sort_keys: false
+  });
+  
+  t.true(sorted_result.acknowledged);
+  t.true(unsorted_result.acknowledged);
+  t.is(sorted_result.inserted_count, 10000);
+  t.is(unsorted_result.inserted_count, 10000);
+  
+  // Sorted version should generally be faster or similar
+  // (This is a performance hint, not a strict requirement)
+  t.true(sorted_result.performance.duration_ms <= unsorted_result.performance.duration_ms * 1.5);
+});
+
+// Error handling tests
+test('bulk_insert_optimized should reject invalid parameters', async t => {
+  await t.throwsAsync(
+    () => bulk_insert_optimized('', TEST_COLLECTION, []),
+    { message: /Database name and collection name are required/ }
+  );
+  
+  await t.throwsAsync(
+    () => bulk_insert_optimized(TEST_DATABASE, '', []),
+    { message: /Database name and collection name are required/ }
+  );
+  
+  await t.throwsAsync(
+    () => bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, []),
+    { message: /Documents must be a non-empty array/ }
+  );
+  
+  await t.throwsAsync(
+    () => bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, null),
+    { message: /Documents must be a non-empty array/ }
+  );
+});
+
+test('bulk_insert_optimized should handle duplicate IDs correctly', async t => {
+  const documents = [
+    { _id: 'duplicate_id', name: 'Document 1' },
+    { _id: 'duplicate_id', name: 'Document 2' }
+  ];
+  
+  await t.throwsAsync(
+    () => bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, documents),
+    { message: /Document with _id duplicate_id already exists/ }
+  );
+});
+
+// Data integrity tests
+test('bulk_insert_optimized should preserve document data integrity', async t => {
+  const original_documents = generate_test_documents(1000, { include_id: true });
+  
+  await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, original_documents);
+  
+  // Verify all documents were inserted correctly
+  for (const original_doc of original_documents) {
+    const retrieved_doc = await find_one(TEST_DATABASE, TEST_COLLECTION, { _id: original_doc._id });
+    
+    t.truthy(retrieved_doc);
+    t.is(retrieved_doc.name, original_doc.name);
+    t.is(retrieved_doc.index, original_doc.index);
+    t.is(retrieved_doc.category, original_doc.category);
+    t.is(retrieved_doc.active, original_doc.active);
+    t.deepEqual(retrieved_doc.metadata, original_doc.metadata);
+    t.truthy(retrieved_doc._created_at);
+    t.truthy(retrieved_doc._updated_at);
+  }
+});
+
+// Concurrent read safety tests
+test('bulk_insert_optimized should allow concurrent reads', async t => {
+  // Insert initial data
+  const initial_docs = generate_test_documents(1000, { include_id: true });
+  await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, initial_docs);
+  
+  // Start bulk insert of additional data
+  const additional_docs = generate_test_documents(10000);
+  const bulk_insert_promise = bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, additional_docs);
+  
+  // Perform concurrent reads while bulk insert is running
+  const read_promises = [];
+  for (let i = 0; i < 10; i++) {
+    read_promises.push(
+      find_one(TEST_DATABASE, TEST_COLLECTION, { _id: initial_docs[i]._id })
+    );
+  }
+  
+  // Wait for both bulk insert and reads to complete
+  const [bulk_result, ...read_results] = await Promise.all([
+    bulk_insert_promise,
+    ...read_promises
+  ]);
+  
+  // Verify bulk insert succeeded
+  t.true(bulk_result.acknowledged);
+  t.is(bulk_result.inserted_count, 10000);
+  
+  // Verify all reads succeeded and returned correct data
+  read_results.forEach((doc, index) => {
+    t.truthy(doc);
+    t.is(doc._id, initial_docs[index]._id);
+    t.is(doc.name, initial_docs[index].name);
+  });
+});
+
+// Performance monitoring tests
+test('bulk_insert_with_metrics should provide detailed performance metrics', async t => {
+  const documents = generate_test_documents(5000);
+  
+  const result = await bulk_insert_with_metrics(TEST_DATABASE, TEST_COLLECTION, documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 5000);
+  t.truthy(result.performance);
+  t.truthy(result.performance.memory_usage);
+  
+  // Verify performance metrics structure
+  t.true(typeof result.performance.duration_ms === 'number');
+  t.true(typeof result.performance.documents_per_second === 'number');
+  t.true(typeof result.performance.memory_usage.start_heap_mb === 'number');
+  t.true(typeof result.performance.memory_usage.end_heap_mb === 'number');
+  t.true(typeof result.performance.memory_usage.delta_heap_mb === 'number');
+  t.true(typeof result.performance.memory_usage.peak_heap_mb === 'number');
+});
+
+// Non-blocking bulk insert tests
+test('non_blocking_bulk_insert should process in chunks', async t => {
+  const documents = generate_test_documents(25000);
+  
+  const result = await non_blocking_bulk_insert(TEST_DATABASE, TEST_COLLECTION, documents, {
+    chunk_size: 5000
+  });
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 25000);
+  t.is(result.inserted_ids.length, 25000);
+  t.truthy(result.performance);
+  t.true(result.performance.duration_ms > 0);
+  t.true(result.performance.documents_per_second > 0);
+});
+
+// Stress tests
+test('bulk_insert_optimized should handle very large documents', async t => {
+  const large_documents = [];
+  for (let i = 0; i < 1000; i++) {
+    large_documents.push({
+      name: `Large Document ${i}`,
+      large_content: 'x'.repeat(10000), // 10KB per document
+      nested_data: {
+        level1: {
+          level2: {
+            level3: {
+              data: Array.from({ length: 100 }, (_, j) => ({ id: j, value: `value_${j}` }))
+            }
+          }
+        }
+      }
+    });
+  }
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, large_documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 1000);
+  t.true(result.performance.documents_per_second > 100);
+});
+
+test('bulk_insert_optimized should maintain performance with mixed document sizes', async t => {
+  const mixed_documents = [];
+  
+  // Small documents
+  for (let i = 0; i < 5000; i++) {
+    mixed_documents.push({
+      type: 'small',
+      index: i,
+      data: 'small_content'
+    });
+  }
+  
+  // Medium documents
+  for (let i = 0; i < 3000; i++) {
+    mixed_documents.push({
+      type: 'medium',
+      index: i,
+      data: 'x'.repeat(1000),
+      metadata: Array.from({ length: 50 }, (_, j) => ({ key: `key_${j}`, value: `value_${j}` }))
+    });
+  }
+  
+  // Large documents
+  for (let i = 0; i < 2000; i++) {
+    mixed_documents.push({
+      type: 'large',
+      index: i,
+      data: 'x'.repeat(5000),
+      large_array: Array.from({ length: 200 }, (_, j) => ({ id: j, content: `content_${j}` }))
+    });
+  }
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, mixed_documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 10000);
+  t.true(result.performance.documents_per_second > 2000);
+  
+  // Verify data integrity for different document types
+  const small_doc = await find_one(TEST_DATABASE, TEST_COLLECTION, { type: 'small' });
+  const medium_doc = await find_one(TEST_DATABASE, TEST_COLLECTION, { type: 'medium' });
+  const large_doc = await find_one(TEST_DATABASE, TEST_COLLECTION, { type: 'large' });
+  
+  t.truthy(small_doc);
+  t.is(small_doc.type, 'small');
+  t.truthy(medium_doc);
+  t.is(medium_doc.type, 'medium');
+  t.is(medium_doc.metadata.length, 50);
+  t.truthy(large_doc);
+  t.is(large_doc.type, 'large');
+  t.is(large_doc.large_array.length, 200);
+});
+
+// Edge case tests
+test('bulk_insert_optimized should handle documents with special characters', async t => {
+  const special_documents = [
+    { name: 'Document with "quotes"', content: 'Content with \n newlines and \t tabs' },
+    { name: 'Document with émojis 🚀', content: 'Unicode content: café, naïve, résumé' },
+    { name: 'Document with JSON', content: '{"nested": "json", "array": [1, 2, 3]}' },
+    { name: 'Document with HTML', content: '<div>HTML content</div>' },
+    { name: 'Document with null values', nullable_field: null, undefined_field: undefined }
+  ];
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, special_documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 5);
+  
+  // Verify special characters are preserved
+  const docs = await find(TEST_DATABASE, TEST_COLLECTION, {});
+  t.is(docs.length, 5);
+  
+  const emoji_doc = docs.find(doc => doc.name.includes('émojis'));
+  t.truthy(emoji_doc);
+  t.true(emoji_doc.content.includes('café'));
+});
+
+test('bulk_insert_optimized should handle empty and minimal documents', async t => {
+  const minimal_documents = [
+    {},
+    { single_field: 'value' },
+    { _id: 'custom_id_1' },
+    { _id: 'custom_id_2', empty_object: {}, empty_array: [] }
+  ];
+  
+  const result = await bulk_insert_optimized(TEST_DATABASE, TEST_COLLECTION, minimal_documents);
+  
+  t.true(result.acknowledged);
+  t.is(result.inserted_count, 4);
+  
+  // Verify all documents were inserted with proper timestamps
+  const docs = await find(TEST_DATABASE, TEST_COLLECTION, {});
+  t.is(docs.length, 4);
+  
+  docs.forEach(doc => {
+    t.truthy(doc._id);
+    t.truthy(doc._created_at);
+    t.truthy(doc._updated_at);
+  });
+});

package/tests/server/lib/operations/admin.test.js

@@ -44,7 +44,9 @@ test.beforeEach(async () => {
 test.afterEach(async () => {
   if (server) {
     await server.cleanup();
-    server.close();
+    await new Promise((resolve) => {
+      server.close(resolve);
+    });
     server = null;
   }
   

package/dist/server/lib/batched_write_queue.js

@@ -1 +0,0 @@
-import l from"./processing_lane.js";import d from"./logger.js";const{create_context_logger:p}=d("batched_write_queue");class h{constructor(t={}){this.batch_size=t.batch_size||100,this.batch_timeout=t.batch_timeout||10,this.lane_count=t.lane_count||4,this.queue_limit=t.queue_limit||1e4,this.overflow_strategy=t.overflow_strategy||"block",this.lanes=Array(this.lane_count).fill(null).map((s,o)=>new l({batch_size:this.batch_size,batch_timeout:this.batch_timeout,lane_id:o})),this.shutting_down=!1,this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:0,max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0,lane_distribution:new Array(this.lane_count).fill(0)},this.log=p()}async enqueue_write_operation(t,s={}){if(this.shutting_down)throw new Error("Server shutting down");if(this.get_current_queue_depth()>=this.queue_limit){if(this.overflow_strategy==="drop")throw new Error("Queue full, operation dropped");this.overflow_strategy==="block"&&await this.wait_for_queue_space()}const i={operation_fn:t,context:s,enqueued_at:Date.now()},_=this.get_lane_for_operation(i),e=this.lanes[_];this.stats.total_operations++,this.stats.lane_distribution[_]++,this.update_queue_depth_stats(),this.log.debug("Operation enqueued to lane",{lane_id:_,total_operations:this.stats.total_operations,context:s});try{const a=await e.add_operation(i);this.stats.completed_operations++;const r=Date.now()-i.enqueued_at;return this.stats.total_wait_time_ms+=r,a}catch(a){throw this.stats.failed_operations++,a}}get_lane_for_operation(t){const s=t.context||{},o=s.collection||"",i=s.document_id||s.id||"",_=`${o}:${i}`;let e=0;for(let r=0;r<_.length;r++){const c=_.charCodeAt(r);e=(e<<5)-e+c,e=e&e}return Math.abs(e)%this.lane_count}get_current_queue_depth(){return this.lanes.reduce((t,s)=>t+s.stats.current_batch_size,0)}update_queue_depth_stats(){this.stats.current_queue_depth=this.get_current_queue_depth(),this.stats.current_queue_depth>this.stats.max_queue_depth&&(this.stats.max_queue_depth=this.stats.current_queue_depth)}async wait_for_queue_space(){const o=Date.now();for(;this.get_current_queue_depth()>=this.queue_limit;){if(Date.now()-o>5e3)throw new Error("Queue full, timeout waiting for space");if(await new Promise(i=>setTimeout(i,10)),this.shutting_down)throw new Error("Server shutting down")}}async flush_all_batches(){const t=this.lanes.map(s=>s.flush_batch());await Promise.all(t)}get_stats(){const t=this.lanes.map(e=>e.get_stats()),s=this.stats.completed_operations>0?Math.round(this.stats.total_wait_time_ms/this.stats.completed_operations):0,o=t.reduce((e,a)=>e+a.total_batch_processing_time_ms,0),i=this.stats.completed_operations>0?Math.round(o/this.stats.completed_operations):0,_=this.stats.lane_distribution.map((e,a)=>({lane_id:a,operations:e,percentage:this.stats.total_operations>0?Math.round(e/this.stats.total_operations*100):0}));return{total_operations:this.stats.total_operations,completed_operations:this.stats.completed_operations,failed_operations:this.stats.failed_operations,current_queue_depth:this.get_current_queue_depth(),max_queue_depth:this.stats.max_queue_depth,avg_wait_time_ms:s,avg_processing_time_ms:i,success_rate:this.stats.total_operations>0?Math.round(this.stats.completed_operations/this.stats.total_operations*100):100,lane_count:this.lane_count,batch_size:this.batch_size,batch_timeout:this.batch_timeout,lane_distribution:this.stats.lane_distribution,lane_utilization:_,lane_stats:t,total_batches_processed:t.reduce((e,a)=>e+a.batches_processed,0),avg_batch_size:t.length>0?Math.round(t.reduce((e,a)=>e+a.avg_batch_size,0)/t.length):0}}clear_stats(){this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:this.get_current_queue_depth(),max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0,lane_distribution:new Array(this.lane_count).fill(0)},this.lanes.forEach(t=>t.clear_stats())}async shutdown(){this.log.info("Shutting down batched write queue",{pending_operations:this.get_current_queue_depth(),lane_count:this.lane_count}),this.shutting_down=!0,await this.flush_all_batches();const t=this.lanes.map(s=>s.shutdown());await Promise.all(t),this.log.info("Batched write queue shutdown complete")}}let n=null;const g=u=>(n||(n=new h(u)),n),f=async()=>{n&&(await n.shutdown(),n=null)};var q=h;export{q as default,g as get_batched_write_queue,f as shutdown_batched_write_queue};

package/dist/server/lib/processing_lane.js

@@ -1 +0,0 @@
-import c from"./logger.js";const{create_context_logger:n}=c("processing_lane");class o{constructor(t={}){this.batch_size=t.batch_size||100,this.batch_timeout=t.batch_timeout||10,this.lane_id=t.lane_id||0,this.current_batch=[],this.processing=!1,this.shutting_down=!1,this.batch_timeout_handle=null,this.stats={total_operations:0,completed_operations:0,failed_operations:0,batches_processed:0,current_batch_size:0,max_batch_size:0,total_batch_wait_time_ms:0,total_batch_processing_time_ms:0},this.log=n(`lane_${this.lane_id}`)}async add_operation(t){if(this.shutting_down)throw new Error("Processing lane shutting down");return new Promise((a,s)=>{if(this.shutting_down){s(new Error("Processing lane shutting down"));return}const e={...t,resolve:a,reject:s,enqueued_at:Date.now(),id:this.generate_operation_id()};this.current_batch.push(e),this.stats.total_operations++,this.stats.current_batch_size=this.current_batch.length,this.stats.current_batch_size>this.stats.max_batch_size&&(this.stats.max_batch_size=this.stats.current_batch_size),this.log.debug("Operation added to batch",{lane_id:this.lane_id,operation_id:e.id,batch_size:this.stats.current_batch_size,context:t.context}),this.current_batch.length>=this.batch_size?this.process_current_batch():this.current_batch.length===1&&this.start_batch_timeout()})}start_batch_timeout(){this.batch_timeout_handle&&clearTimeout(this.batch_timeout_handle),this.batch_timeout_handle=setTimeout(()=>{this.current_batch.length>0&&!this.processing&&(this.log.debug("Batch timeout triggered",{lane_id:this.lane_id,batch_size:this.current_batch.length}),this.process_current_batch())},this.batch_timeout)}async process_current_batch(){if(this.processing||this.current_batch.length===0||this.shutting_down)return;this.batch_timeout_handle&&(clearTimeout(this.batch_timeout_handle),this.batch_timeout_handle=null),this.processing=!0;const t=[...this.current_batch];this.current_batch=[],this.stats.current_batch_size=0;const a=Date.now(),s=Math.min(...t.map(i=>i.enqueued_at)),e=a-s;this.stats.total_batch_wait_time_ms+=e,this.stats.batches_processed++,this.log.debug("Processing batch",{lane_id:this.lane_id,batch_size:t.length,batch_wait_time_ms:e});try{const i=await this.execute_batch_transaction(t),h=Date.now()-a;this.stats.total_batch_processing_time_ms+=h,this.stats.completed_operations+=t.length,t.forEach((_,r)=>{_.resolve(i[r])}),this.log.debug("Batch completed successfully",{lane_id:this.lane_id,batch_size:t.length,batch_wait_time_ms:e,batch_processing_time_ms:h})}catch(i){const h=Date.now()-a;this.stats.total_batch_processing_time_ms+=h,this.stats.failed_operations+=t.length,t.forEach(_=>{_.reject(i)}),this.log.error("Batch processing failed",{lane_id:this.lane_id,batch_size:t.length,batch_wait_time_ms:e,batch_processing_time_ms:h,error:i.message})}this.processing=!1,this.current_batch.length>0&&(this.current_batch.length>=this.batch_size?setImmediate(()=>this.process_current_batch()):this.start_batch_timeout())}async execute_batch_transaction(t){const a=[];for(const s of t)try{const e=await this.execute_with_retry(s.operation_fn,s.context);a.push(e)}catch(e){throw e}return a}async execute_with_retry(t,a,s=3){let e=null;for(let i=1;i<=s;i++)try{return await t()}catch(h){if(e=h,this.is_retryable_error(h)&&i<s){const _=this.calculate_backoff_delay(i);this.log.warn("Operation failed, retrying",{lane_id:this.lane_id,attempt:i,max_retries:s,delay_ms:_,error:h.message,context:a}),await this.sleep(_);continue}break}throw e}is_retryable_error(t){return["MDB_MAP_FULL","MDB_TXN_FULL","MDB_READERS_FULL","EAGAIN","EBUSY"].some(s=>t.message.includes(s)||t.code===s)}calculate_backoff_delay(t){const e=100*Math.pow(2,t-1),i=Math.random()*.1*e;return Math.min(e+i,5e3)}sleep(t){return new Promise(a=>setTimeout(a,t))}generate_operation_id(){return`lane_${this.lane_id}_${Date.now()}_${Math.random().toString(36).substr(2,9)}`}get_stats(){const t=this.stats.batches_processed>0?Math.round(this.stats.total_batch_wait_time_ms/this.stats.batches_processed):0,a=this.stats.batches_processed>0?Math.round(this.stats.total_batch_processing_time_ms/this.stats.batches_processed):0,s=this.stats.batches_processed>0?Math.round(this.stats.completed_operations/this.stats.batches_processed):0;return{lane_id:this.lane_id,...this.stats,avg_batch_wait_time_ms:t,avg_batch_processing_time_ms:a,avg_batch_size:s,success_rate:this.stats.total_operations>0?Math.round(this.stats.completed_operations/this.stats.total_operations*100):100}}clear_stats(){this.stats={total_operations:0,completed_operations:0,failed_operations:0,batches_processed:0,current_batch_size:this.current_batch.length,max_batch_size:0,total_batch_wait_time_ms:0,total_batch_processing_time_ms:0}}async flush_batch(){this.current_batch.length>0&&!this.processing&&await this.process_current_batch()}async shutdown(){for(this.log.info("Shutting down processing lane",{lane_id:this.lane_id,pending_operations:this.current_batch.length,currently_processing:this.processing}),this.shutting_down=!0,this.batch_timeout_handle&&(clearTimeout(this.batch_timeout_handle),this.batch_timeout_handle=null),this.current_batch.length>0&&!this.processing&&await this.process_current_batch();this.processing;)await new Promise(t=>setTimeout(t,10));this.current_batch.forEach(t=>{t.reject(new Error("Processing lane shutting down"))}),this.current_batch=[],this.processing=!1}}var b=o;export{b as default};