@joystick.js/db-canary 0.0.0-canary.2270 → 0.0.0-canary.2272

package/dist/server/lib/auto_index_manager.js

@@ -1 +1 @@
-import{get_database as I}from"./query_engine.js";import{create_index as M,get_indexes as q,drop_index as w}from"./index_manager.js";import{get_settings as S}from"./load_settings.js";import A from"./logger.js";const{create_context_logger:d}=A("auto_index_manager");let _=null,u=new Map,c=new Map,m=null;const k=()=>(_||(_=I().openDB("auto_indexes",{create:!0}),N(),C(),O()),_),g=()=>{if(!_)throw new Error("Auto index database not initialized. Call initialize_auto_index_database first.");return _},f=()=>{try{return S().auto_indexing||{enabled:!0,frequency_threshold:100,performance_threshold_ms:50,max_auto_indexes_per_collection:10,monitoring_window_hours:24,cleanup_unused_after_hours:168,excluded_fields:["_id","created_at"],included_collections:["*"],excluded_collections:[]}}catch{return{enabled:!0,frequency_threshold:100,performance_threshold_ms:50,max_auto_indexes_per_collection:10,monitoring_window_hours:24,cleanup_unused_after_hours:168,excluded_fields:["_id","created_at"],included_collections:["*"],excluded_collections:[]}}},z=e=>{const o=f();return o.excluded_collections.includes(e)?!1:o.included_collections.includes("*")?!0:o.included_collections.includes(e)},Q=e=>!f().excluded_fields.includes(e),R=e=>{const o=[];if(!e||typeof e!="object")return o;for(const[t,a]of Object.entries(e))Q(t)&&o.push(t);return o},E=(e,o,t,a=!1,n=null)=>{const r=d(),s=f();if(!s.enabled||!z(e)||!_)return;const i=R(o),x=new Date;u.has(e)||u.set(e,new Map);const y=u.get(e);for(const h of i){y.has(h)||y.set(h,{query_count:0,total_time_ms:0,avg_time_ms:0,last_queried:x,slow_query_count:0,used_index_count:0});const l=y.get(h);l.query_count++,l.total_time_ms+=t,l.avg_time_ms=l.total_time_ms/l.query_count,l.last_queried=x,t>s.performance_threshold_ms&&l.slow_query_count++,a&&(n===h||n===null)&&l.used_index_count++}r.debug("Query recorded for auto-indexing analysis",{collection:e,fields:i,execution_time_ms:t,used_index:a,indexed_field:n})},B=()=>{const e=d();try{const o=g(),t={};for(const[a,n]of u.entries()){t[a]={};for(const[r,s]of n.entries())t[a][r]={...s,last_queried:s.last_queried.toISOString()}}o.put("query_stats",t),e.debug("Query statistics saved to database")}catch(o){e.error("Failed to save query statistics",{error:o.message})}},C=()=>{const e=d();try{const t=g().get("query_stats");if(t){u.clear();for(const[a,n]of Object.entries(t)){const r=new Map;for(const[s,i]of Object.entries(n))r.set(s,{...i,last_queried:new Date(i.last_queried)});u.set(a,r)}e.debug("Query statistics loaded from database")}}catch(o){e.error("Failed to load query statistics",{error:o.message})}},p=()=>{const e=d();try{const o=g(),t={};for(const[a,n]of c.entries()){t[a]={};for(const[r,s]of n.entries())t[a][r]={...s,created_at:s.created_at.toISOString(),last_used:s.last_used?s.last_used.toISOString():null}}o.put("auto_index_metadata",t),e.debug("Auto index metadata saved to database")}catch(o){e.error("Failed to save auto index metadata",{error:o.message})}},N=()=>{const e=d();try{const t=g().get("auto_index_metadata");if(t){c.clear();for(const[a,n]of Object.entries(t)){const r=new Map;for(const[s,i]of Object.entries(n))r.set(s,{...i,created_at:new Date(i.created_at),last_used:i.last_used?new Date(i.last_used):null});c.set(a,r)}e.debug("Auto index metadata loaded from database")}}catch(o){e.error("Failed to load auto index metadata",{error:o.message})}},v=(e,o)=>{try{const t=c.get(e);return!!(t&&t.has(o))}catch{return!1}},P=(e,o)=>q("default",e).filter(n=>v(e,n.field)).length>=o.max_auto_indexes_per_collection,T=(e,o)=>new Date-e.last_queried<=o,G=(e,o)=>o.some(t=>t.field===e),H=(e,o)=>{const t=e.query_count>=o.frequency_threshold,a=e.avg_time_ms>=o.performance_threshold_ms,n=e.slow_query_count>0;return t||a&&n},J=(e,o)=>e.slow_query_count*2+e.query_count/o.frequency_threshold,K=(e,o,t,a)=>({collection:e,field:o,stats:{...t},priority:J(t,a)}),b=()=>{const e=f(),o=[],t=e.monitoring_window_hours*60*60*1e3;for(const[a,n]of u.entries()){if(P(a,e))continue;const r=q("default",a);for(const[s,i]of n.entries())T(i,t)&&(G(s,r)||H(i,e)&&o.push(K(a,s,i,e)))}return o.sort((a,n)=>n.priority-a.priority)},F=async(e,o,t)=>{const a=d();try{return await M("default",e,o,{sparse:!0}),c.has(e)||c.set(e,new Map),c.get(e).set(o,{created_at:new Date,query_count_at_creation:t.query_count,avg_performance_improvement_ms:0,last_used:null,usage_count:0,auto_created:!0}),p(),a.info("Automatic index created",{collection:e,field:o,query_count:t.query_count,avg_time_ms:t.avg_time_ms,slow_query_count:t.slow_query_count}),!0}catch(n){return a.error("Failed to create automatic index",{collection:e,field:o,error:n.message}),!1}},D=async()=>{const e=d();if(f().enabled)try{const t=b();if(t.length===0){e.debug("No automatic index candidates found");return}e.info("Evaluating automatic index candidates",{candidate_count:t.length});for(const a of t.slice(0,5))await F(a.collection,a.field,a.stats)&&await new Promise(r=>setTimeout(r,100))}catch(t){e.error("Failed to evaluate automatic indexes",{error:t.message})}},L=async()=>{const e=d(),o=f(),t=new Date,a=o.cleanup_unused_after_hours*60*60*1e3;try{for(const[n,r]of c.entries())for(const[s,i]of r.entries())i.last_used?t-i.last_used>a&&(await w("default",n,s),r.delete(s),e.info("Removed unused automatic index",{collection:n,field:s,last_used:i.last_used,usage_count:i.usage_count})):t-i.created_at>a&&(await w("default",n,s),r.delete(s),e.info("Removed unused automatic index",{collection:n,field:s,created_at:i.created_at,usage_count:i.usage_count}));p()}catch(n){e.error("Failed to cleanup unused indexes",{error:n.message})}},U=(e,o)=>{const t=c.get(e);if(t&&t.has(o)){const a=t.get(o);a.last_used=new Date,a.usage_count++}},O=()=>{const e=f();m&&clearInterval(m),e.enabled&&(m=setInterval(async()=>{B(),await D(),await L()},6e4))},j=()=>{m&&(clearInterval(m),m=null)},V=(e=null)=>{if(e){const t=u.get(e);if(!t)return{};const a={};for(const[n,r]of t.entries())a[n]={...r};return a}const o={};for(const[t,a]of u.entries()){o[t]={};for(const[n,r]of a.entries())o[t][n]={...r}}return o},W=()=>{const e={total_auto_indexes:0,collections:{}};for(const[o,t]of c.entries()){e.collections[o]={};for(const[a,n]of t.entries())e.total_auto_indexes++,e.collections[o][a]={...n}}return e},X=async(e=null)=>{const o=d();try{if(e){const t=b().filter(a=>a.collection===e);for(const a of t)await F(a.collection,a.field,a.stats);o.info("Forced index evaluation completed",{collection:e,candidates_processed:t.length})}else await D(),o.info("Forced index evaluation completed for all collections");return{acknowledged:!0}}catch(t){throw o.error("Failed to force index evaluation",{error:t.message}),t}},Y=async(e,o=null)=>{const t=d();try{const a=c.get(e);if(!a)return{acknowledged:!0,removed_count:0};const n=o||Array.from(a.keys());let r=0;for(const s of n)a.has(s)&&(await w("default",e,s),a.delete(s),r++,t.info("Removed automatic index",{collection:e,field:s}));return p(),{acknowledged:!0,removed_count:r}}catch(a){throw t.error("Failed to remove automatic indexes",{collection:e,field_names:o,error:a.message}),a}},Z=()=>{if(j(),_)try{_.remove("query_stats"),_.remove("auto_index_metadata")}catch{}u.clear(),c.clear(),_=null};export{Z as cleanup_auto_index_database,X as force_index_evaluation,g as get_auto_index_database,W as get_auto_index_statistics,V as get_query_statistics,k as initialize_auto_index_database,v as is_auto_created_index,U as record_index_usage,E as record_query,Y as remove_automatic_indexes,O as start_evaluation_timer,j as stop_evaluation_timer};
+import{get_database as I}from"./query_engine.js";import{create_index as M,get_indexes as q,drop_index as w}from"./index_manager.js";import{get_settings as S}from"./load_settings.js";import A from"./logger.js";const{create_context_logger:_}=A("auto_index_manager");let u=null,d=new Map,c=new Map,m=null;const k=()=>(u||(u=I().openDB("auto_indexes",{create:!0}),d.clear(),c.clear(),N(),C(),O()),u),g=()=>{if(!u)throw new Error("Auto index database not initialized. Call initialize_auto_index_database first.");return u},f=()=>{try{return S().auto_indexing||{enabled:!0,frequency_threshold:100,performance_threshold_ms:50,max_auto_indexes_per_collection:10,monitoring_window_hours:24,cleanup_unused_after_hours:168,excluded_fields:["_id","created_at"],included_collections:["*"],excluded_collections:[]}}catch{return{enabled:!0,frequency_threshold:100,performance_threshold_ms:50,max_auto_indexes_per_collection:10,monitoring_window_hours:24,cleanup_unused_after_hours:168,excluded_fields:["_id","created_at"],included_collections:["*"],excluded_collections:[]}}},z=e=>{const o=f();return o.excluded_collections.includes(e)?!1:o.included_collections.includes("*")?!0:o.included_collections.includes(e)},Q=e=>!f().excluded_fields.includes(e),R=e=>{const o=[];if(!e||typeof e!="object")return o;for(const[t,a]of Object.entries(e))Q(t)&&o.push(t);return o},E=(e,o,t,a=!1,n=null)=>{const r=_(),s=f();if(!s.enabled||!z(e)||!u)return;const i=R(o),x=new Date;d.has(e)||d.set(e,new Map);const y=d.get(e);for(const h of i){y.has(h)||y.set(h,{query_count:0,total_time_ms:0,avg_time_ms:0,last_queried:x,slow_query_count:0,used_index_count:0});const l=y.get(h);l.query_count++,l.total_time_ms+=t,l.avg_time_ms=l.total_time_ms/l.query_count,l.last_queried=x,t>s.performance_threshold_ms&&l.slow_query_count++,a&&(n===h||n===null)&&l.used_index_count++}r.debug("Query recorded for auto-indexing analysis",{collection:e,fields:i,execution_time_ms:t,used_index:a,indexed_field:n})},B=()=>{const e=_();try{const o=g(),t={};for(const[a,n]of d.entries()){t[a]={};for(const[r,s]of n.entries())t[a][r]={...s,last_queried:s.last_queried.toISOString()}}o.put("query_stats",t),e.debug("Query statistics saved to database")}catch(o){e.error("Failed to save query statistics",{error:o.message})}},C=()=>{const e=_();try{const t=g().get("query_stats");if(t){d.clear();for(const[a,n]of Object.entries(t)){const r=new Map;for(const[s,i]of Object.entries(n))r.set(s,{...i,last_queried:new Date(i.last_queried)});d.set(a,r)}e.debug("Query statistics loaded from database")}}catch(o){e.error("Failed to load query statistics",{error:o.message})}},p=()=>{const e=_();try{const o=g(),t={};for(const[a,n]of c.entries()){t[a]={};for(const[r,s]of n.entries())t[a][r]={...s,created_at:s.created_at.toISOString(),last_used:s.last_used?s.last_used.toISOString():null}}o.put("auto_index_metadata",t),e.debug("Auto index metadata saved to database")}catch(o){e.error("Failed to save auto index metadata",{error:o.message})}},N=()=>{const e=_();try{const t=g().get("auto_index_metadata");if(t){c.clear();for(const[a,n]of Object.entries(t)){const r=new Map;for(const[s,i]of Object.entries(n))r.set(s,{...i,created_at:new Date(i.created_at),last_used:i.last_used?new Date(i.last_used):null});c.set(a,r)}e.debug("Auto index metadata loaded from database")}}catch(o){e.error("Failed to load auto index metadata",{error:o.message})}},v=(e,o)=>{try{const t=c.get(e);return!!(t&&t.has(o))}catch{return!1}},P=(e,o)=>q("default",e).filter(n=>v(e,n.field)).length>=o.max_auto_indexes_per_collection,T=(e,o)=>new Date-e.last_queried<=o,G=(e,o)=>o.some(t=>t.field===e),H=(e,o)=>{const t=e.query_count>=o.frequency_threshold,a=e.avg_time_ms>=o.performance_threshold_ms,n=e.slow_query_count>0;return t||a&&n},J=(e,o)=>e.slow_query_count*2+e.query_count/o.frequency_threshold,K=(e,o,t,a)=>({collection:e,field:o,stats:{...t},priority:J(t,a)}),b=()=>{const e=f(),o=[],t=e.monitoring_window_hours*60*60*1e3;for(const[a,n]of d.entries()){if(P(a,e))continue;const r=q("default",a);for(const[s,i]of n.entries())T(i,t)&&(G(s,r)||H(i,e)&&o.push(K(a,s,i,e)))}return o.sort((a,n)=>n.priority-a.priority)},F=async(e,o,t)=>{const a=_();try{return await M("default",e,o,{sparse:!0}),c.has(e)||c.set(e,new Map),c.get(e).set(o,{created_at:new Date,query_count_at_creation:t.query_count,avg_performance_improvement_ms:0,last_used:null,usage_count:0,auto_created:!0}),p(),a.info("Automatic index created",{collection:e,field:o,query_count:t.query_count,avg_time_ms:t.avg_time_ms,slow_query_count:t.slow_query_count}),!0}catch(n){return a.error("Failed to create automatic index",{collection:e,field:o,error:n.message}),!1}},D=async()=>{const e=_();if(f().enabled)try{const t=b();if(t.length===0){e.debug("No automatic index candidates found");return}e.info("Evaluating automatic index candidates",{candidate_count:t.length});for(const a of t.slice(0,5))await F(a.collection,a.field,a.stats)&&await new Promise(r=>setTimeout(r,100))}catch(t){e.error("Failed to evaluate automatic indexes",{error:t.message})}},L=async()=>{const e=_(),o=f(),t=new Date,a=o.cleanup_unused_after_hours*60*60*1e3;try{for(const[n,r]of c.entries())for(const[s,i]of r.entries())i.last_used?t-i.last_used>a&&(await w("default",n,s),r.delete(s),e.info("Removed unused automatic index",{collection:n,field:s,last_used:i.last_used,usage_count:i.usage_count})):t-i.created_at>a&&(await w("default",n,s),r.delete(s),e.info("Removed unused automatic index",{collection:n,field:s,created_at:i.created_at,usage_count:i.usage_count}));p()}catch(n){e.error("Failed to cleanup unused indexes",{error:n.message})}},U=(e,o)=>{const t=c.get(e);if(t&&t.has(o)){const a=t.get(o);a.last_used=new Date,a.usage_count++}},O=()=>{const e=f();m&&clearInterval(m),e.enabled&&(m=setInterval(async()=>{B(),await D(),await L()},6e4))},j=()=>{m&&(clearInterval(m),m=null)},V=(e=null)=>{if(e){const t=d.get(e);if(!t)return{};const a={};for(const[n,r]of t.entries())a[n]={...r};return a}const o={};for(const[t,a]of d.entries()){o[t]={};for(const[n,r]of a.entries())o[t][n]={...r}}return o},W=()=>{const e={total_auto_indexes:0,collections:{}};for(const[o,t]of c.entries()){e.collections[o]={};for(const[a,n]of t.entries())e.total_auto_indexes++,e.collections[o][a]={...n}}return e},X=async(e=null)=>{const o=_();try{if(e){const t=b().filter(a=>a.collection===e);for(const a of t)await F(a.collection,a.field,a.stats);o.info("Forced index evaluation completed",{collection:e,candidates_processed:t.length})}else await D(),o.info("Forced index evaluation completed for all collections");return{acknowledged:!0}}catch(t){throw o.error("Failed to force index evaluation",{error:t.message}),t}},Y=async(e,o=null)=>{const t=_();try{const a=c.get(e);if(!a)return{acknowledged:!0,removed_count:0};const n=o||Array.from(a.keys());let r=0;for(const s of n)a.has(s)&&(await w("default",e,s),a.delete(s),r++,t.info("Removed automatic index",{collection:e,field:s}));return p(),{acknowledged:!0,removed_count:r}}catch(a){throw t.error("Failed to remove automatic indexes",{collection:e,field_names:o,error:a.message}),a}},Z=()=>{if(j(),d.clear(),c.clear(),u){try{u.remove("query_stats"),u.remove("auto_index_metadata")}catch{}u=null}};export{Z as cleanup_auto_index_database,X as force_index_evaluation,g as get_auto_index_database,W as get_auto_index_statistics,V as get_query_statistics,k as initialize_auto_index_database,v as is_auto_created_index,U as record_index_usage,E as record_query,Y as remove_automatic_indexes,O as start_evaluation_timer,j as stop_evaluation_timer};

package/dist/server/lib/bulk_insert_optimizer.js

@@ -0,0 +1 @@
+import{get_database as f,build_collection_key as b,check_and_grow_map_size as v}from"./query_engine.js";import{get_write_queue as B}from"./write_queue.js";import"./auto_index_manager.js";import R from"./logger.js";const{create_context_logger:w}=R("bulk_insert_optimizer"),C=100*1024*1024,U=1e3,D=1e4,I=(e,r=100)=>{const t=e.slice(0,Math.min(r,e.length)),n=t.reduce((o,s)=>o+Buffer.byteLength(JSON.stringify(s),"utf8"),0);return Math.ceil(n/t.length)},P=(e,r)=>{const t=e*r,n=2,o=1024*1024*1024*10;return Math.max(t*n,o)},J=async e=>{const r=w();if(e.length===0)return;const t=I(e),n=P(e.length,t);r.info("Pre-allocating map size for bulk insert",{document_count:e.length,avg_document_size:t,required_map_size:n,required_map_size_gb:Math.round(n/(1024*1024*1024)*100)/100}),await v();const o=f();if(o.resize)try{o.resize(n),r.info("Map size pre-allocated successfully",{new_map_size:n,new_map_size_gb:Math.round(n/(1024*1024*1024)*100)/100})}catch(s){r.warn("Failed to pre-allocate map size",{error:s.message})}},O=(e,r=C)=>{const t=[];let n=[],o=0;for(const s of e){const i=Buffer.byteLength(JSON.stringify(s),"utf8");o+i>r&&n.length>0?(t.push(n),n=[s],o=i):(n.push(s),o+=i)}return n.length>0&&t.push(n),t},T=(()=>{let e=Date.now()*1e3;return()=>(++e).toString(36).padStart(12,"0")})(),A=(e,r,t)=>e.map(n=>({...n,_id:n._id||T()})).sort((n,o)=>{const s=b(r,t,n._id),i=b(r,t,o._id);return s.localeCompare(i)}),y=(e,r,t)=>{const n=new Date().toISOString();return e.map(o=>{const s=o._id||T(),i={...o,_id:s,_created_at:o._created_at||n,_updated_at:o._updated_at||n},l=JSON.stringify(i);return{key:b(r,t,s),value:l,document_id:s}})},E=async(e,r)=>{const t=[];return await e.transaction(()=>{for(const{key:n,value:o,document_id:s}of r){if(e.get(n))throw new Error(`Document with _id ${s} already exists`);e.put(n,o),t.push(s)}}),t},F=async function*(e,r,t,n=U){const o=f();for(let s=0;s<e.length;s+=n){const i=e.slice(s,s+n),l=y(i,r,t);yield await E(o,l),i.length=0,l.length=0;const c=Math.floor(s/n);e.length>=5e6?(c%5===0&&global.gc&&(global.gc(),await new Promise(a=>setTimeout(a,100))),await new Promise(a=>setImmediate(a))):e.length>=1e6?(c%8===0&&global.gc&&(global.gc(),await new Promise(a=>setTimeout(a,75))),await new Promise(a=>setImmediate(a))):e.length>1e5?(c%25===0&&global.gc&&(global.gc(),await new Promise(a=>setTimeout(a,25))),await new Promise(a=>setImmediate(a))):c%10===0&&await new Promise(a=>setImmediate(a))}},G=()=>!1,Z=e=>{},j=async(e,r)=>{w().debug("Index rebuilding skipped (not implemented)",{database:e,collection:r})},p=async(e,r,t,n={})=>{const{disable_indexing:o=!0,pre_allocate_map_size:s=!0,sort_keys:i=!0,stream_processing:l=!0,batch_size:_=U}=n,c=w(),a=Date.now(),h=process.memoryUsage();if(!e||!r)throw new Error("Database name and collection name are required");if(!Array.isArray(t)||t.length===0)throw new Error("Documents must be a non-empty array");c.info("Starting optimized bulk insert",{database:e,collection:r,document_count:t.length,options:n});let k=!1;try{s&&await J(t),o&&(k=G());let u=t;i&&(u=A(t,e,r));const m=[];let d=0;if(l)for await(const g of F(u,e,r,_))m.push(...g),d+=g.length,d%D===0&&c.info("Bulk insert progress",{processed:d,total:t.length,percentage:Math.round(d/t.length*100)});else{const g=O(u),q=f();for(const L of g){const N=y(L,e,r),S=await E(q,N);m.push(...S),d+=S.length,d%D===0&&c.info("Bulk insert progress",{processed:d,total:t.length,percentage:Math.round(d/t.length*100)})}}o&&await j(e,r);const z=Date.now(),M=process.memoryUsage(),x={duration_ms:z-a,documents_per_second:Math.round(t.length/((z-a)/1e3)),memory_delta_mb:Math.round((M.heapUsed-h.heapUsed)/(1024*1024)),peak_memory_mb:Math.round(M.heapUsed/(1024*1024))};return c.info("Optimized bulk insert completed",{database:e,collection:r,inserted_count:m.length,performance:x}),{acknowledged:!0,inserted_count:m.length,inserted_ids:m,performance:x}}catch(u){throw c.error("Optimized bulk insert failed",{database:e,collection:r,error:u.message}),u}finally{o&&Z(k)}},H=async(e,r,t,n={})=>{const{chunk_size:o=1e4}=n,s={acknowledged:!0,inserted_count:0,inserted_ids:[],performance:{duration_ms:0,documents_per_second:0,memory_delta_mb:0,peak_memory_mb:0}},i=Date.now();for(let _=0;_<t.length;_+=o){const c=t.slice(_,_+o),a=await p(e,r,c,n);s.inserted_count+=a.inserted_count,s.inserted_ids.push(...a.inserted_ids),await new Promise(h=>setImmediate(h))}const l=Date.now();return s.performance.duration_ms=l-i,s.performance.documents_per_second=Math.round(t.length/((l-i)/1e3)),s},V=async(e,r,t,n={})=>{const o=Date.now(),s=process.memoryUsage(),i=await p(e,r,t,n),l=Date.now(),_=process.memoryUsage();return{...i,performance:{...i.performance,total_duration_ms:l-o,memory_usage:{start_heap_mb:Math.round(s.heapUsed/(1024*1024)),end_heap_mb:Math.round(_.heapUsed/(1024*1024)),delta_heap_mb:Math.round((_.heapUsed-s.heapUsed)/(1024*1024)),peak_heap_mb:Math.round(_.heapUsed/(1024*1024))}}}},$=async(e,r,t,n={})=>{const o=B(),s={operation:"bulk_insert_optimized",database:e,collection:r,document_count:t.length};return await o.enqueue_write_operation(()=>p(e,r,t,n),s)};export{$ as bulk_insert,p as bulk_insert_optimized,V as bulk_insert_with_metrics,I as calculate_average_document_size,P as calculate_bulk_map_size,O as create_size_based_batches,H as non_blocking_bulk_insert,y as pre_encode_documents,A as sort_documents_by_key};

package/dist/server/lib/memory_efficient_bulk_insert.js

@@ -0,0 +1 @@
+import{bulk_insert_optimized as z}from"./bulk_insert_optimizer.js";import k from"./logger.js";const{create_context_logger:U}=k("memory_efficient_bulk_insert"),M=async function*(r,n={}){const{batch_size:a=1e3,document_template:c="minimal",test_id:i=Date.now().toString(36)}=n;for(let s=0;s<r;s+=a){const d=Math.min(a,r-s),l=[];for(let _=0;_<d;_++){const e=s+_;let o;c==="minimal"?o={_id:`mem_${i}_${e.toString().padStart(8,"0")}`,idx:e,cat:e%50,val:e%1e3}:c==="medium"?o={_id:`mem_${i}_${e.toString().padStart(8,"0")}`,name:`Document ${e}`,index:e,category:`category_${e%100}`,active:e%2===0,priority:e%5,score:Math.random()*100,created_timestamp:Date.now()+e}:c==="large"&&(o={_id:`mem_${i}_${e.toString().padStart(8,"0")}`,name:`Large Document ${e}`,index:e,category:`category_${e%100}`,subcategory:`subcategory_${e%20}`,active:e%2===0,priority:e%5,score:Math.random()*100,created_timestamp:Date.now()+e,description:`This is a large document with index ${e} for performance testing purposes.`,metadata:{created_by:`user_${e%1e3}`,department:`dept_${e%50}`,project:`project_${e%200}`,tags:[`tag_${e%10}`,`tag_${(e+1)%10}`]},measurements:Array.from({length:5},(f,p)=>({timestamp:Date.now()+e+p,value:Math.random()*1e3}))}),l.push(o)}yield l,await new Promise(_=>setImmediate(_))}},x=async(r,n,a,c={})=>{const{generation_batch_size:i=1e3,insert_batch_size:s=250,document_template:d="minimal",disable_indexing:l=!0,pre_allocate_map_size:_=!0,sort_keys:e=!0}=c,o=U(),f=Date.now(),p=process.memoryUsage();o.info("Starting memory-efficient bulk insert",{database:r,collection:n,document_count:a,generation_batch_size:i,insert_batch_size:s,document_template:d});const h=[];let b=0,m=0;try{for await(const w of M(a,{batch_size:i,document_template:d})){const $=await z(r,n,w,{disable_indexing:l,pre_allocate_map_size:m===0?_:!1,sort_keys:e,stream_processing:!0,batch_size:s});if(h.push(...$.inserted_ids),b+=$.inserted_count,m++,w.length=0,m%10===0){const t=process.memoryUsage();o.info("Memory-efficient bulk insert progress",{processed:b,total:a,percentage:Math.round(b/a*100),current_heap_mb:Math.round(t.heapUsed/(1024*1024)),batches_processed:m})}a>=1e7?(m%20===0&&global.gc&&(global.gc(),await new Promise(t=>setTimeout(t,25))),m%5===0&&await new Promise(t=>setImmediate(t))):a>=5e6?(m%10===0&&global.gc&&(global.gc(),await new Promise(t=>setTimeout(t,50))),m%2===0&&await new Promise(t=>setImmediate(t))):a>=1e6?(m%10===0&&global.gc&&(global.gc(),await new Promise(t=>setTimeout(t,50))),await new Promise(t=>setImmediate(t))):await new Promise(t=>setImmediate(t))}const g=Date.now(),u=process.memoryUsage(),y={duration_ms:g-f,documents_per_second:Math.round(a/((g-f)/1e3)),memory_usage:{start_heap_mb:Math.round(p.heapUsed/(1024*1024)),end_heap_mb:Math.round(u.heapUsed/(1024*1024)),delta_heap_mb:Math.round((u.heapUsed-p.heapUsed)/(1024*1024)),peak_heap_mb:Math.round(u.heapUsed/(1024*1024))}};return o.info("Memory-efficient bulk insert completed",{database:r,collection:n,inserted_count:h.length,performance:y}),{acknowledged:!0,inserted_count:h.length,inserted_ids:h,performance:y}}catch(g){throw o.error("Memory-efficient bulk insert failed",{database:r,collection:n,error:g.message}),g}},D=(r,n="minimal",a=1e3)=>{const c={minimal:50,medium:200,large:500},i=c[n]||c.minimal,s=Math.round(a*i/(1024*1024)),d=Math.round(r*i/(1024*1024)),l=s*3+100;return{avg_document_size_bytes:i,total_data_size_mb:d,batch_memory_mb:s,estimated_peak_memory_mb:l,recommended_batch_size:r>=1e7?2e3:r>=5e6?1e3:r>=1e6?750:1e3}};export{D as estimate_memory_usage,M as generate_documents_streaming,x as memory_efficient_bulk_insert};

package/dist/server/lib/write_queue.js

@@ -1 +1 @@
-import c from"./logger.js";import{get_batched_write_queue as l,shutdown_batched_write_queue as p}from"./batched_write_queue.js";const{create_context_logger:h}=c("write_queue");class d{constructor(){this.queue=[],this.processing=!1,this.shutting_down=!1,this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:0,max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0},this.log=h()}async enqueue_write_operation(t,e={}){if(this.shutting_down)throw new Error("Server shutting down");return new Promise((i,a)=>{if(this.shutting_down){a(new Error("Server shutting down"));return}const s={operation_fn:t,context:e,resolve:i,reject:a,enqueued_at:Date.now(),id:this.generate_operation_id()};this.queue.push(s),this.stats.total_operations++,this.stats.current_queue_depth=this.queue.length,this.stats.current_queue_depth>this.stats.max_queue_depth&&(this.stats.max_queue_depth=this.stats.current_queue_depth),this.log.debug("Write operation enqueued",{operation_id:s.id,queue_depth:this.stats.current_queue_depth,context:e}),this.process_queue()})}async process_queue(){if(!(this.processing||this.queue.length===0||this.shutting_down)){for(this.processing=!0;this.queue.length>0&&!this.shutting_down;){const t=this.queue.shift();this.stats.current_queue_depth=this.queue.length;const e=Date.now()-t.enqueued_at;this.stats.total_wait_time_ms+=e;const i=Date.now();try{this.log.debug("Processing write operation",{operation_id:t.id,wait_time_ms:e,context:t.context});const a=await this.execute_with_retry(t.operation_fn,t.context),s=Date.now()-i;this.stats.total_processing_time_ms+=s,this.stats.completed_operations++,this.log.debug("Write operation completed",{operation_id:t.id,wait_time_ms:e,processing_time_ms:s,context:t.context}),t.resolve(a)}catch(a){const s=Date.now()-i;this.stats.total_processing_time_ms+=s,this.stats.failed_operations++,this.log.error("Write operation failed",{operation_id:t.id,wait_time_ms:e,processing_time_ms:s,error:a.message,context:t.context}),t.reject(a)}}this.processing=!1}}async execute_with_retry(t,e,i=3){let a=null;for(let s=1;s<=i;s++)try{return await t()}catch(_){if(a=_,this.is_retryable_error(_)&&s<i){const u=this.calculate_backoff_delay(s);this.log.warn("Write operation failed, retrying",{attempt:s,max_retries:i,delay_ms:u,error:_.message,context:e}),await this.sleep(u);continue}break}throw a}is_retryable_error(t){return["MDB_MAP_FULL","MDB_TXN_FULL","MDB_READERS_FULL","EAGAIN","EBUSY"].some(i=>t.message.includes(i)||t.code===i)}calculate_backoff_delay(t){const a=100*Math.pow(2,t-1),s=Math.random()*.1*a;return Math.min(a+s,5e3)}sleep(t){return new Promise(e=>setTimeout(e,t))}generate_operation_id(){return`${Date.now()}-${Math.random().toString(36).substr(2,9)}`}get_stats(){const t=this.stats.completed_operations>0?Math.round(this.stats.total_wait_time_ms/this.stats.completed_operations):0,e=this.stats.completed_operations>0?Math.round(this.stats.total_processing_time_ms/this.stats.completed_operations):0;return{...this.stats,avg_wait_time_ms:t,avg_processing_time_ms:e,success_rate:this.stats.total_operations>0?Math.round(this.stats.completed_operations/this.stats.total_operations*100):100}}clear_stats(){this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:this.queue.length,max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0}}async shutdown(){for(this.log.info("Shutting down write queue",{pending_operations:this.queue.length,currently_processing:this.processing}),this.shutting_down=!0;this.processing;)await this.sleep(50);this.queue.forEach(t=>{t.reject(new Error("Server shutting down"))}),this.queue=[],this.processing=!1}}let o=null,n=!0;const q=r=>{if(!o)if(n){const t=l(r);o=new m(t)}else o=new d;return o},y=async()=>{o&&(await o.shutdown(),o=null),n&&await p()},f=r=>{n=r};class m{constructor(t){this.batched_queue=t,this.log=h("write_queue_wrapper")}async enqueue_write_operation(t,e={}){return this.batched_queue.enqueue_write_operation(t,e)}get_stats(){const t=this.batched_queue.get_stats();return{total_operations:t.total_operations,completed_operations:t.completed_operations,failed_operations:t.failed_operations,current_queue_depth:t.current_queue_depth,max_queue_depth:t.max_queue_depth,avg_wait_time_ms:t.avg_wait_time_ms,avg_processing_time_ms:t.avg_processing_time_ms,success_rate:t.success_rate}}clear_stats(){this.batched_queue.clear_stats()}async shutdown(){await this.batched_queue.shutdown()}is_retryable_error(t){return["MDB_MAP_FULL","MDB_TXN_FULL","MDB_READERS_FULL","EAGAIN","EBUSY"].some(i=>t.message.includes(i)||t.code===i)}calculate_backoff_delay(t){const a=100*Math.pow(2,t-1),s=Math.random()*.1*a;return Math.min(a+s,5e3)}generate_operation_id(){return`${Date.now()}-${Math.random().toString(36).substr(2,9)}`}}export{q as get_write_queue,f as set_batched_queue_enabled,y as shutdown_write_queue};
+import _ from"./logger.js";const{create_context_logger:u}=_("write_queue");class h{constructor(){this.queue=[],this.processing=!1,this.shutting_down=!1,this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:0,max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0},this.log=u()}async enqueue_write_operation(t,s={}){if(this.shutting_down)throw new Error("Server shutting down");return new Promise((o,i)=>{if(this.shutting_down){i(new Error("Server shutting down"));return}const e={operation_fn:t,context:s,resolve:o,reject:i,enqueued_at:Date.now(),id:this.generate_operation_id()};this.queue.push(e),this.stats.total_operations++,this.stats.current_queue_depth=this.queue.length,this.stats.current_queue_depth>this.stats.max_queue_depth&&(this.stats.max_queue_depth=this.stats.current_queue_depth),this.log.debug("Write operation enqueued",{operation_id:e.id,queue_depth:this.stats.current_queue_depth,context:s}),this.process_queue()})}async process_queue(){if(!(this.processing||this.queue.length===0||this.shutting_down)){for(this.processing=!0;this.queue.length>0&&!this.shutting_down;){const t=this.queue.shift();this.stats.current_queue_depth=this.queue.length;const s=Date.now()-t.enqueued_at;this.stats.total_wait_time_ms+=s;const o=Date.now();try{this.log.debug("Processing write operation",{operation_id:t.id,wait_time_ms:s,context:t.context});const i=await this.execute_with_retry(t.operation_fn,t.context),e=Date.now()-o;this.stats.total_processing_time_ms+=e,this.stats.completed_operations++,this.log.debug("Write operation completed",{operation_id:t.id,wait_time_ms:s,processing_time_ms:e,context:t.context}),t.resolve(i)}catch(i){const e=Date.now()-o;this.stats.total_processing_time_ms+=e,this.stats.failed_operations++,this.log.error("Write operation failed",{operation_id:t.id,wait_time_ms:s,processing_time_ms:e,error:i.message,context:t.context}),t.reject(i)}}this.processing=!1}}async execute_with_retry(t,s,o=3){let i=null;for(let e=1;e<=o;e++)try{return await t()}catch(n){if(i=n,this.is_retryable_error(n)&&e<o){const a=this.calculate_backoff_delay(e);this.log.warn("Write operation failed, retrying",{attempt:e,max_retries:o,delay_ms:a,error:n.message,context:s}),await this.sleep(a);continue}break}throw i}is_retryable_error(t){return["MDB_MAP_FULL","MDB_TXN_FULL","MDB_READERS_FULL","EAGAIN","EBUSY"].some(o=>t.message.includes(o)||t.code===o)}calculate_backoff_delay(t){const i=100*Math.pow(2,t-1),e=Math.random()*.1*i;return Math.min(i+e,5e3)}sleep(t){return new Promise(s=>setTimeout(s,t))}generate_operation_id(){return`${Date.now()}-${Math.random().toString(36).substr(2,9)}`}get_stats(){const t=this.stats.completed_operations>0?Math.round(this.stats.total_wait_time_ms/this.stats.completed_operations):0,s=this.stats.completed_operations>0?Math.round(this.stats.total_processing_time_ms/this.stats.completed_operations):0;return{...this.stats,avg_wait_time_ms:t,avg_processing_time_ms:s,success_rate:this.stats.total_operations>0?Math.round(this.stats.completed_operations/this.stats.total_operations*100):100}}clear_stats(){this.stats={total_operations:0,completed_operations:0,failed_operations:0,current_queue_depth:this.queue.length,max_queue_depth:0,total_wait_time_ms:0,total_processing_time_ms:0}}async shutdown(){for(this.log.info("Shutting down write queue",{pending_operations:this.queue.length,currently_processing:this.processing}),this.shutting_down=!0;this.processing;)await this.sleep(50);this.queue.forEach(t=>{t.reject(new Error("Server shutting down"))}),this.queue=[],this.processing=!1}}let r=null;const p=()=>(r||(r=new h),r),d=async()=>{r&&(await r.shutdown(),r=null)};export{p as get_write_queue,d as shutdown_write_queue};

package/package.json

@@ -1,16 +1,21 @@
 {
   "name": "@joystick.js/db-canary",
   "type": "module",
-  "version": "0.0.0-canary.2270",
-  "canary_version": "0.0.0-canary.2269",
+  "version": "0.0.0-canary.2272",
+  "canary_version": "0.0.0-canary.2271",
   "description": "JoystickDB - A minimalist database server for the Joystick framework",
   "main": "./dist/server/index.js",
   "scripts": {
     "build": "node ./.build/index.js",
     "release": "node increment_version.js && npm run build && npm publish",
     "start": "node src/server/index.js",
-    "test": "NODE_ENV=test ava --serial",
+    "test": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=8192' ava --serial --timeout=10m",
     "test:watch": "NODE_ENV=test ava --watch",
+    "test:performance": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/*.test.js",
+    "test:enterprise": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/bulk_insert_enterprise_*.test.js",
+    "test:benchmarks": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=16384' ava --serial --timeout=30m tests/performance/bulk_insert_benchmarks.test.js",
+    "test:1m": "NODE_ENV=test NODE_OPTIONS='--expose-gc --max-old-space-size=8192' ava --serial --timeout=15m tests/performance/bulk_insert_1m_test.js",
+    "test:runner": "node test_runner.js",
     "build:types": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types src/**/*.js",
     "build:types:client": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types/client src/client/*.js",
     "build:types:server": "tsc --declaration --emitDeclarationOnly --allowJs --outDir types/server src/server/**/*.js"
@@ -33,7 +38,8 @@
   },
   "ava": {
     "files": [
-      "tests/**/*.test.js"
+      "tests/**/*.test.js",
+      "!tests/performance/*.test.js"
     ],
     "verbose": true
   },

package/src/server/lib/auto_index_manager.js

@@ -34,6 +34,10 @@ const initialize_auto_index_database = () => {
     const main_db = get_database();
     auto_index_db = main_db.openDB('auto_indexes', { create: true });
     
+    // Clear in-memory data first to ensure clean state
+    query_stats.clear();
+    auto_index_metadata.clear();
+    
     load_auto_index_metadata();
     load_query_stats();
     start_evaluation_timer();
@@ -713,18 +717,21 @@ const remove_automatic_indexes = async (collection_name, field_names = null) =>
 
 const cleanup_auto_index_database = () => {
   stop_evaluation_timer();
+  
+  // Clear in-memory data first
+  query_stats.clear();
+  auto_index_metadata.clear();
+  
   if (auto_index_db) {
     try {
-      // Clear database entries instead of saving them
+      // Clear database entries to prevent data persistence between tests
       auto_index_db.remove('query_stats');
       auto_index_db.remove('auto_index_metadata');
     } catch (error) {
       // Ignore errors during cleanup
     }
+    auto_index_db = null;
   }
-  query_stats.clear();
-  auto_index_metadata.clear();
-  auto_index_db = null;
 };
 
 export {