@joystick.js/db-canary 0.0.0-canary.2266 → 0.0.0-canary.2268
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import n from"cluster";import h from"os";import{EventEmitter as d}from"events";import{writeFileSync as
|
|
1
|
+
import n from"cluster";import h from"os";import{EventEmitter as d}from"events";import{writeFileSync as u}from"fs";import{load_settings as c,get_settings as p,get_port_configuration as l}from"../lib/load_settings.js";import{restore_backup as w,start_backup_schedule as g,stop_backup_schedule as f}from"../lib/backup_manager.js";import m from"../lib/logger.js";import{initialize_database as k,check_and_grow_map_size as y,cleanup_database as b}from"../lib/query_engine.js";import{setup_authentication as v,verify_password as q,initialize_auth_manager as D}from"../lib/auth_manager.js";import x from"../lib/operations/insert_one.js";import E from"../lib/operations/update_one.js";import T from"../lib/operations/delete_one.js";import S from"../lib/operations/delete_many.js";import z from"../lib/operations/bulk_write.js";import W from"../lib/operations/find_one.js";import F from"../lib/operations/find.js";import N from"../lib/operations/count_documents.js";import P from"../lib/operations/create_index.js";import O from"../lib/operations/drop_index.js";import R from"../lib/operations/get_indexes.js";import{start_http_server as A,stop_http_server as I,is_setup_required as j}from"../lib/http_server.js";import{is_development_mode as L,display_development_startup_message as M}from"../lib/development_mode.js";import{initialize_api_key_manager as $}from"../lib/api_key_manager.js";class H extends d{constructor(e={}){super(),this.workers=new Map,this.write_queue=[],this.processing_writes=!1,this.authenticated_sessions=new Map,this.worker_count=e.worker_count||h.cpus().length,this.port=e.port||1983,this.settings_file=e.settings_file||"settings.db.json",this.settings=null,this.pending_writes=new Map,this.write_id_counter=0,this.shutting_down=!1,this.master_id=`master_${Date.now()}_${Math.random()}`;const{create_context_logger:t}=m("master");this.log=t({port:this.port,worker_count:this.worker_count,master_id:this.master_id}),this.setup_master()}setup_master(){n.setupPrimary({exec:new URL("./index.js",import.meta.url).pathname,args:[],silent:!1}),n.on("exit",(e,t,s)=>{this.log.warn("Worker died",{worker_pid:e.process.pid,exit_code:t,signal:s}),this.handle_worker_death(e)}),n.on("message",(e,t)=>{this.handle_worker_message(e,t)})}get_database_path(){let e="./data";try{const t=p();t?.data_path&&(e=t.data_path)}catch{}return e}async initialize_core_systems(){const e=this.get_database_path();k(e),D(),await $(),this.log.info("Database and auth manager initialized")}async handle_startup_restore(){if(this.settings?.restore_from)try{this.log.info("Startup restore requested",{backup_filename:this.settings.restore_from});const e=await w(this.settings.restore_from);this.log.info("Startup restore completed",{backup_filename:this.settings.restore_from,duration_ms:e.duration_ms}),this.remove_restore_from_settings()}catch(e){this.log.error("Startup restore failed",{backup_filename:this.settings.restore_from,error:e.message}),this.log.info("Continuing with existing database after restore failure")}}remove_restore_from_settings(){const e={...this.settings};delete e.restore_from,u(this.settings_file,JSON.stringify(e,null,2)),this.settings=c(this.settings_file),this.log.info("Removed restore_from from settings after successful restore")}start_backup_scheduling(){if(this.settings?.s3)try{g(),this.log.info("Backup scheduling started")}catch(e){this.log.warn("Failed to start backup scheduling",{error:e.message})}}async start_setup_server(){if(j())try{const{http_port:e}=l();await A(e)&&this.log.info("HTTP setup server started",{http_port:e})}catch(e){this.log.warn("Failed to start HTTP setup server",{error:e.message})}}spawn_all_workers(){for(let e=0;e<this.worker_count;e++)this.spawn_worker()}display_development_message(){if(L()){const{tcp_port:e,http_port:t}=l();M(e,t)}}async start(){const e=Date.now();try{this.settings=c(this.settings_file),this.log.info("Settings loaded successfully",{settings_file:this.settings_file}),await this.initialize_core_systems(),await this.handle_startup_restore(),this.start_backup_scheduling(),await this.start_setup_server(),this.spawn_all_workers(),this.display_development_message();const t=Date.now()-e;this.log.info("Master process started successfully",{workers_spawned:this.worker_count,startup_duration_ms:t})}catch(t){this.log.error("Failed to start master process",{error:t.message}),process.exit(1)}}spawn_worker(){const e=Date.now();this.log.info("Spawning worker");const t=n.fork({WORKER_PORT:this.port,WORKER_SETTINGS:JSON.stringify(this.settings)});this.workers.set(t.id,{worker:t,connections:0,last_heartbeat:Date.now(),status:"starting"});const s=Date.now()-e;return this.log.info("Worker spawned successfully",{worker_id:t.id,worker_pid:t.process.pid,spawn_duration_ms:s}),t}handle_worker_death(e){this.workers.delete(e.id),this.shutting_down||(this.log.info("Respawning worker after death",{dead_worker_id:e.id,respawn_delay_ms:1e3}),setTimeout(()=>{this.spawn_worker()},1e3))}handle_worker_message(e,t){switch(t.type){case"worker_ready":this.handle_worker_ready_for_config(e,t);break;case"server_ready":this.handle_worker_server_ready(e,t);break;case"write_request":this.handle_write_request(e,t);break;case"auth_request":this.handle_auth_request(e,t);break;case"setup_request":this.handle_setup_request(e,t);break;case"connection_count":this.update_worker_connections(e,t);break;case"heartbeat":this.handle_worker_heartbeat(e,t);break;default:this.log.warn("Unknown message type received",{message_type:t.type,worker_id:e.id})}}handle_worker_ready_for_config(e,t){this.log.info("Worker ready for config, sending configuration",{worker_id:e.id,worker_pid:e.process.pid,master_id:this.master_id}),e.send({type:"config",data:{port:this.port,settings:this.settings,master_id:this.master_id}})}handle_worker_server_ready(e,t){const s=this.workers.get(e.id);s&&(s.status="ready",this.log.info("Worker server ready",{worker_id:e.id,worker_pid:e.process.pid}))}async handle_write_request(e,t){if(this.shutting_down){e.send({type:"write_response",data:{write_id:t.data.write_id,success:!1,error:"Server is shutting down"}});return}const{write_id:s,op_type:r,data:i,socket_id:o}=t.data;try{const a={write_id:s,worker_id:e.id,op_type:r,data:i,socket_id:o,timestamp:Date.now()};this.write_queue.push(a),this.process_write_queue()}catch(a){e.send({type:"write_response",data:{write_id:s,success:!1,error:a.message}})}}async process_write_queue(){if(!(this.processing_writes||this.write_queue.length===0)){for(this.processing_writes=!0;this.write_queue.length>0;){const e=this.write_queue.shift();await this.execute_write_operation(e)}this.processing_writes=!1,this.shutting_down&&this.write_queue.length===0&&this.emit("writes_completed")}}async execute_write_operation(e){const{write_id:t,worker_id:s,op_type:r,data:i,socket_id:o}=e,a=this.workers.get(s);if(!a){this.log.error("Worker not found for write operation",{worker_id:s});return}try{const _=await this.perform_database_operation(r,i);a.worker.send({type:"write_response",data:{write_id:t,success:!0,result:_}}),this.broadcast_write_notification(r,i,s)}catch(_){this.log.error("Write operation failed",{write_id:t,op_type:r,worker_id:s,error_message:_.message}),a.worker.send({type:"write_response",data:{write_id:t,success:!1,error:_.message}})}}async perform_database_operation(e,t){const s=Date.now();this.log.info("Executing database operation",{op_type:e});try{let r;const i=t.database||"default";switch(e){case"insert_one":r=await x(i,t.collection,t.document,t.options);break;case"update_one":r=await E(i,t.collection,t.filter,t.update,t.options);break;case"delete_one":r=await T(i,t.collection,t.filter,t.options);break;case"delete_many":r=await S(i,t.collection,t.filter,t.options);break;case"bulk_write":r=await z(i,t.collection,t.operations,t.options);break;case"find_one":r=await W(i,t.collection,t.filter,t.options);break;case"find":r=await F(i,t.collection,t.filter,t.options);break;case"count_documents":r=await N(i,t.collection,t.filter,t.options);break;case"create_index":r=await P(i,t.collection,t.field,t.options);break;case"drop_index":r=await O(i,t.collection,t.field);break;case"get_indexes":r=await R(i,t.collection);break;default:throw new Error(`Unsupported database operation: ${e}`)}const o=Date.now()-s;return this.log.log_operation(e,o,{result:r}),["find_one","find","count_documents","get_indexes"].includes(e)||setImmediate(()=>y()),r}catch(r){const i=Date.now()-s;throw this.log.error("Database operation failed",{op_type:e,duration_ms:i,error_message:r.message}),r}}broadcast_write_notification(e,t,s){const r={type:"write_notification",data:{op_type:e,data:t,timestamp:Date.now()}};for(const[i,o]of this.workers)i!==s&&o.status==="ready"&&o.worker.send(r)}async handle_auth_request(e,t){const{auth_id:s,socket_id:r,password:i}=t.data;try{const o=await q(i,"cluster_client");o&&this.authenticated_sessions.set(r,{authenticated_at:Date.now(),worker_id:e.id}),e.send({type:"auth_response",data:{auth_id:s,success:o,message:o?"Authentication successful":"Authentication failed"}})}catch(o){e.send({type:"auth_response",data:{auth_id:s,success:!1,message:`Authentication error: ${o.message}`}})}}handle_setup_request(e,t){const{setup_id:s,socket_id:r}=t.data;try{const i=v(),o=`===
|
|
2
2
|
JoystickDB Setup
|
|
3
3
|
|
|
4
4
|
Your database has been setup. Follow the instructions below carefully to avoid issues.
|
|
@@ -17,4 +17,4 @@ const client = joystickdb.client({
|
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
await client.ping();
|
|
20
|
-
===`;e.send({type:"setup_response",data:{setup_id:s,success:!0,password:i,instructions:o,message:"Authentication setup completed successfully"}})}catch(i){e.send({type:"setup_response",data:{setup_id:s,success:!1,error:i.message}})}}update_worker_connections(e,t){const s=this.workers.get(e.id);s&&(s.connections=t.data.count)}handle_worker_heartbeat(e,t){const s=this.workers.get(e.id);s&&(s.last_heartbeat=Date.now())}get_cluster_stats(){const e={master_pid:process.pid,worker_count:this.workers.size,total_connections:0,write_queue_length:this.write_queue.length,authenticated_sessions:this.authenticated_sessions.size,workers:[]};for(const[t,s]of this.workers)e.total_connections+=s.connections,e.workers.push({id:t,pid:s.worker.process.pid,connections:s.connections,status:s.status,last_heartbeat:s.last_heartbeat});return e}async stop_http_server_gracefully(){try{await
|
|
20
|
+
===`;e.send({type:"setup_response",data:{setup_id:s,success:!0,password:i,instructions:o,message:"Authentication setup completed successfully"}})}catch(i){e.send({type:"setup_response",data:{setup_id:s,success:!1,error:i.message}})}}update_worker_connections(e,t){const s=this.workers.get(e.id);s&&(s.connections=t.data.count)}handle_worker_heartbeat(e,t){const s=this.workers.get(e.id);s&&(s.last_heartbeat=Date.now())}get_cluster_stats(){const e={master_pid:process.pid,worker_count:this.workers.size,total_connections:0,write_queue_length:this.write_queue.length,authenticated_sessions:this.authenticated_sessions.size,workers:[]};for(const[t,s]of this.workers)e.total_connections+=s.connections,e.workers.push({id:t,pid:s.worker.process.pid,connections:s.connections,status:s.status,last_heartbeat:s.last_heartbeat});return e}async stop_http_server_gracefully(){try{await I(),this.log.info("HTTP server stopped")}catch(e){this.log.warn("Failed to stop HTTP server",{error:e.message})}}stop_backup_scheduling_gracefully(){try{f(),this.log.info("Backup scheduling stopped")}catch(e){this.log.warn("Failed to stop backup scheduling",{error:e.message})}}send_shutdown_signals(){for(const[e,t]of this.workers)try{t.worker.send({type:"shutdown"})}catch(s){this.log.warn("Error sending shutdown signal to worker",{worker_id:e,error:s.message})}}async wait_for_pending_writes(){this.write_queue.length!==0&&(this.log.info("Waiting for pending writes to complete",{pending_writes:this.write_queue.length}),await new Promise(e=>{const t=setTimeout(()=>{this.log.warn("Timeout waiting for writes to complete, proceeding with shutdown"),e()},process.env.NODE_ENV==="test"?1e3:5e3);this.once("writes_completed",()=>{clearTimeout(t),e()})}))}disconnect_all_workers(){for(const[e,t]of this.workers)try{t.worker.disconnect()}catch(s){this.log.warn("Error disconnecting worker",{worker_id:e,error:s.message})}}force_kill_remaining_workers(){for(const[e,t]of this.workers){this.log.warn("Force killing worker after timeout",{worker_id:e});try{t.worker.kill("SIGKILL")}catch(s){this.log.warn("Error force killing worker",{worker_id:e,error:s.message})}}this.workers.clear()}async wait_for_workers_to_exit(){const e=process.env.NODE_ENV==="test"?500:3e3;await new Promise(t=>{const s=setTimeout(()=>{this.force_kill_remaining_workers(),t()},e),r=()=>{this.workers.size===0?(clearTimeout(s),t()):setTimeout(r,50)};r()})}cleanup_database_connections(){try{b(),this.log.info("Database cleanup completed")}catch(e){this.log.warn("Error during database cleanup",{error:e.message})}}clear_internal_state(){this.authenticated_sessions.clear(),this.write_queue.length=0,this.pending_writes.clear()}perform_test_environment_cleanup(){if(process.env.NODE_ENV==="test")try{for(const e in n.workers){const t=n.workers[e];if(t&&!t.isDead()){this.log.info("Force killing remaining cluster worker",{worker_id:e,worker_pid:t.process.pid});try{t.kill("SIGKILL")}catch(s){this.log.warn("Error force killing remaining worker",{worker_id:e,error:s.message})}}}for(const e in n.workers)delete n.workers[e];n.removeAllListeners(),this.log.info("Aggressive cluster cleanup completed for test environment")}catch(e){this.log.warn("Error during aggressive cluster cleanup",{error:e.message})}}async shutdown(){const e=Date.now();this.log.info("Initiating graceful shutdown"),this.shutting_down=!0,await this.stop_http_server_gracefully(),this.stop_backup_scheduling_gracefully(),this.send_shutdown_signals(),await this.wait_for_pending_writes(),this.log.info("All writes completed, disconnecting workers"),this.disconnect_all_workers(),await this.wait_for_workers_to_exit(),this.cleanup_database_connections(),this.clear_internal_state(),this.perform_test_environment_cleanup();const t=Date.now()-e;this.log.info("Shutdown complete",{shutdown_duration_ms:t})}}var pe=H;export{pe as default};
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@joystick.js/db-canary",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.0.0-canary.
|
|
5
|
-
"canary_version": "0.0.0-canary.
|
|
4
|
+
"version": "0.0.0-canary.2268",
|
|
5
|
+
"canary_version": "0.0.0-canary.2267",
|
|
6
6
|
"description": "JoystickDB - A minimalist database server for the Joystick framework",
|
|
7
7
|
"main": "./dist/server/index.js",
|
|
8
8
|
"scripts": {
|
|
@@ -22,6 +22,7 @@ import delete_many from '../lib/operations/delete_many.js';
|
|
|
22
22
|
import bulk_write from '../lib/operations/bulk_write.js';
|
|
23
23
|
import find_one from '../lib/operations/find_one.js';
|
|
24
24
|
import find from '../lib/operations/find.js';
|
|
25
|
+
import count_documents from '../lib/operations/count_documents.js';
|
|
25
26
|
import create_index_operation from '../lib/operations/create_index.js';
|
|
26
27
|
import drop_index_operation from '../lib/operations/drop_index.js';
|
|
27
28
|
import get_indexes_operation from '../lib/operations/get_indexes.js';
|
|
@@ -528,6 +529,10 @@ class ClusterMaster extends EventEmitter {
|
|
|
528
529
|
result = await find(database_name, data.collection, data.filter, data.options);
|
|
529
530
|
break;
|
|
530
531
|
|
|
532
|
+
case 'count_documents':
|
|
533
|
+
result = await count_documents(database_name, data.collection, data.filter, data.options);
|
|
534
|
+
break;
|
|
535
|
+
|
|
531
536
|
case 'create_index':
|
|
532
537
|
result = await create_index_operation(database_name, data.collection, data.field, data.options);
|
|
533
538
|
break;
|
|
@@ -548,7 +553,7 @@ class ClusterMaster extends EventEmitter {
|
|
|
548
553
|
this.log.log_operation(op_type, duration_ms, { result });
|
|
549
554
|
|
|
550
555
|
// NOTE: Trigger map size check after write operations (not read operations).
|
|
551
|
-
const read_operations = ['find_one', 'find', 'get_indexes'];
|
|
556
|
+
const read_operations = ['find_one', 'find', 'count_documents', 'get_indexes'];
|
|
552
557
|
if (!read_operations.includes(op_type)) {
|
|
553
558
|
setImmediate(() => check_and_grow_map_size());
|
|
554
559
|
}
|