@joystick.js/db-canary 0.0.0-canary.2253 → 0.0.0-canary.2255
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import n from"cluster";import
|
|
1
|
+
import n from"cluster";import h from"os";import{EventEmitter as d}from"events";import{writeFileSync as p}from"fs";import{load_settings as c,get_settings as u,get_port_configuration as l}from"../lib/load_settings.js";import{restore_backup as w,start_backup_schedule as g,stop_backup_schedule as f}from"../lib/backup_manager.js";import m from"../lib/logger.js";import{initialize_database as k,check_and_grow_map_size as y,cleanup_database as b}from"../lib/query_engine.js";import{setup_authentication as v,verify_password as q,initialize_auth_manager as D}from"../lib/auth_manager.js";import x from"../lib/operations/insert_one.js";import E from"../lib/operations/update_one.js";import T from"../lib/operations/delete_one.js";import S from"../lib/operations/delete_many.js";import z from"../lib/operations/bulk_write.js";import W from"../lib/operations/find_one.js";import F from"../lib/operations/find.js";import N from"../lib/operations/create_index.js";import P from"../lib/operations/drop_index.js";import O from"../lib/operations/get_indexes.js";import{start_http_server as R,stop_http_server as A,is_setup_required as I}from"../lib/http_server.js";import{is_development_mode as j,display_development_startup_message as L}from"../lib/development_mode.js";import{initialize_api_key_manager as M}from"../lib/api_key_manager.js";class $ extends d{constructor(e={}){super(),this.workers=new Map,this.write_queue=[],this.processing_writes=!1,this.authenticated_sessions=new Map,this.worker_count=e.worker_count||h.cpus().length,this.port=e.port||1983,this.settings_file=e.settings_file||"settings.db.json",this.settings=null,this.pending_writes=new Map,this.write_id_counter=0,this.shutting_down=!1,this.master_id=`master_${Date.now()}_${Math.random()}`;const{create_context_logger:t}=m("master");this.log=t({port:this.port,worker_count:this.worker_count,master_id:this.master_id}),this.setup_master()}setup_master(){n.setupPrimary({exec:new URL("./index.js",import.meta.url).pathname,args:[],silent:!1}),n.on("exit",(e,t,s)=>{this.log.warn("Worker died",{worker_pid:e.process.pid,exit_code:t,signal:s}),this.handle_worker_death(e)}),n.on("message",(e,t)=>{this.handle_worker_message(e,t)})}get_database_path(){let e="./data";try{const t=u();t?.data_path&&(e=t.data_path)}catch{}return e}async initialize_core_systems(){const e=this.get_database_path();k(e),D(),await M(),this.log.info("Database and auth manager initialized")}async handle_startup_restore(){if(this.settings?.restore_from)try{this.log.info("Startup restore requested",{backup_filename:this.settings.restore_from});const e=await w(this.settings.restore_from);this.log.info("Startup restore completed",{backup_filename:this.settings.restore_from,duration_ms:e.duration_ms}),this.remove_restore_from_settings()}catch(e){this.log.error("Startup restore failed",{backup_filename:this.settings.restore_from,error:e.message}),this.log.info("Continuing with existing database after restore failure")}}remove_restore_from_settings(){const e={...this.settings};delete e.restore_from,p(this.settings_file,JSON.stringify(e,null,2)),this.settings=c(this.settings_file),this.log.info("Removed restore_from from settings after successful restore")}start_backup_scheduling(){if(this.settings?.s3)try{g(),this.log.info("Backup scheduling started")}catch(e){this.log.warn("Failed to start backup scheduling",{error:e.message})}}async start_setup_server(){if(I())try{const{http_port:e}=l();await R(e)&&this.log.info("HTTP setup server started",{http_port:e})}catch(e){this.log.warn("Failed to start HTTP setup server",{error:e.message})}}spawn_all_workers(){for(let e=0;e<this.worker_count;e++)this.spawn_worker()}display_development_message(){if(j()){const{tcp_port:e,http_port:t}=l();L(e,t)}}async start(){const e=Date.now();try{this.settings=c(this.settings_file),this.log.info("Settings loaded successfully",{settings_file:this.settings_file}),await this.initialize_core_systems(),await this.handle_startup_restore(),this.start_backup_scheduling(),await this.start_setup_server(),this.spawn_all_workers(),this.display_development_message();const t=Date.now()-e;this.log.info("Master process started successfully",{workers_spawned:this.worker_count,startup_duration_ms:t})}catch(t){this.log.error("Failed to start master process",{error:t.message}),process.exit(1)}}spawn_worker(){const e=Date.now();this.log.info("Spawning worker");const t=n.fork({WORKER_PORT:this.port,WORKER_SETTINGS:JSON.stringify(this.settings)});this.workers.set(t.id,{worker:t,connections:0,last_heartbeat:Date.now(),status:"starting"});const s=Date.now()-e;return this.log.info("Worker spawned successfully",{worker_id:t.id,worker_pid:t.process.pid,spawn_duration_ms:s}),t}handle_worker_death(e){this.workers.delete(e.id),this.shutting_down||(this.log.info("Respawning worker after death",{dead_worker_id:e.id,respawn_delay_ms:1e3}),setTimeout(()=>{this.spawn_worker()},1e3))}handle_worker_message(e,t){switch(t.type){case"worker_ready":this.handle_worker_ready_for_config(e,t);break;case"server_ready":this.handle_worker_server_ready(e,t);break;case"write_request":this.handle_write_request(e,t);break;case"auth_request":this.handle_auth_request(e,t);break;case"setup_request":this.handle_setup_request(e,t);break;case"connection_count":this.update_worker_connections(e,t);break;case"heartbeat":this.handle_worker_heartbeat(e,t);break;default:this.log.warn("Unknown message type received",{message_type:t.type,worker_id:e.id})}}handle_worker_ready_for_config(e,t){this.log.info("Worker ready for config, sending configuration",{worker_id:e.id,worker_pid:e.process.pid,master_id:this.master_id}),e.send({type:"config",data:{port:this.port,settings:this.settings,master_id:this.master_id}})}handle_worker_server_ready(e,t){const s=this.workers.get(e.id);s&&(s.status="ready",this.log.info("Worker server ready",{worker_id:e.id,worker_pid:e.process.pid}))}async handle_write_request(e,t){if(this.shutting_down){e.send({type:"write_response",data:{write_id:t.data.write_id,success:!1,error:"Server is shutting down"}});return}const{write_id:s,op_type:r,data:i,socket_id:o}=t.data;try{const a={write_id:s,worker_id:e.id,op_type:r,data:i,socket_id:o,timestamp:Date.now()};this.write_queue.push(a),this.process_write_queue()}catch(a){e.send({type:"write_response",data:{write_id:s,success:!1,error:a.message}})}}async process_write_queue(){if(!(this.processing_writes||this.write_queue.length===0)){for(this.processing_writes=!0;this.write_queue.length>0;){const e=this.write_queue.shift();await this.execute_write_operation(e)}this.processing_writes=!1,this.shutting_down&&this.write_queue.length===0&&this.emit("writes_completed")}}async execute_write_operation(e){const{write_id:t,worker_id:s,op_type:r,data:i,socket_id:o}=e,a=this.workers.get(s);if(!a){this.log.error("Worker not found for write operation",{worker_id:s});return}try{const _=await this.perform_database_operation(r,i);a.worker.send({type:"write_response",data:{write_id:t,success:!0,result:_}}),this.broadcast_write_notification(r,i,s)}catch(_){this.log.error("Write operation failed",{write_id:t,op_type:r,worker_id:s,error_message:_.message}),a.worker.send({type:"write_response",data:{write_id:t,success:!1,error:_.message}})}}async perform_database_operation(e,t){const s=Date.now();this.log.info("Executing database operation",{op_type:e});try{let r;const i=t.database||"default";switch(e){case"insert_one":r=await x(i,t.collection,t.document,t.options);break;case"update_one":r=await E(i,t.collection,t.filter,t.update,t.options);break;case"delete_one":r=await T(i,t.collection,t.filter,t.options);break;case"delete_many":r=await S(i,t.collection,t.filter,t.options);break;case"bulk_write":r=await z(i,t.collection,t.operations,t.options);break;case"find_one":r=await W(i,t.collection,t.filter,t.options);break;case"find":r=await F(i,t.collection,t.filter,t.options);break;case"create_index":r=await N(i,t.collection,t.field,t.options);break;case"drop_index":r=await P(i,t.collection,t.field);break;case"get_indexes":r=await O(i,t.collection);break;default:throw new Error(`Unsupported database operation: ${e}`)}const o=Date.now()-s;return this.log.log_operation(e,o,{result:r}),["find_one","find","get_indexes"].includes(e)||setImmediate(()=>y()),r}catch(r){const i=Date.now()-s;throw this.log.error("Database operation failed",{op_type:e,duration_ms:i,error_message:r.message}),r}}broadcast_write_notification(e,t,s){const r={type:"write_notification",data:{op_type:e,data:t,timestamp:Date.now()}};for(const[i,o]of this.workers)i!==s&&o.status==="ready"&&o.worker.send(r)}async handle_auth_request(e,t){const{auth_id:s,socket_id:r,password:i}=t.data;try{const o=await q(i,"cluster_client");o&&this.authenticated_sessions.set(r,{authenticated_at:Date.now(),worker_id:e.id}),e.send({type:"auth_response",data:{auth_id:s,success:o,message:o?"Authentication successful":"Authentication failed"}})}catch(o){e.send({type:"auth_response",data:{auth_id:s,success:!1,message:`Authentication error: ${o.message}`}})}}handle_setup_request(e,t){const{setup_id:s,socket_id:r}=t.data;try{const i=v(),o=`===
|
|
2
2
|
JoystickDB Setup
|
|
3
3
|
|
|
4
4
|
Your database has been setup. Follow the instructions below carefully to avoid issues.
|
|
@@ -17,4 +17,4 @@ const client = joystickdb.client({
|
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
await client.ping();
|
|
20
|
-
===`;e.send({type:"setup_response",data:{setup_id:s,success:!0,password:i,instructions:o,message:"Authentication setup completed successfully"}})}catch(i){e.send({type:"setup_response",data:{setup_id:s,success:!1,error:i.message}})}}update_worker_connections(e,t){const s=this.workers.get(e.id);s&&(s.connections=t.data.count)}handle_worker_heartbeat(e,t){const s=this.workers.get(e.id);s&&(s.last_heartbeat=Date.now())}get_cluster_stats(){const e={master_pid:process.pid,worker_count:this.workers.size,total_connections:0,write_queue_length:this.write_queue.length,authenticated_sessions:this.authenticated_sessions.size,workers:[]};for(const[t,s]of this.workers)e.total_connections+=s.connections,e.workers.push({id:t,pid:s.worker.process.pid,connections:s.connections,status:s.status,last_heartbeat:s.last_heartbeat});return e}async stop_http_server_gracefully(){try{await
|
|
20
|
+
===`;e.send({type:"setup_response",data:{setup_id:s,success:!0,password:i,instructions:o,message:"Authentication setup completed successfully"}})}catch(i){e.send({type:"setup_response",data:{setup_id:s,success:!1,error:i.message}})}}update_worker_connections(e,t){const s=this.workers.get(e.id);s&&(s.connections=t.data.count)}handle_worker_heartbeat(e,t){const s=this.workers.get(e.id);s&&(s.last_heartbeat=Date.now())}get_cluster_stats(){const e={master_pid:process.pid,worker_count:this.workers.size,total_connections:0,write_queue_length:this.write_queue.length,authenticated_sessions:this.authenticated_sessions.size,workers:[]};for(const[t,s]of this.workers)e.total_connections+=s.connections,e.workers.push({id:t,pid:s.worker.process.pid,connections:s.connections,status:s.status,last_heartbeat:s.last_heartbeat});return e}async stop_http_server_gracefully(){try{await A(),this.log.info("HTTP server stopped")}catch(e){this.log.warn("Failed to stop HTTP server",{error:e.message})}}stop_backup_scheduling_gracefully(){try{f(),this.log.info("Backup scheduling stopped")}catch(e){this.log.warn("Failed to stop backup scheduling",{error:e.message})}}send_shutdown_signals(){for(const[e,t]of this.workers)try{t.worker.send({type:"shutdown"})}catch(s){this.log.warn("Error sending shutdown signal to worker",{worker_id:e,error:s.message})}}async wait_for_pending_writes(){this.write_queue.length!==0&&(this.log.info("Waiting for pending writes to complete",{pending_writes:this.write_queue.length}),await new Promise(e=>{const t=setTimeout(()=>{this.log.warn("Timeout waiting for writes to complete, proceeding with shutdown"),e()},process.env.NODE_ENV==="test"?1e3:5e3);this.once("writes_completed",()=>{clearTimeout(t),e()})}))}disconnect_all_workers(){for(const[e,t]of this.workers)try{t.worker.disconnect()}catch(s){this.log.warn("Error disconnecting worker",{worker_id:e,error:s.message})}}force_kill_remaining_workers(){for(const[e,t]of this.workers){this.log.warn("Force killing worker after timeout",{worker_id:e});try{t.worker.kill("SIGKILL")}catch(s){this.log.warn("Error force killing worker",{worker_id:e,error:s.message})}}this.workers.clear()}async wait_for_workers_to_exit(){const e=process.env.NODE_ENV==="test"?500:3e3;await new Promise(t=>{const s=setTimeout(()=>{this.force_kill_remaining_workers(),t()},e),r=()=>{this.workers.size===0?(clearTimeout(s),t()):setTimeout(r,50)};r()})}cleanup_database_connections(){try{b(),this.log.info("Database cleanup completed")}catch(e){this.log.warn("Error during database cleanup",{error:e.message})}}clear_internal_state(){this.authenticated_sessions.clear(),this.write_queue.length=0,this.pending_writes.clear()}perform_test_environment_cleanup(){if(process.env.NODE_ENV==="test")try{for(const e in n.workers){const t=n.workers[e];if(t&&!t.isDead()){this.log.info("Force killing remaining cluster worker",{worker_id:e,worker_pid:t.process.pid});try{t.kill("SIGKILL")}catch(s){this.log.warn("Error force killing remaining worker",{worker_id:e,error:s.message})}}}for(const e in n.workers)delete n.workers[e];n.removeAllListeners(),this.log.info("Aggressive cluster cleanup completed for test environment")}catch(e){this.log.warn("Error during aggressive cluster cleanup",{error:e.message})}}async shutdown(){const e=Date.now();this.log.info("Initiating graceful shutdown"),this.shutting_down=!0,await this.stop_http_server_gracefully(),this.stop_backup_scheduling_gracefully(),this.send_shutdown_signals(),await this.wait_for_pending_writes(),this.log.info("All writes completed, disconnecting workers"),this.disconnect_all_workers(),await this.wait_for_workers_to_exit(),this.cleanup_database_connections(),this.clear_internal_state(),this.perform_test_environment_cleanup();const t=Date.now()-e;this.log.info("Shutdown complete",{shutdown_duration_ms:t})}}var de=$;export{de as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{encode_message as f}from"./tcp_protocol.js";import{get_write_forwarder as
|
|
1
|
+
import{encode_message as f}from"./tcp_protocol.js";import{get_write_forwarder as g}from"./write_forwarder.js";import{get_replication_manager as h}from"./replication_manager.js";import{check_and_grow_map_size as x}from"./query_engine.js";import{performance_monitor as v}from"./performance_monitor.js";import D from"./logger.js";import q from"./operations/insert_one.js";import A from"./operations/update_one.js";import b from"./operations/delete_one.js";import I from"./operations/delete_many.js";import $ from"./operations/bulk_write.js";import C from"./operations/find_one.js";import E from"./operations/find.js";import L from"./operations/create_index.js";import U from"./operations/drop_index.js";import Z from"./operations/get_indexes.js";import j from"./operations/admin.js";const{create_context_logger:d}=D("operation_dispatcher"),k=e=>e.length>64,B=e=>["admin","config","local"].includes(e.toLowerCase()),F=e=>/^[a-zA-Z0-9_-]+$/.test(e),G=e=>!e||typeof e!="string"||k(e)||B(e)?!1:F(e),H=()=>({ok:0,error:"Authentication required"}),J=()=>({ok:0,error:"Invalid database name. Database names must be alphanumeric with underscores/hyphens, max 64 characters, and cannot be reserved names (admin, config, local)."}),_=(e,n)=>{const o=f(n);e.write(o)},p=(e,n,o,s,t,r,c,i)=>{v.log_structured_operation(e,n,o,s,t,r,c,i)},w=(e,n,o,s,t,r,c,i,l=null)=>{const m={client_id:n,op:o,collection:s,duration_ms:t,status:r,request_size:c};r==="success"?(m.response_size=i,e.info("Database operation completed",m)):(m.error=l,e.error("Database operation failed",m))},K=async(e,n,o)=>{switch(e){case"insert_one":return await q(n,o.collection,o.document,o.options);case"update_one":return await A(n,o.collection,o.filter,o.update,o.options);case"delete_one":return await b(n,o.collection,o.filter,o.options);case"delete_many":return await I(n,o.collection,o.filter,o.options);case"bulk_write":return await $(n,o.collection,o.operations,o.options);case"find_one":return await C(n,o.collection,o.filter,o.options);case"find":return await E(n,o.collection,o.filter,o.options);case"create_index":return await L(n,o.collection,o.field,o.options);case"drop_index":return await U(n,o.collection,o.field);case"get_indexes":return await Z(n,o.collection);default:throw new Error(`Unsupported operation: ${e}`)}},M=(e,n)=>e==="find_one"?{ok:1,document:n}:e==="find"?{ok:1,documents:n}:{ok:1,...n},N=e=>!["find","find_one","get_indexes"].includes(e),O=(e,n)=>{if(!N(e))return;h().queue_replication(e,n.collection,n),setImmediate(()=>x())},P=(e,n,o,s,t,r)=>{const c=d(),i=Date.now()-t,l=M(n,s),u=f(l).length;_(e,l),p(e.id,n,o.collection,i,"success",null,r,u),w(c,e.id,n,o.collection,i,"success",r,u),O(n,o)},Q=(e,n,o,s,t,r)=>{const c=d(),i=Date.now()-t;p(e.id,n,o.collection,i,"error",s.message,r,0),w(c,e.id,n,o.collection,i,"error",r,0,s.message);const l={ok:0,error:s.message};_(e,l)},me=async(e,n,o,s,t=0,r=null,c=null)=>{const i=Date.now();if(!s(e)){const a=H();_(e,a),p(e.id,n,null,0,"error","Authentication required",t,0);return}const l=o.database||"default";if(!G(l)){const a=J();_(e,a),p(e.id,n,o.collection,0,"error","Invalid database name",t,0);return}if(!await g().forward_operation(e,n,o))try{const a=await K(n,l,o);P(e,n,o,a,i,t)}catch(a){Q(e,n,o,a,i,t)}},R=()=>({ok:!1,error:"Authentication required"}),S=(e,n)=>e?{ok:1,...n}:{ok:!0,...n},T=e=>({ok:0,error:`Admin operation failed: ${e}`}),pe=async(e,n,o,s=null,t=null)=>{if(!o(e)){const r=R();_(e,r);return}try{const r=n?.admin_action,i=await j(r,n||{},s,t),l=S(r,i);_(e,l)}catch(r){const c=T(r.message);_(e,c)}},ue=e=>{const n=Date.now(),o={ok:1,response_time_ms:Date.now()-n};_(e,o)};export{pe as handle_admin_operation,me as handle_database_operation,ue as handle_ping_operation};
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@joystick.js/db-canary",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.0.0-canary.
|
|
5
|
-
"canary_version": "0.0.0-canary.
|
|
4
|
+
"version": "0.0.0-canary.2255",
|
|
5
|
+
"canary_version": "0.0.0-canary.2254",
|
|
6
6
|
"description": "JoystickDB - A minimalist database server for the Joystick framework",
|
|
7
7
|
"main": "./dist/server/index.js",
|
|
8
8
|
"scripts": {
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import insert_one from '../lib/operations/insert_one.js';
|
|
19
19
|
import update_one from '../lib/operations/update_one.js';
|
|
20
20
|
import delete_one from '../lib/operations/delete_one.js';
|
|
21
|
+
import delete_many from '../lib/operations/delete_many.js';
|
|
21
22
|
import bulk_write from '../lib/operations/bulk_write.js';
|
|
22
23
|
import find_one from '../lib/operations/find_one.js';
|
|
23
24
|
import find from '../lib/operations/find.js';
|
|
@@ -511,6 +512,10 @@ class ClusterMaster extends EventEmitter {
|
|
|
511
512
|
result = await delete_one(database_name, data.collection, data.filter, data.options);
|
|
512
513
|
break;
|
|
513
514
|
|
|
515
|
+
case 'delete_many':
|
|
516
|
+
result = await delete_many(database_name, data.collection, data.filter, data.options);
|
|
517
|
+
break;
|
|
518
|
+
|
|
514
519
|
case 'bulk_write':
|
|
515
520
|
result = await bulk_write(database_name, data.collection, data.operations, data.options);
|
|
516
521
|
break;
|
|
@@ -439,6 +439,10 @@ export const handle_admin_operation = async (socket, data, check_authentication,
|
|
|
439
439
|
* @param {net.Socket} socket - Client socket connection
|
|
440
440
|
*/
|
|
441
441
|
export const handle_ping_operation = (socket) => {
|
|
442
|
-
const
|
|
442
|
+
const start_time = Date.now();
|
|
443
|
+
const ping_response = {
|
|
444
|
+
ok: 1,
|
|
445
|
+
response_time_ms: Date.now() - start_time
|
|
446
|
+
};
|
|
443
447
|
send_encoded_response(socket, ping_response);
|
|
444
448
|
};
|