itsi-scheduler 0.1.5 → 0.1.19

data/ext/itsi_server/src/server/serve_strategy/cluster_mode.rs

@@ -1,14 +1,13 @@
-use crate::server::{
-    itsi_server::Server, lifecycle_event::LifecycleEvent, listener::Listener,
-    process_worker::ProcessWorker,
-};
+use crate::ruby_types::itsi_server::itsi_server_config::ItsiServerConfig;
+use crate::server::signal::SIGNAL_HANDLER_CHANNEL;
+use crate::server::{lifecycle_event::LifecycleEvent, process_worker::ProcessWorker};
 use itsi_error::{ItsiError, Result};
-use itsi_rb_helpers::{call_without_gvl, create_ruby_thread};
-use itsi_tracing::{error, info, warn};
-use nix::{
-    libc::{self, exit},
-    unistd::Pid,
+use itsi_rb_helpers::{
+    call_proc_and_log_errors, call_with_gvl, call_without_gvl, create_ruby_thread,
 };
+use itsi_tracing::{error, info, warn};
+use magnus::Value;
+use nix::{libc::exit, unistd::Pid};
 
 use std::{
     sync::{atomic::AtomicUsize, Arc},
@@ -19,10 +18,9 @@ use tokio::{
     sync::{broadcast, watch, Mutex},
     time::{self, sleep},
 };
-use tracing::instrument;
+use tracing::{debug, instrument};
 pub(crate) struct ClusterMode {
-    pub listeners: Arc<Vec<Arc<Listener>>>,
-    pub server: Arc<Server>,
+    pub server_config: Arc<ItsiServerConfig>,
     pub process_workers: parking_lot::Mutex<Vec<ProcessWorker>>,
     pub lifecycle_channel: broadcast::Sender<LifecycleEvent>,
 }
@@ -32,15 +30,8 @@ static CHILD_SIGNAL_SENDER: parking_lot::Mutex<Option<watch::Sender<()>>> =
     parking_lot::Mutex::new(None);
 
 impl ClusterMode {
-    pub fn new(
-        server: Arc<Server>,
-        listeners: Arc<Vec<Arc<Listener>>>,
-        lifecycle_channel: broadcast::Sender<LifecycleEvent>,
-    ) -> Self {
-        if let Some(f) = server.before_fork.lock().take() {
-            f();
-        }
-        let process_workers = (0..server.workers)
+    pub fn new(server_config: Arc<ItsiServerConfig>) -> Self {
+        let process_workers = (0..server_config.server_params.read().workers)
             .map(|_| ProcessWorker {
                 worker_id: WORKER_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
                 ..Default::default()
@@ -48,10 +39,9 @@ impl ClusterMode {
             .collect();
 
         Self {
-            listeners,
-            server,
+            server_config,
             process_workers: parking_lot::Mutex::new(process_workers),
-            lifecycle_channel,
+            lifecycle_channel: SIGNAL_HANDLER_CHANNEL.0.clone(),
         }
     }
 
@@ -73,13 +63,56 @@ impl ClusterMode {
     ) -> Result<()> {
         match lifecycle_event {
             LifecycleEvent::Start => Ok(()),
+            LifecycleEvent::PrintInfo => {
+                self.print_info().await?;
+                Ok(())
+            }
             LifecycleEvent::Shutdown => {
+                self.server_config.stop_watcher()?;
                 self.shutdown().await?;
                 Ok(())
             }
             LifecycleEvent::Restart => {
-                for worker in self.process_workers.lock().iter() {
-                    worker.reboot(self.clone()).await?;
+                self.server_config.dup_fds()?;
+                self.shutdown().await.ok();
+                info!("Shutdown complete. Calling reload exec");
+                self.server_config.reload_exec()?;
+                Ok(())
+            }
+            LifecycleEvent::Reload => {
+                let should_reexec = self.server_config.clone().reload(true)?;
+                if should_reexec {
+                    self.server_config.dup_fds()?;
+                    self.shutdown().await.ok();
+                    self.server_config.reload_exec()?;
+                }
+                let mut workers_to_load = self.server_config.server_params.read().workers;
+                let mut next_workers = Vec::new();
+                for worker in self.process_workers.lock().drain(..) {
+                    if workers_to_load == 0 {
+                        worker.graceful_shutdown(self.clone()).await
+                    } else {
+                        workers_to_load -= 1;
+                        worker.reboot(self.clone()).await?;
+                        next_workers.push(worker);
+                    }
+                }
+                self.process_workers.lock().extend(next_workers);
+                while workers_to_load > 0 {
+                    let mut workers = self.process_workers.lock();
+                    let worker = ProcessWorker {
+                        worker_id: WORKER_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
+                        ..Default::default()
+                    };
+                    let worker_clone = worker.clone();
+                    let self_clone = self.clone();
+                    create_ruby_thread(move || {
+                        call_without_gvl(move || {
+                            worker_clone.boot(self_clone).ok();
+                        })
+                    });
+                    workers.push(worker);
+                    workers_to_load -= 1
                 }
                 Ok(())
             }
@@ -106,8 +139,10 @@ impl ClusterMode {
                 };
                 if let Some(dropped_worker) = worker {
                     dropped_worker.request_shutdown();
-                    let force_kill_time =
-                        Instant::now() + Duration::from_secs_f64(self.server.shutdown_timeout);
+                    let force_kill_time = Instant::now()
+                        + Duration::from_secs_f64(
+                            self.server_config.server_params.read().shutdown_timeout,
+                        );
                     while dropped_worker.is_alive() && force_kill_time > Instant::now() {
                         tokio::time::sleep(Duration::from_millis(100)).await;
                     }
@@ -121,13 +156,20 @@ impl ClusterMode {
                 for worker in self.process_workers.lock().iter() {
                     worker.force_kill();
                 }
+                error!("Force shutdown!");
                 unsafe { exit(0) };
             }
+            LifecycleEvent::ChildTerminated => {
+                CHILD_SIGNAL_SENDER.lock().as_ref().inspect(|i| {
+                    i.send(()).ok();
+                });
+                Ok(())
+            }
         }
     }
 
     pub async fn shutdown(&self) -> Result<()> {
-        let shutdown_timeout = self.server.shutdown_timeout;
+        let shutdown_timeout = self.server_config.server_params.read().shutdown_timeout;
         let workers = self.process_workers.lock().clone();
 
         workers.iter().for_each(|worker| worker.request_shutdown());
@@ -152,7 +194,7 @@ impl ClusterMode {
 
         tokio::select! {
             _ = monitor_handle => {
-              info!("All children exited early, exit normally")
+              debug!("All children exited early, exit normally")
             }
             _ = sleep(Duration::from_secs_f64(shutdown_timeout)) => {
                 warn!("Graceful shutdown timeout reached, force killing remaining children");
@@ -160,37 +202,76 @@ impl ClusterMode {
             }
         }
 
-        Err(ItsiError::Break())
+        Err(ItsiError::Break)
     }
 
-    pub fn receive_signal(signal: i32) {
-        match signal {
-            libc::SIGCHLD => {
-                CHILD_SIGNAL_SENDER.lock().as_ref().inspect(|i| {
-                    i.send(()).ok();
-                });
-            }
-            _ => {
-                // Handle other signals
+    pub async fn print_info(self: Arc<Self>) -> Result<()> {
+        println!("Itsi Cluster Info:");
+        println!("Master PID: {:?}", Pid::this());
+        if let Some(memory_limit) = self.server_config.server_params.read().worker_memory_limit {
+            println!("Worker Memory Limit: {}", memory_limit);
+        }
+
+        if self.server_config.watcher_fd.is_some() {
+            println!("File Watcher Enabled: true",);
+            if let Some(watchers) = self
+                .server_config
+                .server_params
+                .read()
+                .notify_watchers
+                .as_ref()
+            {
+                for watcher in watchers {
+                    println!(
+                        "Watching path: {} => {}",
+                        watcher.0,
+                        watcher
+                            .1
+                            .iter()
+                            .map(|path| path.join(","))
+                            .collect::<Vec<String>>()
+                            .join(" ")
+                    );
+                }
             }
         }
+        println!(
+            "Silent Mode: {}",
+            self.server_config.server_params.read().silence
+        );
+        println!(
+            "Preload: {}",
+            self.server_config.server_params.read().preload
+        );
+        let workers = self.process_workers.lock().clone();
+        for worker in workers {
+            worker.print_info()?;
+            sleep(Duration::from_millis(50)).await;
+        }
+        Ok(())
     }
 
     pub fn stop(&self) -> Result<()> {
-        unsafe { libc::signal(libc::SIGCHLD, libc::SIG_DFL) };
-
         for worker in self.process_workers.lock().iter() {
             if worker.is_alive() {
                 worker.force_kill();
             }
         }
-
         Ok(())
     }
 
     #[instrument(skip(self), fields(mode = "cluster", pid=format!("{:?}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
         info!("Starting in Cluster mode");
+        if let Some(proc) = self
+            .server_config
+            .server_params
+            .read()
+            .hooks
+            .get("before_fork")
+        {
+            call_with_gvl(|_| call_proc_and_log_errors(proc.clone()))
+        }
         self.process_workers
             .lock()
             .iter()
@@ -199,8 +280,6 @@ impl ClusterMode {
         let (sender, mut receiver) = watch::channel(());
         *CHILD_SIGNAL_SENDER.lock() = Some(sender);
 
-        unsafe { libc::signal(libc::SIGCHLD, Self::receive_signal as usize) };
-
         let mut lifecycle_rx = self.lifecycle_channel.subscribe();
         let self_ref = self.clone();
 
@@ -212,14 +291,15 @@ impl ClusterMode {
               _ = receiver.changed() => {
                 let mut workers = self_ref.process_workers.lock();
                 workers.retain(|worker| {
-                  worker.boot_if_dead(Arc::clone(&self_ref))
+                  worker.boot_if_dead(self_ref.clone())
                 });
                 if workers.is_empty() {
                     warn!("No workers running. Send SIGTTIN to increase worker count");
                 }
               }
               _ = memory_check_interval.tick() => {
-                if let Some(memory_limit) = self_ref.server.worker_memory_limit {
+                let worker_memory_limit = self_ref.server_config.server_params.read().worker_memory_limit;
+                if let Some(memory_limit) = worker_memory_limit {
                   let largest_worker = {
                     let workers = self_ref.process_workers.lock();
                     workers.iter().max_by(|wa, wb| wa.memory_usage().cmp(&wb.memory_usage())).cloned()
@@ -228,6 +308,9 @@ impl ClusterMode {
                     if let Some(current_mem_usage) = largest_worker.memory_usage(){
                       if current_mem_usage > memory_limit {
                         largest_worker.reboot(self_ref.clone()).await.ok();
+                        if let Some(hook) = self_ref.server_config.server_params.read().hooks.get("after_memory_threshold_reached") {
+                          call_with_gvl(|_|  hook.call::<_, Value>((largest_worker.pid(),)).ok() );
+                        }
                       }
                     }
                   }
@@ -237,7 +320,7 @@ impl ClusterMode {
                 Ok(lifecycle_event) => {
                   if let Err(e) = self_ref.clone().handle_lifecycle_event(lifecycle_event).await{
                     match e {
-                      ItsiError::Break() => break,
+                      ItsiError::Break => break,
                       _ => error!("Error in handle_lifecycle_event {:?}", e)
                     }
                   }
@@ -248,7 +331,12 @@ impl ClusterMode {
             }
           }
         });
-
+        self.server_config
+            .server_params
+            .write()
+            .listeners
+            .lock()
+            .drain(..);
         Ok(())
     }
 }

data/ext/itsi_server/src/server/serve_strategy/mod.rs

@@ -1,27 +1,30 @@
+use std::sync::Arc;
+
 use cluster_mode::ClusterMode;
 use itsi_error::Result;
 use single_mode::SingleMode;
-use std::sync::Arc;
+
 pub mod cluster_mode;
 pub mod single_mode;
 
+#[derive(Clone)]
 pub(crate) enum ServeStrategy {
     Single(Arc<SingleMode>),
     Cluster(Arc<ClusterMode>),
 }
 
 impl ServeStrategy {
-    pub fn run(&self) -> Result<()> {
+    pub fn run(self) -> Result<()> {
         match self {
-            ServeStrategy::Single(single_router) => single_router.clone().run(),
-            ServeStrategy::Cluster(cluster_router) => cluster_router.clone().run(),
+            ServeStrategy::Single(single_router) => single_router.run(),
+            ServeStrategy::Cluster(cluster_router) => cluster_router.run(),
         }
     }
 
     pub(crate) fn stop(&self) -> Result<()> {
         match self {
-            ServeStrategy::Single(single_router) => single_router.clone().stop(),
-            ServeStrategy::Cluster(cluster_router) => cluster_router.clone().stop(),
+            ServeStrategy::Single(single_router) => single_router.stop(),
+            ServeStrategy::Cluster(cluster_router) => cluster_router.stop(),
         }
     }
 }