itsi-server 0.2.16 → 0.2.17

data/ext/itsi_server/src/server/serve_strategy/single_mode.rs

@@ -4,7 +4,10 @@ use crate::{
         lifecycle_event::LifecycleEvent,
         request_job::RequestJob,
         serve_strategy::acceptor::{Acceptor, AcceptorArgs},
-        signal::{SHUTDOWN_REQUESTED, SIGNAL_HANDLER_CHANNEL},
+        signal::{
+            send_lifecycle_event, subscribe_runtime_to_signals, unsubscribe_runtime,
+            SHUTDOWN_REQUESTED,
+        },
         thread_worker::{build_thread_workers, ThreadWorker},
     },
 };
@@ -29,22 +32,20 @@ use std::{
 };
 use tokio::{
     runtime::{Builder as RuntimeBuilder, Runtime},
-    sync::{
-        broadcast,
-        watch::{self},
-    },
+    sync::watch::{self},
     task::JoinSet,
 };
 use tracing::instrument;
 
 pub struct SingleMode {
+    pub worker_id: usize,
     pub executor: Builder<TokioExecutor>,
     pub server_config: Arc<ItsiServerConfig>,
-    pub(crate) lifecycle_channel: broadcast::Sender<LifecycleEvent>,
     pub restart_requested: AtomicBool,
     pub status: RwLock<HashMap<u8, (u64, u64)>>,
 }
 
+#[derive(PartialEq, Debug)]
 pub enum RunningPhase {
     Running,
     ShutdownPending,
@@ -53,31 +54,45 @@ pub enum RunningPhase {
 
 impl SingleMode {
     #[instrument(parent=None, skip_all)]
-    pub fn new(server_config: Arc<ItsiServerConfig>) -> Result<Self> {
+    pub fn new(server_config: Arc<ItsiServerConfig>, worker_id: usize) -> Result<Self> {
         server_config.server_params.read().preload_ruby()?;
-        let mut executor = Builder::new(TokioExecutor::new());
-        executor
-            .http1()
-            .header_read_timeout(server_config.server_params.read().header_read_timeout)
-            .writev(true)
-            .timer(TokioTimer::new());
-        executor
-            .http2()
-            .max_concurrent_streams(100)
-            .max_local_error_reset_streams(100)
-            .enable_connect_protocol()
-            .max_header_list_size(10 * 1024 * 1024)
-            .max_send_buf_size(16 * 1024 * 1024);
+        let executor = {
+            let mut executor = Builder::new(TokioExecutor::new());
+            let server_params = server_config.server_params.read();
+            let mut http1_executor = executor.http1();
+
+            http1_executor
+                .header_read_timeout(server_params.header_read_timeout)
+                .pipeline_flush(server_params.pipeline_flush)
+                .timer(TokioTimer::new());
+
+            if let Some(writev) = server_params.writev {
+                http1_executor.writev(writev);
+            }
+
+            executor
+                .http2()
+                .max_concurrent_streams(server_params.max_concurrent_streams)
+                .max_local_error_reset_streams(server_params.max_local_error_reset_streams)
+                .max_header_list_size(server_params.max_header_list_size)
+                .max_send_buf_size(server_params.max_send_buf_size)
+                .enable_connect_protocol();
+            executor
+        };
 
         Ok(Self {
+            worker_id,
             executor,
             server_config,
-            lifecycle_channel: SIGNAL_HANDLER_CHANNEL.0.clone(),
             restart_requested: AtomicBool::new(false),
             status: RwLock::new(HashMap::new()),
         })
     }
 
+    pub fn is_zero_worker(&self) -> bool {
+        self.worker_id == 0
+    }
+
     pub fn build_runtime(&self) -> Runtime {
         let mut builder: RuntimeBuilder = if self
             .server_config
@@ -103,7 +118,7 @@ impl SingleMode {
 
     pub fn stop(&self) -> Result<()> {
         SHUTDOWN_REQUESTED.store(true, std::sync::atomic::Ordering::SeqCst);
-        self.lifecycle_channel.send(LifecycleEvent::Shutdown).ok();
+        send_lifecycle_event(LifecycleEvent::Shutdown);
         Ok(())
     }
 
@@ -182,7 +197,7 @@ impl SingleMode {
                         .unwrap();
                     let receiver = self.clone();
                     monitor_runtime.block_on({
-                        let mut lifecycle_rx = receiver.lifecycle_channel.subscribe();
+                        let mut lifecycle_rx = subscribe_runtime_to_signals();
                         let receiver = receiver.clone();
                         let thread_workers = thread_workers.clone();
                         async move {
@@ -201,18 +216,19 @@ impl SingleMode {
                                   }
                                   lifecycle_event = lifecycle_rx.recv() => {
                                       match lifecycle_event {
-                                          Ok(LifecycleEvent::Restart) => {
+                                          Ok(LifecycleEvent::Restart) | Ok(LifecycleEvent::Reload) => {
                                               receiver.restart().await.ok();
                                           }
-                                          Ok(LifecycleEvent::Reload) => {
-                                              receiver.reload().await.ok();
-                                          }
                                           Ok(LifecycleEvent::Shutdown) => {
                                             break;
                                           }
                                           Ok(LifecycleEvent::PrintInfo) => {
                                             receiver.print_info(thread_workers.clone()).await.ok();
                                           }
+                                          Err(e) => {
+                                            debug!("Lifecycle channel closed: {:?}, exiting single mode monitor loop", e);
+                                            break;
+                                          }
                                           _ => {}
                                       }
                                   }
@@ -227,13 +243,15 @@ impl SingleMode {
 
     #[instrument(name="worker", parent=None, skip(self), fields(pid=format!("{}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
-        let (thread_workers, job_sender, nonblocking_sender) =
-            build_thread_workers(self.server_config.server_params.read().clone(), Pid::this())
-                .inspect_err(|e| {
-                    if let Some(err_val) = e.value() {
-                        print_rb_backtrace(err_val);
-                    }
-                })?;
+        let (thread_workers, job_sender, nonblocking_sender) = build_thread_workers(
+            self.server_config.server_params.read().clone(),
+            self.worker_id,
+        )
+        .inspect_err(|e| {
+            if let Some(err_val) = e.value() {
+                print_rb_backtrace(err_val);
+            }
+        })?;
 
         let worker_count = thread_workers.len();
         info!(
@@ -244,6 +262,7 @@ impl SingleMode {
         let shutdown_timeout = self.server_config.server_params.read().shutdown_timeout;
         let (shutdown_sender, _) = watch::channel(RunningPhase::Running);
         let monitor_thread = self.clone().start_monitors(thread_workers.clone());
+        let is_zero_worker = self.is_zero_worker();
         if monitor_thread.is_none() {
             error!("Failed to start monitor thread");
             return Err(ItsiError::new("Failed to start monitor thread"));
@@ -253,106 +272,123 @@ impl SingleMode {
             return Ok(());
         }
         let runtime = self.build_runtime();
-        let result = runtime.block_on(
-          async  {
-              let mut listener_task_set = JoinSet::new();
-              let server_params = self.server_config.server_params.read().clone();
-              if let Err(err) = server_params.initialize_middleware().await {
-                  error!("Failed to initialize middleware: {}", err);
-                  return Err(ItsiError::new("Failed to initialize middleware"))
-              }
-              let tokio_listeners = server_params.listeners.lock()
-                  .drain(..)
-                  .map(|list| {
-                    Arc::new(list.into_tokio_listener())
-                  })
-                  .collect::<Vec<_>>();
-
-              tokio_listeners.iter().cloned().for_each(|listener| {
-                  let shutdown_sender = shutdown_sender.clone();
-                  let job_sender = job_sender.clone();
-                  let nonblocking_sender = nonblocking_sender.clone();
-
-                  let mut lifecycle_rx = self.lifecycle_channel.subscribe();
-                  let mut shutdown_receiver = shutdown_sender.subscribe();
-                  let mut acceptor = Acceptor{
-                      acceptor_args: Arc::new(
-                        AcceptorArgs{
-                          strategy: self.clone(),
-                          listener_info: listener.listener_info(),
-                          shutdown_receiver: shutdown_sender.subscribe(),
-                          job_sender: job_sender.clone(),
-                          nonblocking_sender: nonblocking_sender.clone(),
-                          server_params: server_params.clone()
-                        }
-                      ),
-                      join_set: JoinSet::new()
-                  };
-
-                  let shutdown_rx_for_acme_task = shutdown_receiver.clone();
-                  let acme_task_listener_clone = listener.clone();
-                  listener_task_set.spawn(async move {
-                      acme_task_listener_clone.spawn_acme_event_task(shutdown_rx_for_acme_task).await;
-                  });
-
-                  listener_task_set.spawn(async move {
-                      loop {
-                          tokio::select! {
-                              accept_result = listener.accept() => {
-                                  match accept_result {
-                                      Ok(accepted) => acceptor.serve_connection(accepted).await,
-                                      Err(e) => debug!("Listener.accept failed: {:?}", e)
-                                  }
-                              },
-                              _ = shutdown_receiver.changed() => {
-                                  debug!("Shutdown requested via receiver");
-                                  break;
-                              },
-                              lifecycle_event = lifecycle_rx.recv() => {
-                                  match lifecycle_event {
-                                      Ok(LifecycleEvent::Shutdown) => {
-                                          debug!("Received LifecycleEvent::Shutdown");
-                                          let _ = shutdown_sender.send(RunningPhase::ShutdownPending);
-                                          for _ in 0..worker_count {
-                                              let _ = job_sender.send(RequestJob::Shutdown).await;
-                                              let _ = nonblocking_sender.send(RequestJob::Shutdown).await;
-                                          }
-                                          break;
-                                      },
-                                      Err(e) =>  error!("Error receiving lifecycle event: {:?}", e),
-                                      _ => ()
+        let result = runtime.block_on(async {
+            let mut listener_task_set = JoinSet::new();
+            let server_params = self.server_config.server_params.read().clone();
+            if let Err(err) = server_params.initialize_middleware().await {
+                error!("Failed to initialize middleware: {}", err);
+                return Err(ItsiError::new("Failed to initialize middleware"));
+            }
+            let tokio_listeners = server_params
+                .listeners
+                .lock()
+                .drain(..)
+                .map(|list| Arc::new(list.into_tokio_listener(is_zero_worker)))
+                .collect::<Vec<_>>();
+
+            tokio_listeners.iter().cloned().for_each(|listener| {
+                let shutdown_sender = shutdown_sender.clone();
+                let job_sender = job_sender.clone();
+                let nonblocking_sender = nonblocking_sender.clone();
+
+                let mut lifecycle_rx = subscribe_runtime_to_signals();
+                let mut shutdown_receiver = shutdown_sender.subscribe();
+                let mut acceptor = Acceptor {
+                    acceptor_args: Arc::new(AcceptorArgs {
+                        strategy: self.clone(),
+                        listener_info: listener.listener_info(),
+                        shutdown_receiver: shutdown_sender.subscribe(),
+                        job_sender: job_sender.clone(),
+                        nonblocking_sender: nonblocking_sender.clone(),
+                        server_params: server_params.clone(),
+                    }),
+                    join_set: JoinSet::new(),
+                };
+
+                let shutdown_rx_for_acme_task = shutdown_receiver.clone();
+                let acme_task_listener_clone = listener.clone();
+
+                let mut after_accept_wait: Option<Duration> = None::<Duration>;
+
+                if cfg!(target_os = "macos") {
+                    after_accept_wait = if server_params.workers > 1 {
+                        Some(Duration::from_nanos(10 * server_params.workers as u64))
+                    } else {
+                        None
+                    };
+                };
+
+                listener_task_set.spawn(async move {
+                    acme_task_listener_clone
+                        .spawn_acme_event_task(shutdown_rx_for_acme_task)
+                        .await;
+                });
+
+                listener_task_set.spawn(async move {
+                    loop {
+                        // Process any pending signals before select
+                        tokio::select! {
+                            accept_result = listener.accept() => {
+                                match accept_result {
+                                    Ok(accepted) => acceptor.serve_connection(accepted).await,
+                                    Err(e) => debug!("Listener.accept failed: {:?}", e)
+                                }
+                                if cfg!(target_os = "macos") {
+                                  if let Some(after_accept_wait) = after_accept_wait{
+                                    tokio::time::sleep(after_accept_wait).await;
                                   }
-                              }
-                          }
-                      }
-                      acceptor.join().await;
-                  });
-              });
-
-              if self.is_single_mode() {
+                                }
+                            },
+                            _ = shutdown_receiver.changed() => {
+                                debug!("Shutdown requested via receiver");
+                                break;
+                            },
+                            lifecycle_event = lifecycle_rx.recv() => {
+                                match lifecycle_event {
+                                    Ok(LifecycleEvent::Shutdown) => {
+                                        debug!("Received LifecycleEvent::Shutdown");
+                                        let _ = shutdown_sender.send(RunningPhase::ShutdownPending);
+                                        break;
+                                    },
+                                    Err(e) => {
+                                      debug!("Lifecycle channel closed: {:?}, exiting accept loop", e);
+                                      break
+                                    },
+                                    _ => ()
+                                }
+                            }
+                        }
+                    }
+                    acceptor.join().await;
+                });
+            });
+
+            if self.is_single_mode() {
                 self.invoke_hook("after_start");
-              }
+            }
 
-              while let Some(_res) = listener_task_set.join_next().await {}
-              drop(tokio_listeners);
+            while let Some(_res) = listener_task_set.join_next().await {}
+            drop(tokio_listeners);
 
-              Ok::<(), ItsiError>(())
-          });
+            Ok::<(), ItsiError>(())
+        });
 
         debug!("Single mode runtime exited.");
 
+        for _i in 0..thread_workers.len() {
+            job_sender.send_blocking(RequestJob::Shutdown).unwrap();
+            nonblocking_sender
+                .send_blocking(RequestJob::Shutdown)
+                .unwrap();
+        }
         if result.is_err() {
-            for _i in 0..thread_workers.len() {
-                job_sender.send_blocking(RequestJob::Shutdown).unwrap();
-                nonblocking_sender
-                    .send_blocking(RequestJob::Shutdown)
-                    .unwrap();
-            }
-            self.lifecycle_channel.send(LifecycleEvent::Shutdown).ok();
+            send_lifecycle_event(LifecycleEvent::Shutdown);
         }
 
         shutdown_sender.send(RunningPhase::Shutdown).ok();
         runtime.shutdown_timeout(Duration::from_millis(100));
+        unsubscribe_runtime();
+
         debug!("Shutdown timeout finished.");
 
         let deadline = Instant::now() + Duration::from_secs_f64(shutdown_timeout);
@@ -384,26 +420,6 @@ impl SingleMode {
     pub fn is_single_mode(&self) -> bool {
         self.server_config.server_params.read().workers == 1
     }
-    /// Attempts to reload the config "live"
-    /// Not that when running in single mode this will not unload
-    /// old code. If you need a clean restart, use the `restart` (SIGHUP) method instead
-    pub async fn reload(&self) -> Result<()> {
-        if !self.server_config.check_config().await {
-            return Ok(());
-        }
-        let should_reexec = self.server_config.clone().reload(false)?;
-        if should_reexec {
-            if self.is_single_mode() {
-                self.invoke_hook("before_restart");
-            }
-            self.server_config.dup_fds()?;
-            self.server_config.reload_exec()?;
-        }
-        self.restart_requested.store(true, Ordering::SeqCst);
-        self.stop()?;
-        self.server_config.server_params.read().preload_ruby()?;
-        Ok(())
-    }
 
     pub fn invoke_hook(&self, hook_name: &str) {
         if let Some(hook) = self.server_config.server_params.read().hooks.get(hook_name) {

data/ext/itsi_server/src/server/signal.rs

@@ -1,22 +1,50 @@
-use std::sync::{
-    atomic::{AtomicBool, AtomicI8},
-    LazyLock,
+use std::{
+    collections::VecDeque,
+    sync::atomic::{AtomicBool, AtomicI8},
 };
 
 use nix::libc::{self, sighandler_t};
-use tokio::sync::{self, broadcast};
+use parking_lot::Mutex;
+use tokio::sync::broadcast;
 
 use super::lifecycle_event::LifecycleEvent;
 
 pub static SIGINT_COUNT: AtomicI8 = AtomicI8::new(0);
 pub static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);
-pub static SIGNAL_HANDLER_CHANNEL: LazyLock<(
-    broadcast::Sender<LifecycleEvent>,
-    broadcast::Receiver<LifecycleEvent>,
-)> = LazyLock::new(|| sync::broadcast::channel(5));
+pub static SIGNAL_HANDLER_CHANNEL: Mutex<Option<broadcast::Sender<LifecycleEvent>>> =
+    Mutex::new(None);
+
+pub static PENDING_QUEUE: Mutex<VecDeque<LifecycleEvent>> = Mutex::new(VecDeque::new());
+
+pub fn subscribe_runtime_to_signals() -> broadcast::Receiver<LifecycleEvent> {
+    let mut guard = SIGNAL_HANDLER_CHANNEL.lock();
+    if let Some(sender) = guard.as_ref() {
+        return sender.subscribe();
+    }
+    let (sender, receiver) = broadcast::channel(5);
+    let sender_clone = sender.clone();
+    std::thread::spawn(move || {
+        std::thread::sleep(std::time::Duration::from_millis(50));
+        for event in PENDING_QUEUE.lock().drain(..) {
+            sender_clone.send(event).ok();
+        }
+    });
+
+    guard.replace(sender);
+
+    receiver
+}
+
+pub fn unsubscribe_runtime() {
+    SIGNAL_HANDLER_CHANNEL.lock().take();
+}
 
 pub fn send_lifecycle_event(event: LifecycleEvent) {
-    SIGNAL_HANDLER_CHANNEL.0.send(event).ok();
+    if let Some(sender) = SIGNAL_HANDLER_CHANNEL.lock().as_ref() {
+        sender.send(event).ok();
+    } else {
+        PENDING_QUEUE.lock().push_back(event);
+    }
 }
 
 fn receive_signal(signum: i32, _: sighandler_t) {