itsi 0.1.11 → 0.1.12

data/crates/itsi_server/src/server/serve_strategy/single_mode.rs

@@ -1,36 +1,43 @@
 use crate::{
-    request::itsi_request::ItsiRequest,
+    ruby_types::itsi_server::itsi_server_config::ItsiServerConfig,
     server::{
         io_stream::IoStream,
-        itsi_server::{RequestJob, Server},
+        itsi_service::{IstiServiceInner, ItsiService},
         lifecycle_event::LifecycleEvent,
-        listener::{Listener, ListenerInfo},
+        listener::ListenerInfo,
+        request_job::RequestJob,
+        signal::SIGNAL_HANDLER_CHANNEL,
         thread_worker::{build_thread_workers, ThreadWorker},
     },
 };
-use http::Request;
-use hyper::{body::Incoming, service::service_fn};
 use hyper_util::{
     rt::{TokioExecutor, TokioIo, TokioTimer},
     server::conn::auto::Builder,
 };
 use itsi_error::{ItsiError, Result};
-use itsi_rb_helpers::print_rb_backtrace;
+use itsi_rb_helpers::{
+    call_with_gvl, call_without_gvl, create_ruby_thread, funcall_no_ret, print_rb_backtrace,
+};
 use itsi_tracing::{debug, error, info};
+use magnus::value::ReprValue;
 use nix::unistd::Pid;
-use parking_lot::Mutex;
+use parking_lot::RwLock;
 use std::{
+    collections::HashMap,
     num::NonZeroU8,
-    panic,
     pin::Pin,
-    sync::Arc,
-    time::{Duration, Instant},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 use tokio::{
     runtime::{Builder as RuntimeBuilder, Runtime},
     sync::{
         broadcast,
-        watch::{self, Sender},
+        watch::{self},
     },
     task::JoinSet,
 };
@@ -38,11 +45,10 @@ use tracing::instrument;
 
 pub struct SingleMode {
     pub executor: Builder<TokioExecutor>,
-    pub server: Arc<Server>,
-    pub sender: async_channel::Sender<RequestJob>,
-    pub(crate) listeners: Mutex<Vec<Listener>>,
-    pub(crate) thread_workers: Arc<Vec<ThreadWorker>>,
+    pub server_config: Arc<ItsiServerConfig>,
     pub(crate) lifecycle_channel: broadcast::Sender<LifecycleEvent>,
+    pub restart_requested: AtomicBool,
+    pub status: RwLock<HashMap<u8, (u64, u64)>>,
 }
 
 pub enum RunningPhase {
@@ -52,36 +58,29 @@ pub enum RunningPhase {
 }
 
 impl SingleMode {
-    #[instrument(parent=None, skip_all, fields(pid=format!("{:?}", Pid::this())))]
-    pub(crate) fn new(
-        server: Arc<Server>,
-        listeners: Vec<Listener>,
-        lifecycle_channel: broadcast::Sender<LifecycleEvent>,
-    ) -> Result<Self> {
-        let (thread_workers, sender) = build_thread_workers(
-            server.clone(),
-            Pid::this(),
-            NonZeroU8::try_from(server.threads).unwrap(),
-            server.app.clone(),
-            server.scheduler_class.clone(),
-        )
-        .inspect_err(|e| {
-            if let Some(err_val) = e.value() {
-                print_rb_backtrace(err_val);
-            }
-        })?;
+    #[instrument(parent=None, skip_all)]
+    pub fn new(server_config: Arc<ItsiServerConfig>) -> Result<Self> {
+        server_config.server_params.read().preload_ruby()?;
         Ok(Self {
             executor: Builder::new(TokioExecutor::new()),
-            listeners: Mutex::new(listeners),
-            server,
-            sender,
-            thread_workers,
-            lifecycle_channel,
+            server_config,
+            lifecycle_channel: SIGNAL_HANDLER_CHANNEL.0.clone(),
+            restart_requested: AtomicBool::new(false),
+            status: RwLock::new(HashMap::new()),
         })
     }
 
     pub fn build_runtime(&self) -> Runtime {
-        let mut builder: RuntimeBuilder = RuntimeBuilder::new_current_thread();
+        let mut builder: RuntimeBuilder = if self
+            .server_config
+            .server_params
+            .read()
+            .multithreaded_reactor
+        {
+            RuntimeBuilder::new_multi_thread()
+        } else {
+            RuntimeBuilder::new_current_thread()
+        };
         builder
             .thread_name("itsi-server-accept-loop")
             .thread_stack_size(3 * 1024 * 1024)
@@ -96,27 +95,171 @@ impl SingleMode {
         Ok(())
     }
 
-    #[instrument(parent=None, skip(self), fields(pid=format!("{}", Pid::this())))]
+    pub async fn print_info(&self, thread_workers: Arc<Vec<Arc<ThreadWorker>>>) -> Result<()> {
+        println!(" └─ Worker");
+        println!(
+            "    - binds: {:?}",
+            self.server_config.server_params.read().binds
+        );
+
+        println!(
+            "    ─ script_name: {:?}",
+            self.server_config.server_params.read().script_name
+        );
+        println!(
+            "    ─ streaming body: {:?}",
+            self.server_config.server_params.read().streamable_body
+        );
+        println!(
+            "    ─ multithreaded runtime: {:?}",
+            self.server_config
+                .server_params
+                .read()
+                .multithreaded_reactor
+        );
+        println!(
+            "    ─ scheduler: {:?}",
+            self.server_config.server_params.read().scheduler_class
+        );
+        println!(
+            "    ─ OOB GC Response threadhold: {:?}",
+            self.server_config
+                .server_params
+                .read()
+                .oob_gc_responses_threshold
+        );
+        for worker in thread_workers.iter() {
+            println!("   └─ - Thread : {:?}", worker.id);
+            println!("       - # Requests Processed: {:?}", worker.request_id);
+            println!(
+                "       - Last Request Started: {:?} ago",
+                if worker.current_request_start.load(Ordering::Relaxed) == 0 {
+                    Duration::from_secs(0)
+                } else {
+                    SystemTime::now()
+                        .duration_since(
+                            UNIX_EPOCH
+                                + Duration::from_secs(
+                                    worker.current_request_start.load(Ordering::Relaxed),
+                                ),
+                        )
+                        .unwrap_or(Duration::from_secs(0))
+                }
+            );
+            call_with_gvl(|_| {
+                if let Some(thread) = worker.thread.read().as_ref() {
+                    if let Ok(backtrace) = thread.funcall::<_, _, Vec<String>>("backtrace", ()) {
+                        println!("       - Backtrace:");
+                        for line in backtrace {
+                            println!("       -   {}", line);
+                        }
+                    }
+                }
+            })
+        }
+
+        Ok(())
+    }
+
+    pub fn start_monitors(
+        self: Arc<Self>,
+        thread_workers: Arc<Vec<Arc<ThreadWorker>>>,
+    ) -> magnus::Thread {
+        call_with_gvl(move |_| {
+            create_ruby_thread(move || {
+                call_without_gvl(move || {
+                    let monitor_runtime = RuntimeBuilder::new_current_thread()
+                        .enable_time()
+                        .build()
+                        .unwrap();
+                    let receiver = self.clone();
+                    monitor_runtime.block_on({
+                        let mut lifecycle_rx = receiver.lifecycle_channel.subscribe();
+                        let receiver = receiver.clone();
+                        let thread_workers = thread_workers.clone();
+                        async move {
+                            loop {
+                                tokio::select! {
+                                  _ = tokio::time::sleep(Duration::from_secs(1)) => {
+                                    let mut status_lock = receiver.status.write();
+                                    thread_workers.iter().for_each(|worker| {
+                                        let worker_entry = status_lock.entry(worker.id);
+                                        let data = (
+                                            worker.request_id.load(Ordering::Relaxed),
+                                            worker.current_request_start.load(Ordering::Relaxed),
+                                        );
+                                        worker_entry.or_insert(data);
+                                    });
+                                  }
+                                  lifecycle_event = lifecycle_rx.recv() => {
+                                      match lifecycle_event {
+                                          Ok(LifecycleEvent::Restart) => {
+                                              receiver.restart().ok();
+                                          }
+                                          Ok(LifecycleEvent::Reload) => {
+                                              receiver.reload().ok();
+                                          }
+                                          Ok(LifecycleEvent::Shutdown) => {
+                                            break;
+                                          }
+                                          Ok(LifecycleEvent::PrintInfo) => {
+                                            receiver.print_info(thread_workers.clone()).await.ok();
+                                          }
+                                          _ => {}
+                                      }
+                                  }
+                                }
+                            }
+                        }
+                    })
+                })
+            })
+        })
+    }
+
+    #[instrument(name="worker", parent=None, skip(self), fields(pid=format!("{}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
         let mut listener_task_set = JoinSet::new();
         let runtime = self.build_runtime();
 
-        runtime.block_on(async {
-              let tokio_listeners = self
-                  .listeners.lock()
+        let (thread_workers, job_sender) = build_thread_workers(
+            self.server_config.server_params.read().clone(),
+            Pid::this(),
+            NonZeroU8::try_from(self.server_config.server_params.read().threads).unwrap(),
+        )
+        .inspect_err(|e| {
+            if let Some(err_val) = e.value() {
+                print_rb_backtrace(err_val);
+            }
+        })?;
+
+        info!(
+            pid = format!("{}", Pid::this()),
+            threads = thread_workers.len(),
+            binds = format!("{:?}", self.server_config.server_params.read().binds)
+        );
+
+        let (shutdown_sender, _) = watch::channel(RunningPhase::Running);
+        let thread = self.clone().start_monitors(thread_workers.clone());
+        runtime.block_on(
+          async  {
+              let server_params = self.server_config.server_params.read().clone();
+              server_params.middleware.get().unwrap().initialize_layers().await?;
+              let tokio_listeners = server_params.listeners.lock()
                   .drain(..)
                   .map(|list| {
                     Arc::new(list.into_tokio_listener())
                   })
                   .collect::<Vec<_>>();
-              let (shutdown_sender, _) = watch::channel(RunningPhase::Running);
+
               for listener in tokio_listeners.iter() {
                   let mut lifecycle_rx = self.lifecycle_channel.subscribe();
                   let listener_info = Arc::new(listener.listener_info());
                   let self_ref = self.clone();
                   let listener = listener.clone();
                   let shutdown_sender = shutdown_sender.clone();
-
+                  let job_sender = job_sender.clone();
+                  let workers_clone = thread_workers.clone();
                   let listener_clone = listener.clone();
                   let mut shutdown_receiver = shutdown_sender.subscribe();
                   let shutdown_receiver_clone = shutdown_receiver.clone();
@@ -134,8 +277,9 @@ impl SingleMode {
                                 let strategy = strategy_clone.clone();
                                 let listener_info = listener_info.clone();
                                 let shutdown_receiver = shutdown_receiver.clone();
+                                let job_sender = job_sender.clone();
                                 acceptor_task_set.spawn(async move {
-                                  strategy.serve_connection(accept_result, listener_info, shutdown_receiver).await;
+                                  strategy.serve_connection(accept_result, job_sender, listener_info, shutdown_receiver).await;
                                 });
                               },
                               Err(e) => debug!("Listener.accept failed {:?}", e),
@@ -144,16 +288,18 @@ impl SingleMode {
                               break;
                             }
                             lifecycle_event = lifecycle_rx.recv() => match lifecycle_event{
-                              Ok(lifecycle_event) => {
-                                if let Err(e) = self_ref.handle_lifecycle_event(lifecycle_event, shutdown_sender.clone()).await{
-                                  match e {
-                                    ItsiError::Break() => break,
-                                    _ => error!("Error in handle_lifecycle_event {:?}", e)
-                                  }
+                              Ok(LifecycleEvent::Shutdown) => {
+                                shutdown_sender.send(RunningPhase::ShutdownPending).unwrap();
+                                // Tell any in-progress connections to stop accepting new requests
+                                tokio::time::sleep(Duration::from_millis(25)).await;
+                                // Tell workers to stop processing requests once they've flushed their buffers.
+                                for _i in 0..workers_clone.len() {
+                                  job_sender.send(RequestJob::Shutdown).await.unwrap();
                                 }
-
+                                break;
                               },
                               Err(e) => error!("Error receiving lifecycle_event: {:?}", e),
+                              _ => {}
                           }
                         }
                     }
@@ -164,8 +310,31 @@ impl SingleMode {
 
               while let Some(_res) = listener_task_set.join_next().await {}
 
-            });
+              Ok::<(), ItsiError>(())
+          })?;
+
+        shutdown_sender.send(RunningPhase::Shutdown).ok();
+        let deadline = Instant::now()
+            + Duration::from_secs_f64(self.server_config.server_params.read().shutdown_timeout);
+
         runtime.shutdown_timeout(Duration::from_millis(100));
+
+        loop {
+            if thread_workers
+                .iter()
+                .all(|worker| call_with_gvl(move |_| !worker.poll_shutdown(deadline)))
+            {
+                funcall_no_ret(thread, "join", ()).ok();
+                break;
+            }
+            sleep(Duration::from_millis(50));
+        }
+
+        if self.restart_requested.load(Ordering::SeqCst) {
+            self.restart_requested.store(false, Ordering::SeqCst);
+            info!("Worker restarting");
+            self.run()?;
+        }
         debug!("Runtime has shut down");
         Ok(())
     }
@@ -173,36 +342,32 @@ impl SingleMode {
     pub(crate) async fn serve_connection(
         &self,
         stream: IoStream,
+        job_sender: async_channel::Sender<RequestJob>,
         listener: Arc<ListenerInfo>,
         shutdown_channel: watch::Receiver<RunningPhase>,
     ) {
-        let sender_clone = self.sender.clone();
         let addr = stream.addr();
         let io: TokioIo<Pin<Box<IoStream>>> = TokioIo::new(Box::pin(stream));
-        let server = self.server.clone();
         let executor = self.executor.clone();
         let mut shutdown_channel_clone = shutdown_channel.clone();
-        let server = server.clone();
         let mut executor = executor.clone();
         let mut binding = executor.http1();
         let shutdown_channel = shutdown_channel_clone.clone();
+
+        let service = ItsiService {
+            inner: Arc::new(IstiServiceInner {
+                sender: job_sender.clone(),
+                server_params: self.server_config.server_params.read().clone(),
+                listener,
+                addr: addr.to_string(),
+                shutdown_channel: shutdown_channel.clone(),
+            }),
+        };
         let mut serve = Box::pin(
             binding
-                .timer(TokioTimer::new())
+                .timer(TokioTimer::new()) // your existing timer
                 .header_read_timeout(Duration::from_secs(1))
-                .serve_connection_with_upgrades(
-                    io,
-                    service_fn(move |hyper_request: Request<Incoming>| {
-                        ItsiRequest::process_request(
-                            hyper_request,
-                            sender_clone.clone(),
-                            server.clone(),
-                            listener.clone(),
-                            addr.clone(),
-                            shutdown_channel.clone(),
-                        )
-                    }),
-                ),
+                .serve_connection_with_upgrades(io, service),
         );
 
         tokio::select! {
@@ -213,7 +378,7 @@ impl SingleMode {
                       debug!("Connection closed normally")
                     },
                     Err(res) => {
-                      debug!("Connection finished with error: {:?}", res)
+                      debug!("Connection closed abruptly: {:?}", res)
                     }
                 }
                 serve.as_mut().graceful_shutdown();
@@ -221,6 +386,7 @@ impl SingleMode {
             // A lifecycle event triggers shutdown.
             _ = shutdown_channel_clone.changed() => {
                 // Initiate graceful shutdown.
+                info!("Starting graceful shutdown");
                 serve.as_mut().graceful_shutdown();
 
                 // Now await the connection to finish shutting down.
@@ -231,46 +397,25 @@ impl SingleMode {
         }
     }
 
-    pub async fn handle_lifecycle_event(
-        &self,
-        lifecycle_event: LifecycleEvent,
-        shutdown_sender: Sender<RunningPhase>,
-    ) -> Result<()> {
-        info!("Handling lifecycle event: {:?}", lifecycle_event);
-        if let LifecycleEvent::Shutdown = lifecycle_event {
-            //1. Stop accepting new connections.
-            shutdown_sender.send(RunningPhase::ShutdownPending).ok();
-            tokio::time::sleep(Duration::from_millis(25)).await;
-
-            //2. Break out of work queues.
-            for worker in &*self.thread_workers {
-                worker.request_shutdown().await;
-            }
-
-            tokio::time::sleep(Duration::from_millis(25)).await;
-
-            //3. Wait for all threads to finish.
-            let deadline = Instant::now() + Duration::from_secs_f64(self.server.shutdown_timeout);
-            while Instant::now() < deadline {
-                let alive_threads = self
-                    .thread_workers
-                    .iter()
-                    .filter(|worker| worker.poll_shutdown(deadline))
-                    .count();
-                if alive_threads == 0 {
-                    break;
-                }
-                tokio::time::sleep(Duration::from_millis(200)).await;
-            }
-
-            //4. Force shutdown any stragglers
-            shutdown_sender.send(RunningPhase::Shutdown).ok();
-            self.thread_workers.iter().for_each(|worker| {
-                worker.poll_shutdown(deadline);
-            });
-
-            return Err(ItsiError::Break());
+    /// Attempts to reload the config "live"
+    /// Not that when running in single mode this will not unload
+    /// old code. If you need a clean restart, use the `restart` (SIGUSR2) method instead
+    pub fn reload(&self) -> Result<()> {
+        let should_reexec = self.server_config.clone().reload(false)?;
+        if should_reexec {
+            self.server_config.dup_fds()?;
+            self.server_config.reload_exec()?;
         }
+        self.restart_requested.store(true, Ordering::SeqCst);
+        self.stop()?;
+        self.server_config.server_params.read().preload_ruby()?;
+        Ok(())
+    }
+
+    /// Restart the server while keeping connections open.
+    pub fn restart(&self) -> Result<()> {
+        self.server_config.dup_fds()?;
+        self.server_config.reload_exec()?;
         Ok(())
     }
 }

data/crates/itsi_server/src/server/signal.rs

@@ -30,9 +30,18 @@ fn receive_signal(signum: i32, _: sighandler_t) {
                     .ok();
             }
         }
+        libc::SIGINFO => {
+            SIGNAL_HANDLER_CHANNEL
+                .0
+                .send(LifecycleEvent::PrintInfo)
+                .ok();
+        }
         libc::SIGUSR1 => {
             SIGNAL_HANDLER_CHANNEL.0.send(LifecycleEvent::Restart).ok();
         }
+        libc::SIGUSR2 => {
+            SIGNAL_HANDLER_CHANNEL.0.send(LifecycleEvent::Reload).ok();
+        }
         libc::SIGTTIN => {
             SIGNAL_HANDLER_CHANNEL
                 .0
@@ -45,6 +54,12 @@ fn receive_signal(signum: i32, _: sighandler_t) {
                 .send(LifecycleEvent::DecreaseWorkers)
                 .ok();
         }
+        libc::SIGCHLD => {
+            SIGNAL_HANDLER_CHANNEL
+                .0
+                .send(LifecycleEvent::ChildTerminated)
+                .ok();
+        }
         _ => {}
     }
 }
@@ -54,10 +69,12 @@ pub fn reset_signal_handlers() -> bool {
     unsafe {
         libc::signal(libc::SIGTERM, receive_signal as usize);
         libc::signal(libc::SIGINT, receive_signal as usize);
+        libc::signal(libc::SIGINFO, receive_signal as usize);
         libc::signal(libc::SIGUSR1, receive_signal as usize);
         libc::signal(libc::SIGUSR2, receive_signal as usize);
         libc::signal(libc::SIGTTIN, receive_signal as usize);
         libc::signal(libc::SIGTTOU, receive_signal as usize);
+        libc::signal(libc::SIGCHLD, receive_signal as usize);
     }
     true
 }
@@ -66,9 +83,11 @@ pub fn clear_signal_handlers() {
     unsafe {
         libc::signal(libc::SIGTERM, libc::SIG_DFL);
         libc::signal(libc::SIGINT, libc::SIG_DFL);
+        libc::signal(libc::SIGINFO, libc::SIG_DFL);
         libc::signal(libc::SIGUSR1, libc::SIG_DFL);
         libc::signal(libc::SIGUSR2, libc::SIG_DFL);
         libc::signal(libc::SIGTTIN, libc::SIG_DFL);
         libc::signal(libc::SIGTTOU, libc::SIG_DFL);
+        libc::signal(libc::SIGCHLD, libc::SIG_DFL);
     }
 }