itsi-scheduler 0.1.5 → 0.2.2

data/ext/itsi_server/src/server/serve_strategy/single_mode.rs

@@ -1,42 +1,53 @@
 use crate::{
-    request::itsi_request::ItsiRequest,
+    ruby_types::itsi_server::itsi_server_config::ItsiServerConfig,
     server::{
+        binds::listener::ListenerInfo,
         io_stream::IoStream,
-        itsi_server::{RequestJob, Server},
         lifecycle_event::LifecycleEvent,
-        listener::{Listener, TokioListener},
+        request_job::RequestJob,
+        signal::{SHUTDOWN_REQUESTED, SIGNAL_HANDLER_CHANNEL},
         thread_worker::{build_thread_workers, ThreadWorker},
     },
+    services::itsi_http_service::{ItsiHttpService, ItsiHttpServiceInner},
 };
-use http::Request;
-use hyper::{body::Incoming, service::service_fn};
 use hyper_util::{
     rt::{TokioExecutor, TokioIo, TokioTimer},
     server::conn::auto::Builder,
 };
 use itsi_error::{ItsiError, Result};
+use itsi_rb_helpers::{
+    call_with_gvl, call_without_gvl, create_ruby_thread, funcall_no_ret, print_rb_backtrace,
+};
 use itsi_tracing::{debug, error, info};
+use magnus::{value::ReprValue, Value};
 use nix::unistd::Pid;
+use parking_lot::RwLock;
 use std::{
-    num::NonZeroU8,
+    collections::HashMap,
     pin::Pin,
-    sync::Arc,
-    time::{Duration, Instant},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 use tokio::{
     runtime::{Builder as RuntimeBuilder, Runtime},
-    sync::broadcast,
+    sync::{
+        broadcast,
+        watch::{self},
+    },
     task::JoinSet,
 };
 use tracing::instrument;
 
 pub struct SingleMode {
     pub executor: Builder<TokioExecutor>,
-    pub server: Arc<Server>,
-    pub sender: async_channel::Sender<RequestJob>,
-    pub(crate) listeners: Arc<Vec<Arc<Listener>>>,
-    pub(crate) thread_workers: Arc<Vec<ThreadWorker>>,
+    pub server_config: Arc<ItsiServerConfig>,
     pub(crate) lifecycle_channel: broadcast::Sender<LifecycleEvent>,
+    pub restart_requested: AtomicBool,
+    pub status: RwLock<HashMap<u8, (u64, u64)>>,
 }
 
 pub enum RunningPhase {
@@ -46,202 +57,425 @@ pub enum RunningPhase {
 }
 
 impl SingleMode {
-    #[instrument(parent=None, skip_all, fields(pid=format!("{:?}", Pid::this())))]
-    pub(crate) fn new(
-        server: Arc<Server>,
-        listeners: Arc<Vec<Arc<Listener>>>,
-        lifecycle_channel: broadcast::Sender<LifecycleEvent>,
-    ) -> Result<Self> {
-        let (thread_workers, sender) = build_thread_workers(
-            Pid::this(),
-            NonZeroU8::try_from(server.threads).unwrap(),
-            server.app,
-            server.scheduler_class.clone(),
-        )?;
+    #[instrument(parent=None, skip_all)]
+    pub fn new(server_config: Arc<ItsiServerConfig>) -> Result<Self> {
+        server_config.server_params.read().preload_ruby()?;
         Ok(Self {
             executor: Builder::new(TokioExecutor::new()),
-            listeners,
-            server,
-            sender,
-            thread_workers,
-            lifecycle_channel,
+            server_config,
+            lifecycle_channel: SIGNAL_HANDLER_CHANNEL.0.clone(),
+            restart_requested: AtomicBool::new(false),
+            status: RwLock::new(HashMap::new()),
         })
     }
 
     pub fn build_runtime(&self) -> Runtime {
-        let mut builder: RuntimeBuilder = RuntimeBuilder::new_current_thread();
+        let mut builder: RuntimeBuilder = if self
+            .server_config
+            .server_params
+            .read()
+            .multithreaded_reactor
+        {
+            RuntimeBuilder::new_multi_thread()
+        } else {
+            RuntimeBuilder::new_current_thread()
+        };
         builder
             .thread_name("itsi-server-accept-loop")
             .thread_stack_size(3 * 1024 * 1024)
-            .enable_io()
-            .enable_time()
+            .enable_all()
             .build()
             .expect("Failed to build Tokio runtime")
     }
 
     pub fn stop(&self) -> Result<()> {
+        SHUTDOWN_REQUESTED.store(true, std::sync::atomic::Ordering::SeqCst);
+        self.lifecycle_channel.send(LifecycleEvent::Shutdown).ok();
+        Ok(())
+    }
+
+    pub async fn print_info(&self, thread_workers: Arc<Vec<Arc<ThreadWorker>>>) -> Result<()> {
+        println!(" └─ Worker");
+        println!(
+            "    - binds: {:?}",
+            self.server_config.server_params.read().binds
+        );
+
+        println!(
+            "    ─ streaming body: {:?}",
+            self.server_config.server_params.read().streamable_body
+        );
+        println!(
+            "    ─ multithreaded runtime: {:?}",
+            self.server_config
+                .server_params
+                .read()
+                .multithreaded_reactor
+        );
+        println!(
+            "    ─ scheduler: {:?}",
+            self.server_config.server_params.read().scheduler_class
+        );
+        println!(
+            "    ─ OOB GC Response threadhold: {:?}",
+            self.server_config
+                .server_params
+                .read()
+                .oob_gc_responses_threshold
+        );
+        for worker in thread_workers.iter() {
+            println!("   └─ - Thread : {:?}", worker.id);
+            println!("       - # Requests Processed: {:?}", worker.request_id);
+            println!(
+                "       - Last Request Started: {:?} ago",
+                if worker.current_request_start.load(Ordering::Relaxed) == 0 {
+                    Duration::from_secs(0)
+                } else {
+                    SystemTime::now()
+                        .duration_since(
+                            UNIX_EPOCH
+                                + Duration::from_secs(
+                                    worker.current_request_start.load(Ordering::Relaxed),
+                                ),
+                        )
+                        .unwrap_or(Duration::from_secs(0))
+                }
+            );
+            call_with_gvl(|_| {
+                if let Some(thread) = worker.thread.read().as_ref() {
+                    if let Ok(backtrace) = thread.funcall::<_, _, Vec<String>>("backtrace", ()) {
+                        println!("       - Backtrace:");
+                        for line in backtrace {
+                            println!("       -   {}", line);
+                        }
+                    }
+                }
+            })
+        }
+
         Ok(())
     }
 
-    #[instrument(parent=None, skip(self))]
+    pub fn start_monitors(
+        self: Arc<Self>,
+        thread_workers: Arc<Vec<Arc<ThreadWorker>>>,
+    ) -> Option<magnus::Thread> {
+        call_with_gvl(move |_| {
+            create_ruby_thread(move || {
+                call_without_gvl(move || {
+                    let monitor_runtime = RuntimeBuilder::new_current_thread()
+                        .enable_all()
+                        .build()
+                        .unwrap();
+                    let receiver = self.clone();
+                    monitor_runtime.block_on({
+                        let mut lifecycle_rx = receiver.lifecycle_channel.subscribe();
+                        let receiver = receiver.clone();
+                        let thread_workers = thread_workers.clone();
+                        async move {
+                            loop {
+                                tokio::select! {
+                                  _ = tokio::time::sleep(Duration::from_secs(1)) => {
+                                    let mut status_lock = receiver.status.write();
+                                    thread_workers.iter().for_each(|worker| {
+                                        let worker_entry = status_lock.entry(worker.id);
+                                        let data = (
+                                            worker.request_id.load(Ordering::Relaxed),
+                                            worker.current_request_start.load(Ordering::Relaxed),
+                                        );
+                                        worker_entry.or_insert(data);
+                                    });
+                                  }
+                                  lifecycle_event = lifecycle_rx.recv() => {
+                                      match lifecycle_event {
+                                          Ok(LifecycleEvent::Restart) => {
+                                              receiver.restart().await.ok();
+                                          }
+                                          Ok(LifecycleEvent::Reload) => {
+                                              receiver.reload().await.ok();
+                                          }
+                                          Ok(LifecycleEvent::Shutdown) => {
+                                            break;
+                                          }
+                                          Ok(LifecycleEvent::PrintInfo) => {
+                                            receiver.print_info(thread_workers.clone()).await.ok();
+                                          }
+                                          _ => {}
+                                      }
+                                  }
+                                }
+                            }
+                        }
+                    })
+                })
+            })
+        })
+    }
+
+    #[instrument(name="worker", parent=None, skip(self), fields(pid=format!("{}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
         let mut listener_task_set = JoinSet::new();
-        let self_ref = Arc::new(self);
-        self_ref.build_runtime().block_on(async {
-
-          for listener in self_ref.listeners.clone().iter() {
-              let listener = Arc::new(listener.to_tokio_listener());
-              let mut lifecycle_rx = self_ref.lifecycle_channel.subscribe();
-              let self_ref = self_ref.clone();
-              let listener = listener.clone();
-              let (shutdown_sender, mut shutdown_receiver) = tokio::sync::watch::channel::<RunningPhase>(RunningPhase::Running);
-              let listener_clone = listener.clone();
-
-              tokio::spawn(async move {
-                listener_clone.spawn_state_task().await;
-              });
-
-              listener_task_set.spawn(async move {
-                let strategy = self_ref.clone();
-                loop {
-                    tokio::select! {
-                        accept_result = listener.accept() => match accept_result {
-                          Ok(accept_result) => {
-                            if let Err(e) = strategy.serve_connection(accept_result, listener.clone(), shutdown_receiver.clone()).await {
-                              error!("Error in serve_connection {:?}", e)
+        let runtime = self.build_runtime();
+
+        let (thread_workers, job_sender, nonblocking_sender) =
+            build_thread_workers(self.server_config.server_params.read().clone(), Pid::this())
+                .inspect_err(|e| {
+                    if let Some(err_val) = e.value() {
+                        print_rb_backtrace(err_val);
+                    }
+                })?;
+
+        info!(
+            threads = thread_workers.len(),
+            binds = format!("{:?}", self.server_config.server_params.read().binds)
+        );
+
+        let (shutdown_sender, _) = watch::channel(RunningPhase::Running);
+        let monitor_thread = self.clone().start_monitors(thread_workers.clone());
+        if monitor_thread.is_none() {
+            error!("Failed to start monitor thread");
+            return Err(ItsiError::new("Failed to start monitor thread"));
+        }
+        let monitor_thread = monitor_thread.unwrap();
+        if SHUTDOWN_REQUESTED.load(Ordering::SeqCst) {
+            return Ok(());
+        }
+        let result = runtime.block_on(
+          async  {
+              let server_params = self.server_config.server_params.read().clone();
+              if let Err(err) = server_params.initialize_middleware().await {
+                  error!("Failed to initialize middleware: {}", err);
+                  return Err(ItsiError::new("Failed to initialize middleware"))
+              }
+              let tokio_listeners = server_params.listeners.lock()
+                  .drain(..)
+                  .map(|list| {
+                    Arc::new(list.into_tokio_listener())
+                  })
+                  .collect::<Vec<_>>();
+
+              for listener in tokio_listeners.iter() {
+                  let mut lifecycle_rx = self.lifecycle_channel.subscribe();
+
+                  let listener_info = Arc::new(listener.listener_info());
+                  let self_ref = self.clone();
+                  let listener = listener.clone();
+                  let shutdown_sender = shutdown_sender.clone();
+                  let job_sender = job_sender.clone();
+                  let nonblocking_sender = nonblocking_sender.clone();
+                  let workers_clone = thread_workers.clone();
+                  let listener_clone = listener.clone();
+                  let mut shutdown_receiver = shutdown_sender.subscribe();
+                  let shutdown_receiver_clone = shutdown_receiver.clone();
+                  listener_task_set.spawn(async move {
+                    listener_clone.spawn_state_task(shutdown_receiver_clone).await;
+                  });
+
+                  listener_task_set.spawn(async move {
+                    let strategy_clone = self_ref.clone();
+                    let mut acceptor_task_set = JoinSet::new();
+                    loop {
+                        tokio::select! {
+                            accept_result = listener.accept() => match accept_result {
+                              Ok(accept_result) => {
+                                let strategy = strategy_clone.clone();
+                                let listener_info = listener_info.clone();
+                                let shutdown_receiver = shutdown_receiver.clone();
+                                let job_sender = job_sender.clone();
+                                let nonblocking_sender = nonblocking_sender.clone();
+                                acceptor_task_set.spawn(async move {
+                                  strategy.serve_connection(accept_result, job_sender, nonblocking_sender, listener_info, shutdown_receiver).await;
+                                });
+                              },
+                              Err(e) => debug!("Listener.accept failed {:?}", e),
+                            },
+                            _ = shutdown_receiver.changed() => {
+                              break;
                             }
-                          },
-                          Err(e) => debug!("Listener.accept failed {:?}", e),
-                        },
-                        _ = shutdown_receiver.changed() => {
-                          break;
+                            lifecycle_event = lifecycle_rx.recv() => match lifecycle_event{
+                              Ok(LifecycleEvent::Shutdown) => {
+                                debug!("Received lifecycle event: {:?}", lifecycle_event);
+                                shutdown_sender.send(RunningPhase::ShutdownPending).unwrap();
+                                tokio::time::sleep(Duration::from_millis(25)).await;
+                                for _i in 0..workers_clone.len() {
+                                  job_sender.send(RequestJob::Shutdown).await.unwrap();
+                                  nonblocking_sender.send(RequestJob::Shutdown).await.unwrap();
+                                }
+                                break;
+                              },
+                              Err(e) => error!("Error receiving lifecycle_event: {:?}", e),
+                              _ => {}
+                          }
                         }
-                        lifecycle_event = lifecycle_rx.recv() => match lifecycle_event{
-                          Ok(lifecycle_event) => {
-                            if let Err(e) = strategy.handle_lifecycle_event(lifecycle_event, shutdown_sender.clone()).await{
-                              match e {
-                                ItsiError::Break() => break,
-                                _ => error!("Error in handle_lifecycle_event {:?}", e)
-                              }
-                            }
+                    }
 
-                          },
-                          Err(e) => error!("Error receiving lifecycle_event: {:?}", e),
-                      }
+                    let deadline = Instant::now()
+                        + Duration::from_secs_f64(self_ref.server_config.server_params.read().shutdown_timeout);
+                    tokio::select! {
+                        _ = async {
+                            while let Some(_res) = acceptor_task_set.join_next().await {}
+                        } => {},
+                        _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => {},
                     }
-                }
-                if let Ok(listener) = Arc::try_unwrap(listener){
-                  listener.unbind();
-                }
-            });
+                });
 
-          }
+              }
 
-          while let Some(_res) = listener_task_set.join_next().await {}
-        });
+              if self.is_single_mode() {
+                self.invoke_hook("after_start");
+              }
 
-        Ok(())
+              while let Some(_res) = listener_task_set.join_next().await {}
+
+              // Explicitly drop all listeners to ensure file descriptors are released
+              drop(tokio_listeners);
+
+              Ok::<(), ItsiError>(())
+          });
+
+        debug!("Single mode runtime exited.");
+
+        if result.is_err() {
+            for _i in 0..thread_workers.len() {
+                job_sender.send_blocking(RequestJob::Shutdown).unwrap();
+                nonblocking_sender
+                    .send_blocking(RequestJob::Shutdown)
+                    .unwrap();
+            }
+            self.lifecycle_channel.send(LifecycleEvent::Shutdown).ok();
+        }
+
+        shutdown_sender.send(RunningPhase::Shutdown).ok();
+        let deadline = Instant::now()
+            + Duration::from_secs_f64(self.server_config.server_params.read().shutdown_timeout);
+
+        runtime.shutdown_timeout(Duration::from_millis(100));
+
+        debug!("Shutdown timeout finished.");
+        loop {
+            if thread_workers
+                .iter()
+                .all(|worker| call_with_gvl(move |_| !worker.poll_shutdown(deadline)))
+            {
+                funcall_no_ret(monitor_thread, "join", ()).ok();
+                break;
+            }
+            sleep(Duration::from_millis(50));
+        }
+
+        if self.is_single_mode() {
+            self.invoke_hook("before_shutdown");
+        }
+
+        if self.restart_requested.load(Ordering::SeqCst) {
+            self.restart_requested.store(false, Ordering::SeqCst);
+            info!("Worker restarting");
+            self.run()?;
+        }
+
+        debug!("Runtime has shut down");
+        result
+    }
+
+    pub fn is_single_mode(&self) -> bool {
+        self.server_config.server_params.read().workers == 1
     }
 
     pub(crate) async fn serve_connection(
         &self,
         stream: IoStream,
-        listener: Arc<TokioListener>,
-        shutdown_channel: tokio::sync::watch::Receiver<RunningPhase>,
-    ) -> Result<()> {
-        let sender_clone = self.sender.clone();
+        job_sender: async_channel::Sender<RequestJob>,
+        nonblocking_sender: async_channel::Sender<RequestJob>,
+        listener: Arc<ListenerInfo>,
+        shutdown_channel: watch::Receiver<RunningPhase>,
+    ) {
         let addr = stream.addr();
         let io: TokioIo<Pin<Box<IoStream>>> = TokioIo::new(Box::pin(stream));
-        let server = self.server.clone();
         let executor = self.executor.clone();
         let mut shutdown_channel_clone = shutdown_channel.clone();
-        tokio::spawn(async move {
-            let server = server.clone();
-            let mut executor = executor.clone();
-            let mut binding = executor.http1();
-            let shutdown_channel = shutdown_channel_clone.clone();
-            let mut serve = Box::pin(
-                binding
-                    .timer(TokioTimer::new())
-                    .header_read_timeout(Duration::from_secs(1))
-                    .serve_connection_with_upgrades(
-                        io,
-                        service_fn(move |hyper_request: Request<Incoming>| {
-                            ItsiRequest::process_request(
-                                hyper_request,
-                                sender_clone.clone(),
-                                server.clone(),
-                                listener.clone(),
-                                addr.clone(),
-                                shutdown_channel.clone(),
-                            )
-                        }),
-                    ),
-            );
+        let mut executor = executor.clone();
+        let mut binding = executor.http1();
+        let shutdown_channel = shutdown_channel_clone.clone();
 
-            tokio::select! {
-                // Await the connection finishing naturally.
-                res = &mut serve => {
-                    match res{
-                        Ok(()) => {
-                          debug!("Connection closed normally")
-                        },
-                        Err(res) => {
-                          debug!("Connection finished with error: {:?}", res)
-                        }
-                    }
-                    serve.as_mut().graceful_shutdown();
-                },
-                // A lifecycle event triggers shutdown.
-                _ = shutdown_channel_clone.changed() => {
-                    // Initiate graceful shutdown.
-                    serve.as_mut().graceful_shutdown();
-                    // Now await the connection to finish shutting down.
-                    if let Err(e) = serve.await {
-                        debug!("Connection shutdown error: {:?}", e);
+        let service = ItsiHttpService {
+            inner: Arc::new(ItsiHttpServiceInner {
+                sender: job_sender.clone(),
+                nonblocking_sender: nonblocking_sender.clone(),
+                server_params: self.server_config.server_params.read().clone(),
+                listener,
+                addr: addr.to_string(),
+                shutdown_channel: shutdown_channel.clone(),
+            }),
+        };
+        let mut serve = Box::pin(
+            binding
+                .timer(TokioTimer::new())
+                .header_read_timeout(self.server_config.server_params.read().header_read_timeout)
+                .serve_connection_with_upgrades(io, service),
+        );
+
+        tokio::select! {
+            // Await the connection finishing naturally.
+            res = &mut serve => {
+                match res{
+                    Ok(()) => {
+                      debug!("Connection closed normally")
+                    },
+                    Err(res) => {
+                      debug!("Connection closed abruptly: {:?}", res)
                     }
                 }
-            }
-        });
-        Ok(())
-    }
+                serve.as_mut().graceful_shutdown();
+            },
+            // A lifecycle event triggers shutdown.
+            _ = shutdown_channel_clone.changed() => {
+                // Initiate graceful shutdown.
+                serve.as_mut().graceful_shutdown();
 
-    pub async fn handle_lifecycle_event(
-        &self,
-        lifecycle_event: LifecycleEvent,
-        shutdown_sender: tokio::sync::watch::Sender<RunningPhase>,
-    ) -> Result<()> {
-        if let LifecycleEvent::Shutdown = lifecycle_event {
-            shutdown_sender
-                .send(RunningPhase::ShutdownPending)
-                .expect("Failed to send shutdown pending signal");
-            let deadline = Instant::now() + Duration::from_secs_f64(self.server.shutdown_timeout);
-            for worker in &*self.thread_workers {
-                worker.request_shutdown().await;
-            }
-            while Instant::now() < deadline {
-                tokio::time::sleep(Duration::from_millis(50)).await;
-                let alive_threads = self
-                    .thread_workers
-                    .iter()
-                    .filter(|worker| worker.poll_shutdown(deadline))
-                    .count();
-                if alive_threads == 0 {
-                    break;
+                // Now await the connection to finish shutting down.
+                if let Err(e) = serve.await {
+                    debug!("Connection shutdown error: {:?}", e);
                 }
-                tokio::time::sleep(Duration::from_millis(200)).await;
             }
+        }
+    }
 
-            info!("Sending shutdown signal");
-            shutdown_sender
-                .send(RunningPhase::Shutdown)
-                .expect("Failed to send shutdown signal");
-            self.thread_workers.iter().for_each(|worker| {
-                worker.poll_shutdown(deadline);
-            });
+    /// Attempts to reload the config "live"
+    /// Not that when running in single mode this will not unload
+    /// old code. If you need a clean restart, use the `restart` (SIGHUP) method instead
+    pub async fn reload(&self) -> Result<()> {
+        if !self.server_config.check_config().await {
+            return Ok(());
+        }
+        let should_reexec = self.server_config.clone().reload(false)?;
+        if should_reexec {
+            if self.is_single_mode() {
+                self.invoke_hook("before_restart");
+            }
+            self.server_config.dup_fds()?;
+            self.server_config.reload_exec()?;
+        }
+        self.restart_requested.store(true, Ordering::SeqCst);
+        self.stop()?;
+        self.server_config.server_params.read().preload_ruby()?;
+        Ok(())
+    }
 
-            return Err(ItsiError::Break());
+    pub fn invoke_hook(&self, hook_name: &str) {
+        if let Some(hook) = self.server_config.server_params.read().hooks.get(hook_name) {
+            call_with_gvl(|_| hook.call::<_, Value>(()).ok());
+        }
+    }
+    /// Restart the server while keeping connections open.
+    pub async fn restart(&self) -> Result<()> {
+        if !self.server_config.check_config().await {
+            return Ok(());
+        }
+        if self.is_single_mode() {
+            self.invoke_hook("before_restart");
         }
+        self.server_config.dup_fds()?;
+        self.server_config.reload_exec()?;
         Ok(())
     }
 }