itsi 0.1.9 → 0.1.11

data/crates/itsi_server/src/server/serve_strategy/cluster_mode.rs

@@ -3,8 +3,11 @@ use crate::server::{
     process_worker::ProcessWorker,
 };
 use itsi_error::{ItsiError, Result};
-use itsi_rb_helpers::{call_without_gvl, create_ruby_thread};
+use itsi_rb_helpers::{
+    call_proc_and_log_errors, call_with_gvl, call_without_gvl, create_ruby_thread,
+};
 use itsi_tracing::{error, info, warn};
+use magnus::Value;
 use nix::{
     libc::{self, exit},
     unistd::Pid,
@@ -19,9 +22,9 @@ use tokio::{
     sync::{broadcast, watch, Mutex},
     time::{self, sleep},
 };
-use tracing::instrument;
+use tracing::{debug, instrument};
 pub(crate) struct ClusterMode {
-    pub listeners: Arc<Vec<Arc<Listener>>>,
+    pub listeners: parking_lot::Mutex<Vec<Listener>>,
     pub server: Arc<Server>,
     pub process_workers: parking_lot::Mutex<Vec<ProcessWorker>>,
     pub lifecycle_channel: broadcast::Sender<LifecycleEvent>,
@@ -34,12 +37,9 @@ static CHILD_SIGNAL_SENDER: parking_lot::Mutex<Option<watch::Sender<()>>> =
 impl ClusterMode {
     pub fn new(
         server: Arc<Server>,
-        listeners: Arc<Vec<Arc<Listener>>>,
+        listeners: Vec<Listener>,
         lifecycle_channel: broadcast::Sender<LifecycleEvent>,
     ) -> Self {
-        if let Some(f) = server.before_fork.lock().take() {
-            f();
-        }
         let process_workers = (0..server.workers)
             .map(|_| ProcessWorker {
                 worker_id: WORKER_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
@@ -48,7 +48,7 @@ impl ClusterMode {
             .collect();
 
         Self {
-            listeners,
+            listeners: parking_lot::Mutex::new(listeners),
             server,
             process_workers: parking_lot::Mutex::new(process_workers),
             lifecycle_channel,
@@ -152,7 +152,7 @@ impl ClusterMode {
 
         tokio::select! {
             _ = monitor_handle => {
-              info!("All children exited early, exit normally")
+              debug!("All children exited early, exit normally")
             }
             _ = sleep(Duration::from_secs_f64(shutdown_timeout)) => {
                 warn!("Graceful shutdown timeout reached, force killing remaining children");
@@ -191,6 +191,9 @@ impl ClusterMode {
     #[instrument(skip(self), fields(mode = "cluster", pid=format!("{:?}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
         info!("Starting in Cluster mode");
+        if let Some(proc) = self.server.hooks.get("before_fork") {
+            call_with_gvl(|_| call_proc_and_log_errors(proc.clone()))
+        }
         self.process_workers
             .lock()
             .iter()
@@ -228,6 +231,9 @@ impl ClusterMode {
                     if let Some(current_mem_usage) = largest_worker.memory_usage(){
                       if current_mem_usage > memory_limit {
                         largest_worker.reboot(self_ref.clone()).await.ok();
+                        if let Some(hook) = self_ref.server.hooks.get("after_memory_threshold_reached") {
+                          call_with_gvl(|_|  hook.call::<_, Value>((largest_worker.pid(),)).ok() );
+                        }
                       }
                     }
                   }

data/crates/itsi_server/src/server/serve_strategy/single_mode.rs

@@ -15,10 +15,13 @@ use hyper_util::{
     server::conn::auto::Builder,
 };
 use itsi_error::{ItsiError, Result};
+use itsi_rb_helpers::print_rb_backtrace;
 use itsi_tracing::{debug, error, info};
 use nix::unistd::Pid;
+use parking_lot::Mutex;
 use std::{
     num::NonZeroU8,
+    panic,
     pin::Pin,
     sync::Arc,
     time::{Duration, Instant},
@@ -37,7 +40,7 @@ pub struct SingleMode {
     pub executor: Builder<TokioExecutor>,
     pub server: Arc<Server>,
     pub sender: async_channel::Sender<RequestJob>,
-    pub(crate) listeners: Arc<Vec<Arc<Listener>>>,
+    pub(crate) listeners: Mutex<Vec<Listener>>,
     pub(crate) thread_workers: Arc<Vec<ThreadWorker>>,
     pub(crate) lifecycle_channel: broadcast::Sender<LifecycleEvent>,
 }
@@ -52,18 +55,24 @@ impl SingleMode {
     #[instrument(parent=None, skip_all, fields(pid=format!("{:?}", Pid::this())))]
     pub(crate) fn new(
         server: Arc<Server>,
-        listeners: Arc<Vec<Arc<Listener>>>,
+        listeners: Vec<Listener>,
         lifecycle_channel: broadcast::Sender<LifecycleEvent>,
     ) -> Result<Self> {
         let (thread_workers, sender) = build_thread_workers(
+            server.clone(),
             Pid::this(),
             NonZeroU8::try_from(server.threads).unwrap(),
             server.app.clone(),
             server.scheduler_class.clone(),
-        )?;
+        )
+        .inspect_err(|e| {
+            if let Some(err_val) = e.value() {
+                print_rb_backtrace(err_val);
+            }
+        })?;
         Ok(Self {
             executor: Builder::new(TokioExecutor::new()),
-            listeners,
+            listeners: Mutex::new(listeners),
             server,
             sender,
             thread_workers,
@@ -83,82 +92,81 @@ impl SingleMode {
     }
 
     pub fn stop(&self) -> Result<()> {
-        self.lifecycle_channel
-            .send(LifecycleEvent::Shutdown)
-            .expect("Failed to send shutdown event");
+        self.lifecycle_channel.send(LifecycleEvent::Shutdown).ok();
         Ok(())
     }
 
     #[instrument(parent=None, skip(self), fields(pid=format!("{}", Pid::this())))]
     pub fn run(self: Arc<Self>) -> Result<()> {
         let mut listener_task_set = JoinSet::new();
-        let self_ref = Arc::new(self);
-        let runtime = self_ref.build_runtime();
+        let runtime = self.build_runtime();
 
         runtime.block_on(async {
-          let tokio_listeners = self_ref
-              .listeners
-              .iter()
-              .map(|list| Arc::new(list.to_tokio_listener()))
-              .collect::<Vec<_>>();
-          let (shutdown_sender, _) = watch::channel::<RunningPhase>(RunningPhase::Running);
-          for listener in tokio_listeners.iter() {
-              let mut lifecycle_rx = self_ref.lifecycle_channel.subscribe();
-              let listener_info = Arc::new(listener.listener_info());
-              let self_ref = self_ref.clone();
-              let listener = listener.clone();
-              let shutdown_sender = shutdown_sender.clone();
-
+              let tokio_listeners = self
+                  .listeners.lock()
+                  .drain(..)
+                  .map(|list| {
+                    Arc::new(list.into_tokio_listener())
+                  })
+                  .collect::<Vec<_>>();
+              let (shutdown_sender, _) = watch::channel(RunningPhase::Running);
+              for listener in tokio_listeners.iter() {
+                  let mut lifecycle_rx = self.lifecycle_channel.subscribe();
+                  let listener_info = Arc::new(listener.listener_info());
+                  let self_ref = self.clone();
+                  let listener = listener.clone();
+                  let shutdown_sender = shutdown_sender.clone();
 
-              let listener_clone = listener.clone();
-              let mut shutdown_receiver = shutdown_sender.clone().subscribe();
-              let shutdown_receiver_clone = shutdown_receiver.clone();
-              listener_task_set.spawn(async move {
-                listener_clone.spawn_state_task(shutdown_receiver_clone).await;
-              });
+                  let listener_clone = listener.clone();
+                  let mut shutdown_receiver = shutdown_sender.subscribe();
+                  let shutdown_receiver_clone = shutdown_receiver.clone();
+                  listener_task_set.spawn(async move {
+                    listener_clone.spawn_state_task(shutdown_receiver_clone).await;
+                  });
 
-              listener_task_set.spawn(async move {
-                let strategy = self_ref.clone();
-                loop {
-                    tokio::select! {
-                        accept_result = listener.accept() => match accept_result {
-                          Ok(accept_result) => {
-                            match strategy.serve_connection(accept_result, listener_info.clone(), shutdown_receiver.clone()).await {
-                              Ok(_) => {
-                                debug!("Connection accepted and served");
+                  listener_task_set.spawn(async move {
+                    let strategy_clone = self_ref.clone();
+                    let mut acceptor_task_set = JoinSet::new();
+                    loop {
+                        tokio::select! {
+                            accept_result = listener.accept() => match accept_result {
+                              Ok(accept_result) => {
+                                let strategy = strategy_clone.clone();
+                                let listener_info = listener_info.clone();
+                                let shutdown_receiver = shutdown_receiver.clone();
+                                acceptor_task_set.spawn(async move {
+                                  strategy.serve_connection(accept_result, listener_info, shutdown_receiver).await;
+                                });
                               },
-                              Err(e) => error!("Error in serve_connection {:?}", e)
-                            }
-                          },
-                          Err(e) => debug!("Listener.accept failed {:?}", e),
-                        },
-                        _ = shutdown_receiver.changed() => {
-                          break;
-                        }
-                        lifecycle_event = lifecycle_rx.recv() => match lifecycle_event{
-                          Ok(lifecycle_event) => {
-                            if let Err(e) = strategy.handle_lifecycle_event(lifecycle_event, shutdown_sender.clone()).await{
-                              match e {
-                                ItsiError::Break() => break,
-                                _ => error!("Error in handle_lifecycle_event {:?}", e)
-                              }
+                              Err(e) => debug!("Listener.accept failed {:?}", e),
+                            },
+                            _ = shutdown_receiver.changed() => {
+                              break;
                             }
+                            lifecycle_event = lifecycle_rx.recv() => match lifecycle_event{
+                              Ok(lifecycle_event) => {
+                                if let Err(e) = self_ref.handle_lifecycle_event(lifecycle_event, shutdown_sender.clone()).await{
+                                  match e {
+                                    ItsiError::Break() => break,
+                                    _ => error!("Error in handle_lifecycle_event {:?}", e)
+                                  }
+                                }
 
-                          },
-                          Err(e) => error!("Error receiving lifecycle_event: {:?}", e),
-                      }
+                              },
+                              Err(e) => error!("Error receiving lifecycle_event: {:?}", e),
+                          }
+                        }
                     }
-                }
-            });
+                    while let Some(_res) = acceptor_task_set.join_next().await {}
+                });
 
-          }
+              }
 
-          while let Some(_res) = listener_task_set.join_next().await {}
+              while let Some(_res) = listener_task_set.join_next().await {}
 
-        });
+            });
         runtime.shutdown_timeout(Duration::from_millis(100));
-
-        info!("Runtime has shut down");
+        debug!("Runtime has shut down");
         Ok(())
     }
 
@@ -167,62 +175,60 @@ impl SingleMode {
         stream: IoStream,
         listener: Arc<ListenerInfo>,
         shutdown_channel: watch::Receiver<RunningPhase>,
-    ) -> Result<()> {
+    ) {
         let sender_clone = self.sender.clone();
         let addr = stream.addr();
         let io: TokioIo<Pin<Box<IoStream>>> = TokioIo::new(Box::pin(stream));
         let server = self.server.clone();
         let executor = self.executor.clone();
         let mut shutdown_channel_clone = shutdown_channel.clone();
-        tokio::spawn(async move {
-            let server = server.clone();
-            let mut executor = executor.clone();
-            let mut binding = executor.http1();
-            let shutdown_channel = shutdown_channel_clone.clone();
-            let mut serve = Box::pin(
-                binding
-                    .timer(TokioTimer::new())
-                    .header_read_timeout(Duration::from_secs(1))
-                    .serve_connection_with_upgrades(
-                        io,
-                        service_fn(move |hyper_request: Request<Incoming>| {
-                            ItsiRequest::process_request(
-                                hyper_request,
-                                sender_clone.clone(),
-                                server.clone(),
-                                listener.clone(),
-                                addr.clone(),
-                                shutdown_channel.clone(),
-                            )
-                        }),
-                    ),
-            );
+        let server = server.clone();
+        let mut executor = executor.clone();
+        let mut binding = executor.http1();
+        let shutdown_channel = shutdown_channel_clone.clone();
+        let mut serve = Box::pin(
+            binding
+                .timer(TokioTimer::new())
+                .header_read_timeout(Duration::from_secs(1))
+                .serve_connection_with_upgrades(
+                    io,
+                    service_fn(move |hyper_request: Request<Incoming>| {
+                        ItsiRequest::process_request(
+                            hyper_request,
+                            sender_clone.clone(),
+                            server.clone(),
+                            listener.clone(),
+                            addr.clone(),
+                            shutdown_channel.clone(),
+                        )
+                    }),
+                ),
+        );
 
-            tokio::select! {
-                // Await the connection finishing naturally.
-                res = &mut serve => {
-                    match res{
-                        Ok(()) => {
-                          debug!("Connection closed normally")
-                        },
-                        Err(res) => {
-                          debug!("Connection finished with error: {:?}", res)
-                        }
-                    }
-                    serve.as_mut().graceful_shutdown();
-                },
-                // A lifecycle event triggers shutdown.
-                _ = shutdown_channel_clone.changed() => {
-                    // Initiate graceful shutdown.
-                    serve.as_mut().graceful_shutdown();
-                    // Now await the connection to finish shutting down.
-                    if let Err(e) = serve.await {
-                        debug!("Connection shutdown error: {:?}", e);
+        tokio::select! {
+            // Await the connection finishing naturally.
+            res = &mut serve => {
+                match res{
+                    Ok(()) => {
+                      debug!("Connection closed normally")
+                    },
+                    Err(res) => {
+                      debug!("Connection finished with error: {:?}", res)
                     }
                 }
+                serve.as_mut().graceful_shutdown();
+            },
+            // A lifecycle event triggers shutdown.
+            _ = shutdown_channel_clone.changed() => {
+                // Initiate graceful shutdown.
+                serve.as_mut().graceful_shutdown();
+
+                // Now await the connection to finish shutting down.
+                if let Err(e) = serve.await {
+                    debug!("Connection shutdown error: {:?}", e);
+                }
             }
-        });
-        Ok(())
+        }
     }
 
     pub async fn handle_lifecycle_event(
@@ -232,13 +238,20 @@ impl SingleMode {
     ) -> Result<()> {
         info!("Handling lifecycle event: {:?}", lifecycle_event);
         if let LifecycleEvent::Shutdown = lifecycle_event {
+            //1. Stop accepting new connections.
             shutdown_sender.send(RunningPhase::ShutdownPending).ok();
-            let deadline = Instant::now() + Duration::from_secs_f64(self.server.shutdown_timeout);
+            tokio::time::sleep(Duration::from_millis(25)).await;
+
+            //2. Break out of work queues.
             for worker in &*self.thread_workers {
                 worker.request_shutdown().await;
             }
+
+            tokio::time::sleep(Duration::from_millis(25)).await;
+
+            //3. Wait for all threads to finish.
+            let deadline = Instant::now() + Duration::from_secs_f64(self.server.shutdown_timeout);
             while Instant::now() < deadline {
-                tokio::time::sleep(Duration::from_millis(50)).await;
                 let alive_threads = self
                     .thread_workers
                     .iter()
@@ -250,7 +263,7 @@ impl SingleMode {
                 tokio::time::sleep(Duration::from_millis(200)).await;
             }
 
-            info!("Sending shutdown signal");
+            //4. Force shutdown any stragglers
             shutdown_sender.send(RunningPhase::Shutdown).ok();
             self.thread_workers.iter().for_each(|worker| {
                 worker.poll_shutdown(deadline);

data/crates/itsi_server/src/server/signal.rs

@@ -11,10 +11,7 @@ pub static SIGNAL_HANDLER_CHANNEL: LazyLock<(
 )> = LazyLock::new(|| sync::broadcast::channel(5));
 
 pub fn send_shutdown_event() {
-    SIGNAL_HANDLER_CHANNEL
-        .0
-        .send(LifecycleEvent::Shutdown)
-        .expect("Failed to send shutdown event");
+    SIGNAL_HANDLER_CHANNEL.0.send(LifecycleEvent::Shutdown).ok();
 }
 
 pub static SIGINT_COUNT: AtomicI8 = AtomicI8::new(0);

data/crates/itsi_server/src/server/thread_worker.rs

@@ -1,4 +1,4 @@
-use super::itsi_server::RequestJob;
+use super::itsi_server::{RequestJob, Server};
 use crate::{request::itsi_request::ItsiRequest, ITSI_SERVER};
 use itsi_rb_helpers::{
     call_with_gvl, call_without_gvl, create_ruby_thread, kill_threads, HeapVal, HeapValue,
@@ -24,6 +24,7 @@ use std::{
 use tokio::{runtime::Builder as RuntimeBuilder, sync::watch};
 use tracing::instrument;
 pub struct ThreadWorker {
+    pub server: Arc<Server>,
     pub id: String,
     pub app: Opaque<Value>,
     pub receiver: Arc<async_channel::Receiver<RequestJob>>,
@@ -48,8 +49,9 @@ static CLASS_FIBER: Lazy<RClass> = Lazy::new(|ruby| {
 
 pub struct TerminateWakerSignal(bool);
 
-#[instrument(name = "Boot", parent=None, skip(threads, app, pid, scheduler_class))]
+#[instrument(name = "Boot", parent=None, skip(server, threads, app, pid, scheduler_class))]
 pub fn build_thread_workers(
+    server: Arc<Server>,
     pid: Pid,
     threads: NonZeroU8,
     app: HeapVal,
@@ -65,6 +67,7 @@ pub fn build_thread_workers(
                 .map(|id| {
                     info!(pid = pid.as_raw(), id, "Thread");
                     ThreadWorker::new(
+                        server.clone(),
                         format!("{:?}#{:?}", pid, id),
                         app,
                         receiver_ref.clone(),
@@ -83,10 +86,7 @@ pub fn load_app(
     scheduler_class: Option<String>,
 ) -> Result<(Opaque<Value>, Option<Opaque<Value>>)> {
     call_with_gvl(|ruby| {
-        let app = Opaque::from(
-            app.funcall::<_, _, Value>(*ID_CALL, ())
-                .expect("Couldn't load app"),
-        );
+        let app = Opaque::from(app.funcall::<_, _, Value>(*ID_CALL, ())?);
         let scheduler_class = if let Some(scheduler_class) = scheduler_class {
             Some(Opaque::from(
                 ruby.module_kernel()
@@ -100,6 +100,7 @@ pub fn load_app(
 }
 impl ThreadWorker {
     pub fn new(
+        server: Arc<Server>,
         id: String,
         app: Opaque<Value>,
         receiver: Arc<async_channel::Receiver<RequestJob>>,
@@ -107,6 +108,7 @@ impl ThreadWorker {
         scheduler_class: Option<Opaque<Value>>,
     ) -> Result<Self> {
         let mut worker = Self {
+            server,
             id,
             app,
             receiver,
@@ -125,7 +127,7 @@ impl ThreadWorker {
             Ok(_) => {}
             Err(err) => error!("Failed to send shutdown request: {}", err),
         };
-        info!("Requesting shutdown");
+        debug!("Requesting shutdown");
     }
 
     #[instrument(skip(self, deadline), fields(id = self.id))]
@@ -140,7 +142,7 @@ impl ThreadWorker {
                 if thread.funcall::<_, _, bool>(*ID_ALIVE, ()).unwrap_or(false) {
                     return true;
                 }
-                info!("Thread has shut down");
+                debug!("Thread has shut down");
             }
             self.thread.write().take();
 
@@ -154,17 +156,23 @@ impl ThreadWorker {
         let receiver = self.receiver.clone();
         let terminated = self.terminated.clone();
         let scheduler_class = self.scheduler_class;
+        let server = self.server.clone();
         call_with_gvl(|_| {
             *self.thread.write() = Some(
                 create_ruby_thread(move || {
                     if let Some(scheduler_class) = scheduler_class {
-                        if let Err(err) =
-                            Self::fiber_accept_loop(id, app, receiver, scheduler_class, terminated)
-                        {
+                        if let Err(err) = Self::fiber_accept_loop(
+                            server,
+                            id,
+                            app,
+                            receiver,
+                            scheduler_class,
+                            terminated,
+                        ) {
                             error!("Error in fiber_accept_loop: {:?}", err);
                         }
                     } else {
-                        Self::accept_loop(id, app, receiver, terminated);
+                        Self::accept_loop(server, id, app, receiver, terminated);
                     }
                 })
                 .into(),
@@ -180,6 +188,7 @@ impl ThreadWorker {
         receiver: &Arc<async_channel::Receiver<RequestJob>>,
         terminated: &Arc<AtomicBool>,
         waker_sender: &watch::Sender<TerminateWakerSignal>,
+        oob_gc_responses_threshold: Option<u64>,
     ) -> magnus::block::Proc {
         let leader = leader.clone();
         let receiver = receiver.clone();
@@ -243,10 +252,15 @@ impl ThreadWorker {
                 }
 
                 let yield_result = if receiver.is_empty() {
+                    let should_gc = if let Some(oob_gc_threshold) = oob_gc_responses_threshold {
+                        idle_counter = (idle_counter + 1) % oob_gc_threshold;
+                        idle_counter == 0
+                    } else {
+                        false
+                    };
                     waker_sender.send(TerminateWakerSignal(false)).unwrap();
-                    idle_counter = (idle_counter + 1) % 100;
                     call_with_gvl(|ruby| {
-                        if idle_counter == 0 {
+                        if should_gc {
                             ruby.gc_start();
                         }
                         scheduler.funcall::<_, _, Value>(*ID_BLOCK, (thread_current, None::<u8>))
@@ -264,6 +278,8 @@ impl ThreadWorker {
 
     #[instrument(skip_all, fields(thread_worker=id))]
     pub fn fiber_accept_loop(
+        server: Arc<Server>,
+
         id: String,
         app: Opaque<Value>,
         receiver: Arc<async_channel::Receiver<RequestJob>>,
@@ -273,10 +289,16 @@ impl ThreadWorker {
         let ruby = Ruby::get().unwrap();
         let (waker_sender, waker_receiver) = watch::channel(TerminateWakerSignal(false));
         let leader: Arc<Mutex<Option<RequestJob>>> = Arc::new(Mutex::new(None));
-        let server = ruby.get_inner(&ITSI_SERVER);
-        let scheduler_proc =
-            Self::build_scheduler_proc(app, &leader, &receiver, &terminated, &waker_sender);
-        let (scheduler, scheduler_fiber) = server.funcall::<_, _, (Value, Value)>(
+        let server_class = ruby.get_inner(&ITSI_SERVER);
+        let scheduler_proc = Self::build_scheduler_proc(
+            app,
+            &leader,
+            &receiver,
+            &terminated,
+            &waker_sender,
+            server.oob_gc_responses_threshold,
+        );
+        let (scheduler, scheduler_fiber) = server_class.funcall::<_, _, (Value, Value)>(
             "start_scheduler_loop",
             (scheduler_class, scheduler_proc),
         )?;
@@ -337,21 +359,31 @@ impl ThreadWorker {
 
     #[instrument(skip_all, fields(thread_worker=id))]
     pub fn accept_loop(
+        server: Arc<Server>,
         id: String,
         app: Opaque<Value>,
         receiver: Arc<async_channel::Receiver<RequestJob>>,
         terminated: Arc<AtomicBool>,
     ) {
         let ruby = Ruby::get().unwrap();
-        let server = ruby.get_inner(&ITSI_SERVER);
+        let server_class = ruby.get_inner(&ITSI_SERVER);
+        let mut idle_counter = 0;
         call_without_gvl(|| loop {
+            if receiver.is_empty() {
+                if let Some(oob_gc_threshold) = server.oob_gc_responses_threshold {
+                    idle_counter = (idle_counter + 1) % oob_gc_threshold;
+                    if idle_counter == 0 {
+                        ruby.gc_start();
+                    }
+                };
+            }
             match receiver.recv_blocking() {
                 Ok(RequestJob::ProcessRequest(request)) => {
                     if terminated.load(Ordering::Relaxed) {
                         break;
                     }
                     call_with_gvl(|_ruby| {
-                        request.process(&ruby, server, app).ok();
+                        request.process(&ruby, server_class, app).ok();
                     })
                 }
                 Ok(RequestJob::Shutdown) => {

data/crates/itsi_server/src/server/tls.rs

@@ -63,7 +63,7 @@ pub fn configure_tls(
                 .map(|s| s.to_string())
                 .or_else(|| (*ITSI_ACME_CONTACT_EMAIL).as_ref().ok().map(|s| s.to_string()))
                 .ok_or_else(|| itsi_error::ItsiError::ArgumentError(
-                    "acme_cert query param or ITSI_ACME_CONTACT_EMAIL must be set before you can auto-generate let's encrypt certificates".to_string(),
+                    "acme_email query param or ITSI_ACME_CONTACT_EMAIL must be set before you can auto-generate let's encrypt certificates".to_string(),
                 ))?;
 
             let acme_config = AcmeConfig::new(domains)