itsi 0.1.20 → 0.2.3

data/crates/itsi_server/src/ruby_types/itsi_server/itsi_server_config.rs

@@ -7,14 +7,14 @@ use crate::{
     },
 };
 use derive_more::Debug;
-use futures::executor::block_on;
+use itsi_error::ItsiError;
 use itsi_rb_helpers::{call_with_gvl, print_rb_backtrace, HeapValue};
 use itsi_tracing::{set_format, set_level, set_target, set_target_filters};
 use magnus::{
     block::Proc,
     error::Result,
     value::{LazyId, ReprValue},
-    RArray, RHash, Ruby, Symbol, Value,
+    RArray, RHash, Ruby, Symbol, TryConvert, Value,
 };
 use nix::{
     fcntl::{fcntl, FcntlArg, FdFlag},
@@ -25,7 +25,11 @@ use std::{
     collections::HashMap,
     os::fd::{AsRawFd, OwnedFd, RawFd},
     path::PathBuf,
-    sync::{Arc, OnceLock},
+    str::FromStr,
+    sync::{
+        atomic::{AtomicBool, Ordering::Relaxed},
+        Arc, OnceLock,
+    },
     time::Duration,
 };
 use tracing::{debug, error};
@@ -71,8 +75,33 @@ pub struct ServerParams {
     #[debug(skip)]
     pub(crate) listeners: Mutex<Vec<Listener>>,
     listener_info: Mutex<HashMap<String, i32>>,
+    pub itsi_server_token_preference: ItsiServerTokenPreference,
+    pub preloaded: AtomicBool,
+    socket_opts: SocketOpts,
+    preexisting_listeners: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+pub enum ItsiServerTokenPreference {
+    Version,
+    Name,
+    None,
+}
+
+impl FromStr for ItsiServerTokenPreference {
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        Ok(match s {
+            "version" => ItsiServerTokenPreference::Version,
+            "name" => ItsiServerTokenPreference::Name,
+            "none" => ItsiServerTokenPreference::None,
+            _ => ItsiServerTokenPreference::Version,
+        })
+    }
+
+    type Err = ItsiError;
 }
 
+#[derive(Debug, Clone)]
 pub struct SocketOpts {
     pub reuse_address: bool,
     pub reuse_port: bool,
@@ -83,8 +112,10 @@ pub struct SocketOpts {
 
 impl ServerParams {
     pub fn preload_ruby(self: &Arc<Self>) -> Result<()> {
+        if self.preloaded.load(Relaxed) {
+            return Ok(());
+        }
         call_with_gvl(|ruby| -> Result<()> {
-            debug!("Preloading Ruby");
             if self
                 .scheduler_class
                 .as_ref()
@@ -93,9 +124,17 @@ impl ServerParams {
                 debug!("Loading Itsi Scheduler");
                 ruby.require("itsi/scheduler")?;
             }
-            let routes_raw = self
+            let result_pair = self
                 .middleware_loader
-                .call::<_, Option<Value>>(())
+                .call::<(), RArray>(())
+                .inspect_err(|e| {
+                    eprintln!("Error loading middleware: {:?}", e);
+                    if let Some(err_value) = e.value() {
+                        print_rb_backtrace(err_value);
+                    }
+                })?;
+            let routes_raw = result_pair
+                .entry::<Option<Value>>(0)
                 .inspect_err(|e| {
                     eprintln!("Error loading middleware: {:?}", e);
                     if let Some(err_value) = e.value() {
@@ -103,9 +142,22 @@ impl ServerParams {
                     }
                 })?
                 .map(|mw| mw.into());
-            debug!("Middleware routes returned");
+            let error_lines = result_pair.entry::<Option<RArray>>(1).inspect_err(|e| {
+                eprintln!("Error loading middleware: {:?}", e);
+                if let Some(err_value) = e.value() {
+                    print_rb_backtrace(err_value);
+                }
+            })?;
+            if error_lines.is_some_and(|r| !r.is_empty()) {
+                let errors: Vec<String> =
+                    Vec::<String>::try_convert(error_lines.unwrap().as_value())?;
+                ItsiServerConfig::print_config_errors(errors);
+                return Err(magnus::Error::new(
+                    magnus::exception::runtime_error(),
+                    "Failed to set middleware",
+                ));
+            }
             let middleware = MiddlewareSet::new(routes_raw)?;
-            debug!("Middleware loaded");
             self.middleware.set(middleware).map_err(|_| {
                 magnus::Error::new(
                     magnus::exception::runtime_error(),
@@ -114,6 +166,7 @@ impl ServerParams {
             })?;
             Ok(())
         })?;
+        self.preloaded.store(true, Relaxed);
         Ok(())
     }
 
@@ -154,11 +207,11 @@ impl ServerParams {
             .transpose()?
             .unwrap_or_default();
         let preload: bool = rb_param_hash.fetch("preload")?;
-        let request_timeout: Option<u64> = rb_param_hash.fetch("request_timeout")?;
-        let request_timeout = request_timeout.map(Duration::from_secs);
+        let request_timeout: Option<f64> = rb_param_hash.fetch("request_timeout")?;
+        let request_timeout = request_timeout.map(Duration::from_secs_f64);
         let header_read_timeout: Duration = rb_param_hash
-            .fetch::<_, Option<u64>>("header_read_timeout")?
-            .map(Duration::from_secs)
+            .fetch::<_, Option<f64>>("header_read_timeout")?
+            .map(Duration::from_secs_f64)
             .unwrap_or(Duration::from_secs(1));
 
         let notify_watchers: Option<Vec<(String, Vec<Vec<String>>)>> =
@@ -226,6 +279,12 @@ impl ServerParams {
             .map(|s| s.parse())
             .collect::<itsi_error::Result<Vec<Bind>>>()?;
 
+        let itsi_server_token_preference: String = rb_param_hash
+            .fetch("itsi_server_token_preference")
+            .unwrap_or_default();
+        let itsi_server_token_preference: ItsiServerTokenPreference =
+            itsi_server_token_preference.parse()?;
+
         let socket_opts = SocketOpts {
             reuse_address,
             reuse_port,
@@ -233,10 +292,42 @@ impl ServerParams {
             nodelay,
             recv_buffer_size,
         };
-        let listeners = if let Some(preexisting_listeners) =
-            rb_param_hash.delete::<_, Option<String>>("listeners")?
-        {
-            let bind_to_fd_map: HashMap<String, i32> = serde_json::from_str(&preexisting_listeners)
+        let preexisting_listeners = rb_param_hash.delete::<_, Option<String>>("listeners")?;
+
+        let params = ServerParams {
+            workers,
+            worker_memory_limit,
+            silence,
+            multithreaded_reactor,
+            pin_worker_cores,
+            shutdown_timeout,
+            hooks,
+            preload,
+            request_timeout,
+            header_read_timeout,
+            notify_watchers,
+            threads,
+            scheduler_threads,
+            streamable_body,
+            scheduler_class,
+            oob_gc_responses_threshold,
+            binds,
+            itsi_server_token_preference,
+            socket_opts,
+            preexisting_listeners,
+            listener_info: Mutex::new(HashMap::new()),
+            listeners: Mutex::new(Vec::new()),
+            middleware_loader: middleware_loader.into(),
+            middleware: OnceLock::new(),
+            preloaded: AtomicBool::new(false),
+        };
+
+        Ok(params)
+    }
+
+    pub fn setup_listeners(&self) -> Result<()> {
+        let listeners = if let Some(preexisting_listeners) = self.preexisting_listeners.as_ref() {
+            let bind_to_fd_map: HashMap<String, i32> = serde_json::from_str(preexisting_listeners)
                 .map_err(|e| {
                     magnus::Error::new(
                         magnus::exception::standard_error(),
@@ -244,24 +335,24 @@ impl ServerParams {
                     )
                 })?;
 
-            binds
+            self.binds
                 .iter()
                 .cloned()
                 .map(|bind| {
                     if let Some(fd) = bind_to_fd_map.get(&bind.listener_address_string()) {
-                        Listener::inherit_fd(bind, *fd, &socket_opts)
+                        Listener::inherit_fd(bind, *fd, &self.socket_opts)
                     } else {
-                        Listener::build(bind, &socket_opts)
+                        Listener::build(bind, &self.socket_opts)
                     }
                 })
                 .collect::<std::result::Result<Vec<Listener>, _>>()?
                 .into_iter()
                 .collect::<Vec<_>>()
         } else {
-            binds
+            self.binds
                 .iter()
                 .cloned()
-                .map(|b| Listener::build(b, &socket_opts))
+                .map(|b| Listener::build(b, &self.socket_opts))
                 .collect::<std::result::Result<Vec<Listener>, _>>()?
                 .into_iter()
                 .collect::<Vec<_>>()
@@ -276,29 +367,9 @@ impl ServerParams {
             })
             .collect::<Result<HashMap<String, i32>>>()?;
 
-        Ok(ServerParams {
-            workers,
-            worker_memory_limit,
-            silence,
-            multithreaded_reactor,
-            pin_worker_cores,
-            shutdown_timeout,
-            hooks,
-            preload,
-            request_timeout,
-            header_read_timeout,
-            notify_watchers,
-            threads,
-            scheduler_threads,
-            streamable_body,
-            scheduler_class,
-            oob_gc_responses_threshold,
-            binds,
-            listener_info: Mutex::new(listener_info),
-            listeners: Mutex::new(listeners),
-            middleware_loader: middleware_loader.into(),
-            middleware: OnceLock::new(),
-        })
+        *self.listener_info.lock() = listener_info;
+        *self.listeners.lock() = listeners;
+        Ok(())
     }
 }
 
@@ -310,28 +381,34 @@ impl ItsiServerConfig {
         itsi_config_proc: Option<Proc>,
     ) -> Result<Self> {
         let itsi_config_proc = Arc::new(itsi_config_proc.map(HeapValue::from));
-        let server_params = Self::combine_params(
+        match Self::combine_params(
             ruby,
             cli_params,
             itsifile_path.as_ref(),
             itsi_config_proc.clone(),
-        )?;
-
-        cli_params.delete::<_, Value>(Symbol::new("listeners"))?;
+        ) {
+            Ok(server_params) => {
+                cli_params.delete::<_, Value>(Symbol::new("listeners"))?;
 
-        let watcher_fd = if let Some(watchers) = server_params.notify_watchers.clone() {
-            file_watcher::watch_groups(watchers)?
-        } else {
-            None
-        };
-
-        Ok(ItsiServerConfig {
-            cli_params: Arc::new(cli_params.into()),
-            server_params: RwLock::new(server_params.clone()).into(),
-            itsi_config_proc,
-            itsifile_path,
-            watcher_fd: watcher_fd.into(),
-        })
+                let watcher_fd = if let Some(watchers) = server_params.notify_watchers.clone() {
+                    file_watcher::watch_groups(watchers)?
+                } else {
+                    None
+                };
+
+                Ok(ItsiServerConfig {
+                    cli_params: Arc::new(cli_params.into()),
+                    server_params: RwLock::new(server_params.clone()).into(),
+                    itsi_config_proc,
+                    itsifile_path,
+                    watcher_fd: watcher_fd.into(),
+                })
+            }
+            Err(err) => Err(magnus::Error::new(
+                magnus::exception::standard_error(),
+                format!("Error loading initial configuration {:?}", err),
+            )),
+        }
     }
 
     /// Reload
@@ -397,7 +474,7 @@ impl ItsiServerConfig {
         Ok(())
     }
 
-    pub fn get_config_errors(&self) -> Option<Vec<String>> {
+    pub async fn get_config_errors(&self) -> Option<Vec<String>> {
         let rb_param_hash = call_with_gvl(|ruby| {
             let inner = self
                 .itsi_config_proc
@@ -424,8 +501,13 @@ impl ItsiServerConfig {
                         let err_val = call_with_gvl(|_| format!("{}", err));
                         return Some(vec![err_val]);
                     }
-                    if let Err(err) =
-                        block_on(params_arc.middleware.get().unwrap().initialize_layers())
+
+                    if let Err(err) = params_arc
+                        .middleware
+                        .get()
+                        .unwrap()
+                        .initialize_layers()
+                        .await
                     {
                         let err_val = call_with_gvl(|_| format!("{}", err));
                         return Some(vec![err_val]);
@@ -477,8 +559,8 @@ impl ItsiServerConfig {
         }
     }
 
-    pub fn check_config(&self) -> bool {
-        if let Some(errors) = self.get_config_errors() {
+    pub async fn check_config(&self) -> bool {
+        if let Some(errors) = self.get_config_errors().await {
             Self::print_config_errors(errors);
             return false;
         }

data/crates/itsi_server/src/ruby_types/itsi_server.rs

@@ -42,6 +42,7 @@ impl ItsiServer {
 
     #[instrument(skip(self))]
     pub fn start(&self) -> Result<()> {
+        self.config.lock().server_params.read().setup_listeners()?;
         let result = if self.config.lock().server_params.read().silence {
             run_silently(|| self.build_and_run_strategy())
         } else {
@@ -49,6 +50,7 @@ impl ItsiServer {
             self.build_and_run_strategy()
         };
         if let Err(e) = result {
+            error!("Error starting server: {:?}", e);
             if let Some(err_value) = e.value() {
                 print_rb_backtrace(err_value);
             }

data/crates/itsi_server/src/server/binds/bind.rs

@@ -10,6 +10,7 @@ use std::{
     path::PathBuf,
     str::FromStr,
 };
+use tracing::{instrument, Level};
 
 #[derive(Debug, Clone)]
 pub enum BindAddress {
@@ -86,6 +87,7 @@ impl std::fmt::Debug for Bind {
 impl FromStr for Bind {
     type Err = ItsiError;
 
+    #[instrument(ret(level = Level::DEBUG))]
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let (protocol, remainder) = if let Some((proto, rest)) = s.split_once("://") {
             (proto.parse::<BindProtocol>()?, rest)
@@ -180,6 +182,7 @@ fn parse_bind_options(query: &str) -> HashMap<String, String> {
 }
 
 /// Attempts to resolve a hostname into an IP address.
+#[instrument(ret(level = Level::DEBUG))]
 fn resolve_hostname(hostname: &str) -> Option<IpAddr> {
     (hostname, 0)
         .to_socket_addrs()

data/crates/itsi_server/src/server/binds/listener.rs

@@ -418,8 +418,12 @@ fn connect_tcp_socket(addr: IpAddr, port: u16, socket_opts: &SocketOpts) -> Resu
         .set_recv_buffer_size(socket_opts.recv_buffer_size)
         .ok();
     socket.set_only_v6(false).ok();
-    socket.bind(&socket_address.into())?;
-    socket.listen(socket_opts.listen_backlog as i32)?;
+    if let Err(e) = socket.bind(&socket_address.into()) {
+        error!("Failed to bind socket: {}", e);
+    };
+    if let Err(e) = socket.listen(socket_opts.listen_backlog as i32) {
+        error!("Failed to listen on socket: {}", e);
+    };
     Ok(socket.into())
 }
 
@@ -430,8 +434,11 @@ fn connect_unix_socket(path: &PathBuf, socket_opts: &SocketOpts) -> Result<UnixL
 
     let socket_address = socket2::SockAddr::unix(path)?;
 
-    socket.bind(&socket_address)?;
-    socket.listen(socket_opts.listen_backlog as i32)?;
-
+    if let Err(e) = socket.bind(&socket_address) {
+        error!("Failed to bind socket: {}", e);
+    };
+    if let Err(e) = socket.listen(socket_opts.listen_backlog as i32) {
+        error!("Failed to listen on socket: {}", e);
+    };
     Ok(socket.into())
 }

data/crates/itsi_server/src/server/binds/tls.rs

@@ -3,8 +3,10 @@ use itsi_acme::{AcmeAcceptor, AcmeConfig, AcmeState};
 use itsi_error::Result;
 use itsi_tracing::info;
 use locked_dir_cache::LockedDirCache;
+use rcgen::ExtendedKeyUsagePurpose;
 use rcgen::{
-    BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, KeyPair, SanType,
+    BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
 };
 use rustls::{
     pki_types::{CertificateDer, PrivateKeyDer},
@@ -228,13 +230,14 @@ pub fn generate_ca_signed_cert(
         .push(DnType::CommonName, domains[0].clone());
 
     ee_params.use_authority_key_identifier_extension = true;
+    ee_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
 
     let ee_cert = ee_params.signed_by(&ee_key, &ca_cert, &ca_kp).unwrap();
     let ee_cert_der = ee_cert.der().to_vec();
     let ee_cert = CertificateDer::from(ee_cert_der);
-    let ca_cert = CertificateDer::from(ca_cert.der().to_vec());
+
     Ok((
-        vec![ee_cert, ca_cert],
+        vec![ee_cert],
         PrivateKeyDer::try_from(ee_key.serialize_der()).unwrap(),
     ))
 }
@@ -253,12 +256,17 @@ fn get_or_create_local_dev_ca() -> Result<(String, String)> {
 
         Ok((key_pem, cert_pem))
     } else {
-        let subject_alt_names = vec!["dev.itsi.fyi".to_string(), "localhost".to_string()];
+        let subject_alt_names = vec!["ca.itsi.fyi".to_string(), "localhost".to_string()];
         let mut params = CertificateParams::new(subject_alt_names)?;
         let mut distinguished_name = DistinguishedName::new();
-        distinguished_name.push(DnType::CommonName, "Itsi Development CA");
+        distinguished_name.push(DnType::CommonName, "ca.itsi.fyi");
         params.distinguished_name = distinguished_name;
         params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+        params.key_usages = vec![
+            KeyUsagePurpose::KeyCertSign,
+            KeyUsagePurpose::CrlSign,
+            KeyUsagePurpose::DigitalSignature, // useful for OCSP/CRL signing
+        ];
         let key_pair = KeyPair::generate()?;
         let cert = params.self_signed(&key_pair)?;
 

data/crates/itsi_server/src/server/middleware_stack/middlewares/allow_list.rs

@@ -1,23 +1,23 @@
+use super::{token_source::TokenSource, ErrorResponse, FromValue, MiddlewareLayer};
 use crate::{
     server::http_message_types::{HttpRequest, HttpResponse, RequestExt},
     services::itsi_http_service::HttpRequestContext,
 };
-
-use super::{ErrorResponse, FromValue, MiddlewareLayer};
-
 use async_trait::async_trait;
 use either::Either;
 use itsi_error::ItsiError;
 use magnus::error::Result;
 use regex::RegexSet;
 use serde::Deserialize;
-use std::sync::OnceLock;
+use std::{collections::HashMap, sync::OnceLock};
+use tracing::debug;
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct AllowList {
     #[serde(skip_deserializing)]
     pub allowed_ips: OnceLock<RegexSet>,
     pub allowed_patterns: Vec<String>,
+    pub trusted_proxies: HashMap<String, TokenSource>,
     #[serde(default = "forbidden_error_response")]
     pub error_response: ErrorResponse,
 }
@@ -42,7 +42,14 @@ impl MiddlewareLayer for AllowList {
         context: &mut HttpRequestContext,
     ) -> Result<Either<HttpRequest, HttpResponse>> {
         if let Some(allowed_ips) = self.allowed_ips.get() {
-            if !allowed_ips.is_match(&context.addr) {
+            let addr = if self.trusted_proxies.contains_key(&context.addr) {
+                let source = self.trusted_proxies.get(&context.addr).unwrap();
+                source.extract_token(&req).unwrap_or(&context.addr)
+            } else {
+                &context.addr
+            };
+            if !allowed_ips.is_match(addr) {
+                debug!(target: "middleware::allow_list", "IP address {} is not allowed", addr);
                 return Ok(Either::Right(
                     self.error_response
                         .to_http_response(req.accept().into())

data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_api_key.rs

@@ -11,6 +11,7 @@ use async_trait::async_trait;
 use either::Either;
 use magnus::error::Result;
 use serde::Deserialize;
+use tracing::debug;
 
 type PasswordHash = String;
 
@@ -51,6 +52,8 @@ impl MiddlewareLayer for AuthAPIKey {
             }
             TokenSource::Query(query_name) => req.query_param(query_name),
         } {
+            debug!(target: "middleware::auth_api_key", "API Key Retrieved. Anonymous {}", self.key_id_source.is_none());
+
             if let Some(key_id) = self.key_id_source.as_ref() {
                 let key_id = match &key_id {
                     TokenSource::Header { name, prefix } => {
@@ -66,17 +69,21 @@ impl MiddlewareLayer for AuthAPIKey {
                     }
                     TokenSource::Query(query_name) => req.query_param(query_name),
                 };
+                debug!(target: "middleware::auth_api_key", "Key ID Retrieved");
                 if let Some(hash) = key_id.and_then(|kid| self.valid_keys.get(kid)) {
+                    debug!(target: "middleware::auth_api_key", "Key for ID found");
                     if password_hasher::verify_password_hash(submitted_key, hash).is_ok_and(|v| v) {
                         return Ok(Either::Left(req));
                     }
                 }
-            } else if self.valid_keys.iter().any(|(_key_id, key)| {
+            } else if self.valid_keys.values().any(|key| {
                 password_hasher::verify_password_hash(submitted_key, key).is_ok_and(|v| v)
             }) {
                 return Ok(Either::Left(req));
             }
         }
+
+        debug!(target: "middleware::auth_api_key", "Failed to authenticate API key");
         Ok(Either::Right(
             self.error_response
                 .to_http_response(req.accept().into())

data/crates/itsi_server/src/server/middleware_stack/middlewares/auth_basic.rs

@@ -8,6 +8,7 @@ use magnus::error::Result;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::str;
+use tracing::debug;
 
 use crate::{
     server::http_message_types::{HttpRequest, HttpResponse, RequestExt},
@@ -48,6 +49,7 @@ impl MiddlewareLayer for AuthBasic {
         let auth_header = req.header("Authorization");
 
         if !auth_header.is_some_and(|header| header.starts_with("Basic ")) {
+            debug!(target: "middleware::auth_basic", "Basic auth failed. Authorization Header doesn't start with 'Basic '");
             return Ok(Either::Right(self.basic_auth_failed_response()));
         }
 
@@ -57,6 +59,7 @@ impl MiddlewareLayer for AuthBasic {
         let decoded = match general_purpose::STANDARD.decode(encoded_credentials) {
             Ok(bytes) => bytes,
             Err(_) => {
+                debug!(target: "middleware::auth_basic", "Basic auth failed. Decoding failed");
                 return Ok(Either::Right(self.basic_auth_failed_response()));
             }
         };
@@ -64,6 +67,7 @@ impl MiddlewareLayer for AuthBasic {
         let decoded_str = match str::from_utf8(&decoded) {
             Ok(s) => s,
             Err(_) => {
+                debug!(target: "middleware::auth_basic", "Basic auth failed. Decoding failed");
                 return Ok(Either::Right(self.basic_auth_failed_response()));
             }
         };
@@ -71,6 +75,7 @@ impl MiddlewareLayer for AuthBasic {
         let mut parts = decoded_str.splitn(2, ':');
         let username = parts.next().unwrap_or("");
         let password = parts.next().unwrap_or("");
+
         match self.credential_pairs.get(username) {
             Some(expected_password_hash) => {
                 match verify_password_hash(password, expected_password_hash) {
@@ -78,7 +83,10 @@ impl MiddlewareLayer for AuthBasic {
                     _ => Ok(Either::Right(self.basic_auth_failed_response())),
                 }
             }
-            None => Ok(Either::Right(self.basic_auth_failed_response())),
+            None => {
+                debug!(target: "middleware::auth_basic", "Basic auth failed. Username {} not found", username);
+                Ok(Either::Right(self.basic_auth_failed_response()))
+            }
         }
     }
 }