itsi 0.1.0 → 0.1.3

data/Rakefile

@@ -9,65 +9,54 @@ $LOAD_PATH.unshift(File.expand_path('server/lib', __dir__))
 
 GEMS = [
   {
+    shortname: :scheduler,
     name: 'itsi-scheduler',
     dir: 'gems/scheduler', # subfolder that holds gem code
     gemspec: 'itsi-scheduler.gemspec',
     rust_name: 'itsi_scheduler' # name of the ext subfolder
   },
   {
+    shortname: :server,
     name: 'itsi-server',
     dir: 'gems/server',
     gemspec: 'itsi-server.gemspec',
     rust_name: 'itsi_server'
   }
 ]
+SHARED_TASKS = %i[compile compile:dev test]
 
-Minitest::TestTask.create do |t|
-  t.test_globs = ['**/test/**/*.rb']
-  t.warning = true
-  t.verbose = true
-end
-
-namespace :scheduler do
-  desc 'Run tasks in the scheduler directory'
-  task :default do
-    sh 'cd gems/scheduler && rake'
-  end
+GEMS.each do |gem|
+  namespace gem[:shortname] do
+    desc "Run tasks in the #{gem[:dir]} directory"
+    task :default do
+      sh "cd #{gem[:dir]} && rake"
+    end
 
-  task :compile do
-    sh 'cd gems/scheduler && rake compile'
+    SHARED_TASKS.each do |task|
+      task task do
+        sh "cd #{gem[:dir]} && rake #{task}"
+      end
+    end
   end
 end
 
-namespace :server do
-  desc 'Run tasks in the server directory'
-  task :default do
-    sh 'cd gems/server && rake'
-  end
-
-  task :compile do
-    sh 'cd gems/server && rake compile'
+SHARED_TASKS.each do |task|
+  desc "#{task} in all Gem directories"
+  task task do
+    GEMS.each do |gem|
+      Rake::Task["#{gem[:shortname]}:#{task}"].invoke
+    end
   end
+  Rake::Task[task].enhance([:sync_crates])
 end
 
-desc 'Compile in both scheduler and server directories'
-task :compile do
-  Rake::Task['scheduler:compile'].invoke
-  Rake::Task['server:compile'].invoke
-end
-
-Rake::Task[:compile].enhance([:sync_crates])
 Rake::Task[:build].enhance([:build_all])
 
 task :sync_crates do
   require 'fileutils'
-
   GEMS.each do |gem_info|
     Dir.chdir('crates') do
-      to_sync = Dir['*'].select do |fn|
-        rust_name = fn.split('/', 2).last
-        rust_name == gem_info[:rust_name] || GEMS.none? { |g| g[:rust_name] == rust_name }
-      end.each do |to_sync|
+      to_sync = Dir['*'].each do |to_sync|
         system("rsync -q -av #{to_sync}/ ../#{gem_info[:dir]}/ext/#{to_sync} --delete")
       end
     end

data/crates/itsi_error/Cargo.toml

@@ -7,3 +7,5 @@ edition = "2024"
 thiserror = "2.0.11"
 magnus = { version = "0.7.1" }
 rcgen = "0.13.2"
+nix = "0.29.0"
+httparse = "1.10.1"

data/crates/itsi_error/src/from.rs

@@ -0,0 +1,70 @@
+use crate::ItsiError;
+use std::ffi::NulError;
+
+pub static CLIENT_CONNECTION_CLOSED: &str = "Client disconnected";
+
+impl From<httparse::Error> for ItsiError {
+    fn from(err: httparse::Error) -> Self {
+        ItsiError::ArgumentError(err.to_string())
+    }
+}
+
+impl From<nix::errno::Errno> for ItsiError {
+    fn from(err: nix::errno::Errno) -> Self {
+        ItsiError::ArgumentError(err.to_string())
+    }
+}
+
+impl From<std::io::Error> for ItsiError {
+    fn from(err: std::io::Error) -> Self {
+        ItsiError::ArgumentError(err.to_string())
+    }
+}
+
+impl From<rcgen::Error> for ItsiError {
+    fn from(err: rcgen::Error) -> Self {
+        ItsiError::ArgumentError(err.to_string())
+    }
+}
+
+impl From<NulError> for ItsiError {
+    fn from(err: NulError) -> Self {
+        ItsiError::ArgumentError(err.to_string())
+    }
+}
+
+impl From<magnus::Error> for ItsiError {
+    fn from(err: magnus::Error) -> Self {
+        match err.error_type() {
+            magnus::error::ErrorType::Jump(tag) => ItsiError::Jump(tag.to_string()),
+            magnus::error::ErrorType::Error(_exception_class, cow) => {
+                ItsiError::ArgumentError(cow.to_string())
+            }
+            magnus::error::ErrorType::Exception(exception) => {
+                ItsiError::ArgumentError(exception.to_string())
+            }
+        }
+    }
+}
+
+impl From<ItsiError> for magnus::Error {
+    fn from(err: ItsiError) -> Self {
+        match err {
+            ItsiError::InvalidInput(msg) => magnus::Error::new(magnus::exception::arg_error(), msg),
+            ItsiError::InternalServerError(msg) => {
+                magnus::Error::new(magnus::exception::exception(), msg)
+            }
+            ItsiError::UnsupportedProtocol(msg) => {
+                magnus::Error::new(magnus::exception::arg_error(), msg)
+            }
+            ItsiError::ArgumentError(msg) => {
+                magnus::Error::new(magnus::exception::arg_error(), msg)
+            }
+            ItsiError::Jump(msg) => magnus::Error::new(magnus::exception::local_jump_error(), msg),
+            ItsiError::Break() => magnus::Error::new(magnus::exception::interrupt(), "Break"),
+            ItsiError::ClientConnectionClosed => {
+                magnus::Error::new(magnus::exception::eof_error(), CLIENT_CONNECTION_CLOSED)
+            }
+        }
+    }
+}

data/crates/itsi_error/src/lib.rs

@@ -1,49 +1,22 @@
+pub mod from;
 use thiserror::Error;
 
 pub type Result<T> = std::result::Result<T, ItsiError>;
 
 #[derive(Error, Debug)]
 pub enum ItsiError {
-    #[error("Invalid input")]
+    #[error("Invalid input {0}")]
     InvalidInput(String),
-    #[error("Internal server error")]
-    InternalServerError,
-    #[error("Unsupported protocol")]
+    #[error("Internal server error {0}")]
+    InternalServerError(String),
+    #[error("Unsupported protocol {0}")]
     UnsupportedProtocol(String),
-    #[error("Argument error")]
+    #[error("Argument error: {0}")]
     ArgumentError(String),
+    #[error("Client Connection Closed")]
+    ClientConnectionClosed,
     #[error("Jump")]
     Jump(String),
-}
-
-impl From<ItsiError> for magnus::Error {
-    fn from(err: ItsiError) -> Self {
-        magnus::Error::new(magnus::exception::runtime_error(), err.to_string())
-    }
-}
-
-impl From<std::io::Error> for ItsiError {
-    fn from(err: std::io::Error) -> Self {
-        ItsiError::ArgumentError(err.to_string())
-    }
-}
-
-impl From<rcgen::Error> for ItsiError {
-    fn from(err: rcgen::Error) -> Self {
-        ItsiError::ArgumentError(err.to_string())
-    }
-}
-
-impl From<magnus::Error> for ItsiError {
-    fn from(err: magnus::Error) -> Self {
-        match err.error_type() {
-            magnus::error::ErrorType::Jump(tag) => ItsiError::Jump(tag.to_string()),
-            magnus::error::ErrorType::Error(_exception_class, cow) => {
-                ItsiError::ArgumentError(cow.to_string())
-            }
-            magnus::error::ErrorType::Exception(exception) => {
-                ItsiError::ArgumentError(exception.to_string())
-            }
-        }
-    }
+    #[error("Break")]
+    Break(),
 }

data/crates/itsi_instrument_entry/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "itsi_instrument_entry"
+version = "0.1.0"
+edition = "2021"
+authors = ["Wouter Coppieters <wc@pico.net.nz>"]
+license = "MIT"
+publish = false
+
+
+[lib]
+proc-macro = true
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = { version = "1.0", features = ["full"] }

data/crates/itsi_instrument_entry/src/lib.rs

@@ -0,0 +1,31 @@
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use syn::{parse_macro_input, ItemFn};
+
+#[proc_macro_attribute]
+pub fn instrument_with_entry(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr_tokens = TokenStream2::from(attr);
+    let input_fn = parse_macro_input!(item as ItemFn);
+    let attrs = input_fn.attrs;
+    let vis = input_fn.vis;
+    let sig = input_fn.sig;
+    let block = input_fn.block;
+    let output = quote! {
+        #[cfg(debug_assertions)]
+        #[tracing::instrument(#attr_tokens)]
+        #(#attrs)*
+        #vis #sig {
+            tracing::trace!("");
+            #block
+        }
+
+        #[cfg(not(debug_assertions))]
+        #(#attrs)*
+        #vis #sig {
+            #block
+        }
+    };
+
+    output.into()
+}

data/crates/itsi_rb_helpers/Cargo.toml

@@ -4,5 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+cfg-if = "1.0.0"
 magnus = { version = "0.7.1", features = ["rb-sys", "bytes"] }
+nix = "0.29.0"
 rb-sys = "0.9.105"

data/crates/itsi_rb_helpers/src/heap_value.rs

@@ -0,0 +1,121 @@
+use magnus::IntoValue;
+use magnus::rb_sys::AsRawValue;
+use magnus::value::BoxValue;
+use magnus::{Ruby, Value, value::ReprValue};
+use std::fmt::{self, Debug, Formatter};
+use std::ops::Deref;
+
+/// HeapVal is a wrapper for heap-allocated magnus ReprVa;ies
+/// that is marked as thread-safe(Send and Sync)
+/// It's up to the user to actually ensure this though,
+/// typically by only interacting with the value from a thread which
+/// holds the GVL.
+pub struct HeapValue<T>(pub BoxValue<T>)
+where
+    T: ReprValue;
+
+impl<T> PartialEq for HeapValue<T>
+where
+    T: ReprValue,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.0.as_raw() == other.0.as_raw()
+    }
+}
+
+impl<T> Deref for HeapValue<T>
+where
+    T: ReprValue,
+{
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl<T> HeapValue<T>
+where
+    T: ReprValue,
+{
+    pub fn inner(self) -> T {
+        *self.0
+    }
+}
+
+impl<T> IntoValue for HeapValue<T>
+where
+    T: ReprValue,
+{
+    fn into_value_with(self, _: &Ruby) -> Value {
+        self.0.into_value()
+    }
+}
+
+impl<T> From<T> for HeapValue<T>
+where
+    T: ReprValue,
+{
+    fn from(value: T) -> Self {
+        HeapValue(BoxValue::new(value))
+    }
+}
+
+impl<T> Clone for HeapValue<T>
+where
+    T: ReprValue + Clone,
+{
+    fn clone(&self) -> Self {
+        HeapValue(BoxValue::new(*self.0.deref()))
+    }
+}
+
+impl<T> Debug for HeapValue<T>
+where
+    T: ReprValue + Debug,
+{
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self.0)
+    }
+}
+
+unsafe impl<T> Send for HeapValue<T> where T: ReprValue {}
+unsafe impl<T> Sync for HeapValue<T> where T: ReprValue {}
+
+/// HeapVal is a wrapper for heap-allocated magnus Values
+/// that is marked as thread-safe(Send and Sync)
+/// It's up to the user to actually ensure this though,
+/// typically by only interacting with the value from a thread which
+/// holds the GVL.
+pub struct HeapVal(HeapValue<Value>);
+impl Deref for HeapVal {
+    type Target = Value;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl IntoValue for HeapVal {
+    fn into_value_with(self, _: &Ruby) -> Value {
+        self.0.into_value()
+    }
+}
+
+impl From<Value> for HeapVal {
+    fn from(value: Value) -> Self {
+        HeapVal(HeapValue(BoxValue::new(value)))
+    }
+}
+
+impl Clone for HeapVal {
+    fn clone(&self) -> Self {
+        HeapVal(HeapValue(BoxValue::new(*self.0.deref())))
+    }
+}
+
+impl Debug for HeapVal {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self.0)
+    }
+}

data/crates/itsi_rb_helpers/src/lib.rs

@@ -1,23 +1,43 @@
-use std::{os::raw::c_void, ptr::null_mut};
+use std::{os::raw::c_void, ptr::null_mut, sync::Arc};
 
+use magnus::{
+    RArray, Ruby, Thread, Value,
+    rb_sys::FromRawValue,
+    value::{LazyId, ReprValue},
+};
 use rb_sys::{
-    rb_thread_call_with_gvl, rb_thread_call_without_gvl, rb_thread_create, rb_thread_wakeup,
+    rb_thread_call_with_gvl, rb_thread_call_without_gvl, rb_thread_create, rb_thread_schedule,
+    rb_thread_wakeup,
 };
 
-pub fn create_ruby_thread<F>(f: F)
+mod heap_value;
+pub use heap_value::{HeapVal, HeapValue};
+static ID_FORK: LazyId = LazyId::new("fork");
+static ID_LIST: LazyId = LazyId::new("list");
+static ID_EQ: LazyId = LazyId::new("==");
+static ID_ALIVE: LazyId = LazyId::new("alive?");
+static ID_THREAD_VARIABLE_GET: LazyId = LazyId::new("thread_variable_get");
+
+pub fn schedule_thread() {
+    unsafe {
+        rb_thread_schedule();
+    };
+}
+pub fn create_ruby_thread<F>(f: F) -> Thread
 where
-    F: FnOnce() -> u64 + Send + 'static,
+    F: FnOnce() + Send + 'static,
 {
     extern "C" fn trampoline<F>(ptr: *mut c_void) -> u64
     where
-        F: FnOnce() -> u64,
+        F: FnOnce(),
     {
         // Reconstruct the boxed Option<F> that holds our closure.
         let boxed_closure: Box<Option<F>> = unsafe { Box::from_raw(ptr as *mut Option<F>) };
         // Extract the closure. (The Option should be Some; panic otherwise.)
         let closure = (*boxed_closure).expect("Closure already taken");
         // Call the closure and return its result.
-        closure()
+        closure();
+        0
     }
 
     // Box the closure (wrapped in an Option) to create a stable pointer.
@@ -26,7 +46,10 @@ where
 
     // Call rb_thread_create with our trampoline and boxed closure.
     unsafe {
-        rb_thread_wakeup(rb_thread_create(Some(trampoline::<F>), ptr));
+        let thread = rb_thread_create(Some(trampoline::<F>), ptr);
+        rb_thread_wakeup(thread);
+        rb_thread_schedule();
+        Thread::from_value(Value::from_raw(thread)).unwrap()
     }
 }
 
@@ -67,18 +90,18 @@ where
 
 pub fn call_with_gvl<F, R>(f: F) -> R
 where
-    F: FnOnce() -> R,
+    F: FnOnce(Ruby) -> R,
 {
     extern "C" fn trampoline<F, R>(arg: *mut c_void) -> *mut c_void
     where
-        F: FnOnce() -> R,
+        F: FnOnce(Ruby) -> R,
     {
         // 1) Reconstruct the Box that holds our closure
         let closure_ptr = arg as *mut Option<F>;
         let closure = unsafe { (*closure_ptr).take().expect("Closure already taken") };
 
         // 2) Call the user’s closure
-        let result = closure();
+        let result = closure(Ruby::get().unwrap());
 
         // 3) Box up the result so we can return a pointer to it
         let boxed_result = Box::new(result);
@@ -96,3 +119,60 @@ where
     let result_box = unsafe { Box::from_raw(raw_result_ptr as *mut R) };
     *result_box
 }
+
+pub fn fork(after_fork: Arc<Option<impl Fn()>>) -> Option<i32> {
+    let ruby = Ruby::get().unwrap();
+    let fork_result = ruby
+        .module_kernel()
+        .funcall::<_, _, Option<i32>>(*ID_FORK, ())
+        .unwrap();
+    if fork_result.is_none() {
+        if let Some(f) = &*after_fork {
+            f()
+        }
+    }
+    fork_result
+}
+
+pub fn kill_threads<T>(threads: Vec<T>)
+where
+    T: ReprValue,
+{
+    for thr in &threads {
+        let alive: bool = thr
+            .funcall(*ID_ALIVE, ())
+            .expect("Failed to check if thread is alive");
+        if !alive {
+            eprintln!("Thread killed");
+            break;
+        }
+        eprintln!("Killing thread {:?}", thr.as_value());
+        thr.funcall::<_, _, Value>("terminate", ())
+            .expect("Failed to kill thread");
+    }
+}
+
+pub fn terminate_non_fork_safe_threads() {
+    let ruby = Ruby::get().unwrap();
+    let thread_class = ruby.class_thread();
+    let current: Thread = ruby.thread_current();
+    let threads: RArray = thread_class
+        .funcall(*ID_LIST, ())
+        .expect("Failed to list Ruby threads");
+
+    let non_fork_safe_threads = threads
+        .into_iter()
+        .filter_map(|v| {
+            let v_thread = Thread::from_value(v).unwrap();
+            let non_fork_safe = !v_thread
+                .funcall::<_, _, bool>(*ID_EQ, (current,))
+                .unwrap_or(false)
+                && !v_thread
+                    .funcall::<_, _, bool>(*ID_THREAD_VARIABLE_GET, (ruby.sym_new("fork_safe"),))
+                    .unwrap_or(false);
+            if non_fork_safe { Some(v_thread) } else { None }
+        })
+        .collect::<Vec<_>>();
+
+    kill_threads(non_fork_safe_threads);
+}

data/crates/itsi_scheduler/Cargo.toml

@@ -10,7 +10,15 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.6.2" }
+magnus = { version = "0.7.1", features = ["rb-sys", "bytes"] }
+derive_more = { version = "2.0.1", features = ["debug"] }
 itsi_tracing = { path = "../itsi_tracing" }
 itsi_rb_helpers = { path = "../itsi_rb_helpers" }
 itsi_error = { path = "../itsi_error" }
+itsi_instrument_entry = { path = "../itsi_instrument_entry" }
+parking_lot = "0.12.3"
+mio = { version = "1.0.3", features = ["os-poll", "os-ext"] }
+rb-sys = "0.9.105"
+bytes = "1.10.1"
+nix = "0.29.0"
+tracing = "0.1.41"

data/crates/itsi_scheduler/extconf.rb

@@ -3,4 +3,4 @@
 require "mkmf"
 require "rb_sys/mkmf"
 
-create_rust_makefile("itsi_scheduler/itsi_scheduler")
+create_rust_makefile("itsi/scheduler/itsi_scheduler")

data/crates/itsi_scheduler/src/itsi_scheduler/io_helpers.rs

@@ -0,0 +1,56 @@
+use std::os::fd::RawFd;
+
+use itsi_error::{ItsiError, Result};
+use mio::Interest;
+use nix::libc::{fcntl, poll, pollfd, F_GETFL, F_SETFL, O_NONBLOCK};
+
+use super::Readiness;
+
+pub fn set_nonblocking(fd: RawFd) -> itsi_error::Result<()> {
+    unsafe {
+        let flags = fcntl(fd, F_GETFL);
+        if flags < 0 {
+            return Err(ItsiError::ArgumentError(format!(
+                "fcntl(F_GETFL) error for fd {}: {}",
+                fd,
+                std::io::Error::last_os_error()
+            )));
+        }
+        let new_flags = flags | O_NONBLOCK;
+        if fcntl(fd, F_SETFL, new_flags) < 0 {
+            return Err(ItsiError::ArgumentError(format!(
+                "fcntl(F_SETFL) error for fd {}: {}",
+                fd,
+                std::io::Error::last_os_error()
+            )));
+        }
+    }
+    Ok(())
+}
+
+pub fn poll_readiness(fd: RawFd, events: i16) -> Option<Readiness> {
+    let mut pfd = pollfd {
+        fd,
+        events,
+        revents: 0,
+    };
+    let ret = unsafe { poll(&mut pfd as *mut pollfd, 1, 0) };
+    if ret > 0 {
+        return Some(Readiness(pfd.revents));
+    }
+    None
+}
+
+pub fn build_interest(events: i16) -> Result<Interest> {
+    let mut interest_opt = None;
+    if events & 1 != 0 {
+        interest_opt = Some(Interest::READABLE);
+    }
+    if events & 4 != 0 {
+        interest_opt = Some(match interest_opt {
+            Some(i) => i | Interest::WRITABLE,
+            None => Interest::WRITABLE,
+        });
+    }
+    interest_opt.ok_or_else(|| ItsiError::ArgumentError("No valid event specified".to_owned()))
+}

data/crates/itsi_scheduler/src/itsi_scheduler/io_waiter.rs

@@ -0,0 +1,44 @@
+use derive_more::Debug;
+use mio::{event::Source, unix::SourceFd, Interest, Token};
+use std::os::fd::RawFd;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct IoWaiter {
+    pub fd: RawFd,
+    pub readiness: i16,
+    pub token: Token,
+}
+
+impl IoWaiter {
+    pub fn new(fd: RawFd, readiness: i16, token: Token) -> Self {
+        Self {
+            fd,
+            readiness,
+            token,
+        }
+    }
+}
+
+impl Source for IoWaiter {
+    fn register(
+        &mut self,
+        registry: &mio::Registry,
+        token: Token,
+        interests: Interest,
+    ) -> std::io::Result<()> {
+        SourceFd(&self.fd).register(registry, token, interests)
+    }
+
+    fn reregister(
+        &mut self,
+        registry: &mio::Registry,
+        token: Token,
+        interests: Interest,
+    ) -> std::io::Result<()> {
+        SourceFd(&self.fd).reregister(registry, token, interests)
+    }
+
+    fn deregister(&mut self, registry: &mio::Registry) -> std::io::Result<()> {
+        SourceFd(&self.fd).deregister(registry)
+    }
+}