prosody 0.1.1

data/ext/prosody/src/scheduler/processor.rs

@@ -0,0 +1,166 @@
+//! Provides functionality to schedule and execute Rust functions within the
+//! Ruby runtime.
+//!
+//! This module contains the implementation of `RubyProcessor`, which interfaces
+//! with a Ruby `AsyncTaskProcessor` class to manage task execution,
+//! cancellation, and lifecycle management.
+
+use crate::bridge::Bridge;
+use crate::scheduler::cancellation::CancellationToken;
+use crate::scheduler::result::ResultSender;
+use crate::util::ThreadSafeValue;
+use crate::{ROOT_MOD, id};
+use educe::Educe;
+use magnus::value::ReprValue;
+use magnus::{Class, Error, Module, RClass, Ruby, Value};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering::Relaxed;
+
+/// A thread-safe wrapper around a Ruby `AsyncTaskProcessor` instance.
+///
+/// This struct provides an interface for submitting Rust functions to be
+/// executed in the Ruby runtime, managing task lifecycle, and handling task
+/// cleanup upon shutdown.
+#[derive(Clone, Educe)]
+#[educe(Debug)]
+pub struct RubyProcessor {
+    /// Thread-safe reference to the Ruby `AsyncTaskProcessor` instance
+    processor: Arc<ThreadSafeValue>,
+
+    /// Bridge for communicating between Rust and Ruby threads
+    bridge: Bridge,
+
+    /// Flag indicating whether the processor is in shutdown state
+    is_shutdown: Arc<AtomicBool>,
+}
+
+impl RubyProcessor {
+    /// Creates a new `RubyProcessor` instance.
+    ///
+    /// Instantiates a Ruby `AsyncTaskProcessor` class and starts it, making it
+    /// ready to accept tasks.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `bridge` - Bridge for communicating between Rust and Ruby threads
+    ///
+    /// # Returns
+    ///
+    /// A new `RubyProcessor` instance
+    ///
+    /// # Errors
+    ///
+    /// Returns a Magnus error if:
+    /// - The `AsyncTaskProcessor` Ruby class cannot be found
+    /// - Instantiation of the processor fails
+    /// - Starting the processor fails
+    pub fn new(ruby: &Ruby, bridge: Bridge) -> Result<Self, Error> {
+        let module = ruby.get_inner(&ROOT_MOD);
+        let logger: Value = module.funcall(id!(ruby, "logger"), ())?;
+
+        let class: RClass = module.const_get(id!(ruby, "AsyncTaskProcessor"))?;
+        let instance: Value = class.new_instance((logger,))?;
+        let _: Value = instance.funcall(id!(ruby, "start"), ())?;
+        Ok(Self {
+            processor: Arc::new(ThreadSafeValue::new(instance, bridge.clone())),
+            bridge,
+            is_shutdown: Arc::default(),
+        })
+    }
+
+    /// Submits a function to be executed in the Ruby runtime.
+    ///
+    /// This method packages a Rust function along with metadata and callback
+    /// mechanisms to be executed by the Ruby processor. It handles
+    /// propagation of tracing context and ensures proper result handling.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `task_id` - Unique identifier for the task
+    /// * `carrier` - OpenTelemetry context carrier for trace propagation
+    /// * `result_receiver` - Channel for reporting task execution results
+    /// * `function` - The Rust function to execute in Ruby's context
+    ///
+    /// # Returns
+    ///
+    /// A `CancellationToken` that can be used to cancel the task if needed
+    ///
+    /// # Errors
+    ///
+    /// Returns a Magnus error if:
+    /// - The processor is shutting down
+    /// - The function submission to Ruby fails
+    pub fn submit<F>(
+        &self,
+        ruby: &Ruby,
+        task_id: &str,
+        carrier: HashMap<String, String>,
+        event_context: HashMap<String, String>,
+        result_receiver: ResultSender,
+        function: F,
+    ) -> Result<CancellationToken, Error>
+    where
+        F: FnOnce(&Ruby) -> Result<(), Error> + Send + 'static,
+    {
+        if self.is_shutdown.load(Relaxed) {
+            return Err(Error::new(
+                ruby.exception_runtime_error(),
+                String::from("Processor is shutting down"),
+            ));
+        }
+
+        // Convert the result sender into a Ruby callback
+        let callback = result_receiver.into_proc(ruby);
+
+        // Wrap the function in a Ruby block that can be executed later
+        let mut maybe_function = Some(function);
+        let block = ruby.proc_from_fn(move |ruby, _, _| {
+            if let Some(function) = maybe_function.take() {
+                function(ruby)
+            } else {
+                Ok(())
+            }
+        });
+
+        // Convert the tracing context carrier and event context to Ruby hashes
+        let carrier = ruby.hash_from_iter(carrier);
+        let event_context = ruby.hash_from_iter(event_context);
+
+        // Call the Ruby method: def submit(task_id, carrier, event_context, callback,
+        // &task_block)
+        let token = CancellationToken::new(
+            self.processor.get(ruby).funcall_with_block(
+                id!(ruby, "submit"),
+                (task_id, carrier, event_context, callback),
+                block,
+            )?,
+            self.bridge.clone(),
+        );
+
+        Ok(token)
+    }
+
+    /// Stops the processor, preventing it from accepting new tasks.
+    ///
+    /// This method is idempotent - calling it multiple times will only
+    /// stop the processor once.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    ///
+    /// # Errors
+    ///
+    /// Returns a Magnus error if the stop operation fails in Ruby
+    pub fn stop(&self, ruby: &Ruby) -> Result<(), Error> {
+        if !self.is_shutdown.fetch_or(true, Relaxed) {
+            let _: Value = self.processor.get(ruby).funcall(id!(ruby, "stop"), ())?;
+        }
+
+        Ok(())
+    }
+}

data/ext/prosody/src/scheduler/result.rs

@@ -0,0 +1,197 @@
+//! # Result Communication
+//!
+//! Provides asynchronous result communication between Ruby and Rust.
+//!
+//! This module implements a channel-based system for Ruby task results to be
+//! communicated back to Rust. It supports different error categories
+//! (transient vs permanent) and ensures one-time result delivery through
+//! atomic guarantees.
+
+use crate::id;
+use atomic_take::AtomicTake;
+use educe::Educe;
+use magnus::block::Proc;
+use magnus::value::ReprValue;
+use magnus::{Error, Ruby, TryConvert, Value, kwargs};
+use prosody::error::{ClassifyError, ErrorCategory};
+use thiserror::Error;
+use tokio::sync::oneshot;
+use tracing::debug;
+
+/// Creates a paired sender and receiver for task result communication.
+///
+/// Creates a one-shot channel wrapped with `AtomicTake` to ensure
+/// the result is only sent once, preventing duplicate delivery.
+///
+/// # Returns
+///
+/// A tuple containing a connected `ResultSender` and `ResultReceiver` pair.
+pub fn result_channel() -> (ResultSender, ResultReceiver) {
+    let (result_tx, result_rx) = oneshot::channel();
+    let result_tx = AtomicTake::new(result_tx);
+    (ResultSender { result_tx }, ResultReceiver { result_rx })
+}
+
+/// Sends task results from Ruby back to Rust.
+///
+/// Wraps a one-shot sender in an `AtomicTake` to guarantee that the result
+/// can only be sent once, preventing accidental duplicate sends.
+#[derive(Educe)]
+#[educe(Debug)]
+pub struct ResultSender {
+    #[educe(Debug(ignore))]
+    result_tx: AtomicTake<oneshot::Sender<Result<(), ProcessingError>>>,
+}
+
+/// Receives task results in Rust from Ruby.
+///
+/// Wraps a one-shot receiver that allows asynchronously waiting
+/// for a task to complete or fail.
+#[derive(Educe)]
+#[educe(Debug)]
+pub struct ResultReceiver {
+    #[educe(Debug(ignore))]
+    result_rx: oneshot::Receiver<Result<(), ProcessingError>>,
+}
+
+impl ResultSender {
+    /// Sends a result from Ruby to Rust.
+    ///
+    /// Takes a Ruby result value, interprets it as a success or failure,
+    /// and sends it through the channel. For failures, it extracts meaningful
+    /// error messages from the Ruby exception and categorizes them as
+    /// permanent or transient.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `is_success` - Boolean indicating if the operation succeeded
+    /// * `result` - For successes, the return value; for failures, the Ruby
+    ///   exception
+    ///
+    /// # Returns
+    ///
+    /// `true` if the result was sent successfully, `false` if the result had
+    /// already been sent or the receiver was dropped.
+    pub fn send(&self, ruby: &Ruby, is_success: bool, result: Value) -> bool {
+        let Some(result_tx) = self.result_tx.take() else {
+            debug!("result was already sent");
+            return false;
+        };
+
+        if is_success {
+            if result_tx.send(Ok(())).is_err() {
+                debug!("discarding result; receiver went away");
+            }
+
+            return true;
+        }
+
+        // For error results, determine if the error is permanent by calling
+        // the `permanent?` method on the Ruby exception
+        let is_permanent = result.funcall(id!(ruby, "permanent?"), ()).unwrap_or(false);
+
+        // Extract a detailed error message from the Ruby exception
+        let error_string: String = result
+            .funcall(id!(ruby, "full_message"), (kwargs!("highlight" => false),))
+            .or_else(|_| result.funcall(id!(ruby, "inspect"), ()))
+            .unwrap_or_else(|_| result.to_string());
+
+        let error = if is_permanent {
+            ProcessingError::Permanent(error_string)
+        } else {
+            ProcessingError::Transient(error_string)
+        };
+
+        if result_tx.send(Err(error)).is_err() {
+            debug!("discarding result; receiver went away");
+            return false;
+        }
+
+        true
+    }
+
+    /// Converts this sender into a Ruby Proc for callback-style integration.
+    ///
+    /// Creates a Ruby Proc that, when called with a success status and result
+    /// value, will send that result through this sender back to Rust.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    ///
+    /// # Returns
+    ///
+    /// A Ruby `Proc` that can be passed to Ruby code as a callback
+    pub fn into_proc(self, ruby: &Ruby) -> Proc {
+        ruby.proc_from_fn(move |ruby, args, _block| match args {
+            [is_success, result, ..] => {
+                let was_sent = self.send(ruby, bool::try_convert(*is_success)?, *result);
+                Ok(was_sent)
+            }
+            _ => Err(Error::new(
+                ruby.exception_arg_error(),
+                format!("Expected two arguments but received {}", args.len()),
+            )),
+        })
+    }
+}
+
+impl ResultReceiver {
+    /// Asynchronously waits for a result from Ruby.
+    ///
+    /// Awaits the task completion in Ruby and returns its result.
+    /// This consumes the receiver, ensuring the result can only be awaited
+    /// once.
+    ///
+    /// # Returns
+    ///
+    /// `Ok(())` if the task completed successfully
+    ///
+    /// # Errors
+    ///
+    /// Returns a `ProcessingError` if:
+    /// - The task failed (with either a permanent or transient error)
+    /// - The channel was closed unexpectedly (e.g., the Ruby VM terminated)
+    pub async fn receive(self) -> Result<(), ProcessingError> {
+        self.result_rx.await.map_err(|_| ProcessingError::Closed)?
+    }
+}
+
+/// Errors that can occur during task processing.
+///
+/// Represents error scenarios when processing tasks between Ruby and Rust,
+/// with categorization that informs handling strategies (e.g., retry policies).
+#[derive(Clone, Debug, Error)]
+pub enum ProcessingError {
+    /// A temporary error that may succeed on retry.
+    #[error("transient error: {0}")]
+    Transient(String),
+
+    /// A permanent error that will not succeed on retry.
+    #[error("permanent error: {0}")]
+    Permanent(String),
+
+    /// The result channel was closed before receiving a result.
+    ///
+    /// This typically happens when the Ruby VM terminates while a task is in
+    /// progress.
+    #[error("result channel has been closed")]
+    Closed,
+}
+
+impl ClassifyError for ProcessingError {
+    /// Classifies errors to determine retry behavior.
+    ///
+    /// Maps error types to their appropriate retry categories:
+    /// - Transient errors are retryable
+    /// - Permanent errors should not be retried
+    /// - Channel closure is treated as a transient error, allowing for retries
+    ///   when communication is restored
+    fn classify_error(&self) -> ErrorCategory {
+        match self {
+            ProcessingError::Transient(_) | ProcessingError::Closed => ErrorCategory::Transient,
+            ProcessingError::Permanent(_) => ErrorCategory::Permanent,
+        }
+    }
+}

data/ext/prosody/src/tracing_util.rs

@@ -0,0 +1,56 @@
+//! OpenTelemetry tracing utilities for Ruby-Rust integration.
+//!
+//! This module provides shared utilities for extracting and propagating
+//! OpenTelemetry context across the Ruby-Rust boundary for distributed tracing.
+use crate::id;
+use magnus::value::ReprValue;
+use magnus::{Error, Module, RModule, Ruby, Value};
+use opentelemetry::Context;
+use opentelemetry::propagation::{TextMapCompositePropagator, TextMapPropagator};
+use std::collections::HashMap;
+
+/// Extracts OpenTelemetry context from Ruby's current tracing environment.
+///
+/// This function extracts the current OpenTelemetry context from Ruby and
+/// converts it to a Rust context that can be used for distributed tracing.
+/// The context is extracted using Ruby's OpenTelemetry propagation mechanism
+/// and then parsed using the provided propagator.
+///
+/// # Arguments
+///
+/// * `ruby` - Reference to the Ruby VM
+/// * `propagator` - The OpenTelemetry propagator to use for context extraction
+///
+/// # Returns
+///
+/// The extracted OpenTelemetry context that can be used as a parent for new
+/// spans.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - OpenTelemetry module is not available in Ruby
+/// - Context extraction or propagation fails
+/// - Ruby-to-Rust type conversion fails
+///
+/// # Example
+///
+/// ```rust
+/// let context = extract_opentelemetry_context(ruby, &propagator)?;
+/// let span = info_span!("operation");
+/// span.set_parent(context);
+/// ```
+pub fn extract_opentelemetry_context(
+    ruby: &Ruby,
+    propagator: &TextMapCompositePropagator,
+) -> Result<Context, Error> {
+    let carrier = ruby.hash_new();
+    let otel_class: RModule = ruby.class_module().const_get(id!(ruby, "OpenTelemetry"))?;
+    let propagator_obj: Value = otel_class.funcall(id!(ruby, "propagation"), ())?;
+    let _: Value = propagator_obj.funcall(id!(ruby, "inject"), (carrier,))?;
+
+    let carrier: HashMap<String, String> = carrier.to_hash_map()?;
+    let context = propagator.extract(&carrier);
+
+    Ok(context)
+}

data/ext/prosody/src/util.rs

@@ -0,0 +1,219 @@
+//! # Utilities for Ruby/Rust interoperability
+//!
+//! This module provides utilities for safe and efficient interaction between
+//! Rust and Ruby, particularly focusing on thread-safety and efficient symbol
+//! handling.
+
+use crate::bridge::Bridge;
+use crate::logging::Logger;
+use crate::{BRIDGE, RUNTIME, TRACING_INIT};
+use magnus::value::BoxValue;
+use magnus::{Ruby, Value};
+use prosody::tracing::initialize_tracing;
+use std::mem::{ManuallyDrop, forget};
+use tokio::runtime::{EnterGuard, Handle};
+use tracing::warn;
+
+/// Creates a static Ruby identifier (symbol) for efficient reuse.
+///
+/// This macro creates a lazily-initialized static Ruby identifier from a string
+/// literal. Using this macro for frequently accessed Ruby method names or
+/// symbols avoids repeatedly converting strings to Ruby symbols at runtime.
+///
+/// This macro requires a `ruby: &Ruby` parameter to enforce that it can only
+/// be used within a Ruby thread context, ensuring thread safety.
+///
+/// # Examples
+///
+/// ```
+/// let method_name = id!(ruby, "to_s");
+/// // Use method_name with Ruby function calls
+/// ```
+#[macro_export]
+macro_rules! id {
+    ($ruby:expr, $str:expr) => {{
+        static VAL: magnus::value::LazyId = magnus::value::LazyId::new($str);
+        let _ruby: &magnus::Ruby = $ruby; // Enforce that ruby is &Ruby type
+        *VAL
+    }};
+}
+
+/// A thread-safe wrapper around a Ruby value.
+///
+/// Provides a way to safely share Ruby values between threads by wrapping
+/// them in a type that implements both `Send` and `Sync`. This type enforces
+/// that the underlying Ruby value is only accessed within a Ruby thread
+/// context.
+///
+/// # Drop safety
+///
+/// [`BoxValue`] must be dropped on a Ruby thread because its `Drop` impl calls
+/// `rb_gc_unregister_address`. This type handles that automatically: if dropped
+/// on a Ruby thread, the value is cleaned up normally. If dropped on a
+/// non-Ruby thread, the value is sent through the bridge for deferred cleanup
+/// on the Ruby thread. If the bridge is unavailable (shutdown), the value is
+/// leaked with a warning rather than risking a segfault.
+#[derive(Debug)]
+pub struct ThreadSafeValue {
+    value: ManuallyDrop<RubyDrop>,
+    bridge: Bridge,
+}
+
+// SAFETY: The underlying value can only be accessed from a Ruby thread
+// (enforced by requiring `&Ruby` in `get`). Drop is safe across threads: it
+// sends cleanup through the bridge when not on a Ruby thread, falling back
+// to a leak if the bridge is unavailable.
+unsafe impl Send for ThreadSafeValue {}
+
+// SAFETY: `get` requires `&Ruby`, ensuring the value is only read on a Ruby
+// thread. The inner `BoxValue` is never mutated after construction.
+unsafe impl Sync for ThreadSafeValue {}
+
+impl ThreadSafeValue {
+    /// Creates a new thread-safe wrapper around a Ruby value.
+    ///
+    /// # Arguments
+    ///
+    /// * `value` - The Ruby value to wrap
+    /// * `bridge` - The bridge used to defer cleanup onto the Ruby thread
+    pub fn new(value: Value, bridge: Bridge) -> Self {
+        Self {
+            value: ManuallyDrop::new(RubyDrop::new(value)),
+            bridge,
+        }
+    }
+
+    /// Gets a reference to the wrapped Ruby value.
+    ///
+    /// This method ensures that access to the Ruby value only happens
+    /// within a Ruby thread context by requiring a `Ruby` reference.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - A reference to the Ruby VM
+    pub fn get(&self, ruby: &Ruby) -> &Value {
+        self.value.get(ruby)
+    }
+}
+
+impl Drop for ThreadSafeValue {
+    fn drop(&mut self) {
+        // SAFETY: `drop` is called exactly once per value, so this
+        // `ManuallyDrop::take` cannot double-free.
+        let inner = unsafe { ManuallyDrop::take(&mut self.value) };
+
+        // On a Ruby thread: safe to drop directly.
+        if RubyDrop::can_drop() {
+            drop(inner);
+            return;
+        }
+
+        // On a non-Ruby thread: send to the bridge for cleanup on the Ruby
+        // thread. When the closure runs, `inner` is dropped on the Ruby thread
+        // and `RubyDrop::Drop` takes the normal cleanup path.
+        //
+        // If the send fails (bridge shut down), the `SendError` owns the
+        // closure which owns `inner`. When `SendError` drops here on the
+        // non-Ruby thread, `RubyDrop::Drop` takes the leak path and emits
+        // its own warning — no need to warn here too.
+        let _ = self.bridge.send(Box::new(move |_ruby| drop(inner)));
+    }
+}
+
+/// Wraps a [`BoxValue`] and ensures it is dropped on a Ruby thread.
+///
+/// `BoxValue::drop` calls `rb_gc_unregister_address`, which must only run on
+/// a Ruby thread. `RubyDrop` handles this safely: on a Ruby thread it drops
+/// normally; on any other thread it leaks with a warning rather than
+/// segfaulting. It has no knowledge of channels or bridges, so there is no
+/// risk of infinite recursion in `Drop`.
+#[derive(Debug)]
+struct RubyDrop(ManuallyDrop<BoxValue<Value>>);
+
+// SAFETY: Either dropped on a Ruby thread (normal cleanup) or leaked on a
+// non-Ruby thread (forget + warn). Never calls into Ruby from the wrong thread.
+unsafe impl Send for RubyDrop {}
+
+impl RubyDrop {
+    fn new(value: Value) -> Self {
+        Self(ManuallyDrop::new(BoxValue::new(value)))
+    }
+
+    /// Gets a reference to the wrapped Ruby value.
+    ///
+    /// This method ensures that access to the Ruby value only happens
+    /// within a Ruby thread context by requiring a `Ruby` reference.
+    ///
+    /// # Arguments
+    ///
+    /// * `_ruby` - A reference to the Ruby VM
+    fn get(&self, _ruby: &Ruby) -> &Value {
+        &self.0
+    }
+
+    /// Returns `true` if the current thread is a Ruby thread and it is safe
+    /// to call `rb_gc_unregister_address` (i.e. drop the inner [`BoxValue`]).
+    fn can_drop() -> bool {
+        Ruby::get().is_ok()
+    }
+}
+
+impl Drop for RubyDrop {
+    fn drop(&mut self) {
+        // SAFETY: `drop` is called exactly once per value.
+        let inner = unsafe { ManuallyDrop::take(&mut self.0) };
+
+        if Ruby::get().is_ok() {
+            drop(inner);
+            return;
+        }
+
+        warn!(
+            "leaked a Ruby value because it was dropped on a non-Ruby thread; this is safe but \
+             indicates a value outlived its expected scope"
+        );
+        forget(inner);
+    }
+}
+
+/// Ensures a Tokio runtime context exists, entering one if necessary.
+///
+/// Only creates a runtime guard when not already in a runtime context, avoiding
+/// `EnterGuard` ordering violations that panic.
+///
+/// Lazily initializes the bridge and tracing subsystems on first call. This
+/// deferred initialization is critical for fork safety—each process gets its
+/// own bridge channels and tracing state rather than inheriting stale handles
+/// from the parent.
+///
+/// # Returns
+///
+/// `Some(EnterGuard)` if we entered a new runtime (hold the guard), or `None`
+/// if already in a runtime context.
+///
+/// # Examples
+///
+/// ```rust
+/// let _guard = ensure_runtime_context(ruby);
+/// // Safe to perform async operations
+/// ```
+pub fn ensure_runtime_context(ruby: &Ruby) -> Option<EnterGuard<'static>> {
+    let guard = Handle::try_current().is_err().then(|| RUNTIME.enter());
+
+    // Set up the bridge for Ruby-Rust communication
+    let bridge = BRIDGE.get_or_init(|| Bridge::new(ruby));
+
+    // Initialize tracing for observability
+    #[allow(clippy::print_stderr, reason = "logger has not been initialized yet")]
+    TRACING_INIT.get_or_init(|| {
+        let maybe_logger = Logger::new(ruby, bridge.clone())
+            .inspect_err(|error| eprintln!("failed to create logger: {error:#}"))
+            .ok();
+
+        if let Err(error) = initialize_tracing(maybe_logger) {
+            eprintln!("failed to initialize tracing: {error:#}");
+        }
+    });
+
+    guard
+}