prosody 0.1.1

data/Rakefile

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rb_sys/cargo/metadata"
+require "rb_sys/extensiontask"
+
+task build: :compile
+
+GEMSPEC = Gem::Specification.load("prosody.gemspec")
+
+begin
+  RbSys::ExtensionTask.new("prosody", GEMSPEC) do |ext|
+    ext.lib_dir = "lib/prosody"
+  end
+rescue RbSys::CargoMetadataError
+  # This is expected for the source gem, which can't be installed directly
+  warn "Source gem cannot be installed directly, must be a supported platform"
+end
+
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "standard/rake"
+
+task default: %i[compile spec standard]

data/ext/prosody/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+edition = "2024"
+license = "MIT"
+name = "prosody"
+publish = false
+version = "0.1.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+atomic-take.workspace = true
+bumpalo = { workspace = true, features = ["collections"] }
+educe.workspace = true
+futures.workspace = true
+magnus = { workspace = true, default-features = false }
+opentelemetry.workspace = true
+prosody = { workspace = true }
+rb-sys.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde-untagged.workspace = true
+serde_magnus.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync"] }
+tokio-stream.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
+
+
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+tikv-jemallocator = { workspace = true, features = [
+    "disable_initial_exec_tls",
+] }
+
+
+[lints]
+workspace = true

data/ext/prosody/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("prosody/prosody")

data/ext/prosody/src/admin.rs

@@ -0,0 +1,171 @@
+//! # Admin Client Module
+//!
+//! Provides administrative capabilities for Kafka through the Prosody library.
+//! This module implements Ruby bindings for creating and deleting Kafka topics.
+
+use crate::bridge::Bridge;
+use crate::util::ensure_runtime_context;
+use crate::{ROOT_MOD, id};
+use magnus::value::ReprValue;
+use magnus::{Error, Module, Object, Ruby, Value, function, method};
+use prosody::admin::{AdminConfiguration, ProsodyAdminClient, TopicConfiguration};
+use std::sync::Arc;
+use tracing::Span;
+
+/// Ruby wrapper for the Prosody admin client.
+///
+/// This struct provides administrative operations for Kafka topics, such as
+/// creating and deleting topics. It wraps the Rust `ProsodyAdminClient` and
+/// uses the bridge mechanism to handle asynchronous operations from Ruby.
+#[magnus::wrap(class = "Prosody::AdminClient")]
+pub struct AdminClient {
+    /// The underlying Prosody admin client
+    client: Arc<ProsodyAdminClient>,
+    /// Bridge for executing asynchronous operations from Ruby
+    bridge: Bridge,
+    /// PID at construction time, used to detect post-fork usage
+    pid: u32,
+}
+
+impl AdminClient {
+    /// Creates a new admin client with the specified bootstrap servers.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `bootstrap_servers` - List of Kafka bootstrap server addresses
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Magnus::Error` if:
+    /// - The client cannot be created with the provided bootstrap servers
+    /// - The bridge is not initialized
+    #[allow(clippy::needless_pass_by_value)]
+    pub fn new(ruby: &Ruby, bootstrap_servers: Vec<String>) -> Result<Self, Error> {
+        let _guard = ensure_runtime_context(ruby);
+        let admin_config = AdminConfiguration::new(bootstrap_servers)
+            .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))?;
+
+        let client = Arc::new(
+            ProsodyAdminClient::new(&admin_config)
+                .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))?,
+        );
+
+        let bridge = crate::BRIDGE
+            .get()
+            .ok_or(Error::new(
+                ruby.exception_runtime_error(),
+                "Bridge not initialized",
+            ))?
+            .clone();
+
+        Ok(Self {
+            client,
+            bridge,
+            pid: std::process::id(),
+        })
+    }
+
+    fn check_fork(ruby: &Ruby, this: &Self) -> Result<(), Error> {
+        if std::process::id() != this.pid {
+            return Err(Error::new(
+                ruby.exception_runtime_error(),
+                "Prosody::AdminClient cannot be used after fork. Create a new client in the child \
+                 process.",
+            ));
+        }
+        Ok(())
+    }
+
+    /// Creates a new Kafka topic.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `this` - The admin client instance
+    /// * `name` - Name of the topic to create
+    /// * `partition_count` - Number of partitions for the topic
+    /// * `replication_factor` - Replication factor for the topic
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Magnus::Error` if:
+    /// - The topic creation fails
+    /// - There's an issue with the asynchronous execution
+    pub fn create_topic(
+        ruby: &Ruby,
+        this: &Self,
+        name: String,
+        partition_count: u16,
+        replication_factor: u16,
+    ) -> Result<(), Error> {
+        Self::check_fork(ruby, this)?;
+        let topic_config = TopicConfiguration::builder()
+            .name(name)
+            .partition_count(partition_count)
+            .replication_factor(replication_factor)
+            .build()
+            .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))?;
+
+        let client = this.client.clone();
+        let future = async move { client.create_topic(&topic_config).await };
+
+        this.bridge
+            .wait_for(ruby, future, Span::current())?
+            .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))
+    }
+
+    /// Deletes a Kafka topic.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM
+    /// * `this` - The admin client instance
+    /// * `name` - Name of the topic to delete
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Magnus::Error` if:
+    /// - The topic deletion fails
+    /// - There's an issue with the asynchronous execution
+    pub fn delete_topic(ruby: &Ruby, this: &Self, name: String) -> Result<(), Error> {
+        Self::check_fork(ruby, this)?;
+        let client = this.client.clone();
+        let future = async move { client.delete_topic(&name).await };
+
+        this.bridge
+            .wait_for(ruby, future, Span::current())?
+            .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))
+    }
+}
+
+/// Initializes the admin module by registering the `Prosody::AdminClient`
+/// class.
+///
+/// # Arguments
+///
+/// * `ruby` - Reference to the Ruby VM
+///
+/// # Errors
+///
+/// Returns a `Magnus::Error` if class or method definition fails
+pub fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.get_inner(&ROOT_MOD);
+    let class_id = id!(ruby, "AdminClient");
+    let class = module.define_class(class_id, ruby.class_object())?;
+
+    class.define_singleton_method("new", function!(AdminClient::new, 1))?;
+    class.define_method(
+        id!(ruby, "create_topic"),
+        method!(AdminClient::create_topic, 3),
+    )?;
+    class.define_method(
+        id!(ruby, "delete_topic"),
+        method!(AdminClient::delete_topic, 1),
+    )?;
+
+    // Make the admin client class private
+    let _: Value = module.funcall(id!(ruby, "private_constant"), (class_id,))?;
+
+    Ok(())
+}

data/ext/prosody/src/bridge/callback.rs

@@ -0,0 +1,60 @@
+//! This module provides functionality to bridge between Rust and Ruby, allowing
+//! asynchronous operations in Rust to communicate with the Ruby runtime.
+//!
+//! It includes mechanisms for executing Ruby code from Rust threads and for
+//! managing asynchronous callback communication between the two languages.
+
+use crate::bridge::Bridge;
+use crate::id;
+use crate::util::ThreadSafeValue;
+use magnus::value::ReprValue;
+use magnus::{Error, IntoValue, Ruby, Value};
+
+/// Handles asynchronous callbacks from Rust to Ruby using a thread-safe queue.
+///
+/// This struct provides a way to send values back to Ruby from asynchronous
+/// Rust operations, ensuring thread safety by wrapping the Ruby queue in a
+/// thread-safe container.
+pub struct AsyncCallback {
+    /// A thread-safe wrapper around a Ruby Queue object
+    queue: ThreadSafeValue,
+}
+
+impl AsyncCallback {
+    /// Creates a new `AsyncCallback` from a Ruby Queue object.
+    ///
+    /// # Arguments
+    ///
+    /// * `queue` - A Ruby Queue object that will receive values from Rust
+    /// * `bridge` - The bridge used to defer cleanup of the wrapped queue
+    ///   value onto the Ruby thread when the callback is dropped
+    pub fn from_queue(queue: Value, bridge: Bridge) -> Self {
+        Self {
+            queue: ThreadSafeValue::new(queue, bridge),
+        }
+    }
+
+    /// Completes the callback by pushing a value to the associated Ruby queue.
+    ///
+    /// This consumes the callback, preventing it from being used multiple
+    /// times.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - A reference to the Ruby VM
+    /// * `value` - The value to push to the queue, which will be converted to a
+    ///   Ruby value
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the Ruby function call fails
+    pub fn complete<V>(self, ruby: &Ruby, value: V) -> Result<(), Error>
+    where
+        V: IntoValue,
+    {
+        self.queue
+            .get(ruby)
+            .funcall(id!(ruby, "push"), (value.into_value_with(ruby),))
+            .map(|_: Value| ())
+    }
+}

data/ext/prosody/src/bridge/mod.rs

@@ -0,0 +1,332 @@
+//! # Bridge Module
+//!
+//! Facilitates communication between Ruby and Rust, particularly for handling
+//! asynchronous operations. This module creates a dedicated Ruby thread that
+//! processes functions sent from Rust, allowing async Rust code to interact
+//! safely with the Ruby runtime.
+
+use crate::bridge::callback::AsyncCallback;
+use crate::gvl::{GvlError, without_gvl};
+use crate::{ROOT_MOD, RUNTIME, id};
+use atomic_take::AtomicTake;
+use educe::Educe;
+use futures::executor::block_on;
+use magnus::value::{Lazy, ReprValue};
+use magnus::{Error, Module, RClass, Ruby, Value};
+use std::any::Any;
+use thiserror::Error;
+use tokio::select;
+use tokio::sync::mpsc::error::SendError;
+use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel};
+use tokio::sync::oneshot;
+use tracing::{Instrument, Span, debug, error, warn};
+
+/// Helper module for handling callbacks from async Rust code to Ruby.
+mod callback;
+
+/// Maximum number of commands to process in a single poll operation.
+const POLL_BATCH_SIZE: usize = 16;
+
+/// Lazily initialized reference to Ruby's Thread class.
+#[allow(clippy::expect_used)]
+pub static THREAD_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+    ruby.class_object()
+        .const_get(id!(ruby, "Thread"))
+        .expect("Failed to load Thread class")
+});
+
+/// Lazily initialized reference to Ruby's `Thread::Queue` class.
+#[allow(clippy::expect_used)]
+pub static QUEUE_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+    ruby.get_inner(&THREAD_CLASS)
+        .const_get(id!(ruby, "Queue"))
+        .expect("Failed to load Queue class")
+});
+
+/// Type alias for a boxed closure that can be executed in a Ruby context.
+pub(crate) type RubyFunction = Box<dyn FnOnce(&Ruby) + Send>;
+
+/// A wrapper for results returned from async operations to Ruby.
+///
+/// Uses `AtomicTake` to ensure the value can only be extracted once.
+#[derive(Debug)]
+#[magnus::wrap(class = "Prosody::DynamicResult")]
+struct DynamicResult(AtomicTake<Box<dyn Any + Send>>);
+
+/// Bridge between Ruby and Rust for async operations.
+///
+/// Creates a dedicated Ruby thread that processes functions sent from Rust,
+/// allowing async Rust code to interact safely with the Ruby runtime.
+#[derive(Educe, Clone)]
+#[educe(Debug)]
+pub struct Bridge {
+    /// Channel sender for submitting functions to be executed in the Ruby
+    /// context.
+    #[educe(Debug(ignore))]
+    tx: UnboundedSender<RubyFunction>,
+}
+
+impl Bridge {
+    /// Creates a new Bridge.
+    ///
+    /// The internal command channel is unbounded. In typical request/response
+    /// flows, callers wait for Ruby to process commands, which tends to bound
+    /// queue growth in practice. However, fire-and-forget uses (e.g. from
+    /// `Drop` implementations on non-Ruby threads) can enqueue without
+    /// blocking, so growth is unbounded if the Ruby bridge thread is stalled.
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM.
+    ///
+    /// # Returns
+    ///
+    /// A new `Bridge` instance.
+    pub fn new(ruby: &Ruby) -> Self {
+        let (tx, mut rx) = unbounded_channel();
+
+        // Create a dedicated Ruby thread to process functions
+        ruby.thread_create_from_fn(move |ruby| {
+            loop {
+                let Err(error) = poll(&mut rx, ruby) else {
+                    continue;
+                };
+
+                if matches!(error, BridgeError::Shutdown) {
+                    debug!("shutting down Ruby bridge");
+                    break;
+                }
+
+                error!("error during Ruby bridge poll: {error:#}");
+            }
+        });
+
+        Self { tx }
+    }
+
+    /// Runs a function in the Ruby context and returns its result
+    /// asynchronously.
+    ///
+    /// # Arguments
+    ///
+    /// * `function` - Closure to run in the Ruby context.
+    ///
+    /// # Returns
+    ///
+    /// The result of the function wrapped in a `Result`.
+    ///
+    /// # Errors
+    ///
+    /// Returns `BridgeError::Shutdown` if the bridge has been shut down.
+    pub async fn run<F, T>(&self, function: F) -> Result<T, BridgeError>
+    where
+        F: FnOnce(&Ruby) -> T + Send + 'static,
+        T: Send + 'static,
+    {
+        let (result_tx, result_rx) = oneshot::channel();
+
+        // Wrap the function to capture its result
+        let function = |ruby: &Ruby| {
+            let result = function(ruby);
+            if result_tx.send(result).is_err() {
+                error!("failed to send Ruby bridge result");
+            }
+        };
+
+        self.tx
+            .send(Box::new(function))
+            .map_err(|_| BridgeError::Shutdown)?;
+
+        result_rx.await.map_err(|_| BridgeError::Shutdown)
+    }
+
+    /// Waits for a future to complete and returns its result to Ruby.
+    ///
+    /// This method bridges asynchronous Rust code with synchronous Ruby code
+    /// by:
+    /// 1. Creating a Ruby Queue to receive the result
+    /// 2. Spawning a Rust task to await the future
+    /// 3. Yielding until the result is pushed to the Queue
+    ///
+    /// # Arguments
+    ///
+    /// * `ruby` - Reference to the Ruby VM.
+    /// * `future` - Future to await.
+    ///
+    /// # Returns
+    ///
+    /// The output of the future.
+    ///
+    /// # Errors
+    ///
+    /// Returns a Magnus error if there's an issue with the Ruby interaction or
+    /// type conversion.
+    pub fn wait_for<Fut>(&self, ruby: &Ruby, future: Fut, span: Span) -> Result<Fut::Output, Error>
+    where
+        Fut: Future + Send + 'static,
+        Fut::Output: Send,
+    {
+        let cloned_span = span.clone();
+        let _enter = cloned_span.enter();
+
+        // Create a Ruby Queue to receive the result
+        let queue: Value = ruby.get_inner(&QUEUE_CLASS).funcall(id!(ruby, "new"), ())?;
+        let callback = AsyncCallback::from_queue(queue, self.clone());
+        let tx = self.tx.clone();
+
+        // Spawn a task to await the future and send the result back to Ruby
+        RUNTIME.spawn(
+            async move {
+                let result = future.await;
+                let function = move |ruby: &Ruby| {
+                    let value = DynamicResult(AtomicTake::new(Box::new(result)));
+                    if let Err(error) = callback.complete(ruby, value) {
+                        error!("failed to complete Ruby callback: {error:#}");
+                    }
+                };
+
+                if let Err(error) = tx.send(Box::new(function)) {
+                    error!("failed to send callback to Ruby: {error:#}");
+                }
+            }
+            .instrument(span),
+        );
+
+        // Yield until the result is available, then extract and return it
+        queue
+            .funcall::<_, _, &DynamicResult>(id!(ruby, "pop"), ())?
+            .0
+            .take()
+            .ok_or_else(|| Error::new(ruby.exception_runtime_error(), "result already taken"))?
+            .downcast::<Fut::Output>()
+            .map_err(|_| Error::new(ruby.exception_runtime_error(), "failed to downcast result"))
+            .map(|output| *output)
+    }
+
+    /// Synchronously enqueues a function for execution on the Ruby thread.
+    ///
+    /// Unlike [`run`](Self::run), this method does not await a result and can
+    /// be called from synchronous contexts such as `Drop` implementations.
+    /// Returns `Err` only if the bridge receiver has been dropped (shutdown).
+    pub(crate) fn send(&self, function: RubyFunction) -> Result<(), SendError<RubyFunction>> {
+        self.tx.send(function)
+    }
+}
+
+/// Polls for and executes functions in the Ruby context.
+///
+/// This function processes commands from the receiver channel, allowing them to
+/// be executed in the Ruby context. Since the GVL is expensive to acquire, this
+/// executes multiple commands in a single poll operation to reduce the
+/// overhead.
+///
+/// # Arguments
+///
+/// * `rx` - Receiver for commands to execute.
+/// * `ruby` - Reference to the Ruby VM.
+///
+/// # Returns
+///
+/// `Ok(())` if commands were processed successfully.
+///
+/// # Errors
+///
+/// Returns a `BridgeError` if there was an issue with polling or executing
+/// commands.
+fn poll(rx: &mut UnboundedReceiver<RubyFunction>, ruby: &Ruby) -> Result<(), BridgeError> {
+    // Set up cancellation channel
+    let (cancel_tx, cancel_rx) = oneshot::channel();
+    let mut maybe_cancel_tx = Some(cancel_tx);
+
+    // Function to be executed without the GVL (Global VM Lock)
+    let poll_fn = || {
+        // Wait for either a command or cancellation
+        let maybe_command = block_on(async {
+            select! {
+                _ = cancel_rx => Err(BridgeError::Cancelled),
+                result = rx.recv() => Ok(result),
+            }
+        })?;
+
+        let first_command = maybe_command.ok_or(BridgeError::Shutdown)?;
+
+        // Batch up to POLL_BATCH_SIZE commands
+        let mut commands = Vec::with_capacity(POLL_BATCH_SIZE);
+        commands.push(first_command);
+
+        while commands.len() < POLL_BATCH_SIZE {
+            if let Ok(command) = rx.try_recv() {
+                commands.push(command);
+            } else {
+                break;
+            }
+        }
+
+        Result::<_, BridgeError>::Ok(commands)
+    };
+
+    // Function to cancel polling if needed
+    let cancel_fn = || {
+        let Some(cancel_tx) = maybe_cancel_tx.take() else {
+            return;
+        };
+
+        if cancel_tx.send(()).is_err() {
+            warn!("Failed to cancel poll operation");
+        }
+    };
+
+    // Execute poll function without the GVL
+    let commands = without_gvl(poll_fn, cancel_fn)??;
+
+    // Execute each command in the Ruby context
+    for command in commands {
+        command(ruby);
+    }
+
+    Ok(())
+}
+
+/// Errors that can occur during bridge operations.
+#[derive(Debug, Error)]
+pub enum BridgeError {
+    /// An error occurred while polling with the GVL released.
+    #[error("an error occurred while polling: {0:#}")]
+    Gvl(#[from] GvlError),
+
+    /// Failed to create an async task.
+    #[error("failed to create async task: {0:#}")]
+    TaskCreate(String),
+
+    /// An async task was dropped before completion.
+    #[error("async task was dropped before completion")]
+    TaskDropped,
+
+    /// A command was cancelled by the Ruby runtime.
+    #[error("command was cancelled by Ruby runtime")]
+    Cancelled,
+
+    /// The Ruby bridge was shut down.
+    #[error("Ruby bridge was shutdown")]
+    Shutdown,
+}
+
+/// Initializes the bridge module in Ruby.
+///
+/// # Arguments
+///
+/// * `ruby` - Reference to the Ruby VM.
+///
+/// # Returns
+///
+/// `Ok(())` if initialization was successful.
+///
+/// # Errors
+///
+/// Returns a Magnus error if there's an issue with class definition.
+pub fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.get_inner(&ROOT_MOD);
+    module.define_class("DynamicResult", ruby.class_object())?;
+
+    Ok(())
+}