numaflow_ruby 0.1.0

data/Cargo.toml

@@ -0,0 +1,11 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/numaflow_ruby"]
+resolver = "2"
+
+
+[profile.release]
+debug = true

data/README.md

@@ -0,0 +1,31 @@
+# NumaflowRuby
+
+TODO: Delete this and the text below, and describe your gem
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/numaflow_ruby`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/numaflow_ruby.

data/Rakefile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+require "rb_sys/extensiontask"
+
+task build: :compile
+
+GEMSPEC = Gem::Specification.load("numaflow_ruby.gemspec")
+
+RbSys::ExtensionTask.new("numaflow_ruby", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/numaflow_ruby"
+end
+
+task default: %i[compile spec rubocop]

data/ext/numaflow_ruby/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "numaflow_ruby"
+version = "0.1.0"
+edition = "2021"
+authors = ["Kyle Cooke <kyle@playerdata.com>"]
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.7" }
+rb-sys = "0.9.116"
+numaflow = { git = "https://github.com/PlayerData/numaflow-rs.git" , branch = "main" }
+tokio = { version = "^1.45.1" , features = ["rt"]}
+tonic = "0.13.1"
+chrono = "0.4.41"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_magnus = { version = "0.9" }
+log = "0.4.27"
+env_logger = "0.11.8"
+simple_logger = "5.0.0"
+
+tower = "0.5.2"
+hyper-util = "0.1.16"
+prost-types = "0.13.5"
+tokio-stream = "0.1.17"

data/ext/numaflow_ruby/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("numaflow_ruby/numaflow_ruby")

data/ext/numaflow_ruby/src/lib.rs

@@ -0,0 +1,34 @@
+use magnus::{prelude::*, Error};
+use simple_logger::SimpleLogger;
+
+mod map;
+mod spec_helper;
+mod time;
+
+#[magnus::init]
+fn init(ruby: &magnus::Ruby) -> Result<(), Error> {
+    // env_logger::Builder::from_default_env().target(env_logger::Target::Stdout).build();
+    SimpleLogger::new().init().unwrap();
+
+    let module = ruby.define_module("NumaflowRuby")?;
+    module.define_singleton_method(
+        "rust_start_map_server",
+        magnus::function!(map::start_map_server, 0),
+    )?;
+    module.define_singleton_method(
+        "rust_get_next_message",
+        magnus::function!(map::get_next_message, 1),
+    )?;
+    module.define_class("MessageResponder", ruby.class_object())?;
+    module.define_singleton_method(
+        "rust_respond_to_message",
+        magnus::function!(map::respond_to_message, 2),
+    )?;
+    module.define_class("MessageSource", ruby.class_object())?;
+    module.define_singleton_method(
+        "rust_send_message_get_response",
+        magnus::function!(spec_helper::send_message_get_response, 3),
+    )?;
+    log::trace!("Magnus module NumaflowRuby initialized");
+    Ok(())
+}

data/ext/numaflow_ruby/src/map.rs

@@ -0,0 +1,263 @@
+use crate::time::time_into_ruby_value;
+use magnus::{IntoValue, RArray, RHash, Ruby, Value};
+use numaflow::map::{MapRequest, Message, Server};
+use rb_sys::rb_thread_call_without_gvl;
+use std::env;
+use std::ops::DerefMut;
+use std::sync::Mutex;
+use tokio::runtime::Runtime;
+
+type Request = (MapRequest, tokio::sync::oneshot::Sender<Vec<Message>>);
+struct ResponseValue(magnus::Value);
+impl TryFrom<ResponseValue> for Message {
+    type Error = magnus::Error;
+
+    fn try_from(value: ResponseValue) -> Result<Self, Self::Error> {
+        log::trace!("attempting conversion of value: {:?}", value.0);
+        let Some(hash) = RHash::from_value(value.0) else {
+            return Err(magnus::Error::new(
+                magnus::exception::type_error(),
+                "returned value must be Array of Hash or Hash",
+            ));
+        };
+        let tags = hash.lookup2(magnus::symbol::Symbol::new("tags"), None::<String>)?;
+        let keys = hash.lookup2(magnus::symbol::Symbol::new("keys"), None::<String>)?;
+        let value = hash.fetch(magnus::symbol::Symbol::new("value"))?;
+
+        Ok(Message { keys, value, tags })
+    }
+}
+
+impl IntoValue for MyMapRequest {
+    fn into_value_with(self, handle: &Ruby) -> Value {
+        let out = RHash::new();
+        out.aset("value", self.0.value.into_value_with(handle))
+            .unwrap();
+        out.aset(
+            "keys",
+            RArray::from_vec(self.0.keys).into_value_with(handle),
+        )
+        .unwrap();
+        if let Ok(t) = time_into_ruby_value(self.0.watermark) {
+            out.aset("watermark", t.into_value_with(handle)).unwrap();
+        }
+        if let Ok(t) = time_into_ruby_value(self.0.eventtime) {
+            out.aset("event_time", t.into_value_with(handle)).unwrap();
+        }
+        out.aset("headers", self.0.headers.into_value_with(handle))
+            .unwrap();
+        out.into_value()
+    }
+}
+pub struct MyMapRequest(MapRequest);
+
+struct MessageSource {
+    rx: tokio::sync::mpsc::Receiver<Request>,
+    rt: tokio::runtime::Runtime,
+    cancel_tx: tokio::sync::broadcast::Sender<()>,
+}
+
+#[magnus::wrap(
+    class = "NumaflowRuby::MessageSource",
+    free_immediately,
+    frozen_shareable
+)]
+pub struct MutMessageSource(Mutex<MessageSource>);
+
+#[magnus::wrap(
+    class = "NumaflowRuby::MessageResponder",
+    free_immediately,
+    frozen_shareable
+)]
+pub struct MessageResponder(Mutex<Option<tokio::sync::oneshot::Sender<Vec<Message>>>>);
+
+struct MsgWait<'a> {
+    message_source: &'a mut MessageSource,
+    received_message: Option<Request>,
+}
+
+unsafe extern "C" fn wait_for_recv(
+    message_wait_ptr: *mut ::std::os::raw::c_void,
+) -> *mut ::std::os::raw::c_void {
+    let message_wait = (message_wait_ptr as *mut MsgWait).as_mut().unwrap();
+    let mut cancel_rx = message_wait.message_source.cancel_tx.subscribe();
+
+    // Use select to wait for either receiver or cancellation
+    message_wait.received_message = message_wait.message_source.rt.block_on(async {
+        tokio::select! {
+            msg = message_wait.message_source.rx.recv() => msg,
+            _ = cancel_rx.recv() => None,
+        }
+    });
+
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn unblock_fn(message_wait_ptr: *mut ::std::os::raw::c_void) {
+    let message_wait = (message_wait_ptr as *mut MsgWait).as_mut().unwrap();
+    let _ = message_wait.message_source.cancel_tx.send(());
+}
+
+pub fn get_next_message(
+    message_source: &MutMessageSource,
+) -> Result<(MyMapRequest, MessageResponder), magnus::Error> {
+    let mut guard = message_source.0.lock().unwrap();
+    let message_source = guard.deref_mut();
+    match message_source.rx.try_recv() {
+        Ok(request) => {
+            log::trace!("Received request");
+            // return Ok(request.0.value);
+            return Ok((
+                MyMapRequest(request.0),
+                MessageResponder(Mutex::new(Some(request.1))),
+            ));
+        }
+        Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
+            log::trace!("No messages available");
+        }
+        Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
+            log::error!("Message channel closed");
+            return Err(magnus::Error::new(
+                magnus::exception::runtime_error(),
+                "Message channel closed",
+            ));
+        }
+    };
+    let mut message_wait = MsgWait {
+        message_source,
+        received_message: None,
+    };
+    unsafe {
+        rb_thread_call_without_gvl(
+            Some(wait_for_recv),
+            (&mut message_wait) as *mut MsgWait as *mut ::std::os::raw::c_void,
+            Some(unblock_fn),
+            (&mut message_wait) as *mut MsgWait as *mut ::std::os::raw::c_void,
+        );
+    };
+    match message_wait.received_message {
+        Some(request) => {
+            log::trace!(
+                "Received request after blocking, rx is closed: {}",
+                request.1.is_closed()
+            );
+            Ok((
+                MyMapRequest(request.0),
+                MessageResponder(Mutex::new(Some(request.1))),
+            ))
+        }
+        None => {
+            log::error!("No messages received after blocking");
+            Err(magnus::Error::new(
+                magnus::exception::runtime_error(),
+                "Message channel closed",
+            ))
+        }
+    }
+}
+
+pub fn respond_to_message(
+    response: Value,
+    response_sender: &MessageResponder,
+) -> Result<(), magnus::Error> {
+    log::trace!(
+        "respond_to_message called with response: {:?}, responder: {:?}, responder closed: {}",
+        response,
+        response_sender.0,
+        response_sender
+            .0
+            .lock()
+            .unwrap()
+            .as_ref()
+            .unwrap()
+            .is_closed()
+    );
+    let messages_array = RArray::to_ary(response)?;
+    let responses = messages_array
+        .into_iter()
+        .map(ResponseValue)
+        .map(Message::try_from)
+        .collect::<Result<Vec<_>, _>>()?;
+    log::trace!("Responding with messages: {responses:?}");
+
+    let sender = response_sender
+        .0
+        .lock()
+        .unwrap()
+        .take()
+        .ok_or(magnus::Error::new(
+            magnus::exception::runtime_error(),
+            "response sender has already been used",
+        ))?;
+    sender.send(responses).map_err(|_| {
+        magnus::Error::new(
+            magnus::exception::runtime_error(),
+            "server had stopped waiting for response",
+        )
+    })
+}
+
+pub fn start_map_server() -> Result<MutMessageSource, magnus::Error> {
+    let rt = Runtime::new().expect("Failed to create Tokio runtime");
+    let (tx, rx) = tokio::sync::mpsc::channel::<Request>(100);
+    let (cancel_tx, _) = tokio::sync::broadcast::channel(1);
+
+    rt.block_on(async {
+        let mut server = Server::new(RubyMapServer {
+            message_handler: tx,
+        });
+        if let Ok(info_path) = env::var("NUMAFLOW_SERVER_INFO_FILE_PATH") {
+            server = server.with_server_info_file(info_path);
+        }
+        if let Ok(socket_path) = env::var("NUMAFLOW_SERVER_SOCKET_PATH") {
+            server = server.with_socket_file(socket_path);
+        }
+        println!("Starting map server... {server:?}");
+        tokio::spawn(async move {
+            server.start().await.expect("Failed to start map server");
+        });
+    });
+
+    Ok(MutMessageSource(Mutex::new(MessageSource {
+        rx,
+        rt,
+        cancel_tx,
+    })))
+}
+
+#[derive(Debug)]
+struct RubyMapServer {
+    message_handler: tokio::sync::mpsc::Sender<Request>,
+}
+
+#[derive(Debug)]
+struct DropDetector {
+    msg: &'static str,
+}
+impl Drop for DropDetector {
+    fn drop(&mut self) {
+        log::warn!("drop warning {}", self.msg);
+    }
+}
+
+#[tonic::async_trait]
+impl numaflow::map::Mapper for RubyMapServer {
+    async fn map(&self, input: MapRequest) -> Vec<Message> {
+        let (responder_tx, responder_rx) = tokio::sync::oneshot::channel();
+        log::trace!("Received map request: {:?}", input.value);
+        self.message_handler
+            .send((input, responder_tx))
+            .await
+            .expect("Failed to send map request");
+        log::trace!("sent map request, waiting for response...");
+        let dd = DropDetector {
+            msg: "map call is being dropped",
+        };
+        let Ok(r) = responder_rx.await else {
+            log::error!("Failed to receive response for map request");
+            panic!("Failed to receive response for map request");
+        };
+        log::trace!("Received map response: {:?}, {:?}", r, dd);
+        r
+    }
+}

data/ext/numaflow_ruby/src/spec_helper.rs

@@ -0,0 +1,137 @@
+use magnus::IntoValue;
+use numaflow::servers::map::{self as proto, MapResponse};
+use rb_sys::rb_thread_call_without_gvl;
+use std::ptr::null_mut;
+use std::sync::{Arc, Mutex};
+use tokio::net::UnixStream;
+use tokio::sync::mpsc;
+use tokio_stream::wrappers::ReceiverStream;
+use tonic::transport::Uri;
+use tower::service_fn;
+
+type Cancellable = tokio::sync::broadcast::Sender<()>;
+type Request = (
+    (String, Vec<String>, Vec<u8>, Cancellable),
+    Option<MapResponse>,
+);
+
+pub(crate) fn send_message_get_response(
+    uds_socket_path: String,
+    keys: Vec<String>,
+    value: Vec<u8>,
+) -> magnus::Value {
+    let mut cancel_tx = tokio::sync::broadcast::channel(1).0;
+    let request_arc: Arc<Mutex<Request>> = std::sync::Arc::new(Mutex::new((
+        (uds_socket_path, keys, value, cancel_tx.clone()),
+        None,
+    )));
+    let request_arc_ptr = Arc::into_raw(request_arc.clone()) as *mut ::std::os::raw::c_void;
+    unsafe {
+        rb_thread_call_without_gvl(
+            Some(run_the_async_bit),
+            request_arc_ptr,
+            Some(unblock_fn),
+            &mut cancel_tx as *mut Cancellable as *mut ::std::os::raw::c_void,
+        );
+    };
+    let Some(response) = request_arc.lock().unwrap().1.clone() else {
+        return None::<()>.into_value();
+    };
+    let mut out = Vec::new();
+    for response in response.results.iter() {
+        let response_value = magnus::RHash::new();
+        response_value
+            .aset("keys", magnus::RArray::from_vec(response.keys.clone()))
+            .unwrap();
+        response_value
+            .aset("value", magnus::RArray::from_vec(response.value.clone()))
+            .unwrap();
+        response_value
+            .aset("tags", magnus::RArray::from_vec(response.tags.clone()))
+            .unwrap();
+        out.push(response_value.into_value());
+    }
+    magnus::RArray::from_slice(&out).into_value()
+}
+
+unsafe extern "C" fn run_the_async_bit(
+    request_arc_ptr: *mut ::std::os::raw::c_void,
+) -> *mut ::std::os::raw::c_void {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let request_arc: Arc<Mutex<Request>> = Arc::from_raw(request_arc_ptr as _);
+    let mut request = request_arc.lock().unwrap();
+    let mut subscribe = request.0 .3.subscribe();
+    let response = rt.block_on(async {
+        tokio::select! {
+            resp = do_send_message_get_response(request.0.0.clone(), request.0.1.clone(), request.0.2.clone()) => Some(resp),
+            _ = subscribe.recv() => None,
+            }
+    });
+    request.1 = response;
+    null_mut()
+}
+
+unsafe extern "C" fn unblock_fn(cancel_ptr: *mut ::std::os::raw::c_void) {
+    let cancellable: &mut Cancellable = (cancel_ptr as *mut Cancellable).as_mut().unwrap();
+    let _ = cancellable.send(());
+}
+
+async fn do_send_message_get_response(
+    uds_socket_path: String,
+    keys: Vec<String>,
+    value: Vec<u8>,
+) -> MapResponse {
+    // https://github.com/hyperium/tonic/blob/master/examples/src/uds/client.rs
+    let channel = tonic::transport::Endpoint::try_from("http://[::]:50051")
+        .unwrap()
+        .connect_with_connector(service_fn(move |_: Uri| {
+            // https://rust-lang.github.io/async-book/03_async_await/01_chapter.html#async-lifetimes
+            let sock_file = uds_socket_path.clone();
+            async move {
+                Ok::<_, std::io::Error>(hyper_util::rt::TokioIo::new(
+                    UnixStream::connect(sock_file).await?,
+                ))
+            }
+        }))
+        .await
+        .unwrap();
+
+    let mut client = numaflow::servers::map::map_client::MapClient::new(channel);
+    let request = proto::MapRequest {
+        request: Some(proto::map_request::Request {
+            keys,
+            value,
+            watermark: Some(prost_types::Timestamp::default()),
+            event_time: Some(prost_types::Timestamp::default()),
+            headers: Default::default(),
+        }),
+        id: "".to_string(),
+        handshake: None,
+        status: None,
+    };
+
+    let (tx, rx) = mpsc::channel(2);
+    let handshake_request = proto::MapRequest {
+        request: None,
+        id: "".to_string(),
+        handshake: Some(proto::Handshake { sot: true }),
+        status: None,
+    };
+
+    tx.send(handshake_request).await.unwrap();
+    tx.send(request).await.unwrap();
+
+    let resp = client.map_fn(ReceiverStream::new(rx)).await.unwrap();
+    let mut resp = resp.into_inner();
+
+    let handshake_response = resp.message().await.unwrap();
+    assert!(handshake_response.is_some());
+
+    let handshake_response = handshake_response.unwrap();
+    assert!(handshake_response.handshake.is_some());
+
+    let actual_response = resp.message().await.unwrap();
+    assert!(actual_response.is_some());
+
+    actual_response.unwrap()
+}

data/ext/numaflow_ruby/src/time.rs

@@ -0,0 +1,18 @@
+use chrono::{DateTime, Utc};
+use serde::Serialize;
+use serde_magnus::serialize;
+
+#[derive(Serialize)]
+struct RubyTimeUtc {
+    sec: i64,
+    nsec: i32,
+}
+
+pub fn time_into_ruby_value(time: DateTime<Utc>) -> Result<magnus::Value, magnus::Error> {
+    let delta = time.signed_duration_since(DateTime::UNIX_EPOCH);
+    let ts = RubyTimeUtc {
+        sec: delta.num_seconds(),
+        nsec: delta.subsec_nanos(),
+    };
+    serialize(&ts)
+}

data/lib/numaflow_ruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module NumaflowRuby
+  VERSION = "0.1.0"
+end

data/lib/numaflow_ruby.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "numaflow_ruby/version"
+require_relative "numaflow_ruby/numaflow_ruby"
+
+# NumaflowRuby is a Ruby wrapper for Numaflow rust library, allowing you to create map servers in Ruby.
+module NumaflowRuby
+  class Error < StandardError; end
+
+  def self.start_map_server(map_handler) # rubocop:disable Metrics/MethodLength
+    Thread.new do
+      msg_source = NumaflowRuby.rust_start_map_server
+      loop do
+        val, responder = NumaflowRuby.rust_get_next_message(msg_source)
+        begin
+          result = map_handler.call(val)
+          NumaflowRuby.rust_respond_to_message(result, responder)
+        rescue StandardError
+          Kernel.exit 1
+        end
+      end
+    end
+  end
+end

data/sig/numaflow_ruby.rbs

@@ -0,0 +1,4 @@
+module NumaflowRuby
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: numaflow_ruby
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kyle Cooke
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2025-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.39
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.39
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: Ruby bindings for numaflow sdk
+email:
+- kyle@playerdata.com
+executables: []
+extensions:
+- ext/numaflow_ruby/Cargo.toml
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- README.md
+- Rakefile
+- ext/numaflow_ruby/Cargo.toml
+- ext/numaflow_ruby/extconf.rb
+- ext/numaflow_ruby/src/lib.rs
+- ext/numaflow_ruby/src/map.rs
+- ext/numaflow_ruby/src/spec_helper.rs
+- ext/numaflow_ruby/src/time.rs
+- lib/numaflow_ruby.rb
+- lib/numaflow_ruby/version.rb
+- sig/numaflow_ruby.rbs
+homepage: https://github.com/PlayerData/numaflow_ruby
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/PlayerData/numaflow_ruby
+  source_code_uri: https://github.com/PlayerData/numaflow_ruby
+  changelog_uri: https://github.com/PlayerData/numaflow_ruby/tree/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.11
+requirements: []
+rubygems_version: 3.5.22
+signing_key: 
+specification_version: 4
+summary: Ruby bindings for numaflow sdk
+test_files: []