omq-backend-rust 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c552d7ca25d216c752732579deddf1cf357a8bd1b71cfec2be322f5e1d1645ef
-  data.tar.gz: 76e666f63269f5d69384aa04a6298144b78171c4f69d397a98ebfcb5ff1ccf9e
+  metadata.gz: b22c5a39d466ddff0ea811bfadff93fe58e64c3ea08bb213f61419e5efd13cc6
+  data.tar.gz: '09ef0d2306516c3809a957f61e234360d9f2bbe34b0389cf874c8d5cdc87abc8'
 SHA512:
-  metadata.gz: 4a2974b9286be09db0671bf8784346e88bc58cbcdf00a3a22d58309a8a852e33164d044565315fcde4f869a11e3f294d38ed313f45196967259a1d2e46bfad81
-  data.tar.gz: e406b0651722cff4b807096ec15b5d8cb9cfd8e5e314285250af47ecaf119b79af3e0aa46265cab53e1b4d6b7ca6eea38f7b03cd31205ed0575d186dd43f9bff
+  metadata.gz: bae7c0181f01e714d3fbbb11546bb84dc6642d77661734e5d5eaa5467af153a5437b0a9af1ff0d1b8f894609928d7952a83bcb492ef299e327253acdc6a1f85d
+  data.tar.gz: b787de3962b49bf2c784596f8fd82901e0323e74270915ba051359eb73a46022389726cea92b1a6364c7780eaf296a457d8355d90507503a5690f4de4de8410b

data/CHANGELOG.md

@@ -2,6 +2,33 @@
 
 ## [Unreleased]
 
+## [0.1.7] - 2026-08-02
+
+### Added
+
+- Restored Rust/Ruby backend interop coverage.
+
+### Changed
+
+- Updated to `omq-tokio` 0.21.0.
+
+### Fixed
+
+- Drained the Ruby-to-Tokio send pump before native socket close.
+- Released Rust send ring slots before awaiting native sends, avoiding
+  high-throughput stalls under `omq-tokio` 0.21.0 backpressure.
+- Waited for Rust peer registration before resolving `peer_connected`.
+
+## [0.1.6] - 2026-08-01
+
+### Added
+
+- Updated to `omq-tokio` 0.20.3 and `omq-proto` 0.25.0.
+- Enabled OMQ.rs `zstd+tcp://` by default.
+- Forwarded compression bind/connect kwargs to current OMQ.rs options:
+  `dict:`, `auto_dict:`, `compression_threshold:`, `max_recv_dict_size:`,
+  `compression_offload_threshold:`, and zstd `level:`.
+
 ## [0.1.5] - 2026-07-31
 
 ### Fixed

data/README.md

@@ -58,6 +58,18 @@ SCATTER/GATHER, CHANNEL.
 - **NULL** (default)
 - **CURVE** (CurveZMQ, via [Nuckle](https://github.com/paddor/nuckle))
 
+## Compressed transports
+
+The Rust backend supports `lz4+tcp://` and `zstd+tcp://`. Compression
+configuration uses the same bind/connect kwargs as the Ruby transports:
+`level:` for zstd compression level (`-8..4`), `dict:` for a static
+send-side dictionary, and `auto_dict: true` for automatic dictionary
+training.
+
+Automatic dictionary training is off by default.
+OMQ.rs stores compression configuration per socket, so pass these kwargs on
+the first `bind` or `connect` for a Rust-backed socket.
+
 ## Development
 
 ```sh

data/ext/omq_backend_rust/Cargo.toml

@@ -11,16 +11,17 @@ name       = "omq_backend_rust"
 crate-type = ["cdylib"]
 
 [features]
-default = ["plain", "curve", "lz4"]
+default = ["plain", "curve", "lz4", "zstd"]
 plain   = ["omq-tokio/plain"]
 curve   = ["omq-tokio/curve"]
 lz4     = ["omq-tokio/lz4"]
+zstd    = ["omq-tokio/zstd"]
 
 [dependencies]
-omq-proto = { version = "=0.24.1", default-features = false }
-omq-tokio = { version = "=0.20.2", default-features = false }
+omq-proto = { version = "=0.25.0", default-features = false }
+omq-tokio = { version = "=0.21.0", default-features = false }
 tokio     = { version = "1.52.0", features = ["rt", "rt-multi-thread", "time", "sync", "io-util", "net"] }
-yring     = { version = "0.3.8", features = ["async"] }
+yring     = { version = "0.3.11", features = ["async"] }
 
 bytes   = "1.12.0"
 flume   = { version = "0.12", default-features = false, features = ["async"] }

data/ext/omq_backend_rust/src/options.rs

@@ -48,6 +48,29 @@ pub fn build_options(ruby: &Ruby, hash: RHash) -> Result<omq_tokio::Options, Err
     if let Some(v) = get_opt::<i64>(ruby, hash, "rcvbuf")? {
         opts.recv_buffer_size = Some(v as usize);
     }
+    if let Some(v) = get_opt_bytes(ruby, hash, "compression_dict")? {
+        if !v.is_empty() {
+            opts.compression_dict = Some(Bytes::from(v));
+        }
+    }
+    if let Some(v) = get_opt::<bool>(ruby, hash, "compression_auto_train")? {
+        opts.compression_auto_train = v;
+    }
+    if let Some(v) = get_opt::<i64>(ruby, hash, "compression_threshold")? {
+        opts.compression_threshold = Some(v as usize);
+    }
+    if let Some(v) = get_opt::<i64>(ruby, hash, "compression_level")? {
+        opts.compression_level = Some(v as i32);
+    }
+    if let Some(v) = get_opt::<i64>(ruby, hash, "compression_dict_capacity")? {
+        opts.compression_dict_capacity = Some(v as usize);
+    }
+    if let Some(v) = get_opt::<i64>(ruby, hash, "max_recv_dict_size")? {
+        opts.max_recv_dict_size = Some(v as usize);
+    }
+    if let Some(v) = get_opt::<i64>(ruby, hash, "compression_offload_threshold")? {
+        opts.compression_offload_threshold = if v < 0 { None } else { Some(v as usize) };
+    }
     if let Some(v) = get_opt::<String>(ruby, hash, "on_mute")? {
         opts.on_mute = match v.as_str() {
             "drop_newest" | "drop" => omq_tokio::OnMute::DropNewest,

data/ext/omq_backend_rust/src/runtime.rs

@@ -299,20 +299,15 @@ pub fn materialize(
     let job: Job = Box::new(move || {
         let sock = Arc::new(InnerSocket::new(socket_type, options));
 
-        const SEND_YIELD_INTERVAL: u32 = 256;
         let s = sock.clone();
         let sn = send_notify.clone();
         let send_pump = tokio::spawn(async move {
-            futures::pin_mut!(send_cons);
-            let mut batch = 0u32;
+            let mut send_cons = send_cons;
             while let Some(msg) = futures::StreamExt::next(&mut send_cons).await {
-                let _ = s.send(msg).await;
+                send_cons.release();
                 sn.notify();
-                batch += 1;
-                if batch >= SEND_YIELD_INTERVAL {
-                    batch = 0;
-                    tokio::task::yield_now().await;
-                }
+                let _ = s.send(msg).await;
+                tokio::task::yield_now().await;
             }
             sn.notify();
         });
@@ -343,6 +338,7 @@ pub fn materialize(
         });
 
         let monitor_sock = sock.clone();
+        let peer_ready_sock = sock.clone();
         let monitor_pump = tokio::spawn(async move {
             let mut stream = monitor_sock.monitor();
             let mut peer_count: u32 = 0;
@@ -359,6 +355,7 @@ pub fn materialize(
                                 had_peers = true;
                                 if !peer_connected_fired {
                                     peer_connected_fired = true;
+                                    let _ = peer_ready_sock.connections().await;
                                     peer_connected_notify.force_wake();
                                 }
                             }
@@ -374,6 +371,12 @@ pub fn materialize(
                                     subscriber_joined_notify.force_wake();
                                 }
                             }
+                            omq_tokio::MonitorEvent::JoinReceived { .. } => {
+                                if !subscriber_joined_fired {
+                                    subscriber_joined_fired = true;
+                                    subscriber_joined_notify.force_wake();
+                                }
+                            }
                             _ => {}
                         }
 
@@ -432,24 +435,59 @@ pub fn destroy_socket(
     io_threads: usize,
     sock: Arc<InnerSocket>,
     send_prod: Mutex<yring::AsyncProducer<omq_tokio::Message>>,
-    send_pump: JoinHandle<()>,
+    mut send_pump: JoinHandle<()>,
     recv_pump: JoinHandle<()>,
     monitor_pump: JoinHandle<()>,
     linger: Option<Duration>,
 ) {
     recv_pump.abort();
     monitor_pump.abort();
-    send_pump.abort();
-    drop(send_prod);
     let Ok(handle) = (|| -> std::result::Result<Handle, ()> { Ok(ensure_runtime(io_threads)) })()
     else {
+        send_pump.abort();
+        drop(send_prod);
         return;
     };
     let close_timeout = linger
         .unwrap_or(Duration::from_secs(30))
         .max(Duration::from_millis(10));
-    handle.spawn(async move {
+    let fut = async move {
+        drop(send_prod);
+        if tokio::time::timeout(close_timeout, &mut send_pump)
+            .await
+            .is_err()
+        {
+            send_pump.abort();
+            let _ = send_pump.await;
+        }
+
         let s = Arc::try_unwrap(sock).unwrap_or_else(|arc| (*arc).clone());
         let _ = tokio::time::timeout(close_timeout, s.close()).await;
+    };
+
+    let (otx, orx) = flume::bounded::<()>(1);
+    handle.spawn(async move {
+        fut.await;
+        let _ = otx.send(());
     });
+
+    struct RecvBox {
+        rx: flume::Receiver<()>,
+    }
+
+    extern "C" fn blocking_recv(data: *mut libc::c_void) -> *mut libc::c_void {
+        let rd = unsafe { &mut *(data as *mut RecvBox) };
+        let _ = rd.rx.recv();
+        std::ptr::null_mut()
+    }
+
+    let mut rd = RecvBox { rx: orx };
+    unsafe {
+        rb_sys::rb_thread_call_without_gvl(
+            Some(blocking_recv),
+            &mut rd as *mut RecvBox as *mut libc::c_void,
+            None,
+            std::ptr::null_mut(),
+        );
+    }
 }

data/lib/omq/rust/engine.rb

@@ -19,12 +19,13 @@ module OMQ
         @peer_connected = Async::Promise.new
         @all_peers_gone = Async::Promise.new
         @subscriber_joined = Async::Promise.new
-        @connections    = []
+        @connections    = {}
         @closed         = false
         @parent_task    = nil
         @on_io_thread   = false
         @materialized   = false
         @recv_sentinels = 0
+        @compression_options = {}
 
         @native = Native::RustSocket.new(socket_type.to_s)
 
@@ -47,16 +48,18 @@ module OMQ
       end
 
 
-      def bind(endpoint, parent: nil, **)
+      def bind(endpoint, parent: nil, **opts)
         capture_parent_task(parent: parent)
+        apply_endpoint_options!(opts)
         ensure_materialized
         resolved = @native.bind(endpoint)
         URI.parse(resolved)
       end
 
 
-      def connect(endpoint, parent: nil, **)
+      def connect(endpoint, parent: nil, **opts)
         capture_parent_task(parent: parent)
+        apply_endpoint_options!(opts)
         ensure_materialized
         @native.connect(endpoint)
         URI.parse(endpoint)
@@ -219,6 +222,7 @@ module OMQ
             monitor_io.wait_readable
             monitor_io.read_nonblock(256, exception: false)
             while (data = @native.try_recv_monitor)
+              track_connection_event(data)
               @monitor_queue.enqueue(MonitorEvent.new(**data))
             end
           end
@@ -226,6 +230,23 @@ module OMQ
       end
 
 
+      def track_connection_event(data)
+        detail = data[:detail] || {}
+        connection_id = detail[:connection_id]
+
+        case data[:type]
+        when :handshake_succeeded
+          @connections[connection_id || Object.new] = true
+        when :disconnected
+          if connection_id
+            @connections.delete(connection_id)
+          else
+            @connections.shift
+          end
+        end
+      end
+
+
       def extract_options
         h = {}
         h["send_hwm"]           = @options.send_hwm
@@ -241,6 +262,7 @@ module OMQ
         h["sndbuf"]             = @options.sndbuf
         h["rcvbuf"]             = @options.rcvbuf
         h["on_mute"]            = @options.on_mute.to_s
+        h.merge!(@compression_options)
 
         ri = @options.reconnect_interval
         if ri.is_a?(Range)
@@ -256,6 +278,88 @@ module OMQ
       end
 
 
+      def apply_endpoint_options!(opts)
+        compression = extract_endpoint_compression_options(opts)
+        return if compression.empty?
+
+        if @materialized
+          existing = compression.keys.to_h do |key|
+            [key, @compression_options.fetch(key, default_compression_option(key))]
+          end
+          return if compression == existing
+
+          raise ArgumentError,
+            "Rust backend compression options must be set before first bind/connect"
+        end
+
+        @compression_options.merge!(compression)
+      end
+
+
+      def extract_endpoint_compression_options(opts)
+        out = {}
+
+        if opts.key?(:level)
+          validate_zstd_level!(opts[:level])
+          out["compression_level"] = opts[:level]
+        end
+        out["compression_dict"] = opts[:dict].b if opts.key?(:dict) && opts[:dict]
+
+        if opts.key?(:auto_dict)
+          auto_dict = opts[:auto_dict]
+          if auto_dict && opts[:dict]
+            raise ArgumentError, "cannot combine auto_dict: and dict:"
+          end
+
+          case auto_dict
+          when nil, false
+            out["compression_auto_train"] = false
+          when true
+            out["compression_auto_train"] = true
+          when Hash
+            if auto_dict.key?(:trigger)
+              raise ArgumentError,
+                "Rust backend does not support auto_dict: trigger"
+            end
+            validate_positive!("auto_dict capacity", auto_dict[:capacity]) if auto_dict[:capacity]
+            out["compression_auto_train"] = true
+            out["compression_dict_capacity"] = auto_dict[:capacity] if auto_dict[:capacity]
+          else
+            raise TypeError, "auto_dict: must be true, false, or a Hash; got #{auto_dict.class}"
+          end
+        end
+
+        if opts.key?(:compression_threshold)
+          out["compression_threshold"] = opts[:compression_threshold]
+        end
+        out["max_recv_dict_size"] = opts[:max_recv_dict_size] if opts.key?(:max_recv_dict_size)
+        if opts.key?(:compression_offload_threshold)
+          out["compression_offload_threshold"] = opts[:compression_offload_threshold] || -1
+        end
+
+        out
+      end
+
+
+      def default_compression_option(key)
+        key == "compression_auto_train" ? false : nil
+      end
+
+
+      def validate_positive!(label, value)
+        return if value.respond_to?(:positive?) && value.positive?
+
+        raise ArgumentError, "#{label} must be positive"
+      end
+
+
+      def validate_zstd_level!(level)
+        return if level.is_a?(Integer) && (-8..4).cover?(level)
+
+        raise ArgumentError, "zstd compression level must be -8..4, got #{level.inspect}"
+      end
+
+
       def extract_mechanism(h)
         mech = @options.mechanism
         case mech

data/lib/omq/rust/version.rb

@@ -2,6 +2,6 @@
 
 module OMQ
   module Rust
-    VERSION = "0.1.5"
+    VERSION = "0.1.7"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omq-backend-rust
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Patrik Wenger