omq 0.8.0 → 0.10.0

data/lib/omq/pub_sub.rb

@@ -2,7 +2,7 @@
 
 module OMQ
   class PUB < Socket
-    include ZMTP::Writable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0, conflate: false)
       _init_engine(:PUB, linger: linger, conflate: conflate)
@@ -13,7 +13,7 @@ module OMQ
   # SUB socket.
   #
   class SUB < Socket
-    include ZMTP::Readable
+    include Readable
 
     # @return [String] subscription prefix to subscribe to everything
     #
@@ -50,8 +50,8 @@ module OMQ
   end
 
   class XPUB < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0)
       _init_engine(:XPUB, linger: linger)
@@ -60,8 +60,8 @@ module OMQ
   end
 
   class XSUB < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     # @param endpoints [String, nil]
     # @param linger [Integer]

data/lib/omq/push_pull.rb

@@ -2,7 +2,7 @@
 
 module OMQ
   class PUSH < Socket
-    include ZMTP::Writable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0, send_hwm: nil, send_timeout: nil)
       _init_engine(:PUSH, linger: linger, send_hwm: send_hwm, send_timeout: send_timeout)
@@ -11,7 +11,7 @@ module OMQ
   end
 
   class PULL < Socket
-    include ZMTP::Readable
+    include Readable
 
     def initialize(endpoints = nil, linger: 0, recv_hwm: nil, recv_timeout: nil)
       _init_engine(:PULL, linger: linger, recv_hwm: recv_hwm, recv_timeout: recv_timeout)

data/lib/omq/radio_dish.rb

@@ -2,7 +2,7 @@
 
 module OMQ
   class RADIO < Socket
-    include ZMTP::Writable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0, conflate: false)
       _init_engine(:RADIO, linger: linger, conflate: conflate)
@@ -45,7 +45,7 @@ module OMQ
   end
 
   class DISH < Socket
-    include ZMTP::Readable
+    include Readable
 
     def initialize(endpoints = nil, linger: 0, group: nil)
       _init_engine(:DISH, linger: linger)

data/lib/omq/reactor.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "async"
+
+module OMQ
+  # Shared IO reactor for the Ruby backend.
+  #
+  # When user code runs inside an Async reactor, engine tasks are
+  # spawned directly under the caller's Async task. When no reactor
+  # is available (e.g. bare Thread.new), a single shared IO thread
+  # hosts all engine tasks — mirroring libzmq's IO thread.
+  #
+  # Engines obtain the IO thread's root task via {.root_task} and
+  # use it as their @parent_task. Blocking operations from the main
+  # thread are dispatched to the IO thread via {.run}.
+  #
+  module Reactor
+    @mutex      = Mutex.new
+    @thread     = nil
+    @root_task  = nil
+    @work_queue = nil
+    @lingers    = Hash.new(0) # linger value → count of active sockets
+
+    class << self
+      # Returns the root Async task inside the shared IO thread.
+      # Starts the thread exactly once (double-checked lock).
+      #
+      # @return [Async::Task]
+      #
+      def root_task
+        return @root_task if @root_task
+        @mutex.synchronize do
+          return @root_task if @root_task
+          ready       = Thread::Queue.new
+          @work_queue = Async::Queue.new
+          @thread     = Thread.new { run_reactor(ready) }
+          @thread.name = "omq-io"
+          @root_task = ready.pop
+          at_exit { stop! }
+        end
+        @root_task
+      end
+
+
+      # Runs a block inside the Async reactor.
+      #
+      # Inside an Async reactor: runs directly.
+      # Outside: dispatches to the shared IO thread and blocks
+      # the calling thread until the result is available.
+      #
+      # @return [Object] the block's return value
+      #
+      def run(&block)
+        if Async::Task.current?
+          yield
+        else
+          result = Thread::Queue.new
+          root_task # ensure started
+          @work_queue.push([block, result])
+          status, value = result.pop
+          raise value if status == :error
+          value
+        end
+      end
+
+
+      # Registers a socket's linger value.
+      #
+      # @param seconds [Numeric, nil] linger value
+      #
+      def track_linger(seconds)
+        @lingers[seconds || 0] += 1
+      end
+
+
+      # Unregisters a socket's linger value.
+      #
+      # @param seconds [Numeric, nil] linger value
+      #
+      def untrack_linger(seconds)
+        key = seconds || 0
+        @lingers[key] -= 1
+        @lingers.delete(key) if @lingers[key] <= 0
+      end
+
+
+      # Stops the shared IO thread.
+      #
+      # @return [void]
+      #
+      def stop!
+        return unless @thread&.alive?
+        max_linger = @lingers.empty? ? 0 : @lingers.keys.max
+        @work_queue&.push(nil)
+        @thread&.join(max_linger + 1)
+        @thread     = nil
+        @root_task  = nil
+        @work_queue = nil
+        @lingers    = Hash.new(0)
+      end
+
+      private
+
+      # Runs the shared Async reactor.
+      #
+      # Processes work items dispatched via {.run} while engine
+      # tasks (accept loops, pumps, etc.) run as transient children.
+      #
+      # @param ready [Thread::Queue] receives the root task once started
+      #
+      def run_reactor(ready)
+        Async do |task|
+          ready.push(task)
+          loop do
+            item = @work_queue.dequeue
+            break if item.nil?
+            block, result = item
+            task.async do
+              result.push([:ok, block.call])
+            rescue => e
+              result.push([:error, e])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omq/readable.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module OMQ
+  # Pure Ruby Readable mixin. Dequeues messages from the engine's recv queue.
+  #
+  module Readable
+    # Maximum messages to prefetch from the recv queue per drain.
+    RECV_BATCH_SIZE = 64
+
+    # Receives the next message. Returns from a local prefetch
+    # buffer when available, otherwise drains up to
+    # {RECV_BATCH_SIZE} messages from the recv queue in one
+    # synchronized dequeue.
+    #
+    # @return [Array<String>] message parts
+    # @raise [IO::TimeoutError] if read_timeout exceeded
+    #
+    def receive
+      @recv_mutex.synchronize { @recv_buffer.shift } || fill_recv_buffer
+    end
+
+    # Waits until the socket is readable.
+    #
+    # @param timeout [Numeric, nil] timeout in seconds
+    # @return [true]
+    #
+    def wait_readable(timeout = @options.read_timeout)
+      true
+    end
+
+    private
+
+    def fill_recv_buffer
+      batch = Reactor.run { with_timeout(@options.read_timeout) { @engine.dequeue_recv_batch(RECV_BATCH_SIZE) } }
+      msg = batch.shift
+      @recv_mutex.synchronize { @recv_buffer.concat(batch) } unless batch.empty?
+      msg
+    end
+  end
+end

data/lib/omq/req_rep.rb

@@ -2,8 +2,8 @@
 
 module OMQ
   class REQ < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0)
       _init_engine(:REQ, linger: linger)
@@ -12,8 +12,8 @@ module OMQ
   end
 
   class REP < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0)
       _init_engine(:REP, linger: linger)

data/lib/omq/router_dealer.rb

@@ -2,8 +2,8 @@
 
 module OMQ
   class DEALER < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0)
       _init_engine(:DEALER, linger: linger)
@@ -14,8 +14,8 @@ module OMQ
   # ROUTER socket.
   #
   class ROUTER < Socket
-    include ZMTP::Readable
-    include ZMTP::Writable
+    include Readable
+    include Writable
 
     def initialize(endpoints = nil, linger: 0)
       _init_engine(:ROUTER, linger: linger)

data/lib/omq/routing/channel.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # CHANNEL socket routing: exclusive 1-to-1 bidirectional.
+    #
+    class Channel
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine     = engine
+        @connection = nil
+        @recv_queue = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @send_queue = Async::LimitedQueue.new(engine.options.send_hwm)
+        @tasks          = []
+        @send_pump_idle = true
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      # @raise [RuntimeError] if a connection already exists
+      #
+      def connection_added(connection)
+        raise "CHANNEL allows only one peer" if @connection
+        @connection = connection
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+        start_send_pump(connection) unless connection.is_a?(Transport::Inproc::DirectPipe)
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        if @connection == connection
+          @connection = nil
+          @send_pump&.stop
+          @send_pump = nil
+        end
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        conn = @connection
+        if conn.is_a?(Transport::Inproc::DirectPipe) && conn.direct_recv_queue
+          conn.send_message(parts)
+        else
+          @send_queue.enqueue(parts)
+        end
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+      def send_pump_idle? = @send_pump_idle
+
+      private
+
+      def start_send_pump(conn)
+        @send_pump = @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            @send_pump_idle = true
+            batch = [@send_queue.dequeue]
+            @send_pump_idle = false
+            Routing.drain_send_queue(@send_queue, batch)
+            batch.each { |parts| conn.write_message(parts) }
+            conn.flush
+          end
+        rescue *CONNECTION_LOST
+          @engine.connection_lost(conn)
+        end
+        @tasks << @send_pump
+      end
+    end
+  end
+end

data/lib/omq/routing/client.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # CLIENT socket routing: round-robin send, fair-queue receive.
+    #
+    # Same as DEALER — no envelope manipulation.
+    #
+    class Client
+      include RoundRobin
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine     = engine
+        @recv_queue = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @tasks      = []
+        init_round_robin(engine)
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        signal_connection_available
+        update_direct_pipe
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+        update_direct_pipe
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        enqueue_round_robin(parts)
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+    end
+  end
+end

data/lib/omq/routing/dealer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # DEALER socket routing: round-robin send, fair-queue receive.
+    #
+    # No envelope manipulation — messages pass through unchanged.
+    #
+    class Dealer
+      include RoundRobin
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine     = engine
+        @recv_queue = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @tasks      = []
+        init_round_robin(engine)
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        signal_connection_available
+        update_direct_pipe
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+        update_direct_pipe
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        enqueue_round_robin(parts)
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+    end
+  end
+end

data/lib/omq/routing/dish.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # DISH socket routing: group-based receive from RADIO peers.
+    #
+    # Sends JOIN/LEAVE commands to connected RADIO peers.
+    # Receives two-frame messages (group + body) from RADIO.
+    #
+    class Dish
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine      = engine
+        @connections = []
+        @recv_queue  = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @groups      = Set.new
+        @tasks       = []
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        # Send existing group memberships to new peer
+        @groups.each do |group|
+          connection.send_command(Protocol::ZMTP::Codec::Command.join(group))
+        end
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+      end
+
+      # DISH is read-only.
+      #
+      def enqueue(_parts)
+        raise "DISH sockets cannot send"
+      end
+
+      # Joins a group.
+      #
+      # @param group [String]
+      #
+      def join(group)
+        @groups << group
+        @connections.each do |conn|
+          conn.send_command(Protocol::ZMTP::Codec::Command.join(group))
+        end
+      end
+
+      # Leaves a group.
+      #
+      # @param group [String]
+      #
+      def leave(group)
+        @groups.delete(group)
+        @connections.each do |conn|
+          conn.send_command(Protocol::ZMTP::Codec::Command.leave(group))
+        end
+      end
+
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+    end
+  end
+end

data/lib/omq/routing/fan_out.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # Mixin for routing strategies that fan-out to subscribers.
+    #
+    # Manages per-connection subscription sets, subscription command
+    # listeners, and a send pump that delivers to all matching peers.
+    #
+    # Including classes must call `init_fan_out(engine)` from
+    # their #initialize.
+    #
+    module FanOut
+      attr_reader :subscriber_joined
+
+      private
+
+      def init_fan_out(engine)
+        @connections        = []
+        @subscriptions      = {} # connection => Set of prefixes
+        @send_queue         = Async::LimitedQueue.new(engine.options.send_hwm)
+        @send_pump_started  = false
+        @send_pump_idle     = true
+        @conflate           = engine.options.conflate
+        @subscriber_joined  = Async::Promise.new
+        @written            = Set.new
+        @latest             = {} if @conflate
+      end
+
+      # @return [Boolean] whether the connection is subscribed to the topic
+      #
+      def subscribed?(conn, topic)
+        subs = @subscriptions[conn]
+        return false unless subs
+        subs.any? { |prefix| topic.start_with?(prefix) }
+      end
+
+      # Called when a subscription command is received from a peer.
+      # Override in subclasses to expose subscriptions to the
+      # application (e.g. XPUB enqueues to recv_queue).
+      #
+      # @param conn [Connection]
+      # @param prefix [String]
+      #
+      def on_subscribe(conn, prefix)
+        @subscriptions[conn] << prefix.b.freeze
+        @subscriber_joined.resolve(conn) unless @subscriber_joined.resolved?
+      end
+
+      # Called when a cancel command is received from a peer.
+      # Override in subclasses (e.g. XPUB enqueues to recv_queue).
+      #
+      # @param conn [Connection]
+      # @param prefix [String]
+      #
+      def on_cancel(conn, prefix)
+        @subscriptions[conn]&.delete(prefix)
+      end
+
+      # @return [Boolean] true when the send pump is idle (not sending a batch)
+      def send_pump_idle? = @send_pump_idle
+
+
+      def start_send_pump
+        @send_pump_started = true
+        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            @send_pump_idle = true
+            batch = [@send_queue.dequeue]
+            @send_pump_idle = false
+            Routing.drain_send_queue(@send_queue, batch)
+
+            @written.clear
+
+            if @conflate
+              # Keep only the last matching message per connection.
+              @latest.clear
+              batch.each do |parts|
+                topic = parts.first || EMPTY_BINARY
+                @connections.each do |conn|
+                  next unless subscribed?(conn, topic)
+                  @latest[conn] = parts
+                end
+              end
+              @latest.each do |conn, parts|
+                begin
+                  conn.write_message(parts)
+                  @written << conn
+                rescue *CONNECTION_LOST
+                end
+              end
+            else
+              batch.each do |parts|
+                topic = parts.first || EMPTY_BINARY
+                @connections.each do |conn|
+                  next unless subscribed?(conn, topic)
+                  begin
+                    conn.write_message(parts)
+                    @written << conn
+                  rescue *CONNECTION_LOST
+                  end
+                end
+              end
+            end
+
+            @written.each do |conn|
+              conn.flush
+            rescue *CONNECTION_LOST
+            end
+          end
+        end
+      end
+
+      def start_subscription_listener(conn)
+        @tasks << @engine.spawn_pump_task(annotation: "subscription listener") do
+          loop do
+            frame = conn.read_frame
+            next unless frame.command?
+            cmd = Protocol::ZMTP::Codec::Command.from_body(frame.body)
+            case cmd.name
+            when "SUBSCRIBE" then on_subscribe(conn, cmd.data)
+            when "CANCEL"    then on_cancel(conn, cmd.data)
+            end
+          end
+        rescue *CONNECTION_LOST
+          @engine.connection_lost(conn)
+        end
+      end
+    end
+  end
+end