omq 0.12.0 → 0.14.0

data/lib/omq/routing/req.rb

@@ -8,43 +8,45 @@ module OMQ
     #
     class Req
       include RoundRobin
+      include FairRecv
 
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine          = engine
-        @recv_queue      = Routing.build_queue(engine.options.recv_hwm, :block)
+        @recv_queue      = FairQueue.new
         @tasks           = []
         @state           = :ready        # :ready or :waiting_reply
         init_round_robin(engine)
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
 
       # @param connection [Connection]
       #
       def connection_added(connection)
         @connections << connection
-        signal_connection_available
-        update_direct_pipe
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
+        add_fair_recv_connection(connection) do |msg|
           @state = :ready
           msg.first&.empty? ? msg[1..] : msg
         end
-        @tasks << task if task
-        start_send_pump unless @send_pump_started
+        add_round_robin_send_connection(connection)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
-        update_direct_pipe
+        @recv_queue.remove_queue(connection)
+        remove_round_robin_send_connection(connection)
       end
 
+
       # @param parts [Array<String>]
       #
       def enqueue(parts)
@@ -53,6 +55,10 @@ module OMQ
         enqueue_round_robin(parts)
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)
@@ -64,7 +70,6 @@ module OMQ
       # REQ prepends empty delimiter frame on the wire.
       #
       def transform_send(parts) = [EMPTY_BINARY, *parts]
-
     end
   end
 end

data/lib/omq/routing/round_robin.rb

@@ -8,16 +8,22 @@ module OMQ
     # for the first connection, Array#cycle handles round-robin,
     # and a new Promise is created when all connections drop.
     #
+    # Each connected peer gets its own bounded send queue and a
+    # dedicated send pump fiber, ensuring HWM is enforced per peer.
+    #
     # Including classes must call `init_round_robin(engine)` from
     # their #initialize.
     #
     module RoundRobin
-      # @return [Boolean] true when the send pump is idle (not sending a batch)
-      def send_pump_idle? = @send_pump_idle
+      # @return [Boolean] true when the staging queue and all per-connection
+      #   send queues are empty
+      #
+      def send_queues_drained?
+        @staging_queue.empty? && @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
-
       # Initializes round-robin state for the including class.
       #
       # @param engine [Engine]
@@ -26,11 +32,37 @@ module OMQ
         @connections          = []
         @cycle                = @connections.cycle
         @connection_available = Async::Promise.new
-        @send_queue           = Routing.build_queue(engine.options.send_hwm, :block)
-        @send_pump_started    = false
-        @send_pump_idle       = true
+        @conn_queues          = {}  # connection => send queue
+        @conn_send_tasks      = {}  # connection => send pump task
         @direct_pipe          = nil
-        @written              = Set.new
+        @staging_queue        = Routing.build_queue(@engine.options.send_hwm, :block)
+      end
+
+
+      # Creates a per-connection send queue and starts its send pump.
+      # Call from #connection_added after appending to @connections.
+      #
+      # @param conn [Connection]
+      #
+      def add_round_robin_send_connection(conn)
+        update_direct_pipe
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[conn] = q
+        drain_staging_to(q)
+        start_conn_send_pump(conn, q)
+        signal_connection_available
+      end
+
+
+      # Stops the per-connection send pump and removes the queue.
+      # Call from #connection_removed.
+      #
+      # @param conn [Connection]
+      #
+      def remove_round_robin_send_connection(conn)
+        update_direct_pipe
+        @conn_queues.delete(conn)
+        @conn_send_tasks.delete(conn)&.stop
       end
 
 
@@ -56,15 +88,33 @@ module OMQ
       end
 
 
-      # Enqueues directly to the inproc peer's recv queue if possible,
-      # otherwise falls back to the send queue for the send pump.
+      # Enqueues directly to the inproc peer's recv queue if possible.
+      # When peers are connected, picks the next one round-robin and
+      # enqueues into its per-connection send queue (blocking if full).
+      # When no peers are connected yet, buffers in a staging queue
+      # (bounded by send_hwm) — drained into the first peer's queue
+      # when it connects.
       #
       def enqueue_round_robin(parts)
         pipe = @direct_pipe
         if pipe&.direct_recv_queue
           pipe.send_message(transform_send(parts))
+        elsif @connections.empty?
+          @staging_queue.enqueue(parts)
         else
-          @send_queue.enqueue(parts)
+          conn = next_connection
+          @conn_queues[conn].enqueue(parts)
+        end
+      end
+
+
+      # Drains the staging queue into the given per-connection queue.
+      # Called when the first peer connects, to deliver messages that
+      # were enqueued before any connection existed.
+      #
+      def drain_staging_to(q)
+        while (msg = @staging_queue.dequeue(timeout: 0))
+          q.enqueue(msg)
         end
       end
 
@@ -93,66 +143,35 @@ module OMQ
       def transform_send(parts) = parts
 
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+      # Starts a dedicated send pump for one connection.
+      # Batches messages for throughput; flushes after each batch.
+      # Calls Engine#connection_lost on disconnect so reconnect fires.
+      #
+      # @param conn [Connection]
+      # @param q [Async::LimitedQueue] the connection's send queue
+      #
+      def start_conn_send_pump(conn, q)
+        task = @engine.spawn_pump_task(annotation: "send pump") do
           loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            if batch.size == 1
-              send_with_retry(batch[0])
-            else
-              send_batch(batch)
-            end
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            write_batch(conn, batch)
+          rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+            @engine.connection_lost(conn)
+            break
           end
         end
+        @conn_send_tasks[conn] = task
+        @tasks << task
       end
 
 
-      # Sends a single message, retrying on a new connection if
-      # the current one is lost.
-      #
-      # @param parts [Array<String>]
-      #
-      def send_with_retry(parts)
-        conn = next_connection
-        conn.send_message(transform_send(parts))
-      rescue *CONNECTION_LOST
-        @engine.connection_lost(conn)
-        retry
-      end
-
-
-      # Sends a batch of messages, writing without flushing for
-      # throughput. Falls back to #send_with_retry on failure.
-      #
-      # @param batch [Array<Array<String>>]
-      #
-      def send_batch(batch)
-        @written.clear
-        batch.each_with_index do |parts, i|
-          conn = next_connection
-          begin
-            conn.write_message(transform_send(parts))
-            @written << conn
-          rescue *CONNECTION_LOST
-            @engine.connection_lost(conn)
-            @written.each do |c|
-              c.flush
-            rescue *CONNECTION_LOST
-            end
-            @written.clear
-            send_with_retry(parts)
-            batch[(i + 1)..].each { |p| send_with_retry(p) }
-            return
-          end
-        end
-        @written.each do |conn|
+      def write_batch(conn, batch)
+        if batch.size == 1
+          conn.send_message(transform_send(batch[0]))
+        else
+          batch.each { |parts| conn.write_message(transform_send(parts)) }
           conn.flush
-        rescue *CONNECTION_LOST
         end
       end
     end

data/lib/omq/routing/router.rb

@@ -11,23 +11,23 @@ module OMQ
     # routing identity on send.
     #
     class Router
+      include FairRecv
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine                  = engine
-        @recv_queue              = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue              = Routing.build_queue(engine.options.send_hwm, :block)
+        @recv_queue              = FairQueue.new
         @connections_by_identity = {}
         @identity_by_connection  = {}
+        @conn_queues             = {}  # connection => per-connection send queue
+        @conn_send_tasks         = {}  # connection => send pump task
         @tasks                   = []
-        @send_pump_started       = false
-        @send_pump_idle          = true
-        @written                 = Set.new
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
@@ -37,74 +37,58 @@ module OMQ
         @connections_by_identity[identity] = connection
         @identity_by_connection[connection] = identity
 
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
-          [identity, *msg]
-        end
-        @tasks << task if task
+        add_fair_recv_connection(connection) { |msg| [identity, *msg] }
 
-        start_send_pump unless @send_pump_started
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        @conn_send_tasks[connection] = ConnSendPump.start(@engine, connection, q, @tasks)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         identity = @identity_by_connection.delete(connection)
         @connections_by_identity.delete(identity) if identity
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
-      # Enqueues a message for sending.
+
+      # Enqueues a message for sending. The first frame is the routing identity.
       #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
+        identity = parts.first
         if @engine.options.router_mandatory?
-          identity = parts.first
           unless @connections_by_identity[identity]
             raise SocketError, "no route to identity #{identity.inspect}"
           end
         end
-        @send_queue.enqueue(parts)
+        conn = @connections_by_identity[identity]
+        return unless conn  # silently drop if peer disconnected
+        @conn_queues[conn]&.enqueue(parts[1..])
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
+      #
       def stop
         @tasks.each(&:stop)
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
-
-      private
-
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-            batch.each do |parts|
-              identity = parts.first
-              conn     = @connections_by_identity[identity]
-              next unless conn # silently drop (peer may have disconnected)
-              begin
-                conn.write_message(parts[1..])
-                @written << conn
-              rescue *CONNECTION_LOST
-                # will be cleaned up
-              end
-            end
-
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-              # will be cleaned up
-            end
-          end
-        end
+
+      # @return [Boolean] true when all per-connection send queues are empty
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
       end
+
     end
   end
 end

data/lib/omq/routing/sub.rb

@@ -12,13 +12,14 @@ module OMQ
       #
       def initialize(engine)
         @engine        = engine
-        @connections   = []
-        @recv_queue    = Routing.build_queue(engine.options.recv_hwm, engine.options.on_mute)
+        @connections   = Set.new
+        @recv_queue    = FairQueue.new
         @subscriptions = Set.new
         @tasks         = []
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
       attr_reader :recv_queue
 
@@ -29,22 +30,29 @@ module OMQ
         @subscriptions.each do |prefix|
           connection.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
         end
-        task = @engine.start_recv_pump(connection, @recv_queue)
+        conn_q    = Routing.build_queue(@engine.options.recv_hwm, @engine.options.on_mute)
+        signaling = SignalingQueue.new(conn_q, @recv_queue)
+        @recv_queue.add_queue(connection, conn_q)
+        task = @engine.start_recv_pump(connection, signaling)
         @tasks << task if task
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
+        @recv_queue.remove_queue(connection)
       end
 
+
       # SUB is read-only.
       #
       def enqueue(_parts)
         raise "SUB sockets cannot send"
       end
 
+
       # Subscribes to a topic prefix.
       #
       # @param prefix [String]
@@ -56,6 +64,7 @@ module OMQ
         end
       end
 
+
       # Unsubscribes from a topic prefix.
       #
       # @param prefix [String]
@@ -67,11 +76,15 @@ module OMQ
         end
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
+      #
       def stop
         @tasks.each(&:stop)
         @tasks.clear
       end
-
     end
   end
 end

data/lib/omq/routing/xpub.rb

@@ -8,6 +8,9 @@ module OMQ
     # the application as data frames: \x01 + prefix for subscribe,
     # \x00 + prefix for unsubscribe.
     #
+    # The recv_queue is a simple bounded queue (not a FairQueue) because
+    # messages come from subscription commands, not from peer data pumps.
+    #
     class XPub
       include FanOut
 
@@ -20,9 +23,10 @@ module OMQ
         init_fan_out(engine)
       end
 
+
       # @return [Async::LimitedQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
@@ -30,22 +34,29 @@ module OMQ
         @connections << connection
         @subscriptions[connection] = Set.new
         start_subscription_listener(connection)
-        start_send_pump unless @send_pump_started
+        add_fan_out_send_connection(connection)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
         @subscriptions.delete(connection)
+        remove_fan_out_send_connection(connection)
       end
 
+
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        fan_out_enqueue(parts)
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)
@@ -61,6 +72,7 @@ module OMQ
         @recv_queue.enqueue(["\x01#{prefix}".b])
       end
 
+
       # Expose unsubscription to application as data message.
       #
       def on_cancel(conn, prefix)

data/lib/omq/routing/xsub.rb

@@ -5,78 +5,104 @@ module OMQ
     # XSUB socket routing: like SUB but subscriptions sent as data messages.
     #
     # Subscriptions are sent as data frames: \x01 + prefix for subscribe,
-    # \x00 + prefix for unsubscribe.
+    # \x00 + prefix for unsubscribe. Each connected PUB gets its own send
+    # queue so subscription commands are delivered independently per peer.
     #
     class XSub
 
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine            = engine
-        @connections       = []
-        @recv_queue        = Routing.build_queue(engine.options.recv_hwm, engine.options.on_mute)
-        @send_queue        = Routing.build_queue(engine.options.send_hwm, :block)
-        @tasks             = []
-        @send_pump_started = false
-        @send_pump_idle    = true
+        @engine          = engine
+        @connections     = Set.new
+        @recv_queue      = FairQueue.new
+        @conn_queues     = {}  # connection => per-connection send queue
+        @conn_send_tasks = {}  # connection => send pump task
+        @tasks           = []
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
         @connections << connection
-        task = @engine.start_recv_pump(connection, @recv_queue)
+
+        conn_q    = Routing.build_queue(@engine.options.recv_hwm, @engine.options.on_mute)
+        signaling = SignalingQueue.new(conn_q, @recv_queue)
+        @recv_queue.add_queue(connection, conn_q)
+        task = @engine.start_recv_pump(connection, signaling)
         @tasks << task if task
-        start_send_pump unless @send_pump_started
+
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        start_conn_send_pump(connection, q)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
+
+      # Enqueues a subscription command (fan-out to all connected PUBs).
+      #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        @connections.each { |conn| @conn_queues[conn]&.enqueue(parts) }
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
+
+      # @return [Boolean] true when all per-connection send queues are empty
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+      def start_conn_send_pump(conn, q)
+        task = @engine.spawn_pump_task(annotation: "send pump") do
           loop do
-            @send_pump_idle = true
-            parts = @send_queue.dequeue
-            @send_pump_idle = false
+            parts = q.dequeue
             frame = parts.first&.b
             next if frame.nil? || frame.empty?
-
             flag   = frame.getbyte(0)
             prefix = frame.byteslice(1..) || "".b
-
-            case flag
-            when 0x01
-              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix)) }
-            when 0x00
-              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix)) }
+            begin
+              case flag
+              when 0x01
+                conn.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
+              when 0x00
+                conn.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix))
+              end
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
             end
           end
         end
+        @conn_send_tasks[conn] = task
+        @tasks << task
       end
     end
   end