omq 0.12.0 → 0.13.0

data/lib/omq/routing/rep.rb

@@ -9,52 +9,60 @@ module OMQ
     # on send.
     #
     class Rep
-      # @param engine [Engine]
-      #
+      include FairRecv
+
       EMPTY_FRAME = "".b.freeze
 
+      # @param engine [Engine]
+      #
       def initialize(engine)
-        @engine            = engine
-        @recv_queue        = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue        = Routing.build_queue(engine.options.send_hwm, :block)
-        @pending_replies   = []
-        @tasks             = []
-        @send_pump_started = false
-        @send_pump_idle    = true
-        @written           = Set.new
+        @engine          = engine
+        @recv_queue      = FairQueue.new
+        @pending_replies = []
+        @conn_queues     = {}  # connection => per-connection send queue
+        @conn_send_tasks = {}  # connection => send pump task
+        @tasks           = []
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
+        add_fair_recv_connection(connection) do |msg|
           delimiter = msg.index(&:empty?) || msg.size
           envelope  = msg[0, delimiter]
           body      = msg[(delimiter + 1)..] || []
           @pending_replies << { conn: connection, envelope: envelope }
           body
         end
-        @tasks << task if task
-        start_send_pump unless @send_pump_started
+
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        @conn_send_tasks[connection] = ConnSendPump.start(@engine, connection, q, @tasks)
       end
 
       # @param connection [Connection]
       #
       def connection_removed(connection)
-        # Remove any pending replies for this connection
         @pending_replies.reject! { |r| r[:conn] == connection }
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
-      # Enqueues a reply for sending.
+      # Enqueues a reply. Routes to the connection that sent the matching
+      # request by consuming the next pending_reply entry.
       #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        reply_info = @pending_replies.shift
+        return unless reply_info
+        conn = reply_info[:conn]
+        @conn_queues[conn]&.enqueue([*reply_info[:envelope], EMPTY_FRAME, *parts])
       end
 
       def stop
@@ -62,40 +70,12 @@ module OMQ
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
-
-      private
-
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-            batch.each do |parts|
-              reply_info = @pending_replies.shift
-              next unless reply_info
-              conn = reply_info[:conn]
-              begin
-                conn.write_message([*reply_info[:envelope], EMPTY_FRAME, *parts])
-                @written << conn
-              rescue *CONNECTION_LOST
-                # connection lost mid-write
-              end
-            end
-
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-              # connection lost mid-flush
-            end
-          end
-        end
+      # True when all per-connection send queues are empty.
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
       end
+
     end
   end
 end

data/lib/omq/routing/req.rb

@@ -8,41 +8,40 @@ module OMQ
     #
     class Req
       include RoundRobin
+      include FairRecv
 
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine          = engine
-        @recv_queue      = Routing.build_queue(engine.options.recv_hwm, :block)
+        @recv_queue      = FairQueue.new
         @tasks           = []
         @state           = :ready        # :ready or :waiting_reply
         init_round_robin(engine)
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
 
       # @param connection [Connection]
       #
       def connection_added(connection)
         @connections << connection
-        signal_connection_available
-        update_direct_pipe
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
+        add_fair_recv_connection(connection) do |msg|
           @state = :ready
           msg.first&.empty? ? msg[1..] : msg
         end
-        @tasks << task if task
-        start_send_pump unless @send_pump_started
+        add_round_robin_send_connection(connection)
       end
 
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
-        update_direct_pipe
+        @recv_queue.remove_queue(connection)
+        remove_round_robin_send_connection(connection)
       end
 
       # @param parts [Array<String>]
@@ -64,7 +63,6 @@ module OMQ
       # REQ prepends empty delimiter frame on the wire.
       #
       def transform_send(parts) = [EMPTY_BINARY, *parts]
-
     end
   end
 end

data/lib/omq/routing/round_robin.rb

@@ -8,16 +8,22 @@ module OMQ
     # for the first connection, Array#cycle handles round-robin,
     # and a new Promise is created when all connections drop.
     #
+    # Each connected peer gets its own bounded send queue and a
+    # dedicated send pump fiber, ensuring HWM is enforced per peer.
+    #
     # Including classes must call `init_round_robin(engine)` from
     # their #initialize.
     #
     module RoundRobin
-      # @return [Boolean] true when the send pump is idle (not sending a batch)
-      def send_pump_idle? = @send_pump_idle
+      # True when the staging queue and all per-connection send queues
+      # are empty. Used by Engine#drain_send_queues during linger.
+      #
+      def send_queues_drained?
+        @staging_queue.empty? && @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
-
       # Initializes round-robin state for the including class.
       #
       # @param engine [Engine]
@@ -26,11 +32,37 @@ module OMQ
         @connections          = []
         @cycle                = @connections.cycle
         @connection_available = Async::Promise.new
-        @send_queue           = Routing.build_queue(engine.options.send_hwm, :block)
-        @send_pump_started    = false
-        @send_pump_idle       = true
+        @conn_queues          = {}  # connection => send queue
+        @conn_send_tasks      = {}  # connection => send pump task
         @direct_pipe          = nil
-        @written              = Set.new
+        @staging_queue        = Routing.build_queue(@engine.options.send_hwm, :block)
+      end
+
+
+      # Creates a per-connection send queue and starts its send pump.
+      # Call from #connection_added after appending to @connections.
+      #
+      # @param conn [Connection]
+      #
+      def add_round_robin_send_connection(conn)
+        update_direct_pipe
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[conn] = q
+        drain_staging_to(q)
+        start_conn_send_pump(conn, q)
+        signal_connection_available
+      end
+
+
+      # Stops the per-connection send pump and removes the queue.
+      # Call from #connection_removed.
+      #
+      # @param conn [Connection]
+      #
+      def remove_round_robin_send_connection(conn)
+        update_direct_pipe
+        @conn_queues.delete(conn)
+        @conn_send_tasks.delete(conn)&.stop
       end
 
 
@@ -56,15 +88,33 @@ module OMQ
       end
 
 
-      # Enqueues directly to the inproc peer's recv queue if possible,
-      # otherwise falls back to the send queue for the send pump.
+      # Enqueues directly to the inproc peer's recv queue if possible.
+      # When peers are connected, picks the next one round-robin and
+      # enqueues into its per-connection send queue (blocking if full).
+      # When no peers are connected yet, buffers in a staging queue
+      # (bounded by send_hwm) — drained into the first peer's queue
+      # when it connects.
       #
       def enqueue_round_robin(parts)
         pipe = @direct_pipe
         if pipe&.direct_recv_queue
           pipe.send_message(transform_send(parts))
+        elsif @connections.empty?
+          @staging_queue.enqueue(parts)
         else
-          @send_queue.enqueue(parts)
+          conn = next_connection
+          @conn_queues[conn]&.enqueue(parts)
+        end
+      end
+
+
+      # Drains the staging queue into the given per-connection queue.
+      # Called when the first peer connects, to deliver messages that
+      # were enqueued before any connection existed.
+      #
+      def drain_staging_to(q)
+        while (msg = @staging_queue.dequeue(timeout: 0))
+          q.enqueue(msg)
         end
       end
 
@@ -93,66 +143,34 @@ module OMQ
       def transform_send(parts) = parts
 
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            if batch.size == 1
-              send_with_retry(batch[0])
-            else
-              send_batch(batch)
-            end
-          end
-        end
-      end
-
-
-      # Sends a single message, retrying on a new connection if
-      # the current one is lost.
+      # Starts a dedicated send pump for one connection.
+      # Batches messages for throughput; flushes after each batch.
+      # Calls Engine#connection_lost on disconnect so reconnect fires.
       #
-      # @param parts [Array<String>]
+      # @param conn [Connection]
+      # @param q [Async::LimitedQueue] the connection's send queue
       #
-      def send_with_retry(parts)
-        conn = next_connection
-        conn.send_message(transform_send(parts))
-      rescue *CONNECTION_LOST
-        @engine.connection_lost(conn)
-        retry
-      end
-
-
-      # Sends a batch of messages, writing without flushing for
-      # throughput. Falls back to #send_with_retry on failure.
-      #
-      # @param batch [Array<Array<String>>]
-      #
-      def send_batch(batch)
-        @written.clear
-        batch.each_with_index do |parts, i|
-          conn = next_connection
-          begin
-            conn.write_message(transform_send(parts))
-            @written << conn
-          rescue *CONNECTION_LOST
+      def start_conn_send_pump(conn, q)
+        task = @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            write_batch(conn, batch)
+          rescue Protocol::ZMTP::Error, *CONNECTION_LOST
             @engine.connection_lost(conn)
-            @written.each do |c|
-              c.flush
-            rescue *CONNECTION_LOST
-            end
-            @written.clear
-            send_with_retry(parts)
-            batch[(i + 1)..].each { |p| send_with_retry(p) }
-            return
+            break
           end
         end
-        @written.each do |conn|
+        @conn_send_tasks[conn] = task
+        @tasks << task
+      end
+
+      def write_batch(conn, batch)
+        if batch.size == 1
+          conn.send_message(transform_send(batch[0]))
+        else
+          batch.each { |parts| conn.write_message(transform_send(parts)) }
           conn.flush
-        rescue *CONNECTION_LOST
         end
       end
     end

data/lib/omq/routing/router.rb

@@ -11,23 +11,22 @@ module OMQ
     # routing identity on send.
     #
     class Router
+      include FairRecv
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine                  = engine
-        @recv_queue              = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue              = Routing.build_queue(engine.options.send_hwm, :block)
+        @recv_queue              = FairQueue.new
         @connections_by_identity = {}
         @identity_by_connection  = {}
+        @conn_queues             = {}  # connection => per-connection send queue
+        @conn_send_tasks         = {}  # connection => send pump task
         @tasks                   = []
-        @send_pump_started       = false
-        @send_pump_idle          = true
-        @written                 = Set.new
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
@@ -37,12 +36,11 @@ module OMQ
         @connections_by_identity[identity] = connection
         @identity_by_connection[connection] = identity
 
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
-          [identity, *msg]
-        end
-        @tasks << task if task
+        add_fair_recv_connection(connection) { |msg| [identity, *msg] }
 
-        start_send_pump unless @send_pump_started
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        @conn_send_tasks[connection] = ConnSendPump.start(@engine, connection, q, @tasks)
       end
 
       # @param connection [Connection]
@@ -50,20 +48,25 @@ module OMQ
       def connection_removed(connection)
         identity = @identity_by_connection.delete(connection)
         @connections_by_identity.delete(identity) if identity
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
-      # Enqueues a message for sending.
+      # Enqueues a message for sending. The first frame is the routing identity.
       #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
+        identity = parts.first
         if @engine.options.router_mandatory?
-          identity = parts.first
           unless @connections_by_identity[identity]
             raise SocketError, "no route to identity #{identity.inspect}"
           end
         end
-        @send_queue.enqueue(parts)
+        conn = @connections_by_identity[identity]
+        return unless conn  # silently drop if peer disconnected
+        @conn_queues[conn]&.enqueue(parts[1..])
       end
 
       def stop
@@ -71,40 +74,12 @@ module OMQ
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
-
-      private
-
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-            batch.each do |parts|
-              identity = parts.first
-              conn     = @connections_by_identity[identity]
-              next unless conn # silently drop (peer may have disconnected)
-              begin
-                conn.write_message(parts[1..])
-                @written << conn
-              rescue *CONNECTION_LOST
-                # will be cleaned up
-              end
-            end
-
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-              # will be cleaned up
-            end
-          end
-        end
+      # True when all per-connection send queues are empty.
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
       end
+
     end
   end
 end

data/lib/omq/routing/sub.rb

@@ -12,13 +12,13 @@ module OMQ
       #
       def initialize(engine)
         @engine        = engine
-        @connections   = []
-        @recv_queue    = Routing.build_queue(engine.options.recv_hwm, engine.options.on_mute)
+        @connections   = Set.new
+        @recv_queue    = FairQueue.new
         @subscriptions = Set.new
         @tasks         = []
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
       attr_reader :recv_queue
 
@@ -29,7 +29,10 @@ module OMQ
         @subscriptions.each do |prefix|
           connection.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
         end
-        task = @engine.start_recv_pump(connection, @recv_queue)
+        conn_q    = Routing.build_queue(@engine.options.recv_hwm, @engine.options.on_mute)
+        signaling = SignalingQueue.new(conn_q, @recv_queue)
+        @recv_queue.add_queue(connection, conn_q)
+        task = @engine.start_recv_pump(connection, signaling)
         @tasks << task if task
       end
 
@@ -37,6 +40,7 @@ module OMQ
       #
       def connection_removed(connection)
         @connections.delete(connection)
+        @recv_queue.remove_queue(connection)
       end
 
       # SUB is read-only.
@@ -71,7 +75,6 @@ module OMQ
         @tasks.each(&:stop)
         @tasks.clear
       end
-
     end
   end
 end

data/lib/omq/routing/xpub.rb

@@ -8,6 +8,9 @@ module OMQ
     # the application as data frames: \x01 + prefix for subscribe,
     # \x00 + prefix for unsubscribe.
     #
+    # The recv_queue is a simple bounded queue (not a FairQueue) because
+    # messages come from subscription commands, not from peer data pumps.
+    #
     class XPub
       include FanOut
 
@@ -22,7 +25,7 @@ module OMQ
 
       # @return [Async::LimitedQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
@@ -30,7 +33,7 @@ module OMQ
         @connections << connection
         @subscriptions[connection] = Set.new
         start_subscription_listener(connection)
-        start_send_pump unless @send_pump_started
+        add_fan_out_send_connection(connection)
       end
 
       # @param connection [Connection]
@@ -38,12 +41,13 @@ module OMQ
       def connection_removed(connection)
         @connections.delete(connection)
         @subscriptions.delete(connection)
+        remove_fan_out_send_connection(connection)
       end
 
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        fan_out_enqueue(parts)
       end
 
       #