omq 0.12.0 → 0.13.0

data/lib/omq/routing/fair_queue.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # Per-connection recv queue aggregator.
+    #
+    # Maintains one bounded queue per connected peer. #dequeue
+    # returns the next available message from any peer in fair
+    # round-robin order, blocking until one arrives.
+    #
+    # Recv pumps do not enqueue directly — they write through a
+    # SignalingQueue wrapper, which also wakes a blocked #dequeue.
+    #
+    class FairQueue
+      def initialize
+        @queues    = []              # ordered list of per-connection inner queues
+        @mapping   = {}              # connection => inner queue
+        @cycle     = @queues.cycle  # live reference — sees adds/removes
+        @condition = Async::Condition.new
+        @pending   = 0              # signals received before #dequeue waits
+        @closed    = false
+      end
+
+      # Registers a per-connection queue. Called when a connection is added.
+      #
+      # @param conn [Connection]
+      # @param q [Async::LimitedQueue]
+      #
+      def add_queue(conn, q)
+        @mapping[conn] = q
+        @queues << q
+      end
+
+      # Removes the per-connection queue for a disconnected peer.
+      #
+      # If the queue is empty it is removed immediately. If it still has
+      # pending messages it is kept in @queues so the application can drain
+      # them via #dequeue; it will be cleaned up lazily by try_dequeue once
+      # it is empty.
+      #
+      # @param conn [Connection]
+      #
+      def remove_queue(conn)
+        q = @mapping.delete(conn)
+        return unless q
+        @queues.delete(q) if q.empty?
+        # Non-empty orphaned queues stay in @queues until drained
+      end
+
+      # Wakes a blocked #dequeue. Called by SignalingQueue after each enqueue.
+      #
+      def signal
+        @pending += 1
+        @condition.signal
+      end
+
+      # Returns the next message from any per-connection queue, in fair
+      # round-robin order. Blocks until a message is available.
+      #
+      # Pass +timeout: 0+ for a non-blocking poll (returns nil immediately
+      # if no messages are available).
+      #
+      # @param timeout [Numeric, nil] 0 = non-blocking, nil = block forever
+      # @return [Array<String>, nil]
+      #
+      def dequeue(timeout: nil)
+        return try_dequeue if timeout == 0
+
+        loop do
+          return nil if @closed && @queues.all?(&:empty?)
+
+          msg = try_dequeue
+          return msg if msg
+
+          if @pending > 0
+            @pending -= 1
+            next
+          end
+
+          @condition.wait
+        end
+      end
+
+      # Injects a nil sentinel to unblock a waiting #dequeue.
+      # Called by Engine on close or fatal error.
+      #
+      def push(nil_sentinel)
+        @closed = true
+        @condition.signal
+      end
+
+      # @return [Boolean]
+      #
+      def empty?
+        @queues.all?(&:empty?)
+      end
+
+      private
+
+      # Tries each per-connection queue once in round-robin order.
+      # Returns the first message found, or nil if all are empty.
+      # Lazily removes empty orphaned queues (disconnected peers that have
+      # been fully drained).
+      #
+      def try_dequeue
+        @queues.size.times do
+          q = begin
+                @cycle.next
+              rescue StopIteration
+                @cycle = @queues.cycle
+                break
+              end
+          msg = q.dequeue(timeout: 0)
+          return msg if msg
+          if q.empty? && !@mapping.value?(q)
+            @queues.delete(q)
+            break
+          end
+        end
+        nil
+      end
+    end
+
+
+    # Wraps a per-connection bounded queue so that each #enqueue also
+    # signals the FairQueue to wake a blocked #dequeue.
+    #
+    class SignalingQueue
+      def initialize(inner, fair_queue)
+        @inner = inner
+        @fair  = fair_queue
+      end
+
+      def enqueue(msg)
+        @inner.enqueue(msg)
+        @fair.signal
+      end
+
+      def dequeue(timeout: nil) = @inner.dequeue(timeout: timeout)
+      def empty?                = @inner.empty?
+      def push(item)            = @inner.push(item)
+    end
+  end
+end

data/lib/omq/routing/fair_recv.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # Mixin that adds per-connection recv queue setup for fair-queued sockets.
+    #
+    # Including classes must have @engine, @recv_queue (FairQueue), and @tasks.
+    #
+    module FairRecv
+      private
+
+      # Creates a per-connection recv queue, registers it with @recv_queue,
+      # and starts a recv pump for the connection. Called from #connection_added.
+      #
+      # @param conn [Connection]
+      # @yield [msg] optional per-message transform
+      #
+      def add_fair_recv_connection(conn, &transform)
+        conn_q    = Routing.build_queue(@engine.options.recv_hwm, :block)
+        signaling = SignalingQueue.new(conn_q, @recv_queue)
+        @recv_queue.add_queue(conn, conn_q)
+        task      = @engine.start_recv_pump(conn, signaling, &transform)
+        @tasks << task if task
+      end
+    end
+  end
+end

data/lib/omq/routing/fan_out.rb

@@ -5,7 +5,13 @@ module OMQ
     # Mixin for routing strategies that fan-out to subscribers.
     #
     # Manages per-connection subscription sets, subscription command
-    # listeners, and a send pump that delivers to all matching peers.
+    # listeners, and per-connection send queues/pumps that deliver
+    # to each matching peer independently.
+    #
+    # HWM is enforced per subscriber: each connection gets its own
+    # bounded send queue. DropQueues (for :drop_newest/:drop_oldest)
+    # silently drop messages for a slow subscriber without affecting
+    # others. LimitedQueues (for :block) block the publisher.
     #
     # Including classes must call `init_fan_out(engine)` from
     # their #initialize.
@@ -13,24 +19,26 @@ module OMQ
     module FanOut
       attr_reader :subscriber_joined
 
-      # @return [Boolean] true when the send pump is idle (not sending a batch)
-      def send_pump_idle? = @send_pump_idle
+      # True when all per-connection send queues are empty.
+      # Used by Engine#drain_send_queues during linger.
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
       def init_fan_out(engine)
-        @connections        = []
+        @connections        = Set.new
         @subscriptions      = {} # connection => Set of prefixes
-        @send_queue         = Routing.build_queue(engine.options.send_hwm, :block)
-        @on_mute            = engine.options.on_mute
-        @send_pump_started  = false
-        @send_pump_idle     = true
+        @conn_queues        = {} # connection => per-connection send queue
+        @conn_send_tasks    = {} # connection => send pump task
         @conflate           = engine.options.conflate
         @subscriber_joined  = Async::Promise.new
-        @written            = Set.new
         @latest             = {} if @conflate
       end
 
+
       # @return [Boolean] whether the connection is subscribed to the topic
       #
       def subscribed?(conn, topic)
@@ -61,77 +69,50 @@ module OMQ
         @subscriptions[conn]&.delete(prefix)
       end
 
-      # Returns true if the connection is muted (recv queue full) and
-      # the on_mute strategy says to drop rather than block.
+
+      # Creates a per-connection send queue and starts its send pump.
+      # Call from #connection_added.
       #
-      def muted?(conn)
-        return false if @on_mute == :block
-        q = conn.direct_recv_queue if conn.respond_to?(:direct_recv_queue)
-        q&.respond_to?(:limited?) && q.limited?
+      # @param conn [Connection]
+      #
+      def add_fan_out_send_connection(conn)
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[conn] = q
+        start_conn_send_pump(conn, q)
       end
 
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-
-            if @conflate
-              # Keep only the last matching message per connection.
-              @latest.clear
-              batch.each do |parts|
-                topic = parts.first || EMPTY_BINARY
-                @connections.each do |conn|
-                  next unless subscribed?(conn, topic)
-                  @latest[conn] = parts
-                end
-              end
-              @latest.each do |conn, parts|
-                next if muted?(conn)
-                begin
-                  conn.write_message(parts)
-                  @written << conn
-                rescue *CONNECTION_LOST
-                end
-              end
-            else
-              batch.each do |parts|
-                topic      = parts.first || EMPTY_BINARY
-                wire_bytes = nil
-
-                @connections.each do |conn|
-                  next unless subscribed?(conn, topic)
-                  next if muted?(conn)
-                  begin
-                    if conn.respond_to?(:curve?) && conn.curve?
-                      conn.write_message(parts)
-                    elsif conn.respond_to?(:write_wire)
-                      wire_bytes ||= Protocol::ZMTP::Codec::Frame.encode_message(parts)
-                      conn.write_wire(wire_bytes)
-                    else
-                      conn.write_message(parts)
-                    end
-                    @written << conn
-                  rescue *CONNECTION_LOST
-                  end
-                end
-              end
-            end
+      # Stops the per-connection send pump and removes the queue.
+      # Call from #connection_removed.
+      #
+      # @param conn [Connection]
+      #
+      def remove_fan_out_send_connection(conn)
+        @conn_queues.delete(conn)
+        @conn_send_tasks.delete(conn)&.stop
+      end
 
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-            end
-          end
+
+      # Fans a message out to every connected peer's send queue.
+      # Subscription filtering happens in the per-connection send pump so
+      # that late-arriving subscriptions (e.g. inproc connect-before-subscribe)
+      # are respected: a message enqueued before the async subscription listener
+      # has processed SUBSCRIBE commands will still be delivered correctly.
+      #
+      # Per-connection queues use :block (Async::LimitedQueue) for
+      # backpressure: when a subscriber's queue is full, the publisher
+      # yields until the send pump drains it. This matches the old
+      # shared-queue behavior and keeps the publisher fiber-friendly.
+      #
+      # @param parts [Array<String>]
+      #
+      def fan_out_enqueue(parts)
+        @connections.each do |conn|
+          @conn_queues[conn]&.enqueue(parts)
         end
       end
 
+
       def start_subscription_listener(conn)
         @tasks << @engine.spawn_pump_task(annotation: "subscription listener") do
           loop do
@@ -147,6 +128,66 @@ module OMQ
           @engine.connection_lost(conn)
         end
       end
+
+
+      # Starts a dedicated send pump for one subscriber connection.
+      # Uses write_wire (pre-encoded bytes) for non-CURVE TCP connections
+      # to avoid re-encoding the same message N times during fan-out.
+      # In conflate mode, drains the batch and keeps only the latest
+      # message per topic before writing.
+      #
+      # @param conn [Connection]
+      # @param q [Async::LimitedQueue, DropQueue]
+      #
+      def start_conn_send_pump(conn, q)
+        use_wire = conn.respond_to?(:write_wire) && !(conn.respond_to?(:curve?) && conn.curve?)
+        task     = @conflate ? start_conn_send_pump_conflate(conn, q) : start_conn_send_pump_normal(conn, q, use_wire)
+        @conn_send_tasks[conn] = task
+        @tasks << task
+      end
+
+      def start_conn_send_pump_normal(conn, q, use_wire)
+        @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            conn.flush if write_matching_batch(conn, batch, use_wire)
+          rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+            @engine.connection_lost(conn)
+            break
+          end
+        end
+      end
+
+      def write_matching_batch(conn, batch, use_wire)
+        sent = false
+        batch.each do |parts|
+          next unless subscribed?(conn, parts.first || EMPTY_BINARY)
+          use_wire ? conn.write_wire(Protocol::ZMTP::Codec::Frame.encode_message(parts)) : conn.write_message(parts)
+          sent = true
+        end
+        sent
+      end
+
+
+      def start_conn_send_pump_conflate(conn, q)
+        @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            # Keep only the latest message that matches the subscription.
+            latest = batch.reverse.find { |parts| subscribed?(conn, parts.first || EMPTY_BINARY) }
+            next unless latest
+            begin
+              conn.write_message(latest)
+              conn.flush
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/omq/routing/pair.rb

@@ -4,25 +4,28 @@ module OMQ
   module Routing
     # PAIR socket routing: exclusive 1-to-1 bidirectional.
     #
-    # Only one peer connection is allowed. Messages flow through
-    # internal send/recv queues backed by Async::LimitedQueue.
+    # Only one peer connection is allowed. Send and recv queues are
+    # created per-connection (and destroyed on disconnection) so
+    # HWM is consistent with multi-peer socket types.
     #
     class Pair
+      include FairRecv
 
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine     = engine
-        @connection = nil
-        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue = Routing.build_queue(engine.options.send_hwm, :block)
+        @engine         = engine
+        @connection     = nil
+        @recv_queue     = FairQueue.new
+        @send_queue     = nil   # created per-connection
+        @staging_queue  = Routing.build_queue(@engine.options.send_hwm, :block)
+        @send_pump      = nil
         @tasks          = []
-        @send_pump_idle = true
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       # @raise [RuntimeError] if a connection already exists
@@ -30,9 +33,16 @@ module OMQ
       def connection_added(connection)
         raise "PAIR allows only one peer" if @connection
         @connection = connection
-        task = @engine.start_recv_pump(connection, @recv_queue)
-        @tasks << task if task
-        start_send_pump(connection) unless connection.is_a?(Transport::Inproc::DirectPipe)
+
+        add_fair_recv_connection(connection)
+
+        unless connection.is_a?(Transport::Inproc::DirectPipe)
+          @send_queue = Routing.build_queue(@engine.options.send_hwm, :block)
+          while (msg = @staging_queue.dequeue(timeout: 0))
+            @send_queue.enqueue(msg)
+          end
+          start_send_pump(connection)
+        end
       end
 
       # @param connection [Connection]
@@ -40,6 +50,8 @@ module OMQ
       def connection_removed(connection)
         if @connection == connection
           @connection = nil
+          @recv_queue.remove_queue(connection)
+          @send_queue = nil
           @send_pump&.stop
           @send_pump = nil
         end
@@ -51,8 +63,10 @@ module OMQ
         conn = @connection
         if conn.is_a?(Transport::Inproc::DirectPipe) && conn.direct_recv_queue
           conn.send_message(parts)
-        else
+        elsif @send_queue
           @send_queue.enqueue(parts)
+        else
+          @staging_queue.enqueue(parts)
         end
       end
 
@@ -62,22 +76,27 @@ module OMQ
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
+      # True when the staging and send queues are empty.
+      #
+      def send_queues_drained?
+        @staging_queue.empty? && (@send_queue.nil? || @send_queue.empty?)
+      end
 
       private
 
       def start_send_pump(conn)
         @send_pump = @engine.spawn_pump_task(annotation: "send pump") do
           loop do
-            @send_pump_idle = true
             batch = [@send_queue.dequeue]
-            @send_pump_idle = false
             Routing.drain_send_queue(@send_queue, batch)
-            batch.each { |parts| conn.write_message(parts) }
-            conn.flush
+            begin
+              batch.each { |parts| conn.write_message(parts) }
+              conn.flush
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
+            end
           end
-        rescue *CONNECTION_LOST
-          @engine.connection_lost(conn)
         end
         @tasks << @send_pump
       end

data/lib/omq/routing/pub.rb

@@ -5,7 +5,8 @@ module OMQ
     # PUB socket routing: fan-out to all subscribers.
     #
     # Listens for SUBSCRIBE/CANCEL commands from peers.
-    # Drops messages if a subscriber's connection write fails.
+    # Each subscriber gets its own bounded send queue; slow subscribers
+    # are muted via the socket's on_mute strategy (drop by default).
     #
     class Pub
       include FanOut
@@ -18,10 +19,6 @@ module OMQ
         init_fan_out(engine)
       end
 
-      # @return [Async::LimitedQueue]
-      #
-      attr_reader :send_queue
-
       # PUB is write-only.
       #
       def recv_queue
@@ -34,7 +31,7 @@ module OMQ
         @connections << connection
         @subscriptions[connection] = Set.new
         start_subscription_listener(connection)
-        start_send_pump unless @send_pump_started
+        add_fan_out_send_connection(connection)
       end
 
       # @param connection [Connection]
@@ -42,12 +39,13 @@ module OMQ
       def connection_removed(connection)
         @connections.delete(connection)
         @subscriptions.delete(connection)
+        remove_fan_out_send_connection(connection)
       end
 
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        fan_out_enqueue(parts)
       end
 
       #

data/lib/omq/routing/pull.rb

@@ -5,28 +5,29 @@ module OMQ
     # PULL socket routing: fair-queue receive from PUSH peers.
     #
     class Pull
+      include FairRecv
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine     = engine
-        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
+        @recv_queue = FairQueue.new
         @tasks      = []
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
       attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
-        task = @engine.start_recv_pump(connection, @recv_queue)
-        @tasks << task if task
+        add_fair_recv_connection(connection)
       end
 
       # @param connection [Connection]
       #
       def connection_removed(connection)
+        @recv_queue.remove_queue(connection)
         # recv pump stops on EOFError
       end
 

data/lib/omq/routing/push.rb

@@ -16,11 +16,6 @@ module OMQ
       end
 
 
-      # @return [Async::LimitedQueue]
-      #
-      attr_reader :send_queue
-
-
       # PUSH is write-only.
       #
       def recv_queue
@@ -32,9 +27,7 @@ module OMQ
       #
       def connection_added(connection)
         @connections << connection
-        signal_connection_available
-        update_direct_pipe
-        start_send_pump unless @send_pump_started
+        add_round_robin_send_connection(connection)
         start_reaper(connection)
       end
 
@@ -43,7 +36,7 @@ module OMQ
       #
       def connection_removed(connection)
         @connections.delete(connection)
-        update_direct_pipe
+        remove_round_robin_send_connection(connection)
       end
 
 
@@ -54,7 +47,7 @@ module OMQ
       end
 
 
-      # Stops all background tasks (send pump, reapers).
+      # Stops all background tasks (send pumps, reapers).
       #
       def stop
         @tasks.each(&:stop)
@@ -76,8 +69,6 @@ module OMQ
           @engine.connection_lost(conn)
         end
       end
-
-
     end
   end
 end