omq 0.12.0 → 0.14.0

data/lib/omq/routing/fan_out.rb

@@ -5,40 +5,52 @@ module OMQ
     # Mixin for routing strategies that fan-out to subscribers.
     #
     # Manages per-connection subscription sets, subscription command
-    # listeners, and a send pump that delivers to all matching peers.
+    # listeners, and per-connection send queues/pumps that deliver
+    # to each matching peer independently.
+    #
+    # HWM is enforced per subscriber: each connection gets its own
+    # bounded send queue. DropQueues (for :drop_newest/:drop_oldest)
+    # silently drop messages for a slow subscriber without affecting
+    # others. LimitedQueues (for :block) block the publisher.
     #
     # Including classes must call `init_fan_out(engine)` from
     # their #initialize.
     #
     module FanOut
+      # @return [Async::Promise] resolves when the first subscriber joins
+      #
       attr_reader :subscriber_joined
 
-      # @return [Boolean] true when the send pump is idle (not sending a batch)
-      def send_pump_idle? = @send_pump_idle
+      # @return [Boolean] true when all per-connection send queues are empty
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
       def init_fan_out(engine)
-        @connections        = []
+        @connections        = Set.new
         @subscriptions      = {} # connection => Set of prefixes
-        @send_queue         = Routing.build_queue(engine.options.send_hwm, :block)
-        @on_mute            = engine.options.on_mute
-        @send_pump_started  = false
-        @send_pump_idle     = true
+        @subscribe_all      = Set.new # connections subscribed to "" (match-all fast path)
+        @conn_queues        = {} # connection => per-connection send queue
+        @conn_send_tasks    = {} # connection => send pump task
         @conflate           = engine.options.conflate
         @subscriber_joined  = Async::Promise.new
-        @written            = Set.new
         @latest             = {} if @conflate
       end
 
+
       # @return [Boolean] whether the connection is subscribed to the topic
       #
       def subscribed?(conn, topic)
+        return true if @subscribe_all.include?(conn)
         subs = @subscriptions[conn]
         return false unless subs
         subs.any? { |prefix| topic.start_with?(prefix) }
       end
 
+
       # Called when a subscription command is received from a peer.
       # Override in subclasses to expose subscriptions to the
       # application (e.g. XPUB enqueues to recv_queue).
@@ -48,9 +60,11 @@ module OMQ
       #
       def on_subscribe(conn, prefix)
         @subscriptions[conn] << prefix.b.freeze
+        @subscribe_all.add(conn) if prefix.empty?
         @subscriber_joined.resolve(conn) unless @subscriber_joined.resolved?
       end
 
+
       # Called when a cancel command is received from a peer.
       # Override in subclasses (e.g. XPUB enqueues to recv_queue).
       #
@@ -59,79 +73,54 @@ module OMQ
       #
       def on_cancel(conn, prefix)
         @subscriptions[conn]&.delete(prefix)
+        @subscribe_all.delete(conn) if prefix.empty?
       end
 
-      # Returns true if the connection is muted (recv queue full) and
-      # the on_mute strategy says to drop rather than block.
+
+      # Creates a per-connection send queue and starts its send pump.
+      # Call from #connection_added.
+      #
+      # @param conn [Connection]
       #
-      def muted?(conn)
-        return false if @on_mute == :block
-        q = conn.direct_recv_queue if conn.respond_to?(:direct_recv_queue)
-        q&.respond_to?(:limited?) && q.limited?
+      def add_fan_out_send_connection(conn)
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[conn] = q
+        start_conn_send_pump(conn, q)
       end
 
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-
-            if @conflate
-              # Keep only the last matching message per connection.
-              @latest.clear
-              batch.each do |parts|
-                topic = parts.first || EMPTY_BINARY
-                @connections.each do |conn|
-                  next unless subscribed?(conn, topic)
-                  @latest[conn] = parts
-                end
-              end
-              @latest.each do |conn, parts|
-                next if muted?(conn)
-                begin
-                  conn.write_message(parts)
-                  @written << conn
-                rescue *CONNECTION_LOST
-                end
-              end
-            else
-              batch.each do |parts|
-                topic      = parts.first || EMPTY_BINARY
-                wire_bytes = nil
-
-                @connections.each do |conn|
-                  next unless subscribed?(conn, topic)
-                  next if muted?(conn)
-                  begin
-                    if conn.respond_to?(:curve?) && conn.curve?
-                      conn.write_message(parts)
-                    elsif conn.respond_to?(:write_wire)
-                      wire_bytes ||= Protocol::ZMTP::Codec::Frame.encode_message(parts)
-                      conn.write_wire(wire_bytes)
-                    else
-                      conn.write_message(parts)
-                    end
-                    @written << conn
-                  rescue *CONNECTION_LOST
-                  end
-                end
-              end
-            end
+      # Stops the per-connection send pump and removes the queue.
+      # Call from #connection_removed.
+      #
+      # @param conn [Connection]
+      #
+      def remove_fan_out_send_connection(conn)
+        @subscribe_all.delete(conn)
+        @conn_queues.delete(conn)
+        @conn_send_tasks.delete(conn)&.stop
+      end
 
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-            end
-          end
+
+      # Fans a message out to every connected peer's send queue.
+      # Subscription filtering happens in the per-connection send pump so
+      # that late-arriving subscriptions (e.g. inproc connect-before-subscribe)
+      # are respected: a message enqueued before the async subscription listener
+      # has processed SUBSCRIBE commands will still be delivered correctly.
+      #
+      # Per-connection queues use :block (Async::LimitedQueue) for
+      # backpressure: when a subscriber's queue is full, the publisher
+      # yields until the send pump drains it. This matches the old
+      # shared-queue behavior and keeps the publisher fiber-friendly.
+      #
+      # @param parts [Array<String>]
+      #
+      def fan_out_enqueue(parts)
+        @connections.each do |conn|
+          @conn_queues[conn].enqueue(parts)
         end
       end
 
+
       def start_subscription_listener(conn)
         @tasks << @engine.spawn_pump_task(annotation: "subscription listener") do
           loop do
@@ -139,14 +128,78 @@ module OMQ
             next unless frame.command?
             cmd = Protocol::ZMTP::Codec::Command.from_body(frame.body)
             case cmd.name
-            when "SUBSCRIBE" then on_subscribe(conn, cmd.data)
-            when "CANCEL"    then on_cancel(conn, cmd.data)
+            when "SUBSCRIBE"
+              on_subscribe(conn, cmd.data)
+            when "CANCEL"
+              on_cancel(conn, cmd.data)
             end
           end
         rescue *CONNECTION_LOST
           @engine.connection_lost(conn)
         end
       end
+
+
+      # Starts a dedicated send pump for one subscriber connection.
+      # Uses write_wire (pre-encoded bytes) for non-encrypted TCP connections
+      # to avoid re-encoding the same message N times during fan-out.
+      # In conflate mode, drains the batch and keeps only the latest
+      # message per topic before writing.
+      #
+      # @param conn [Connection]
+      # @param q [Async::LimitedQueue, DropQueue]
+      #
+      def start_conn_send_pump(conn, q)
+        use_wire = conn.respond_to?(:write_wire) && !conn.encrypted?
+        task     = @conflate ? start_conn_send_pump_conflate(conn, q) : start_conn_send_pump_normal(conn, q, use_wire)
+        @conn_send_tasks[conn] = task
+        @tasks << task
+      end
+
+
+      def start_conn_send_pump_normal(conn, q, use_wire)
+        @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            conn.flush if write_matching_batch(conn, batch, use_wire)
+          rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+            @engine.connection_lost(conn)
+            break
+          end
+        end
+      end
+
+
+      def write_matching_batch(conn, batch, use_wire)
+        sent = false
+        batch.each do |parts|
+          next unless subscribed?(conn, parts.first || EMPTY_BINARY)
+          use_wire ? conn.write_wire(Protocol::ZMTP::Codec::Frame.encode_message(parts)) : conn.write_message(parts)
+          sent = true
+        end
+        sent
+      end
+
+
+      def start_conn_send_pump_conflate(conn, q)
+        @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            batch = [q.dequeue]
+            Routing.drain_send_queue(q, batch)
+            # Keep only the latest message that matches the subscription.
+            latest = batch.reverse.find { |parts| subscribed?(conn, parts.first || EMPTY_BINARY) }
+            next unless latest
+            begin
+              conn.write_message(latest)
+              conn.flush
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/omq/routing/pair.rb

@@ -4,25 +4,29 @@ module OMQ
   module Routing
     # PAIR socket routing: exclusive 1-to-1 bidirectional.
     #
-    # Only one peer connection is allowed. Messages flow through
-    # internal send/recv queues backed by Async::LimitedQueue.
+    # Only one peer connection is allowed. Send and recv queues are
+    # created per-connection (and destroyed on disconnection) so
+    # HWM is consistent with multi-peer socket types.
     #
     class Pair
+      include FairRecv
 
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine     = engine
-        @connection = nil
-        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue = Routing.build_queue(engine.options.send_hwm, :block)
+        @engine         = engine
+        @connection     = nil
+        @recv_queue     = FairQueue.new
+        @send_queue     = nil   # created per-connection
+        @staging_queue  = Routing.build_queue(@engine.options.send_hwm, :block)
+        @send_pump      = nil
         @tasks          = []
-        @send_pump_idle = true
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       # @raise [RuntimeError] if a connection already exists
@@ -30,54 +34,77 @@ module OMQ
       def connection_added(connection)
         raise "PAIR allows only one peer" if @connection
         @connection = connection
-        task = @engine.start_recv_pump(connection, @recv_queue)
-        @tasks << task if task
-        start_send_pump(connection) unless connection.is_a?(Transport::Inproc::DirectPipe)
+
+        add_fair_recv_connection(connection)
+
+        unless connection.is_a?(Transport::Inproc::DirectPipe)
+          @send_queue = Routing.build_queue(@engine.options.send_hwm, :block)
+          while (msg = @staging_queue.dequeue(timeout: 0))
+            @send_queue.enqueue(msg)
+          end
+          start_send_pump(connection)
+        end
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         if @connection == connection
           @connection = nil
+          @recv_queue.remove_queue(connection)
+          @send_queue = nil
           @send_pump&.stop
           @send_pump = nil
         end
       end
 
+
       # @param parts [Array<String>]
       #
       def enqueue(parts)
         conn = @connection
         if conn.is_a?(Transport::Inproc::DirectPipe) && conn.direct_recv_queue
           conn.send_message(parts)
-        else
+        elsif @send_queue
           @send_queue.enqueue(parts)
+        else
+          @staging_queue.enqueue(parts)
         end
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
+
+      # @return [Boolean] true when the staging and send queues are empty
+      #
+      def send_queues_drained?
+        @staging_queue.empty? && (@send_queue.nil? || @send_queue.empty?)
+      end
 
       private
 
       def start_send_pump(conn)
         @send_pump = @engine.spawn_pump_task(annotation: "send pump") do
           loop do
-            @send_pump_idle = true
             batch = [@send_queue.dequeue]
-            @send_pump_idle = false
             Routing.drain_send_queue(@send_queue, batch)
-            batch.each { |parts| conn.write_message(parts) }
-            conn.flush
+            begin
+              batch.each { |parts| conn.write_message(parts) }
+              conn.flush
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
+            end
           end
-        rescue *CONNECTION_LOST
-          @engine.connection_lost(conn)
         end
         @tasks << @send_pump
       end

data/lib/omq/routing/pub.rb

@@ -5,7 +5,8 @@ module OMQ
     # PUB socket routing: fan-out to all subscribers.
     #
     # Listens for SUBSCRIBE/CANCEL commands from peers.
-    # Drops messages if a subscriber's connection write fails.
+    # Each subscriber gets its own bounded send queue; slow subscribers
+    # are muted via the socket's on_mute strategy (drop by default).
     #
     class Pub
       include FanOut
@@ -18,9 +19,6 @@ module OMQ
         init_fan_out(engine)
       end
 
-      # @return [Async::LimitedQueue]
-      #
-      attr_reader :send_queue
 
       # PUB is write-only.
       #
@@ -28,28 +26,36 @@ module OMQ
         raise "PUB sockets cannot receive"
       end
 
+
       # @param connection [Connection]
       #
       def connection_added(connection)
         @connections << connection
         @subscriptions[connection] = Set.new
         start_subscription_listener(connection)
-        start_send_pump unless @send_pump_started
+        add_fan_out_send_connection(connection)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
         @subscriptions.delete(connection)
+        remove_fan_out_send_connection(connection)
       end
 
+
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        fan_out_enqueue(parts)
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)

data/lib/omq/routing/pull.rb

@@ -5,37 +5,45 @@ module OMQ
     # PULL socket routing: fair-queue receive from PUSH peers.
     #
     class Pull
+      include FairRecv
       # @param engine [Engine]
       #
       def initialize(engine)
         @engine     = engine
-        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
+        @recv_queue = FairQueue.new
         @tasks      = []
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
       attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
-        task = @engine.start_recv_pump(connection, @recv_queue)
-        @tasks << task if task
+        add_fair_recv_connection(connection)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
+        @recv_queue.remove_queue(connection)
         # recv pump stops on EOFError
       end
 
+
       # PULL is read-only.
       #
       def enqueue(_parts)
         raise "PULL sockets cannot send"
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
       #
       def stop
         @tasks.each(&:stop)

data/lib/omq/routing/push.rb

@@ -16,11 +16,6 @@ module OMQ
       end
 
 
-      # @return [Async::LimitedQueue]
-      #
-      attr_reader :send_queue
-
-
       # PUSH is write-only.
       #
       def recv_queue
@@ -32,9 +27,7 @@ module OMQ
       #
       def connection_added(connection)
         @connections << connection
-        signal_connection_available
-        update_direct_pipe
-        start_send_pump unless @send_pump_started
+        add_round_robin_send_connection(connection)
         start_reaper(connection)
       end
 
@@ -43,7 +36,7 @@ module OMQ
       #
       def connection_removed(connection)
         @connections.delete(connection)
-        update_direct_pipe
+        remove_round_robin_send_connection(connection)
       end
 
 
@@ -54,7 +47,7 @@ module OMQ
       end
 
 
-      # Stops all background tasks (send pump, reapers).
+      # Stops all background tasks (send pumps, reapers).
       #
       def stop
         @tasks.each(&:stop)
@@ -76,8 +69,6 @@ module OMQ
           @engine.connection_lost(conn)
         end
       end
-
-
     end
   end
 end

data/lib/omq/routing/rep.rb

@@ -9,93 +9,83 @@ module OMQ
     # on send.
     #
     class Rep
-      # @param engine [Engine]
-      #
+      include FairRecv
+
       EMPTY_FRAME = "".b.freeze
 
+
+      # @param engine [Engine]
+      #
       def initialize(engine)
-        @engine            = engine
-        @recv_queue        = Routing.build_queue(engine.options.recv_hwm, :block)
-        @send_queue        = Routing.build_queue(engine.options.send_hwm, :block)
-        @pending_replies   = []
-        @tasks             = []
-        @send_pump_started = false
-        @send_pump_idle    = true
-        @written           = Set.new
+        @engine          = engine
+        @recv_queue      = FairQueue.new
+        @pending_replies = []
+        @conn_queues     = {}  # connection => per-connection send queue
+        @conn_send_tasks = {}  # connection => send pump task
+        @tasks           = []
       end
 
-      # @return [Async::LimitedQueue]
+
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
-        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
+        add_fair_recv_connection(connection) do |msg|
           delimiter = msg.index(&:empty?) || msg.size
           envelope  = msg[0, delimiter]
           body      = msg[(delimiter + 1)..] || []
           @pending_replies << { conn: connection, envelope: envelope }
           body
         end
-        @tasks << task if task
-        start_send_pump unless @send_pump_started
+
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        @conn_send_tasks[connection] = ConnSendPump.start(@engine, connection, q, @tasks)
       end
 
+
       # @param connection [Connection]
       #
       def connection_removed(connection)
-        # Remove any pending replies for this connection
         @pending_replies.reject! { |r| r[:conn] == connection }
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
-      # Enqueues a reply for sending.
+
+      # Enqueues a reply. Routes to the connection that sent the matching
+      # request by consuming the next pending_reply entry.
       #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        reply_info = @pending_replies.shift
+        return unless reply_info
+        conn = reply_info[:conn]
+        @conn_queues[conn]&.enqueue([*reply_info[:envelope], EMPTY_FRAME, *parts])
       end
 
+
+      # Stops all background tasks.
+      #
+      # @return [void]
+      #
       def stop
         @tasks.each(&:stop)
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
-
-      private
-
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
-          loop do
-            @send_pump_idle = true
-            batch = [@send_queue.dequeue]
-            @send_pump_idle = false
-            Routing.drain_send_queue(@send_queue, batch)
-
-            @written.clear
-            batch.each do |parts|
-              reply_info = @pending_replies.shift
-              next unless reply_info
-              conn = reply_info[:conn]
-              begin
-                conn.write_message([*reply_info[:envelope], EMPTY_FRAME, *parts])
-                @written << conn
-              rescue *CONNECTION_LOST
-                # connection lost mid-write
-              end
-            end
-
-            @written.each do |conn|
-              conn.flush
-            rescue *CONNECTION_LOST
-              # connection lost mid-flush
-            end
-          end
-        end
+
+      # @return [Boolean] true when all per-connection send queues are empty
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
       end
+
     end
   end
 end