omq 0.11.0 → 0.13.0

data/lib/omq/routing/xpub.rb

@@ -8,6 +8,9 @@ module OMQ
     # the application as data frames: \x01 + prefix for subscribe,
     # \x00 + prefix for unsubscribe.
     #
+    # The recv_queue is a simple bounded queue (not a FairQueue) because
+    # messages come from subscription commands, not from peer data pumps.
+    #
     class XPub
       include FanOut
 
@@ -15,14 +18,14 @@ module OMQ
       #
       def initialize(engine)
         @engine     = engine
-        @recv_queue = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
         @tasks      = []
         init_fan_out(engine)
       end
 
       # @return [Async::LimitedQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
@@ -30,7 +33,7 @@ module OMQ
         @connections << connection
         @subscriptions[connection] = Set.new
         start_subscription_listener(connection)
-        start_send_pump unless @send_pump_started
+        add_fan_out_send_connection(connection)
       end
 
       # @param connection [Connection]
@@ -38,12 +41,13 @@ module OMQ
       def connection_removed(connection)
         @connections.delete(connection)
         @subscriptions.delete(connection)
+        remove_fan_out_send_connection(connection)
       end
 
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        fan_out_enqueue(parts)
       end
 
       #

data/lib/omq/routing/xsub.rb

@@ -5,45 +5,57 @@ module OMQ
     # XSUB socket routing: like SUB but subscriptions sent as data messages.
     #
     # Subscriptions are sent as data frames: \x01 + prefix for subscribe,
-    # \x00 + prefix for unsubscribe.
+    # \x00 + prefix for unsubscribe. Each connected PUB gets its own send
+    # queue so subscription commands are delivered independently per peer.
     #
     class XSub
 
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine            = engine
-        @connections       = []
-        @recv_queue        = Async::LimitedQueue.new(engine.options.recv_hwm)
-        @send_queue        = Async::LimitedQueue.new(engine.options.send_hwm)
-        @tasks             = []
-        @send_pump_started = false
-        @send_pump_idle    = true
+        @engine          = engine
+        @connections     = Set.new
+        @recv_queue      = FairQueue.new
+        @conn_queues     = {}  # connection => per-connection send queue
+        @conn_send_tasks = {}  # connection => send pump task
+        @tasks           = []
       end
 
-      # @return [Async::LimitedQueue]
+      # @return [FairQueue]
       #
-      attr_reader :recv_queue, :send_queue
+      attr_reader :recv_queue
 
       # @param connection [Connection]
       #
       def connection_added(connection)
         @connections << connection
-        task = @engine.start_recv_pump(connection, @recv_queue)
+
+        conn_q    = Routing.build_queue(@engine.options.recv_hwm, @engine.options.on_mute)
+        signaling = SignalingQueue.new(conn_q, @recv_queue)
+        @recv_queue.add_queue(connection, conn_q)
+        task = @engine.start_recv_pump(connection, signaling)
         @tasks << task if task
-        start_send_pump unless @send_pump_started
+
+        q = Routing.build_queue(@engine.options.send_hwm, :block)
+        @conn_queues[connection] = q
+        start_conn_send_pump(connection, q)
       end
 
       # @param connection [Connection]
       #
       def connection_removed(connection)
         @connections.delete(connection)
+        @recv_queue.remove_queue(connection)
+        @conn_queues.delete(connection)
+        @conn_send_tasks.delete(connection)&.stop
       end
 
+      # Enqueues a subscription command (fan-out to all connected PUBs).
+      #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        @send_queue.enqueue(parts)
+        @connections.each { |conn| @conn_queues[conn]&.enqueue(parts) }
       end
 
       #
@@ -52,31 +64,35 @@ module OMQ
         @tasks.clear
       end
 
-      def send_pump_idle? = @send_pump_idle
+      # True when all per-connection send queues are empty.
+      #
+      def send_queues_drained?
+        @conn_queues.values.all?(&:empty?)
+      end
 
       private
 
-      def start_send_pump
-        @send_pump_started = true
-        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+      def start_conn_send_pump(conn, q)
+        task = @engine.spawn_pump_task(annotation: "send pump") do
           loop do
-            @send_pump_idle = true
-            parts = @send_queue.dequeue
-            @send_pump_idle = false
+            parts = q.dequeue
             frame = parts.first&.b
             next if frame.nil? || frame.empty?
-
             flag   = frame.getbyte(0)
             prefix = frame.byteslice(1..) || "".b
-
-            case flag
-            when 0x01
-              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix)) }
-            when 0x00
-              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix)) }
+            begin
+              case flag
+              when 0x01 then conn.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
+              when 0x00 then conn.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix))
+              end
+            rescue Protocol::ZMTP::Error, *CONNECTION_LOST
+              @engine.connection_lost(conn)
+              break
             end
           end
         end
+        @conn_send_tasks[conn] = task
+        @tasks << task
       end
     end
   end

data/lib/omq/routing.rb

@@ -3,6 +3,10 @@
 require "async"
 require "async/queue"
 require "async/limited_queue"
+require_relative "drop_queue"
+require_relative "routing/fair_queue"
+require_relative "routing/fair_recv"
+require_relative "routing/conn_send_pump"
 
 module OMQ
   # Routing strategies for each ZMQ socket type.
@@ -14,6 +18,42 @@ module OMQ
     # Shared frozen empty binary string to avoid repeated allocations.
     EMPTY_BINARY = "".b.freeze
 
+    # Plugin registry for socket types not built into omq.
+    # Populated by sister gems via +Routing.register+.
+    #
+    @registry = {}
+
+    class << self
+      # Registers a routing strategy class for a socket type.
+      # Called by omq-draft (and other plugins) at require time.
+      #
+      # @param socket_type [Symbol] e.g. :RADIO, :CLIENT
+      # @param strategy_class [Class]
+      #
+      def register(socket_type, strategy_class)
+        @registry[socket_type] = strategy_class
+      end
+    end
+
+    # Builds a send or recv queue based on the mute strategy.
+    #
+    # @param hwm [Integer] high water mark
+    # @param on_mute [Symbol] :block, :drop_newest, or :drop_oldest
+    # @return [Async::LimitedQueue, DropQueue]
+    #
+    def self.build_queue(hwm, on_mute)
+      return Async::Queue.new if hwm.nil? || hwm == 0
+
+      case on_mute
+      when :block
+        Async::LimitedQueue.new(hwm)
+      when :drop_newest, :drop_oldest
+        DropQueue.new(hwm, strategy: on_mute)
+      else
+        raise ArgumentError, "unknown on_mute strategy: #{on_mute.inspect}"
+      end
+    end
+
     # Drains all available messages from +queue+ into +batch+ without
     # blocking. Call after the initial blocking dequeue.
     #
@@ -49,17 +89,10 @@ module OMQ
       when :SUB    then Sub
       when :XPUB   then XPub
       when :XSUB   then XSub
-      when :PUSH    then Push
-      when :PULL    then Pull
-      when :CLIENT  then Client
-      when :SERVER  then Server
-      when :RADIO   then Radio
-      when :DISH    then Dish
-      when :SCATTER then Scatter
-      when :GATHER  then Gather
-      when :PEER    then Peer
-      when :CHANNEL then Channel
-      else raise ArgumentError, "unknown socket type: #{socket_type}"
+      when :PUSH   then Push
+      when :PULL   then Pull
+      else
+        @registry[socket_type] or raise ArgumentError, "unknown socket type: #{socket_type.inspect}"
       end
     end
   end

data/lib/omq/socket.rb

@@ -36,8 +36,8 @@ module OMQ
       :heartbeat_ttl,         :heartbeat_ttl=,
       :heartbeat_timeout,     :heartbeat_timeout=,
       :max_message_size,      :max_message_size=,
-      :mechanism,             :mechanism=,
-      :tls_context,           :tls_context=
+      :on_mute,               :on_mute=,
+      :mechanism,             :mechanism=
 
 
     # Creates a new socket and binds it to the given endpoint.
@@ -47,7 +47,7 @@ module OMQ
     # @return [Socket]
     #
     def self.bind(endpoint, **opts)
-      new(nil, **opts).tap { |s| s.bind(endpoint) }
+      new("@#{endpoint}", **opts)
     end
 
 
@@ -58,7 +58,7 @@ module OMQ
     # @return [Socket]
     #
     def self.connect(endpoint, **opts)
-      new(nil, **opts).tap { |s| s.connect(endpoint) }
+      new(">#{endpoint}", **opts)
     end
 
 
@@ -141,6 +141,46 @@ module OMQ
     end
 
 
+    # Yields lifecycle events for this socket.
+    #
+    # Spawns a background fiber that reads from an internal event queue.
+    # The block receives {MonitorEvent} instances until the socket is
+    # closed or the returned task is stopped.
+    #
+    # @yield [event] called for each lifecycle event
+    # @yieldparam event [MonitorEvent]
+    # @return [Async::Task] the monitor task (call +#stop+ to end early)
+    #
+    # @example
+    #   task = socket.monitor do |event|
+    #     case event
+    #     in type: :connected, endpoint:
+    #       puts "peer up: #{endpoint}"
+    #     in type: :disconnected, endpoint:
+    #       puts "peer down: #{endpoint}"
+    #     end
+    #   end
+    #   # later:
+    #   task.stop
+    #
+    def monitor(&block)
+      ensure_parent_task
+      queue = Async::Queue.new
+      @engine.monitor_queue = queue
+      Reactor.run do
+        @engine.parent_task.async(transient: true, annotation: "monitor") do
+          while (event = queue.dequeue)
+            block.call(event)
+          end
+        rescue Async::Stop
+        ensure
+          @engine.monitor_queue = nil
+          block.call(MonitorEvent.new(type: :monitor_stopped))
+        end
+      end
+    end
+
+
     # Disable auto-reconnect for connected endpoints.
     def reconnect_enabled=(val)
       @engine.reconnect_enabled = val
@@ -226,13 +266,14 @@ module OMQ
     #
     def _init_engine(socket_type, linger:, send_hwm: nil, recv_hwm: nil,
                      send_timeout: nil, recv_timeout: nil, conflate: false,
-                     backend: nil)
+                     on_mute: nil, backend: nil)
       @options = Options.new(linger: linger)
       @options.send_hwm      = send_hwm     if send_hwm
       @options.recv_hwm      = recv_hwm     if recv_hwm
       @options.send_timeout   = send_timeout if send_timeout
       @options.recv_timeout   = recv_timeout if recv_timeout
       @options.conflate       = conflate
+      @options.on_mute        = on_mute      if on_mute
       @recv_buffer = []
       @recv_mutex  = Mutex.new
       @engine      = case backend

data/lib/omq/transport/inproc/direct_pipe.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Transport
+    module Inproc
+      # A direct in-process pipe that transfers Ruby arrays through queues.
+      #
+      # Implements the same interface as Connection so routing strategies
+      # can use it transparently.
+      #
+      # When a routing strategy sets {#direct_recv_queue} on a pipe,
+      # {#send_message} enqueues directly into the peer's recv queue,
+      # bypassing the intermediate pipe queues and the recv pump task.
+      # This reduces inproc from 3 queue hops to 2 (send_queue →
+      # recv_queue), eliminating the internal pipe queue in between.
+      #
+      class DirectPipe
+        # @return [String] peer's socket type
+        #
+        attr_reader :peer_socket_type
+
+
+        # @return [String] peer's identity
+        #
+        attr_reader :peer_identity
+
+
+        # @return [DirectPipe, nil] the other end of this pipe pair
+        #
+        attr_accessor :peer
+
+
+        # @return [Async::LimitedQueue, nil] when set, {#send_message}
+        #   enqueues directly here instead of using the internal queue
+        #
+        attr_reader :direct_recv_queue
+
+
+        # @return [Proc, nil] optional transform applied before
+        #   enqueuing into {#direct_recv_queue}
+        #
+        attr_accessor :direct_recv_transform
+
+
+        # @param send_queue [Async::Queue, nil] outgoing command queue
+        #   (nil for non-PUB/SUB types that don't exchange commands)
+        # @param receive_queue [Async::Queue, nil] incoming command queue
+        # @param peer_identity [String]
+        # @param peer_type [String]
+        #
+        def initialize(send_queue: nil, receive_queue: nil, peer_identity:, peer_type:)
+          @send_queue            = send_queue
+          @receive_queue         = receive_queue
+          @peer_identity         = peer_identity || "".b
+          @peer_socket_type      = peer_type
+          @closed                = false
+          @peer                  = nil
+          @direct_recv_queue     = nil
+          @direct_recv_transform = nil
+          @pending_direct        = nil
+        end
+
+
+        # Sets the direct recv queue. Drains any messages that were
+        # buffered before the queue was available.
+        #
+        def direct_recv_queue=(queue)
+          @direct_recv_queue = queue
+          if queue && @pending_direct
+            @pending_direct.each { |msg| queue.enqueue(msg) }
+            @pending_direct = nil
+          end
+        end
+
+
+        # Sends a multi-frame message.
+        #
+        # @param parts [Array<String>]
+        # @return [void]
+        #
+        def send_message(parts)
+          raise IOError, "closed" if @closed
+          if @direct_recv_queue
+            @direct_recv_queue.enqueue(apply_transform(parts))
+          elsif @send_queue
+            @send_queue.enqueue(parts)
+          else
+            (@pending_direct ||= []) << apply_transform(parts)
+          end
+        end
+
+
+        alias write_message send_message
+
+
+        # No-op — inproc has no IO buffer to flush.
+        #
+        def flush = nil
+
+
+        # Receives a multi-frame message.
+        #
+        # @return [Array<String>]
+        # @raise [EOFError] if closed
+        #
+        def receive_message
+          loop do
+            item = @receive_queue.dequeue
+            raise EOFError, "connection closed" if item.nil?
+            if item.is_a?(Array) && item.first == :command
+              yield Protocol::ZMTP::Codec::Frame.new(item[1].to_body, command: true) if block_given?
+              next
+            end
+            return item
+          end
+        end
+
+
+        # Sends a command via the internal command queue.
+        # Only available for PUB/SUB-family pipes.
+        #
+        # @param command [Protocol::ZMTP::Codec::Command]
+        #
+        def send_command(command)
+          raise IOError, "closed" if @closed
+          @send_queue.enqueue([:command, command])
+        end
+
+
+        # Reads one command frame from the internal command queue.
+        # Used by PUB/XPUB subscription listeners.
+        #
+        # @return [Protocol::ZMTP::Codec::Frame]
+        #
+        def read_frame
+          loop do
+            item = @receive_queue.dequeue
+            raise EOFError, "connection closed" if item.nil?
+            if item.is_a?(Array) && item.first == :command
+              return Protocol::ZMTP::Codec::Frame.new(item[1].to_body, command: true)
+            end
+          end
+        end
+
+
+        # Closes this pipe end.
+        #
+        def close
+          return if @closed
+          @closed = true
+          @send_queue&.enqueue(nil) # close sentinel
+        end
+
+        private
+
+        def apply_transform(parts)
+          @direct_recv_transform ? @direct_recv_transform.call(parts).freeze : parts
+        end
+      end
+    end
+  end
+end